python爬虫入门（1）----- requests

介绍

requests是python实现的简单易用的HTTP库，使用起来比urllib简洁很多

基本使用

requests.get("http://www.baidu.com")
requests.post("http://www.baidu.com")
requests.put("http://www.baidu.com")
requests.delete("http://www.baidu.com")
requests.request("get", "http://www.baidu.com")

get

def get(url, params=None, **kwargs):
        r"""Sends a GET request.

        :param url: URL for the new :class:`Request` object.
        :param params: (optional) Dictionary, list of tuples or bytes to send
            in the body of the :class:`Request`.
        :param \*\*kwargs: Optional arguments that ``request`` takes.
        :return: :class:`Response <Response>` object
        :rtype: requests.Response
        """

        kwargs.setdefault('allow_redirects', True)
        return request('get', url, params=params, **kwargs)

下面凡科微传单获取模板的接口为例子

 import requests

    param = {
    "cmd": "getTemplate"，
    "scrollIndex": 0
    }
    header = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36"
    }//通过ua识别是否是爬虫
    rep = requests.get("https://cd.fkw.com/ajax/flyerhome.jsp", params=param, headers=header)
    rep.encoding = 'utf8'
    print(rep.text)

post

def post(url, data=None, json=None, **kwargs):
        r"""Sends a POST request.

        :param url: URL for the new :class:`Request` object.
        :param data: (optional) Dictionary, list of tuples, bytes, or file-like
            object to send in the body of the :class:`Request`.
        :param json: (optional) json data to send in the body of the :class:`Request`.
        :param \*\*kwargs: Optional arguments that ``request`` takes.
        :return: :class:`Response <Response>` object
        :rtype: requests.Response
        """

        return request('post', url, data=data, json=json, **kwargs)

一样以凡科微传单接口为例

 import requests

    data = {
    "cmd": "getTemplate"，
    "scrollIndex": 0
    }
    header = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36"
    }
    rep = requests.post("https://cd.fkw.com/ajax/flyerhome.jsp", data=data, headers=header)
    rep.encoding = 'utf8'
    print(rep.text)

会话对象

在上面操作中request不会持有cookie对象导致每次请求都是新的会话，requests库提供了session的解决方案，下面以凡科登录和登录状态下获取模板为例

import requests
    import _md5
    import json
    import re

    s = requests.session()

    md5 = _md5.md5()
    md5.update("pwd".encode("utf8"))
    pwd = md5.hexdigest()
    data = {
    "cmd": "loginCorpNew",
    "cacct": "username",
    "pwd": pwd
    }
    header = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36"
    }
    rep = s.post("https://i.fkw.com/ajax/login_h.jsp?dogSrc=3", data=data, headers=header)
    login = json.loads(rep.text)
    tokenStr = login.get("_TOKEN")
    print(tokenStr)
    pattern = "value='(.+)'"
    matcher = re.search(pattern, rep.text)
    if matcher:
        token = matcher.group(1)
        print(token)
        param = {
        "cmd": "getTemplate",
        "_TOKEN": token,
        "scrollIndex": 0
        }
        rep = s.get("https://i.cd.fkw.com/ajax/flyerTemplate_h.jsp", params=param, headers=header)
        print(rep.text)

参考文献

https://cuiqingcai.com/2556.html
http://docs.python-requests.org/en/master/api/