requests模块的一些总结

一.cookie模拟登录的三种方式

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#author tom

import  requests
post_url='http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=20192012211'
post_data={"email":"302624476@qq.com","password":"xxx"}
headers={"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36"}
session=requests.session()
#使用session发送post请求,cookie保存在其中
session.post(url=post_url,data=post_data,headers=headers)

#在session请求登录之后才能获取的网址
r=session.get("http://www.renren.com/323264874/profile",headers=headers)

with open('renren.html','w',encoding='utf-8') as f:
    f.write(r.content.decode())

# request模拟登陆的三种方式:
#方式一:
# 先实例化一个session对象,用session先发起一个post请求,session就保存了cookie,再用session发起登录后的请求

#方式二
#浏览器挡登陆后拿到cookie
# 在headers中添加cookie的键值对

# 方式三
#在请求方法中添加cookies参数,接收字典形式的cookies
# 字典形式的cookie中的键是cookie中name对应的值,值是cookie中value对应的之
#和方式二有点类似,就是拿到cookie,改造成字典,然后在请求时候设置cookies参数

　　关于方式三的扩展：

1. requests.utils.dict_from_cookiejar()

def test_url(ip,is_http,redis=None):
    pro = {TYPES[is_http]:ip}
    #if redis == None:
    #    redis = redis.StrictRedis(REDIS_SERVER,REDIS_PORT,DB_FOR_IP)
    time = 0
    flag= False
    try:
            #print "test url:",i,ip,pro
        r = None
        cookie_old = None
        if STORE_COOKIE and redis != None:
            cookie_old = redis.get(ip)
            #print "old cookie:",cookie
            if cookie_old != None and cookie_old != "None" and cookie_old != "{}":
                #print "use cookie"
                log.debug("PID:%d IP:%s use old cookies:%s " % (os.getpid(),ip,cookie_old))
                cookies = cookiejar_from_dict(json.loads(cookie_old))
                r = requests.get(TEST_URL,proxies=pro,cookies=cookies,timeout=SOKCET_TIMEOUT)
            else:
                if USE_DEFAULT_COOKIE:
                    rand_cookies = {"bid":random_str()}
                    log.debug("PID:%d IP:%s use random cookies:%s " % (os.getpid(),ip,str(rand_cookies)))
                    cookie = cookiejar_from_dict(rand_cookies)
                    r = requests.get(TEST_URL,proxies=pro,cookies=cookie,timeout=SOKCET_TIMEOUT)
                else:
                    r = requests.get(TEST_URL,proxies=pro,timeout=SOKCET_TIMEOUT)
        else:
            if USE_DEFAULT_COOKIE:
                cookie = cookiejar_from_dict({"bid":random_str()})
                r = requests.get(TEST_URL,proxies=pro,cookies=cookie,timeout=SOKCET_TIMEOUT)
            else:
                r = requests.get(TEST_URL,proxies=pro,timeout=SOKCET_TIMEOUT)
        time += r.elapsed.microseconds/1000
        log.debug("PID:%d Test IP:%s result:%d time:%d type:%s" % (os.getpid(),ip,r.status_code,time,TYPES[is_http]))
        if r.ok:
            flag = True
            if STORE_COOKIE and redis != None:
                #print "new cookies:",r.cookies
                if r.cookies != None :
                    cookie = json.dumps(dict_from_cookiejar(r.cookies))
                    if cookie and cookie != "{}" and cookie_old != cookie:
                        log.debug("PID:%d IP:%s new cookies:%s old cookies:%s" % (os.getpid(),ip,cookie,cookie_old))
                        redis.set(ip,cookie)
    except Exception as e:
        log.debug("PID:%d error:%s" % (os.getpid(),e.message))
    return flag,time

　　可参考：http://codingdict.com/sources/py/requests.utils/18814.html

二.requests编码问题

　　1.response.text

• 类型:str
• 解码方式:根据HTTP头部对应的编码做出有根据的推测,推测文本的编码内容
• 如何修改编码方式:response.encoding="指定的编码方式"

　　2.response.content

• 类型:bytes
• 解码类型:没有指定
• 如何修改编码方式:response.content.deocode("编码方式")

注意:更建议使用第二种,原汁原味,过早的预处理不是好结果

三.url的编码解码

　　1.requests.utils.unquote(url)

　　2.requests.utils.quote(url)