爬虫学习--Requests库详解 Day2

什么是Requests

Requests是用python语言编写，基于urllib，采用Apache2 licensed开源协议的HTTP库，它比urllib更加方便，可以节约我们大量的工作，完全满足HTTP测试需求。
一句话总结：它是Python实现的简单易用的HTTP库

安装Requests  pip install requests

验证没有报错，表示已经成功的安装了

实例引入

 import requests
 response = requests.get('https://www.baidu.com')
 print(type(response))
 print(response.status_code) # 状态码
 print(type(response.text))
 print(response.text) # 响应的内容
 print(response.cookies) # 获取cookie

各种请求方式

 import requests
 print(requests.post('http://httpbin.org/post')) #
 print(requests.put('http://httpbin.org/put'))
 print(requests.delete('http://httpbin.org/delete'))
 print(requests.head('http://httpbin.org/get'))
 print(requests.options('http://httpbin.org/get'))

Requests库请求具体怎么用的
基本GET请求-----------------------------------------------------------------------------------------------------------
基本写法

 import requests

 response = requests.get('http://httpbin.org/get')
 print(response.text) # 请求头啊，请求的IP地址，请求的链接

带参数的GET请求

 import requests

 response = requests.get('http://httpbin.org/get?name=germey&age=22')
 print(response.text)

也可以用字典的形式传参 params

 import requests

 data = {
     'name':'xiaohu',
     'age':''
 }
 response = requests.get('http://httpbin.org/get',params=data)
 print(response.text)

解析json

 import requests

 response = requests.get('http://httpbin.org/get')
 print(type(response.text))
 print(response.json())
 print(type(response.json())) # 它是一个字典的类型

区别用json.loads与直接.json有什么不一样,结果其实是一样的

 import requests
 import json

 response = requests.get('http://httpbin.org/get')
 print(type(response.text))
 print(response.json())
 print(json.loads(response.text)) #区别用json.loads与直接.json有什么不一样,结果其实是一样的
 print(type(response.json())) # 它是一个字典的类型

获取二进制数据

 import requests

 response = requests.get('https://github.com/favicon.ico')
 print(type(response.text),type(response.content))  # .content是二进制内容
 print(response.text)
 print(response.content)

获取到图片把它保存

  import requests
 response = requests.get('https://github.com/favicon.ico')
 with open('favicon','wb') as f:
     f.write(response.content)
     f.close()

 添加headers 如果不加headers 报错500
 import requests

 headers = {
     'User-Agent':'Mozilla/5.0(Macintosh;intel Mac OS X 10_11_4)AppleWebKit/537.36(KHTML,like Gecko)Chrome/52.0.2743.116 Safari/537.36'
 }
 response = requests.get("https://www.zhihu.com/explore",headers=headers)
 print(response.text)

基本POST请求

 import requests

 data = {
     'name':'xiaohu',
     'age':'',
     'job':'IT'
 }
 response = requests.post('http://httpbin.org/post',data=data)
 print(response.text)

 import requests

 data = {
     'name':'xiaohu',
     'age':'',
     'job':'IT'
 }
 headers = {
     'User-Agent':'Mozilla/5.0(Macintosh;intel Mac OS X 10_11_4)AppleWebKit/537.36(KHTML,like Gecko)Chrome/52.0.2743.116 Safari/537.36'
 }
 response = requests.post('http://httpbin.org/post',data=data,headers=headers)
 print(response.json())

响应
response属性

 import requests

 response = requests.get('http://www.jianshu.com')
 print(type(response.status_code),response.status_code) # 状态码
 print(type(response.headers),response.headers) # 请求头
 print(type(response.cookies),response.cookies) #
 print(type(response.url),response.url)
 print(type(response.history),response.history) # 访问的历史记录

高级操作
文件上传

 import requests

 files = {
     'file':open('favicon','rb')
 }
 response = requests.post('http://httpbin.org/post',files=files)
 print(response.text)

获取cookie

 import requests

 response = requests.get('https://www.baidu.com')
 print(response.cookies)
 print(type(response.cookies))
 for key,value in response.cookies.items():
     print(key+'='+value)

会话维持

 import requests
 requests.get('http://httpbin.org/cookies/set/number/1165872335') # 为网站设置一个cookies
 response = requests.get('http://httpbin.org/cookies') # 再用get访问这个cookies
 print(response.text) # 为空，因为这里进行了两次get请求，相当于两个浏览器分别设置cookies和访问cookies,相对独立的

改进

 import requests
 S = requests.Session() # 声明对象
 S.get('http://httpbin.org/cookies/set/number/1165872335') # 实现在同一个浏览器进行设置rookies和访问rookies
 response = S.get('http://httpbin.org/cookies') # 再用get访问这个cookies
 print(response.text) # 此时不为空

证书验证

 import requests
 from requests.packages import urllib3
 urllib3.disable_warnings() # 消除警告信息

 response = requests.get('https://www.12306.cn',verify=False) # verify=False 不需要验证进入，但是有警告
 print(response.status_code)

 import requests

 response = requests.get('https://www.12306.cn',cert=('/path/server.crt','/path/key')) # 指定的证书
 print(response.status_code)

代理设置

 import requests
 import socks5

 proxies = {
     "http":"socks5://127.0.0.1:8080",
     "https":"socks5://127.0.0.1:8080",
 }

 response = requests.get('https://www.taobao.com',proxies=proxies)
 print(response.status_code)

超时设置

 import requests
 from requests.exceptions import ReadTimeout

 try:
     response = requests.get('https://www.httpbin.org/get',timeout = 0.2)
     print(response.status_code)
 except ReadTimeout:
     print("timeout")

认证设置

 import requests
 from requests.auth import HTTPBasicAuth

 r = requests.get('http://127.27.34.24:9001',auth=HTTPBasicAuth('user',''))
 print(r.status_code)

 # 第二种方式
 import requests

 r = requests.get('http://127.27.34.24:9001',auth=('user',''))
 print(r.status_code)

异常处理

import requests
from requests.exceptions import ReadTimeout,HTTPError,RequestException,ConnectionError
try:
    response = requests.get('http://httpbin.org/get',timeout = 0.2)
    print(response.status_code)
except ReadTimeout:
    print('Timeout')
except HTTPError:
    print('Http error')
except ConnectionError:
    print('Connection Error')
except RequestException:
    print('error')