urllib库使用方法

这周打算把学过的内容重新总结一下，便于以后翻阅查找资料。

urllib库是python的内置库，不需要单独下载。其主要分为四个模块：

1.urllib.request——请求模块

2.urllib.error——异常处理模块

3.urllib.parse——url解析模块

4.urllib.robotparser——用来识别网站的robot.txt文件（看看哪些内容是可以爬的，不常用）

1.urlopen

import urllib.request

response = urllib.request.urlopen('http://www.baidu.com')
print(response.read().decode('utf-8'))

超时读取
import socket
import urllib.request
import urllib.error

try:
    response = urllib.request.urlopen('http://httpbin.org/get', timeout=0.1)
except urllib.error.URLError as e:
    if isinstance(e.reason, socket.timeout):
        print('TIME OUT')

响应内容分析
import urllib.request

response = urllib.request.urlopen('https://www.python.org')
print(type(response))
print(response.status)
print(response.getheaders())
print(response.getheader('Server'))

<class 'http.client.HTTPResponse'>
200
[('Server', 'nginx'), ('Content-Type', 'text/html; charset=utf-8'), ('X-Frame-Options', 'SAMEORIGIN'), ('x-xss-protection', '1; mode=block'), ('X-Clacks-Overhead', 'GNU Terry Pratchett'), ('Via', '1.1 varnish'), ('Content-Length', '50069'), ('Accept-Ranges', 'bytes'), ('Date', 'Mon, 26 Nov 2018 02:44:49 GMT'), ('Via', '1.1 varnish'), ('Age', '3121'), ('Connection', 'close'), ('X-Served-By', 'cache-iad2150-IAD, cache-sjc3143-SJC'), ('X-Cache', 'MISS, HIT'), ('X-Cache-Hits', '0, 254'), ('X-Timer', 'S1543200290.644687,VS0,VE0'), ('Vary', 'Cookie'), ('Strict-Transport-Security', 'max-age=63072000; includeSubDomains')]
nginx

2. request

用来传递更多的请求参数，url,headers,data, method
from urllib import request, parse

url = 'http://httpbin.org/post'
headers = {
    'User-Agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)',
    'Host': 'httpbin.org'
}
dict = {
    'name': 'Germey'
}
data = bytes(parse.urlencode(dict), encoding='utf8')
req = request.Request(url=url, data=data, headers=headers, method='POST')
response = request.urlopen(req)
print(response.read().decode('utf-8'))

{
  "args": {},
  "data": "",
  "files": {},
  "form": {
    "name": "Germey"
  },
  "headers": {
    "Accept-Encoding": "identity",
    "Connection": "close",
    "Content-Length": "11",
    "Content-Type": "application/x-www-form-urlencoded",
    "Host": "httpbin.org",
    "User-Agent": "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"
  },
  "json": null,
  "origin": "117.136.66.101",
  "url": "http://httpbin.org/post"
}

另一种方法添加headers

from urllib import request, parse

url = 'http://httpbin.org/post'
dict = {
    'name': 'asd'
}
data = bytes(parse.urlencode(dict), encoding='utf8')
req = request.Request(url=url, data=data, method='POST')
req.add_header('User-Agent', 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)')  #如果有好多headers,可以循环添加
response = request.urlopen(req)
print(response.read().decode('utf-8'))

3. Handler

代理
import urllib.request

proxy_handler = urllib.request.ProxyHandler({
    'http': 'http://127.0.0.1:9743',
    'https': 'https://127.0.0.1:9743'
})
opener = urllib.request.build_opener(proxy_handler)
response = opener.open('https://www.httpbin,org/get')
print(response.read())

4. Cookie

获取cookie
import http.cookiejar, urllib.request

cookie = http.cookiejar.CookieJar()
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open('http://www.baidu.com')
for item in cookie:
    print(item.name+"="+item.value)

存储Cookie
import http.cookiejar, urllib.request
filename = "cookie.txt"
cookie = http.cookiejar.MozillaCookieJar(filename)
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open('http://www.baidu.com')
cookie.save(ignore_discard=True, ignore_expires=True)

获取Cookie
import http.cookiejar, urllib.request
cookie = http.cookiejar.MozillaCookieJar()
cookie.load('cookie.txt', ignore_discard=True, ignore_expires=True)
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open('http://www.baidu.com')
print(response.read().decode('utf-8'))

5. UrlError 和 HttpError

from urllib import request, error

try:
    response = request.urlopen('http://cuiqingcai.com/index.htm')
except error.HTTPError as e:
    print(e.reason, e.code, e.headers, sep='\n')
except error.URLError as e:
    print(e.reason)
else:
    print('Request Successfully')

对于urllib.parse，因为用的不多，就先不写了。上传一个文档链接吧，万一哪天用了还可以看看。

https://files.cnblogs.com/files/zhangguoxv/urllib%E8%AE%B2%E8%A7%A3.zip