urllib(最基本的)库的应用

Urllib库

---

python内置的http请求库

1、urllib.request 请求模块

2、urllib.error 异常处理模块(try,catch)

3、urllib.parse url解析模块

4、urllib.robotparser robots.txr解析模块

---

urlopen

get请求

import urllib.request
reponse=urllib.request.urlopen("http://www.baidu.com")
print(reponse.read().decode('utf-8'))#.read()读请求内容

post请求

import urllib.parse#貌似加不加都行
import urllib.request
data=bytes(urllib.parse.urlencode({'name':'汪国强'}),encoding='utf-8')
response=urllib.request.urlopen('http://httpbin.org/post',data=data)
print(response.read().decode('utf-8'))

urllib.error

import urllib.request
import socket
import urllib.error
try:
    response=urllib.request.urlopen('http://httpbin.org/get',timeout=0.01)
except urllib.error.URLError as e:  #超时属于URLError
    if isinstance(e.reason,socket.timeout):
        print('timeout')

---

对响应的一些处理

状态码、响应头

import urllib.request
import socket
import urllib.error
response=urllib.request.urlopen('http://www.baidu.com')
print(response.status)
print('-----------------')
print(response.getheaders())
print('-----------------')
print(response.getheader('Server'))

得到：

200        状态码
-----------------

响应头
[('Date', 'Mon, 25 Dec 2017 09:59:01 GMT'), ('Content-Type', 'text/html; charset=utf-8'), ('Transfer-Encoding', 'chunked'), ('Connection', 'Close'), ('Vary', 'Accept-Encoding'), ('Set-Cookie', 'BAIDUID=C941C9CEBE13F4D6264663E5A10D4603:FG=1; expires=Thu, 31-Dec-37 23:55:55 GMT; max-age=2147483647; path=/; domain=.baidu.com'), ('Set-Cookie', 'BIDUPSID=C941C9CEBE13F4D6264663E5A10D4603; expires=Thu, 31-Dec-37 23:55:55 GMT; max-age=2147483647; path=/; domain=.baidu.com'), ('Set-Cookie', 'PSTM=1514195941; expires=Thu, 31-Dec-37 23:55:55 GMT; max-age=2147483647; path=/; domain=.baidu.com'), ('Set-Cookie', 'BDSVRTM=0; path=/'), ('Set-Cookie', 'BD_HOME=0; path=/'), ('Set-Cookie', 'H_PS_PSSID=25394_1453_21119_25178_22157; path=/; domain=.baidu.com'), ('P3P', 'CP=" OTI DSP COR IVA OUR IND COM "'), ('Cache-Control', 'private'), ('Cxy_all', 'baidu+e8e6fa769a31bd4f787c267655da18e6'), ('Expires', 'Mon, 25 Dec 2017 09:58:11 GMT'), ('X-Powered-By', 'HPHP'), ('Server', 'BWS/1.1'), ('X-UA-Compatible', 'IE=Edge,chrome=1'), ('BDPAGETYPE', '1'), ('BDQID', '0xc958493100031c89'), ('BDUSERID', '0')]

-----------------

指定的响应头内容

BWS/1.1

---

如果想在请求时加上请求头怎么办？

import urllib.request
import urllib.parse
head={
    "Host": "httpbin.org",
    "Upgrade-Insecure-Requests": "",
}
dic={'name':''}
data=bytes(urllib.parse.urlencode(dic),encoding='utf-8')
request=urllib.request.Request('http://httpbin.org/post',data=data,headers=head,method='POST')
response=urllib.request.urlopen(request)
print(response.read().decode('utf-8'))

或者使用request.add_header()

import urllib.request,parser

dic={'name':''}
data=bytes(urllib.parse.urlencode(dic),encoding='utf-8')
request=urllib.request.Request('http://httpbin.org/post',data=data,method='POST')
request.add_header(
    "Upgrade-Insecure-Requests", ""
)
response=urllib.request.urlopen(request)
print(response.read().decode('utf-8'))

---

Handler

代理

使用代理ip

import urllib.request
proxy_handler=urllib.request.ProxyHandler({
    'http':'http://116.199.115.78:80/'
})
opener=urllib.request.build_opener(proxy_handler)
response=opener.open('http://httpbin.org/ip')
print(response.read().decode('utf-8'))