python2.x urllib2和urllib的使用

1.最简单用法

　　urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,...)

 import urllib2
 import urllib

 response = urllib2.urlopen("http://www.baidu.com")

 print 'getcode():',response.getcode()
 print 'geturl():',response.geturl()
 print 'url:',response.url
 print 'headers:\n',response.headers
 print 'msg:',response.msg

 #-------------------------------------out--------------------------------------
 getcode(): 200
 geturl(): http://www.baidu.com
 url: http://www.baidu.com
 headers:
 Date: Thu, 29 Dec 2016 06:28:36 GMT
 Content-Type: text/html; charset=utf-8
 Transfer-Encoding: chunked
 Connection: Close
 Vary: Accept-Encoding
 Set-Cookie: BAIDUID=9A1E663B4C3AB33D11266F0D865A1F59:FG=1; expires=Thu, 31-Dec-37 23:55:55 GMT; max-age=2147483647; path=/; domain=.baidu.com
 Set-Cookie: BIDUPSID=9A1E663B4C3AB33D11266F0D865A1F59; expires=Thu, 31-Dec-37 23:55:55 GMT; max-age=2147483647; path=/; domain=.baidu.com
 Set-Cookie: PSTM=1482992916; expires=Thu, 31-Dec-37 23:55:55 GMT; max-age=2147483647; path=/; domain=.baidu.com
 Set-Cookie: BDSVRTM=0; path=/
 Set-Cookie: BD_HOME=0; path=/
 Set-Cookie: H_PS_PSSID=21858_1464_21112_17001_21553_20930; path=/; domain=.baidu.com
 P3P: CP=" OTI DSP COR IVA OUR IND COM "
 Cache-Control: private
 Cxy_all: baidu+0ba0b09e0fa305471b5e3b42c352570f
 Expires: Thu, 29 Dec 2016 06:27:54 GMT
 X-Powered-By: HPHP
 Server: BWS/1.1
 X-UA-Compatible: IE=Edge,chrome=1
 BDPAGETYPE: 1
 BDQID: 0x889c1bcd00004be7
 BDUSERID: 0

 msg: OK

获取html内容

 print response.read()     #以str字符串形式返回整个页面
 print response.readline() #每执行一次返回一行
 print response.readlines() #以列表形式返回

2.  构造Request 设置headers

 def set_headers():
     #构造Request,设置headers
     #__init__(self, url, data=None, headers={},origin_req_host=None, unverifiable=False)
     import urllib2
     headers = {'User-Agent':'liubi-Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'}
     request = urllib2.Request("http://localhost:5000/urllib2testget",headers=headers)

     response = urllib2.urlopen(request)
     print request.headers
     #追加一个header
     request.add_header("addheader","nice")
     response = urllib2.urlopen(request)
     print request.headers

 set_headers()

 #--------------------------------输出:

 {'User-agent': 'liubi-Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'}
 {"a": "", "": ""}
 ------------------------------------------------
 {'Addheader': 'nice', 'User-agent': 'liubi-Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'}
 {"a": "", "": ""}

3.发送get请求,发送post请求

 def get_post():
     #get方式
     import urllib2
     import urllib
     headers = {'User-Agent':'liu bi'}
     values = {"username":"diaosir_get","password":"diao123_get"}
     data = urllib.urlencode(values)
     print '---------------------get:'
     url = "http://localhost:5000/urllib2testget"
     get_url=url+"?"+data
     request = urllib2.Request(get_url,headers=headers)
     response = urllib2.urlopen(request)
     print json.loads(response.read())
     print '---------------------post:'
     url = "http://localhost:5000/urllib2testpost"
     request = urllib2.Request(url,data,headers=headers)
     response = urllib2.urlopen(request)
     print json.loads(response.read())

 get_post()

 #---------------------------------------------------------输出:
 ---------------------get:
 {u'username': u'diaosir_get', u'password': u'diao123_get'}
 ---------------------post:
 {u'username': u'diaosir_get', u'password': u'diao123_get'}

post&get

4.代理模式设置

def set_proxies():
    #1.proxy_handler
    #2.创建operner
    #3.安装opener[非必须]
    #4.拿operner去请求url
    enable_proxy = True
    proxy_handler = urllib2.ProxyHandler({"http":'http://120.24.73.165:3128'})
    null_proxy_handler = urllib2.ProxyHandler({})
    if enable_proxy:
        opener = urllib2.build_opener(proxy_handler)#挂载opener
    else:
        opener = urllib2.build_opener(null_proxy_handler)
    request = urllib2.Request('http://www.baidu.com')
    print '---------------------不使用代理'
    response = urllib2.urlopen(request)
    print response.getcode(),request.host
    print '---------------------使用代理'
    response = opener.open(request)
    print response.getcode(),request.host

#----------------------------------------------------------输出
---------------------不使用代理
200 www.baidu.com
---------------------使用代理
200 120.24.73.165:3128

5.debug模式, 代码中urllib2.build_opener中的httpsHandler需要去掉，

 def debug_set():
     #代理，调试
     import  urllib2,urllib
     proxy_handler = urllib2.ProxyHandler({"http":'http://192.168.1.108:89'})

     #debuglog的使用
     httpHandler = urllib2.HTTPHandler(debuglevel=1)
     opener = urllib2.build_opener(httpHandler, httpsHandler,)
     urllib2.install_opener(opener)
     request = urllib2.Request('http://127.0.0.1:5000/urllib2testget?a=2&b=3',headers={'User-Agent':'liubi00'})
     response = opener.open(request)
     print response.getcode(),response.read()

 #-------------------------------------------输出:
 send: 'GET /urllib2testget?a=2&b=3 HTTP/1.1\r\nAccept-Encoding: identity\r\nHost: 127.0.0.1:5000\r\nConnection: close\r\nUser-Agent: liubi00\r\n\r\n'
 reply: 'HTTP/1.0 200 OK\r\n'
 header: Content-Type: text/html; charset=utf-8
 header: Content-Length: 20
 header: Server: Werkzeug/0.11.11 Python/2.7.12
 header: Date: Fri, 30 Dec 2016 15:12:40 GMT
 200 {"a": "", "b": ""}

6.获取cookie存到cookie.txt

import cookielib
import  urllib2

def get_cookie():
    filename = 'cookie.txt'
    #声明一个MozillaCookieJar对象实例来保存cookie，之后写入文件
    cookie = cookielib.MozillaCookieJar(filename)
    #利用urllib2库的HTTPCookieProcessor对象来创建cookie处理器
    handler = urllib2.HTTPCookieProcessor(cookie)
    #通过handler来构建opener
    opener = urllib2.build_opener(handler,)
    request = urllib2.Request('http://www.baidu.com')
    request.add_header('User-Agent','fuckyou')
    response = opener.open(request)
    #保存cookie到文件
    cookie.save(ignore_discard=True, ignore_expires=True)
    print response.getcode()

get_cookie()

#----------------------------------------------输出:
200

7.通过cookie请求，更多查看http://www.cnblogs.com/sysu-blackbear/p/3629770.html

 import cookielib
 import urllib2
 def use_cookie():
     #cookie--从cookies.txt读取cookies,携带cookies请求
     cookie_file = 'cookie.txt'
     #创建MozillaCookieJar实例对象
     cookie = cookielib.MozillaCookieJar(cookie_file)
     #从文件中读取cookie内容到变量
     cookie.load( ignore_discard=True, ignore_expires=True)
     #创建请求的request
     req = urllib2.Request("http://www.baidu.com")
     #利用urllib2的build_opener方法创建一个opener
     opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
     response = opener.open(req)
     print response.read()

8.异常处理

 def deal_errors():
     #异常处理
     import urllib2
     #HTTPError
     req = urllib2.Request('http://blog.csdn.net/cqcre')
     try:
         urllib2.urlopen(req)
     except urllib2.HTTPError, e:
         print e.code
         print e.reason

     #URLError
     requset = urllib2.Request('http://www.xxxxx.com')
     try:
         urllib2.urlopen(requset)
     except urllib2.URLError, e:
         print e.reason

     #HTTPERROR&URLERROR
     req = urllib2.Request('http://blog.csdn.net/cqcre')
     try:
         urllib2.urlopen(req)
     except urllib2.URLError, e:
         if hasattr(e,"code"):
             print e.code
         if hasattr(e,"reason"):
             print e.reason
     else:
         print "OK"