urllib 获取页面或发送信息

#! /usr/bin/env python3

# -*- coding:utf-8 -*-

#urllib提供了一系列用于操作URL的功能。

#urllib的request模块可以非常方便地抓取URL内容，也就是发送一个GET请求到指定的页面，然后返回HTTP的响应：

#例如，对豆瓣的一个URL ： https://api.douban.com/v2/book/2129650 进行抓取，并返回响应：

from urllib import request

with request.urlopen('https://api.douban.com/v2/book/2129650') as f :

data=f.read()

print('Status:',f.status,f.reason)

for k,v in f.getheaders():

print('%s:%s' % (k,v))

print('Data:',data.decode('utf-8'))

'''

Status: 200 OK

Date:Wed, 10 May 2017 07:15:35 GMT

Content-Type:application/json; charset=utf-8

Content-Length:2055

Connection:close

Vary:Accept-Encoding

X-Ratelimit-Remaining2:99

X-Ratelimit-Limit2:100

Expires:Sun, 1 Jan 2006 01:00:00 GMT

Pragma:no-cache

Cache-Control:must-revalidate, no-cache, private

Set-Cookie:bid=tJuu061vOU0; Expires=Thu, 10-May-18 07:15:35 GMT; Domain=.douban.com; Path=/

X-DOUBAN-NEWBID:tJuu061vOU0

X-DAE-Node:sindar7d

X-DAE-App:book

Server:dae

Data: {"rating":{"max":10,"numRaters":16,"average":"7.4","min":0},"subtitle":"","author":["廖雪峰编著"],"pubdate":"2007-6","tags":[{"count":21,"name":"spring","title":"spring"},{"count":13,"name":"Java","title":"Java"},{"count":6,"name":"javaee","title":"javaee"},{"count":5,"name":"j2ee","title":"j2ee"},{"count":4,"name":"计算机","title":"计算机"},{"count":3,"name":"POJO","title":"POJO"},{"count":3,"name":"藏书","title":"藏书"},{"count":3,"name":"编程","title":"编程"}],"origin_title":"","image":"https://img3.doubanio.com\/mpic\/s2648230.jpg","binding":"","translator":[],"catalog":"","pages":"509","images":{"small":"https://img3.doubanio.com\/spic\/s2648230.jpg","large":"https://img3.doubanio.com\/lpic\/s2648230.jpg","medium":"https://img3.doubanio.com\/mpic\/s2648230.jpg"},"alt":"https:\/\/book.douban.com\/subject\/2129650\/","id":"2129650","publisher":"电子工业","isbn10":"7121042622","isbn13":"9787121042621","title":"Spring 2.0核心技术与最佳实践","url":"https:\/\/api.douban.com\/v2\/book\/2129650","alt_title":"","author_intro":"","summary":"本书注重实践而又深入理论，由浅入深且详细介绍了Spring 2.0框架的几乎全部的内容，并重点突出2.0版本的新特性。本书将为读者展示如何应用Spring 2.0框 架创建灵活高效的JavaEE应用，并提供了一个真正可直接部署的完整的Web应用程序——Live在线书店(http:\/\/www.livebookstore.net)。\n在介绍Spring框架的同时，本书还介绍了与Spring相关的大量第三方框架，涉及领域全面，实用性强。本书另一大特色是实用性强 ，易于上手，以实际项目为出发点，介绍项目开发中应遵循的最佳开发模式。\n本书还介绍了大量实践性极强的例子，并给出了完整的配置步骤，几乎覆盖了Spring 2.0版本的新特性。\n本书适合有一定Java基础的读者，对JavaEE开发人员特别有帮助。本书既可以作为Spring 2.0的学习指南，也可以作为实际项目开发的参考手册。","price":"59.80元"}

'''

#可以看到HTTP响应的头和JSON数据

#如果我们想要模拟浏览器发送get请求，就需要使用request对象，通过往request对象添加HTTP头，就可以把请求伪装成浏览器。

#例，模拟iPhone 6 去请求豆瓣首页：

from urllib import request

req=request.Request('http://www.douban.com/')

req.add_header('User-Agent', 'Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25')

with request.urlopen(req) as f :

print('Status:',f.status,f.reason)

for k,v in f.getheaders():

print('%s:%s' % (k,v))

print('Data:',f.read().decode('utf-8'))

#这样豆瓣会返回合适iPhone的移动版网页：

'''

...

<meta name="viewport" content="width=device-width, user-scalable=no, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0">

<meta name="format-detection" content="telephone=no">

<link rel="apple-touch-icon" sizes="57x57" href="http://img4.douban.com/pics/cardkit/launcher/57.png" />

...

'''

#Post

#如果要以POST发送一个请求，只需要把参数data以bytes形式传入。

#我们模拟一个微博登录，先读取登录的邮箱和口令，然后按照weibo.cn 的登录页的格式以username=xxx&password=xxx 的编码传入：

from urllib import request,parse

print('Login to weibo.cn...')

email=input('Email:')

passwd=input('Password:')

login_data=parse.urlencode([

('username',email),

('password',passwd),

('entry','mweibo'),

('client_id','l'),

('ec',''),

('pagerefer','https://passport.weibo.cn/signin/welcome?entry=mweibo&r=http%3A%2F%2Fm.weibo.cn%2F')

])

req=request.Request('https://passport.weibo.cn/sso/login')

req.add_header('Origin','https://passport.weibo.cn')

req.add_header('User-Agent', 'Mozilla/6.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/8.0 Mobile/10A5376e Safari/8536.25')

req.add_header('Referer', 'https://passport.weibo.cn/signin/login?entry=mweibo&res=wel&wm=3349&r=http%3A%2F%2Fm.weibo.cn%2F')

with request.urlopen(req,data=login_data.encode('utf-8')) as f:

print('Status:',f.status,f.reason)

for k,v in f.getheaders():

print('%s:%s' % (k,v))

print('Data:',f.read().decode('utf-8'))

#如果登录成功，我们获得的响应如下：

'''

Status: 200 OK

Server: nginx/1.2.0

...

Set-Cookie: SSOLoginState=1432620126; path=/; domain=weibo.cn

...

Data: {"retcode":20000000,"msg":"","data":{...,"uid":"1658384301"}}

'''

#如果登录失败，我们获得的响应如下：

#...

#Data: {"retcode":50011015,"msg":"\u7528\u6237\u540d\u6216\u5bc6\u7801\u9519\u8bef","data":{"username":"example@python.org","errline":536}}

#Handler

#如果还需要更复杂的控制，比如通过一个Proxy去访问网站，我们需要利用ProxyHandler来处理，例：

proxy_handler=urllib.request.ProxyHandler({'http':'http://www.example.com:3128/'})

proxy_auth_handler=urllib.request.ProxyBasicAuthHandler()

proxy_auth_handler.add_password('realm','host','username','password')

opener=urllib.request.build_opener(proxy_handler,proxy_auth_handler)

with opener.open('http://www.example.com/login.html') as f

pass

#urllib提供的功能就是利用程序去执行各种Http请求。如果要模拟浏览器完成特定功能，需要把请求伪装成浏览器。伪装的方法是先监控浏览器发出的请求，再根据

#浏览器的请求头来伪装，User-Agent头就是用来标识浏览器的。