python学习-爬虫

转载自 静觅的博客

最普通下载网页

 import urrlib2
 response = urllib2.urlopen("http://www.baidu.com")
 print response.read()

Post方式

 import urllib
 import urllib2

 values = {"username":"*****", "password":"*****"}
 url = "   "
 request = urllib2.Request(url,urllib.urlencode(values))
 response = urllib2.urlopen(request)
 print response.read()

设置headers

import urllib
import urllib2  

url = 'http://www.server.com/login'
user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
values = {'username' : 'cqc',  'password' : 'XXXX' }
headers = { 'User-Agent' : user_agent }
data = urllib.urlencode(values)
request = urllib2.Request(url, data, headers)
response = urllib2.urlopen(request)
page = response.read() 

Get方式

 import urllib2
 import urllib

 values = {}
 values["username"] =
 values["password"] =
 data = urlencode(values)
 url =
 geturl = url + "?" + data
 request = urllib2.Request(geturl)
 response = urllib2.urlopen(request)
 print response.read()

设置代理

 import urllib2
 enable_proxy = True
 proxy_handler = urllib2.ProxyHandler({"http" : 'http://some-proxy.com:8080'})
 null_proxy_handler = urllib2.ProxyHandler({})
 if enable_proxy:
     opener = urllib2.build_opener(proxy_handler)
 else:
     opener = urllib2.build_opener(null_proxy_handler)
 urllib2.install_opener(opener)

设置延时

 import urllib2
 response = urllib2.urlopen('http://www.baidu.com',data, 10)

异常处理

 import urllib2

 req = urllib2.Request('http://blog.csdn.net/cqcre')
 try:
     urllib2.urlopen(req)
 except urllib2.URLError, e:
     if hasattr(e,"code"):
         print e.code
     if hasattr(e,"reason"):
         print e.reason
 else:
     print "OK"

设置cookie

 import urllib
 import urllib2
 import cookielib

 filename = 'cookie.txt'
 #声明一个MozillaCookieJar对象实例来保存cookie，之后写入文件
 cookie = cookielib.MozillaCookieJar(filename)
 opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
 postdata = urllib.urlencode({
             'stuid':'',
             'pwd':''
         })
 #登录教务系统的URL
 loginUrl = 'http://jwxt.sdu.edu.cn:7890/pls/wwwbks/bks_login2.login'
 #模拟登录，并把cookie保存到变量
 result = opener.open(loginUrl,postdata)
 #保存cookie到cookie.txt中
 cookie.save(ignore_discard=True, ignore_expires=True)
 #利用cookie请求访问另一个网址，此网址是成绩查询网址
 gradeUrl = 'http://jwxt.sdu.edu.cn:7890/pls/wwwbks/bkscjcx.curscopre'
 #请求访问成绩查询网址
 result = opener.open(gradeUrl)
 print result.read()