用代理IP进行简单的爬虫——爬高匿代理网站

用西刺代理网站的IP爬高匿代理网站

 import re
 import _thread
 from time import sleep,ctime
 from urllib.request import urlopen
 from urllib.request import Request
 from urllib.request import ProxyHandler
 from urllib.request import build_opener
 from lxml import etree

 url = "http://www.kuaidaili.com/free/"
 #设置代理IP
 proxy = {'http':'113.123.13.149:808'}
 proxy_support = ProxyHandler(proxy)
 opener = build_opener(proxy_support)
 #设置访问协议头
 opener.addheaders = [('User-agent','Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6')]
 r = opener.open(url)
 html = r.read().decode('utf-8')
 print(html)
 selector = etree.HTML(html)
 links = selector.xpath('//tr/td/text()')
 for link in links:
     print(link)