python 爬取网页内的代理服务器列表（需调整优化）

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 # @Date    : 2017-08-30 20:38:23
 # @Author  : EnderZhou (zptxwd@gmail.com)
 # @Link    : http://www.cnblogs.com/enderzhou/
 # @Version : $Id$

 import requests
 from bs4 import BeautifulSoup as bs

 # 这种爬取网页内容中的列表的方式复用性差，不同的网站需要针对性的修改。每次使用均需要填写更换header头。后续将编写适用性更强的版本。

 url = 'http://www.kuaidaili.com/free/inha/'

 headers = {
 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
 'Accept-Encoding':'gzip, deflate',
 'Accept-Language':'zh-CN,zh;q=0.8,zh-TW;q=0.6',
 'AlexaToolbar-ALX_NS_PH':'AlexaToolbar/alx-4.0.1',
 'Cache-Control':'max-age=0',
 'Connection':'keep-alive',
 'Cookie':'yd_cookie=a0d0f393-2812-44d0b1453fbf740f3ce870820ada37151e8c; _ydclearance=dd0b3de069ce8a768712e248-d97e-4bd9-8284-f2ef598da35b-1504104455; channelid=0; sid=1504099004948599; _ga=GA1.2.742898386.1504074603; _gid=GA1.2.583101265.1504074603; Hm_lvt_7ed65b1cc4b810e9fd37959c9bb51b31=1504074603,1504097260; Hm_lpvt_7ed65b1cc4b810e9fd37959c9bb51b31=1504099719',
 'Host':'www.kuaidaili.com',
 'Upgrade-Insecure-Requests':'',
 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36',
 }

 def proxy_check(types,ip,port):
     headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'}
     url = 'http://www.whatismyip.com.tw/'
     proxy = {}
     proxy[types.lower()] = '%s:%s' % (ip,port)
     print proxy
     try:
         r = requests.get(url,headers=headers,proxies=proxy)
         soup = bs(r.content,'html.parser')
         chack_ip = soup.find_all(name='b')
         print chack_ip[0].string+':'+port
     except Exception,e:
         # print e
         pass

 # proxy_check('http','183.62.11.242','8088')#可用于测试代理验证模块

 def main():
     r = requests.get(url=url,headers=headers)
     tr_soup = bs(r.content,'html.parser')
     tr = tr_soup.find_all(name='tr')
     for i in tr:
         # print i
         td_soup = bs(str(i),'html.parser')
         td = td_soup.find_all(name='td')
         if len(td) != 0:
             ip = str(td[0].string)
             port = str(td[1].string)
             types = str(td[3].string)
             proxy_check(types,ip,port)

 if __name__ == '__main__':
     main()