python 批量爬取代理ip

 import urllib.request
 import re
 import time
 import random

 def getResponse(url):
     req = urllib.request.Request(url)
     req.add_header("User-Agent","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36")

     resp = urllib.request.urlopen(req)
     return resp.read()

 def getHtml(url,charSet = "utf-8"):
     return getResponse(url).decode(charSet)

 def createOpenner(ipList):
     m_proxy = urllib.request.ProxyHandler({"http":random.choice(ipList)})
     openner = urllib.request.build_opener(m_proxy)
     urllib.request.install_opener(openner)

 #
 def getProxyList(url,iPage = 10):
     ipList = []
     for i in range(1,iPage+1):
         html_str = getHtml(url+str(i))
         ip = re.findall("IP\">((?:\d{1,3}\.){3}(?:\d{1,3}))(?:[\s\S]{0,50})\"PORT\">(\d{2,4})", html_str)
         for addr in ip:
             ipList.append(addr[0]+":"+addr[1])
         time.sleep(2)

     return ipList

 ipList = getProxyList("http://www.kuaidaili.com/free/outha/",1)#爬取1页

 print(ipList)

快代理："IP\">((?:\d{1,3}\.){3}(?:\d{1,3}))(?:[\s\S]*?)\"PORT\">(\d{2,4})"     #下划线处原来是[\s\S]*，不带问号，后果是默认的贪婪模式。

只能取到一个地址，加上问号开启非贪婪模式。

西刺："((?:\d{1,3}\.){3}(?:\d{1,3}))(?:[\s\S]*?)(\d{2,4})"

返回ipList列表：['46.101.3.126:8118', '177.207.234.14:80', '113.255.49.49:80', '52.59.18.222:80', '36.81.0.138:8080', '54.165.24.194:80', '115.252.35.104:8080', '136.169.58.21:8080', '51.254.106.65:80', '178.238.213.246:8080', '49.205.212.243:8080', '137.135.166.225:8131', '168.63.24.174:8138', '179.243.46.131:8080', '186.90.160.245:8080']