python-爬免费ip并验证其可行性

前言

最近在重新温习python基础-正则，感觉正则很强大，不过有点枯燥，想着，就去应用正则，找点有趣的事玩玩

00xx01---代理IP

有好多免费的ip,不过一个一个保存太难了，也不可能，还是用我们的python爬取吧

00xx02---正则提取ip

 import requests
 import re

 #防反爬
 headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36" }

 url = "https://www.xicidaili.com/nn/1"

 response = requests.get(url,headers=headers)
     # print(response.text)

 html = response.text
 #print(html)

 #re.S忽略换行的干扰
 ips = re.findall("<td>(\d+\.\d+\.\d+\.\d+)</td>",html,re.S)
 ports = re.findall(("<td>(\d+)</td>"),html,re.S)
 print(ips)
 print(ports)

00xx03---拼接IP和端口

 import requests
 import re

 #防反爬
 headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36" }

 url = "https://www.xicidaili.com/nn/1"

 response = requests.get(url,headers=headers)
     # print(response.text)

 html = response.text
 # print(html)

 #re.S忽略换行的干扰
 ips = re.findall("<td>(\d+\.\d+\.\d+\.\d+)</td>",html,re.S)
 ports = re.findall(("<td>(\d+)</td>"),html,re.S)
 #print(ips)
 #print(ports)
 for ip in zip(ips,ports ):  #提取拼接ip和端口
     print(ip)

00xx03---验证IP可行性

思路：带着ip和端口去访问一个网站，百度就可以

 import requests
 import re

 headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36" }
 for i in range(1,1000):
     #网址
     url = "https://www.xicidaili.com/nn/{}".format(i)

     response = requests.get(url,headers=headers)
     # print(response.text)

     html = response.text

     #re.S忽略换行的干扰
     ips = re.findall("<td>(\d+\.\d+\.\d+\.\d+)</td>",html,re.S)
     ports = re.findall(("<td>(\d+)</td>"),html,re.S)
     # print(ips)
     # print(ports)
     for ip in zip(ips,ports ):  #提取拼接ip和端口
         proxies = {
             "http":"http://" + ip[0] + ":" + ip[1],
             "https":"http://" + ip[0] + ":" + ip[1]
         }
         try:
             res = requests.get("http://www.baidu.com",proxies=proxies,timeout = 3)  #访问网站等待3s没有反应，自动断开
             print(ip,"能使用")
             with open("ip.text",mode="a+") as f:
                 f.write(":".join(ip))  #写入ip.text文本
                 f.write("\n") #换行
         except Exception as e:   #捕捉错误异常
             print(ip,"不能使用")

00xx04---写入文本

 import requests
 import re

 #防反爬
 headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36" }

 url = "https://www.xicidaili.com/nn/1"

 response = requests.get(url,headers=headers)
     # print(response.text)

 html = response.text
 # print(html)

 #re.S忽略换行的干扰
 ips = re.findall("<td>(\d+\.\d+\.\d+\.\d+)</td>",html,re.S)
 ports = re.findall(("<td>(\d+)</td>"),html,re.S)
 #print(ips)
 #print(ports)
 for ip in zip(ips,ports ):  #提取拼接ip和端口
     print(ip)
     proxies = {
             "http":"http://" + ip[0] + ":" + ip[1],
             "https":"http://" + ip[0] + ":" + ip[1]
         }
     try:
         res = requests.get("http://www.baidu.com",proxies=proxies,timeout = 3)  #访问网站等待3s没有反应，自动断开
         print(ip,"能使用")
         with open("ip.text",mode="a+") as f:
             f.write(":".join(ip))  #写入ip.text文本
             f.write("\n") #换行
     except Exception as e:   #捕捉错误异常
         print(ip,"不能使用")

爬了一页，才几个能用，有3000多页，不可能手动的

00xx05---批量爬

 #!/usr/bin/env python3
 # coding:utf-8
 # 2019/11/18 22:38
 #lanxing
 import requests
 import re

 #防反爬
 headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36" }
 for i in range(1,3000):  #爬3000个网页
     #网站
     url = "https://www.xicidaili.com/nn/{}".format(i)

     response = requests.get(url,headers=headers)
         # print(response.text)

     html = response.text
     # print(html)

     #re.S忽略换行的干扰
     ips = re.findall("<td>(\d+\.\d+\.\d+\.\d+)</td>",html,re.S)
     ports = re.findall(("<td>(\d+)</td>"),html,re.S)
     #print(ips)
     #print(ports)
     for ip in zip(ips,ports ):  #提取拼接ip和端口
         print(ip)
         proxies = {
                 "http":"http://" + ip[0] + ":" + ip[1],
                 "https":"http://" + ip[0] + ":" + ip[1]
             }
         try:
             res = requests.get("http://www.baidu.com",proxies=proxies,timeout = 3)  #访问网站等待3s没有反应，自动断开
             print(ip,"能使用")
             with open("ip.text",mode="a+") as f:
                 f.write(":".join(ip))  #写入ip.text文本
                 f.write("\n") #换行
         except Exception as e:   #捕捉错误异常
             print(ip,"不能使用")

00xx06---最后

哈哈，感觉爬的速度太慢了，毕竟是单线程，如果要快速爬，可以试试用多线程爬取，

以后再补充完善代码吧