Python爬取代理ip

 # -*- coding:utf-8 -*-
 #author : willowj
 import urllib
 import urllib2
 from bs4 import BeautifulSoup
 import re
 import bs4

 import sys

 reload(sys)
 sys.setdefaultencoding('utf8') 

 def ip_test(ip,url= "https://www.baidu.com"):
     #test ip if can be used
     #url = "http://ip.chinaz.com/getip.aspx"# 默认测试网址
     ip1="http://"+ip
     try :
         res = urllib.urlopen(url,proxies={'http:':ip1}).read() #尝试代理访问
         print 'ok',ip1 #,res
         return True
     except Exception,e:
         print "failed"
         return False

 def get_iphtml_inyoudaili():
     url='http://www.youdaili.net'
     html=urllib2.urlopen(url)
     code=html.read()
     #href="http://www.youdaili.net/Daili/http/26672.html" title="12月27号 最新代理http服务器ip地址"
     regexp='href="(.*?)" .*?最新代理http服务器ip地址'
     pat=re.compile(regexp)
     met=re.findall(pat,code)
     print met[0]
     #最新代理http服务器ip地址 html
     return met[0]

 def getIps(url):
     #getip from website, test,and  return,save aviable ips in 'ips.txt'
     htmlip=urllib2.urlopen(url)
     codeip=htmlip.read()

     regexpip='([1-9][0-9]{0,2}\.\S*?)@HTTP#'  #IP样式
     pat_ip=re.compile(regexpip) 

     met_ip=re.findall(pat_ip,codeip)

     ips=[]
     file_open=open('ips.txt','w')
     for x in met_ip:
         print x
         if ip_test(x):
             ips.append(x)
             file_open.write(x+'\n')
     file_open.close()
     #print ips,'youdaili'
     return ips

 def saveIps(list):
     file_open=open('ips.txt','w')
     for ip in list:
         file_open.write(ip+'\n')
     file_open.close()

 def read_ips(file='ips.txt'):
     '''读取IP 以list返回'''
     file_open=open(file)
     lines=file_open.readlines()
     ips=[]
     for line in  lines:
         ip=line.strip("\n")
         ips.append(ip)
     print ips
     return ips

 if __name__=="__main__":

     ips = getIps(get_iphtml_inyoudaili())

     saveIps(ips)