[Golang] 一个简易代理池

　　晚上写了一个代理池，就是在一个代理网站上爬取代理ip和端口以及测试是否可用。接下来可能考虑扩展成一个比较大的 golang实现的代理池。

　　简易版代码：

 package main

 import (
     "os"
     "fmt"
     log "github.com/Sirupsen/logrus"
     "io/ioutil"
     "strings"
 )

 type New struct {
     Prefix string
     NewId string
     Title string
     Time string
     Content string
     Subject string
 }

 type Subject struct {
     Name string
     Url string
 }

 func CreateDir(PathName string) error {
     err := os.Mkdir(PathName, 0777)
     if err != nil && !os.IsExist(err) {
         return err
     }
     return nil
 }

 func AppendFile(SavePath string, FileName string, buf string) {
     out, err := os.OpenFile(SavePath+FileName, os.O_WRONLY, 0644)
     defer out.Close()
     if err != nil {
         log.Errorln(err.Error())
         return
     }
     offset, err := out.Seek(0, os.SEEK_END)
     if err != nil {
         log.Errorln(err.Error())
         return
     }
     _, err = out.WriteAt([]byte(buf), offset)
     if err != nil {
         log.Errorln(err.Error())
         return
     }
     log.Warnln("Save file finished. Locate in ", SavePath + FileName)
 }

 func PathExists(path string) bool {
     _, err := os.Stat(path)
     if err == nil {
         return true
     }
     if os.IsNotExist(err) {
         return false
     }
     return false
 }

 func SaveFile(SavePath string, FileName string, buf string) {
     out, err := os.Create(SavePath + FileName)
     defer out.Close()
     fmt.Fprintf(out, "%s", buf)
     if err != nil {
         log.Errorln(err.Error())
         return
     }
     log.Warnln("Save file finished. Locate in ", SavePath + FileName)
 }

 func ReadAll(path string) ([]byte, error) {
     f, err := os.Open(path)
     if err != nil {
         return nil, err
     }
     defer f.Close()
     return ioutil.ReadAll(f)
 }

 func ReadFile(path string) []string {
     var fp interface{}
     fp, err := ReadAll(path)
     if err != nil {
         log.Errorln(err.Error())
         return nil
     }
     fp = string(fp.([]byte))
     return strings.Split(fp.(string), "\n")
 }

 package main

 import (
     log "github.com/Sirupsen/logrus"
     "math/rand"
     "net/http"
     "net/url"
     "time"
 )

 var userAgent = [...]string {
     "Mozilla/5.0 (compatible, MSIE 10.0, Windows NT, DigExt)",
     "Mozilla/4.0 (compatible, MSIE 7.0, Windows NT 5.1, 360SE)",
     "Mozilla/4.0 (compatible, MSIE 8.0, Windows NT 6.0, Trident/4.0)",
     "Mozilla/5.0 (compatible, MSIE 9.0, Windows NT 6.1, Trident/5.0,",
     "Opera/9.80 (Windows NT 6.1, U, en) Presto/2.8.131 Version/11.11",
     "Mozilla/4.0 (compatible, MSIE 7.0, Windows NT 5.1, TencentTraveler 4.0)",
     "Mozilla/5.0 (Windows, U, Windows NT 6.1, en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
     "Mozilla/5.0 (Macintosh, Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
     "Mozilla/5.0 (Macintosh, U, Intel Mac OS X 10_6_8, en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50",
     "Mozilla/5.0 (Linux, U, Android 3.0, en-us, Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
     "Mozilla/5.0 (iPad, U, CPU OS 4_3_3 like Mac OS X, en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
     "Mozilla/4.0 (compatible, MSIE 7.0, Windows NT 5.1, Trident/4.0, SE 2.X MetaSr 1.0, SE 2.X MetaSr 1.0, .NET CLR 2.0.50727, SE 2.X MetaSr 1.0)",
     "Mozilla/5.0 (iPhone, U, CPU iPhone OS 4_3_3 like Mac OS X, en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5",
     "MQQBrowser/26 Mozilla/5.0 (Linux, U, Android 2.3.7, zh-cn, MB200 Build/GRJ22, CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
 }

 func GetRandomUserAgent() string {
     var r = rand.New(rand.NewSource(time.Now().UnixNano()))
     return userAgent[r.Intn(len(userAgent))]
 }

 func GetFakeHeader(request *http.Request) {
     request.Header.Set("User-Agent", GetRandomUserAgent())
     request.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
     request.Header.Set("Connection", "keep-alive")
     //request.Header.Set("Accept-Encoding", "gzip, deflate")
 }

 func GetByProxy(proxyAddr string, Url string) (*http.Response, error) {
     timeout := time.Duration(10 * time.Second)
     request, err := http.NewRequest(http.MethodGet, Url, nil)
     GetFakeHeader(request)
     if err != nil {
         return nil, err
     }
     proxy, err := url.Parse(proxyAddr)
     if err != nil {
         log.Errorln(err.Error())
     }
     client := &http.Client{
         Transport: &http.Transport{
             Proxy: http.ProxyURL(proxy),
         },
         Timeout: timeout,
     }
     return client.Do(request)
 }

 func GetByDirectory(Url string) (*http.Response, error) {
     timeout := time.Duration(10 * time.Second)
     request, err := http.NewRequest(http.MethodGet, Url, nil)
     GetFakeHeader(request)
     if err != nil {
         return nil, err
     }
     client := http.Client{
         Timeout: timeout,
     }
     return client.Do(request)
 }

 package main

 import (
     log "github.com/Sirupsen/logrus"
     "strconv"
     "regexp"
     "github.com/opesun/goquery"
     "time"
 )

 const (
     SAVE_PATH = "kproxy.orz"
     PROXY_URL = "http://www.kuaidaili.com/free/inha/"
 )
 var (
     IP_REGEXP = regexp.MustCompile(`[\d]+\.[\d]+\.[\d]+\.[\d]+\n\s+[\d]+`)
     IP_DETAIL_REGEXP = regexp.MustCompile(`[\d]+\.[\d]+\.[\d]+\.[\d]+`)
     INT_REGEXP = regexp.MustCompile(`\s[\d]+`)
 )

 func UrlGetter(num int) string {
     return PROXY_URL + strconv.Itoa(num)
 }

 func GetProxy(Url string) {
     nod, err := goquery.ParseUrl(Url)
     if err != nil {
         log.Errorln(err.Error())
         return
     }
     ret := nod.Text()
     ips := IP_REGEXP.FindAll([]byte(ret), -1)
     var port []string = make([]string, len(ips))
     var str string = ""
     for i := 0; i < len(ips); i++ {
         port[i] = string(INT_REGEXP.FindAll(ips[i], -1)[0])[1:]
         ips[i] = IP_DETAIL_REGEXP.FindAll(ips[i], -1)[0]
         str += string(ips[i])+":"+port[i]+"\n"
     }
     AppendFile("./", SAVE_PATH, str)
 }

 func main() {
     log.Infoln("Start getting proxy ...")
     SaveFile("./", SAVE_PATH, "")
     for i := 1; i <= 500; i++ {
         log.Println(UrlGetter(i))
         GetProxy(UrlGetter(i))
         time.Sleep(time.Second*5)
     }
 }

这里有个python来测试：

 import urllib
 import urllib2
 import os
 import socket

 AIM_URL = ''
 PROXY_PATH = './kproxy.orz'

 class MyException(Exception):
     pass

 def read_file(path):
     if not os.path.exists(path):
         print 'path : \''+ path + '\' not find.'
         return []
     content = ''
     try:
         with open(path, 'r') as fp:
             content += reduce(lambda x,y:x+y, fp)
     finally:
         fp.close()
     return content.split('\n')

 socket.setdefaulttimeout(5)
 proxies = read_file(PROXY_PATH)
 print len(proxies)
 for pp in proxies:
     try:
         print 'http://'+pp
         inforMation = urllib.urlopen(AIM_URL, proxies={'http': 'http://'+pp})
     except urllib2.URLError, e:
         if isinstance(e.reason, socket.timeout):
             pass
     except:
         pass
     finally:
         pass