go 爬虫

 package main  

    import (
        "fmt"
        "io"
        "io/ioutil"
        "net/http"
        "os"
        "regexp"
        "strconv"
        "strings"
        "time"
    )  

    var (
        // regHref       = `((ht|f)tps?)://[w]{0,3}.baidu.com/link\?[a-zA-z=0-9-\s]*`
        regTitle      = `<title[\sa-zA-z="-]*>([^x00-xff]|[\sa-zA-Z=-：|，？"])*</title>`
        regCheckTitle = `(为什么|怎么)*.*([G|g][O|o][L|l][A|a][N|n][G|g]).*(怎么|实现|如何|为什么).*`
    )  

    func main() {
       if checkFile("./data/", "url.txt").Size() == 0 {
            fistStart()
            main()
        }         else {
        Timer()
        }
    }  

    func Timer() {
        t := time.NewTimer(time.Second * 1)
        <-t.C
        fmt.Print("\n\n\n执行爬抓\n\n")
        f, _ := os.OpenFile("./data/url.txt", os.O_CREATE|os.O_APPEND|os.O_RDWR, 0666)
        file, _ := ioutil.ReadAll(f)
        pageCont, _ := pageVisit(strings.Split(string(file), "\n")[0])
        if checkRegexp(checkRegexp(pageCont, regTitle, 0).(string), regCheckTitle, 0).(string) != "" {
            fmt.Print(checkRegexp(checkRegexp(pageCont, regTitle, 0).(string), regCheckTitle, 0).(string))
            fmt.Print("\n有效内容 => " + checkRegexp(pageCont, regTitle, 0).(string))
        }
        fmt.Print("\n\n待爬抓网址共" + strconv.Itoa(len(strings.Split(string(file), "\n"))-1) + "个 => " + strings.Split(string(file), "\n")[0] + "\n")
        DelFirstText("./data/url.txt")
        Timer()
    }  

    func fistStart() {
        var num int
        url := "http://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&tn=39042058_20_oem_dg&wd=golang%E5%AE%9E%E7%8E%B0&oq=golang%2520%25E5%2588%25A0%25E9%2599%25A4%25E6%2595%25B0%25E7%25BB%2584&rsv_pq=d9be28ec0002df1b&rsv_t=8017GWpSLPhDmKilZQ1StC04EVpUAeLEP90NIm%2Bk5pRh5R9o57NHMO8Gaxm1TtSOo%2FvtJj%2B98%2Fsc&rqlang=cn&rsv_enter=1&inputT=3474&rsv_sug3=16&rsv_sug1=11&rsv_sug7=100&rsv_sug2=0&rsv_sug4=4230"
        resp, _ := http.Get(url)
        defer resp.Body.Close()
        body, _ := ioutil.ReadAll(resp.Body)
        reg := regexp.MustCompile(`((ht|f)tps?)://[w]{0,3}.baidu.com/link\?[a-zA-z=0-9-\s]*`)
        f, _ := os.OpenFile("./data/url.txt", os.O_CREATE|os.O_APPEND|os.O_RDWR, 0666)
        defer f.Close()
        for _, d := range reg.FindAllString(string(body), -1) {
            ff, _ := os.OpenFile("./data/url.txt", os.O_RDWR, 0666)
            file, _ := ioutil.ReadAll(ff)
            dd := strings.Split(d, "")
            dddd := ""
            for _, ddd := range dd {
                if ddd == "?" {
                    ddd = `\?`
                }
                dddd += ddd
            }
            if checkRegexp(string(file), dddd, 0).(string) == "" {
                io.WriteString(f, d+"\n")
                fmt.Print("\n收集地址：" + d + "\n")
                num++
            }
            // fmt.Print(string(file))
            ff.Close()
        }
        fmt.Print("\n首次收集网络地址：" + strconv.Itoa(len(reg.FindAllString(string(body), -1))) + "\n")
        fmt.Print("\n去重后网络地址数：" + strconv.Itoa(num))
        fmt.Print("\n\n首次储存成功！\n")
    }  

    func pageVisit(url string) (page string, body []byte) {
        resp, _ := http.Get(url)
        defer resp.Body.Close()
        body, _ = ioutil.ReadAll(resp.Body)
        page = string(body)
        return
    }  

    func checkFile(dir string, file string) os.FileInfo {
        list, _ := ioutil.ReadDir(dir)
        for _, info := range list {
            if info.Name() == file {
                return info
            }
        }
        return list[0]
    }  

    func saveFile(file string, cont string) {
        f, _ := os.OpenFile(file, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
        defer f.Close()
        io.WriteString(f, cont)
    }  

    func checkRegexp(cont string, reg string, style int) (result interface{}) {
        check := regexp.MustCompile(reg)
        switch style {
        case 0:
            result = check.FindString(cont)
        case 1:
            result = check.FindAllString(cont, -1)
        default:
            result = check.FindAll([]byte(cont), -1)
        }
        return
    }  

    func DelFirstText(file string) {
        var text = ""
        f, _ := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0666)
        files, _ := ioutil.ReadAll(f)
        var ss = strings.Split(string(files), "\n")
        for i := 1; i < len(ss)-1; i++ {
            text += ss[i] + "\n"
        }
        defer f.Close()
        ioutil.WriteFile(file, []byte(text), 0666)
        fmt.Print("\n\n删除该地址 => " + ss[0])
    }