pyspider示例代码四：搜索引擎爬取

搜索引擎爬取

#!/usr/bin/env python
# -*- encoding: utf- -*-
# Created on -- ::
# Project: __git_lab_fix

from pyspider.libs.base_handler import *

class Handler(BaseHandler):
    crawl_config = {
    }

    @every(minutes= * )
    def on_start(self):
        list = ['bigsec', 'password', 'email', 'tongdun', 'vpn', 'address', 'pop3',
                'smtp', 'imap', 'zhengxin', 'jdbc', 'mysql', 'credit', 'access_token', 'client_secret',
                'privatekey', 'secret_key', 'xiecheng', 'ctrip', 'tongcheng']
        for u in list:
            url = 'https://gitlab.com/search?group_id=&scope=issues&search=' + u
            self.crawl(url, callback=self.index_page)

    @config(age=)
    def index_page(self, response):
        self.crawl(response.doc('.next > a').attr.href,callback = self.index_page)
        for each in response.doc('h4 > a[href^="http"]').items():
            # print each.text()
            self.crawl(each.attr.href, callback=self.detail_page)

    @config(etag = True)
    def detail_page(self, response):
        for each in response.doc('.detail-page-description').items():
            return {
                "app":"githack",
                "origin":"gitlab.net",
                "code": each.text(),
            }

#!/usr/bin/env python
# -*- encoding: utf- -*-
# Created on -- ::
# Project: __git_zhibo

from pyspider.libs.base_handler import *

class Handler(BaseHandler):
    crawl_config = {
    }

    @every(minutes=  * )
    def on_start(self):
        list = ['douyu', 'panda', 'zhanqi', 'longzhu', 'huya', 'yy', 'momo', 'tv']
        for qu in list:
            url = 'https://github.com/search?p=1&q=' + qu + '&type=Code&utf8'
            self.crawl(url, callback=self.index_page)

    @config(age=)
    def index_page(self, response):
        self.crawl(response.doc('.next_page').attr.href,callback = self.index_page)
        flag =
        for each in response.doc('.title > a').items():
            flag +=
            if flag %  == :
                self.crawl(each.attr.href, callback=self.into_page)

    @config(age=,etag = True)
    def into_page(self, response):
        for each in response.doc('table').items():
            return{
                "app":"githack",
                "origin":"github.net",
                "code": each.text(),
            }