爬虫框架Scrapy之案例一

阳光热线问政平台

http://wz.sun0769.com/index.php/question/questionType?type=4

爬取投诉帖子的编号、帖子的url、帖子的标题，和帖子里的内容。

items.py

import scrapy

class SunwzItem(scrapy.Item):
    number = scrapy.Field()
    url = scrapy.Field()
    title = scrapy.Field()
    content = scrapy.Field()

spiders/sunwz.py

# -*- coding: utf-8 -*-

from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from Sunwz.items import SunwzItem

class SunwzSpider(CrawlSpider):
    name = 'sunwz'
    num = 0
    allow_domain = ['http://wz.sun0769.com/']
    start_urls = ['http://wz.sun0769.com/index.php/question/questionType?type=4']
    rules = {
        Rule(LinkExtractor(allow='page')),
        Rule(LinkExtractor(allow='/index\.php/question/questionType\?type=4$')),
        Rule(LinkExtractor(allow='/html/question/\d+/\d+\.shtml$'), follow = True, callback='parse_content')
    }

    xpathDict = {
        'title': '//div[contains(@class, "pagecenter p3")]/div/div/div[contains(@class,"cleft")]/strong/text()',
        'content': '//div[contains(@class, "c1 text14_2")]/text()',
        'content_first': '//div[contains(@class, "contentext")]/text()'
    }

    def parse_content(self, response):
        item = SunwzItem()
        content = response.xpath(self.xpathDict['content_first']).extract()
        if len(content) == 0:
            content = response.xpath(self.xpathDict['content']).extract()[0]
        else:
            content = content[0]
        title = response.xpath(self.xpathDict['title']).extract()[0]
        title_list = title.split(' ')
        number = title_list[-1]
        number = number.split(':')[-1]
        url = response.url
        item['url'] = url
        item['number'] = number
        item['title'] = title
        item['content'] = content

        yield item

pipelines.py

import json
import codecs

class JsonWriterPipeline(object):

    def __init__(self):
        self.file = codecs.open('sunwz.json', 'w', encoding='utf-8')

    def process_item(self, item, spider):
        line = json.dumps(dict(item), ensure_ascii=False) + "\n"
        self.file.write(line)
        return item

    def spider_closed(self, spider):
        self.file.close()

settings.py

ITEM_PIPELINES = {
    'Sunwz.pipelines.JsonWriterPipeline': 300,
}

在项目根目录下新建main.py文件,用于调试

from scrapy import cmdline
cmdline.execute('scrapy crawl sunwz'.split())

执行程序

py2 main.py