scrapy 爬取自己的博客

定义项目

# -*- coding: utf-8 -*-
# items.py

import scrapy

class LianxiCnblogsItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    url = scrapy.Field()
    title = scrapy.Field()
    article = scrapy.Field()
    post_date = scrapy.Field()

定义爬虫

# -*- coding: utf-8 -*-
# spider/cnblogs_spider.py

from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor

from lianxi_cnblogs.items import LianxiCnblogsItem

class MininovaSpider(CrawlSpider):

    name = 'cnblogs'
    allowed_domains = ['cnblogs.com']
    start_urls = ['http://www.cnblogs.com/hhh5460/default.html?page=%s' i+1 for i in range(11)] # 分页
    rules = [Rule(LinkExtractor(allow=['/p/\d+\.html']), 'parse_cnblogs')]

    def parse_cnblogs(self, response):
        res = LianxiCnblogsItem()
        res['url'] = response.url
        res['title'] = response.xpath("//h1/a/text()").extract()
        res['article'] = response.xpath("//div[@id=topics]").extract()
        res['post_date'] = response.xpath("//span[@id='post-date']/text()").extract()
        return res
        

运行爬虫

$ scrapy crawl cnblogs -o results.json