scrapy框架爬取笔趣阁完整版

继续上一篇,这一次的爬取了小说内容

pipelines.py

import csv

class ScrapytestPipeline(object):

# 爬虫文件中提取数据的方法每yield一次item，就会运行一次

# 该方法为固定名称函数

def process_item(self, item, spider):

# csv保存

with open(item['book_name'] + '.csv', "a", newline='', encoding='gb18030') as csvfile:

# print(item)

writer = csv.writer(csvfile)

name = item['section_name']

content = item['section_content']

writer.writerows([[name, content]])

return item

==============================

item.py

import scrapy

设置 爬取的key field

class BiqugeItem_detail(scrapy.Item):

section_link = scrapy.Field()

section_name = scrapy.Field()

section_content = scrapy.Field()

book_name = scrapy.Field()

==============================

biquge.py

-- coding: utf-8 --

import time

import copy

import scrapy

自定义spider类，继承scrapy.spider

from scrapytest.items import BiqugeItem_detail

class BiqugeSpider(scrapy.Spider):

# 爬虫名字

name = 'biquge'

# 允许爬取的范围，防止爬虫爬到别的网站

allowed_domains = ['xbiquge.la']

# 开始爬取的url地址

start_urls = ['http://www.xbiquge.la/xiaoshuodaquan/']

# 数据提取的方法，接受下载中间件传过来的response
def parse(self, response):
    # 分组
    li_list = response.xpath('//div[@class="novellist"]//a')
    i = 0
    for li in li_list:
        # 创建一个数据字典
        dict_data = {}
        # 利用scrapy封装好的xpath选择器定位元素，并通过extract()或extract_first()来获取结果
        dict_data['name'] = li.xpath('.//text()').extract_first()  # 书名
        dict_data['link'] = li.xpath('.//@href').extract_first()  # 书链接
        # yield dict_data
        if i < 3:
            yield scrapy.Request(dict_data['link'], callback=self.parse_detail, meta={'dict_data': dict_data})
        i += 1  书籍计数

def parse_detail(self, response):
    book = {}
    # book['book_name'] = response.meta['dict_data']['name']
    section_data = BiqugeItem_detail()
    section_data['book_name'] = response.meta['dict_data']['name']
    section_list = response.xpath('//*[@id="list"]/dl/dd/a')
    i = 0
    for section in section_list:
        section_data['section_link'] = 'http://www.xbiquge.la' + section.xpath('./@href').extract_first()
        section_data['section_name'] = section.xpath('./text()').extract_first()
        # book['detail'] = section_data
        # yield book
        # if i < 3:
        time.sleep(0.01)
        yield scrapy.Request(section_data['section_link'], callback=self.parse_content, meta=copy.deepcopy({'section_data': section_data}))
        i += 1  # 章节计数

def parse_content(self, response):
    section_data = response.meta['section_data']
    section_data['section_content'] = response.xpath('//*[@id="content"]/text()').extract()
    yield section_data

=======================================================================================

保存的表格格式,因为是异步,所以章节是乱序的,按章节排序即可