scrapy文件管道

安装scrapy

pip install scrapy

新建项目

(python36) E:\www>scrapy startproject fileDownload
New Scrapy project 'fileDownload', using template directory 'c:\users\brady\.conda\envs\python36\lib\site-packages\scrapy\templates\project', created in:
    E:\www\fileDownload

You can start your first spider with:
    cd fileDownload
    scrapy genspider example example.com

(python36) E:\www>

(python36) E:\www>scrapy startproject fileDownload
New Scrapy project 'fileDownload', using template directory 'c:\users\brady\.conda\envs\python36\lib\site-packages\scrapy\templates\project', created in:
    E:\www\fileDownload

You can start your first spider with:
    cd fileDownload
    scrapy genspider example example.com

(python36) E:\www>

编辑爬虫提取内容

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

from  fileDownload.items import  FiledownloadItem

class PexelsSpider(CrawlSpider):
    name = 'pexels'
    allowed_domains = ['www.pexels.com']
    start_urls = ['https://www.pexels.com/photo/white-concrete-building-2559175/']

    rules = (
        Rule(LinkExtractor(allow=r'/photo/'), callback='parse_item', follow=True),
    )

    def parse_item(self, response):
        print(response.url)
        url = response.xpath("//img[contains(@src,'photos')]/@src").extract()
        item = FiledownloadItem()
        try:
            item['file_urls'] = url
            print("爬取到图片列表 " + url)
            yield item
        except Exception as  e:
            print(str(e))

配置item

class FiledownloadItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    file_urls = scrapy.Field()

　　

setting.py

启用文件管道

'scrapy.pipelines.files.FilesPipeline':2  文件管道

FILES_STORE=''  //存储路径

item里面

file_urls = scrapy.Field()

files = scrapy.field()

爬虫里面 改为file_urls参数传递到管道

重写文件管道 保存文件名为图片原名

pipelines.php里面 新建自己图片管道，继承图片管道

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

from scrapy.pipelines.files import  FilesPipeline
class FiledownloadPipeline(object):
    def process_item(self, item, spider):
        tmp = item['file_urls']
        item['file_urls'] = []

        for i in tmp:
            if "?" in i:
                item['file_urls'].append(i.split('?')[0])
            else:
                item['file_urls'].append(i)
        print(item)
        return item

class  MyFilesPipeline(FilesPipeline):
    def file_path(self, request, response=None, info=None):
        file_path = request.url
        file_path = file_path.split('/')[-1]
        print("下载图片"+ file_path)
        return 'full/%s' % (file_path)

setting.py 改为启用自己文件管道

ITEM_PIPELINES = {
    'fileDownload.pipelines.FiledownloadPipeline': 1,
    'fileDownload.pipelines.MyFilesPipeline': 2,
    #'scrapy.pipelines.files.FilesPipeline':2
}

获取套图

# -*- coding: utf-8 -*-
from time import sleep

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

class AngelSpider(CrawlSpider):
    name = 'angel'
    allowed_domains = ['angelimg.spbeen.com']
    start_urls = ['http://angelimg.spbeen.com/']

    base_url = "http://angelimg.spbeen.com"
    rules = (
        Rule(LinkExtractor(allow=r'^http://angelimg.spbeen.com/ang/\d+$'), callback='parse_item', follow=False),
    )

    def parse_item(self, response):
        item = response.meta.get('item',False)
        if item:
            pass
        else:
            item = {}
            item['files'] = []
            item['file_urls'] = []
        print(response.url)
        img_url = response.xpath('.//div[@id="content"]/a/img/@src').extract_first()
        item['file_urls'].append(img_url)

        # 如果有下一页 请求下一页，没有数据丢回管道
        next_url = response.xpath('.//div[@class="page"]//a[contains(@class,"next")]/@href').extract_first()

        if next_url:
            next_url = self.base_url + next_url
            yield scrapy.Request(next_url,callback=self.parse_item,meta={'item':item})
        else:
            print(item)
            yield item
    def parse_next_response(self,response,):
        item = response.meta.get('item')
        print(item,response.url)

　　

　　github地址

https://github.com/brady-wang/spider-fileDownload