# -*- coding: utf-8 -*-
import re
from time import sleep import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule class AngelSpider(CrawlSpider):
name = 'angel'
allowed_domains = ['angelimg.spbeen.com']
start_urls = ['http://angelimg.spbeen.com/'] base_url = "http://angelimg.spbeen.com"
rules = (
Rule(LinkExtractor(allow=r'^http://angelimg.spbeen.com/ang/\d+$'), callback='parse_item', follow=False),
) def parse_item(self, response):
print(response.url)
item = response.meta.get('item',False)
if item:
pass
else:
item = {}
item['files'] = []
item['file_urls'] = []
dir_name = response.xpath('.//div[@class="article"]/h2/text()').extract_first()
item['dir_name'] = dir_name.split('【')[0]
item['dir_name'] = re.sub(u"([^\u4e00-\u9fa5\u0030-\u0039\u0041-\u005a\u0061-\u007a])","", item['dir_name']) img_url = response.xpath('.//div[@id="content"]/a/img/@src').extract_first()
item['file_urls'].append(img_url)
# 如果有下一页 请求下一页,没有数据丢回管道
next_url = response.xpath('.//div[@class="page"]//a[contains(@class,"next")]/@href').extract_first() #sleep(1)
if next_url:
next_url = self.base_url + next_url
yield scrapy.Request(next_url,callback=self.parse_item,meta={'item':item})
else:
yield item

  

管道 继承文件管道

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import hashlib
import os from scrapy.pipelines.files import FilesPipeline class AngelimgPipeline(object):
def process_item(self, item, spider):
return item from scrapy.http import Request
from scrapy.utils.python import to_bytes class DealFilePathPipeline(FilesPipeline):
def get_media_requests(self, item, info):
return [Request(x,meta={'item':item}) for x in item.get(self.files_urls_field, [])] def file_path(self, request, response=None, info=None):
## start of deprecation warning block (can be removed in the future)
def _warn():
from scrapy.exceptions import ScrapyDeprecationWarning
import warnings
warnings.warn('FilesPipeline.file_key(url) method is deprecated, please use '
'file_path(request, response=None, info=None) instead',
category=ScrapyDeprecationWarning, stacklevel=1) # check if called from file_key with url as first argument
if not isinstance(request, Request):
_warn()
url = request
else:
url = request.url # detect if file_key() method has been overridden
if not hasattr(self.file_key, '_base'):
_warn()
return self.file_key(url)
## end of deprecation warning block
item = request.meta.get('item',{})
media_guid = hashlib.sha1(to_bytes(url)).hexdigest() # change to request.url after deprecation
media_ext = os.path.splitext(url)[1] # change to request.url after deprecation
print(item)
return 'full2/{}/{}{}'.format(item['dir_name'],media_guid, media_ext)
return 'full/%s%s' % (media_guid, media_ext) # deprecated
def file_key(self, url):
return self.file_path(url) file_key._base = True

  setting.py

# -*- coding: utf-8 -*-

# Scrapy settings for angelImg project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# https://doc.scrapy.org/en/latest/topics/settings.html
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html BOT_NAME = 'angelImg' SPIDER_MODULES = ['angelImg.spiders']
NEWSPIDER_MODULE = 'angelImg.spiders' # Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'angelImg (+http://www.yourdomain.com)' # Obey robots.txt rules
ROBOTSTXT_OBEY = False # Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16 # Disable cookies (enabled by default)
#COOKIES_ENABLED = False # Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False # Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
# 'Accept-Language': 'en',
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36",
"Referer":"http://angelimg.spbeen.com/"
} # Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'angelImg.middlewares.AngelimgSpiderMiddleware': 543,
#} # Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
# 'angelImg.middlewares.AngelimgDownloaderMiddleware': 543,
#} # Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#} # Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
#'angelImg.pipelines.AngelimgPipeline': 300,
'angelImg.pipelines.DealFilePathPipeline': 200,
#'scrapy.pipelines.files.FilesPipeline': 2
} FILES_STORE='file_doload' # Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

python文件管道 下载图集的更多相关文章

  1. PYTHON文件多线程下载

    其实,在一般的文件编程中,这有两个概念要说明: 第一是,下载一个大文件,将这个大文件多为多线程. 第二是,下载N多小文件,将每个线程指定下载多个小文件. 现在实现的是多线程下载一个大文件. 今天完成了 ...

  2. 2、Python djang 框架下的word Excel TXT Image 等文件的下载

    2.python实现文件下载 (1)方法一.直接用a标签的href+数据库中文件地址,即可下载.缺点:word excel是直接弹框下载,对于image txt 等文件的下载方式是直接在新页面打开. ...

  3. python爬虫之下载文件的方式总结以及程序实例

    python爬虫之下载文件的方式以及下载实例 目录 第一种方法:urlretrieve方法下载 第二种方法:request download 第三种方法:视频文件.大型文件下载 实战演示 第一种方法: ...

  4. Python selenium 文件自动下载 (自动下载器)

    MyGithub:https://github.com/williamzxl 最新代码已经上传到Github,以下版本为stupid版本. 由于在下载过程中需要下载不同文件,所以可以把所有类型放在Va ...

  5. Python 文件操作函数

    这个博客是 Building powerful image classification models using very little data 的前期准备,用于把图片数据按照教程指示放到规定的文 ...

  6. python文件打包格式,pip包管理

    1..whl是python文件的一种打包格式, 在有些情况下,可以将文件的后缀名改为.zip并解压 2.cmd中,提示pip版本太低,先升级pip   pip install --upgrade pi ...

  7. Python文件系统功能:os模块

    Python文件系统功能:os模块 1.os模块方法分类 (1)目录: chdir() 改变工作目录 chroot() 设定当前进程的根目录 listdir() 列出指定目录下的所有文件名 mkdir ...

  8. 利用pyinstaller 打包Python文件

    1.下载安装pyinstaller模块 cmd 命令: pip install pyinstaller cmd命令: pip list 查看自己安装的模块 2.建议把要大包的Python文件单独放到新 ...

  9. 随手用python写一个下载jdk源码爬虫

    最近在研读jdk源码,网上找了下资源,发现都不完整. 后来新发现了一个有完整源码的地方,主要包括了java,c,c++的东西,装逼需要,就想拿来玩玩.但是,找了好多种下载打开的方式,发现都不对.于是, ...

随机推荐

  1. xargs命令学习,于阮一峰博客

    一.标准输入与管道命令 Unix 命令都带有参数,有些命令可以接受"标准输入"(stdin)作为参数. $ cat /etc/passwd | grep root 上面的代码使用了 ...

  2. C/C++ 实现PE文件特征码识别

    PE文件就是我们常说的EXE可执行文件,针对文件特征的识别可以清晰的知道该程序是使用何种编程语言实现的,前提是要有特征库,PE特征识别有多种形式,第一种是静态识别,此方法就是只针对磁盘中文件的特征码字 ...

  3. Python3 学习笔记之 数据类型

  4. 解决spark streaming集成kafka时只能读topic的其中一个分区数据的问题

    1. 问题描述 我创建了一个名称为myTest的topic,该topic有三个分区,在我的应用中spark streaming以direct方式连接kakfa,但是发现只能消费一个分区的数据,多次更换 ...

  5. 微信小程序入门到精通[更新版]

    微信小程序账号与工具 在线文档:https://mp.weixin.qq.com/debug/wxadoc/dev/ 小程序开发者账号注册 微信公众平台:https://mp.weixin.qq.co ...

  6. Shareplex搭建步骤(rman)

    实施例:rman/BCV 环境准备 splex软件上传 源端: #mkdir /quest #chmod -R 755 /quest #chown -R oracle:oinstall /quest ...

  7. Docker:三、深入Docker容器&Asp.net发版

    各位看官,我们前面已经有过两篇文章的研究了,哈哈,都是皮毛... 今天我们来看看docker容器内部,一探究竟... 一.进入docker容器 进入Linux容器非常简单,直接使用如下命令即可: do ...

  8. JVM学习(五)对象的引用类型

    一.引言 前面我们学习了JVM的垃圾回收机制,我们知道了垃圾回收是JVM的自发行为:虽然我们可以通过System.gc() 或Runtime.getRuntime().gc()进行显式调用垃圾回收 , ...

  9. 使用DynamicExpresso实现表达式求值

    之前写了一篇Z.Expressions表达式计算的博客,直到最近才发现Z.Expressions不是免费的.Z.Expressions从2.0开始支持了NetCore,使用一段时期后会提示许可证到期, ...

  10. tomcat在linux下安装

    1.下载地址: https://tomcat.apache.org/download-90.cgi 2.上传linux 3.查看是否上传成功 4.解压: 5.进入后,查看README.md文件,可以查 ...