Python的scrapy之爬取妹子图片

闲来无事，做的一个小爬虫项目

爬虫主程序：

 import scrapy
 from ..items import MeiziItem

 class MztSpider(scrapy.Spider):
     name = 'mzt'
     allowed_domains = ['meizitu.com']
     start_urls = ['http://meizitu.com/']

     def parse(self, response):
         tags = response.xpath(".//*[@class='tags']/span/a")
         for i in tags:
             item = MeiziItem()
             tag_href = i.xpath(".//@href").extract()[0]
             tag_name = i.xpath(".//@title").extract()[0]
             item['tag_name'] = tag_name
             item['tag_href'] = tag_href
             #print(item['tag_name'])
             #yield item
             yield scrapy.Request(url=item['tag_href'], meta={'item': item}, callback=self.parse_page)

     def parse_page(self, response):

         item = response.meta['item']
         # 进入某个标签后，爬取底部分页按钮
         page_lists = response.xpath(".//*[@id='wp_page_numbers']/ul/li")
         # 获取底部分页按钮上的文字，根据文字来判断当前标签页下总共有多少分页
         page_list = page_lists.xpath('.//text()')
         # 如果当前标签页下有多个页面，则再根据第一个按钮是否为“首页”来进行再次提取，因为这里有的页面第一个按钮是首页，有的第一个按钮是“1”
         if len(page_lists) > 0:
             if page_list[0].extract() == '首页':
                 page_num = len(page_lists) - 3
             else:
                 page_num = len(page_lists) - 2
         else:
             page_num = 1

         # 根据当前标签页的链接，来拼成带页码的链接
         if '_' in item['tag_href']:
             index = item['tag_href'][::-1].index('_')
             href_pre = item['tag_href'][:-index]
         else:
             if page_num == 1:
                 href_pre = item['tag_href'].split('.html')[0]
             else:
                 href_pre = item['tag_href'].split('.html')[0] + '_'
         for i in range(1, page_num + 1):
             item = response.meta['item']
             if page_num == 1:
                 href = href_pre + '.html'
             else:
                 href = href_pre + str(i) + '.html'
             item['page_list'] = href
             #yield item
             yield scrapy.Request(url=item['page_list'], meta={'item': item}, callback=self.parse_album)

     def parse_album(self, response):
         albums = response.xpath(".//*[@class='pic']")
         for album in albums:
             item = response.meta['item']
             album_href = album.xpath(".//a/@href").extract()[0]
             album_name = album.xpath(".//a/img/@alt").extract()[0]
             item['album_name'] = album_name
             item['album_href'] = album_href
             #yield item
             yield scrapy.Request(url=item['album_href'], meta={'item': item}, callback=self.parse_img)

     def parse_img(self, response):
         img_list = response.xpath(".//*/p/img")
         for img in img_list:
             item = response.meta['item']
             img_title = img.xpath(".//@alt").extract()[0]
             if img_title == '':
                 for i in range(1, len(img_list + 1)):
                     img_title = item['album_name'] + '_' + str(i)
             else:
                 img_title = img_title
             img_urls = img.xpath(".//@src").extract()
             img_src = img.xpath(".//@src").extract()[0]
             item['img_title'] = img_title
             item['img_src'] = img_src
             item['img_urls'] = img_urls
             yield item

items设置

 import scrapy

 class MeiziItem(scrapy.Item):
     # define the fields for your item here like:
     # name = scrapy.Field()
     # 标签名称
     tag_name = scrapy.Field()
     # 标签链接
     tag_href = scrapy.Field()
     # 进入某标签后的所有链接，加页码的
     page_list = scrapy.Field()
     # 图片专辑名称
     album_name = scrapy.Field()
     # 图片专辑链接
     album_href = scrapy.Field()
     # 照片标题
     img_title = scrapy.Field()
     # 照片链接
     img_src = scrapy.Field()
     # 照片链接集合，用于ImagesPipeline下载图片
     img_urls = scrapy.Field()

输出管道：

print('正在爬取...')
print('老湿机，请耐心等待哟...')
class MeiziPipeline(object):
    def process_item(self, item, spider):
        print('标签名称:',item['tag_name'])
        print('标签链接:',item['tag_href'])
        print('页码:',item['page_list'])
        print('图片专辑名称:',item['album_name'])
        print('图片专辑链接:',item['album_href'])
        print('照片标题:',item['img_title'])
        print('照片链接:',item['img_src'])
        print('照片链接集合:',item['img_urls'])
        print('----------------')
        return item

保存到本地的管道：

import scrapy
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem

class MztImagesPipeline(ImagesPipeline):
    def get_media_requests(self, item, info):
        for image_url in item['img_urls']:
            yield scrapy.Request(image_url)

    def item_completed(self, results, item, info):
        image_paths = [x['path'] for ok, x in results if ok]
        if not image_paths:
            raise DropItem("该Item没有图片")
        return item

setting的设置，往setting.py加入

BOT_NAME = 'meizi'

SPIDER_MODULES = ['meizi.spiders']
NEWSPIDER_MODULE = 'meizi.spiders'

IMAGES_STORE = r'G:\\mzt'    # 图片存储路径
IMAGES_EXPIRES = 90             # 过期天数
IMAGES_MIN_HEIGHT = 100         # 图片的最小高度
IMAGES_MIN_WIDTH = 100          # 图片的最小宽度

爬取的最终结果

本来想把这些图片分门分类的保存，然而不太会，所有的图片全保存在一个文件夹下面，，，