用scrapy爬取搜狗Lofter图片

用scrapy爬取搜狗Lofter图片

# -*- coding: utf-8 -*-
import json

import scrapy
from scrapy.http import Request
from urllib import parse
from scrapy.loader import ItemLoader

from tutorial.items import LofterSpiderItem

class LofterSpider(scrapy.Spider):
    name = "lofter"
    allowed_domains = ["pic.sogou.com"]
    start_urls = ['http://pic.sogou.com/']

    # question的第一页answer的请求url
    start_answer_url = "http://pic.sogou.com/pics/channel/getAllRecomPicByTag.jsp?category=LOFTER&tag=%E5%85%A8%E9%83%A8&start={0}&len=15"

    headers = {
        "HOST": "pic.sogou.com",
        "Referer": "http://pic.sogou.com",
        'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:51.0) Gecko/20100101 Firefox/51.0"
    }

    def parse(self, response):

        yield scrapy.Request(self.start_answer_url.format(0), headers=self.headers,callback=self.parse_url)

    def parse_url(self,response):
        ans_json = json.loads(response.text)
        for ans in ans_json['all_items']:
            image_url = ans['ori_pic_url']
            item_loader = ItemLoader(item=LofterSpiderItem(), response=response)
            item_loader.add_value("lofter_image_url", image_url)
            lofter_item = item_loader.load_item()
            yield lofter_item

        yield scrapy.Request(self.start_answer_url.format(ans_json['startIndex']+15), headers=self.headers,callback=self.parse_url)

settings.py

ITEM_PIPELINES = {
   'tutorial.pipelines.TutorialPipeline': 300,
    'tutorial.pipelines.TutorialImagePipeline': 1,
}
# IMAGES_URLS_FIELD = "front_image_url"
project_dir = os.path.abspath(os.path.dirname(__file__))
IMAGES_STORE = os.path.join(project_dir, 'image')

items.py

class LofterSpiderItem(scrapy.Item):
    lofter_image_url = scrapy.Field(
        output_processor=MapCompose(return_value)
    )