python框架Scrapy中crawlSpider的使用

一、创建Scrapy工程

 #scrapy startproject 工程名
 scrapy startproject demo3

二、进入工程目录，根据爬虫模板生成爬虫文件

 #scrapy genspider -l # 查看可用模板
 #scrapy genspider -t 模板名 爬虫文件名 允许的域名
 scrapy genspider -t crawl test sohu.com

三、设置IP池或用户代理（middlewares.py文件）

 # -*- coding: utf-8 -*-
 # 导入随机模块
 import random
 # 导入有关IP池有关的模块
 from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
 # 导入有关用户代理有关的模块
 from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware

 # IP池
 class HTTPPROXY(HttpProxyMiddleware):
     # 初始化 注意一定是 ip=''
     def __init__(self, ip=''):
         self.ip = ip

     def process_request(self, request, spider):
         item = random.choice(IPPOOL)
         try:
             print("当前的IP是："+item["ipaddr"])
             request.meta["proxy"] = "http://"+item["ipaddr"]
         except Exception as e:
             print(e)
             pass

 # 设置IP池
 IPPOOL = [
     {"ipaddr": "182.117.102.10:8118"},
     {"ipaddr": "121.31.102.215:8123"},
     {"ipaddr": "1222.94.128.49:8118"}
 ]

 # 用户代理
 class USERAGENT(UserAgentMiddleware):
     #初始化 注意一定是 user_agent=''
     def __init__(self, user_agent=''):
         self.user_agent = user_agent

     def process_request(self, request, spider):
         item = random.choice(UPPOOL)
         try:
             print("当前的User-Agent是："+item)
             request.headers.setdefault('User-Agent', item)
         except Exception as e:
             print(e)
             pass

 # 设置用户代理池
 UPPOOL = [
     "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393"
 ]

四、settngs.py配置

 #========================================

 # 设置IP池和用户代理

 # 禁止本地Cookie
 COOKIES_ENABLED = False

 # 下载中间件配置指向（注意这里的工程名字是"demo3",指向DOWNLOADER_MIDDLEWARES = {
     # 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware':123,
     # 'demo3.middlewares.HTTPPROXY' : 125,
     'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 2,
     'demo3.middlewares.USERAGENT': 1
 }

 # 管道指向配置（注意这里的工程名字是"demo3",指向ITEM_PIPELINES = {
     'demo3.pipelines.Demo3Pipeline': 300,
 }

 #============================================

五、定义爬取关注的数据（items.py文件）

 # -*- coding: utf-8 -*-
 import scrapy
 # Define here the models for your scraped items
 #
 # See documentation in:
 # http://doc.scrapy.org/en/latest/topics/items.html
 class Demo3Item(scrapy.Item):
     name = scrapy.Field()
     link = scrapy.Field()

六、爬虫文件编写（test.py）

 # -*- coding: utf-8 -*-
 import scrapy
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule
 from demo3.items import Demo3Item

 class TestSpider(CrawlSpider):
     name = 'test'
     allowed_domains = ['sohu.com']
     start_urls = ['http://www.sohu.com/']

     rules = (
         Rule(LinkExtractor(allow=('http://news.sohu.com'), allow_domains=('sohu.com')), callback='parse_item', follow=False),
         #Rule(LinkExtractor(allow=('.*?/n.*?shtml'),allow_domains=('sohu.com')), callback='parse_item', follow=False),
     )

     def parse_item(self, response):
         i = Demo3Item()
         i['name'] = response.xpath('//div[@class="news"]/h1/a/text()').extract()
         i['link'] = response.xpath('//div[@class="news"]/h1/a/@href').extract()
         return i

七、管道文件编写（pipelines.py）

 # -*- coding: utf-8 -*-
 import codecs
 import json
 # Define your item pipelines here
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
 # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
 class Demo3Pipeline(object):
     def __init__(self):
         self.file = codecs.open("E:/workspace/PyCharm/codeSpace/books/python_web_crawler_book/chapter17/demo3/1.json", "wb", encoding='utf-8')

     def process_item(self, item, spider):
         for j in range(0, len(item["name"])):
             name = item["name"][j]
             link = item["link"][j]
             datas = {"name": name, "link": link}
             i = json.dumps(dict(datas), ensure_ascii=False)
             line = i + '\n'
             self.file.write(line)
         return item
     def close_spider(self, spider):
         self.file.close()

八、测试（scrapy crawl test ）之后，生成了1.json文件