scrapy爬虫案例--爬取阳关热线问政平台

阳光热线问政平台：http://wz.sun0769.com/political/index/politicsNewest?id=1&page=1

爬取最新问政帖子的编号、投诉标题、投诉内容以及处理状态，并存入到Mongodb数据库中。

1、创建项目

scrapy startproject myProject

2、定义Item

items.py

import scrapy

class MyprojectItem(scrapy.Item):
    number = scrapy.Field() #帖子编号
    title = scrapy.Field() #帖子标题
    content = scrapy.Field() #帖子内容
    status = scrapy.Field() #处理状态

3、编写爬虫，提取item

首先使用如下命令生成爬虫文件sun.py

scrapy genspider sun wz.sun0769.com

spiders/sun.py

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from myProject.items import MyprojectItem

class SunSpider(CrawlSpider):
    name = 'sun'
    allowed_domains = ['wz.sun0769.com']
    start_urls = ['http://wz.sun0769.com/political/index/politicsNewest?id=1&page=1']

    rules = (
        Rule(LinkExtractor(allow=r'id=\d+&page=\d+')),#每一页的匹配规则，默认follow=True跟进
        Rule(LinkExtractor(allow=r'politics/index\?id=\d+'), callback='parse_item'),#每个帖子的匹配规则，默认不跟进
    )

    def parse_item(self, response):
        item = MyprojectItem()
        title = response.xpath('//div[@class="mr-three"]/p[@class="focus-details"]/text()').extract()[0] #投诉标题
        status = response.xpath('//div[@class="focus-date clear focus-date-list"]/span[3]/text()').extract()[0].split()[1] #处理状态
        number = response.xpath('//div[@class="focus-date clear focus-date-list"]/span[4]/text()').extract()[0].split('：')[-1] #投诉编号
        content = response.xpath('//div[@class="details-box"]/pre/text()').extract()[0] #投诉内容
        item['title'] = title
        item['status'] = status
        item['number'] = number
        item['content'] = content
        yield item

4、编写中间件文件middlewares.pyd中的下载中间件

给每个请求随机选择一个User-Agent，并且随机选择一个代理ip，另外写一个脚本定时获取代理ip和port，并存入redis数据库(比如使用芝麻ip)

import random
import json
import redis

from myProject.settings import USER_AGENTS

class MyprojectDownloaderMiddleware:
    def __init__(self):
        super().__init__()
        self.r = redis.StrictRedis()#创建redis数据库连接客户端，用于取里面存储的代理Ip和port

    def process_request(self, request, spider):
        proxy_list = json.loads(self.r.get('proxy_list').decode())
        proxy = random.choice(proxy_list)
        request.headers['User-Agent'] = random.choice(USER_AGENTS) #设置user-agent
        request.meta['proxy'] ='http://'+proxy['ip']+':'+str(proxy['port']) #设置代理ip和port
        return None

5、存储数据

编写pipelines.py文件

import pymongo

class MyprojectPipeline:
    def open_spider(self,spider):
        self.client = pymongo.MongoClient('mongodb://localhost:27017')#创建mongodb数据库连接客户端
        self.db = self.client.test #选择test数据库
        self.collection = self.db.sunInfo #选择sunInfo集合存储数据

    def process_item(self, item, spider):
        self.collection.insert_one(dict(item))
        return item

    def close_spider(self,spider):
        self.client.close()

6、编写settings.py

BOT_NAME = 'myProject'

SPIDER_MODULES = ['myProject.spiders']
NEWSPIDER_MODULE = 'myProject.spiders'

ROBOTSTXT_OBEY = False
COOKIES_ENABLED = False
DOWNLOAD_DELAY = 1.0#设置延迟，减轻服务器压力，从而降低被查封的风险

#设置下载中间件
DOWNLOADER_MIDDLEWARES = {
   'myProject.middlewares.MyprojectDownloaderMiddleware': 543,
}

#设置item管道
ITEM_PIPELINES = {
   'myProject.pipelines.MyprojectPipeline': 300,
}

#可供选择的User-Agent，可通过网上查找更多，随机切换，起到反爬虫作用
USER_AGENTS = [
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5"
    ]

7、开启爬虫

scrapy crawl sun