封装IP池和用户代理相应的类（python3）

一、middlewares.py源代码：

 # -*- coding: utf-8 -*-
 # 导入随机模块
 import random
 # 导入有关IP池有关的模块
 from scrapy.contrib.downloadermiddleware.httpproxy import HttpProxyMiddleware
 # 导入有关用户代理有关的模块
 from scrapy.contrib.downloadermiddleware.useragent import UserAgentMiddleware

 # IP池
 class HTTPPROXY(HttpProxyMiddleware):
     # 初始化 注意一定是 ip=''
     def __init__(self, ip=''):
         self.ip = ip

     def process_request(self, request, spider):
         item = random.choice(IPPOOL)
         print("当前的IP是："+item["ipaddr"])
         request.meta["proxy"] = "http://"+item["ipaddr"]

 # 设置IP池
 IPPOOL = [
     {"ipaddr": "182.117.102.10:8118"},
     {"ipaddr": "121.31.102.215:8123"},
     {"ipaddr": "1222.94.128.49:8118"}
 ]

 # 用户代理
 class USERAGENT(UserAgentMiddleware):
     #初始化 注意一定是 user_agent=''
     def __init__(self, user_agent=''):
         self.user_agent = user_agent

     def process_request(self, request, spider):
         item = random.choice(UPPOOL)
         try:
             print("当前的User-Agent是："+item)
             request.headers.setdefault('User-Agent', item)
         except Exception as e:
             print(e)
             pass

 # 设置用户代理池
 UPPOOL = [
     "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393"
 ]

二、settings.py文件设置，添加下面代码：

 #==============================================

 # 禁止Cookie
 COOKIES_ENABLED = False

 # 下载中间件指向
 DOWNLOADER_MIDDLEWARES = {
     # 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware':123,
     # '工程名.middlewares.HTTPPROXY' : 125,
     'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 2,
     '工程名.middlewares.USERAGENT': 1
 }

 # 管道指向
 ITEM_PIPELINES = {
     '工程名.pipelines.管道中对应的类名': 300,
 }

 # 注意将Obey robots关闭（上面已经开启了，所以需要找到并设置为False）
 ROBOTSTXT_OBEY = False

 #==============================================

三、总结一些需要的东西

1.在引入（有关IP池有关的和关用户代理的）模块是注意这个

 # 导入有关IP池有关的模块
 from scrapy.contrib.downloadermiddleware.httpproxy import HttpProxyMiddleware
 # 导入有关用户代理有关的模块
 from scrapy.contrib.downloadermiddleware.useragent import UserAgentMiddleware

"contrib"不能丢了

2.配置下载中间件的连接信息时，注意文件指向，也就是调用类的指向

3.“.”表示这一级目录，".."表示上一级目录