下载中间件--随机IP代理以及随机User_Agent

下载中间件随机IP代理以及随机User_Agent

1.在settings.py中设置开启代理功能

# 设置下载中间件
DOWNLOADER_MIDDLEWARES = {
   # 随机的 User-Agent
   'douban.middlewares.DoubanUserAgent': 100,
   # 随机的 Proxy
   'douban.middlewares.DoubanProxy': 200,
}

# 代理列表值
# User_Agent 列表
User_Agent_lists = [
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36 OPR/37.0.2178.32',
    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2 (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586',
    'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 BIDUBrowser/8.3 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36 Core/1.47.277.400 QQBrowser/9.4.7658.400',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 UBrowser/5.6.12150.8 Safari/537.36',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.154 Safari/537.36 LBBROWSER',
    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36 TheWorld 7',
    'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36',
]

# 代理列表
PROXIES = [
    {"ip_port": "210.16.189.75:888", "user_passwd": "fa319:fa319"},
    # ... ...
    # {"ip_prot": "ip:端口", "user_passwd": ""},     # 没有用户名密码的，直接将用户密码留空即可
]

# 注释之前设置的 User-Agent
# DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   # 'Accept-Language': 'en',
#   'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36',
# }

2.编辑middlewares.py 中间件文件

import random
import base64
from scrapy import signals
from .settings import User_Agent_lists
from .settings import PROXIES

class DoubanUserAgent(object):
    # process_request必要方法
    def process_request(self, request, spider):
        # 从列表中随机拿取一个 User_Agent 设置为头
        useragent = random.choice(User_Agent_lists)

        # 将随机 User_Agent 放入请求中
        request.headers.setdefault("User-Agent", useragent)

class DoubanProxy(object):
    # process_request必要方法
    def process_request(self, request, spider):
        proxy = random.choice(PROXIES)

        if proxy['user_passwd'] is None:
            # 代理需要写在 request 的 meta 信息中
            request.meta["proxy"] = "http://" + proxy['ip_port']
        else:
            # 对账户进行 base64 的编码转换
            base64_userpasswd = base64.b64encode(proxy["user_passwd"].encode("utf-8")).decode()

            # 代理的 IP 地址
            request.meta["proxy"] = "http://" + proxy['ip_port']

            # 代理的 用户名密码
            request.headers["Proxy-Authorization"] = 'Basic ' + base64_userpasswd