middlewares.py

1.数据需要selenium解析后才能呈现

# middlewares.py
from scrapy import signals
import scrapy
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time

class ChromedriverMiddleware(object):
    def process_request(self, request, spider):
        chrome_options = Options()
        chrome_options.add_argument('--headless')  # 使用无头谷歌浏览器模式
        chrome_options.add_argument('--disable-gpu')
        chrome_options.add_argument('--no-sandbox')
        # 指定谷歌浏览器路径
        self.driver = webdriver.Chrome(chrome_options=chrome_options,
                                       executable_path='/root/zx/spider/driver/chromedriver')
        if request.url != 'https://www.aqistudy.cn/historydata/':
            self.driver.get(request.url)
            time.sleep(1)
            html = self.driver.page_source
            self.driver.quit()
            return scrapy.http.HtmlResponse(url=request.url, body=html.encode('utf-8'), encoding='utf-8',
                                            request=request)

# setting.py
DOWNLOADER_MIDDLEWARES = {
    'driver.middlewares.ChromedriverMiddleware': 543,
}

2.使用代理

# middlewares.py
from scrapy.utils.python import to_bytes
import base64

class ProxyMiddleware(object):
    def process_request(self, request, spider):
        # 全部请求都使用代理
        PROXIES = [
            {'ip_port': '111.11.228.75:80', 'user_pass': 'root@123456'},
            {'ip_port': '120.198.243.22:80', 'user_pass': 'root@123456'},
            {'ip_port': '111.8.60.9:8123', 'user_pass': 'root@123456'},
            {'ip_port': '101.71.27.120:80', 'user_pass': 'root@123456'},
            {'ip_port': '122.96.59.104:80', 'user_pass': 'root@123456'},
            {'ip_port': '122.224.249.122:8088', 'user_pass': 'root@123456'},
        ]
        proxy = random.choice(PROXIES)
        if proxy['user_pass'] is not None:
            request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port'])
            encoded_user_pass = base64.b64encode(to_bytes(proxy['user_pass']))
            request.headers['Proxy-Authorization'] = to_bytes('Basic ' + encoded_user_pass)
        else:
            request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port'])

    def process_exception(self, request, response, spider):
        # 请求报错是执行代理
        def process_request(self, request, spider):
            PROXIES = [
                {'ip_port': '111.11.228.75:80', 'user_pass': 'root@123456'},
                {'ip_port': '120.198.243.22:80', 'user_pass': 'root@123456'},
                {'ip_port': '111.8.60.9:8123', 'user_pass': 'root@123456'},
                {'ip_port': '101.71.27.120:80', 'user_pass': 'root@123456'},
                {'ip_port': '122.96.59.104:80', 'user_pass': 'root@123456'},
                {'ip_port': '122.224.249.122:8088', 'user_pass': 'root@123456'},
            ]
            proxy = random.choice(PROXIES)
            if proxy['user_pass'] is not None:
                request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port'])
                encoded_user_pass = base64.b64encode(to_bytes(proxy['user_pass']))
                request.headers['Proxy-Authorization'] = to_bytes('Basic ' + encoded_user_pass)
            else:
                request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port'])

# setting.py
DOWNLOADER_MIDDLEWARES = {
    'driver.middlewares.ProxyMiddleware': 543,
    # 设置不参与scrapy的自动重试的动作   请求出错时才使用代理
    'scrapy.downloadermiddlewares.retry.RetryMiddleware': None
}

# 禁止重试
RETRY_ENABLED = False
# 下载超时
DOWNLOAD_TIMEOUT = 10
# 下载重试次数
RETRY_TIMES=5

# spider.py  在请求中使用代理
yield scrapy.FormRequest(self.start_url, method="POST", formdata=self.form_data,meta={'proxy':"http://59.44.247.194:9797"})

# proxy_pool 设置代理
class MyproxiesSpiderMiddleware(object):

    def process_request(self, request, spider):
        proxies = requests.get('http://127.0.0.1:5000/get').content.decode('utf-8')
        print(proxies)
        if 'pool' in proxies:
            time.sleep(80)
            print('没有可用的的代理,等60秒')
            proxies = requests.get('http://127.0.0.1:5000/get').content.decode('utf-8')
            request.meta["proxy"] = "http://{}".format(proxies)
        else:
            request.meta["proxy"] = "http://{}".format(proxies)
        # request.meta["proxy"] = "http://36.249.49.43:9999"