python实现并发获取html的几种方式

1.线程池

from concurrent.futures import ThreadPoolExecutor
import requests
from fake_useragent import UserAgent

def task(url):
    res = requests.get(url,
                       headers={"User-Agent": UserAgent().chrome})
    print(f"{url}--{res}")

url_list = ["http://www.baidu.com",
            "http://www.acfun.cn",
            "http://www.bilibili.com",
            "http://www.zhihu.com",
            "http://www.douban.com"]

executor = ThreadPoolExecutor(max_workers=3)

tasks = [executor.submit(task, url) for url in url_list]
executor.shutdown(wait=True)

　　

2.asyncio+aiohttp

import asyncio
import aiohttp

async def task(url):
    async with aiohttp.request("GET", url) as response:
        res = await response.text()
        print(res)

url_list = ["http://www.baidu.com",
            "http://www.acfun.cn",
            "http://www.bilibili.com",
            "http://www.zhihu.com",
            "http://www.douban.com"]

tasks = [task(url) for url in url_list]
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))

　　

3.gevent

from gevent import monkey
monkey.patch_all()
import gevent
from fake_useragent import UserAgent
import requests

def task(url):
    res = requests.get(url,
                       headers={"User-Agent": UserAgent().chrome})
    res.encoding = res.apparent_encoding
    print(f"{url}--{res.text}")

url_list = ["http://www.baidu.com",
            "http://www.acfun.cn",
            "http://www.bilibili.com",
            "http://www.zhihu.com",
            "http://www.douban.com"]

gevent.joinall([gevent.spawn(task, url) for url in url_list])

　　

4.tornado

from tornado.httpclient import AsyncHTTPClient
from tornado.httpclient import HTTPRequest
from tornado import ioloop
from functools import partial
from fake_useragent import UserAgent

count = None
def handler_response(response):
    global count
    count -= 1
    if response.error:
        print("error")
    else:
        print(response.effective_url)
    if count == 0:
        ioloop.IOLoop.current().stop()

def func(url_list):
    global count
    count = len(url_list)
    for url in url_list:
        client = AsyncHTTPClient()
        client.fetch(HTTPRequest(url, headers={"User-Agent": UserAgent().chrome}), callback=handler_response)

url_list = ["http://www.baidu.com",
            "http://www.acfun.cn",
            "http://www.bilibili.com",
            "http://www.zhihu.com",
            "http://www.douban.com"]

ioloop.IOLoop.current().run_sync(partial(func, url_list))
ioloop.IOLoop.current().start()

　　

5. twisted

from twisted.internet import reactor  # 事件循环（终止条件，所有的socket都已经移除）
from twisted.internet import defer  # defer.Deferred 特殊的socket对象（不发请求，手动移除）
from twisted.web.client import getPage  # 用于创建socket对象（下载完成，自动从事件循环中移除）
from fake_useragent import UserAgent

# 1. 利用getPage创建socket
# 2. 将socket添加到事件循环
# 3. 开始事件循环（内部发送请求，并接受相应。当所有的socket请求完成之后，终止事件循环）

@defer.inlineCallbacks  # 添加到事件循环
def task(url):
    # 创建socket,注意url要转换成字节。headers中的"User-Agent"也要是字节
    d = getPage(bytes(url, encoding="utf-8"), headers={b"User-Agent": UserAgent().chrome})
    # 获取到的html页面会自动传到response函数的第一个参数里面
    d.addCallback(response)
    yield d

def response(content):
    print(str(content, encoding="utf-8"))

def done(*args):
    reactor.stop()
url_list = ["http://www.baidu.com",
            "http://www.acfun.cn",
            "http://www.bilibili.com",
            "http://www.zhihu.com",
            "http://www.douban.com"]
defer_list = []
for url in url_list:
    t = task(url)
    defer_list.append(t)
# 监听defer_list里面的任务是否执行完毕
d = defer.DeferredList(defer_list)
#如果执行完毕，执行对应的回调函数
d.addBoth(done)
# 启动事件循环
reactor.run()