基于socket实现http请求

异步非阻塞模块原理

# 基于socket实现http请求
import socket
# 多路IO复用模块
import select

socket_list= []
url_list = ["http://www.baidu.com", ...]
for url in url_list:
    sk = socket.socket()
    # 设置非阻塞模式
    sk.setblocking(False)
    try:
        sk.connect((url, 80))
    except BlockingIOError as e:
        print(e)
    # 为每个url请求都创建一个socket对象并把它放到一个列表里
    socket_list.append(sk)

# 事件循环
# tornado/twisted事件循环的原理就是通过这种方式来实现的
while True:
    # 监测socket是否连接成功了
    # 监测socket是否可以进行接受数据了
    r, w, e = select.select(socket_list, socket_list, [], 0.05)
    for obj in w:
        obj.send("GET / HTTP1.1\r\nhost:...")
    for obj in r:
        response = obj.recv(4096)
        print(response)

自定制一个基于socket实现的异步非阻塞模块

import socket
import select
import uuid

# 定义一个Request类 对socket对象进行二次封装
class Request(object):
    def __init__(self, sock, info):
        self.sock = sock
        self.info = info

    def fileno(self):
        return self.sock.fileno()

class Lou(object):

    def __init__(self):
        self.sock_list = []
        self.conn_list = []

    def add_request(self, req_info):
        sk = socket.socket()
        sk.setblocking(False)
        try:
            sk.connect((req_info['host'], req_info['port']))
        except BlockingIOError as e:
            pass
        obj = Request(sk, req_info)
        self.conn_list.append(obj)
        self.sock_list.append(obj)

    def run(self):
        while True:
            # 只要列表中的对象有fileno方法就可以作为select的参数
            r, w, e = select.select(self.sock_list, self.conn_list, [], 0.05)
            for obj in w:
                data = "GET %s HTTP/1.1\r\nhost:%s\r\n\r\n" % (obj.info["path"], obj.info["host"])
                obj.sock.send(data.encode("utf-8"))
                self.conn_list.remove(obj)
            for obj in r:
                response = obj.sock.recv(8192)
                print(obj.info["host"], response)
                self.sock_list.remove(obj)
                obj.info["callback"](response)
            if not self.sock_list:
                break

def done(response):
    filename = uuid.uuid4()
    with open("%s.html"%filename, "wb") as f:
        f.write(response)

url_list = [
    {"host": "www.baidu.com", "port": 80, "path": "/", "callback": done},
    {"host": "www.cnblogs.com", "port": 80, "path": "/", "callback": done},
    {"host": "www.bing.com", "port": 80, "path": "/", "callback": done},
]

l = Lou()
for req_info in url_list:
    l.add_request(req_info)

l.run()