190221协程与IO模型

一、协程

• 又称微线程
• 协程是一种用户态的轻量级的线程
• 在单线程下实现的并发，例如：yield
  优点：
• 无需线程上下文切换的开销
• 无需原子操作锁定及同步的开销
• 方便切换控制流，简化编程模型
• 高并发，高扩展性，低成本：一个CPU可支持上万的协程
  缺点：
• 无法利用多核资源
• 进行阻塞操作会阻塞掉整个程序

import time,queue

def consumer(name):
    print('--->starting eating bun...')
    while True:
        new_bun = yield
        print("[%s] is eating bun %s" %(name,new_bun))

def producer():
    r = con.__next__()
    r = con2.__next__()
    n = 0
    while n < 5:
        n += 1
        con.send(n)
        con2.send(n)
        print("\033[32;1m[producer]\033[0m is making bun %s" %n)

if __name__ == '__main__':
    con = consumer("c1")
    con2 = consumer("c2")
    p = producer()

二、greenlet 手动切换

# Author:Li Dongfei

from greenlet import greenlet

def test1():
    print(12)
    gr2.switch()  #切换至gr2
    print(34)
    gr2.switch()

def test2():
    print(56)
    gr1.switch()
    print(78)

gr1 = greenlet(test1)  #启动一个协程
gr2 = greenlet(test2)
gr1.switch()  #切换至gr1

三、genent 自动切换

• 遇到sleep自动切换

# Author:Li Dongfei
import gevent

def foo():
    print("Running in foo")
    gevent.sleep(2)
    print("Explicit context switch to foo again")

def bar():
    print("Explicit精确的 context to bar")
    gevent.sleep(1)
    print("Implicit context switch back to bar")

def func():
    print("in the func")
    gevent.sleep(0)
    print("in the func again")

gevent.joinall([
    gevent.spawn(foo),
    gevent.spawn(bar),
    gevent.spawn(func),
])

• 协程并发爬网页

# Author:Li Dongfei

from gevent import monkey; monkey.patch_all()  #把当前程序所有的io操作单独做标记，让gevent识别
import gevent
from urllib.request import urlopen

def f(url):
    print('GET: %s' %url)
    resp = urlopen(url)
    data = resp.read()
    f = open("url.html", "wb")
    f.write(data)
    f.close()
    print('%d bytes received from %s.' %(len(data), url))

gevent.joinall([
    gevent.spawn(f, 'https://www.python.org'),
    gevent.spawn(f, 'https://www.github.com'),
    gevent.spawn(f, 'https://www.aliyun.com'),
])

• 通过gevent实现单线程的多socket并发
• server

# Author:Li Dongfei

import sys,socket,time,gevent
from gevent import socket,monkey
monkey.patch_all()

def server(port):
    s = socket.socket()
    s.bind(('localhost', port))
    s.listen(500)
    while True:
        cli, addr = s.accept()
        gevent.spawn(handle_request, cli)

def handle_request(conn):
    try:
        while True:
            data = conn.recv(1024)
            print("recv:", data)
            conn.send(data)
            if not data:
                conn.shutdown(socket.SHUT_WR)
    except Exception as ex:
        print(ex)
    finally:
        conn.close()

if __name__ == '__main__':
    server(8001)

• client

# Author:Li Dongfei
import socket

HOST = 'localhost'
PORT = 8001

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
while True:
    msg = bytes(input(">>:"), encoding="utf-8")
    s.sendall(msg)
    data = s.recv(1024)
    print('Received', data)

s.close()

四、I/O模型

• 阻塞
• 非阻塞
• I/O多路复用
• 信号驱动
• 异步
• Python select socket server代码示例

# Author:Li Dongfei
import select, socket, sys, queue
server = socket.socket()
server.bind(('localhost', 9000))
server.listen(1000)
server.setblocking(False)  #设置为非阻塞

msg_dic = {}
inputs = [server,]
outputs = []

while True:
    readable, writeable, exceptional = select.select(inputs, outputs, inputs)
    print(readable,writeable,exceptional)
    for r in readable:
        if r is server:
            conn, addr = server.accept()
            print("new connect", addr)
            inputs.append(conn)
            msg_dic[conn] = queue.Queue()
        else:
            data = r.recv(1024)
            print("recv data",data)
            msg_dic[r].put(data)
            outputs.append(r)

    for w in writeable:
        data_to_clinet =msg_dic[w].get()
        w.send(data_to_clinet)
        outputs.remove(w)

    for e in exceptional:
        if e in outputs:
            outputs.remove(e)
        inputs.remove(e)
        del msg_dic[e]

五、selector模块

# Author:Li Dongfei
import selectors, socket

sel = selectors.DefaultSelector()

def accept(sock, mask):
    conn, addr = sock.accept()
    print('accepted', conn, 'from', addr)
    conn.setblocking(False)
    sel.register(conn, selectors.EVENT_READ, read)

def read(conn, mask):
    data = conn.recv(1024)
    if data:
        print('echoing', repr(data), 'to', conn)
        conn.send(data)
    else:
        print('closing', conn)
        sel.unregister(conn)
        conn.close()

sock = socket.socket()
sock.bind(('localhost', 9000))
sock.listen(1000)
sock.setblocking(False)
sel.register(sock, selectors.EVENT_READ, accept)

while True:
    events = sel.select()
    for key, mask in events:
        callback = key.data
        callback(key.fileobj, mask)