进程池线程池 协程 gvent 单线程实现并发套接字

1.基于多线程实现套接字服务端支持并发

服务端

from socket import *
from threading import Thread

def comunicate(conn):
    while True:  # 通信循环
        try:
            data = conn.recv(1024)
            if len(data) == 0: break
            conn.send(data.upper())
        except ConnectionResetError:
            break
    conn.close()

def server(ip, port, backlog=5):
    server = socket(AF_INET, SOCK_STREAM)
    server.bind((ip, port))
    server.listen(backlog)

    while True:  # 链接循环
        conn, client_addr = server.accept()
        print(client_addr)

        # 通信
        t=Thread(target=comunicate,args=(conn,))
        t.start()

if __name__ == '__main__':
    s=Thread(target=server,args=('127.0.0.1',8081))
    s.start()

客户端

from socket import *

client=socket(AF_INET,SOCK_STREAM)
client.connect(('127.0.0.1',8081))

while True:
    msg=input('>>: ').strip()
    if len(msg) == 0:continue
    client.send(msg.encode('utf-8'))
    data=client.recv(1024)
    print(data.decode('utf-8'))

2.进程池

使用进程池就是对启动进程数加以限制

from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor
import time,random,os

def task(name,n):
    print('%s%s is running'%(name,os.getpid()))
    time.sleep(random.randint(1,3))
    return n

if __name__ == '__main__':
    # print(os.cpu_count())#打印cpu核数
    p=ProcessPoolExecutor(4)#进程池预留了4个进程，可以提供开启。后续一次性可以同时开启4个进程，进程号不会变。
    # 如果不传参数（数字），那么默认最多开启的进程数为电脑的核数
    l=[]
    for i in range(20):
        #同步提交
        # res=p.submit(task,'进程pid:').result()
        # print(res)
        #异步提交
        future=p.submit(task,'进程pid:',i)#submit是异步提交,直接传位置参数，或关键字参数
        l.append(future)
    p.shutdown(wait=True)#关闭进程池的入口，并且在原地等待进程池内所有任务运行完毕
    for future in l:
        print(future.result())
    print('主')

不使用回调函数进行异步提交，自行解析

import time, os, random
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
from threading import Thread
import requests

def get(url):
    print('%s GET %s' % (os.getpid(), url))
    time.sleep(random.randint(1, 3))
    response = requests.get(url)
    if response.status_code == 200:
        return response.text
    else:
        return '下载失败'

def parse(res):
    print('%s解析结果为%s' % (os.getpid(), len(res)))

if __name__ == '__main__':
    urls = [
        'https://www.baidu.com',
        'https://www.sina.com.cn',
        'https://www.tmall.com',
        'https://www.jd.com',
        'https://www.python.org',
        'https://www.bilibili.com',
        'https://www.youku.com',
        'https://www.baidu.com',
        'https://www.baidu.com',
        'https://www.baidu.com',
    ]
    p = ProcessPoolExecutor(4)
    l = []
    for url in urls:
        future = p.submit(get, url)
        l.append(future)
    p.shutdown(wait=True)
    for future in l:
        res = future.result()
        parse(res)
    print('主')

异步一般与回调函数连用

使用回调函数

import time, os, random
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
from threading import Thread
import requests

def get(url):
    print('%s GET %s' % (os.getpid(), url))
    time.sleep(random.randint(1, 3))
    response = requests.get(url)
    if response.status_code == 200:
        return response.text
    else:
        return '下载失败'

def parse(res):
    res=res.result()
    print('%s解析结果为%s' % (os.getpid(), len(res)))

if __name__ == '__main__':
    urls = [
        'https://www.baidu.com',
        'https://www.sina.com.cn',
        'https://www.tmall.com',
        'https://www.jd.com',
        'https://www.python.org',
        'https://www.bilibili.com',
        'https://www.youku.com',
        'https://www.baidu.com',
        'https://www.baidu.com',
        'https://www.baidu.com',
    ]
    p = ProcessPoolExecutor(4)
    start=time.time()
    for url in urls:
        future = p.submit(get, url)
        future.add_done_callback(parse)#parse会在任务完毕后触发，然后接收一个参数future对象
    p.shutdown(wait=True)
    print(time.time()-start)
    print('主%s'%os.getpid())

使用线程池开多线程进行任务

线程池和进程池的用法很相似

import time, os, random
from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
from threading import Thread,current_thread
import requests

def get(url):
    print('%s GET %s' % (current_thread().name, url))
    time.sleep(random.randint(1, 3))
    response = requests.get(url)
    if response.status_code == 200:
        return response.text
    else:
        return '下载失败'

def parse(res):
    res=res.result()
    print('%s解析结果为%s' % (current_thread().name, len(res)))

if __name__ == '__main__':
    urls = [
        'https://www.baidu.com',
        'https://www.sina.com.cn',
        'https://www.tmall.com',
        'https://www.jd.com',
        'https://www.python.org',
        'https://www.bilibili.com',
        'https://www.youku.com',
        'https://www.baidu.com',
        'https://www.baidu.com',
        'https://www.baidu.com',
    ]
    p = ThreadPoolExecutor(4)
    start=time.time()
    for url in urls:
        future = p.submit(get, url)
        future.add_done_callback(parse)#parse会在任务完毕后触发，然后接收一个参数future对象
        #如果开启的是进程，那么来干parse这件事的是主进程
        #如果开启的是线程，那么线程池里的线程也可以来做parse，谁有空谁做
    p.shutdown(wait=True)
    print(time.time()-start)
    print('主%s'%current_thread().name)

3.协程

使用协程的目标是想要在单线程下实现并发

实现协程的原理是：切换+保存状态

注意：协程是程序员意淫出来的东西，操作系统里只有进程和线程的概念。

在单线程下实现多个任务间遇到IO就切换可以降低单线程的IO时间，从而最大限度地提升单线程的效率。

不遇到IO就进行切换的话并不能提升单线程的效率，这样不算是成功的协程。

4.gevent

要实现单线程下的协程，需要gevent模块

from gevent import spawn,sleep
from gevent import monkey,joinall
monkey.patch_all()#这样gevent就能识别所有IO行为
import time

#gevent不能识别本身以外的IO行为,为了监听所有的IO行为，要导入monkey
def play(name):
    print('%s play1'%name)
    time.sleep(3)
    print('%s play2'%name)
def eat(name):
    print('%s eat1'%name)
    time.sleep(5)
    print('%s eat2'%name)

start=time.time()
g1=spawn(play,'刘清正')#这里是异步提交
g2=spawn(eat,'刘清正')

joinall([g1,g2])
stop=time.time()
print(stop-start)

5.单线程实现并发套接字