并发编程 - 线程 - 1.线程queue/2.线程池进程池/3.异步调用与回调机制

1.线程queue :会有锁
    q=queue.Queue(3)
    q.get()
    q.put()

先进先出 队列
后进先出 堆栈
优先级队列

 """先进先出 队列"""
 import queue
 q=queue.Queue(3) #先进先出->队列

 q.put('first')
 q.put(2)
 # q.put('third')
 # q.put(4)
 q.put(4,block=False) #q.put_nowait(4)
 # q.put_nowait(4)
 # q.put(4,block=True)  # True 阻塞 False 不阻塞 直接告诉你 队列满了
 # q.put(4,block=True,timeout=3) # 阻塞等待3秒 还没有拿走数据就抛异常
 #
 print(q.get())
 print(q.get())
 print(q.get())
 print(q.get(block=True,timeout=2))    # false 不阻塞没有数据就抛异常  默认是阻塞 block=True
 print(q.get_nowait()) # 相当于block=false
 # def get(self, block=True, timeout=None):

 """后进先出 堆栈"""
 import queue
 q=queue.LifoQueue(3)  #后进先出->堆栈
 q.put('first')
 q.put(2)
 q.put('third')

 print(q.get())
 print(q.get())
 print(q.get())

 """优先级队列 """
 import queue
 q=queue.PriorityQueue(3) #优先级队列

 q.put((10,{'alice':12}))  # 数字越小 优先级越高 优先拿出来
 q.put((40,'two'))
 q.put((30,'three'))

 print(q.get())
 print(q.get())
 print(q.get())

2.线程池进程池：
client server 是IO 操作应该用多线程
计算密集型: 用多进程
io密集型：用多线程

池：对数目加以限制，保证机器正常运行

 from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor
 import os,time,random

 def task(name):
     print('name:%s pid:%s run' %(name,os.getpid()))
     time.sleep(random.randint(1,3))

 if __name__ == '__main__':
     pool=ProcessPoolExecutor(4)  # 不指定 默认是cpu的核数
     # pool=ThreadPoolExecutor(5)

     for i in range(10):
         pool.submit(task,'egon%s' %i)  # 异步调用池子收了10个任务，但同一时间只有4个任务在进行

     pool.shutdown(wait=True)  # 类似join  代表往池子里面丢任务的入口封死了 计数器-1

     print('主')
 """
 主                         # # 异步调用池子收了10个任务，但同一时间只有4个任务在进行
 name:egon0 pid:60056 run     # 只有4个pid
 name:egon1 pid:64700 run
 name:egon2 pid:59940 run
 name:egon3 pid:60888 run

 name:egon4 pid:60888 run

 name:egon5 pid:60056 run
 name:egon6 pid:60888 run

 name:egon7 pid:60056 run
 name:egon8 pid:64700 run
 name:egon9 pid:59940 run
 """
 # pool.shutdown(wait=True) # 代表往池子里面丢任务的入口封死了 计数器-1
 """
 name:egon0 pid:57124 run
 name:egon1 pid:62252 run
 name:egon2 pid:55736 run
 name:egon3 pid:62060 run
 name:egon4 pid:57124 run
 name:egon5 pid:62252 run
 name:egon6 pid:55736 run
 name:egon7 pid:55736 run
 name:egon8 pid:62060 run
 name:egon9 pid:55736 run
 主
 """

 from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor
 from threading import currentThread
 import os,time,random

 def task():
     print('name:%s pid:%s run' %(currentThread().getName(),os.getpid()))
     time.sleep(random.randint(1,3))

 if __name__ == '__main__':
     pool=ThreadPoolExecutor(5)

     for i in range(10):
         pool.submit(task)

     pool.shutdown(wait=True)

     print('主')
 """
 name:ThreadPoolExecutor-0_0 pid:61508 run
 name:ThreadPoolExecutor-0_1 pid:61508 run
 name:ThreadPoolExecutor-0_2 pid:61508 run
 name:ThreadPoolExecutor-0_3 pid:61508 run
 name:ThreadPoolExecutor-0_4 pid:61508 run
 name:ThreadPoolExecutor-0_2 pid:61508 run
 name:ThreadPoolExecutor-0_4 pid:61508 run
 name:ThreadPoolExecutor-0_0 pid:61508 run
 name:ThreadPoolExecutor-0_3 pid:61508 run
 name:ThreadPoolExecutor-0_1 pid:61508 run
 主
 """

3.异步调用与回调机制：
提交任务的两种方式：
    同步调用：提交完任务后，就在原地等待任务执行完毕，拿到结果，再执行下一行代码,导致程序是串行执行，效率低
    异步调用：提交完任务后，不等待任务执行完毕。异步调用+回调机制  自动触发叫回调

 """同步调用"""
 from concurrent.futures import ThreadPoolExecutor
 import time
 import random

 def la(name):
     print('%s is laing' %name)
     time.sleep(random.randint(3,5))
     res=random.randint(7,13)*'#'
     return {'name':name,'res':res}

 def weigh(shit):
     name=shit['name']
     size=len(shit['res'])
     print('%s 拉了 《%s》kg' %(name,size))

 if __name__ == '__main__':
     pool=ThreadPoolExecutor(13)

     shit1=pool.submit(la,'alex').result()
     weigh(shit1)

     shit2=pool.submit(la,'wupeiqi').result()
     weigh(shit2)

     shit3=pool.submit(la,'yuanhao').result()
     weigh(shit3)

 """异步调用 + 回调机制  自动触发叫回调"""
 from concurrent.futures import ThreadPoolExecutor
 import time
 import random

 def la(name):
     print('%s is laing' %name)
     time.sleep(random.randint(3,5))
     res=random.randint(7,13)*'#'
     return {'name':name,'res':res}
     # weigh({'name':name,'res':res})  # 这样写不好  所有功能 写在一起了

 def weigh(shit):
     shit=shit.result()  # 拿到是 对象 需要result()
     name=shit['name']
     size=len(shit['res'])
     print('%s 拉了 《%s》kg' %(name,size))

 if __name__ == '__main__':
     pool=ThreadPoolExecutor(13)

     # pool.submit(la, 'alex')
     # pool.submit(la, 'wupeiqi')
     # pool.submit(la, 'yuanhao')

     pool.submit(la,'alex').add_done_callback(weigh) # 实现了程序的解耦合
     pool.submit(la,'wupeiqi').add_done_callback(weigh)
     pool.submit(la,'yuanhao').add_done_callback(weigh)

4.异步调用与回调机制应用：
pip3 install requests
    requests

异步调用+回调机制的 应用场景：

from concurrent.futures import ThreadPoolExecutor
import requests
import time

def get(url):   # io操作  基于线程 数目有限 用线程池
    print('GET %s' %url)
    response=requests.get(url)
    time.sleep(3)
    return {'url':url,'content':response.text}

def parse(res):
    res=res.result()
    print('%s parse res is %s' %(res['url'],len(res['content'])))

if __name__ == '__main__':
    urls=[
        'http://www.cnblogs.com/linhaifeng',
        'https://www.python.org',
        'https://www.openstack.org',
    ]

    pool=ThreadPoolExecutor(2)

    for url in urls:
        pool.submit(get,url).add_done_callback(parse)