046.Python协程

协程

1 生成器

初始化生成器函数 返回生成器对象,简称生成器

def gen():
        for i in range(10):
                #yield  返回便能够保留状态
                yield i
mygen = gen()
for i in mygen:
         print(i)

执行

[root@node10 python]# python3 test.py
0
1
2
3
4
5
6
7
8
9

使用next定义遍历里次数

def gen():
        for i in range(10):
                yield i

# 初始化生成器函数 返回生成器对象,简称生成器
mygen = gen()
for i in range (3):
        res = next(mygen)
        print (res)

执行

[root@node10 python]# python3 test.py
0
1
2

2 用协程改写生产者消费者模型

def producer():
    for i in range(100):
        yield i 

def consumer():
    g = producer()
    for i in g:
        print(i)

consumer()

3 协程的具体实现

switch 一般遇到阻塞时,可以手动调用该函数进行任务切换

缺点:不能够自动规避io,即不能自动实现遇到阻塞就切换

from greenlet import greenlet
import time
def plane():
        print ("plane one")
        print ("Plane two")
def fly():
        print ("fly to newyork")
        print ("fly to beijing")
g1 = greenlet(plane)
g2 = greenlet(fly)
g1.switch()

在执行之前，需要安装greenlet模块

[root@node10 python]# pip-3 install wheel

[root@node10 python]# pip-3 install gevent

执行python

[root@node10 python]# python3 test.py
plane one
Plane two

添加阻塞，并配置一个swith

import time
def plane():
        print ("plane one")
        g2.switch()
        time.sleep(2)
        print ("Plane two")
def fly():
        print ("fly to newyork")
        time.sleep(2)
        print ("fly to beijing")
g1 = greenlet(plane)
g2 = greenlet(fly)
g1.switch()

执行

[root@node10 python]# python3 test.py
plane one
fly to newyork
fly to beijing

有阻塞不能启动切换

from greenlet import greenlet
import time
def plane():
        print ("plane one")
        g2.switch()
        time.sleep(2)
        print ("Plane two")
def fly():
        print ("fly to newyork")
        time.sleep(2)
        print ("fly to beijing")
        g1.switch()
g1 = greenlet(plane)
g2 = greenlet(fly)
g1.switch()

执行

[root@node10 python]# python3 test.py
plane one
fly to newyork
fly to beijing
Plane two

4 使用gevent

缺陷:不能够识别time.sleep 阻塞

from greenlet import greenlet
import gevent
import time
def plane():
        print ("plane one")
        time.sleep(2)
        print ("Plane two")
def fly():
        print ("fly to newyork")
        time.sleep(2)
        print ("fly to beijing")
# 利用gevent 创建协程对象g1
g1 = gevent.spawn(plane)
# 利用gevent 创建协程对象g2
g2 = gevent.spawn(fly)
g1.join() #阻塞,直到g1协程任务执行完毕
g2.join() #阻塞,直到g2协程任务执行完毕
print("主线程执行完毕")

执行

[root@node10 python]# python3 test.py
plane one
Plane two
fly to newyork
fly to beijing
主线程执行完毕

阻塞没有生效

进阶改造

5 用gevent.sleep 取代 time.sleep()

from greenlet import greenlet
import gevent
import time
def plane():
        print ("plane one")
        gevent.sleep(2)
        print ("Plane two")
def fly():
        print ("fly to newyork")
        gevent.sleep(2)
        print ("fly to beijing")
# 利用gevent 创建协程对象g1
g1 = gevent.spawn(plane)
# 利用gevent 创建协程对象g2
g2 = gevent.spawn(fly)
g1.join() #阻塞,直到g1协程任务执行完毕
g2.join() #阻塞,直到g2协程任务执行完毕
print("主线程执行完毕")

执行，自动实现任务切换

[root@node10 python]# python3 test.py
plane one
fly to newyork
Plane two
fly to beijing
主线程执行完毕

终极解决不识别问题

6 引入ba patch_all

下面所有引入的模块所包含的阻塞,重新识别出来.

from greenlet import greenlet
from gevent import monkey
monkey.patch_all()
import gevent
import time
def plane():
        print ("plane one")
        time.sleep(2)
        print ("Plane two")
def fly():
        print ("fly to newyork")
        time.sleep(2)
        print ("fly to beijing")
# 利用gevent 创建协程对象g1
g1 = gevent.spawn(plane)
# 利用gevent 创建协程对象g2
g2 = gevent.spawn(fly)
g1.join() #阻塞,直到g1协程任务执行完毕
g2.join() #阻塞,直到g2协程任务执行完毕
print("主线程执行完毕")

执行

[root@node10 python]# python3 test.py
plane one
fly to newyork
Plane two
fly to beijing
主线程执行完毕

7 协程案例

1. spawn(函数,参数1,参数2,参数3....) 启动切换一个协程
2. join() 阻塞,直到某个协成执行完毕
3. joinall() 等待所有协成执行任务完毕
   • g1.join() g2.join() 可以通过joinall简写
   • gevent.joinall( [g1,g2] ) 等价于 1; 参数是一个列表;
4. value 获取协成返回值

oinall value函数的用法

rom gevent import monkey;monkey.patch_all()
import time
import gevent
def plane():
        print ("plane one")
        time.sleep(2)
        print ("Plane two")
        return ("有两架飞机")
def fly():
        print ("fly to newyork")
        time.sleep(2)
        print ("fly to beijing")
        return ("fly two place")
g1 = gevent.spawn(plane)
g2 = gevent.spawn(fly)
gevent.joinall( [g1,g2]  )
# 获取协成的返回值
print(g1.value)
print(g2.value)
print("主线程执行完毕")

执行

plane one
fly to newyork
Plane two
fly to beijing
有两架飞机
fly two place
主线程执行完毕

利用协程爬取页面数据

安装request模块

[root@node10 python]# pip-3 install requests

import gevent
import requests
import time
# 抓取网站信息,返回响应对象
print ("<++++++++++++抓取网站信息,返回响应对象+++++++++++++++++>")
response = requests.get("http://www.baidu.com")
print(response)
# 获取状态码
print ("<++++++++++++获取状态码+++++++++++++++++>")
res = response.status_code
print(res)
# 获取字符编码集 apparent_encoding
print ("<++++++++++++获取字符编码集 apparent_encoding+++++++++++++++++>")
res_code = response.apparent_encoding
print(res_code)
# 设置编码集
print ("<++++++++++++++设置编码集+++++++++++++++++>")
response.encoding = res_code
print ("<++++++++++++获取网页里面的内容+++++++++++++++++>")
res = response.text
print(res)

import re
strvar = r'<img hidefocus=true src="https://www.baidu.com/img/bd_logo1.png"  width=270 height=129>'
obj = re.search("src=(.*?) ",strvar)
res = obj.group()
print (res)
res = obj.groups()
print (res)
res = obj.groups()[0]
print (res)

执行

<++++++++++++抓取网站信息,返回响应对象+++++++++++++++++>
<Response [200]>
<++++++++++++获取状态码+++++++++++++++++>
200
<++++++++++++获取字符编码集 apparent_encoding+++++++++++++++++>
utf-8
<++++++++++++++设置编码集+++++++++++++++++>
<++++++++++++获取网页里面的内容+++++++++++++++++>
<!DOCTYPE html>
<!--STATUS OK--><html> <head><meta http-equiv=content-type content=text/html;charset=utf-8><meta http-equiv=X-UA-Compatible content=IE=Edge><meta content=always name=referrer><link rel=stylesheet type=text/css href=http://s1.bdstatic.com/r/www/cache/bdorz/baidu.min.css><title>百度一下，你就知道</title></head> <body link=#0000cc> <div id=wrapper> <div id=head> <div class=head_wrapper> <div class=s_form> <div class=s_form_wrapper> <div id=lg> <img hidefocus=true src=//www.baidu.com/img/bd_logo1.png width=270 height=129> </div> <form id=form name=f action=//www.baidu.com/s class=fm> <input type=hidden name=bdorz_come value=1> <input type=hidden name=ie value=utf-8> <input type=hidden name=f value=8> <input type=hidden name=rsv_bp value=1> <input type=hidden name=rsv_idx value=1> <input type=hidden name=tn value=baidu><span class="bg s_ipt_wr"><input id=kw name=wd class=s_ipt value maxlength=255 autocomplete=off autofocus></span><span class="bg s_btn_wr"><input type=submit id=su value=百度一下 class="bg s_btn"></span> </form> </div> </div> <div id=u1> <a href=http://news.baidu.com name=tj_trnews class=mnav>新闻</a> <a href=http://www.hao123.com name=tj_trhao123 class=mnav>hao123</a> <a href=http://map.baidu.com name=tj_trmap class=mnav>地图</a> <a href=http://v.baidu.com name=tj_trvideo class=mnav>视频</a> <a href=http://tieba.baidu.com name=tj_trtieba class=mnav>贴吧</a> <noscript> <a href=http://www.baidu.com/bdorz/login.gif?login&amp;tpl=mn&amp;u=http%3A%2F%2Fwww.baidu.com%2f%3fbdorz_come%3d1 name=tj_login class=lb>登录</a> </noscript> <script>document.write('<a href="http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u='+ encodeURIComponent(window.location.href+ (window.location.search === "" ? "?" : "&")+ "bdorz_come=1")+ '" name="tj_login" class="lb">登录</a>');</script> <a href=//www.baidu.com/more/ name=tj_briicon class=bri style="display: block;">更多产品</a> </div> </div> </div> <div id=ftCon> <div id=ftConw> <p id=lh> <a href=http://home.baidu.com>关于百度</a> <a href=http://ir.baidu.com>About Baidu</a> </p> <p id=cp>&copy;2017&nbsp;Baidu&nbsp;<a href=http://www.baidu.com/duty/>使用百度前必读</a>&nbsp; <a href=http://jianyi.baidu.com/ class=cp-feedback>意见反馈</a>&nbsp;京ICP证030173号&nbsp; <img src=//www.baidu.com/img/gs.gif> </p> </div> </div> </div> </body> </html>

src="https://www.baidu.com/img/bd_logo1.png"
('"https://www.baidu.com/img/bd_logo1.png"',)
"https://www.baidu.com/img/bd_logo1.png"

爬虫实例

import gevent
import requests
import time
# 抓取网站信息,返回响应对象
response = requests.get("http://www.baidu.com")
print(response)
# 获取状态码
res = response.status_code
print(res)
# 获取字符编码集 apparent_encoding
res_code = response.apparent_encoding
print(res_code)
# 设置编码集
response.encoding = res_code
res = response.text
print(res)

url_list = [
"http://www.baidu.com",
"http://www.4399.com",
"http://www.7k7k.com",
"http://www.jingdong.com",
"http://www.taobao.com",
]
def get_url(url):
        response = requests.get(url)
        if response.status_code == 200:
                pass
                # print(response.text)

# (1) 正常方式爬取数据
startime = time.time()
for i in url_list:
        get_url(i)
endtime = time.time()
print("<=1=1=1=1=1=1=1=1=>")
print(endtime-startime)

# (2) 用协程爬取数据 更快
startime = time.time()
lst = []
for i in url_list:
        g = gevent.spawn(get_url,i)
        lst.append(g)

gevent.joinall(lst)
endtime = time.time()
print("<=2=2=2=2=2=2=2=2=2=>")
print(endtime - startime)

执行

<Response [200]>
200
utf-8
<!DOCTYPE html>
<!--STATUS OK--><html> <head><meta http-equiv=content-type content=text/html;charset=utf-8><meta http-equiv=X-UA-Compatible content=IE=Edge><meta content=always name=referrer><link rel=stylesheet type=text/css href=http://s1.bdstatic.com/r/www/cache/bdorz/baidu.min.css><title>百度一下，你就知道</title></head> <body link=#0000cc> <div id=wrapper> <div id=head> <div class=head_wrapper> <div class=s_form> <div class=s_form_wrapper> <div id=lg> <img hidefocus=true src=//www.baidu.com/img/bd_logo1.png width=270 height=129> </div> <form id=form name=f action=//www.baidu.com/s class=fm> <input type=hidden name=bdorz_come value=1> <input type=hidden name=ie value=utf-8> <input type=hidden name=f value=8> <input type=hidden name=rsv_bp value=1> <input type=hidden name=rsv_idx value=1> <input type=hidden name=tn value=baidu><span class="bg s_ipt_wr"><input id=kw name=wd class=s_ipt value maxlength=255 autocomplete=off autofocus></span><span class="bg s_btn_wr"><input type=submit id=su value=百度一下 class="bg s_btn"></span> </form> </div> </div> <div id=u1> <a href=http://news.baidu.com name=tj_trnews class=mnav>新闻</a> <a href=http://www.hao123.com name=tj_trhao123 class=mnav>hao123</a> <a href=http://map.baidu.com name=tj_trmap class=mnav>地图</a> <a href=http://v.baidu.com name=tj_trvideo class=mnav>视频</a> <a href=http://tieba.baidu.com name=tj_trtieba class=mnav>贴吧</a> <noscript> <a href=http://www.baidu.com/bdorz/login.gif?login&amp;tpl=mn&amp;u=http%3A%2F%2Fwww.baidu.com%2f%3fbdorz_come%3d1 name=tj_login class=lb>登录</a> </noscript> <script>document.write('<a href="http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u='+ encodeURIComponent(window.location.href+ (window.location.search === "" ? "?" : "&")+ "bdorz_come=1")+ '" name="tj_login" class="lb">登录</a>');</script> <a href=//www.baidu.com/more/ name=tj_briicon class=bri style="display: block;">更多产品</a> </div> </div> </div> <div id=ftCon> <div id=ftConw> <p id=lh> <a href=http://home.baidu.com>关于百度</a> <a href=http://ir.baidu.com>About Baidu</a> </p> <p id=cp>&copy;2017&nbsp;Baidu&nbsp;<a href=http://www.baidu.com/duty/>使用百度前必读</a>&nbsp; <a href=http://jianyi.baidu.com/ class=cp-feedback>意见反馈</a>&nbsp;京ICP证030173号&nbsp; <img src=//www.baidu.com/img/gs.gif> </p> </div> </div> </div> </body> </html>

<=1=1=1=1=1=1=1=1=>
14.512077331542969
<=2=2=2=2=2=2=2=2=2=>
6.321309566497803

协程的速度比较快