async 异步抓取 花瓣网高清大图 30s爬取500张

废话 不多说，直接上代码，不懂得看注释

先安装  pip install aiohttp

 "异步抓取花瓣网图片"

 # pip install aiohttp
 import requests
 import aiohttp
 import asyncio
 import time
 import os

 headers = {
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36',
     "X-Request": "JSON",
     "Accept": "application/json",
     "X-Requested-With": "XMLHttpRequest"
 }

 def get_image_urls():
     """获取图片链接"""
     print('开始获取图片链接，请耐心等待......')
     image_id = 2551285279
     for page in range(1,26):
         url = "https://huaban.com/boards/41743806/?jzwfs8ej&max="+ str(image_id) +"&limit=20&wfl=1"
         response = requests.get(url,headers=headers).json()
         pins = response['board']['pins']
         end_pins_id = pins[-1]['pin_id']
         for i in pins:
             key = i['file']['key']
             urls.append('http://hbimg.huabanimg.com/'+key)
         image_id = end_pins_id  # 下一页url中需要的参数

 #特殊的函數：该函数调用后,函数内部的程序语句不会被执行,但是该函数调用会返回一个协程对象
 async def get_audio_data(url):
   #使用aiohttp进行请求发送
     #实例化了一个发送网络请求的对象
     async with aiohttp.ClientSession() as s:
       #该函数内部的异步操作必须使用await进行修饰
         async with await s.get(url=url,headers=headers) as response:
             audio_data = await response.read()  #read()返回的是二进制形式的响应数据
             return {'data':audio_data,'url':url}

 #任务对象的回调函数,进行数据的持久化存储
 def saveData(task):
     dic_obj = task.result()
     name = dic_obj['url'].split('/')[-1]
     data = dic_obj['data']
     image_dir = 'images'
     if not os.path.exists(image_dir):
         os.mkdir(image_dir)
     with open(os.path.join(image_dir,name) + '.jpg','wb') as fp:
         fp.write(data)
     print(name+'下载完毕!')

 if __name__ == '__main__':
     start_time = time.clock()
     urls = []
     tasks = []
     get_image_urls()
     for url in urls:
         #调用该特殊函数,让其返回一个协程对象
         c = get_audio_data(url)
         #将协程对象封装到任务对象中
         task = asyncio.ensure_future(c)
         # 给任务对象绑定回调函数
         task.add_done_callback(saveData)
         #将任务对象添加到列表中
         tasks.append(task)
     #创建一个事件循环对象
     loop = asyncio.get_event_loop()
     #将任务对象列表注册到事件循环对象中,并且开启事件循环
     loop.run_until_complete(asyncio.wait(tasks))
     end_time = time.clock()
     print('抓取{}张图片，共计用时{}秒'.format(len(tasks),end_time-start_time))

注：window最大线程数 512，所以任务数不要超过这个值，否则 抛出异常