selenium实现百度图片爬取

因为是百度图片是瀑布流ajax异步上传的数据，所以这里用到抓包工具来抓取链接（fiddler）

好了直接上代码，

 from selenium import webdriver
 from selenium.webdriver.common.by import By
 import requests,time
 from queue import Queue
 from urllib import request
 import os,gevent
 from lxml import etree

 def get_img(html):
     html = html.get()

     html = etree.HTML(html)

     img_url = html.xpath('//div[@id="imgid"]/div[last()]//li/@data-objurl')
     # print(img_url)
     path = './baidupic/'
     if not os.path.exists(path):
         os.makedirs(path)

     for url in img_url:
         print(url)
         # response = requests.get(url)
         # img = response.content
         try:
             fname = url.split('/')[-1]
             request.urlretrieve(url,os.path.join(path, fname))
             print('下载成功')
         except:
             print('图片不存在')

 def get_page():
     #创建数据队列
     q = Queue()

     #百度图片搜索地址
     base_url = 'https://image.baidu.com/'
     #返回浏览器对象
     browser = webdriver.Chrome(executable_path=r'C:\Users\zhaozhi\Desktop\chromedriver.exe')
     #模拟访问
     browser.get(base_url)
     #输入搜索关键字
     browser.find_element_by_id('kw').send_keys('美女')
     #按键
     browser.find_element_by_class_name('s_search').click()
     # time.sleep(2)
     for i in range(10):
         browser.execute_script('window.scrollTo(0,document.body.scrollHeight)')
         # time.sleep(2)
         # html = browser.page_source

         q.put(browser.page_source)
     # browser.close()
     # print(browser.page_source)
     g_list=[]
     for i  in range(20):
         g= gevent.spawn(get_img,q)
         g_list.append(g)

     gevent.joinall(g_list)

 # browser.save_screenshot('baidupic.png')
 # print(browser.page_source)
 # browser.find_element(By_)

 if __name__ == '__main__':
     get_page()