python+selenium爬取关键字搜索google图片

 # -*- coding: utf-8 -*-

 import json
 import os
 import time
 from multiprocessing import Pool
 import multiprocessing
 import requests
 from selenium import webdriver

 def get_image_links(keyword, num_requested = 1000):
     """get image links with selenium
     """
     number_of_scrolls = int(num_requested/400) + 1
     img_urls = set()#设置为集合，自动去除重复链接
     chrome_options = webdriver.ChromeOptions()
     # chrome_options.add_argument('--headless')#设置无头浏览器
     # chrome_options.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"')
     # chrome_options.add_argument("lang=en_US")#设置语言
     # prefs = {"profile.managed_default_content_settings.images":2}
     # chrome_options.add_experimental_option("prefs",prefs)#配置不加载图片
     driver = webdriver.Chrome(chrome_options=chrome_options)
     driver.maximize_window()
     search_query = keyword
     url = "https://www.google.com/search?q="+search_query+"&source=lnms&tbm=isch"
     driver.get(url)
     for _ in range(number_of_scrolls):
         for i in range(5):
             # multiple scrolls needed to show all 400 images
             driver.execute_script("window.scrollBy(0, 100000)")
             time.sleep(1)
         time.sleep(5)#等待页面刷新，否则有可能元素不可见
         try:
             # driver.find_element_by_xpath("//input[@value='Show more results']").click()＃浏览器的中英文版本不同
             driver.find_element_by_xpath("//input[@value='显示更多结果']").click()
         except Exception as e:
             print("reach the end of page ")
             break

     # with open('page.html','w') as f:
     #     f.write(driver.page_source)
     imgs = driver.find_elements_by_xpath('//div[contains(@class,"rg_meta")]')#模糊定位
     for i,img in enumerate(imgs):
         img_url = json.loads(img.get_attribute('innerHTML'))["ou"]
         img_urls.add(img_url)
     driver.quit()
     print("finish getting all image urls!")

     return img_urls

 def download(urls,download_dir):
     '''download images
     '''
     print("start downloading images!")
     for url in urls:
         filename=os.path.join(download_dir,os.path.basename(url))
         try:
             r = requests.get(url, stream=True, timeout=60)
             r.raise_for_status()
             with open(filename, 'wb') as f:
                 f.write(r.content)
         except Exception:
             continue
     print("finish downloading images!")

 keywords = ['girl','boy']
 download_dir = './images/'
 download_dirs = []
 for keyword in keywords:
     path = os.path.join(download_dir,keyword)
     download_dirs.append(path)
     if not os.path.exists(path):
         os.makedirs(path)

 # for keyword in main_keywords:
 #     image_urls = get_image_links(keyword)
 #     download(image_urls,download_dir)

 ###################################
 # get image links/MultiProcess
 ###################################
 img_urls=[]
 multiprocessing.freeze_support()
 p = Pool(4) # default number of process is the number of cores of your CPU, change it by yourself
 for keyword in keywords:
     img_urls.append(p.apply_async(get_image_links, (keyword,)))
 #img_urls:[<multiprocessing.pool.ApplyResult object at 0x7f536925fcc0>, <multiprocessing.pool.ApplyResult object at 0x7f536925fd68>]
 for i,urls in enumerate(img_urls):
     img_urls[i]=urls.get()
 p.close()
 p.join()

 # # ###################################
 # # # download images/MultiProcess
 # # ###################################
 p = Pool(4) # default number of process is the number of cores of your CPU, change it by yourself
 for i,urls in enumerate(img_urls):
     p.apply_async(download, [urls,download_dirs[i]])
 p.close()
 p.join()