selenium+chrome抓取淘宝搜索抓娃娃关键页面

最近迷上了抓娃娃，去富国海底世界抓了不少，完全停不下来，还下各种抓娃娃的软件，梦想着有一天买个抓娃娃的机器存家里~.~

今天顺便抓了下马爸爸家抓娃娃机器的信息，晚辈只是觉得翻得手酸，本来100页的数据，就抓了56条，还希望马爸爸莫怪。。。。

有对爬虫的感兴趣的媛友，可以作为参考哦！

 # coding:utf-8

 import pymongo
 import time
 import re
 from selenium import webdriver
 from bs4 import BeautifulSoup
 from selenium.webdriver.common.by import By
 from selenium.common.exceptions import TimeoutException
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 # 使用无界面浏览器
 from selenium.webdriver.chrome.options import Options
 # 创建驱动对象
 chrome_options = webdriver.ChromeOptions()
 # 设置代理
 # chrome_options.add_argument('--proxy-server=http://ip:port')
 # 加载浏览器头
 # 更换头部
 chrome_options.add_argument('User-Agent="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36"')
 # 不加载图片
 prefs = {"profile.managed_default_content_settings.images":2}
 chrome_options.add_experimental_option("prefs",prefs)
 # 设置无界面浏览器
 chrome_options = Options()
 chrome_options.add_argument('--headless')
 chrome_options.add_argument('--disable--gpu')
 driver = webdriver.Chrome(chrome_options=chrome_options)

 # 设置显示等待时间
 wait = WebDriverWait(driver,1)

 # 翻页 在底部输入框输入页数进行翻页
 def next_page(page_num):
     try:
         print ('正在翻第----%s------页' %page_num)
         # 获得页码输入框  try是否超时报错
         input = wait.until(
             EC.presence_of_element_located((By.CSS_SELECTOR,'#mainsrp-pager > div > div > div > div.form > input'))
         )
         # 清空输入框的内容
         input.clear()
         # 输入框输入页码
         input.send_keys(page_num)
         # 获得确定按钮
         btn = driver.find_element_by_class_name('.btn J_Submit')
         # 点击确定按钮
         btn.click()
         # 根据对应页码高亮部分判断是否成功
         # 获得高亮部分
         wait.until(
             EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > ul > li.item.active > span'))
         )
         parse_html(driver.page_source)
     except TimeoutException:
         # 报错重新请求
         return next_page(page_num)

 def save_mongo(data):
     client = pymongo.MongoClient('60.205.211.210',27017)
     # 连接数据库
     db = client.test
     # 连接集合
     collection = db.taobao
     # 插入数据
     collection.insert(data)

 def parse_html(html):
     html_page = BeautifulSoup(html,'lxml')
     items = html_page.select('.m-itemlist .items .item')
     for item in items:
         # 图片url
         img_url = item.select('.pic .pic-link')[0].attrs['href']
         if img_url.startswith('//'):
             img_url = 'https:' + img_url
         # print (img_url)
         # 价格
         price = item.select('.price')[0].text.strip()[1:]
         # print (price)
         # 销量
         sales = re.search(r'(\d+)',item.select('.deal-cnt')[0].text).group(1)
         # print (sales)
         # 商品描述
         desc = item.select('.title .J_ClickStat')[0].text.strip()
         # print (desc)
         # 店铺
         shop = item.select('.shop .dsrs + span')[0].text.strip()
         # print (shop)
         # 地址
         address = item.select('.location')[0].text.strip()
         # print(address)
         product_data = {
             'img_url':img_url,
             'price':price,
             'desc':desc,
             'sales':sales,
             'shop':shop,
             'address':address
         }
         print (product_data)
         save_mongo(product_data)

 def get_page(url):
     try:
         driver.get(url)
         search_input = wait.until(
             # 参数为元祖
             EC.presence_of_element_located((By.ID,'q'))
         )
         search_input.send_keys(u'抓娃娃')
         btn = driver.find_element_by_class_name('btn-search')
         btn.click()
         # 实现翻页 需要把总页数return出去
         total_pages = driver.find_element_by_css_selector('#mainsrp-pager > div > div > div > div.total').text
         total_pages_num = re.compile(r'(\d+)').search(total_pages).group(1)
         # print (total_pages_num)
         parse_html(driver.page_source)
         return total_pages_num
     except TimeoutException as e:
         print (e.msg)

 def main():
     url = 'https://www.taobao.com'
     try:
         page_num = get_page(url)
         # for i in range(2,int(page_num)+1):
         for i in range(2, int(page_num) - 96):
             next_page(i)
     except Exception as e:
         print (e.message)
     finally:
         driver.close()

 if __name__ == '__main__':
     main()

要是环境配置好了，可以直接跑一下哦！

下面是mongo数据库数据：