Python爬虫系列-Selenium+Chrome/PhantomJS爬取淘宝美食

1.搜索关键字

利用Selenium驱动浏览器搜索关键字，得到查询后的商品列表

2.分析页码并翻页

得到商品页码数，模拟翻页，得到后续页面的商品列表

3.分析提取商品内容

利用PyQuery分析源码，解析得到商品列表

4.存储至MongoDB

将商品列表信息存储到数据库MongoDB

无法解决登录的问题，所以代码还是需要修改

from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from pyquery import PyQuery as pq
from config import *
import pymongo
import re

client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB]

browser = webdriver.Chrome()
wait = WebDriverWait(browser, 10) 

def search():
  try:
    browser.get('https://www.taobao.com')
    input = wait.until(
      EC.presence_of_element_located((By.CSS_SELECTOR, "#q"))
      )
    submit = wait.until(
      EC.element_to_be_clickable((By.CSS_SELECTOR,"#J_TSearchForm > div.search-button > button"))
      )
    input.send_keys('美食')
    submit.click()
    total = wait.until(
      EC.presence_of_element_located((By.CSS_SELECTOR, "#mainsrp-pager > div > div > div > div.total"))
    )
    get_products()
    return total.text
  except TimeoutException:
    return search()

def next_page(page_number):
  try:
    input = wait.until(
      EC.presence_of_element_located((By.CSS_SELECTOR, "#mainsrp-pager > div > div > div > div.form > input"))
     )
    submit = wait.until(
    EC.element_to_be_clickable((By.CSS_SELECTOR,"#mainsrp-pager > div > div > div > div.form > span.btn.J_Submit"))
        )
    input.clear()
    input.send_keys(page_number)
    submit.click()
    wait.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR,'#mainsrp-pager > div > div > div > ul > li.item.active > span'),str(page_number)))
    get_products()
  except TimeoutException:
    next_page(page_number)

def get_products():
  wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'#mainsrp-itemlist .items .item')))
  html = browser.page_source
  doc = pq(html)
  items = doc('#mainsrp-itemlist .items .item').items()
  for item in items:
    product = {
      'image': item.find('.pic .img').attr('src'),
      'price': item.find('.price').text(),
      'deal': item.find('.deal-cnt').text()[:-3],
      'title': item.find('.title').text(),
      'shop': item.find('.shop').text(),
      'location':item.find('location').text()
    }
    print(product)
    save_to_mongo(product)

def save_to_mongo(result):
  try:
    if db[MONGO_TABLE].insert(result):
      print('存储到MONGODB成功',result)
  except Exception:
    print('存储到MongoDB错误！',result)

def main():
  total = search()
  total = int(re.compile('(\d+)').search(total).group(1))
  for i in range(2, total+1):
    next_page(i)
  browser.close()

if __name__=='__main__':
  main()

from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from pyquery import PyQuery as pq
from config_phantomjs import *
import pymongo
import re

client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB]

browser = webdriver.PhantomJS(service_args=SERVICE_ARGS)
wait = WebDriverWait(browser, 10)
browser.set_window_size(1400,900)

def search():
  print('正在搜索')
  try:
    browser.get('https://www.taobao.com')
    input = wait.until(
      EC.presence_of_element_located((By.CSS_SELECTOR, "#q"))
      )
    submit = wait.until(
      EC.element_to_be_clickable((By.CSS_SELECTOR,"#J_TSearchForm > div.search-button > button"))
      )
    input.send_keys('美食')
    submit.click()
    total = wait.until(
      EC.presence_of_element_located((By.CSS_SELECTOR, "#mainsrp-pager > div > div > div > div.total"))
    )
    get_products()
    return total.text
  except TimeoutException:
    return search()

def next_page(page_number):
  print('正在翻页',page_number)
  try:
    input = wait.until(
      EC.presence_of_element_located((By.CSS_SELECTOR, "#mainsrp-pager > div > div > div > div.form > input"))
     )
    submit = wait.until(
    EC.element_to_be_clickable((By.CSS_SELECTOR,"#mainsrp-pager > div > div > div > div.form > span.btn.J_Submit"))
        )
    input.clear()
    input.send_keys(page_number)
    submit.click()
    wait.until(EC.text_to_be_present_in_element((By.CSS_SELECTOR,'#mainsrp-pager > div > div > div > ul > li.item.active > span'),str(page_number)))
    get_products()
  except TimeoutException:
    next_page(page_number)

def get_products():
  wait.until(EC.presence_of_element_located((By.CSS_SELECTOR,'#mainsrp-itemlist .items .item')))
  html = browser.page_source
  doc = pq(html)
  items = doc('#mainsrp-itemlist .items .item').items()
  for item in items:
    product = {
      'image': item.find('.pic .img').attr('src'),
      'price': item.find('.price').text(),
      'deal': item.find('.deal-cnt').text()[:-3],
      'title': item.find('.title').text(),
      'shop': item.find('.shop').text(),
      'location':item.find('location').text()
    }
    print(product)
    save_to_mongo(product)

def save_to_mongo(result):
  try:
    if db[MONGO_TABLE].insert(result):
      print('存储到MONGODB成功',result)
  except Exception:
    print('存储到MongoDB错误！',result)

def main():
  try:
    total = search()
    total = int(re.compile('(\d+)').search(total).group(1))
    for i in range(2, total+1):
      next_page(i)
  finally:
    browser.close()

if __name__=='__main__':
  main()