爬虫_淘宝（selenium）

总体来说代码还不是太完美

实现了js渲染网页的解析的一种思路

主要是这个下拉操作，不能一下拉到底，数据是在中间加载进来的，

具体过程都有写注释

 from selenium import webdriver
 import time
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.common.by import By
 from lxml import etree
 import re
 import pymongo

 client = pymongo.MongoClient('127.0.0.1', port=27017)
 db = client.taobao_mark
 collection = db.informations

 def search(url):

     # 设置无头浏览器
     opt = webdriver.ChromeOptions()
     opt.set_headless()
     driver = webdriver.Chrome(options=opt)
     driver.get(url)

     # 使用显示等待加载输入文本框,设置搜索关键字，以'轻薄本'为例
     input_key = WebDriverWait(driver, 10).until(
         EC.presence_of_element_located((By.ID, 'q')))
     input_key.send_keys('轻薄本')

     # 显示等待加载搜索按钮,然后执行点击操作
     search_btn = WebDriverWait(driver, 10).until(
         EC.presence_of_element_located((By.CLASS_NAME, 'btn-search')))
     search_btn.click()

     # 搜索出来的网页内容只有一行与关键字相关，剩下的都是广告
     # 此时发现更多相关内容在//span[@class="see-all"]这个标签的链接中，
     # 通过xpath获取链接，进行跳转
     element = etree.HTML(driver.page_source)
     list_page_url = element.xpath('//span[@class="see-all"]/a/@href')[0]
     list_page_url = 'https://s.taobao.com/' + list_page_url
     driver.close()
     return list_page_url

 def scroll(driver):
     # 如果没有进行动态加载，只能获取四分之一的内容,调用js实现下拉网页进行动态加载
     for y in range(7):
         js='window.scrollBy(0,600)'
         driver.execute_script(js)
         time.sleep(0.5)

 def get_detail_url(url):

     # 用来存储列表页中商品的的链接
     detail_urls = []

     opt = webdriver.ChromeOptions()
     opt.set_headless()
     driver = webdriver.Chrome(options=opt)
     # driver = webdriver.Chrome()
     driver.get(url)

     scroll(driver)

     # 通过简单处理，获取当前最大页数
     max_page = WebDriverWait(driver, 10).until(
         EC.presence_of_element_located((By.XPATH, '//div[@class="total"]'))
         )
     text = max_page.get_attribute('textContent').strip()
     max_page = re.findall('\d+', text, re.S)[0]
     # max_page = int(max_page)
     max_page = 1
     # 翻页操作
     for i in range(1, max_page+1):
         print('正在爬取第%s页' % i)
         # 使用显示等待获取页面跳转页数文本框
         next_page_btn = WebDriverWait(driver, 10).until(
             EC.presence_of_element_located((By.XPATH, '//input[@class="input J_Input"]'))
             )

         # 获取确定按钮，点击后可进行翻页
         submit = WebDriverWait(driver, 10).until(
             EC.element_to_be_clickable((By.XPATH, '//span[@class="btn J_Submit"]'))
             )

         next_page_btn.clear()
         next_page_btn.send_keys(i)
         # 这里直接点击会报错'提示不可点击'，百度说有一个蒙层，
         # 沉睡两秒等待蒙层消失即可点击
         time.sleep(2)
         submit.click()
         scroll(driver)

         urls = WebDriverWait(driver, 10).until(
         EC.presence_of_all_elements_located((By.XPATH, '//a[@class="img-a"]'))
         )
         for url in urls:
             detail_urls.append(url.get_attribute('href'))
     driver.close()
     # 返回所有商品链接列表
     return detail_urls

 def parse_detail(detail_urls):
     parameters = []
     opt = webdriver.ChromeOptions()
     opt.set_headless()
     driver = webdriver.Chrome(options=opt)
     for url in detail_urls:
         parameter = {}

         print('正在解析网址%s' % url)
         driver.get(url)
         html = driver.page_source
         element = etree.HTML(html)

         # 名字name
         name = element.xpath('//div[@class="spu-title"]/text()')
         name = name[0].strip()
         parameter['name'] = name

         # 价格price
         price = element.xpath('//span[@class="price g_price g_price-highlight"]/strong/text()')[0]
         price = str(price)
         parameter['price'] = price

         # 特点specials
         b = element.xpath('//div[@class="item col"]/text()')
         specials = []
         for i in b:
             if re.match('\w', i, re.S):
                 specials.append(i)
         parameter['specials'] = specials

         param = element.xpath('//span[@class="param"]/text()')
         # 尺寸
         size = param[0]
         parameter['size'] = size
         # 笔记本CPU
         cpu = param[1]
         parameter['cpu'] = cpu
         # 显卡
         graphics_card = param[2]
         parameter['graphics_card'] = graphics_card
         # 硬盘容量
         hard_disk = param[3]
         parameter['hard_disk'] = hard_disk
         # 处理器主频
         processor = param[4]
         parameter['processor'] = processor
         # 内存容量
         memory = param[5]
         parameter['memory'] = memory
         parameter['url'] = url
         print(parameter)
         print('='*50)
         save_to_mongo(parameter)
         parameters.append(parameter)
     return parameters

 def save_to_mongo(result):
     try:
         if collection.insert(result):
             print('success save to mongodb', result)
     except Exception:
         print('error to mongo')

 def main():
     url = 'https://www.taobao.com/'
     list_page_url = search(url)
     detail_urls = get_detail_url(list_page_url)
     parameters = parse_detail(detail_urls)

 if __name__ == '__main__':
     main()

运行结果

数据库