爬虫_拉勾网(selenium)

使用selenium进行翻页获取职位链接，再对链接进行解析

会爬取到部分空列表，感觉是网速太慢了，加了time.sleep()还是会有空列表

 from selenium import webdriver
 import requests
 import re
 from lxml import etree
 import time
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.common.by import By

 class LagouSpider(object):
     def __init__(self):
         opt = webdriver.ChromeOptions()
         # 把chrome设置成无界面模式
         opt.set_headless()
         self.driver = webdriver.Chrome(options=opt)
         self.url = 'https://www.lagou.com/jobs/list_爬虫?px=default&city=北京'
         self.headers = {
             'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36'
         }

     def run(self):
         self.driver.get(self.url)
         while True:
             html = ''
             links = []
             html = self.driver.page_source
             links = self.get_one_page_links(html)
             for link in links:
                 print('\n' + link+'\n')
                 self.parse_detail_page(link)

             WebDriverWait(self.driver, 10).until(
                 EC.presence_of_element_located((By.CLASS_NAME, 'pager_next')))
             next_page_btn = self.driver.find_element_by_class_name('pager_next')

             if 'pager_next_disabled' in next_page_btn.get_attribute('class'):
                 break
             else:
                 next_page_btn.click()
             time.sleep(1)

     def get_one_page_links(self, html):
         links = []
         hrefs = self.driver.find_elements_by_xpath('//a[@class="position_link"]')
         for href in hrefs:
             links.append(href.get_attribute('href'))
         return links

     def parse_detail_page(self, url):
         job_information = {}
         response = requests.get(url, headers=self.headers)

         time.sleep(2)
         html = response.text
         html_element = etree.HTML(html)
         job_name = html_element.xpath('//div[@class="job-name"]/@title')
         job_description = html_element.xpath('//dd[@class="job_bt"]//p//text()')
         for index, i in enumerate(job_description):
             job_description[index] = re.sub('\xa0', '', i)
         job_address = html_element.xpath('//div[@class="work_addr"]/a/text()')
         job_salary = html_element.xpath('//span[@class="salary"]/text()')

         # 字符串处理去掉不必要的信息
         for index, i in enumerate(job_address):
             job_address[index] = re.sub('查看地图', '', i)
         while '' in job_address:
             job_address.remove('')

         job_information['job_name'] = job_name
         job_information['job_description'] = job_description
         job_information['job_address'] = job_address
         job_information['job_salary'] = job_salary
         print(job_information)

 def main():
     spider = LagouSpider()
     spider.run()

 if __name__ == '__main__':
     main()

运行结果