爬虫_腾讯招聘（xpath）

和昨天一样的工作量，时间只用了一半，但还是效率有点低了，因为要把两个网页结合起来，所以在列表操作上用了好多时间

 import requests
 from lxml import etree

 headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36'}

 def get_html(url):
     response = requests.get(url, headers=headers)
     response.encoding = response.apparent_encoding
     html = response.text
     return html

 def parse_html(html):
     informations = []
     urls = []
     html_element = etree.HTML(html)
     kinds = html_element.xpath('(//tr[@class="even"]|//tr[@class="odd"])/td[2]/text()')
     '''
     kinds:
     ['技术类', '设计类', '技术类', '技术类', '技术类', '技术类', '技术类', '技术类', '技术类', '产品/项目类']
     '''
     nums = html_element.xpath('(//tr[@class="even"]|//tr[@class="odd"])//td[3]/text()')
     '''
     nums:
     ['2', '1', '2', '1', '2', '2', '1', '2', '1', '1']
     '''
     addresses = html_element.xpath('(//tr[@class="even"]|//tr[@class="odd"])//td[4]/text()')
     '''
     addresses:
     ['深圳', '深圳', '深圳', '深圳', '深圳', '深圳', '深圳', '深圳', '深圳', '深圳']
     '''
     times = html_element.xpath('(//tr[@class="even"]|//tr[@class="odd"])//td[5]/text()')
     '''
     times:
     ['2018-08-04', '2018-08-04', '2018-08-04', '2018-08-04', '2018-08-04', '2018-08-04', '2018-08-04', '2018-08-04', '2018-08-04', '2018-08-04']
     '''
     names = html_element.xpath('(//tr[@class="even"]|//tr[@class="odd"])//a/text()')

     detail_url = html_element.xpath('(//tr[@class="even"]|//tr[@class="odd"])//a/@href')
     for str_url in detail_url:

         url = 'https://hr.tencent.com/' + str(str_url)
         urls.append(url)

     '''
     urls :
     ['https://hr.tencent.com/position_detail.php?id=42917&keywords=python&tid=0&lid=0',
      'https://hr.tencent.com/position_detail.php?id=42908&keywords=python&tid=0&lid=0',
                                  ......
      'https://hr.tencent.com/position_detail.php?id=42832&keywords=python&tid=0&lid=0',
      'https://hr.tencent.com/position_detail.php?id=42628&keywords=python&tid=0&lid=0']
     '''
     for index, name in enumerate(names):
         information = {}
         information['name'] = name
         information['url'] = urls[index]
         information['kind'] = kinds[index]
         information['nums_of_need'] = nums[index]
         information['address'] = addresses[index]
         informations.append(information)
     # print(informations)
     # print(urls)
     return urls, informations

 def parse_detail_page(url):
         #one detail page
     html = get_html(url)
     return html

 def get_all_page(page_nums):
     for i in range(0, page_nums):
         url = 'https://hr.tencent.com/position.php?lid=&tid=&keywords=python&start={0}#a'.format(i*10)
         html = get_html(url)
         urls, informations = parse_html(html)
         # print(informations)
         works = []
         for i, url in enumerate(urls):

             html_detail = parse_detail_page(url)
             html_element = etree.HTML(html_detail)
             work_intro = html_element.xpath('//td[@class="l2"]//text()')
             for index, text in enumerate(work_intro):
                 if text.startswith('工作职责：'):
                     text = text.replace('工作职责：', '')
                     works_detail = {}
                     intros = []
                     for x in range(index+1, len(work_intro)):
                         intro = work_intro[x].strip()
                         if work_intro[x].startswith('工作要求：'):
                             break
                         intros.append(intro)
                     while '' in intros:
                         intros.remove('')
                     works_detail['1_____工作职责：'] = intros
                     works.append(works_detail)
                     # print(intros)
                     '''
                     ['负责NLP与深度学习相关技术的研究与实现；',
                      '负责建设基础的语义分析工具和平台；',
                      '负责搜索系统、知识图谱系统、问答与对话系统的设计与搭建；',
                      '结合实际业务需求与数据，研发高效、稳健、完备的NLP解决方案。']
                     '''

                 if text.startswith('工作要求：'):
                     text = text.replace('工作要求：', '')
                     works_detail = {}
                     requests = []
                     for x in range(index+1, len(work_intro)):
                         intro = work_intro[x].strip()
                         if work_intro[x].startswith('申请岗位'):
                             break
                         requests.append(intro)
                     while '' in requests:
                         requests.remove('')
                     works_detail['2_____工作要求：'] = requests
                     works.append(works_detail)
                     # print(requests)
                     '''
                     ['三年以上自然语言处理经验包括语义表示、搜索、知识图谱、对话系统等；',
                      '扎实的编程基础，至少精通一种编程语言，如C++，Java，python等；',
                      '熟悉深度学习以及常见机器学习算法的原理与算法，能熟练运用聚类、分类、回归、排序等模型解决有挑战性的问题；',
                      '对自然语言处理相关的分词、词性标注、实体识别、句法分析、语义分析等有深入的实践经验；',
                      '有强烈求知欲，对人工智能领域相关技术有热情；', '具有良好的数学基础，良好的英语阅读能力；',
                      '有项目管理经验，与他人合作良好，能够独立有效推动复杂项目。']
                     '''
         return works, informations

 def main():
     works, informations = get_all_page(1)
     for index, information in enumerate(informations):
         list = []
         list.append(works[index*2])
         list.append(works[index*2+1])
         information['duty'] = list
         print(information)

 if __name__ == '__main__':
     main()

目前sublime还输入不了中文，所以把输出注释上，方便看清格式

运行结果：

红色圈出来的是一个字典，包含第一个网页的信息（职位名称，url，位置）和详情页面的职责（工作职责，工作要求），嵌套的可能有点复杂，但目前还没有想到更简明的方法