Scrapy实现腾讯招聘网信息爬取【Python】

一.腾讯招聘网

二.代码实现

　　1.spider爬虫

 # -*- coding: utf-8 -*-
 import scrapy
 from Tencent.items import TencentItem

 class TencentSpider(scrapy.Spider):
     name = 'tencent'
     allowed_domains = ['tencent.com']
     base_url = 'https://hr.tencent.com/position.php?&start='
     offset = 0
     start_urls = [base_url + str(offset)]
     def parse(self, response):
         node_list = response.xpath("//tr[@class='even'] | //tr[@class='odd']")
         for node in node_list:
             # 创建Item对象来保存信息
             item = TencentItem()
             positionName = node.xpath("./td[1]/a/text()").extract()[0]
             if len(node.xpath("./td[2]/text()")):
                 positionType = node.xpath("./td[2]/text()").extract()[0]
             else:
                 positionType = ""
             positionNumber = node.xpath("./td[3]/text()").extract()[0]
             location = node.xpath("./td[4]/text()").extract()[0]
             publishTime = node.xpath("./td[5]/text()").extract()[0]

             # 保存到item中
             item['positionName'] = positionName
             item['positionType'] = positionType
             item['positionNumber'] = positionNumber
             item['location'] = location
             item['publishTime'] = publishTime

             yield item

         # 判断是否需要拼接下一页路径
         if self.offset < 2770:
             self.offset += 10
             # 拼接下一页路径
             url = self.base_url + str(self.offset)
             # dont_filter=True 禁止因域名不同而过滤
             yield scrapy.Request(url, callback=self.parse)

　　2.管道

 # -*- coding: utf-8 -*-

 # Define your item pipelines here
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
 # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 import json

 class TencentPipeline(object):
     def __init__(self):
         self.f = open("Tencent.json", "w")

     def process_item(self, item, spider):
         text = json.dumps(dict(item), ensure_ascii=False) + ",\n"
         self.f.write(text)
         return item

     def close_spider(self, spider):
         self.f.close()

　　3.实体

 # -*- coding: utf-8 -*-

 # Define here the models for your scraped items
 #
 # See documentation in:
 # https://doc.scrapy.org/en/latest/topics/items.html

 import scrapy

 class TencentItem(scrapy.Item):
     # define the fields for your item here like:
     # name = scrapy.Field()
     # 职位名称
     positionName = scrapy.Field()
     # 职位类别
     positionType = scrapy.Field()
     # 人数
     positionNumber = scrapy.Field()
     # 地点
     location = scrapy.Field()
     # 发布时间
     publishTime = scrapy.Field()

三.结果【部分展示】

{"positionName": "15605-动作RPG手游游戏活动策划", "positionType": "产品/项目类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "21645-高级法律顾问", "positionType": "职能类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "HGJ-senior legal counsel(MA)", "positionType": "职能类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "19837-电影新媒体媒介经理", "positionType": "市场类", "positionNumber": "", "location": "北京", "publishTime": "2019-02-14"},
{"positionName": "HGJ-Legal Counsel (Technology Transactions)", "positionType": "职能类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "22989-云服务平台部--服务运营中心总监", "positionType": "技术类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "22989-经营平台产品中心web前端开发", "positionType": "技术类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "22989-视频云高级Web前端开发", "positionType": "技术类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "27554-腾讯音乐商业制片人（上海）", "positionType": "市场类", "positionNumber": "", "location": "上海", "publishTime": "2019-02-14"},
{"positionName": "30361-天天P图图像处理后台开发（上海）", "positionType": "技术类", "positionNumber": "", "location": "上海", "publishTime": "2019-02-14"},
{"positionName": "15573-MMORPG UE4手游资深美术3D设计（上海）", "positionType": "设计类", "positionNumber": "", "location": "上海", "publishTime": "2019-02-14"},
{"positionName": "15573-MMORPG UE4手游3D动画设计师（上海）", "positionType": "设计类", "positionNumber": "", "location": "上海", "publishTime": "2019-02-14"},
{"positionName": "15573-MMORPG UE4手游3D特效美术师（上海）", "positionType": "设计类", "positionNumber": "", "location": "上海", "publishTime": "2019-02-14"},
{"positionName": "15573-MMORPG UE4手游交互设计师(上海)", "positionType": "设计类", "positionNumber": "", "location": "上海", "publishTime": "2019-02-14"},
{"positionName": "AQ-产品安全经理（深圳）", "positionType": "职能类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "AQ-产品安全经理（广州）", "positionType": "职能类", "positionNumber": "", "location": "广州", "publishTime": "2019-02-14"},
{"positionName": "29050-数据安全经理/专家（深圳）", "positionType": "职能类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "29050-数据安全经理/专家（北京）", "positionType": "职能类", "positionNumber": "", "location": "北京", "publishTime": "2019-02-14"},
{"positionName": "AQ-行业合作经理（北京）", "positionType": "职能类", "positionNumber": "", "location": "北京", "publishTime": "2019-02-14"},
{"positionName": "AQ-行业合作经理（深圳）", "positionType": "职能类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "27553-腾讯音乐人曲库运营", "positionType": "内容编辑类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "22086-体育号创作平台产品经理", "positionType": "产品/项目类", "positionNumber": "", "location": "北京", "publishTime": "2019-02-14"},
{"positionName": "22086-体育号CP管理产品经理 ", "positionType": "产品/项目类", "positionNumber": "", "location": "北京", "publishTime": "2019-02-14"},
{"positionName": "22086-体育号内容质量产品经理", "positionType": "产品/项目类", "positionNumber": "", "location": "北京", "publishTime": "2019-02-14"},
{"positionName": "28297-二次元手游本地化策划（深圳）", "positionType": "产品/项目类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "GY0-腾讯云海外商务拓展", "positionType": "市场类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "PCG10-高级产品经理（工具产品方向）", "positionType": "产品/项目类", "positionNumber": "", "location": "成都", "publishTime": "2019-02-14"},
{"positionName": "18432-策略分析师", "positionType": "产品/项目类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "18432-基金高级分析师", "positionType": "产品/项目类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "21309-在线教育运营专家/增长黑客", "positionType": "产品/项目类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "28481-高级医疗商务拓展经理（北京）", "positionType": "市场类", "positionNumber": "", "location": "北京", "publishTime": "2019-02-14"},
{"positionName": "21882-高级医学编辑（深圳/北京）", "positionType": "内容编辑类", "positionNumber": "", "location": "北京", "publishTime": "2019-02-14"},
{"positionName": "18402-MMO手游-平台渠道运营", "positionType": "产品/项目类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "28170-腾讯游戏直播业务管理经理（深圳）", "positionType": "产品/项目类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "28170-腾讯游戏直播内容品牌经理（深圳）", "positionType": "市场类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "PCG10-浏览器阅读中心后台开发工程师（深圳）", "positionType": "技术类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},
{"positionName": "PCG10-浏览器阅读中心前端开发工程师（深圳）", "positionType": "技术类", "positionNumber": "", "location": "深圳", "publishTime": "2019-02-14"},

四.详情