Scrapy：腾讯招聘整站数据爬取

项目地址：https://hr.tencent.com/

步骤一、分析网站结构和待爬取内容

以下省略一万字

步骤二、上代码（不能略了）

1、配置items.py

 import scrapy

 class HrTencentItem(scrapy.Item):
     # define the fields for your item here like:
     # name = scrapy.Field()
     # pass
     position_name = scrapy.Field()#职位名称
     position_type = scrapy.Field()#职位类别
     detail_url = scrapy.Field()
     people_count = scrapy.Field()
     work_city = scrapy.Field()
     release_date = scrapy.Field()#发布时间
     job_description = scrapy.Field()#工作描述
     job_require = scrapy.Field()#工作要求

2、配置settings.py

配置mongo

NEWSPIDER_MODULE = 'hr_tencent.spiders'
MONGO_URL ='localhost'
MONGO_DB ='hrtencent'

切记注册ITEM_PIPELINES

ITEM_PIPELINES = { # 'hr_tencent.pipelines.HrTencentPipeline': 300, 'hr_tencent.pipelines.MongoPipeline': 400, }

3.到spider文件夹里面执行指令 scrapy genspider tencent

4、打开自动生成的tencent.py文件，进行编辑

 # -*- coding: utf-8 -*-
 import scrapy
 from hr_tencent.items import HrTencentItem

 class TencentSpider(scrapy.Spider):
     name = 'tencent'
     allowed_domains = ['hr.tencent.com']
     start_urls = ['https://hr.tencent.com/position.php']
     front_url = "https://hr.tencent.com/"
     def parse(self, response):

         tencenthr = response.xpath('//tr[@class="even"] | //tr[@class="odd"]')
         for job in tencenthr:
             item = HrTencentItem()
             item["position_name"] = job.xpath('.//a/text()').extract_first()
             item["detail_url"] = self.front_url + job.xpath('.//a/@href').extract_first()
             item["position_type"] = job.xpath('.//td[2]/text()').extract_first()
             item["people_count"] = job.xpath('.//td[3]/text()').extract_first()
             item["work_city"] = job.xpath('.//td[4]/text()').extract_first()
             item["release_date"] = job.xpath('.//td[5]/text()').extract_first()
             yield scrapy.Request(url=item["detail_url"], callback=self.detail_parse, meta={"item": item})
         next_url = self.front_url + response.xpath('//div[@class="pagenav"]/a[@id="next"]/@href').extract_first()
         yield scrapy.Request(url=next_url, callback=self.parse)

     def detail_parse(self, response):
         item = response.meta["item"]
         node_list = response.xpath('//ul[@class="squareli"]')
         item["job_description"] = ''.join(node_list[0].xpath("./li/text()").extract())
         item["job_require"] = ''.join(node_list[1].xpath("./li/text()").extract())
         yield item

5、配置pipelines.py文件

 import pymongo

 class MongoPipeline(object):
     def __init__(self,mongo_url,mongo_db):
         self.mongo_url = mongo_url
         self.mongo_db = mongo_db
     @classmethod
     def from_crawler(cls,crawler):
         return cls(
             mongo_url = crawler.settings.get('MONGO_URL'),
             mongo_db=crawler.settings.get('MONGO_DB')

         )
     def open_spider(self,spider):
         self.client = pymongo.MongoClient(self.mongo_url)
         self.db = self.client[self.mongo_db]

     def process_item(self,item,spider):
         name = item.__class__.__name__
         self.db[name].insert(dict(item))
         return item

     def close_spider(self,spider):
         self.client.close()

6、新建一个run.py文件，为了不每次运行都敲指令，直接运行run.py即可

 # -*- coding:utf-8 -*-
 from scrapy import cmdline

 cmdline.execute("scrapy crawl tencent".split())

7、运行本地或服务器上的mongo数据库（远程mongo数据库地址需要自己配）

8、执行run文件数据到手