使用Scrapy框架爬取腾讯新闻

昨晚没事写的爬取腾讯新闻代码，在此贴出，可以参考完善。

# -*- coding: utf-8 -*-
import json

from scrapy import Spider
from scrapy.http import Request
from scrapy.http import Response
from scrapy.http import FormRequest
from scrapy.selector import Selector
from bs4 import BeautifulSoup

from ..items import NewsItem

TencentNewsUrl = 'https://pacaio.match.qq.com/irs/rcd'

# 要闻 https://pacaio.match.qq.com/pc/topNews?callback=__jp0
# https://pacaio.match.qq.com/irs/rcd?cid=108&ext=&token=349ee24cdf9327a050ddad8c166bd3e3&page=1&expIds=&callback=__jp1
# https://new.qq.com/cmsn/20180726/20180726A0QOLA00
# https://new.qq.com/ omn/20180726/20180726A0QOLA.html

class TencentSpider(Spider):
    name = 'tencent'

    def start_requests(self):
        # yield Request(
        #     url='https://pacaio.match.qq.com/pc/topNews?callback=__jp0',
        #     callback=self.parse_contents
        # )

        yield FormRequest(
            url=TencentNewsUrl,
            formdata={
                "cid": "58",
                "token": "c232b098ee7611faeffc46409e836360",
                "ext": "milite",
                "page": "0",
                "expIds": "",
                "callback": "__jp0"
            },
            callback=self.parse_contents,
            meta={
                "page": "0",
                "field": ""
            }
        )

    def parse_contents(self, response: Response):
        try:
            data = json.load(response.text)
        except Exception:
            data = json.loads(response.text[(response.text.find('(') + 1):response.text.rfind(')')])

        # 处理分离网页
        try:
            data = data['data']
        except Exception:
            pass
        for url in data:
            omn = url['vurl']
            if omn.endswith('00') and '/cmsn/' in omn:
                omn = omn.replace('/cmsn/', '/omn/')
                omn = omn[:omn.rfind('00')] + '.html'
                print(omn)
            yield Request(
                url=omn,
                callback=self.parse_news
            )
            break

    def parse_news(self, response: Response):
        news = NewsItem()
        news['url'] = response.url
        soup = BeautifulSoup(response.text, "lxml")
        news['title'] = soup.find('div', class_='LEFT').h1.text
        news['content'] = ''
        article = soup.find_all('p', class_='one-p')
        for sentence in article:
            news['content'] += sentence.text
        return news