[Python] 糗事百科文本数据的抓取

[Python] 糗事百科文本数据的抓取

源码

https://github.com/YouXianMing/QiuShiBaiKeText

import sqlite3
import time
import requests
from regexp_string import *

class QiuShiBaiKeText35:

    db_name = 'qiu_shi_bai_ke_text35.db'
    conn = None

    def prepare(self):
        """
        开始准备数据库相关准备工作
        :return: PiuShiBaiKeText35对象本身
        """

        # 连接数据库,不存在则创建
        self.conn = sqlite3.connect(self.db_name)

        # 不存在表则创建表
        sql_str = """CREATE TABLE IF NOT EXISTS qiu_shi_bai_ke_text (articleId INT8 PRIMARY KEY NOT NULL,
        content TEXT, date TIMESTAMP); """
        self.conn.execute(sql_str)

        # 关闭数据库
        self.conn.close()
        self.conn = None

        return self

    def start(self, max_page=99999):
        """
        开始爬数据
        :param max_page: 最大页码,不设置则为99999
        :return: PiuShiBaiKeText35对象本身
        """

        self.conn = sqlite3.connect(self.db_name)
        self.__qiu_shi_text(max_page)
        self.conn.close()
        self.conn = None

        return self

    def __qiu_shi_text(self, max_page=99999):
        """
        开始扫描
        :param max_page: 最大页码,不设置则为99999
        :return: None
        """

        for i in range(1, max_page):

            url = "http://www.qiushibaike.com/text/page/%s/" % i
            print(url)
            time.sleep(0.5)

            request = requests.get(url)

            if i != 1:
                request = requests.get(url)
                if request.url != url:
                    break

            self.__convert_from_web_string(request.text)

    def __convert_from_web_string(self, web_string):
        """
        获取网页字符串,并用正则表达式进行解析
        :param web_string: 网页字符串
        :return: None
        """

        # 获取列表
        pattern = r"""\d+" target="_blank" class='contentHerf' >.+?</span>"""
        item_list = RegExpString(web_string).get_item_list_with_pattern(pattern)

        # 如果存在列表,则遍历列表
        if item_list:

            for item in item_list:

                # 内容id
                article_id = RegExpString(item).search_with_pattern(r'^\d+').search_result

                # 内容
                article_content = RegExpString(item).search_with_pattern(
                    r'(?<=<span>).+(?=</span>)').search_result
                article_content = RegExpString(article_content).replace_with_pattern(r'<br/>',
                                                                                     "\n").replace_result

                # 打印内容
                print("http://www.qiushibaike.com/article/%s\n%s\n\n" % (article_id, article_content))

                # 先查找有没有这个id的数据
                cursor = self.conn.execute("""SELECT COUNT(*) FROM qiu_shi_bai_ke_text WHERE articleId = '%s';""" % article_id)

                for row in cursor:

                    # 如果查不到数据,则插入数据
                    if row[0] == 0:
                        # 插入语句
                        sql_str = """INSERT INTO qiu_shi_bai_ke_text (articleId, content, date) VALUES ('%s', '%s', '%s');""" % (
                            article_id, article_content, time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()))
                        self.conn.execute(sql_str)

                self.conn.commit()

细节

1. 抓取 http://www.qiushibaike.com/text/ 所有35个页面的文本数据

2. 抓取的数据写进数据库,数据库用的是sqlite3

3. 基于Python3.60版本,其他版本未测试

4. 网络库使用过的 requests (https://github.com/kennethreitz/requests) ,如果没有安装,请使用 pip install requests 安装

效果