python 爬虫实例（三）

问题描述

爬取博客园的首页数据URL【https://home.cnblogs.com/blog/page/1/】，之后写到自己的Excel里面

环境：

OS：Window10

python：3.7

代码

import requests
import os
from bs4 import BeautifulSoup
import xlwt
import xlrd
from xlutils.copy import copy
import threading
import datetime

class BlogHome:

    def __init__(self):
        self.url = "https://home.cnblogs.com/blog/page/{}/"
        self.path = r"C:\pythonProject\Blog"

    def request(self, param):
        url= self.url.format(param)
        r = requests.get(self.url)
        return r.text

    def all_page(self, maxpage):

        # wbk = xlwt.Workbook()
        # sheet = wbk.add_sheet("Data")
        wbk = xlrd.open_workbook(r"C:\Users\peiqiang\Desktop\aaa.xls", formatting_info=True)
        wbCopy = copy(wbk)
        sheet = wbCopy.get_sheet(0)
        row = 4
        for page in range(1, maxpage):
            thread_lock.acquire()
            req = self.request(page)
            reRow = self.getdata(req, sheet, row)
            row = reRow
            thread_lock.release()

        wbCopy.save(r"C:\Users\peiqiang\Desktop\aaa.xls")
        print("書き込みました")

    def getdata(self, req, sheet, row):
        soup = BeautifulSoup(req, "xml")
        all_title = soup.find_all(class_="post_block")
        for title in all_title:
            col = 1
            # title取得
            title_blank = title.find(class_="entry_title").find_all("a")
            print("user：", title_blank[0].string.replace("[", "").replace("]", ""))
            sheet.write(row, col, title_blank[0].string.replace("[", "").replace("]", ""))
            col += 1
            print("title：", title_blank[1].string)
            sheet.write(row, col, title_blank[1].string)
            col += 1

            # 評論個数
            post_comment = title.find(class_="post_comment")
            print("評論個数:", post_comment.string)
            sheet.write(row, col, post_comment.string)
            col += 1
            # 読込個数
            post_view = title.find(class_="post_view")
            print("読込個数:", post_view.string)

            sheet.write(row, col, post_view.string)
            col += 1

            # 推奨個数
            # susume = title.find(class_="entry_footer")
            # print("推奨個数:", susume.string)
            # 発表日付
            postdate = title.find(class_="postdate")
            print("発表日付:", postdate.string)
            sheet.write(row, col, postdate.string)
            col += 1
            # 詳細取得
            entry_summary = title.find(class_="entry_summary")
            print("詳細取得:", entry_summary.string)
            sheet.write(row, col, entry_summary.string)
            col += 1
            row += 1
        return row

    def writeExcel(self, row, col, data):
        wbk = xlwt.Workbook()
        sheet = wbk.add_sheet("Data", cell_overwrite_ok=True)
        sheet.write(row, col, data)
        wbk.save(r"C:\Users\peiqiang\Desktop\aaa.xls")
        print("書き込みました")

    def mkdir(self):
        path = self.path.strip()
        isExist = os.path.exists(path)
        if not isExist:
            print('创建名字叫做', path, '的文件夹')
            os.makedirs(path)
            print('创建成功！')
            return True
        else:
            print(path, '文件夹已经存在了，不再创建')
            return False

    def getBlog(self):

        startTime = datetime.datetime.now()
        print("開始", startTime)
        self.all_page(10)
        endTime = datetime.datetime.now()
        print("実行時間：", (endTime - startTime).seconds)
        print("開始", startTime)
        print("終了", endTime)

thread_lock = threading.BoundedSemaphore(value=10)
blogHome = BlogHome()
blogHome.getBlog()

　　执行上面的代码

Excel上面的数据