python request爬取百度贴吧

 import requests
 import os
 import shutil
 import time

 class PostBarSpider(object):
     def __init__(self, post_bar, page_number, file_dir):
         # 爬取某个贴吧前多少页内容
         self.post_bar = post_bar
         self.page = page_number
         # 保存到哪个目录
         self.file_dir = file_dir
         self.url = "https://tieba.baidu.com/f?kw={}&ie=utf-8&pn={}"
         self.headers = {
             "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
         }

     # 构造url
     def get_url_list(self):
         # url_list = []
         # for i in range(0, self.page):
         #     result = self.url.format(self.post_bar, i * 50)
         #     print(result)
         #     url_list.append(result)
         # return url_list
         return [self.url.format(self.post_bar, i * 50) for i in range(self.page)]

     # 创建保存文件的目录
     def set_dir(self):
         self.file_dir += "/{}/".format(self.post_bar)
         print("保存路径-----{}".format(self.file_dir))
         try:
             if not os.path.exists(self.file_dir):
                 os.makedirs(self.file_dir)
             else:
                 shutil.rmtree(self.file_dir)
                 os.makedirs(self.file_dir)
         except Exception as info:
             print("创建或删除文件夹出现问题")

     def run(self):
         # 设置保存目录
         self.set_dir()
         # 获得url_list
         url_list = self.get_url_list()

         # 遍历ulr_list
         for i in range(0, len(url_list)):
             response = requests.get(url_list[i], headers=self.headers)
             if response.status_code == 200:
                 # 写出数据
                 file_name = self.file_dir + "{}".format("第{}页.txt".format(i + 1))
                 file = open(file_name, "w", encoding="utf-8")
                 file.write(response.content.decode())
                 print("已写入第{}页".format(i + 1))
                 file.close()

 def main():
     start = time.time()
     my_spider = PostBarSpider("李毅", 20, "f:/post_bar_test")
     my_spider.run()
     end = time.time()
     print("耗时-----{}s".format(end - start))

 if __name__ == '__main__':
     main()