[Python]爬取CSDN论坛 标题 2020.2.8

首先新建一个Lei.txt

内容为：

CloudComputing
ParentBlockchainTechnology
Enterprise
DotNET
Java
WebDevelop
VC
VB
Delphi
BCB
Cpp
OtherLanguage
MSSQL
PowerBuilder
Oracle
Linux
Windows
Embedded
GameDevelop
Network_communication
Other
Network_communication
OtherTechnicalForum
AI

爬虫代码如下：

 import requests
 from bs4 import BeautifulSoup
 import io
 import re

 url="https://bbs.csdn.net/forums/Mobile?page="

 def Content(url):
     try:
         kv = {'user-agent': 'Mozilla/5.0'}
         r = requests.get(url, headers=kv)
         r.encoding = r.apparent_encoding
         demo = r.text
         soup = BeautifulSoup(demo, "html.parser")
         text=""
         for a in soup.find_all("a",class_="forums_title"):
             text+=str(a.string).replace("【CSDN 20周年】8场大牛直播+周年T恤免费领", "").replace("小白学习笔记干货，记得点赞哦！", "").replace("有奖征集话题： 区块链大火，对于区块链开发零基础的我来说，要怎么入门呢？", "")
             text+="\n"
         print(text.lstrip())
         write(text.lstrip())
         #print(soup.prettify())
     except:
         print("没有数据了！")

 #写入内容
 def write(contents):
     f=open('E://luntan.txt','a+',encoding='utf-8')
     f.write(contents)
     print('写入成功！')
     f.close()

 #循环写入
 def write_all():
     try:
         f=open('E://Lei.txt','r+',encoding='utf-8')
         for line in f:
             line=line.rstrip("\n")
             for i in range(1,100):
                 url="https://bbs.csdn.net/forums/"+line+"?page="+str(i)
                 Content(url)
     except:
         print("超出页数！")
 if __name__=="__main__":
     write_all()