假期学习【十一】Python爬取百度词条写入csv格式 python 2020.2.10

今天主要完成了根据爬取的txt文档，从百度分类从信息科学类爬取百度词条信息，并写入CSV格式文件。

txt格式文件如图： 为自己爬取内容分词后的结果。

代码如下：

 import requests
 from bs4 import BeautifulSoup
 import csv
 import io
 import re

 url="https://baike.baidu.com/item/"
 id=1
 patton=re.compile(r'.*信息科学分类.*|.*软件.*|.*科技产品.*|.*公司.*|.*互联网人物.*|.*互联网.*|.*科技术语.*|.*技术.*|.*网站.*')

 #写入表头
 def Head():
     with open('E:/bdbk2.csv', 'w', encoding='utf-8', newline='') as csvfile:
         writer = csv.writer(csvfile)
         writer.writerow(["序号", "名称", "属性", "内容", "网址"])

 def read():
     f = open('E:/bdbk2.csv', 'a+', encoding='utf-8', newline='')
     csv_writer = csv.writer(f)
     global url
     f=open("E:/word5.txt",'r+',encoding="utf-8")
     for line in f:
         url=url+line.rstrip("\n")
         try:
             global id, name, nature, content, tag
             kv = {'user-agent': 'Mozilla/5.0'}
             r = requests.get(url, headers=kv)
             r.encoding = "utf-8"
             demo = r.text
             soup = BeautifulSoup(demo, "html.parser")
             print(url)
             # print(soup.prettify())
             tag = soup.find_all("dd", {"id": "open-tag-item"})[0].get_text().replace("（", "").replace("）",
                                                                                                       "").strip().replace(
                 "\n", "")
             name = soup.find_all("h1")[0].get_text().strip()
             nature = soup.find_all("h2")[0].get_text().replace("（", "").replace("）", "").strip()
             if nature == '目录':
                 nature = tag
             content = soup.find_all("div", {"class": "lemma-summary"})[0].get_text().strip().rstrip("]").lstrip("[")
             if name != "百度百科错误页" and nature != "目录" and len(patton.findall(tag)) != 0:
                 print("序号:" + str(id))
                 print("名称:" + name)
                 print("属性:" + nature)
                 print("内容:" + content)
                 print("网址:" + url)
                 csv_writer.writerow([str(id), name, nature, content, url])
                 id += 1
         except:
             print("出错!")
         url = "https://baike.baidu.com/item/"
     f.close()

 if __name__=="__main__":
     Head()
     read()