假期学习【八】首都之窗百姓信件爬虫（完整版）2020.2.6 Python

时间：2020.2.6

今天把昨天做到一半的首都之窗百姓信件爬取完成了。

源码如下：

 import requests
 import io
 from bs4 import BeautifulSoup
 #信1705-1 赵路仓
 kv = {'user-agent': 'Mozilla/5.0'}
 id='AH20010700179'

 def read():
     f=open('E://list.txt','r')
     for line in f:
         id=f.readline().rstrip('\n')
         print(id)
         url1 = "http://www.beijing.gov.cn/hudong/hdjl/com.web.consult.consultDetail.flow?originalId=" + id  # 咨询
         url2 = "http://www.beijing.gov.cn/hudong/hdjl/com.web.suggest.suggesDetail.flow?originalId=" + id  # 建议
         url3 = "http://www.beijing.gov.cn/hudong/hdjl/com.web.complain.complainDetail.flow?originalId=" + id  # 投诉
         parser(url1)
         parser2(url2)
         parser3(url3)
     f.close()

 def write(contents):
     f=open('E://result.txt','a+')
     f.write(contents)
     print(contents,'写入成功！')
     f.close()

 def parser(url):
     try:
         r = requests.get(url, headers=kv)
         print(r.status_code)
         demo = r.text
         soup = BeautifulSoup(demo, "html.parser")
         #print(soup.prettify())
         '''print("标题：", soup.find("strong").get_text().lstrip().rstrip())
         print("来信人：",soup.find_all("div", {"class": "col-xs-10 col-lg-3 col-sm-3 col-md-4 text-muted"})[0].get_text().lstrip('来信人：').lstrip().rstrip())
         print("时间：",soup.find_all("div", {"class": "col-xs-5 col-lg-3 col-sm-3 col-md-3 text-muted"})[0].get_text().lstrip('时间：'))
         print("网友同问：", soup.find_all("div", {"class": "col-xs-4 col-lg-3 col-sm-3 col-md-3 text-muted"})[0].get_text().lstrip().rstrip().lstrip("网友同问:").lstrip().rstrip())
         print("问题：", soup.find_all("div", {"class": "col-xs-12 col-md-12 column p-2 text-muted mx-2"})[0].get_text().lstrip().rstrip())
         print("官方：", soup.find_all("div", {"class": "col-xs-9 col-sm-7 col-md-5 o-font4 my-2"})[0].get_text())
         print("回答时间：",soup.find_all("div", {"class": "col-xs-12 col-sm-3 col-md-3 my-2"})[0].get_text().lstrip('答复时间：'))
         print("回答：", soup.find_all("div", {"class": "col-xs-12 col-md-12 column p-4 text-muted my-3"})[0].get_text().lstrip().rstrip())'''
         if soup.find_all("div", {"class": "col-xs-4 col-lg-3 col-sm-3 col-md-3 text-muted"})[0].get_text().find(
                 '网友同问') != -1:
             write("咨询"+"||")
         write(soup.find("strong").get_text().lstrip().rstrip()+"||")
         write(soup.find_all("div", {"class": "col-xs-10 col-lg-3 col-sm-3 col-md-4 text-muted"})[0].get_text().lstrip('来信人：').lstrip().rstrip()+"||")
         write(soup.find_all("div", {"class": "col-xs-5 col-lg-3 col-sm-3 col-md-3 text-muted"})[0].get_text().lstrip('时间：')+"||")
         write(soup.find_all("div", {"class": "col-xs-4 col-lg-3 col-sm-3 col-md-3 text-muted"})[0].get_text().lstrip().rstrip().lstrip("网友同问:").lstrip().rstrip()+"||")
         write(soup.find_all("div", {"class": "col-xs-12 col-md-12 column p-2 text-muted mx-2"})[0].get_text().lstrip().rstrip()+"||")
         write(soup.find_all("div", {"class": "col-xs-9 col-sm-7 col-md-5 o-font4 my-2"})[0].get_text().lstrip().rstrip()+"||")
         write(soup.find_all("div", {"class": "col-xs-12 col-sm-3 col-md-3 my-2"})[0].get_text().lstrip('答复时间：')+"||")
         write(soup.find_all("div", {"class": "col-xs-12 col-md-12 column p-4 text-muted my-3"})[0].get_text().lstrip().rstrip().replace(" ",""))
         write(soup.find_all("a", {"class": "dex_yes font12"})[0].get_text().lstrip().rstrip().replace(" ", "") + "||")
         write(soup.find_all("a", {"class": "dex_no font12"})[0].get_text().lstrip().rstrip().replace(" ", "") + "||")
         write('\r\n')
     except:
         print("咨询爬取失败！")

 def parser2(url):
     try:
         r = requests.get(url, headers=kv)
         print(r.status_code)
         demo = r.text
         soup = BeautifulSoup(demo, "html.parser")
         #print(soup.prettify())
         '''print("标题：", soup.find("strong").get_text().lstrip().rstrip())
         print("来信人：",soup.find_all("div", {"class": "col-xs-10 col-lg-3 col-sm-3 col-md-4 text-muted"})[0].get_text().lstrip('来信人：').lstrip().rstrip())
         print("时间：",soup.find_all("div", {"class": "col-xs-5 col-lg-3 col-sm-3 col-md-3 text-muted"})[0].get_text().lstrip('时间：'))
         print("网友同问：", soup.find_all("div", {"class": "col-xs-4 col-lg-3 col-sm-3 col-md-3 text-muted"})[0].get_text().lstrip().rstrip().lstrip("网友同问:").lstrip().rstrip())
         print("问题：", soup.find_all("div", {"class": "col-xs-12 col-md-12 column p-2 text-muted mx-2"})[0].get_text().lstrip().rstrip())
         print("官方：", soup.find_all("div", {"class": "col-xs-9 col-sm-7 col-md-5 o-font4 my-2"})[0].get_text())
         print("回答时间：",soup.find_all("div", {"class": "col-xs-12 col-sm-3 col-md-3 my-2"})[0].get_text().lstrip('答复时间：'))
         print("回答：", soup.find_all("div", {"class": "col-xs-12 col-md-12 column p-4 text-muted my-3"})[0].get_text().lstrip().rstrip())'''
         if soup.find_all("div", {"class": "col-xs-4 col-lg-3 col-sm-3 col-md-3 text-muted"})[0].get_text().find('网友支持')!=-1:
             write("建议"+"||")
         write(soup.find("strong").get_text().lstrip().rstrip()+"||")
         write(soup.find_all("div", {"class": "col-xs-10 col-lg-3 col-sm-3 col-md-4 text-muted"})[0].get_text().lstrip('来信人：').lstrip().rstrip()+"||")
         write(soup.find_all("div", {"class": "col-xs-5 col-lg-3 col-sm-3 col-md-3 text-muted"})[0].get_text().lstrip('时间：')+"||")
         write(soup.find_all("div", {"class": "col-xs-4 col-lg-3 col-sm-3 col-md-3 text-muted"})[0].get_text().lstrip().rstrip().lstrip("网友支持:").lstrip().rstrip()+"||")
         write(soup.find_all("div", {"class": "col-xs-12 col-md-12 column p-2 text-muted mx-2"})[0].get_text().lstrip().rstrip()+"||")
         write(soup.find_all("div", {"class": "col-xs-9 col-sm-7 col-md-5 o-font4 my-2"})[0].get_text().lstrip().rstrip()+"||")
         write(soup.find_all("div", {"class": "col-xs-12 col-sm-3 col-md-3 my-2"})[0].get_text().lstrip('答复时间：')+"||")
         write(soup.find_all("div", {"class": "col-xs-12 col-md-12 column p-4 text-muted my-3"})[0].get_text().lstrip().rstrip().replace(" ",""))
         write(soup.find_all("a", {"class": "dex_yes font12"})[0].get_text().lstrip().rstrip().replace(" ", "") + "||")
         write(soup.find_all("a", {"class": "dex_no font12"})[0].get_text().lstrip().rstrip().replace(" ", "") + "||")
         write('\r\n')
     except:
         print("建议爬取失败！")

 def parser3(url):
     try:
         r = requests.get(url, headers=kv)
         print(r.status_code)
         demo = r.text
         soup = BeautifulSoup(demo, "html.parser")
         #print(soup.prettify())
         if soup.find_all("div", {"class": "col-xs-4 col-lg-3 col-sm-3 col-md-3 text-muted"})[0].get_text().find('网友评价')!=-1:
             write("投诉"+"||")
         write(soup.find("strong").get_text().lstrip().rstrip()+"||")
         write(soup.find_all("div", {"class": "col-xs-10 col-lg-3 col-sm-3 col-md-4 text-muted"})[0].get_text().lstrip('来信人：').lstrip().rstrip()+"||")
         write(soup.find_all("div", {"class": "col-xs-5 col-lg-3 col-sm-3 col-md-3 text-muted"})[0].get_text().lstrip('时间：')+"||")
         write(soup.find_all("div", {"class": "col-xs-4 col-lg-3 col-sm-3 col-md-3 text-muted"})[0].get_text().lstrip().rstrip().lstrip("网友评价数:").lstrip().rstrip()+"||")
         write(soup.find_all("div", {"class": "col-xs-12 col-md-12 column p-2 text-muted mx-2"})[0].get_text().lstrip().rstrip()+"||")
         write(soup.find_all("div", {"class": "col-xs-9 col-sm-7 col-md-5 o-font4 my-2"})[0].get_text().lstrip().rstrip()+"||")
         write(soup.find_all("div", {"class": "col-xs-12 col-sm-3 col-md-3 my-2"})[0].get_text().lstrip('答复时间：')+"||")
         write(soup.find_all("div", {"class": "col-xs-12 col-md-12 column p-4 text-muted my-3"})[0].get_text().lstrip().rstrip().replace(" ","")+"||")
         write(soup.find_all("a", {"class": "dex_yes font12"})[0].get_text().lstrip().rstrip().replace(" ", "") + "||")
         write(soup.find_all("a", {"class": "dex_no font12"})[0].get_text().lstrip().rstrip().replace(" ", "") + "||")
         write('\r\n')
     except:
         print("投诉爬取失败！")

 if __name__=="__main__":
     read()

遇到的问题：

　　忽视了read()和readline()读取后面的\n导致不停报错，在开始时没有看到投诉类型的信件和建议类型的信件，后来补上。

爬取的文本内容用||隔开，分别代表信件类型，标题，来信人，问题时间，网友评价，问题内容，回答方，回答时间，回答内容，赞，踩11个属性，存为E：//result.txt文件内。

附上页面的ID（txt格式）百度网盘：https://pan.baidu.com/s/1GvF8Kllvv-vqBblgWnA-LQ

BeautifulSoup的安装和使用可以参考我的博客：https://www.cnblogs.com/zlc364624/p/12264070.html