Python爬取ithome的一所有新闻标题评论数及其他一些信息并存入Excel中。

 # coding=utf-8
 import numpy as np
 import pandas as pd
 import sys

 from selenium import webdriver
 import time
 import requests
 import re
 from openpyxl.workbook import Workbook
 import matplotlib.pyplot as plt
 import matplotlib

 urls = []
 urls_new = []
 titles = []
 titles_new = []
 days = []
 comments = []
 authors = []
 sources = []
 comment = []
 ty = []
 def save_to_file(file_name, contents):
     fh = open(file_name, 'w')
     fh.write(contents)
     fh.close()

 url="https://www.ithome.com/"
 # headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36'
 #                          '(KHTML,like Gecko) Chrome/50.0.2661.102 Safari/537.36 QIHU 360EE'}
 headers={'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0"}
 rep = requests.get(url,headers=headers)
 rep.encoding="utf-8"
 strw=rep.text
 save_to_file('ithome.html', strw)
 p = re.compile(r'<div class="lst lst-1 new-list">(.*?)</div>\s*?</div>')
 m = p.findall(strw)
 print(len(m[0]))
 p = re.compile(r'<div class=\"block \d{4} new-list-\d{1}\"(?: style=\".*?\")?><ul>(.*?)</ul></div>')
 m2 = p.findall(m[0])
 print(len(m2))

 broswer = webdriver.Chrome('D:\谷歌\Google\Chrome\Application\chromedriver.exe')

 for i in m2:
     m2 = re.findall(r'</span><span class=\"title\">.*?href=\"(.*?)\">(?:<.*?>)?(.*?)(?:</font>)?</a></span></li>', i)
     for j in m2:
         urls.append(j[0])
         titles.append(j[1])
 print(len(urls))
 for i in range(len(urls)):
     print(u'读取中' + urls[i])
     broswer.get(urls[i])
     time.sleep(1)
     strw2 = broswer.page_source
     # print(strw2)
     p2 = re.compile(r'https://\w+?.ithome.com/(?:html/)?(.*?)/.*?')
     m2 = p2.findall(urls[i])
     print(m2)
     p = re.compile(u'<span id="pubtime_baidu">(\d*-\d*-\d*).*?</span><span id="source_baidu">'
                    u'来源：<a href=".*?" .*?>(.*?)</a></span><span id="author_baidu">'
                    u'作者：(?:<strong>)?(.*?)(?:</strong>)?</span>.*?<span id="commentcount">(.*?)</span>')
     m = p.findall(strw2)
     print(m)
     if len(m) > 0:
         days.append(m[0][0])
         sources.append(m[0][1])
         authors.append(m[0][2])
         urls_new.append(urls[i])
         comments.append(m[0][3])
         titles_new.append(titles[i])
         ty.append(m2[0])
 print("读取结束")
 data={'日期':days,'作者':authors,'来源':sources,'标题':titles_new,'链接':urls_new,'评论数量':comments,'新闻类型':ty}
 df = pd.DataFrame(data, columns=['日期', '作者', '来源','标题','链接','评论数量','新闻类型'])
 # print(df)
 df.to_excel(r'ShuJuPa.xlsx',sheet_name='数据爬取结果',encoding='gb2312')