爬取笔下wenxue小说

 import urllib.request
 from bs4 import BeautifulSoup
 import  re

 def gethtml(url):
     page=urllib.request.urlopen(url)
     html=page.read().decode('gbk')
     soup=BeautifulSoup(html,"html.parser")
     #print(soup)
     return soup

 def getcontent(soup,load):                            #获取章节内容以及章节名称
     content1=""
     content=re.findall(r'<div id="content"><div id="adright"></div>(.*?)</div>',str(soup))
     for i in range(0,len(content)):
         content1+=content[i]
     content2 = re.sub("</?\w+[^>]*>", "", content1)
     content3=content2.replace('。','。\n\n\0\0\0\0\0\0')
     #以上获取章节内容
     zjname = re.findall(r'<div id="title">(.*?)</div>', str(soup))
     #获取章节名称

     with open(load, 'a', encoding='utf-8') as f:
         f.write("\0\0\0\0\0\0-----------------------------------------------------------"+zjname[0]+"------------------------------------------------------\n\n"+content3)

 def book(soup):
     bookurl=re.findall(r'<td class="odd"><a href="(.*?)">',str(soup))      #get every book url

     for i in range(0,len(bookurl)):
         #找到“[点击阅读]按钮”的链接,以及书名
         print(bookurl[i])
         soup1=gethtml(bookurl[i])
         allcontent=re.findall(r'</a>\xa0\xa0\xa0\xa0<a href="(.*?)">',str(soup1))
         bookname=re.findall(r'<strong>(.*?)全集下载</strong>', str(soup1))

         soup2=gethtml(allcontent[0])
         #打开点击阅读的按钮链接，找到第一章的链接
         firsturl1 = re.findall(r'<dd><a href="(.*?)">.*?</a></dd>', str(soup2))
         headurl=bookurl[i][0:-4].replace("binfo","b")
         firsturl2=headurl+"/"+firsturl1[0]
         print(firsturl2)

         #打开链接，开始爬取内容,同时获取下一章内容，并判断是否到最后一章
         soup3=gethtml(firsturl2)
         k=0
         load="d:/77/%s.txt" % bookname[0]
         try:

             while True:
                 nexturl = re.findall(r'<li><a href="(.*?)">下一页', str(soup3))
                 getcontent(soup3,load)
                 soup3=gethtml(headurl+"/"+nexturl[0])
                 k+=1
                 print("第%d章下载完成" % int(k))
         except:

             print("-------------------第%d本书下载完成---------------" % int(i+1))

 url="http://www.bxwx9.org/modules/article/toplist.php?sort=dayvisit"
 soup=gethtml(url)
 load=book(soup)

保存到txt文件中

2017-05-28

22:58:35