python应用：爬虫实例(静态网页)

爬取起点中文网某本小说实例：

 # -*-coding:utf8-*-
 import requests
 import urllib
 import urllib2
 from bs4 import BeautifulSoup
 import sys
 reload(sys)
 sys.setdefaultencoding('utf-8')       #读写中文内容

 class QDZW:
     def __init__(self, url):
         self.url = url           #网址
         self.user_agent = 'Mozilla/5.0 (Windows NT 6.1; (Windows NT 6.1; WOW64; rv:62.0) Gecko/20100101 Firefox/61.0'      #请求头属性（模拟浏览器访问）
         self.headers = {'User-Agent' : self.user_agent}

     #获取网址静态HTML（如果未访问成功，则持续访问，直到访问成功为止）
     def gethtml(self):

             try:
                 #return requests.get(self.url，headers = self.headers,time_out = 2)
                 request = urllib2.Request(self.url,headers = self.headers)
                 response = urllib2.urlopen(request,time_out = 2)    #设置访问时长限制
                 temp = False
                 return response
             except Exception as e:
                 print e
     #解析HTML内容，获取需要的内容
     def novel(self):
         while True:
             r = self.gethtml()
             while r == 'None':         #r内容为空表示访问超时，则继续访问
                 r = self.gethtml()
             #print r.encoding    #输出编码类型
             soup = BeautifulSoup(r ,"html.parser")     #利用BeautifulSoup解析网址内容，class前加 . id前加 #
             #select输出内容为数组，select参数从body中一级结点开始即可，可逐级叠加定位
             #XXX.text为读取标签中的内容  XXX.attrs['xx']为读取 XXX中属性为xx的值 ，XXX为HTML格式的文本
             section_name = soup.select(' .wrap #j_readMainWrap #j_chapterBox .text-wrap .main-text-wrap .text-head .j_chapterName')[0].text
             print section_name
             section_text = soup.select('.wrap #j_readMainWrap #j_chapterBox .text-wrap .main-text-wrap .read-content')[0].text
             section_text = self.spliting(section_text)
             #print section_text
             self.writing(section_name + "\r\n" + section_text)

             next_url = soup.select('.wrap #j_readMainWrap .chapter-control #j_chapterNext')[0]
             #print next_url.text
             if next_url.text == "下一章":
                 self.url = "https:" + next_url.attrs['href']
             else:
                 break
     #在去掉<p></p>的位置添加换行符号“\r\n”
     def spliting(self, string):
         str_split = string.split()  #以split()中参数为标准，切割字符串
         string = ""
         for text in str_split:
             if text != "":
                 string = string + "    " + text + "\r\n"
         return string
     #输出读取的novel内容
     def writing(self, novel):
         outputs = open(unicode('至尊剑皇.txt','utf8'),'a')      #a为追加型写入
         outputs.write(novel)
         outputs.close()

 qdzw = QDZW('https://read.qidian.com/chapter/JDViC81SWM41/pskSqRrpbXDgn4SMoDUcDQ2')     #参数为初始网址
 qdzw.novel()