Python爬虫之利用BeautifulSoup爬取豆瓣小说（三）——将小说信息写入文件

 #-*-coding:utf-8-*-
 import urllib2
 from bs4 import BeautifulSoup

 class dbxs:

     def __init__(self):
         self.pageIndex = 0
         self.enable = True
         self.file = None
         self.content = []

     #获取html页面的内容
     def getPage(self, pageIndex):
         try:
             #设置代理ip
             enable_proxy = True
             proxy_handler = urllib2.ProxyHandler({'Http': '113.118.170.230:808'})
             null_proxy_handler = urllib2.ProxyHandler({})
             if enable_proxy:
                 opener = urllib2.build_opener(proxy_handler)
             else:
                 opener = urllib2.build_opener(null_proxy_handler)
             urllib2.install_opener(opener)
             #获得页面响应的内容
             url = 'https://www.douban.com/tag/%E5%B0%8F%E8%AF%B4/book' + "?start=" + str(pageIndex)
             #设置请求头部信息，模拟浏览器的行为
             my_headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:55.0)'}
             request =   urllib2.Request(url, headers = my_headers)
             response = urllib2.urlopen(request)
             return response.read()
         except urllib2.URLError, e:
             if hasattr(e, "code"):
                 print e.code
             if hasattr(e, "reason"):
                 print e.reason
                 return None

     #过滤查找这一页的小说名字，信息和评分
     def getContent(self, pageIndex, content):
         pageCode = self.getPage(pageIndex)
         soup = BeautifulSoup(pageCode, 'html.parser')
         #在获得相应的内容中找出所有标签为<dd>的内容（里面包含了我们需要的小说信息）
         contents = soup.find_all('dd')

         if contents:
             for item in contents:
                 title = item.find(class_ = 'title').string.encode('utf-8')
                 info = item.find(class_ = 'desc').string.strip().encode('utf-8')
                 rate = item.find(class_ = 'rating_nums')
                 #通过试验，我们发现某一页可能存在小说没有评分，如果我们不判断rate，那么可能就出现报错
                 if rate:
                     rates = rate.string.encode('utf-8')
                     content.append([title, info, rates])

                 else:
                     content.append([title, info])
         #如果页面不包含<dd>标签，我们应该停止
         else:
             print u"所有页面已加载完"
             self.enable = False

         return content

     #写入文件
     def writeData(self, content):
         self.file = open("bdxs.txt", "w+")   #必须在for循环外面，不然每一次写入都会覆盖之前的数据
         for item in content:
             if len(item) == 3:
                 self.file.write(item[0] + "\n")
                 self.file.write(item[1] + "\n")
                 self.file.write(u"评分:" + item[2] + "\n\n")
             else:
                 self.file.write(item[0] + "\n")
                 self.file.write(item[1] + "\n")
             self.file.write("========================================\n\n")

     #创建一个开始方法
     def start(self):
         x = 1
         while self.enable == True:
             content = self.getContent(self.pageIndex, self.content)
             if self.enable == True:
                 print "正在写入第%s页..." %x
             self.writeData(content)
             self.pageIndex += 15
             x += 1

 DBXS = dbxs()
 DBXS.start()

这段代码我还没理解透彻，比如每一页的小说信息写入完成后，怎么在后面加上第几页，后期我将继续完善它。