python实现网络爬虫下载天涯论坛帖子

最近发现天涯论坛是一个挺有意思的网站，有各种乱七八糟的帖子足以填补无聊时候的空虚感，但是相当不爽的一件事就是天涯的分页模式下想连贯的把楼主的内容看完实在是太心酸了，一个999页的帖子，百分之九十都是无聊网友的灌水，有时候连续翻几十页才能找到楼主的一条内容。所以无聊之下，就打算写一个简单的爬虫，能一次性把某一个帖子下楼主的所有内容一次性的下载下来。好吧，说了这么多废话，现在开始讲点正事。

网页的地址形式：http://bbs.tianya.cn/post-no05-355576-1.shtml，其中.shtml前的1表示这是当前帖子的第一页，我们可以根据第一页的内容解析出最大的页数，然后遍历的去解析每一个页面，获得楼主的全部言论。

网页的源码简单如下图，每一块内容都放在atl-content这个div中，可以根据下面的一个注释来判断是不是楼主的发言，而正文内容放在bbs-content这个div中，如果有图片的话，里面会有图片的链接，实现的过程中我就是根据这两点找到楼主的言论，并把内容提取出来。

为了爬取的高效性，实现的过程中我利用了python的threading模块，下面是threads.py模块，定义了下载解析页面的线程，下载图片的线程以及线程池

import threading
import urllib2
import Queue
import re

thread_lock = threading.RLock()

#下载页面的一个函数，header中没有任何内容也可以顺利的下载，就省去了
def download_page(html_url):
    try:
        req = urllib2.Request(html_url)
        response = urllib2.urlopen(req)
        page = response.read()
        return page
    except Exception:
        print 'download %s failed' % html_url
        return None

#下载图片的一个方法，和上面的函数很像，只不过添加了一个文件头
#因为在测试的过程中发现天涯对于没有如下文件头的图片链接是不会返回正确的图片的
def download_image(image_url, referer):
    try:
        req = urllib2.Request(image_url)
        req.add_header('Host', 'img3.laibafile.cn')
        req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:33.0) Gecko/20100101 Firefox/33.0')
        req.add_header('Accept', 'image/png,image/*;q=0.8,*/*;q=0.5')
        req.add_header('Accept-Language', 'zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3')
        req.add_header('Referer', referer)
        req.add_header('Origin', 'http://bbs.tianya.cn')
        req.add_header('Connection', 'keep-alive')
        response = urllib2.urlopen(req)
        image = response.read()
        return image
    except Exception:
        print 'download %s failed' % image_url
        return None

#下载和解析一个页面的线程类
class download_html_page(threading.Thread):
    #name:线程的名字
    #page_range:用户输入的页面范围
    #page_contents:解析之后楼主的内容
    #img_urls:解析之后楼主贴的图的链接
    #html_url:输入的页面url
    #first_page：第一次已经下载好的页面，主要是考虑效率，不重复下载
    def __init__(self, name, page_range, page_contents, img_urls, html_url, first_page):
        threading.Thread.__init__(self)
        self.name = name
        self.page_range = page_range
        self.page_contents = page_contents
        self.img_urls = img_urls

self.html_url = html_url
        self.first_page = first_page
    
    #判断是不是楼主的内容
    def is_louzhu(self, s):
        result = re.search(r'<!-- <div class="host-ico">(.*?)</div> -->', s, re.S)
        return (result is not None)

#获得页面里属于楼主图片的url
    def get_img_url(self, s, page_url):
        #判断是不是楼主给其他用户的评论，如果是的话，直接过滤掉（本人从不看评论）
        is_louzhu_answer = re.search(r'-{15,}<br>', s, re.S)
        if is_louzhu_answer is None:
            imgurl = re.findall(r'<img.*?original="(?P<imgurl>.*?)".*?/><br>', s, flags = re.S)

url_path = []
            for one_url in imgurl:
                self.img_urls.put(one_url + '|' + page_url)
                path = re.search('\w+\.jpg', one_url).group(0)
                url_path.append('img/' + path)

segments = re.split(r'<img .*?/><br>', s.strip())
            content = segments[0].strip()
            for i in range(len(url_path)):
                content += '\n<img src = "' + url_path[i] + '" />\n<br>'
                content += segments[i+1].strip()
            return content

#解析夜歌页面
    def parse_page(self, html_page, page_url):
        html_page.decode('utf-8')
        Items = re.findall(r'<div class="atl-content">(?P<islouzhu>.+?)<div class="bbs-content.*?">(?P<content>.+?)</div>', html_page, re.S)
        page_content = ''

for item in Items:
            if self.is_louzhu(item[0]):
                one_div = self.get_img_url(item[1], page_url)
                if one_div is not None:
                    page_content += one_div
        return page_content

def run(self):
        while self.page_range.qsize() > 0:
            page_number = self.page_range.get()
            page_url = re.sub('-(\d+?)\.shtml', '-' + str(page_number) + '.shtml', self.html_url)

page_content = ''
            print 'thread %s is downloading %s' % (self.name, page_url)
            if page_url == self.html_url:
                page_content = self.parse_page(self.first_page, page_url)
            else:
                page = download_page(page_url)
                if page is not None:
                    page_content = self.parse_page(page, page_url)
            #thread_lock.acquire()
            #self.page_contents[page_number] = page_content
            #thread_lock.release()
            self.page_contents.put(page_content, page_number)
        self.img_urls.put('finished')

#下载图片的线程
class fetch_img(threading.Thread):
    def __init__(self, name, img_urls, download_img):
        threading.Thread.__init__(self)
        self.name = name
        self.img_urls = img_urls
        self.download_img = download_img

def run(self):
        while True:
            message = self.img_urls.get().split('|')
            img_url = message[0]
            if img_url == 'finished':
                self.img_urls.put('finished')
                break
            else:
                thread_lock.acquire()
                if img_url in self.download_img:
                    thread_lock.release()
                    continue
                else:
                    thread_lock.release()
                    print 'fetching image %s' % img_url
                    referer = message[1]
                    image = download_image(img_url, referer)
                
                    image_name = re.search('\w+\.jpg', img_url).group(0)
                    with open(r'img\%s' % image_name, 'wb') as img:
                        img.write(image)
                    thread_lock.acquire()
                    self.download_img.add(img_url)
                    thread_lock.release()

#定义了一个线程池
class thread_pool:
    def __init__(self, page_range, page_contents, html_url, first_page):
        self.page_range = page_range
        self.page_contents = page_contents
        self.img_urls = Queue.Queue()
        self.html_url = html_url
        self.first_page = first_page
        self.download_img = set()
        
        self.page_thread_pool = []
        self.image_thread_pool = []
        
    def build_thread(self, page, image):
        for i in range(page):
            t = download_html_page('page thread%d' % i, self.page_range, self.page_contents,
                                    self.img_urls, self.html_url, self.first_page)
            self.page_thread_pool.append(t)
        for i in range(image):
            t = fetch_img('image thread%d' % i, self.img_urls, self.download_img)
            self.image_thread_pool.append(t)
        
    def all_start(self):
        for t in self.page_thread_pool:
            t.start()
        for t in self.image_thread_pool:
            t.start()
    
    def all_join(self):
        for t in self.page_thread_pool:
            t.join()
        for t in self.image_thread_pool:
            t.join()

下面是主线程的代码：

# -*- coding: utf-8 -*-  
import re
import Queue
import threads

if __name__ == '__main__':
    html_url = raw_input('enter the url: ')
    html_page = threads.download_page(html_url)

max_page = 0
    title = ''
    if html_page is not None:
        search_title = re.search(r'<span class="s_title"><span style="\S+?">(?P<title>.+?)</span></span>', html_page, re.S)
        title = search_title.groupdict()['title']

search_page = re.findall(r'<a href="/post-\S+?-\d+?-(?P<page>\d+?)\.shtml">(?P=page)</a>', html_page, re.S)
        for page_number in search_page:
            page_number = int(page_number)
            if page_number > max_page:
                max_page = page_number
                
    print 'title：%s' % title
    print 'max page number: %s' % max_page
    
    start_page = 0
    while start_page < 1 or start_page > max_page:
        start_page = input('input the start page number:')
        
    end_page = 0
    while end_page < start_page or end_page > max_page:
        end_page = input('input the end page number:')
        
    page_range = Queue.Queue()
    for i in range(start_page, end_page + 1):
        page_range.put(i)

page_contents = {}
    thread_pool = threads.thread_pool(page_range, page_contents, html_url, html_page)
    thread_pool.build_thread(1, 1)
    thread_pool.all_start()
    thread_pool.all_join()

运行的时候需要手动在python代码的同级路径下创建img的文件夹，用来存放图片，由于本人比较懒，就没有用python生成这个文件夹，最后下载的结果如下图，如果现实乱码的话，需要把网页的编码格式设置为Unicode