requests+xpath+map爬取百度贴吧

 # requests+xpath+map爬取百度贴吧
 # 目标内容:跟帖用户名,跟帖内容,跟帖时间
 # 分解:
 # requests获取网页
 # xpath提取内容
 # map实现多线程爬虫
 import requests
 from requests.exceptions import RequestException
 from lxml import etree
 import json
 from multiprocessing.dummy import Pool as ThreadPool

 def get_html(url):
     try:
         response = requests.get(url)
         if response.status_code == 200:
             return response.text
         else:
             return None
     except RequestException:
         return None

 def parse_html(html):
     selector = etree.HTML(html)
     data = selector.xpath('//div[@class="l_post j_l_post l_post_bright  "]')
     for each in data:
         rs = each.xpath('@data-field')[0]
         rs = json.loads(rs)
         author = rs.get('author').get('user_name')
         author_id = rs.get('content').get('post_id')
         content = each.xpath('div/div/cc/div[@id="post_content_%s"]/text()'% author_id)[0].strip()
         date = rs.get('content').get('date')
         yield {
             'author':author,
             'content':content,
             'date':date
         }

 def save_to_txt(result):
     print('正在存储:',result)

     with open('tieba.txt','a',encoding='utf-8') as f:
         f.write('回帖作者:'+result['author']+'\n')
         f.write('回帖内容:'+result['content']+'\n')
         f.write('回帖时间:'+result['date']+'\n')
         f.write('\n')

 def main(url):
         html = get_html(url)
         if html:
             for result in parse_html(html):
                 save_to_txt(result)

 if __name__=='__main__':

     pool = ThreadPool(4)
     urls=[]
     base_url = 'http://tieba.baidu.com/p/3522395718?pn='
     for page_num in range(1, 21):
         url = base_url + str(page_num)
         urls.append(url)

     pool.map(main,urls)
     pool.close()
     pool.join()