Requests+正则表达式抓取猫眼电影TOP100

spider.py

 # -*- coding:utf-8 -*-
 import requests
 import re
 import json
 import codecs
 from requests.exceptions import RequestException
 from multiprocessing import Pool

 headers = {
     'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0'
 }

 def get_one_page(url):
     try:
         response = requests.get(url,headers=headers)
         if response.status_code == 200:
             return response.text
         return None
     except RequestException:
         return None

 def parse_one_page(html):
     pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?<a.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>', re.S)
     items = re.findall(pattern, html)
     for item in items:
         yield {
             'index': item[0],
             'image': item[1],
             'title': item[2],
             'actor': item[3].strip()[3:],
             'time': item[4].strip()[5:],
             'score': item[5] + item[6]
         }

 def save_to_file(content):
     with codecs.open('result.txt', 'a', 'utf-8') as f:
         f.write(json.dumps(content, ensure_ascii=False) + '\n')

 def main(offset):
     url = 'http://maoyan.com/board/4?offset=' + str(offset)
     html = get_one_page(url)
     for item in parse_one_page(html):
         print json.dumps(item, ensure_ascii=False, encoding='utf-8')
         save_to_file(item)

 if __name__ == '__main__':
     pool = Pool()
     pool.map(main, [i*10 for i in range(10)])