pyspider爬取TripAdvisor

 #!/usr/bin/env python
 # -*- encoding: utf-8 -*-
 # Created on 2017-06-11 10:10:53
 # Project: london

 from pyspider.libs.base_handler import *
 import pymongo

 class Handler(BaseHandler):
     crawl_config = {
     }
     client = pymongo.MongoClient('localhost')
     db = client['trip']

     @every(minutes=24 * 60)
     def on_start(self):
         self.crawl('https://www.tripadvisor.cn/Attractions-g186338-Activities-c47-London_England.html', callback=self.index_page)

     @config(age=10 * 24 * 60 * 60)
     def index_page(self, response):
         for each in response.doc('.listing_title > a').items():
             self.crawl(each.attr.href, callback=self.detail_page)
         next_page = response.doc('.pagination .nav.next').attr.href
         self.crawl(next_page,callback = self.index_page)

     @config(priority=2)
     def detail_page(self, response):
         return {
             "name":response.doc('h1').text(),
             "url": response.url,
             'comment':response.doc('.heading_ratings .taLnk').text(),
             'address':response.doc('.addressReset > span.format_address').text(),
             'phone':response.doc('.phoneNumber').text(),
             'duration':response.doc('#MAP_AND_LISTING > div.main_section.listingbar > div > div.above_fold_listing_details > div > div:nth-child(5) > div > div:nth-child(1)').text(),
             'instruction':response.doc('#MAP_AND_LISTING > div.main_section.listingbar > div > div.above_fold_listing_details > div > div:nth-child(6) > div > b').text()
         }
     def on_result(self,result):
         if result:
             self.save_to_mongo(result)

     def save_to_mongo(self,result):
         if self.db['london'].insert(result):
             print('saved to mongo',result)