scrapy 抓取动态网页

-- coding: utf-8 --

'''

gouwu.sogou.com Spider, Created on Dec, 2014

version: 1.0

author: chenqx @http://chenqx.github.com

See more: http://doc.scrapy.org/en/latest/index.html

'''

import time

from scrapy.selector import Selector

from scrapy.http import Request

from scrapy.contrib.spiders import CrawlSpider

from scrapy.contrib.loader import ItemLoader

from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor

from etao.items import EtaoItem

from etao.lstData import lstData

from selenium import webdriver

class etaoSpider(CrawlSpider):

# name of spiders

name = 'Spider'

allow_domain = ['gouwu.sogou.com']

start_urls = [ ('http://gouwu.sogou.com/shop?query=' + searchWord ) for searchWord in lstData().lst]

link_extractor = {

'page': SgmlLinkExtractor(allow = '/detail/\d+.html.+'),

'page_down': SgmlLinkExtractor(allow = '/shop\?query=.+',),#restrict_xpaths = '//a[@class = "pagination-next"]'

}

_x_query = {

'title': '//p[@class="title"]/a/@title',

'name': '//span[@class="floatR hui61 mt1"]/text()',#//li[2]/a/div[@class="ruyitao-market-name ruyitao-market-name-hightlight"]/text()

'price' : '//span[@class="shopprice font17"]/text()', # 'price' : '//span[@class = "price"]/text()',

}

def init(self):

CrawlSpider.init(self)

# use any browser you wish

self.browser = webdriver.Firefox()

def del(self):

self.browser.close()

def parse(self, response):

#crawl all display page

for link in self.link_extractor['page_down'].extract_links(response):

yield Request(url = link.url, callback=self.parse)

#start browser

self.browser.get(response.url)

#loading time interval

time.sleep(5)

# get the data and write it to scrapy items

etaoItem_loader = ItemLoader(item=EtaoItem(), response = response)

url = str(response.url)

etaoItem_loader.add_value('url', url)

etaoItem_loader.add_xpath('title', self._x_query['title'])

etaoItem_loader.add_xpath('name', self._x_query['name'])

etaoItem_loader.add_xpath('price', self._x_query['price'])

yield etaoItem_loader.load_item()