scrapy爬取极客学院全部课程

 # -*- coding: utf-8 -*-
 # scrapy爬取极客学院全部课程
 import scrapy
 from pyquery import PyQuery as pq
 from jike.items import JikeItem

 class JikespiderSpider(scrapy.Spider):
     name = "jikespider"
     allowed_domains = ["www.jikexueyuan.com"]
     base_url = 'http://www.jikexueyuan.com/course/?pageNum='

     def start_requests(self):
         for page_num in range(1,96):
             url = self.base_url + str(page_num)
             yield scrapy.Request(url, callback=self.parse_index)

     def parse_index(self, response):
         doc = pq(response.text)
         lis = doc('.lesson-list .cf li').items()
         # pyquery心得, 以为pyquery有点问题而导致无法遍历数据结构,
         # 研究发现是'http:' + item('.lessonimg-box a').attr('href')
         # 的问题, href是相对路径没有得到一个有效的请求链接
         for item in lis:
             detail_url = 'http:' + item('.lessonimg-box a').attr('href')
             yield scrapy.Request(url=detail_url,callback=self.parse_detail)

     def parse_detail(self, response):
         item = JikeItem()
         doc = pq(response.text)
         item['title'] = doc('.lesson-teacher .bc-box h2').text()
         item['time'] = doc('.lesson-teacher .bc-box .timebox').text()
         item['content'] = doc('.lesson-teacher .infor-content').text()

         yield item