scrapy实战5 POST方法抓取ajax动态页面(以慕课网APP为例子)：

在手机端打开慕课网，fiddler查看如图注意圈起来的位置

经过分析只有画线的page在变化

上代码：

items.py

 import scrapy

 class ImoocItem(scrapy.Item):
     # define the fields for your item here like:
     # name = scrapy.Field()
     #id
     id=scrapy.Field()
     #标题
     title=scrapy.Field()
     #类别
     tag=scrapy.Field()
     #内容
     content=scrapy.Field()
     #爬取时间
     crawl_time=scrapy.Field()

spiders/IMooc.py

 # -*- coding: utf-8 -*-
 import scrapy
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule
 import json
 from imooc.items import ImoocItem
 from datetime import datetime

 from scrapy.http import FormRequest
 class ImoocSpider(scrapy.Spider):
     name = 'IMooc'
     allowed_domains = ['www.imooc.com']
     headers={
         "User - Agent":"mukewang/5.1.4 (Android 4.4.2; Huawei H60-L02 Build/HDH60-L02),Network WIFI",
         "APP-INFO":"mukewang/5.1.4 (Android 4.4.2; Huawei H60-L02 Build/HDH60-L02),Network WIFI",
         "Host":"www.imooc.com",
         "Connection":"Keep-Alive",
         "Content-Type":"application/x-www-form-urlencoded"
     }
     def start_requests(self):
         url="http://www.imooc.com/api3/articlelist"
         requests = []
         for i in range(1,60):
             formdata={
                 "typeid":"",
                  "page":str(i),
                 }
             request = FormRequest(url, callback=self.parse_item, formdata=formdata,headers=self.headers)
             requests.append(request)
         return requests
     def parse_item(self, response):
         datas=json.loads(response.text)["data"]
         for data in datas:
             item=ImoocItem()
             item["id"]=data["id"]
             item["title"]=data["title"]
             item["tag"]=data["type"]
             item["content"]=data["desc"]
             item["crawl_time"]=datetime.now()
             yield item

pipelines.py

 import pymysql
 class ImoocPipeline(object):
     def process_item(self, item, spider):
         con = pymysql.connect(host="127.0.0.1", user="youusername", passwd="youpassword", db="imooc", charset="utf8")
         cur = con.cursor()
         sql = ("insert into imooc_shouji(id,title,tag,content,crawl_time)"
                "VALUES(%s,%s,%s,%s,%s)")
         lis = (item["id"],item["title"],item["tag"],item["content"],item["crawl_time"])
         cur.execute(sql, lis)
         con.commit()
         cur.close()
         con.close()
         return item

settings.py

 BOT_NAME = 'imooc'

 SPIDER_MODULES = ['imooc.spiders']
 NEWSPIDER_MODULE = 'imooc.spiders'
 ROBOTSTXT_OBEY = False
 DOWNLOAD_DELAY = 5
 ITEM_PIPELINES = {
    'imooc.pipelines.ImoocPipeline': 300,
 }

只爬取python相关的手记如下图：