scrapy+splash 爬取京东动态商品

作业来源:https://edu.cnblogs.com/campus/gzcc/GZCC-16SE1/homework/3159

splash是容器安装的，从docker官网上下载windows下的docker进行安装。

下载完成之后直接点击安装，安装成功后，桌边会出现三个图标：

点击 Docker QuickStart 图标来启动 Docker Toolbox 终端。

使用docker启动服务命令启动Splash服务

docker run -p 8050:8050 scrapinghub/splash

这里我已经开启服务了

打开cmd，在当前目录下开始scrapy爬虫:
scrapy startproject scrapy_examples
在spider文件夹中新建python文件jd_book.py用于编写爬虫
在项目下新建pybook.py用于对数据文件csv处理

京东上的商品是动态加载的，爬取python书籍的前20页，获取每个商品的评论数、书名、简介。

SplashRequest(url,endpoint='execute',args={'lua_source':lua_script},cache_args=['lua_source'])请求页面并执行JS函数渲染页面
endpoint='execute':在页面中执行一些用户自定义的JavaScript代码
args={'lua_source':lua_script}:用户自定义的lua脚本
cache_args=['lua_source']:让Splash服务器缓存该函数
用户自定义的lua脚本中必须包含一个main函数作为程序入口，main函数被调用时会传入一个splash对象（lua中的对象），用户可以调用该对象上的方法操纵Splash。
splash.args属性：用户传入参数的表，通过该属性可以访问用户传入的参数
splash:go方法：类似于在浏览器中打开某url地址的页面，页面所需资源会被加载，并进行JavaScript渲染
splash:wait方法：等待页面渲染，time参数为等待的秒数
splash:runjs方法：在当前页面下，执行一段JavaScript代码
splash:html方法：splash:html()获取当前页面的HTML文本。
middlewares.py随机产生User-Agent添加到每个请求头中
pipelines.py处理爬取的数据并存入数据库
settings.py配置splash服务信息、设置请求延迟反爬虫、添加数据库信息

 # -*- coding:utf-8 -*-
 import scrapy
 from scrapy import Request
 from scrapy_splash import SplashRequest
 from splash_examples.items import PyBooksItem

 lua_script ='''
 function main(splash)
     splash:go(splash.args.url)
     splash:wait(2)
     splash:runjs("document.getElementsByClassName('pn-next')[0].scrollIntoView(true)")
     splash:wait(2)
     return splash.html()
 end
 '''
 class JDBookSpider(scrapy.Spider):
     name = "jd_book"
     allowed_domains = ['search.jd.com']
     base_url = 'https://search.jd.com/Search?keyword=python&enc=utf-8&wq=python'
     def start_requests(self):
         yield Request(self.base_url,callback=self.parse_urls,dont_filter=True)
     def parse_urls(self,response):
         for i in range(20):
             url = '%s&page=%s' % (self.base_url,2*i+1)
             yield SplashRequest(url,
                                 endpoint='execute',
                                 args={'lua_source':lua_script},
                                 cache_args=['lua_source'])
     def parse(self, response):
         for sel in response.css('ul.gl-warp.clearfix>li.gl-item'):
             pyjdbooks = PyBooksItem()
             pyjdbooks['name'] = sel.css('div.p-name').xpath('string(.//em)').extract_first()
             pyjdbooks['comment']=sel.css('div.p-commit').xpath('string(.//a)').extract_first()
             pyjdbooks['promo_words']=sel.css('div.p-name').xpath('string(.//i)').extract_first()
             yield pyjdbooks

jd_book.py

 import scrapy

 class PyBooksItem(scrapy.Item):
     name=scrapy.Field()
     comment=scrapy.Field()
     promo_words=scrapy.Field()

items.py

 from fake_useragent import UserAgent
 # 随机的User-Agent
 class RandomUserAgent(object):
     def process_request(self, request, spider):
         request.headers.setdefault("User-Agent", UserAgent().random)

middlewares.py

 class SplashExamplesPipeline(object):
     def __init__(self):
         self.book_set = set()

     def process_item(self, item, spider):
         if not(item['promo_words']):
             item['promo_words'] = item['name']
         comment = item['comment']
         if comment[-2:] == "万+":
             item['comment'] = str(int(float(comment[:-2])*10000))
         elif comment[-1] == '+':
             item['comment'] = comment[:-1]
         return item

 import pymysql

 class MysqlPipeline(object):
     def __init__(self, host, database, user, password, port):
         self.host = host
         self.database = database
         self.user = user
         self.password = password
         self.port = port

     @classmethod
     def from_crawler(cls, crawler):
         return cls(
             host=crawler.settings.get('MYSQL_HOST'),
             database=crawler.settings.get('MYSQL_DATABASE'),
             user=crawler.settings.get('MYSQL_USER'),
             password=crawler.settings.get('MYSQL_PASSWORD'),
             port=crawler.settings.get('MYSQL_PORT'),
         )

     def open_spider(self, spider):
         self.db = pymysql.connect(self.host, self.user, self.password, self.database, charset='utf8', port=self.port)
         self.cursor = self.db.cursor()

     def close_spider(self, spider):
         self.db.close()

     def process_item(self, item, spider):
         data = dict(item)
         keys = ', '.join(data.keys())
         values = ', '.join(['%s'] * len(data))
         sql = 'insert into books (%s) values (%s)' % (keys, values)
         self.cursor.execute(sql, tuple(data.values()))
         self.db.commit()
         return item

pipelines.py

 BOT_NAME = 'splash_examples'

 SPIDER_MODULES = ['splash_examples.spiders']
 NEWSPIDER_MODULE = 'splash_examples.spiders'

 #Splash服务器地址
 SPLASH_URL = 'http://192.168.99.100:8050'

 #开启Splash的两个下载中间件并调整HttpCompressionMiddleware的次序
 DOWNLOADER_MIDDLEWARES = {
     'splash_examples.middlewares.RandomUserAgent':345,
     'scrapy_splash.SplashCookiesMiddleware': 723,
     'scrapy_splash.SplashMiddleware': 725,
     'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
 }
 #设置去重过滤器
 DUPEFILTER_CLASS='scrapy_splash.SplashAwareDupeFilter'

 #用来支持cache_args
 SPIDER_MIDDLEWARES ={
     'scrapy_splash.SplashDeduplicateArgsMiddleware':100,
 }
 # 使用Splash的Http缓存
 HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage'

 # Obey robots.txt rules
 ROBOTSTXT_OBEY = False

 COOKIES_ENABLED = False
 DOWNLOAD_DELAY = 3

 ITEM_PIPELINES = {
     'splash_examples.pipelines.SplashExamplesPipeline':400,
     'splash_examples.pipelines.MysqlPipeline':543,
 }

 MYSQL_HOST = 'localhost'
 MYSQL_DATABASE = 'pybooks'
 MYSQL_PORT = 3306
 MYSQL_USER = 'root'
 MYSQL_PASSWORD = 'root'

settings.py

在Terminal中执行爬虫:scrapy crawl jd_book -o pybooks.csv
将数据存储到数据库并生成csv文件用于分析可视化

在数据库中查看有1187条信息

做数据分析可视化

 import pandas as pd
 import jieba
 from wordcloud import WordCloud
 import matplotlib.pyplot as plt
 obj = pd.read_csv('pybooks.csv')
 Books = obj.sort_values('comment',ascending=False)[:200]
 promoWords = []
 for promo in Books['promo_words']:
     promoWords.append(promo)
 promoWordsStr = ''.join(promoWords)
 bookTxt = jieba.lcut(promoWordsStr)
 stopwords = ['学习','入门','掌握','教程','图书','使用','全面','推荐','读者','专家']
 bookTxt = [token for token in bookTxt if token not in stopwords]
 bookTxtSet = set(bookTxt)
 txtCount = {}
 for i in bookTxtSet:
     if len(i) == 1:
         continue
     txtCount[i] = bookTxt.count(i)
 txtCount = sorted(txtCount.items(),key=lambda key:key[1],reverse=True)
 TxtStr = ' '.join(bookTxt)
 ciyun = WordCloud(background_color = '#122',width=400,height=300,margin = 1).generate(TxtStr)
 plt.imshow(ciyun)
 plt.axis("off")
 plt.show()
 print(txtCount)

pybook.py