使用scrapy 爬取酷狗音乐歌手及歌曲名并存入mongodb中

备注还没来得及写，共爬取八千多的歌手，每名歌手平均三十首歌曲算，大概二十多万首歌曲

run.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 __author__ = 'Zqf'
 from dingdian_simple.spiders.dingdian_spider import DingdianSimpleSpider
 from scrapy.crawler import CrawlerProcess
 from scrapy.utils.project import get_project_settings

 # 获取settings.py模块的设置
 settings = get_project_settings()
 process = CrawlerProcess(settings=settings)

 # 可以添加多个spider
 process.crawl(DingdianSimpleSpider)

 # 启动爬虫，会阻塞，直到爬取完成
 process.start()

kugou.py

 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import re

 __author__ = 'Zqf'

 import scrapy
 from kugoumusic.items import KugoumusicItem
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Rule

 class KugouSpiders(scrapy.spiders.CrawlSpider):
     name = 'kugou'

     start_urls = ['http://www.kugou.com/']

     rules = (
         Rule(LinkExtractor(allow=['http://www.kugou.com/yy/html/singer.html',
                                   'http://www.kugou.com/yy/singer/index/\d-([a-z]|null)-1.html'])),
         Rule(LinkExtractor(allow=['http://www.kugou.com/yy/singer/home/\d+.html']), callback='parse_item')
     )

     def parse_item(self, response):
         singer = response.xpath('//div/div[@class="clear_fix"]/strong/text()').extract_first()
         print(singer)
         songs = response.xpath('//ul[@id="song_container"]/li//span[@class="text"]/i/text()').extract()
         print(songs)

         item = KugoumusicItem()
         item['singer'] = singer
         item['songs'] = songs

         yield item

items.py

 # -*- coding: utf-8 -*-

 # Define here the models for your scraped items
 #
 # See documentation in:
 # https://doc.scrapy.org/en/latest/topics/items.html

 import scrapy

 class KugoumusicItem(scrapy.Item):
     # define the fields for your item here like:
     # name = scrapy.Field()
     singer = scrapy.Field()
     songs = scrapy.Field()

pipelines.py

 # -*- coding: utf-8 -*-

 # Define your item pipelines here
 #
 # Don't forget to add your pipeline to the ITEM_PIPELINES setting
 # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 from pymongo import MongoClient

 class KugoumusicPipeline(object):

     def open_spider(self, spider):
         # mongo_config = spider.settings['MONGO_CONFIG']
         # host = '127.0.0.1', port = 27017
         self.client = MongoClient(host='127.0.0.1', port=27017)
         self.coll = self.client['student_db']['kugou']
         self.li = []

     def close_spider(self, spider):
         self.insert()
         self.client.close()

     def insert(self):
         self.coll.insert_many(self.li)

     def process_item(self, item, spider):
         if len(self.li) >= 100:
             self.insert()
             self.li = []
             print("成功插入100条数据-------------------------------------")
         else:
             self.li.append(dict(item))

         return item

settings.py

 # -*- coding: utf-8 -*-

 # Scrapy settings for kugoumusic project
 #
 # For simplicity, this file contains only settings considered important or
 # commonly used. You can find more settings consulting the documentation:
 #
 #     https://doc.scrapy.org/en/latest/topics/settings.html
 #     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
 #     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

 BOT_NAME = 'kugoumusic'

 SPIDER_MODULES = ['kugoumusic.spiders']
 NEWSPIDER_MODULE = 'kugoumusic.spiders'

 # MONGO_CONFIG = ['192.168.62.35:1806, '
 #               '192.168.62.240:1806, '
 #               '192.168.62.23:1806, '
 #               '192.168.62.32:1806, '
 #               '192.168.62.25:1806, '
 #               '192.168.62.28:1806, '
 #               '192.168.62.241:1806']

 # MONGO_CONFIG = {
 #     'host': '127.0.0.1',
 #     'port': 27017
     # 'user': 'root',
     # 'password': '123456',
     # 'db': 's1806',
     # 'charset': 'utf8'
 # }
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
 #USER_AGENT = 'kugoumusic (+http://www.yourdomain.com)'

 # Obey robots.txt rules
 ROBOTSTXT_OBEY = False

 # Configure maximum concurrent requests performed by Scrapy (default: 16)
 #CONCURRENT_REQUESTS = 32

 # Configure a delay for requests for the same website (default: 0)
 # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
 # See also autothrottle settings and docs
 #DOWNLOAD_DELAY = 3
 # The download delay setting will honor only one of:
 #CONCURRENT_REQUESTS_PER_DOMAIN = 16
 #CONCURRENT_REQUESTS_PER_IP = 16

 # Disable cookies (enabled by default)
 #COOKIES_ENABLED = False

 # Disable Telnet Console (enabled by default)
 #TELNETCONSOLE_ENABLED = False

 # Override the default request headers:
 DEFAULT_REQUEST_HEADERS = {
     'Connection': 'keep-alive',
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
     'Accept-Encoding': 'gzip, deflate, br',
     'Accept-Language': 'zh-CN,zh;q=0.9',
 }

 # Enable or disable spider middlewares
 # See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
 #SPIDER_MIDDLEWARES = {
 #    'kugoumusic.middlewares.KugoumusicSpiderMiddleware': 543,
 #}

 # Enable or disable downloader middlewares
 # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
 #DOWNLOADER_MIDDLEWARES = {
 #    'kugoumusic.middlewares.KugoumusicDownloaderMiddleware': 543,
 #}

 # Enable or disable extensions
 # See https://doc.scrapy.org/en/latest/topics/extensions.html
 #EXTENSIONS = {
 #    'scrapy.extensions.telnet.TelnetConsole': None,
 #}

 # Configure item pipelines
 # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
 ITEM_PIPELINES = {
    'kugoumusic.pipelines.KugoumusicPipeline': 300,
 }

 # Enable and configure the AutoThrottle extension (disabled by default)
 # See https://doc.scrapy.org/en/latest/topics/autothrottle.html
 #AUTOTHROTTLE_ENABLED = True
 # The initial download delay
 #AUTOTHROTTLE_START_DELAY = 5
 # The maximum download delay to be set in case of high latencies
 #AUTOTHROTTLE_MAX_DELAY = 60
 # The average number of requests Scrapy should be sending in parallel to
 # each remote server
 #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
 # Enable showing throttling stats for every response received:
 #AUTOTHROTTLE_DEBUG = False

 # Enable and configure HTTP caching (disabled by default)
 # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
 #HTTPCACHE_ENABLED = True
 #HTTPCACHE_EXPIRATION_SECS = 0
 #HTTPCACHE_DIR = 'httpcache'
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'