数据存储 mongodb

数据存储 mongodb

 from pymongo import MongoClient
 import os
 base_dir = os.getcwd()
 class MongoPipeline(object):
     # 实现保存到mongo数据库的类，
     collection = 'douban'  # mongo 数据库的 collection 名字

     def __init__(self, mongo_uri, db_name, db_user, db_pass):
         self.mongo_uri = mongo_uri
         self.db_name = db_name
         self.db_user = db_user
         self.db_pass = db_pass

     @classmethod
     def from_crawler(cls, crawler):
         # scrapy 为我们访问settings提供了这样的一个方法，这里，
         # 我们需要从 settings.py 文件中，取得数据库的URI和数据库名称
         return cls(
             mongo_uri=crawler.settings.get('MONGO_URI'),
             db_name=crawler.settings.get('DB_NAME'),
             db_user=crawler.settings.get('DB_USER'),
             db_pass=crawler.settings.get('DB_PASS'))

     def open_spider(self, spider):  # 爬虫启动时调用，连接到数据库
         self.client = MongoClient(self.mongo_uri)
         self.zfdb = self.client[self.db_name]
         self.zfdb.authenticate(self.db_user, self.db_pass)

     def close_spider(self, spider):  # 爬虫关闭时调用，关闭数据库连接
         self.client.close()

     def process_item(self, item, spider):
         self.zfdb[self.collection].insert({"title": item["title"].strip()})
         return item

存储 MongoDB