scrapy基础知识之将item 通过pipeline保存数据到mysql mongoDB：

pipelines.py

class xxPipeline(object):

def process_item(self, item, spider):

con=pymysql.connect(host='localhost,user='',passwd='',db='',charset='utf8')

cur=con.cursor()

sql=("insert into 表名(字段)”"values(%s)")

lis=(item['字段'])

cur.execute(sql,lis)

con.commit()

cur.close()

con.close()

return item

在settings.py中配置

ITEM_PIPELINES = {

’MySpider.pipelines.xxPipeline': 300,

}

需要在mysql中创建数据库以及数据表字段

pipelines.py

import pymongo

class MyspiderPipeline(object):
    def __init__(self):

        # pymongo.MongoClient(host, port) 创建MongoDB链接
        client = pymongo.MongoClient(host="localhost",port=27017)

        # 指向指定的数据库
        db_name = client["dbname"]
        table_name=db_name["tablename"]
        # 获取数据库里存放数据的表名
        self.post = table_name

    def process_item(self, item, spider):
        data = dict(item)
        # 向指定的表里添加数据
        self.post.insert(data)
        return item

settings.py配置

DEFAULT_REQUEST_HEADERS = {
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:53.0) Gecko/20100101 Firefox/53.0'}