Python操作数据库实战
pymysql
# -*- coding: utf-8 -*- """
@Datetime: 2018/12/26
@Author: Zhang Yafei
"""
import pymysql
from DBUtils.PooledDB import PooledDB POOL = PooledDB(
creator=pymysql, # 使用链接数据库的模块
maxconnections=6, # 连接池允许的最大连接数，0和None表示不限制连接数
mincached=2, # 初始化时，链接池中至少创建的空闲的链接，0表示不创建
maxcached=5, # 链接池中最多闲置的链接，0和None不限制
maxshared=3,
# 链接池中最多共享的链接数量，0和None表示全部共享。PS: 无用，因为pymysql和MySQLdb等模块的 threadsafety都为1，所有值无论设置为多少，_maxcached永远为0，所以永远是所有链接都共享。
blocking=True, # 连接池中如果没有可用连接后，是否阻塞等待。True，等待；False，不等待然后报错
maxusage=None, # 一个链接最多被重复使用的次数，None表示无限制
setsession=[], # 开始会话前执行的命令列表。如：["set datestyle to ...", "set time zone ..."]
ping=0,
# ping MySQL服务端，检查是否服务可用。# 如：0 = None = never, 1 = default = whenever it is requested, 2 = when a cursor is created, 4 = when a query is executed, 7 = always
host='127.0.0.1',
port=3306,
user='root',
password='0000',
database='flask_code',
charset='utf8'
) def connect(type=None):
conn = POOL.connection()
cursor = conn.cursor(cursor=type)
return conn, cursor def connect_close(conn, cursor):
cursor.close()
conn.close() def fetchone(sql, arg=list()):
conn, cursor = connect(type)
cursor.execute(sql, arg)
data = cursor.fetchone()
connect_close(conn, cursor)
return data def fetchall(sql, arg=list(), type=pymysql.cursors.DictCursor):
conn, cursor = connect(type)
cursor.execute(sql, arg)
data = cursor.fetchall()
connect_close(conn, cursor)
return data def insert(sql, arg=list()):
conn, cursor = connect()
row = cursor.execute(sql, arg)
conn.commit()
connect_close(conn, cursor)
return row
mysql_helper
sqlite
# -*- coding: utf-8 -*- """
@Datetime: 2019/1/31
@Author: Zhang Yafei
"""
import sqlite3
import settings
import os BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DB_DIR = os.path.join(BASE_DIR, 'data.db') def connect():
'''连接数据库'''
conn = sqlite3.connect(settings.DATABASES.get('sqlite3')) # db不存在时将自动创建db
cursor = conn.cursor()
return conn, cursor def connect_close(conn, cursor):
"""关闭连接"""
cursor.close()
conn.close() def execute(sql, params=tuple()):
conn, cursor = connect()
cursor.execute(sql, params) # 执行这个语句
connect_close(conn, cursor) def fetchone(sql, params=tuple()):
conn, cursor = connect()
result = cursor.execute(sql, params)
data = result.fetchone()
connect_close(conn, cursor)
return data def fetchall(sql, params=tuple()):
conn, cursor = connect()
results = cursor.execute(sql, params)
data = results.fetchall()
connect_close(conn, cursor)
return data
sqlite3_helper
# -*- coding: utf-8 -*- """
@Datetime: 2019/1/31
@Author: Zhang Yafei
"""
import sqlite3
import os BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DB_DIR = os.path.join(BASE_DIR, 'data.db') class SqliteDB(object):
def __init__(self):
self.conn = sqlite3.connect(DB_DIR) # db不存在时将自动创建db
self.cursor = self.conn.cursor() def close(self):
self.cursor.close()
self.conn.close() def execute(self, sql, params=tuple()):
self.cursor.execute(sql, params)
self.close() def fetchone(self, sql, params=tuple()):
result = self.cursor.execute(sql, params)
data = result.fetchone()
self.close()
return data def fetchall(self, sql, params=tuple()):
results = self.cursor.execute(sql, params)
data = results.fetchall()
self.close()
return data if __name__ == '__main__':
sqlite = SqliteDB()
# 1. 建表
sql = '''create table happy(
username text,
password text,
id int)'''
sqlite.execute(sql) # 2. 插入数据
sqlite.execute("INSERT INTO COMPANY (ID,NAME,AGE,ADDRESS,SALARY) \
VALUES (4, 'Mark', 25, 'Rich-Mond ', 65000.00 )") # 3. 更改数据
sqlite.execute("UPDATE COMPANY SET ID=99 WHERE ID=2") # 4. 删除表里面的数据
c.execute("DELETE FROM COMPANY WHERE ID=4")
c.execute("DELETE FROM COMPANY WHERE ID=3") # 5. 查询
data = sqlite.fetchall('select * from label limit 1')
print(data)
# 输出
'''
[('盘龙云海(排毒养颜胶囊)', 509881, '广东深圳龙岗区/女', '昨天吃的，今天就拉肚子了。感觉肚子有点涨痛！不知道效果怎么样~~~~~',
'昨天/吃/的/，/今天/就/拉肚子/SB了/。/感觉/肚子/PB有点/涨痛/SB！/不/知道/效果/怎么样/~/~/~/~/~', '2011-09-30 15:26:00',
'http://ypk.39.net/509881/comment/k0_p...', '昨天/吃/的/，/今天/就/拉肚子/SB了/。/感觉/肚子/PB有点/涨痛/SB！/不/知道/效果/怎么样/~/~/~/~/~',
'昨天/吃/的/，/今天/就/拉肚子/SB了/。/感觉/肚子/PB有点/涨痛/SB！/不/知道/效果/怎么样/~/~/~/~/~')]
'''
sqlite3_helper2
from DBUtils.PooledDB import PooledDB
from settings import DATABASE
import threading lock = threading.Lock() class SqliteHelper(object):
def __init__(self, db_config):
"""
# sqlite3
# 连接数据库文件名，sqlite不支持加密，不使用用户名和密码
import sqlite3
config = {"database": "path/to/your/dbname.db"}
pool = PooledDB(sqlite3, maxcached=50, maxconnections=1000, maxusage=1000, **config)
# mysql
import pymysql
pool = PooledDB(pymysql,5,host='localhost', user='root',passwd='pwd',db='myDB',port=3306) #5为连接池里的最少连接数
# postgressql
import psycopg2
POOL = PooledDB(creator=psycopg2, host="127.0.0.1", port="5342", user, password, database)
# sqlserver
import pymssql
pool = PooledDB(creator=pymssql, host=host, port=port, user=user, password=password, database=database, charset="utf8")
:param type:
"""
self.pool = PooledDB(**db_config)
self.conn = self.pool.connection()
self.cursor = self.conn.cursor() def connect(self):
conn = self.pool.connection()
cursor = self.conn.cursor()
return conn, cursor @staticmethod
def connect_close(conn, cursor):
"""关闭连接"""
conn.close()
cursor.close() def __del__(self):
self.cursor.close()
self.conn.close() def execute(self, sql, params=tuple()):
try:
lock.acquire(True)
self.cursor.execute(sql, params) # 执行这个语句
self.conn.commit()
lock.release()
except Exception as e:
print(e) def execute_many(self, sql, params=tuple()):
try:
lock.acquire(True)
self.cursor.executemany(sql, params)
self.conn.commit()
lock.release()
except Exception as e:
print(e) def fetchone(self, sql, params=tuple()):
try:
lock.acquire(True)
self.cursor.execute(sql, params)
data = self.cursor.fetchone()
lock.release()
return data
except Exception as e:
print(e) def fetchall(self, sql, params=tuple()):
try:
lock.acquire(True)
self.cursor.execute(sql, params)
data = self.cursor.fetchall()
lock.release()
return data
except Exception as e:
print(e)
SqliteHelper3.py
mongodb
# -*- coding: utf-8 -*- """
@Datetime: 2019/1/31
@Author: Zhang Yafei
"""
import json
import pymongo
import pandas as pd class MongoPipeline(object):
"""
mongodb:
save(self, data, collection): 将数据保存到数据库
read(self, data): 读取数据库中指定表格
insert(self, table, dict_data)： 插入数据
delete(self, table, condition)： 删除指定数据
update(self, table, condition, new_dict_data)： 更新指定数据
dbFind(self, table, condition=None): 按条件查找
findAll(self, table)： 查找全部
close(self)： 关闭连接
""" def __init__(self, mongo_db, mongo_uri='localhost'):
self.mongo_uri = mongo_uri
self.mongo_db = mongo_db
self.client = pymongo.MongoClient(self.mongo_uri)
self.db = self.client[self.mongo_db] def close(self):
"""
关闭连接
:return:
"""
self.client.close() def save(self, data, collection):
"""
将数据保存到数据库表
:param data:
:param collection:
:return: None
"""
self.collection = self.db[collection]
try:
if self.collection.insert(json.loads(data.T.to_json()).values()):
print('mongodb insert {} sucess.'.format(collection))
return
except Exception as e:
print('insert error:', e)
import traceback
traceback.print_exc(e) def read(self, table):
"""
读取数据库中的数据
:param table:
:return: dataframe
"""
try:
# 连接数据库
table = self.db[table]
# 读取数据
data = pd.DataFrame(list(table.find()))
return data
except Exception as e:
import traceback
traceback.print_exc(e) def insert(self, table, dict_data):
"""
插入
:param table:
:param dict_data:
:return: None
"""
try:
self.db[table].insert(dict_data)
print("插入成功")
except Exception as e:
print(e) def update(self,table, condition, new_dict_data):
"""
更新
:param table:
:param dict_data:
:param new_dict_data:
:return: None
"""
try:
self.db[table].update(condition, new_dict_data)
print("更新成功")
except Exception as e:
print(e) def delete(self,table, condition):
"""
删除
:param table:
:param dict_data:
:return: None
"""
try:
self.db[table].remove(condition)
print("删除成功")
except Exception as e:
print(e) def dbFind(self, table, condition=None):
"""
按条件查找
:param table:
:param dict_data:
:return: generator dict
"""
data = self.db[table].find(condition)
for item in data:
yield item def findAll(self, table):
"""
查找全部
:param table:
:return: generator dict
"""
for item in self.db[table].find():
yield item if __name__ == '__main__':
mongo = MongoPipeline('flask')
# data = mongo.read('label')
# print(data.head())
condition = {"药品ID": 509881}
data = mongo.dbFind('label', condition)
print(data)
for i in data:
print(i)
# mongo.findAll()
mongo_helper
postgresql
# -*- coding: utf-8 -*-
'''
Datetime: 2019/10/24
author: Zhang Yafei
description: python操作 postgressql数据库
操作流程是：
1）连接PostgreSQL并设定Cursor
2）执行SQL脚本 #有几点需要注意
- sql脚本必须以';'结尾，不可以省略
- 不管sql中有几个参数，都需要用%s代替，只有%s, 不管值是字符还是数字, 一律%s.
- 第二个参数中，一定要传入元组，上述例子中(5)是不行的
- 执行完INSERT, UPDATE, DELETE这样的sql脚本后, 需要conn.commit()提交一下, 才会把数据提交到数据库当中.
3）得到查询结果 可以采用fetchone(), fetchall()和fetchmany() 三种方式。fetchone返回一个tuple或者None, fetchall返回一个
list of tuple，如果没有结果则返回一个空的tuple。fetchmany返回list of tuple， list的长度由size参数决定，
size的默认值是cursor.arraysize， 如果没有结果可以返回，那么返回一个空的list。
'''
import psycopg2 def connect():
conn = psycopg2.connect("dbname=xiaomuchong user=postgres password=0000 host=localhost port=5432")
cur = conn.cursor()
return conn, cur def connect_close(conn, cursor):
"""关闭连接"""
cursor.close()
conn.close() def execute(sql, params=tuple()):
conn, cursor = connect()
cursor.execute(sql, params) # 执行这个语句
conn.commit()
connect_close(conn, cursor) def fetchone(sql, params=tuple()):
conn, cursor = connect()
result = cursor.execute(sql, params)
data = result.fetchone()
connect_close(conn, cursor)
return data def fetchall(sql, params=tuple()):
conn, cursor = connect()
cursor.execute(sql, params)
data = cursor.fetchall()
connect_close(conn, cursor)
return data if __name__ == "__main__":
# create_sql = "CREATE TABLE Employee (name varchar(80), address varchar(80),age int, date date)"
# execute(create_sql) # execute("INSERT INTO Employee VALUES('Gopher', 'China Beijing', 100, '2017-05-27')")
# execute("INSERT INTO Employee VALUES('Zhang', 'China jincheng', 24, '2019-10-24')") # rows = fetchall("SELECT * FROM Employee;")
# for row in rows:
# print('name=' + str(row[0]) + ' address=' + str(row[1]) +
# ' age=' + str(row[2]) + ' date=' + str(row[3])) # execute("UPDATE Employee SET age=12 WHERE name='Gopher'") execute("DELETE FROM Employee WHERE name='Gopher'")
postgresql_helper
# -*- coding: utf-8 -*-
'''
Datetime: 2019/10/24
author: Zhang Yafei
description: python操作 postgressql数据库
操作流程是：
1）连接PostgreSQL并设定Cursor
2）执行SQL脚本 #有几点需要注意
- sql脚本必须以';'结尾，不可以省略
- 不管sql中有几个参数，都需要用%s代替，只有%s, 不管值是字符还是数字, 一律%s.
- 第二个参数中，一定要传入元组，上述例子中(5)是不行的
- 执行完INSERT, UPDATE, DELETE这样的sql脚本后, 需要conn.commit()提交一下, 才会把数据提交到数据库当中.
3）得到查询结果 可以采用fetchone(), fetchall()和fetchmany() 三种方式。fetchone返回一个tuple或者None, fetchall返回一个
list of tuple，如果没有结果则返回一个空的tuple。fetchmany返回list of tuple， list的长度由size参数决定，
size的默认值是cursor.arraysize， 如果没有结果可以返回，那么返回一个空的list。
'''
import psycopg2 class PostgresSql(object):
def __init__(self, dbname, user, password, host='localhost', port=5432):
self.conn = psycopg2.connect(
f"dbname={dbname} user={user} password={password} host={host} port={port}")
self.cursor = self.conn.cursor() def connect_close(self):
"""关闭连接"""
self.cursor.close()
self.conn.close() def execute(self, sql, params=tuple()):
self.cursor.execute(sql, params) # 执行这个语句
self.conn.commit() def fetchone(self, sql, params=tuple()):
self.cursor.execute(sql, params)
data = self.cursor.fetchone()
return data def fetchall(self, sql, params=tuple()):
self.cursor.execute(sql, params)
data = self.cursor.fetchall()
return data if __name__ == "__main__":
postgres = PostgresSql(dbname='xiaomuchong', user='postgres', password='0000', host='localhost', port='5432')
# create_sql = "CREATE TABLE Employee1 (name varchar(80), address varchar(80),age int, date date)"
# postgres.execute(create_sql) # postgres.execute("INSERT INTO Employee1 VALUES('Gopher', 'China Beijing', 100, '2017-05-27')")
# postgres.execute("INSERT INTO Employee1 VALUES('Zhang', 'China jincheng', 24, '2019-10-24')") rows = postgres.fetchall("SELECT * FROM Employee1;")
for row in rows:
print('name=' + str(row[0]) + ' address=' + str(row[1]) +
' age=' + str(row[2]) + ' date=' + str(row[3])) postgres.execute("UPDATE Employee1 SET age=12 WHERE name='Gopher'")
postgres.execute("DELETE FROM Employee1 WHERE name='Gopher'") rows = postgres.fetchall("SELECT * FROM Employee1;")
for row in rows:
print('name=' + str(row[0]) + ' address=' + str(row[1]) +
' age=' + str(row[2]) + ' date=' + str(row[3])) postgres.connect_close()
postgresql_helper_class
from DBUtils.PooledDB import PooledDB class DBPoolHelper(object):
def __init__(self, dbname, user=None, password=None, db_type='postgressql', host='localhost', port=5432):
"""
# sqlite3
# 连接数据库文件名，sqlite不支持加密，不使用用户名和密码
import sqlite3
config = {"datanase": "path/to/your/dbname.db"}
pool = PooledDB(sqlite3, maxcached=50, maxconnections=1000, maxusage=1000, **config)
# mysql
import pymysql
pool = PooledDB(pymysql,5,host='localhost', user='root',passwd='pwd',db='myDB',port=3306) #5为连接池里的最少连接数
# postgressql
import psycopg2
POOL = PooledDB(creator=psycopg2, host="127.0.0.1", port="5342", user, password, database)
# sqlserver
import pymssql
pool = PooledDB(creator=pymssql, host=host, port=port, user=user, password=password, database=database, charset="utf8")
:param type:
"""
if db_type == 'postgressql':
import psycopg2
pool = PooledDB(creator=psycopg2, host=host, port=port, user=user, password=password, database=dbname)
elif db_type == 'mysql':
import pymysql
pool = PooledDB(pymysql, 5, host='localhost', user='root', passwd='pwd', db='myDB',
port=3306) # 5为连接池里的最少连接数
elif db_type == 'sqlite':
import sqlite3
config = {"database": dbname}
pool = PooledDB(sqlite3, maxcached=50, maxconnections=1000, maxusage=1000, **config)
else:
raise Exception('请输入正确的数据库类型, db_type="postgresql" or db_type="mysql" or db_type="sqlite"')
self.conn = pool.connection()
self.cursor = self.conn.cursor() def connect_close(self):
"""关闭连接"""
self.cursor.close()
self.conn.close() def execute(self, sql, params=tuple()):
self.cursor.execute(sql, params) # 执行这个语句
self.conn.commit() def execute_many(self, sql, params=tuple()):
self.cursor.executemany(sql, params)
self.conn.commit() def fetchone(self, sql, params=tuple()):
self.cursor.execute(sql, params)
data = self.cursor.fetchone()
return data def fetchall(self, sql, params=tuple()):
self.cursor.execute(sql, params)
data = self.cursor.fetchall()
return data
postgresql_helper3.py
redis
def redis_init():
settings = get_project_settings()
if settings["REDIS_PARAMS"]:
pool = ConnectionPool(host=settings["REDIS_HOST"], port=settings["REDIS_PORT"],
password=settings["REDIS_PARAMS"]['password'])
else:
pool = ConnectionPool(host=settings["REDIS_HOST"], port=settings["REDIS_PORT"])
conn = Redis(connection_pool=pool)
return conn if __name__ == '__main__':
redis_conn = redis_init()
redis_helper.py
DBUtils
# -*- coding: utf-8 -*- """
Datetime: 2020/07/02
Author: Zhang Yafei
Description:
"""
import time
from functools import wraps
from DBUtils.PooledDB import PooledDB def timeit(func):
"""
装饰器： 判断函数执行时间
:param func:
:return:
""" @wraps(func)
def inner(*args, **kwargs):
start = time.time()
ret = func(*args, **kwargs)
end = time.time() - start
if end < 60:
print(f'花费时间：\t{round(end, 2)}秒')
else:
min, sec = divmod(end, 60)
print(f'花费时间\t{round(min)}分\t{round(sec, 2)}秒')
return ret return inner class DBPoolHelper(object):
def __init__(self, dbname, user=None, password=None, db_type='postgressql', host='localhost', port=5432):
"""
# sqlite3
# 连接数据库文件名，sqlite不支持加密，不使用用户名和密码
import sqlite3
config = {"datanase": "path/to/your/dbname.db"}
pool = PooledDB(sqlite3, maxcached=50, maxconnections=1000, maxusage=1000, **config)
# mysql
import pymysql
pool = PooledDB(pymysql,5,host='localhost', user='root',passwd='pwd',db='myDB',port=3306) #5为连接池里的最少连接数
# postgressql
import psycopg2
POOL = PooledDB(creator=psycopg2, host="127.0.0.1", port="5342", user, password, database)
# sqlserver
import pymssql
pool = PooledDB(creator=pymssql, host=host, port=port, user=user, password=password, database=database, charset="utf8")
:param type:
"""
if db_type == 'postgressql':
import psycopg2
pool = PooledDB(creator=psycopg2, host=host, port=port, user=user, password=password, database=dbname)
elif db_type == 'mysql':
import pymysql
pool = PooledDB(pymysql, 5, host='localhost', user='root', passwd='pwd', db='myDB',
port=3306) # 5为连接池里的最少连接数
elif db_type == 'sqlite':
import sqlite3
config = {"database": dbname}
pool = PooledDB(sqlite3, maxcached=50, maxconnections=1000, maxusage=1000, **config)
else:
raise Exception('请输入正确的数据库类型, db_type="postgresql" or db_type="mysql" or db_type="sqlite"')
self.conn = pool.connection()
self.cursor = self.conn.cursor() def connect_close(self):
"""关闭连接"""
self.cursor.close()
self.conn.close() def execute(self, sql, params=tuple()):
self.cursor.execute(sql, params) # 执行这个语句
self.conn.commit() def execute_many(self, sql, params=tuple()):
self.cursor.executemany(sql, params)
self.conn.commit() def fetchone(self, sql, params=tuple()):
self.cursor.execute(sql, params)
data = self.cursor.fetchone()
return data def fetchall(self, sql, params=tuple()):
self.cursor.execute(sql, params)
data = self.cursor.fetchall()
return data
DBHelper.py
pandas链接数据库
import pandas as pd
from sqlalchemy import create_engine def pandas_db_helper():
"""
'postgresql://postgres:0000@127.0.0.1:5432/xiaomuchong'
"mysql+pymysql://root:0000@127.0.0.1:3306/srld?charset=utf8mb4"
"sqlite: ///sqlite3.db"
"""
engine = create_engine( 'postgresql://postgres:0000@127.0.0.1:5432/xiaomuchong')
conn = engine.connect()
return conn if __name__ == '__main__':
db = pandas_db_helper()
data = pd.read_sql_table("team", con=db, index_col="board_id")
print(data)
pandas_db_helper.py
实战项目
1.pandas+mysql
# -*- coding: utf-8 -*- """
@Datetime: 2018/12/26
@Author: Zhang Yafei
"""
import pymysql
from DBUtils.PooledDB import PooledDB
import pandas as pd
from pymysql.err import InternalError
import numpy as np POOL = PooledDB(
creator=pymysql, # 使用链接数据库的模块
maxconnections=6, # 连接池允许的最大连接数，0和None表示不限制连接数
mincached=2, # 初始化时，链接池中至少创建的空闲的链接，0表示不创建
maxcached=5, # 链接池中最多闲置的链接，0和None不限制
maxshared=3,
# 链接池中最多共享的链接数量，0和None表示全部共享。PS: 无用，因为pymysql和MySQLdb等模块的 threadsafety都为1，所有值无论设置为多少，_maxcached永远为0，所以永远是所有链接都共享。
blocking=True, # 连接池中如果没有可用连接后，是否阻塞等待。True，等待；False，不等待然后报错
maxusage=None, # 一个链接最多被重复使用的次数，None表示无限制
setsession=[], # 开始会话前执行的命令列表。如：["set datestyle to ...", "set time zone ..."]
ping=0,
# ping MySQL服务端，检查是否服务可用。# 如：0 = None = never, 1 = default = whenever it is requested, 2 = when a cursor is created, 4 = when a query is executed, 7 = always
host='127.0.0.1',
port=3306,
user='root',
password='0000',
database='srld',
charset='utf8mb4'
) def connect(type=None):
conn = POOL.connection()
cursor = conn.cursor(cursor=type)
return conn, cursor def connect_close(conn, cursor):
cursor.close()
conn.close() def fetchone(sql, arg=list()):
conn, cursor = connect(type)
cursor.execute(sql, arg)
data = cursor.fetchone()
connect_close(conn, cursor)
return data def fetchall(sql, arg=list(), type=pymysql.cursors.DictCursor):
conn, cursor = connect(type)
cursor.execute(sql, arg)
data = cursor.fetchall()
connect_close(conn, cursor)
return data def insert(sql, arg=list()):
conn, cursor = connect()
row = cursor.execute(sql, arg)
conn.commit()
connect_close(conn, cursor)
return row def insert_sql(row):
pmid_doi, author, journal, year, title, abstract = row sql = """
insert into pubmed(PMID_DOI,author,journal,year,title,abstract) values(%s, %s, %s, %s, %s, %s)
"""
try:
insert(sql, arg=[pmid_doi, author, journal, year, title, abstract])
except InternalError:
# print(pmid_doi, author, journal, year, title, abstract)
print(author, type(author), np.isnan(author))
# pmid_doi = None if np.isnan(pmid_doi) else pmid_doi
# print(author, type(author), np.isnan(author))
author = None if np.isnan(author) else author
print(author)
# journal = None if np.isnan(journal) else journa
# year = None if np.isnan(year) else year
# title = None if np.isnan(title) else title
# abstract = None if np.isnan(abstract) else abstract
# print(author)
# print(pmid_doi, author, journal, year, title, abstract) def main():
df = pd.read_csv('xml副本/documents_1.txt', delimiter='\t', header=None)
df.fillna(0, inplace=True)
df.apply(insert_sql, axis=1) if __name__ == "__main__":
main()
# insert('''insert into pubmed(PMID_DOI,author,journal,year,title,abstract) values('PMID:28231184|DOI:10.3390/foods5040089', 'Moreira-Ascarrunz, Sergio Daniel; Larsson, Hans; Prieto-Linde, Maria Luisa; Johansson, Eva', 'Foods (Basel, Switzerland)', '2016', 'Mineral Nutritional Yield and Nutrient Density of Locally Adapted Wheat Genotypes under Organic Production.','The aim of the present investigation was to investigate the nutritional yield, nutrient density, stability, and adaptability of organically produced wheat for sustainable and nutritional high value food production. This study evaluated the nutritional yield of four minerals (Fe, Zn, Cu, and Mg) in 19 wheat genotypes, selected as being locally adapted under organic agriculture conditions. The new metric of nutritional yield was calculated for each genotype and they were evaluated for stability using the Additive Main effects and Multiplicative Interaction (AMMI) stability analysis and for genotypic value, stability, and adaptability using the Best Linear Unbiased Prediction (BLUP procedure). The results indicated that there were genotypes suitable for production under organic agriculture conditions with satisfactory yields (>4000 kg·ha(-1)). Furthermore, these genotypes showed high nutritional yield and nutrient density for the four minerals studied. Additionally, since these genotypes were stable and adaptable over three environmentally different years, they were designated "balanced genotypes" for the four minerals and for the aforementioned characteristics. Selection and breeding of such "balanced genotypes" may offer an alternative to producing nutritious food under low-input agriculture conditions. Furthermore, the type of evaluation presented here may also be of interest for implementation in research conducted in developing countries, following the objectives of producing enough nutrients for a growing population.')''')
pandas_mysql1.py
# -*- coding: utf-8 -*-
'''
Datetime: 2019/11/07
author: Zhang Yafei
description:
'''
from sqlalchemy import create_engine
import pandas as pd
import os
from multiprocessing import Pool
from mysql_helper import fetchall, insert engine = create_engine(
"mysql+pymysql://root:0000@127.0.0.1:3306/srld?charset=utf8mb4",
max_overflow=0, # 超过连接池大小外最多创建的连接
pool_size=5, # 连接池大小
pool_timeout=30, # 池中没有线程最多等待的时间，否则报错
pool_recycle=-1 # 多久之后对线程池中的线程进行一次连接的回收（重置）
)
conn = engine.connect() def filter_file(file_list):
sql = """ select file_path from data_records """
file_of_database = {file[0] for file in fetchall(sql, type=None)}
files = file_list - file_of_database
print(f'共{len(file_list)}\t已经导入{len(file_of_database)}\t还需导入:{len(files)}')
return files def main(file_path):
table_name = 'pubmed'
# df = pd.read_csv('xml副本/documents_1.txt',delimiter='\t', header=None)
try:
df = pd.read_csv(file_path, delimiter='\t', header=None)
df.columns = ['pmid_doi', 'author', 'journal', 'year', 'title', 'abstract']
df['path'] = file_path
df['year'] = '2017'
df.to_sql(table_name, conn, if_exists='append', index=False)
insert_sql = """ insert into data_records(file_path) values(%s) """
insert(sql=insert_sql, arg=[file_path])
print(f'{file_path}\t导入成功')
except Exception as e:
# insert_sql = """ insert into data_records(file_path, state, remarks) values(%s,%s,%s) """
# print(f'{file_path}文件导入失败')
# insert(sql=insert_sql, arg=[file_path,0,str(e)])
print(f'{file_path}文件导入失败') if __name__ == "__main__":
dir__names = ['data处理/800w-1600w副本', 'data处理/1600w-2600w副本', 'data处理/xmls_news']
# for dir_name in dir__names:
dir_name = dir__names[2]
file_list = {os.path.join(dir_name, file)
for file in os.listdir(dir_name)}
file_list = filter_file(file_list)
for file in file_list:
main(file_path=file) """
pool = Pool()
pool.map(main, file_list)
pool.join()
pool.close()
"""
pandas+mysql2
2.药物数据导入
from sqlalchemy import create_engine
import pandas as pd
import os engine = create_engine(
"mysql+pymysql://root:0000@127.0.0.1:3306/drug?charset=utf8",
max_overflow=0, # 超过连接池大小外最多创建的连接
pool_size=5, # 连接池大小
pool_timeout=30, # 池中没有线程最多等待的时间，否则报错
pool_recycle=-1 # 多久之后对线程池中的线程进行一次连接的回收（重置）
)
conn = engine.connect() def main(file_path, table_name='drugbank'):
df = pd.read_table(file_path, header=None)[:900]
df.columns = ['drugbank_id', 'drug_name']
df['drugbank_id'] = [f'DB1{num}' for num in range(4195,5095)]
# print(df)
df.to_sql(table_name, conn, if_exists='append',index=False)
print(f'{file_path}\t导入成功') if __name__ == "__main__":
main(file_path='drugs.txt')
drugbank_to_Sql.py
3.疾病数据库
# -*- coding: utf-8 -*-
'''
Datetime: 2019/11/08
author: Zhang Yafei
description:
'''
from sqlalchemy import create_engine
import pandas as pd
import os engine = create_engine(
"mysql+pymysql://root:0000@127.0.0.1:3306/disease?charset=utf8mb64",
max_overflow=0, # 超过连接池大小外最多创建的连接
pool_size=5, # 连接池大小
pool_timeout=30, # 池中没有线程最多等待的时间，否则报错
pool_recycle=-1 # 多久之后对线程池中的线程进行一次连接的回收（重置）
)
conn = engine.connect() def main(file_path, table_name='disease'):
df = pd.read_excel(file_path, index_col='#')
# df.columns = ['PMID_DOI', 'author', 'journal', 'year', 'title', 'abstract']
df.to_sql(table_name, conn, if_exists='append',index=False)
print(f'{file_path}\t导入成功') if __name__ == "__main__":
file_list = []
for base_path,dirs, files in os.walk('MalaCards'):
# print(base_path, file_path)
for file in files:
file_list.append(os.path.join(base_path, file))
for file in file_list:
main(file_path=file)
diease_to_sql.py
4.科研文献数据库
# -*- coding: utf-8 -*- """
@Datetime: 2018/12/26
@Author: Zhang Yafei
"""
import pymysql
from DBUtils.PooledDB import PooledDB
import pandas as pd
from pymysql.err import InternalError
import numpy as np POOL = PooledDB(
creator=pymysql, # 使用链接数据库的模块
maxconnections=6, # 连接池允许的最大连接数，0和None表示不限制连接数
mincached=2, # 初始化时，链接池中至少创建的空闲的链接，0表示不创建
maxcached=5, # 链接池中最多闲置的链接，0和None不限制
maxshared=3,
# 链接池中最多共享的链接数量，0和None表示全部共享。PS: 无用，因为pymysql和MySQLdb等模块的 threadsafety都为1，所有值无论设置为多少，_maxcached永远为0，所以永远是所有链接都共享。
blocking=True, # 连接池中如果没有可用连接后，是否阻塞等待。True，等待；False，不等待然后报错
maxusage=None, # 一个链接最多被重复使用的次数，None表示无限制
setsession=[], # 开始会话前执行的命令列表。如：["set datestyle to ...", "set time zone ..."]
ping=0,
# ping MySQL服务端，检查是否服务可用。# 如：0 = None = never, 1 = default = whenever it is requested, 2 = when a cursor is created, 4 = when a query is executed, 7 = always
host='127.0.0.1',
port=3306,
user='root',
password='0000',
database='srld',
charset='utf8mb4'
) def connect(type=None):
conn = POOL.connection()
cursor = conn.cursor(cursor=type)
return conn, cursor def connect_close(conn, cursor):
cursor.close()
conn.close() def fetchone(sql, arg=list()):
conn, cursor = connect(type)
cursor.execute(sql, arg)
data = cursor.fetchone()
connect_close(conn, cursor)
return data def fetchall(sql, arg=list(), type=pymysql.cursors.DictCursor):
conn, cursor = connect(type)
cursor.execute(sql, arg)
data = cursor.fetchall()
connect_close(conn, cursor)
return data def insert(sql, arg=list()):
conn, cursor = connect()
row = cursor.execute(sql, arg)
conn.commit()
connect_close(conn, cursor)
return row def insert_sql(row):
pmid_doi, author, journal, year, title, abstract = row sql = """
insert into pubmed(PMID_DOI,author,journal,year,title,abstract) values(%s, %s, %s, %s, %s, %s)
"""
try:
insert(sql, arg=[pmid_doi, author, journal, year, title, abstract])
except InternalError:
# print(pmid_doi, author, journal, year, title, abstract)
print(author, type(author), np.isnan(author))
# pmid_doi = None if np.isnan(pmid_doi) else pmid_doi
# print(author, type(author), np.isnan(author))
author = None if np.isnan(author) else author
print(author)
# journal = None if np.isnan(journal) else journa
# year = None if np.isnan(year) else year
# title = None if np.isnan(title) else title
# abstract = None if np.isnan(abstract) else abstract
# print(author)
# print(pmid_doi, author, journal, year, title, abstract) def main():
df = pd.read_csv('xml副本/documents_1.txt', delimiter='\t', header=None)
df.fillna(0, inplace=True)
df.apply(insert_sql, axis=1) if __name__ == "__main__":
main()
# insert('''insert into pubmed(PMID_DOI,author,journal,year,title,abstract) values('PMID:28231184|DOI:10.3390/foods5040089', 'Moreira-Ascarrunz, Sergio Daniel; Larsson, Hans; Prieto-Linde, Maria Luisa; Johansson, Eva', 'Foods (Basel, Switzerland)', '2016', 'Mineral Nutritional Yield and Nutrient Density of Locally Adapted Wheat Genotypes under Organic Production.','The aim of the present investigation was to investigate the nutritional yield, nutrient density, stability, and adaptability of organically produced wheat for sustainable and nutritional high value food production. This study evaluated the nutritional yield of four minerals (Fe, Zn, Cu, and Mg) in 19 wheat genotypes, selected as being locally adapted under organic agriculture conditions. The new metric of nutritional yield was calculated for each genotype and they were evaluated for stability using the Additive Main effects and Multiplicative Interaction (AMMI) stability analysis and for genotypic value, stability, and adaptability using the Best Linear Unbiased Prediction (BLUP procedure). The results indicated that there were genotypes suitable for production under organic agriculture conditions with satisfactory yields (>4000 kg·ha(-1)). Furthermore, these genotypes showed high nutritional yield and nutrient density for the four minerals studied. Additionally, since these genotypes were stable and adaptable over three environmentally different years, they were designated "balanced genotypes" for the four minerals and for the aforementioned characteristics. Selection and breeding of such "balanced genotypes" may offer an alternative to producing nutritious food under low-input agriculture conditions. Furthermore, the type of evaluation presented here may also be of interest for implementation in research conducted in developing countries, following the objectives of producing enough nutrients for a growing population.')''')
mysql_helper.py
# -*- coding: utf-8 -*-
'''
Datetime: 2019/11/07
author: Zhang Yafei
description:
'''
from sqlalchemy import create_engine
import pandas as pd
import os
from multiprocessing import Pool
from mysql_helper import fetchall, insert engine = create_engine(
"mysql+pymysql://root:0000@127.0.0.1:3306/srld?charset=utf8mb4",
max_overflow=0, # 超过连接池大小外最多创建的连接
pool_size=5, # 连接池大小
pool_timeout=30, # 池中没有线程最多等待的时间，否则报错
pool_recycle=-1 # 多久之后对线程池中的线程进行一次连接的回收（重置）
)
conn = engine.connect() def filter_file(file_list):
sql = """ select file_path from data_records """
file_of_database = {file[0] for file in fetchall(sql, type=None)}
files = file_list - file_of_database
print(f'共{len(file_list)}\t已经导入{len(file_of_database)}\t还需导入:{len(files)}')
return files def main(file_path):
table_name = 'pubmed'
# df = pd.read_csv('xml副本/documents_1.txt',delimiter='\t', header=None)
try:
df = pd.read_csv(file_path, delimiter='\t', header=None)
df.columns = ['pmid_doi', 'author', 'journal', 'year', 'title', 'abstract']
df['path'] = file_path
df['year'] = '2017'
df.to_sql(table_name, conn, if_exists='append', index=False)
insert_sql = """ insert into data_records(file_path) values(%s) """
insert(sql=insert_sql, arg=[file_path])
print(f'{file_path}\t导入成功')
except Exception as e:
# insert_sql = """ insert into data_records(file_path, state, remarks) values(%s,%s,%s) """
# print(f'{file_path}文件导入失败')
# insert(sql=insert_sql, arg=[file_path,0,str(e)])
print(f'{file_path}文件导入失败') if __name__ == "__main__":
dir__names = ['data处理/800w-1600w副本', 'data处理/1600w-2600w副本', 'data处理/xmls_news']
# for dir_name in dir__names:
dir_name = dir__names[2]
file_list = {os.path.join(dir_name, file)
for file in os.listdir(dir_name)}
file_list = filter_file(file_list)
for file in file_list:
main(file_path=file) """
pool = Pool()
pool.map(main, file_list)
pool.join()
pool.close()
"""
pubmed_data_to_sql.py
# -*- coding: utf-8 -*-
'''
Datetime: 2019/11/08
author: Zhang Yafei
description:
'''
from sqlalchemy import create_engine
import pandas as pd
import os
from mysql_helper import fetchall, insert engine = create_engine(
"mysql+pymysql://root:0000@127.0.0.1:3306/srld?charset=utf8mb4",
max_overflow=0, # 超过连接池大小外最多创建的连接
pool_size=5, # 连接池大小
pool_timeout=30, # 池中没有线程最多等待的时间，否则报错
pool_recycle=-1 # 多久之后对线程池中的线程进行一次连接的回收（重置）
)
conn = engine.connect() def filter_file(file_list):
file_list = set(file_list)
sql = """ select path from sci_records """
file_of_database = {file[0] for file in fetchall(sql, type=None)}
files = file_list - file_of_database
print(f'共{len(file_list)}\t已经导入{len(file_of_database)}\t还需导入:{len(files)}')
return files def main(file_path, table_name='sci_china'):
try:
df = pd.read_csv(file_path, delimiter='\t', encoding='utf-16', error_bad_lines=False)
df = df[['PT', 'CA', 'TI','HO','ID', 'AB','FX','PD','PY','VL','AR']]
df['path'] = 'China/2013'
df.rename({'PT': 'AU', 'CA': 'TI', 'TI':'SO','HO':'DE','ID':'AB','AB':'C1','FX':'CR', 'PD':'PY', 'PY':'VL','VL':'IS','AR':'DI'}, axis=1, inplace=True)
df['PY'] = '2013'
# print(df.head())
df.to_sql(table_name, conn, if_exists='append',index=False)
# insert_sql = """ insert into sci_records(path) values(%s) """
# insert(sql=insert_sql, arg=[file_path])
print(f'{file_path}\t导入成功')
except Exception as e:
print(f'{file_path}\t导入失败') if __name__ == "__main__":
file_list = []
for base_path,dirs, files in os.walk('China'):
for file in files:
if file.endswith('.txt'):
file_list.append(os.path.join(base_path, file))
# main(file_list[0])
# file_list = filter_file(file_list)
file_list = file_list[2000:2080]
for file in file_list:
main(file_path=file)
sci_data_to_sql.py
5.基因数据导入
# -*- coding: utf-8 -*-
'''
Datetime: 2019/11/08
author: Zhang Yafei
description:
'''
from sqlalchemy import create_engine
import pandas as pd
import os engine = create_engine(
"mysql+pymysql://root:0000@127.0.0.1:3306/gene?charset=utf8",
max_overflow=0, # 超过连接池大小外最多创建的连接
pool_size=5, # 连接池大小
pool_timeout=30, # 池中没有线程最多等待的时间，否则报错
pool_recycle=-1 # 多久之后对线程池中的线程进行一次连接的回收（重置）
)
conn = engine.connect() def main(file_path, table_name='arrayexpress'):
# try:
df = pd.read_csv(file_path, delimiter='\t', error_bad_lines=False, encoding='ISO8859-1')
df.to_sql(table_name, conn, if_exists='append',index=False)
# insert_sql = """ insert into sci_records(path) values(%s) """
# insert(sql=insert_sql, arg=[file_path])
print(f'{file_path}\t导入成功')
# except Exception as e:
# print(f'{file_path}\t导入失败') if __name__ == "__main__":
file_list = []
for base_path,dirs, files in os.walk('ArrayExpress数据'):
for file in files:
if file.endswith('.txt'):
file_list.append(os.path.join(base_path, file))
# file_list = filter_file(file_list)
for file in file_list:
main(file_path=file)
arrayexpress_to_sql.py
# -*- coding: utf-8 -*-
'''
Datetime: 2019/11/08
author: Zhang Yafei
description:
'''
from sqlalchemy import create_engine
import pandas as pd
import os engine = create_engine(
"mysql+pymysql://root:0000@127.0.0.1:3306/zacopride?charset=utf8",
max_overflow=0, # 超过连接池大小外最多创建的连接
pool_size=5, # 连接池大小
pool_timeout=30, # 池中没有线程最多等待的时间，否则报错
pool_recycle=-1 # 多久之后对线程池中的线程进行一次连接的回收（重置）
)
conn = engine.connect() def main(file_path):
table_name = file_path.split('\\')[-1].strip('.csv').strip().replace(' ', '_').lower()
df = pd.read_csv(file_path)
df.to_sql(table_name, conn, if_exists='append',index=False)
print(f'{file_path}\t导入成功') if __name__ == "__main__":
file_list = []
for base_path,dirs, files in os.walk('zacopride'):
# print(base_path, file_path)
for file in files:
file_list.append(os.path.join(base_path, file))
# print(file_list)
# main(file_list[0])
for file in file_list:
main(file_path=file)
gene_to_sql.py
# -*- coding: utf-8 -*-
'''
Datetime: 2019/11/08
author: Zhang Yafei
description:
'''
from sqlalchemy import create_engine
import pandas as pd
import os engine = create_engine(
"mysql+pymysql://root:0000@127.0.0.1:3306/gene?charset=utf8",
max_overflow=0, # 超过连接池大小外最多创建的连接
pool_size=5, # 连接池大小
pool_timeout=30, # 池中没有线程最多等待的时间，否则报错
pool_recycle=-1 # 多久之后对线程池中的线程进行一次连接的回收（重置）
)
conn = engine.connect() def main(file_path, table_name='series'):
# try:
df = pd.read_csv(file_path, error_bad_lines=False)
# print(df.columns)
# df['path'] = file_path
df.to_sql(table_name, conn, if_exists='append',index=False)
# insert_sql = """ insert into sci_records(path) values(%s) """
# insert(sql=insert_sql, arg=[file_path])
print(f'{file_path}\t导入成功')
# except Exception as e:
# print(f'{file_path}\t导入失败') if __name__ == "__main__":
file_list = []
for base_path,dirs, files in os.walk('series'):
for file in files:
if file.endswith('.csv'):
file_list.append(os.path.join(base_path, file))
for file in file_list:
main(file_path=file)
series_to_sql.py
Python操作数据库实战的更多相关文章
- Python接口测试实战3（上）- Python操作数据库
如有任何学习问题,可以添加作者微信:lockingfree 课程目录 Python接口测试实战1(上)- 接口测试理论 Python接口测试实战1(下)- 接口测试工具的使用 Python接口测试实战 ...
- python操作数据库
一,安装mysql 如果是windows 用户,mysql 的安装非常简单,直接下载安装文件,双击安装文件一步一步进行操作即可. Linux 下的安装可能会更加简单,除了下载安装包进行安装外,一般的l ...
- python操作数据库之批量导入
python操作数据库之批量导入 Python语法简洁清晰,特色之一是强制用空白符(white space)作为语句缩进. Python具有丰富和强大的库.它常被昵称为胶水语言,能够把用其他语言制作的 ...
- 零基础学Python--------第11章 使用Python操作数据库
第11章 使用Python操作数据库 11.1 数据库编程接口 在项目开发中,数据库应用必不可少.虽然数据库的种类有很多,如SQLite.MySQL.Oracle等,但是它们的功能基本都是一样的,为了 ...
- python操作数据库PostgreSQL
1.简述 python可以操作多种数据库,诸如SQLite.MySql.PostgreSQL等,这里不对所有的数据库操作方法进行赘述,只针对目前项目中用到的PostgreSQL做一下简单介绍,主要包括 ...
- Python操作数据库之 MySQL
Python操作数据库之MySQL 一.安装Python-MySQLdb模块 Python-MySQLdb是一个操作数据库的模块,Python 通过它对 mysql 数据实现各种操作. 如果要源码安装 ...
- Python学习笔记 - day11 - Python操作数据库
MySQL的事务 MySQL的事务支持不是绑定在MySQL服务器本身,而是与存储引擎相关,MySQL的两种引擎如下: 1.MyISAM:不支持事务,用于只读程序提高性能 2.InnoDB:支持ACID ...
- 孤荷凌寒自学python第四十四天Python操作 数据库之准备工作
孤荷凌寒自学python第四十四天Python操作数据库之准备工作 (完整学习过程屏幕记录视频地址在文末,手写笔记在文末) 今天非常激动地开始接触Python的数据库操作的学习了,数据库是系统化设计 ...
- Python操作数据库及hashlib模块
一.hashlib模块 hashlib模块,主要用于加密相关的操作,在python3的版本里,代替了md5和sha模块,主要提供 SHA1, SHA224, SHA256, SHA384, SHA51 ...
随机推荐
- 使用wesocket从 rabbitMQ获取实时数据
rabbitmq支持stomp组件,通过stomp组件和websocket可以从rabbitMQ获取实时数据.这里分享一个demo: 使用时需要引入的js ,用到了sock.js和stomp.js & ...
- 04 - Vue3 UI Framework - 文档页
官网的首页做完了,接下来开始做官网的文档页 返回阅读列表点击 这里 路由设计 先想想我们需要文档页通向哪些地方,这里直接给出我的设计: 所属 子标题 跳转路径 文件名(*.vue) 指南 介绍 /do ...
- 9、Redis五大数据类型---有序集合Zset(sorted set)
一.简介 zset与set异同 相同之处: 都是没有重复元素的字符串集合 不同之处: 有序集合zset的每个成员都关联了一个评分(score),这个评分(score)被用来按照从最低分到最高分的方式排 ...
- 在写易买网时产生的错误 JSTL标签库中<c:choose></c:choose>不能放JSP页面<!-- -->注释
最近在使用JSTL标签库的<c:choose>标签时候,发现在该标签体中加了JSP的<!-- -->注释时,总是会显示报错信息.错误的信息如下: org.apache.jasp ...
- pipeline配置java项目
pipeline { agent { label 'slave' } options { timestamps() disableConcurrentBuilds() buildDiscarder( ...
- 追踪工具strace
目录 一.简介 二.例子 找出程序读取的文件 查看现在程序在做什么 查看程序中哪个部分消耗大量cpu 无法连接服务器查看信息 一.简介 Linux 操作系统有很多用来跟踪程序行为的工具,内核态的函数调 ...
- 原来这才是 Socket ！
关于对 Socket 的认识,大致分为下面几个主题,Socket 是什么,Socket 是如何创建的,Socket 是如何连接并收发数据的,Socket 套接字的删除等. Socket 是什么以及创建 ...
- 禁用copy on write实现全局EAT HOOK
以前写过一个,但是一不小心删除了,哎,就当再次复习复习吧. 首先抛出一个有意思的问题: 已知所有Windows可执行文件exe都会链接子系统ntdll.dll,那么真实内存中有几份ntdll.dll? ...
- Spring核心原理分析之MVC九大组件（1）
本文节选自<Spring 5核心原理> 1 什么是Spring MVC Spring MVC 是 Spring 提供的一个基于 MVC 设计模式的轻量级 Web 开发框架,本质上相当于 S ...
- 前端文本框插件KindEditor
KindEditor 1.进入官网 2.下载 官网下载:http://kindeditor.net/down.php 本地下载:http://files.cnblogs.com/files/wup ...