Python操作数据库实战
pymysql
# -*- coding: utf-8 -*- """
@Datetime: 2018/12/26
@Author: Zhang Yafei
"""
import pymysql
from DBUtils.PooledDB import PooledDB POOL = PooledDB(
creator=pymysql, # 使用链接数据库的模块
maxconnections=6, # 连接池允许的最大连接数，0和None表示不限制连接数
mincached=2, # 初始化时，链接池中至少创建的空闲的链接，0表示不创建
maxcached=5, # 链接池中最多闲置的链接，0和None不限制
maxshared=3,
# 链接池中最多共享的链接数量，0和None表示全部共享。PS: 无用，因为pymysql和MySQLdb等模块的 threadsafety都为1，所有值无论设置为多少，_maxcached永远为0，所以永远是所有链接都共享。
blocking=True, # 连接池中如果没有可用连接后，是否阻塞等待。True，等待；False，不等待然后报错
maxusage=None, # 一个链接最多被重复使用的次数，None表示无限制
setsession=[], # 开始会话前执行的命令列表。如：["set datestyle to ...", "set time zone ..."]
ping=0,
# ping MySQL服务端，检查是否服务可用。# 如：0 = None = never, 1 = default = whenever it is requested, 2 = when a cursor is created, 4 = when a query is executed, 7 = always
host='127.0.0.1',
port=3306,
user='root',
password='0000',
database='flask_code',
charset='utf8'
) def connect(type=None):
conn = POOL.connection()
cursor = conn.cursor(cursor=type)
return conn, cursor def connect_close(conn, cursor):
cursor.close()
conn.close() def fetchone(sql, arg=list()):
conn, cursor = connect(type)
cursor.execute(sql, arg)
data = cursor.fetchone()
connect_close(conn, cursor)
return data def fetchall(sql, arg=list(), type=pymysql.cursors.DictCursor):
conn, cursor = connect(type)
cursor.execute(sql, arg)
data = cursor.fetchall()
connect_close(conn, cursor)
return data def insert(sql, arg=list()):
conn, cursor = connect()
row = cursor.execute(sql, arg)
conn.commit()
connect_close(conn, cursor)
return row
mysql_helper
sqlite
# -*- coding: utf-8 -*- """
@Datetime: 2019/1/31
@Author: Zhang Yafei
"""
import sqlite3
import settings
import os BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DB_DIR = os.path.join(BASE_DIR, 'data.db') def connect():
'''连接数据库'''
conn = sqlite3.connect(settings.DATABASES.get('sqlite3')) # db不存在时将自动创建db
cursor = conn.cursor()
return conn, cursor def connect_close(conn, cursor):
"""关闭连接"""
cursor.close()
conn.close() def execute(sql, params=tuple()):
conn, cursor = connect()
cursor.execute(sql, params) # 执行这个语句
connect_close(conn, cursor) def fetchone(sql, params=tuple()):
conn, cursor = connect()
result = cursor.execute(sql, params)
data = result.fetchone()
connect_close(conn, cursor)
return data def fetchall(sql, params=tuple()):
conn, cursor = connect()
results = cursor.execute(sql, params)
data = results.fetchall()
connect_close(conn, cursor)
return data
sqlite3_helper
# -*- coding: utf-8 -*- """
@Datetime: 2019/1/31
@Author: Zhang Yafei
"""
import sqlite3
import os BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DB_DIR = os.path.join(BASE_DIR, 'data.db') class SqliteDB(object):
def __init__(self):
self.conn = sqlite3.connect(DB_DIR) # db不存在时将自动创建db
self.cursor = self.conn.cursor() def close(self):
self.cursor.close()
self.conn.close() def execute(self, sql, params=tuple()):
self.cursor.execute(sql, params)
self.close() def fetchone(self, sql, params=tuple()):
result = self.cursor.execute(sql, params)
data = result.fetchone()
self.close()
return data def fetchall(self, sql, params=tuple()):
results = self.cursor.execute(sql, params)
data = results.fetchall()
self.close()
return data if __name__ == '__main__':
sqlite = SqliteDB()
# 1. 建表
sql = '''create table happy(
username text,
password text,
id int)'''
sqlite.execute(sql) # 2. 插入数据
sqlite.execute("INSERT INTO COMPANY (ID,NAME,AGE,ADDRESS,SALARY) \
VALUES (4, 'Mark', 25, 'Rich-Mond ', 65000.00 )") # 3. 更改数据
sqlite.execute("UPDATE COMPANY SET ID=99 WHERE ID=2") # 4. 删除表里面的数据
c.execute("DELETE FROM COMPANY WHERE ID=4")
c.execute("DELETE FROM COMPANY WHERE ID=3") # 5. 查询
data = sqlite.fetchall('select * from label limit 1')
print(data)
# 输出
'''
[('盘龙云海(排毒养颜胶囊)', 509881, '广东深圳龙岗区/女', '昨天吃的，今天就拉肚子了。感觉肚子有点涨痛！不知道效果怎么样~~~~~',
'昨天/吃/的/，/今天/就/拉肚子/SB了/。/感觉/肚子/PB有点/涨痛/SB！/不/知道/效果/怎么样/~/~/~/~/~', '2011-09-30 15:26:00',
'http://ypk.39.net/509881/comment/k0_p...', '昨天/吃/的/，/今天/就/拉肚子/SB了/。/感觉/肚子/PB有点/涨痛/SB！/不/知道/效果/怎么样/~/~/~/~/~',
'昨天/吃/的/，/今天/就/拉肚子/SB了/。/感觉/肚子/PB有点/涨痛/SB！/不/知道/效果/怎么样/~/~/~/~/~')]
'''
sqlite3_helper2
from DBUtils.PooledDB import PooledDB
from settings import DATABASE
import threading lock = threading.Lock() class SqliteHelper(object):
def __init__(self, db_config):
"""
# sqlite3
# 连接数据库文件名，sqlite不支持加密，不使用用户名和密码
import sqlite3
config = {"database": "path/to/your/dbname.db"}
pool = PooledDB(sqlite3, maxcached=50, maxconnections=1000, maxusage=1000, **config)
# mysql
import pymysql
pool = PooledDB(pymysql,5,host='localhost', user='root',passwd='pwd',db='myDB',port=3306) #5为连接池里的最少连接数
# postgressql
import psycopg2
POOL = PooledDB(creator=psycopg2, host="127.0.0.1", port="5342", user, password, database)
# sqlserver
import pymssql
pool = PooledDB(creator=pymssql, host=host, port=port, user=user, password=password, database=database, charset="utf8")
:param type:
"""
self.pool = PooledDB(**db_config)
self.conn = self.pool.connection()
self.cursor = self.conn.cursor() def connect(self):
conn = self.pool.connection()
cursor = self.conn.cursor()
return conn, cursor @staticmethod
def connect_close(conn, cursor):
"""关闭连接"""
conn.close()
cursor.close() def __del__(self):
self.cursor.close()
self.conn.close() def execute(self, sql, params=tuple()):
try:
lock.acquire(True)
self.cursor.execute(sql, params) # 执行这个语句
self.conn.commit()
lock.release()
except Exception as e:
print(e) def execute_many(self, sql, params=tuple()):
try:
lock.acquire(True)
self.cursor.executemany(sql, params)
self.conn.commit()
lock.release()
except Exception as e:
print(e) def fetchone(self, sql, params=tuple()):
try:
lock.acquire(True)
self.cursor.execute(sql, params)
data = self.cursor.fetchone()
lock.release()
return data
except Exception as e:
print(e) def fetchall(self, sql, params=tuple()):
try:
lock.acquire(True)
self.cursor.execute(sql, params)
data = self.cursor.fetchall()
lock.release()
return data
except Exception as e:
print(e)
SqliteHelper3.py
mongodb
# -*- coding: utf-8 -*- """
@Datetime: 2019/1/31
@Author: Zhang Yafei
"""
import json
import pymongo
import pandas as pd class MongoPipeline(object):
"""
mongodb:
save(self, data, collection): 将数据保存到数据库
read(self, data): 读取数据库中指定表格
insert(self, table, dict_data)： 插入数据
delete(self, table, condition)： 删除指定数据
update(self, table, condition, new_dict_data)： 更新指定数据
dbFind(self, table, condition=None): 按条件查找
findAll(self, table)： 查找全部
close(self)： 关闭连接
""" def __init__(self, mongo_db, mongo_uri='localhost'):
self.mongo_uri = mongo_uri
self.mongo_db = mongo_db
self.client = pymongo.MongoClient(self.mongo_uri)
self.db = self.client[self.mongo_db] def close(self):
"""
关闭连接
:return:
"""
self.client.close() def save(self, data, collection):
"""
将数据保存到数据库表
:param data:
:param collection:
:return: None
"""
self.collection = self.db[collection]
try:
if self.collection.insert(json.loads(data.T.to_json()).values()):
print('mongodb insert {} sucess.'.format(collection))
return
except Exception as e:
print('insert error:', e)
import traceback
traceback.print_exc(e) def read(self, table):
"""
读取数据库中的数据
:param table:
:return: dataframe
"""
try:
# 连接数据库
table = self.db[table]
# 读取数据
data = pd.DataFrame(list(table.find()))
return data
except Exception as e:
import traceback
traceback.print_exc(e) def insert(self, table, dict_data):
"""
插入
:param table:
:param dict_data:
:return: None
"""
try:
self.db[table].insert(dict_data)
print("插入成功")
except Exception as e:
print(e) def update(self,table, condition, new_dict_data):
"""
更新
:param table:
:param dict_data:
:param new_dict_data:
:return: None
"""
try:
self.db[table].update(condition, new_dict_data)
print("更新成功")
except Exception as e:
print(e) def delete(self,table, condition):
"""
删除
:param table:
:param dict_data:
:return: None
"""
try:
self.db[table].remove(condition)
print("删除成功")
except Exception as e:
print(e) def dbFind(self, table, condition=None):
"""
按条件查找
:param table:
:param dict_data:
:return: generator dict
"""
data = self.db[table].find(condition)
for item in data:
yield item def findAll(self, table):
"""
查找全部
:param table:
:return: generator dict
"""
for item in self.db[table].find():
yield item if __name__ == '__main__':
mongo = MongoPipeline('flask')
# data = mongo.read('label')
# print(data.head())
condition = {"药品ID": 509881}
data = mongo.dbFind('label', condition)
print(data)
for i in data:
print(i)
# mongo.findAll()
mongo_helper
postgresql
# -*- coding: utf-8 -*-
'''
Datetime: 2019/10/24
author: Zhang Yafei
description: python操作 postgressql数据库
操作流程是：
1）连接PostgreSQL并设定Cursor
2）执行SQL脚本 #有几点需要注意
- sql脚本必须以';'结尾，不可以省略
- 不管sql中有几个参数，都需要用%s代替，只有%s, 不管值是字符还是数字, 一律%s.
- 第二个参数中，一定要传入元组，上述例子中(5)是不行的
- 执行完INSERT, UPDATE, DELETE这样的sql脚本后, 需要conn.commit()提交一下, 才会把数据提交到数据库当中.
3）得到查询结果 可以采用fetchone(), fetchall()和fetchmany() 三种方式。fetchone返回一个tuple或者None, fetchall返回一个
list of tuple，如果没有结果则返回一个空的tuple。fetchmany返回list of tuple， list的长度由size参数决定，
size的默认值是cursor.arraysize， 如果没有结果可以返回，那么返回一个空的list。
'''
import psycopg2 def connect():
conn = psycopg2.connect("dbname=xiaomuchong user=postgres password=0000 host=localhost port=5432")
cur = conn.cursor()
return conn, cur def connect_close(conn, cursor):
"""关闭连接"""
cursor.close()
conn.close() def execute(sql, params=tuple()):
conn, cursor = connect()
cursor.execute(sql, params) # 执行这个语句
conn.commit()
connect_close(conn, cursor) def fetchone(sql, params=tuple()):
conn, cursor = connect()
result = cursor.execute(sql, params)
data = result.fetchone()
connect_close(conn, cursor)
return data def fetchall(sql, params=tuple()):
conn, cursor = connect()
cursor.execute(sql, params)
data = cursor.fetchall()
connect_close(conn, cursor)
return data if __name__ == "__main__":
# create_sql = "CREATE TABLE Employee (name varchar(80), address varchar(80),age int, date date)"
# execute(create_sql) # execute("INSERT INTO Employee VALUES('Gopher', 'China Beijing', 100, '2017-05-27')")
# execute("INSERT INTO Employee VALUES('Zhang', 'China jincheng', 24, '2019-10-24')") # rows = fetchall("SELECT * FROM Employee;")
# for row in rows:
# print('name=' + str(row[0]) + ' address=' + str(row[1]) +
# ' age=' + str(row[2]) + ' date=' + str(row[3])) # execute("UPDATE Employee SET age=12 WHERE name='Gopher'") execute("DELETE FROM Employee WHERE name='Gopher'")
postgresql_helper
# -*- coding: utf-8 -*-
'''
Datetime: 2019/10/24
author: Zhang Yafei
description: python操作 postgressql数据库
操作流程是：
1）连接PostgreSQL并设定Cursor
2）执行SQL脚本 #有几点需要注意
- sql脚本必须以';'结尾，不可以省略
- 不管sql中有几个参数，都需要用%s代替，只有%s, 不管值是字符还是数字, 一律%s.
- 第二个参数中，一定要传入元组，上述例子中(5)是不行的
- 执行完INSERT, UPDATE, DELETE这样的sql脚本后, 需要conn.commit()提交一下, 才会把数据提交到数据库当中.
3）得到查询结果 可以采用fetchone(), fetchall()和fetchmany() 三种方式。fetchone返回一个tuple或者None, fetchall返回一个
list of tuple，如果没有结果则返回一个空的tuple。fetchmany返回list of tuple， list的长度由size参数决定，
size的默认值是cursor.arraysize， 如果没有结果可以返回，那么返回一个空的list。
'''
import psycopg2 class PostgresSql(object):
def __init__(self, dbname, user, password, host='localhost', port=5432):
self.conn = psycopg2.connect(
f"dbname={dbname} user={user} password={password} host={host} port={port}")
self.cursor = self.conn.cursor() def connect_close(self):
"""关闭连接"""
self.cursor.close()
self.conn.close() def execute(self, sql, params=tuple()):
self.cursor.execute(sql, params) # 执行这个语句
self.conn.commit() def fetchone(self, sql, params=tuple()):
self.cursor.execute(sql, params)
data = self.cursor.fetchone()
return data def fetchall(self, sql, params=tuple()):
self.cursor.execute(sql, params)
data = self.cursor.fetchall()
return data if __name__ == "__main__":
postgres = PostgresSql(dbname='xiaomuchong', user='postgres', password='0000', host='localhost', port='5432')
# create_sql = "CREATE TABLE Employee1 (name varchar(80), address varchar(80),age int, date date)"
# postgres.execute(create_sql) # postgres.execute("INSERT INTO Employee1 VALUES('Gopher', 'China Beijing', 100, '2017-05-27')")
# postgres.execute("INSERT INTO Employee1 VALUES('Zhang', 'China jincheng', 24, '2019-10-24')") rows = postgres.fetchall("SELECT * FROM Employee1;")
for row in rows:
print('name=' + str(row[0]) + ' address=' + str(row[1]) +
' age=' + str(row[2]) + ' date=' + str(row[3])) postgres.execute("UPDATE Employee1 SET age=12 WHERE name='Gopher'")
postgres.execute("DELETE FROM Employee1 WHERE name='Gopher'") rows = postgres.fetchall("SELECT * FROM Employee1;")
for row in rows:
print('name=' + str(row[0]) + ' address=' + str(row[1]) +
' age=' + str(row[2]) + ' date=' + str(row[3])) postgres.connect_close()
postgresql_helper_class
from DBUtils.PooledDB import PooledDB class DBPoolHelper(object):
def __init__(self, dbname, user=None, password=None, db_type='postgressql', host='localhost', port=5432):
"""
# sqlite3
# 连接数据库文件名，sqlite不支持加密，不使用用户名和密码
import sqlite3
config = {"datanase": "path/to/your/dbname.db"}
pool = PooledDB(sqlite3, maxcached=50, maxconnections=1000, maxusage=1000, **config)
# mysql
import pymysql
pool = PooledDB(pymysql,5,host='localhost', user='root',passwd='pwd',db='myDB',port=3306) #5为连接池里的最少连接数
# postgressql
import psycopg2
POOL = PooledDB(creator=psycopg2, host="127.0.0.1", port="5342", user, password, database)
# sqlserver
import pymssql
pool = PooledDB(creator=pymssql, host=host, port=port, user=user, password=password, database=database, charset="utf8")
:param type:
"""
if db_type == 'postgressql':
import psycopg2
pool = PooledDB(creator=psycopg2, host=host, port=port, user=user, password=password, database=dbname)
elif db_type == 'mysql':
import pymysql
pool = PooledDB(pymysql, 5, host='localhost', user='root', passwd='pwd', db='myDB',
port=3306) # 5为连接池里的最少连接数
elif db_type == 'sqlite':
import sqlite3
config = {"database": dbname}
pool = PooledDB(sqlite3, maxcached=50, maxconnections=1000, maxusage=1000, **config)
else:
raise Exception('请输入正确的数据库类型, db_type="postgresql" or db_type="mysql" or db_type="sqlite"')
self.conn = pool.connection()
self.cursor = self.conn.cursor() def connect_close(self):
"""关闭连接"""
self.cursor.close()
self.conn.close() def execute(self, sql, params=tuple()):
self.cursor.execute(sql, params) # 执行这个语句
self.conn.commit() def execute_many(self, sql, params=tuple()):
self.cursor.executemany(sql, params)
self.conn.commit() def fetchone(self, sql, params=tuple()):
self.cursor.execute(sql, params)
data = self.cursor.fetchone()
return data def fetchall(self, sql, params=tuple()):
self.cursor.execute(sql, params)
data = self.cursor.fetchall()
return data
postgresql_helper3.py
redis
def redis_init():
settings = get_project_settings()
if settings["REDIS_PARAMS"]:
pool = ConnectionPool(host=settings["REDIS_HOST"], port=settings["REDIS_PORT"],
password=settings["REDIS_PARAMS"]['password'])
else:
pool = ConnectionPool(host=settings["REDIS_HOST"], port=settings["REDIS_PORT"])
conn = Redis(connection_pool=pool)
return conn if __name__ == '__main__':
redis_conn = redis_init()
redis_helper.py
DBUtils
# -*- coding: utf-8 -*- """
Datetime: 2020/07/02
Author: Zhang Yafei
Description:
"""
import time
from functools import wraps
from DBUtils.PooledDB import PooledDB def timeit(func):
"""
装饰器： 判断函数执行时间
:param func:
:return:
""" @wraps(func)
def inner(*args, **kwargs):
start = time.time()
ret = func(*args, **kwargs)
end = time.time() - start
if end < 60:
print(f'花费时间：\t{round(end, 2)}秒')
else:
min, sec = divmod(end, 60)
print(f'花费时间\t{round(min)}分\t{round(sec, 2)}秒')
return ret return inner class DBPoolHelper(object):
def __init__(self, dbname, user=None, password=None, db_type='postgressql', host='localhost', port=5432):
"""
# sqlite3
# 连接数据库文件名，sqlite不支持加密，不使用用户名和密码
import sqlite3
config = {"datanase": "path/to/your/dbname.db"}
pool = PooledDB(sqlite3, maxcached=50, maxconnections=1000, maxusage=1000, **config)
# mysql
import pymysql
pool = PooledDB(pymysql,5,host='localhost', user='root',passwd='pwd',db='myDB',port=3306) #5为连接池里的最少连接数
# postgressql
import psycopg2
POOL = PooledDB(creator=psycopg2, host="127.0.0.1", port="5342", user, password, database)
# sqlserver
import pymssql
pool = PooledDB(creator=pymssql, host=host, port=port, user=user, password=password, database=database, charset="utf8")
:param type:
"""
if db_type == 'postgressql':
import psycopg2
pool = PooledDB(creator=psycopg2, host=host, port=port, user=user, password=password, database=dbname)
elif db_type == 'mysql':
import pymysql
pool = PooledDB(pymysql, 5, host='localhost', user='root', passwd='pwd', db='myDB',
port=3306) # 5为连接池里的最少连接数
elif db_type == 'sqlite':
import sqlite3
config = {"database": dbname}
pool = PooledDB(sqlite3, maxcached=50, maxconnections=1000, maxusage=1000, **config)
else:
raise Exception('请输入正确的数据库类型, db_type="postgresql" or db_type="mysql" or db_type="sqlite"')
self.conn = pool.connection()
self.cursor = self.conn.cursor() def connect_close(self):
"""关闭连接"""
self.cursor.close()
self.conn.close() def execute(self, sql, params=tuple()):
self.cursor.execute(sql, params) # 执行这个语句
self.conn.commit() def execute_many(self, sql, params=tuple()):
self.cursor.executemany(sql, params)
self.conn.commit() def fetchone(self, sql, params=tuple()):
self.cursor.execute(sql, params)
data = self.cursor.fetchone()
return data def fetchall(self, sql, params=tuple()):
self.cursor.execute(sql, params)
data = self.cursor.fetchall()
return data
DBHelper.py
pandas链接数据库
import pandas as pd
from sqlalchemy import create_engine def pandas_db_helper():
"""
'postgresql://postgres:0000@127.0.0.1:5432/xiaomuchong'
"mysql+pymysql://root:0000@127.0.0.1:3306/srld?charset=utf8mb4"
"sqlite: ///sqlite3.db"
"""
engine = create_engine( 'postgresql://postgres:0000@127.0.0.1:5432/xiaomuchong')
conn = engine.connect()
return conn if __name__ == '__main__':
db = pandas_db_helper()
data = pd.read_sql_table("team", con=db, index_col="board_id")
print(data)
pandas_db_helper.py
实战项目
1.pandas+mysql
# -*- coding: utf-8 -*- """
@Datetime: 2018/12/26
@Author: Zhang Yafei
"""
import pymysql
from DBUtils.PooledDB import PooledDB
import pandas as pd
from pymysql.err import InternalError
import numpy as np POOL = PooledDB(
creator=pymysql, # 使用链接数据库的模块
maxconnections=6, # 连接池允许的最大连接数，0和None表示不限制连接数
mincached=2, # 初始化时，链接池中至少创建的空闲的链接，0表示不创建
maxcached=5, # 链接池中最多闲置的链接，0和None不限制
maxshared=3,
# 链接池中最多共享的链接数量，0和None表示全部共享。PS: 无用，因为pymysql和MySQLdb等模块的 threadsafety都为1，所有值无论设置为多少，_maxcached永远为0，所以永远是所有链接都共享。
blocking=True, # 连接池中如果没有可用连接后，是否阻塞等待。True，等待；False，不等待然后报错
maxusage=None, # 一个链接最多被重复使用的次数，None表示无限制
setsession=[], # 开始会话前执行的命令列表。如：["set datestyle to ...", "set time zone ..."]
ping=0,
# ping MySQL服务端，检查是否服务可用。# 如：0 = None = never, 1 = default = whenever it is requested, 2 = when a cursor is created, 4 = when a query is executed, 7 = always
host='127.0.0.1',
port=3306,
user='root',
password='0000',
database='srld',
charset='utf8mb4'
) def connect(type=None):
conn = POOL.connection()
cursor = conn.cursor(cursor=type)
return conn, cursor def connect_close(conn, cursor):
cursor.close()
conn.close() def fetchone(sql, arg=list()):
conn, cursor = connect(type)
cursor.execute(sql, arg)
data = cursor.fetchone()
connect_close(conn, cursor)
return data def fetchall(sql, arg=list(), type=pymysql.cursors.DictCursor):
conn, cursor = connect(type)
cursor.execute(sql, arg)
data = cursor.fetchall()
connect_close(conn, cursor)
return data def insert(sql, arg=list()):
conn, cursor = connect()
row = cursor.execute(sql, arg)
conn.commit()
connect_close(conn, cursor)
return row def insert_sql(row):
pmid_doi, author, journal, year, title, abstract = row sql = """
insert into pubmed(PMID_DOI,author,journal,year,title,abstract) values(%s, %s, %s, %s, %s, %s)
"""
try:
insert(sql, arg=[pmid_doi, author, journal, year, title, abstract])
except InternalError:
# print(pmid_doi, author, journal, year, title, abstract)
print(author, type(author), np.isnan(author))
# pmid_doi = None if np.isnan(pmid_doi) else pmid_doi
# print(author, type(author), np.isnan(author))
author = None if np.isnan(author) else author
print(author)
# journal = None if np.isnan(journal) else journa
# year = None if np.isnan(year) else year
# title = None if np.isnan(title) else title
# abstract = None if np.isnan(abstract) else abstract
# print(author)
# print(pmid_doi, author, journal, year, title, abstract) def main():
df = pd.read_csv('xml副本/documents_1.txt', delimiter='\t', header=None)
df.fillna(0, inplace=True)
df.apply(insert_sql, axis=1) if __name__ == "__main__":
main()
# insert('''insert into pubmed(PMID_DOI,author,journal,year,title,abstract) values('PMID:28231184|DOI:10.3390/foods5040089', 'Moreira-Ascarrunz, Sergio Daniel; Larsson, Hans; Prieto-Linde, Maria Luisa; Johansson, Eva', 'Foods (Basel, Switzerland)', '2016', 'Mineral Nutritional Yield and Nutrient Density of Locally Adapted Wheat Genotypes under Organic Production.','The aim of the present investigation was to investigate the nutritional yield, nutrient density, stability, and adaptability of organically produced wheat for sustainable and nutritional high value food production. This study evaluated the nutritional yield of four minerals (Fe, Zn, Cu, and Mg) in 19 wheat genotypes, selected as being locally adapted under organic agriculture conditions. The new metric of nutritional yield was calculated for each genotype and they were evaluated for stability using the Additive Main effects and Multiplicative Interaction (AMMI) stability analysis and for genotypic value, stability, and adaptability using the Best Linear Unbiased Prediction (BLUP procedure). The results indicated that there were genotypes suitable for production under organic agriculture conditions with satisfactory yields (>4000 kg·ha(-1)). Furthermore, these genotypes showed high nutritional yield and nutrient density for the four minerals studied. Additionally, since these genotypes were stable and adaptable over three environmentally different years, they were designated "balanced genotypes" for the four minerals and for the aforementioned characteristics. Selection and breeding of such "balanced genotypes" may offer an alternative to producing nutritious food under low-input agriculture conditions. Furthermore, the type of evaluation presented here may also be of interest for implementation in research conducted in developing countries, following the objectives of producing enough nutrients for a growing population.')''')
pandas_mysql1.py
# -*- coding: utf-8 -*-
'''
Datetime: 2019/11/07
author: Zhang Yafei
description:
'''
from sqlalchemy import create_engine
import pandas as pd
import os
from multiprocessing import Pool
from mysql_helper import fetchall, insert engine = create_engine(
"mysql+pymysql://root:0000@127.0.0.1:3306/srld?charset=utf8mb4",
max_overflow=0, # 超过连接池大小外最多创建的连接
pool_size=5, # 连接池大小
pool_timeout=30, # 池中没有线程最多等待的时间，否则报错
pool_recycle=-1 # 多久之后对线程池中的线程进行一次连接的回收（重置）
)
conn = engine.connect() def filter_file(file_list):
sql = """ select file_path from data_records """
file_of_database = {file[0] for file in fetchall(sql, type=None)}
files = file_list - file_of_database
print(f'共{len(file_list)}\t已经导入{len(file_of_database)}\t还需导入:{len(files)}')
return files def main(file_path):
table_name = 'pubmed'
# df = pd.read_csv('xml副本/documents_1.txt',delimiter='\t', header=None)
try:
df = pd.read_csv(file_path, delimiter='\t', header=None)
df.columns = ['pmid_doi', 'author', 'journal', 'year', 'title', 'abstract']
df['path'] = file_path
df['year'] = '2017'
df.to_sql(table_name, conn, if_exists='append', index=False)
insert_sql = """ insert into data_records(file_path) values(%s) """
insert(sql=insert_sql, arg=[file_path])
print(f'{file_path}\t导入成功')
except Exception as e:
# insert_sql = """ insert into data_records(file_path, state, remarks) values(%s,%s,%s) """
# print(f'{file_path}文件导入失败')
# insert(sql=insert_sql, arg=[file_path,0,str(e)])
print(f'{file_path}文件导入失败') if __name__ == "__main__":
dir__names = ['data处理/800w-1600w副本', 'data处理/1600w-2600w副本', 'data处理/xmls_news']
# for dir_name in dir__names:
dir_name = dir__names[2]
file_list = {os.path.join(dir_name, file)
for file in os.listdir(dir_name)}
file_list = filter_file(file_list)
for file in file_list:
main(file_path=file) """
pool = Pool()
pool.map(main, file_list)
pool.join()
pool.close()
"""
pandas+mysql2
2.药物数据导入
from sqlalchemy import create_engine
import pandas as pd
import os engine = create_engine(
"mysql+pymysql://root:0000@127.0.0.1:3306/drug?charset=utf8",
max_overflow=0, # 超过连接池大小外最多创建的连接
pool_size=5, # 连接池大小
pool_timeout=30, # 池中没有线程最多等待的时间，否则报错
pool_recycle=-1 # 多久之后对线程池中的线程进行一次连接的回收（重置）
)
conn = engine.connect() def main(file_path, table_name='drugbank'):
df = pd.read_table(file_path, header=None)[:900]
df.columns = ['drugbank_id', 'drug_name']
df['drugbank_id'] = [f'DB1{num}' for num in range(4195,5095)]
# print(df)
df.to_sql(table_name, conn, if_exists='append',index=False)
print(f'{file_path}\t导入成功') if __name__ == "__main__":
main(file_path='drugs.txt')
drugbank_to_Sql.py
3.疾病数据库
# -*- coding: utf-8 -*-
'''
Datetime: 2019/11/08
author: Zhang Yafei
description:
'''
from sqlalchemy import create_engine
import pandas as pd
import os engine = create_engine(
"mysql+pymysql://root:0000@127.0.0.1:3306/disease?charset=utf8mb64",
max_overflow=0, # 超过连接池大小外最多创建的连接
pool_size=5, # 连接池大小
pool_timeout=30, # 池中没有线程最多等待的时间，否则报错
pool_recycle=-1 # 多久之后对线程池中的线程进行一次连接的回收（重置）
)
conn = engine.connect() def main(file_path, table_name='disease'):
df = pd.read_excel(file_path, index_col='#')
# df.columns = ['PMID_DOI', 'author', 'journal', 'year', 'title', 'abstract']
df.to_sql(table_name, conn, if_exists='append',index=False)
print(f'{file_path}\t导入成功') if __name__ == "__main__":
file_list = []
for base_path,dirs, files in os.walk('MalaCards'):
# print(base_path, file_path)
for file in files:
file_list.append(os.path.join(base_path, file))
for file in file_list:
main(file_path=file)
diease_to_sql.py
4.科研文献数据库
# -*- coding: utf-8 -*- """
@Datetime: 2018/12/26
@Author: Zhang Yafei
"""
import pymysql
from DBUtils.PooledDB import PooledDB
import pandas as pd
from pymysql.err import InternalError
import numpy as np POOL = PooledDB(
creator=pymysql, # 使用链接数据库的模块
maxconnections=6, # 连接池允许的最大连接数，0和None表示不限制连接数
mincached=2, # 初始化时，链接池中至少创建的空闲的链接，0表示不创建
maxcached=5, # 链接池中最多闲置的链接，0和None不限制
maxshared=3,
# 链接池中最多共享的链接数量，0和None表示全部共享。PS: 无用，因为pymysql和MySQLdb等模块的 threadsafety都为1，所有值无论设置为多少，_maxcached永远为0，所以永远是所有链接都共享。
blocking=True, # 连接池中如果没有可用连接后，是否阻塞等待。True，等待；False，不等待然后报错
maxusage=None, # 一个链接最多被重复使用的次数，None表示无限制
setsession=[], # 开始会话前执行的命令列表。如：["set datestyle to ...", "set time zone ..."]
ping=0,
# ping MySQL服务端，检查是否服务可用。# 如：0 = None = never, 1 = default = whenever it is requested, 2 = when a cursor is created, 4 = when a query is executed, 7 = always
host='127.0.0.1',
port=3306,
user='root',
password='0000',
database='srld',
charset='utf8mb4'
) def connect(type=None):
conn = POOL.connection()
cursor = conn.cursor(cursor=type)
return conn, cursor def connect_close(conn, cursor):
cursor.close()
conn.close() def fetchone(sql, arg=list()):
conn, cursor = connect(type)
cursor.execute(sql, arg)
data = cursor.fetchone()
connect_close(conn, cursor)
return data def fetchall(sql, arg=list(), type=pymysql.cursors.DictCursor):
conn, cursor = connect(type)
cursor.execute(sql, arg)
data = cursor.fetchall()
connect_close(conn, cursor)
return data def insert(sql, arg=list()):
conn, cursor = connect()
row = cursor.execute(sql, arg)
conn.commit()
connect_close(conn, cursor)
return row def insert_sql(row):
pmid_doi, author, journal, year, title, abstract = row sql = """
insert into pubmed(PMID_DOI,author,journal,year,title,abstract) values(%s, %s, %s, %s, %s, %s)
"""
try:
insert(sql, arg=[pmid_doi, author, journal, year, title, abstract])
except InternalError:
# print(pmid_doi, author, journal, year, title, abstract)
print(author, type(author), np.isnan(author))
# pmid_doi = None if np.isnan(pmid_doi) else pmid_doi
# print(author, type(author), np.isnan(author))
author = None if np.isnan(author) else author
print(author)
# journal = None if np.isnan(journal) else journa
# year = None if np.isnan(year) else year
# title = None if np.isnan(title) else title
# abstract = None if np.isnan(abstract) else abstract
# print(author)
# print(pmid_doi, author, journal, year, title, abstract) def main():
df = pd.read_csv('xml副本/documents_1.txt', delimiter='\t', header=None)
df.fillna(0, inplace=True)
df.apply(insert_sql, axis=1) if __name__ == "__main__":
main()
# insert('''insert into pubmed(PMID_DOI,author,journal,year,title,abstract) values('PMID:28231184|DOI:10.3390/foods5040089', 'Moreira-Ascarrunz, Sergio Daniel; Larsson, Hans; Prieto-Linde, Maria Luisa; Johansson, Eva', 'Foods (Basel, Switzerland)', '2016', 'Mineral Nutritional Yield and Nutrient Density of Locally Adapted Wheat Genotypes under Organic Production.','The aim of the present investigation was to investigate the nutritional yield, nutrient density, stability, and adaptability of organically produced wheat for sustainable and nutritional high value food production. This study evaluated the nutritional yield of four minerals (Fe, Zn, Cu, and Mg) in 19 wheat genotypes, selected as being locally adapted under organic agriculture conditions. The new metric of nutritional yield was calculated for each genotype and they were evaluated for stability using the Additive Main effects and Multiplicative Interaction (AMMI) stability analysis and for genotypic value, stability, and adaptability using the Best Linear Unbiased Prediction (BLUP procedure). The results indicated that there were genotypes suitable for production under organic agriculture conditions with satisfactory yields (>4000 kg·ha(-1)). Furthermore, these genotypes showed high nutritional yield and nutrient density for the four minerals studied. Additionally, since these genotypes were stable and adaptable over three environmentally different years, they were designated "balanced genotypes" for the four minerals and for the aforementioned characteristics. Selection and breeding of such "balanced genotypes" may offer an alternative to producing nutritious food under low-input agriculture conditions. Furthermore, the type of evaluation presented here may also be of interest for implementation in research conducted in developing countries, following the objectives of producing enough nutrients for a growing population.')''')
mysql_helper.py
# -*- coding: utf-8 -*-
'''
Datetime: 2019/11/07
author: Zhang Yafei
description:
'''
from sqlalchemy import create_engine
import pandas as pd
import os
from multiprocessing import Pool
from mysql_helper import fetchall, insert engine = create_engine(
"mysql+pymysql://root:0000@127.0.0.1:3306/srld?charset=utf8mb4",
max_overflow=0, # 超过连接池大小外最多创建的连接
pool_size=5, # 连接池大小
pool_timeout=30, # 池中没有线程最多等待的时间，否则报错
pool_recycle=-1 # 多久之后对线程池中的线程进行一次连接的回收（重置）
)
conn = engine.connect() def filter_file(file_list):
sql = """ select file_path from data_records """
file_of_database = {file[0] for file in fetchall(sql, type=None)}
files = file_list - file_of_database
print(f'共{len(file_list)}\t已经导入{len(file_of_database)}\t还需导入:{len(files)}')
return files def main(file_path):
table_name = 'pubmed'
# df = pd.read_csv('xml副本/documents_1.txt',delimiter='\t', header=None)
try:
df = pd.read_csv(file_path, delimiter='\t', header=None)
df.columns = ['pmid_doi', 'author', 'journal', 'year', 'title', 'abstract']
df['path'] = file_path
df['year'] = '2017'
df.to_sql(table_name, conn, if_exists='append', index=False)
insert_sql = """ insert into data_records(file_path) values(%s) """
insert(sql=insert_sql, arg=[file_path])
print(f'{file_path}\t导入成功')
except Exception as e:
# insert_sql = """ insert into data_records(file_path, state, remarks) values(%s,%s,%s) """
# print(f'{file_path}文件导入失败')
# insert(sql=insert_sql, arg=[file_path,0,str(e)])
print(f'{file_path}文件导入失败') if __name__ == "__main__":
dir__names = ['data处理/800w-1600w副本', 'data处理/1600w-2600w副本', 'data处理/xmls_news']
# for dir_name in dir__names:
dir_name = dir__names[2]
file_list = {os.path.join(dir_name, file)
for file in os.listdir(dir_name)}
file_list = filter_file(file_list)
for file in file_list:
main(file_path=file) """
pool = Pool()
pool.map(main, file_list)
pool.join()
pool.close()
"""
pubmed_data_to_sql.py
# -*- coding: utf-8 -*-
'''
Datetime: 2019/11/08
author: Zhang Yafei
description:
'''
from sqlalchemy import create_engine
import pandas as pd
import os
from mysql_helper import fetchall, insert engine = create_engine(
"mysql+pymysql://root:0000@127.0.0.1:3306/srld?charset=utf8mb4",
max_overflow=0, # 超过连接池大小外最多创建的连接
pool_size=5, # 连接池大小
pool_timeout=30, # 池中没有线程最多等待的时间，否则报错
pool_recycle=-1 # 多久之后对线程池中的线程进行一次连接的回收（重置）
)
conn = engine.connect() def filter_file(file_list):
file_list = set(file_list)
sql = """ select path from sci_records """
file_of_database = {file[0] for file in fetchall(sql, type=None)}
files = file_list - file_of_database
print(f'共{len(file_list)}\t已经导入{len(file_of_database)}\t还需导入:{len(files)}')
return files def main(file_path, table_name='sci_china'):
try:
df = pd.read_csv(file_path, delimiter='\t', encoding='utf-16', error_bad_lines=False)
df = df[['PT', 'CA', 'TI','HO','ID', 'AB','FX','PD','PY','VL','AR']]
df['path'] = 'China/2013'
df.rename({'PT': 'AU', 'CA': 'TI', 'TI':'SO','HO':'DE','ID':'AB','AB':'C1','FX':'CR', 'PD':'PY', 'PY':'VL','VL':'IS','AR':'DI'}, axis=1, inplace=True)
df['PY'] = '2013'
# print(df.head())
df.to_sql(table_name, conn, if_exists='append',index=False)
# insert_sql = """ insert into sci_records(path) values(%s) """
# insert(sql=insert_sql, arg=[file_path])
print(f'{file_path}\t导入成功')
except Exception as e:
print(f'{file_path}\t导入失败') if __name__ == "__main__":
file_list = []
for base_path,dirs, files in os.walk('China'):
for file in files:
if file.endswith('.txt'):
file_list.append(os.path.join(base_path, file))
# main(file_list[0])
# file_list = filter_file(file_list)
file_list = file_list[2000:2080]
for file in file_list:
main(file_path=file)
sci_data_to_sql.py
5.基因数据导入
# -*- coding: utf-8 -*-
'''
Datetime: 2019/11/08
author: Zhang Yafei
description:
'''
from sqlalchemy import create_engine
import pandas as pd
import os engine = create_engine(
"mysql+pymysql://root:0000@127.0.0.1:3306/gene?charset=utf8",
max_overflow=0, # 超过连接池大小外最多创建的连接
pool_size=5, # 连接池大小
pool_timeout=30, # 池中没有线程最多等待的时间，否则报错
pool_recycle=-1 # 多久之后对线程池中的线程进行一次连接的回收（重置）
)
conn = engine.connect() def main(file_path, table_name='arrayexpress'):
# try:
df = pd.read_csv(file_path, delimiter='\t', error_bad_lines=False, encoding='ISO8859-1')
df.to_sql(table_name, conn, if_exists='append',index=False)
# insert_sql = """ insert into sci_records(path) values(%s) """
# insert(sql=insert_sql, arg=[file_path])
print(f'{file_path}\t导入成功')
# except Exception as e:
# print(f'{file_path}\t导入失败') if __name__ == "__main__":
file_list = []
for base_path,dirs, files in os.walk('ArrayExpress数据'):
for file in files:
if file.endswith('.txt'):
file_list.append(os.path.join(base_path, file))
# file_list = filter_file(file_list)
for file in file_list:
main(file_path=file)
arrayexpress_to_sql.py
# -*- coding: utf-8 -*-
'''
Datetime: 2019/11/08
author: Zhang Yafei
description:
'''
from sqlalchemy import create_engine
import pandas as pd
import os engine = create_engine(
"mysql+pymysql://root:0000@127.0.0.1:3306/zacopride?charset=utf8",
max_overflow=0, # 超过连接池大小外最多创建的连接
pool_size=5, # 连接池大小
pool_timeout=30, # 池中没有线程最多等待的时间，否则报错
pool_recycle=-1 # 多久之后对线程池中的线程进行一次连接的回收（重置）
)
conn = engine.connect() def main(file_path):
table_name = file_path.split('\\')[-1].strip('.csv').strip().replace(' ', '_').lower()
df = pd.read_csv(file_path)
df.to_sql(table_name, conn, if_exists='append',index=False)
print(f'{file_path}\t导入成功') if __name__ == "__main__":
file_list = []
for base_path,dirs, files in os.walk('zacopride'):
# print(base_path, file_path)
for file in files:
file_list.append(os.path.join(base_path, file))
# print(file_list)
# main(file_list[0])
for file in file_list:
main(file_path=file)
gene_to_sql.py
# -*- coding: utf-8 -*-
'''
Datetime: 2019/11/08
author: Zhang Yafei
description:
'''
from sqlalchemy import create_engine
import pandas as pd
import os engine = create_engine(
"mysql+pymysql://root:0000@127.0.0.1:3306/gene?charset=utf8",
max_overflow=0, # 超过连接池大小外最多创建的连接
pool_size=5, # 连接池大小
pool_timeout=30, # 池中没有线程最多等待的时间，否则报错
pool_recycle=-1 # 多久之后对线程池中的线程进行一次连接的回收（重置）
)
conn = engine.connect() def main(file_path, table_name='series'):
# try:
df = pd.read_csv(file_path, error_bad_lines=False)
# print(df.columns)
# df['path'] = file_path
df.to_sql(table_name, conn, if_exists='append',index=False)
# insert_sql = """ insert into sci_records(path) values(%s) """
# insert(sql=insert_sql, arg=[file_path])
print(f'{file_path}\t导入成功')
# except Exception as e:
# print(f'{file_path}\t导入失败') if __name__ == "__main__":
file_list = []
for base_path,dirs, files in os.walk('series'):
for file in files:
if file.endswith('.csv'):
file_list.append(os.path.join(base_path, file))
for file in file_list:
main(file_path=file)
series_to_sql.py
Python操作数据库实战的更多相关文章
- Python接口测试实战3（上）- Python操作数据库
如有任何学习问题,可以添加作者微信:lockingfree 课程目录 Python接口测试实战1(上)- 接口测试理论 Python接口测试实战1(下)- 接口测试工具的使用 Python接口测试实战 ...
- python操作数据库
一,安装mysql 如果是windows 用户,mysql 的安装非常简单,直接下载安装文件,双击安装文件一步一步进行操作即可. Linux 下的安装可能会更加简单,除了下载安装包进行安装外,一般的l ...
- python操作数据库之批量导入
python操作数据库之批量导入 Python语法简洁清晰,特色之一是强制用空白符(white space)作为语句缩进. Python具有丰富和强大的库.它常被昵称为胶水语言,能够把用其他语言制作的 ...
- 零基础学Python--------第11章 使用Python操作数据库
第11章 使用Python操作数据库 11.1 数据库编程接口 在项目开发中,数据库应用必不可少.虽然数据库的种类有很多,如SQLite.MySQL.Oracle等,但是它们的功能基本都是一样的,为了 ...
- python操作数据库PostgreSQL
1.简述 python可以操作多种数据库,诸如SQLite.MySql.PostgreSQL等,这里不对所有的数据库操作方法进行赘述,只针对目前项目中用到的PostgreSQL做一下简单介绍,主要包括 ...
- Python操作数据库之 MySQL
Python操作数据库之MySQL 一.安装Python-MySQLdb模块 Python-MySQLdb是一个操作数据库的模块,Python 通过它对 mysql 数据实现各种操作. 如果要源码安装 ...
- Python学习笔记 - day11 - Python操作数据库
MySQL的事务 MySQL的事务支持不是绑定在MySQL服务器本身,而是与存储引擎相关,MySQL的两种引擎如下: 1.MyISAM:不支持事务,用于只读程序提高性能 2.InnoDB:支持ACID ...
- 孤荷凌寒自学python第四十四天Python操作 数据库之准备工作
孤荷凌寒自学python第四十四天Python操作数据库之准备工作 (完整学习过程屏幕记录视频地址在文末,手写笔记在文末) 今天非常激动地开始接触Python的数据库操作的学习了,数据库是系统化设计 ...
- Python操作数据库及hashlib模块
一.hashlib模块 hashlib模块,主要用于加密相关的操作,在python3的版本里,代替了md5和sha模块,主要提供 SHA1, SHA224, SHA256, SHA384, SHA51 ...
随机推荐
- MySQL记录锁、间隙锁、临键锁小案例演示
生成间隙(gap)锁.临键(next-key)锁的前提条件 是在 RR 隔离级别下. 有关Mysql记录锁.间隙(gap)锁.临键锁(next-key)锁的一些理论知识之前有写过,详细内容可以看这篇文 ...
- python3.6.4 scrapy框架from PIL import Image报错 from . import _imaging as core
scrapy框架爬取url下载图片时,用ImagesPipeline下载图片 from PIL import Image报错 from . import _imaging as core Import ...
- c#中Array,ArrayList 与List<T>的区别、共性与转换
本文内容来自我写的开源电子书<WoW C#>,现在正在编写中,可以去WOW-Csharp/学习路径总结.md at master · sogeisetsu/WOW-Csharp (gith ...
- java多线程9：线程池
线程池 线程池的优点 我们知道线程的创建和上下文的切换也是需要消耗CPU资源的,所以在多线程任务下,使用线程池的优点就有: 第一:降低资源消耗.通过重复利用已创建的线程降低线程创建和销毁造成的消耗. ...
- tryParse的用法。
tryParse的用法. int.Parse()是一种类容转换:表示将数字内容的字符串转为int类型.如果字符串为空,则抛出ArgumentNullException异常:如果字符串内容不是数字,则抛 ...
- re模块计算器作业
1 - 2 * ( (60-30 +(-40/5) * (9-2*5/3 + 7 /3*99/4*2998 +10 * 568/14 )) - (-4*3)/ (16-3*2) )
- Jenkins安装部署使用图文详解（非常详细）
前言 最近公司需要弄一套自动化运维部署,于是抽空学习了一下,用了两天左右完成Jenkins的安装部署和各种项目的配置化,于是整理一下进行分享. 介绍 Jenkins是一个独立的开源软件项目,是基于Ja ...
- Linux使用tar解压的时候去掉父级目录
去除解压目录结构使用 --strip-components N 如: 压缩文件text.tar 中文件信息为 src/src1/src2/text.txt 运行 tar -zxvf text.tar ...
- 【linux】环境变量生命周期的操作方式
目录 前言 1. 修改环境变量 1.1 手动指定 1.2 临时生效 1.3 永久生效 链接 前言 参考: 李柱明博客 本文主要记录 linux 环境变量配置的生命周期. 如,修改环境变量 PATH 是 ...
- 【LeetCode】代码模板，刷题必会
目录 二分查找 排序的写法 BFS的写法 DFS的写法 回溯法 树 递归 迭代 前序遍历 中序遍历 后序遍历 构建完全二叉树 并查集 前缀树 图遍历 Dijkstra算法 Floyd-Warshall ...