每天迁移MySQL历史数据到历史库Python脚本

#!/usr/bin/env python

# coding:utf-8

#__author__ = 'Logan'

import MySQLdb

import sys

import datetime

import time

class ClassMigrate(object):

def _get_argv(self):

self.usage = """

usage():

python daily_migration.py --source=192.168.1.4:3306/db_name:tab_name/proxy/password \\

--dest=192.168.1.150:13301/db_name_archive:tab_name_201601/proxy/password \\

--delete_strategy=delete --primary_key=auto_id --date_col=ut --time_interval=180

"""

if len(sys.argv) == 1:

print self.usage

sys.exit(1)

elif sys.argv[1] == '--help' or sys.argv[1] == '-h':

print self.usage

sys.exit()

elif len(sys.argv) > 2:

for i in sys.argv[1:]:

_argv = i.split('=')

if _argv[0] == '--source':

_list = _argv[1].split('/')

self.source_host = _list[0].split(':')[0]

self.source_port = int(_list[0].split(':')[1])

self.source_db = _list[1].split(':')[0]

self.source_tab = _list[1].split(':')[1]

self.source_user = _list[2]

self.source_password = _list[3]

elif _argv[0] == '--dest':

_list = _argv[1].split('/')

self.dest_host = _list[0].split(':')[0]

self.dest_port = int(_list[0].split(':')[1])

self.dest_db = _list[1].split(':')[0]

self.dest_tab = _list[1].split(':')[1]

self.dest_user = _list[2]

self.dest_password = _list[3]

elif _argv[0] == '--delete_strategy':

self.deleteStrategy = _argv[1]

if self.deleteStrategy not in ('delete', 'drop'):

print (self.usage)

sys.exit(1)

elif _argv[0] == '--primary_key':

self.pk = _argv[1]

elif _argv[0] == '--date_col':

self.date_col = _argv[1]

elif _argv[0] == '--time_interval':

self.interval = _argv[1]

else:

print (self.usage)

sys.exit(1)

def __init__(self):

self._get_argv()

## --------------------------------------------------------------------

self.sourcedb_conn_str = MySQLdb.connect(host=self.source_host, port=self.source_port, user=self.source_user, passwd=self.source_password, db=self.source_db, charset='utf8')

self.sourcedb_conn_str.autocommit(True)

self.destdb_conn_str = MySQLdb.connect(host=self.dest_host, port=self.dest_port, user=self.dest_user, passwd=self.dest_password, db=self.dest_db, charset='utf8')

self.destdb_conn_str.autocommit(True)

## --------------------------------------------------------------------

self.template_tab = self.source_tab + '_template'

self.step_size = 20000

## --------------------------------------------------------------------

self._migCompleteState = False

self._deleteCompleteState = False

## --------------------------------------------------------------------

self.source_cnt = ''

self.source_min_id = ''

self.source_max_id = ''

self.source_checksum = ''

self.dest_cn = ''

## --------------------------------------------------------------------

self.today = time.strftime("%Y-%m-%d")

# self.today = '2016-05-30 09:59:40'

def sourcedb_query(self, sql, sql_type):

try:

cr = self.sourcedb_conn_str.cursor()

cr.execute(sql)

if sql_type == 'select':

return cr.fetchall()

elif sql_type == 'dml':

rows = self.sourcedb_conn_str.affected_rows()

return rows

else:

return True

except Exception, e:

print (str(e) + " ")

return False

finally:

cr.close()

def destdb_query(self, sql, sql_type, values=''):

try:

cr = self.destdb_conn_str.cursor()

if sql_type == 'select':

cr.execute(sql)

return cr.fetchall()

elif sql_type == 'insertmany':

cr.executemany(sql, values)

rows = self.destdb_conn_str.affected_rows()

return rows

else:

cr.execute(sql)

return True

except Exception, e:

print (str(e) + " ")

return False

finally:

cr.close()

def create_table_from_source(self):

'''''因为tab_name表的数据需要迁移到archive引擎表，所以不适合使用这种方式。 预留作其他用途。'''

try:

sql = "show create table %s;" % self.source_tab

create_str = self.sourcedb_query(sql, 'select')[0][1]

create_str = create_str.replace('CREATE TABLE', 'CREATE TABLE IF NOT EXISTS')

self.destdb_query(create_str, 'ddl')

return True

except Exception, e:

print (str(e) + " ")

return False

def create_table_from_template(self):

try:

sql = 'CREATE TABLE IF NOT EXISTS %s like %s;' % (self.dest_tab, self.template_tab)

state = self.destdb_query(sql, 'ddl')

if state:

return True

else:

return False

except Exception, e:

print (str(e + " ") + " ")

return False

def get_min_max(self):

""" 创建目标表、并获取源表需要迁移的总条数、最小id、最大id """

try:

print ("\nStarting Migrate at -- %s ") % (datetime.datetime.now().__str__())

sql =

"""select count(*),IFNULL(min(%s),-1),IFNULL(max(%s),-1) from %s where %s >= CONCAT(DATE_FORMAT(DATE_ADD('%s', INTERVAL -%s day),'%%Y-%%m-%%d'), ' 00:00:00') \

and %s <= CONCAT(DATE_FORMAT(DATE_ADD('%s', INTERVAL -%s day),'%%Y-%%m-%%d'), ' 23:59:59') """ \

% (self.pk, self.pk, self.source_tab, self.date_col, self.today, self.interval, self.date_col, self.today, self.interval)

q = self.sourcedb_query(sql, 'select')

self.source_cnt = q[0][0]

self.source_min_id = q[0][1]

self.source_max_id = q[0][2]

self.source_checksum = str(self.source_cnt) + '_' + str(self.source_min_id) + '_' + str(self.source_max_id)

if self.source_cnt == 0 or self.source_min_id == -1 or self.source_max_id == -1:

print ("There is 0 record in source table been matched! ")

return False

else:

return True

except Exception, e:

print (str(e) + " ")

return False

def migrate_2_destdb(self):

try:

get_min_max_id = self.get_min_max()

if get_min_max_id:

k = self.source_min_id

desc_sql = "desc %s;" % self.source_tab

# self.filed = []

cols = self.sourcedb_query(desc_sql, 'select')

# for j in cols:

# self.filed.append(j[0])

fileds = "%s," * len(cols) # 源表有多少个字段，就拼凑多少个%s，拼接到insert语句

fileds = fileds.rstrip(',')

while k <= self.source_max_id:

sql = """select * from %s where %s >= %d and %s< %d \

and %s >= CONCAT(DATE_FORMAT(DATE_ADD('%s', INTERVAL -%s day),'%%Y-%%m-%%d'), ' 00:00:00') \

and %s <= CONCAT(DATE_FORMAT(DATE_ADD('%s', INTERVAL -%s day),'%%Y-%%m-%%d'), ' 23:59:59') """\

% (self.source_tab, self.pk, k, self.pk, k+self.step_size, self.date_col, self.today, self.interval, self.date_col, self.today, self.interval)

print ("\n%s ") % sql

starttime = datetime.datetime.now()

results = self.sourcedb_query(sql, 'select')

insert_sql = "insert into " + self.dest_tab + " values (%s)" % fileds

rows = self.destdb_query(insert_sql, 'insertmany', results)

if rows == False:

print ("Insert failed!! ")

else:

print ("Inserted %s rows. ") % rows

endtime = datetime.datetime.now()

timeinterval = endtime - starttime

print("Elapsed :" + str(timeinterval.seconds) + '.' + str(timeinterval.microseconds) + " seconds ")

k += self.step_size

print ("\nInsert complete at -- %s ") % (datetime.datetime.now().__str__())

return True

else:

return False

except Exception, e:

print (str(e) + " ")

return False

def verify_total_cnt(self):

try:

sql =

"""select count(*),IFNULL(min(%s),-1),IFNULL(max(%s),-1) from %s where %s >= CONCAT(DATE_FORMAT(DATE_ADD('%s', INTERVAL -%s day),'%%Y-%%m-%%d'), ' 00:00:00') \

and %s <= CONCAT(DATE_FORMAT(DATE_ADD('%s', INTERVAL -%s day),'%%Y-%%m-%%d'), ' 23:59:59') """ \

% (self.pk, self.pk, self.dest_tab, self.date_col, self.today, self.interval, self.date_col, self.today, self.interval)

dest_result = self.destdb_query(sql, 'select')

self.dest_cnt = dest_result[0][0]

dest_checksum = str(self.dest_cnt) + '_' + str(dest_result[0][1]) + '_' + str(dest_result[0][2])

print ("source_checksum: %s, dest_checksum: %s ") % (self.source_checksum, dest_checksum)

if self.source_cnt == dest_result[0][0] and dest_result[0][0] != 0 and self.source_checksum == dest_checksum:

self._migCompleteState = True

print ("Verify successfully !! ")

else:

print ("Verify failed !! ")

sys.exit(77)

except Exception, e:

print (str(e) + " ")

def drop_daily_partition(self):

try:

if self._migCompleteState:

sql =

"""explain partitions select * from %s where %s >= CONCAT(DATE_FORMAT(DATE_ADD('%s', INTERVAL -%s day),'%%Y-%%m-%%d'), ' 00:00:00')

and %s <= CONCAT(DATE_FORMAT(DATE_ADD('%s', INTERVAL -%s day),'%%Y-%%m-%%d'), ' 23:59:59') """\

% (self.source_tab, self.date_col, self.today, self.interval, self.date_col, self.today, self.interval)

partition_name = self.sourcedb_query(sql, 'select')

partition_name = partition_name[0][3]

sql = """select count(*),IFNULL(min(%s),-1),IFNULL(max(%s),-1) from %s partition (%s)""" \

% (self.pk, self.pk, self.source_tab, partition_name)

q = self.sourcedb_query(sql, 'select')

source_cnt = q[0][0]

source_min_id = q[0][1]

source_max_id = q[0][2]

checksum = str(source_cnt) + '_' + str(source_min_id) + '_' + str(source_max_id)

if source_cnt == 0 or source_min_id == -1 or source_max_id == -1:

print ("There is 0 record in source PARTITION been matched! ")

else:

if checksum == self.source_checksum:

drop_par_sql = "alter table %s drop partition %s;" % (self.source_tab, partition_name)

droped = self.sourcedb_query(drop_par_sql, 'ddl')

if droped:

print (drop_par_sql + " ")

print ("\nDrop partition complete at -- %s ") % (datetime.datetime.now().__str__())

self._deleteCompleteState = True

else:

print (drop_par_sql + " ")

print ("Drop partition failed.. ")

else:

print ("The partition %s checksum failed !! Drop failed !!") % partition_name

sys.exit(77)

except Exception, e:

print (str(e) + " ")

def delete_data(self):

try:

if self._migCompleteState:

k = self.source_min_id

while k <= self.source_max_id:

sql = """delete from %s where %s >= %d and %s< %d \

and %s >= CONCAT(DATE_FORMAT(DATE_ADD('%s', INTERVAL -%s day),'%%Y-%%m-%%d'), ' 00:00:00') \

and %s <= CONCAT(DATE_FORMAT(DATE_ADD('%s', INTERVAL -%s day),'%%Y-%%m-%%d'), ' 23:59:59') """ \

% (self.source_tab, self.pk, k, self.pk, k+self.step_size, self.date_col, self.today, self.interval, self.date_col, self.today, self.interval)

print ("\n%s ") % sql

starttime = datetime.datetime.now()

rows = self.sourcedb_query(sql, 'dml')

if rows == False:

print ("Delete failed!! ")

else:

print ("Deleted %s rows. ") % rows

endtime = datetime.datetime.now()

timeinterval = endtime - starttime

print("Elapsed :" + str(timeinterval.seconds) + '.' + str(timeinterval.microseconds) + " seconds ")

time.sleep(1)

k += self.step_size

print ("\nDelete complete at -- %s ") % (datetime.datetime.now().__str__())

self._deleteCompleteState = True

except Exception, e:

print (str(e) + " ")

def do(self):

tab_create = self.create_table_from_template()

if tab_create:

migration = self.migrate_2_destdb()

if migration:

self.verify_total_cnt()

if self._migCompleteState:

if self.deleteStrategy == 'drop':

self.drop_daily_partition()

else:

self.delete_data()

print ("\n ")

print ("====="*5 + ' ')

print ("source_total_cnt: %s ") % self.source_cnt

print ("dest_total_cnt: %s ") % self.dest_cnt

print ("====="*5 + ' ')

if self._deleteCompleteState:

print ("\nFinal result: Successfully !! ")

sys.exit(88)

else:

print ("\nFinal result: Failed !! ")

sys.exit(254)

else:

print ("Create table failed ! Exiting. . .")

sys.exit(255)

f = ClassMigrate()

f.do()

每天迁移MySQL历史数据到历史库Python脚本的更多相关文章

从零开始学安全(三十五)●mysql 盲注手工自定义python脚本
import requests import string #mysql 手动注入通用脚本适用盲注可以跟具自己的需求更改 def home(): url="url" list ...
迁移mysql数据到oracle上
转自:http://www.cnblogs.com/Warmsunshine/p/4651283.html 我是生成的文件里面的master.sql里面的sql,一个一个拷出来的. 迁移mysql数据 ...
基于binlog来分析mysql的行记录修改情况（python脚本分析）
最近写完mysql flashback,突然发现还有有这种使用场景:有些情况下,可能会统计在某个时间段内,MySQL修改了多少数据量?发生了多少事务?主要是哪些表格发生变动?变动的数量是怎 ...
MySQL利用binlog恢复误操作数据（python脚本）
在人工手动进行一些数据库写操作的时候(比方说数据订正),尤其是一些不可控的批量更新或删除,通常都建议备份后操作.不过不怕万一,就怕一万,有备无患总是好的.在线上或者测试环境误操作导致数据被删除或者更新 ...
mysql更新(三)语句库的操作表的操作
04-初始mysql语句本节课先对mysql的基本语法初体验. 操作文件夹(库) 增 create database db1 charset utf8; 查 # 查看当前创建的数据库 show ...
HBase——使用Put迁移MySql数据到Hbase
先上code: /** * 功能:迁移mysql上电池历史数据到hbase * Created by liuhuichao on 2016/12/6. */ public class MySqlToH ...
mysql数据库从删库到跑路之mysql基础
一数据库是什么之前所学,数据要永久保存,比如用户注册的用户信息,都是保存于文件中,而文件只能存在于某一台机器上. 如果我们不考虑从文件中读取数据的效率问题,并且假设我们的程序所有的组件都运行在一台 ...
使用第三方库连接MySql数据库：PyMysql库和Pandas库
使用PyMysql库和Pandas库链接Mysql 1 系统环境系统版本:Win10 64位 Mysql版本: 8.0.15 MySQL Community Server - GPL pymysql ...
32.修改IK分词器源码来基于mysql热更新词库
主要知识点, 修改IK分词器源码来基于mysql热更新词库一.IK增加新词的原因在第32小节中学习到了直接在es的词库中增加词语,来扩充自已的词库,但是这样做有以下缺点: (1)每次添加完 ...

随机推荐

Linux虚拟机与主机网络连接配置与文件传输
网络配置对于VMware虚拟机 1. 设置linux系统的网络配置,如下(NAT为默认配置,这里采用这一配置) 2. 主机中配置本地连接-属性-共享-勾选红框配置项,如下: 3.重启虚拟机. ...
odoo14在tree、kanban视图上添加dashboard
效果图: 实现代码:js:view的类型原来1个js给拆分成了4个: view, controller, renderer, model 1.view:AbstractView的子类,这是工厂类:类需 ...
Python基础之分离文件名和文件路径
参考链接:https://blog.csdn.net/qq_42110481/article/details/81104182 分离文件名与文件路径: import os if __name__ == ...
学会这十招，轻松搜索github优质项目
大家好,我是青空. 今天我想给大家分享一下使用 GitHub 的一些心得体会.之前我是在分享 GitHub上的一些开源项目,通过这段时间的收集工作,我积累了一些相关的经验在这里分享给大家. 我做了一个 ...
SpringBoot AOP中JoinPoint的用法和通知切点表达式
前言上一篇文章讲解了springboot aop 初步完整的使用和整合这一篇讲解他的接口方法和类 JoinPoint和ProceedingJoinPoint对象 JoinPoint对象封装了Spr ...
String,String Builder,String Buffer-源码
目录 String 源码分析常用的API isEmpty() length() charAt() substring() equals() equals()与"==" inter ...
论文笔记：（2021CVPR）PAConv: Position Adaptive Convolution with Dynamic Kernel Assembling on Point Clouds
目录摘要 1.引言 2.相关工作将点云映射到常规二维或三维栅格(体素) 基于MLPs的点表示学习基于点卷积的点表示学习动态卷积和条件卷积 3.方法 3.1 回顾 3.2 动态内核组装 Weig ...
Dubbo 实现一个Route Factory(用于灰度发布)
Dubbo 可以实现的扩展很多, 官方文档在这: https://dubbo.apache.org/zh/docs/v2.7/dev/impls/ (太简单了....) 下面我们实现一个Route F ...
HCNA Routing&Switching之STP基础
前文我们了解了VLAN动态注册协议GVRP相关话题,回顾请参考https://www.cnblogs.com/qiuhom-1874/p/15113770.html:今天我们来讨论下二层环路和STP相 ...
可怕！字节跳动大牛爆出的“Android进阶指南”，强到无法形容，GitHub已标星“8.5k”！
前言从去年8月份开始,我开始重新找工作,在这期间刷了不少面试题和大牛博客,无意中薅到一份字节跳动大牛整理的一份"Android进阶指南",包含Android面试常见问题.主流技术 ...

每天迁移MySQL历史数据到历史库Python脚本

每天迁移MySQL历史数据到历史库Python脚本的更多相关文章

随机推荐

热门专题