Python--增量循环删除MySQL表数据

需求场景：

有一业务数据库，使用MySQL 5.5版本，每天会写入大量数据，需要不定期将多表中“指定时期前“的数据进行删除，在SQL SERVER中很容易实现，写几个WHILE循环就搞定，虽然MySQL中也存在类似功能，怎奈自己不精通，于是采用Python来实现

话不多少，上脚本：

# coding: utf-8
import MySQLdb
import time
import os

# delete config
# 如果VIEW_OR_RUN = "VIEW",仅生成小批量删除的脚本但不执行
# 如果VIEW_OR_RUN = "RUN",生成小批量删除的脚本并直接调用执行
VIEW_OR_RUN = "VIEW"
DELETE_DATABASE_NAME = ""
DELETE_TABLE_NAME = ""
DELETE_TABLE_KEY = ""
DELETE_CONDITION = ""
DELETE_ROWS_PER_BATCH = 10000
SLEEP_SECOND_PER_BATCH = 0.5

# MySQL Connection Config
Default_MySQL_Host = '192.168.166.169'
Default_MySQL_Port = 3358
Default_MySQL_User = "mysql_admin"
Default_MySQL_Password = 'mysql@Admin@Pwd'

Default_MySQL_Charset = "utf8"
Default_MySQL_Connect_TimeOut = 120
Default_MySQL_Socket = "/export/data/mysql/tmp/mysql.sock"

# Common config:
DATETIME_FORMAT = '%Y-%m-%d %X'
EXEC_DETAIL_FILE = 'exec_detail.txt'
EXEC_SCRIPT_FILE = 'delete_scripts.sql'

def highlight(s):
    return "%s[30;2m%s%s[1m" % (chr(27), s, chr(27))

def print_warning_message(message):
    """
    以红色字体显示消息内容
    :param message: 消息内容
    :return: 无返回值
    """
    message = str(message)
    print(highlight('') + "%s[31;1m%s%s[0m" % (chr(27), message, chr(27)))
    global EXEC_DETAIL_FILE
    write_file(EXEC_DETAIL_FILE, message)

def print_info_message(message):
    """
    以绿色字体输出提醒性的消息
    :param message: 消息内容
    :return: 无返回值
    """
    message = str(message)
    print(highlight('') + "%s[32;2m%s%s[0m" % (chr(27), message, chr(27)))
    global EXEC_DETAIL_FILE
    write_file(EXEC_DETAIL_FILE, message)

def write_file(file_path, message):
    """
    将传入的message追加写入到file_path指定的文件中
    请先创建文件所在的目录
    :param file_path: 要写入的文件路径
    :param message: 要写入的信息
    :return:
    """
    file_handle = open(file_path, 'a')
    file_handle.writelines(message)
    # 追加一个换行以方便浏览
    file_handle.writelines(chr(13))
    file_handle.close()

def get_user_choose_option(input_options, input_message):
    while_flag = True
    choose_option = None
    while while_flag:
        print_info_message(input_message)
        str_input = raw_input("")
        for input_option in input_options:
            if str_input.strip() == input_option:
                choose_option = input_option
                while_flag = False
    return choose_option

def get_mysql_connection():
    """
    根据默认配置返回数据库连接
    :return: 数据库连接
    """
    if Default_MySQL_Host.lower() == 'localhost':
        conn = MySQLdb.connect(
                host=Default_MySQL_Host,
                port=Default_MySQL_Port,
                user=Default_MySQL_User,
                passwd=Default_MySQL_Password,
                connect_timeout=Default_MySQL_Connect_TimeOut,
                charset=Default_MySQL_Charset,
                db=DELETE_DATABASE_NAME,
                unix_socket=Default_MySQL_Socket
        )
    else:
        conn = MySQLdb.connect(
                host=Default_MySQL_Host,
                port=Default_MySQL_Port,
                user=Default_MySQL_User,
                passwd=Default_MySQL_Password,
                connect_timeout=Default_MySQL_Connect_TimeOut,
                charset=Default_MySQL_Charset,
                db=DELETE_DATABASE_NAME
        )
    return conn

def mysql_exec(sql_script, sql_param=None):
    """
    执行传入的脚本，返回影响行数
    :param sql_script:
    :param sql_param:
    :return: 脚本最后一条语句执行影响行数
    """
    try:
        conn = get_mysql_connection()
        print_info_message("在服务器{0}上执行脚本:{1}".format(
                conn.get_host_info(), sql_script))
        cursor = conn.cursor()
        if sql_param is not None:
            cursor.execute(sql_script, sql_param)
        else:
            cursor.execute(sql_script)
        affect_rows = cursor.rowcount
        conn.commit()
        cursor.close()
        conn.close()
        return affect_rows
    except Exception as ex:
        cursor.close()
        conn.rollback()
        raise Exception(str(ex))

def mysql_exec_many(sql_script_list):
    """
    执行传入的脚本，返回影响行数
    :param sql_script_list: 要执行的脚本List,List中每个元素为sql_script, sql_param对
    :return: 返回执行每个脚本影响的行数列表
    """
    try:
        conn = get_mysql_connection()
        exec_result_list = []
        for sql_script, sql_param in sql_script_list:
            print_info_message("在服务器{0}上执行脚本:{1}".format(
                    conn.get_host_info(), sql_script))
            cursor = conn.cursor()
            if sql_param is not None:
                cursor.execute(sql_script, sql_param)
            else:
                cursor.execute(sql_script)
            affect_rows = cursor.rowcount
            exec_result_list.append("影响行数：{0}".format(affect_rows))
        conn.commit()
        cursor.close()
        conn.close()
        return exec_result_list

    except Exception as ex:
        cursor.close()
        conn.rollback()
        raise Exception(str(ex))

def mysql_query(sql_script, sql_param=None):
    """
    执行传入的SQL脚本，并返回查询结果
    :param sql_script:
    :param sql_param:
    :return: 返回SQL查询结果
    """
    try:
        conn = get_mysql_connection()
        print_info_message("在服务器{0}上执行脚本:{1}".format(
                conn.get_host_info(), sql_script))
        cursor = conn.cursor()
        if sql_param is not None:
            cursor.execute(sql_script, sql_param)
        else:
            cursor.execute(sql_script)
        exec_result = cursor.fetchall()
        cursor.close()
        conn.close()
        return exec_result
    except Exception as ex:
        cursor.close()
        conn.close()
        raise Exception(str(ex))

def get_column_info_list(table_name):
    sql_script = """
DESC {0}
""".format(table_name)
    column_info_list = []
    query_result = mysql_query(sql_script=sql_script, sql_param=None)
    for row in query_result:
        column_name = row[0]
        column_type = row[1]
        column_key = row[3]
        column_info = column_name, column_key, column_type
        column_info_list.append(column_info)
    return column_info_list

def get_id_range():
    """
    按照传入的表获取要删除数据最大ID、最小ID、删除总行数
    :return: 返回要删除数据最大ID、最小ID、删除总行数
    """
    global DELETE_TABLE_NAME
    global DELETE_CONDITION
    sql_script = """
SELECT
MAX({2}) AS MAX_ID,
MIN({2}) AS MIN_ID,
COUNT(1) AS Total_Count
FROM {0}
WHERE {1};
""".format(DELETE_TABLE_NAME, DELETE_CONDITION, DELETE_TABLE_KEY)

    query_result = mysql_query(sql_script=sql_script, sql_param=None)
    max_id, min_id, total_count = query_result[0]
    # 此处有一坑，可能出现total_count不为0 但是max_id 和min_id 为None的情况
    # 因此判断max_id和min_id 是否为NULL
    if (max_id is None) or (min_id is None):
        max_id, min_id, total_count = 0, 0, 0
    return max_id, min_id, total_count

def delete_data(current_min_id, current_max_id):
    sql_script = """
DELETE FROM {0}
WHERE {4} <= {1}
and {4} >= {2}
AND {3};
""".format(DELETE_TABLE_NAME,
           current_max_id,
           current_min_id,
           DELETE_CONDITION,
           DELETE_TABLE_KEY
           )
    global EXEC_SCRIPT_FILE
    global VIEW_OR_RUN
    if VIEW_OR_RUN == 'RUN':
        row_count = mysql_exec(sql_script)
        print_info_message("影响行数：{0}".format(row_count))
        time.sleep(SLEEP_SECOND_PER_BATCH)
    else:
        print_info_message("生成删除脚本(未执行)")
        print_info_message(sql_script)
    tmp_script = """
USE {0};
""".format(DELETE_DATABASE_NAME) + sql_script + """
COMMIT;
SELECT SLEEP('{0}');
##=====================================================##
""".format(SLEEP_SECOND_PER_BATCH)
    write_file(file_path=EXEC_SCRIPT_FILE, message=tmp_script)

def loop_delete_data():
    max_id, min_id, total_count = get_id_range()
    if min_id == max_id:
        print_info_message("无数据需要结转")
        return
    current_min_id = min_id
    global DELETE_ROWS_PER_BATCH
    while current_min_id <= max_id:
        print_info_message("*" * 70)
        current_max_id = current_min_id + DELETE_ROWS_PER_BATCH
        delete_data(current_min_id, current_max_id)
        current_percent = (current_max_id - min_id) * 100.0 / (max_id - min_id)
        left_rows = max_id - current_max_id
        if left_rows < 0:
            left_rows = 0
        current_percent_str = "%.2f" % current_percent
        info = "当前进度{0}/{1},剩余{2},进度为{3}%"
        info = info.format(current_max_id,
                           max_id,
                           left_rows,
                           current_percent_str)
        print_info_message(info)
        current_min_id = current_max_id
    print_info_message("*" * 70)
    print_info_message("执行完成")

def check_config():
    try:
        global DELETE_DATABASE_NAME
        global DELETE_TABLE_NAME
        global DELETE_TABLE_KEY
        global DELETE_CONDITION
        global VIEW_OR_RUN

        if str(DELETE_DATABASE_NAME).strip() == "":
            print_warning_message("数据库名不能为空")
            return False
        if str(DELETE_TABLE_NAME).strip() == "":
            print_warning_message("表名不能为空")
            return False
        if str(DELETE_CONDITION).strip() == "":
            print_warning_message("删除条件不能为空")
            return False
        source_columns_info_list = get_column_info_list(DELETE_TABLE_NAME)
        column_count = len(source_columns_info_list)
        primary_key_count = 0
        for column_id in range(column_count):
            source_column_name, source_column_key, source_column_type = source_columns_info_list[column_id]
            if source_column_key.lower() == 'pri':
                primary_key_count += 1
                if not ('int' in str(source_column_type).lower()):
                    print_warning_message("主键不为int或bigint")
                    return False
                else:
                    global DELETE_TABLE_KEY
                    DELETE_TABLE_KEY = source_column_name

        if primary_key_count == 0:
            print_warning_message("未找到主键，不瞒足迁移条件")
            return False

        if primary_key_count > 1:
            print_warning_message("要删除的表使用复合主键，不满足迁移条件")
            return False

        return True
    except Exception as ex:
        print_warning_message("执行出现异常，异常为{0}".format(ex.message))
        return False

def clean_env():
    global DELETE_DATABASE_NAME
    global DELETE_TABLE_NAME
    global DELETE_TABLE_KEY
    global DELETE_CONDITION
    global VIEW_OR_RUN
    DELETE_DATABASE_NAME = ""
    DELETE_TABLE_NAME = ""
    DELETE_TABLE_KEY = ""
    DELETE_CONDITION = ""
    VIEW_OR_RUN = "VIEW"

    if os.path.exists(EXEC_SCRIPT_FILE):
        os.remove(EXEC_SCRIPT_FILE)
    if os.path.exists(EXEC_DETAIL_FILE):
        os.remove(EXEC_DETAIL_FILE)

def user_confirm():
    if VIEW_OR_RUN == 'RUN':
        info = """
您将在服务器{0}上{1}中删除表{2}中数据

删除数据条件为:
DELETE FROM {3}
WHERE {4}
"""
    else:
        info = """
将生成在服务器{0}上{1}中删除表{2}中数据的脚本

删除数据条件为:
DELETE FROM {3}
WHERE {4}
"""
    info = info.format(Default_MySQL_Host,
                       DELETE_DATABASE_NAME,
                       DELETE_TABLE_NAME,
                       DELETE_TABLE_NAME,
                       DELETE_CONDITION)

    if VIEW_OR_RUN == "RUN":
        print_warning_message(info)
    else:
        print_info_message(info)
    input_options = ['yes', 'no']
    input_message = """
请输入yes继续或输入no退出，yes/no?
"""
    user_option = get_user_choose_option(input_options=input_options,
                                         input_message=input_message)
    if user_option == "no":
        return False
    else:
        return True

def delete_table_data(database_name, table_name, delete_condition, is_run, is_need_confirm):
    global DELETE_DATABASE_NAME
    global DELETE_TABLE_NAME
    global DELETE_TABLE_KEY
    global DELETE_CONDITION
    global VIEW_OR_RUN
    DELETE_DATABASE_NAME = database_name
    DELETE_TABLE_NAME = table_name
    DELETE_CONDITION = delete_condition
    DELETE_TABLE_KEY = ''
    if is_run:
        VIEW_OR_RUN = "RUN"
    else:
        VIEW_OR_RUN = "VIEW"
    check_result = check_config()
    if not check_result:
        return
    if is_need_confirm:
        confirm_result = user_confirm()
    else:
        confirm_result = True
    if confirm_result:
        loop_delete_data()

def main():
    clean_env()
    delete_table_data(database_name="db001",
                      table_name="tb2001",
                      delete_condition="dt<'2017-09-01'",
                      is_run=True,
                      is_need_confirm=True)

if __name__ == '__main__':
    main()

执行效果：

实现原理：

由于表存在自增ID，于是给我们增量循环删除的机会，查找出满足删除条件的最大值ID和最小值ID，然后按ID 依次递增，每次小范围内(如10000条)进行删除。

实现优点：

实现“小斧子砍大柴”的效果，事务小，对线上影响较小，打印出当前处理到的“ID”，可以随时关闭，稍微修改下代码便可以从该ID开始，方便。

实现不足：

为防止主从延迟太高，采用每次删除SLEEP1秒的方式，相对比较糙，最好的方式应该是周期扫描这条复制链路，根据延迟调整SLEEP的周期，反正都脚本化，再智能化点又何妨！

本文重点依旧是妹子，不能让诸位看官白跑一趟，是不！！！