【Oracle/Java】批量删除16张十万数据的表单线程耗时45秒多线程耗时38秒

昨天做了插入的单线程多线程比较，今天做个删除的。

单线程批量删除类代码：

package com.hy.delete.singlethread;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;

import org.apache.log4j.Logger;

import com.hy.DBParam;

/**
 * 多表单线程删除
 * Single thread table deleter
 * @author 逆火
 *
 * 2019年11月17日 上午8:42:41
 */
public class BatchDeleter {
    private static Logger log = Logger.getLogger(BatchDeleter.class);

    // Commit size
    private static final int commitSize=10000;

    private String[] tablenames= {    "TestTB01",
            "TestTB02",
            "TestTB03",
            "TestTB04",
            "TestTB05",
            "TestTB06",
            "TestTB07",
            "TestTB08",
            "TestTB09",
            "TestTB10",
            "TestTB11",
            "TestTB12",
            "TestTB13",
            "TestTB14",
            "TestTB15",
            "TestTB16",
          };

    /**
     * 批量插入
     */
    public void batchDelete(String expiredDate) {
        Connection conn = null;
        Statement stmt = null;

        try{
            Class.forName(DBParam.Driver).newInstance();
            conn = DriverManager.getConnection(DBParam.DbUrl, DBParam.User, DBParam.Pswd);
            stmt = conn.createStatement();
            System.out.println("Begin to access "+DBParam.DbUrl+" as "+DBParam.User+"...");

            int index=1;
            for(String table:tablenames) {
                int total=delete(index,table,expiredDate,conn,stmt);
                log.info("#"+index+" "+total+" records deleted from table:'"+table+"'.");
                index++;
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                stmt.close();
                conn.close();
            } catch (SQLException e) {
                log.error("Can't close stmt/conn because of " + e.getMessage());
            }
        }
    }

    /**
     * change seconds to DayHourMinuteSecond format
     * @param stratMs
     * @param endMs
     * @return
     */
    private static String sec2DHMS(long stratMs,long endMs) {
        String retval = null;
        long secondCount=(endMs-stratMs)/1000;

        long days = secondCount / (60 * 60 * 24);
        long hours = (secondCount % (60 * 60 * 24)) / (60 * 60);
        long minutes = (secondCount % (60 * 60)) / 60;
        long seconds = secondCount % 60;

        if (days > 0) {
            retval = days + "d" + hours + "h" + minutes + "m" + seconds + "s";
        } else if (hours > 0) {
            retval = hours + "h" + minutes + "m" + seconds + "s";
        } else if (minutes > 0) {
            retval = minutes + "m" + seconds + "s";
        } else {
            retval = seconds + "s";
        }

        return retval;
    }

    /**
     * 按日期删一张表的记录
     * @param tableIndex
     * @param table
     * @param expiredDate
     * @param conn
     * @param stmt
     * @throws SQLException
     */
    private int delete(int tableIndex,String table,String expiredDate,Connection conn,Statement stmt) throws SQLException {
        int totalDeleted=0;
        int expiredCount=0;

        do {
            String sql="delete from "+table+" WHERE CREATEDTIME < to_date('"+expiredDate+"','yyyy-MM-dd') and rownum<'"+commitSize+"' ";
            int deleted=stmt.executeUpdate(sql);
            //log.info("#"+tableIndex+" "+deleted+" records deleted from table:'"+table+"'.");
            totalDeleted+=deleted;

            expiredCount=queryExpiredCount(table,expiredDate,stmt);
        }while(expiredCount>0);

        return totalDeleted;
    }

    /**
     * 查询过期记录数量
     * @param table
     * @param expiredDate
     * @param conn
     * @param stmt
     * @return
     * @throws SQLException
     */
    private int queryExpiredCount(String table,String expiredDate,Statement stmt) throws SQLException {
        String sql="SELECT COUNT (*) as cnt FROM "+table+" WHERE CREATEDTIME < to_date('"+expiredDate+"','yyyy-MM-dd') and rownum<10 ";

        ResultSet rs = stmt.executeQuery(sql);

        while (rs.next()) {
            int count = rs.getInt("cnt");
            return count;
        }

        return 0;
    }

    public static void main(String[] args) {
        long startTime = System.currentTimeMillis();
        BatchDeleter bd=new BatchDeleter();
        bd.batchDelete("2019-07-17");
        long endTime = System.currentTimeMillis();
        log.info("Time elapsed:" + sec2DHMS(startTime,endTime) );
    }
}

输出：

Begin to access jdbc:oracle:thin:@127.0.0.1:1521:orcl as ufo...
2019-11-17 09:40:44,951 INFO[main]-#1 83077 records deleted from table:'TestTB01'.
2019-11-17 09:40:46,165 INFO[main]-#2 83077 records deleted from table:'TestTB02'.
2019-11-17 09:40:49,874 INFO[main]-#3 83077 records deleted from table:'TestTB03'.
2019-11-17 09:40:53,271 INFO[main]-#4 83077 records deleted from table:'TestTB04'.
2019-11-17 09:40:56,728 INFO[main]-#5 83077 records deleted from table:'TestTB05'.
2019-11-17 09:40:59,420 INFO[main]-#6 83077 records deleted from table:'TestTB06'.
2019-11-17 09:41:02,766 INFO[main]-#7 83077 records deleted from table:'TestTB07'.
2019-11-17 09:41:05,983 INFO[main]-#8 83077 records deleted from table:'TestTB08'.
2019-11-17 09:41:08,704 INFO[main]-#9 83077 records deleted from table:'TestTB09'.
2019-11-17 09:41:11,827 INFO[main]-#10 83077 records deleted from table:'TestTB10'.
2019-11-17 09:41:14,895 INFO[main]-#11 83077 records deleted from table:'TestTB11'.
2019-11-17 09:41:18,155 INFO[main]-#12 83077 records deleted from table:'TestTB12'.
2019-11-17 09:41:20,941 INFO[main]-#13 83077 records deleted from table:'TestTB13'.
2019-11-17 09:41:23,992 INFO[main]-#14 83077 records deleted from table:'TestTB14'.
2019-11-17 09:41:25,462 INFO[main]-#15 83077 records deleted from table:'TestTB15'.
2019-11-17 09:41:28,066 INFO[main]-#16 83077 records deleted from table:'TestTB16'.
2019-11-17 09:41:28,068 INFO[main]-Time elapsed:45s

多线程批量删除管理者类：

package com.hy.delete.multithread;

import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.List;

import org.apache.log4j.Logger;

/**
 * 批量删表管理者
 * @author 逆火
 *
 * 2019年11月17日 上午9:54:46
 */
public class DeleteManager {
    private static Logger log = Logger.getLogger(DeleteManager.class);

    private String[] tablenames= {    "TestTB01",
            "TestTB02",
            "TestTB03",
            "TestTB04",
            "TestTB05",
            "TestTB06",
            "TestTB07",
            "TestTB08",
            "TestTB09",
            "TestTB10",
            "TestTB11",
            "TestTB12",
            "TestTB13",
            "TestTB14",
            "TestTB15",
            "TestTB16",
          };

    private long startTime;// Start time

    private List<DeleteJobInfo> jobInfos;// delete table informations

    /**
     * 批量插入
     */
    public void batchDelete(String expiredDate) {
        startTime=System.currentTimeMillis();
        jobInfos=new ArrayList<DeleteJobInfo>();

        int index=1;
        for(String table:tablenames) {
            DeleteThread dt=new DeleteThread(index,table,expiredDate,this);
            dt.start();
            index++;
        }
    }

    /**
     * Thread report manager "job done."
     * @param tbSN
     * @param tableName
     * @param timeElasped
     */
    public void reportFinished(String tbSN,String tableName,String timeElasped,String deleted) {
        jobInfos.add(new DeleteJobInfo(tbSN,tableName,timeElasped,deleted));

        if(jobInfos.size()==tablenames.length) {
            long endTime = System.currentTimeMillis();
            log.info(">>> Delete jobs finished.( time elapsed： " + sec2DHMS(startTime,endTime)+") <<<");

            log.info("------------ Details ------------");
            log.info("#,table,deleted,time elapsed");
            for(DeleteJobInfo jobInfo:jobInfos) {
                String raw="{0},{1},{2}";
                Object[] arr={jobInfo.tbSn,jobInfo.tableName,jobInfo.deleted,jobInfo.timeElapsed};
                String line=MessageFormat.format(raw, arr);
                log.info(line);
            }
            log.info("------------ Details ------------");

        }else {
            log.info(jobInfos.size()+" deleters completed their jobs.");
        }
    }

    /**
     * 成员内部类，用来做数据结构
     * 参考：https://blog.csdn.net/weixin_42762133/article/details/82890555
     * @author 逆火
     *
     * 2019年11月17日 上午9:22:04
     */
    protected static final class DeleteJobInfo{
        String tbSn;// 表序号
        String tableName;// 表名
        String timeElapsed;// 耗时
        String deleted;// count of deleted records

        public DeleteJobInfo(String tbSn,String tableName,String timeElapsed,String deleted) {
            this.tbSn=tbSn;
            this.tableName=tableName;
            this.timeElapsed=timeElapsed;
            this.deleted=deleted;
        }
    }

    /**
     * change seconds to DayHourMinuteSecond format
     * @param stratMs
     * @param endMs
     * @return
     */
    private static String sec2DHMS(long stratMs,long endMs) {
        String retval = null;
        long secondCount=(endMs-stratMs)/1000;

        long days = secondCount / (60 * 60 * 24);
        long hours = (secondCount % (60 * 60 * 24)) / (60 * 60);
        long minutes = (secondCount % (60 * 60)) / 60;
        long seconds = secondCount % 60;

        if (days > 0) {
            retval = days + "d" + hours + "h" + minutes + "m" + seconds + "s";
        } else if (hours > 0) {
            retval = hours + "h" + minutes + "m" + seconds + "s";
        } else if (minutes > 0) {
            retval = minutes + "m" + seconds + "s";
        } else {
            retval = seconds + "s";
        }

        return retval;
    }

    public static void main(String[] args) {
        DeleteManager dm=new DeleteManager();
        dm.batchDelete("2019-07-17");
    }
}

多线程删除删除者类：

package com.hy.delete.multithread;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;

import org.apache.log4j.Logger;

import com.hy.DBParam;

public class DeleteThread extends Thread {
    private static Logger log = Logger.getLogger(DeleteThread.class);

    // Commit size
    private static final int commitSize=10000;

    private int tableIndex;// table serial number
    private String table;// table will be deleted
    private String expiredDate;// expired date
    private DeleteManager menager; // reference to manager

    public DeleteThread(int tableIndex,String table,String expiredDate,DeleteManager menager) {
        this.tableIndex=tableIndex;
        this.table=table;
        this.expiredDate=expiredDate;
        this.menager=menager;
    }

    public void run() {
        Connection conn = null;
        Statement stmt = null;

        try{
            long startTime = System.currentTimeMillis();

            Class.forName(DBParam.Driver).newInstance();
            conn = DriverManager.getConnection(DBParam.DbUrl, DBParam.User, DBParam.Pswd);
            stmt = conn.createStatement();
            log.info("Begin to access "+DBParam.DbUrl+" as "+DBParam.User+"...");

            int total=delete(tableIndex,table,expiredDate,conn,stmt);
            log.info("#"+tableIndex+" "+total+" records deleted from table:'"+table+"'.");

            long endTime = System.currentTimeMillis();
            String timeElasped=sec2DHMS(startTime,endTime);
            menager.reportFinished(String.valueOf(tableIndex), table, timeElasped,String.valueOf(total));
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                stmt.close();
                conn.close();
            } catch (SQLException e) {
                log.error("Can't close stmt/conn because of " + e.getMessage());
            }
        }
    }

     /**
     * 按日期删一张表的记录
     * @param tableIndex
     * @param table
     * @param expiredDate
     * @param conn
     * @param stmt
     * @throws SQLException
     */
    private int delete(int tableIndex,String table,String expiredDate,Connection conn,Statement stmt) throws SQLException {
        int totalDeleted=0;
        int expiredCount=0;

        do {
            String sql="delete from "+table+" WHERE CREATEDTIME < to_date('"+expiredDate+"','yyyy-MM-dd') and rownum<'"+commitSize+"' ";
            int deleted=stmt.executeUpdate(sql);
            //log.info("#"+tableIndex+" "+deleted+" records deleted from table:'"+table+"'.");
            totalDeleted+=deleted;

            expiredCount=queryExpiredCount(table,expiredDate,stmt);
        }while(expiredCount>0);

        return totalDeleted;
    }

    /**
     * 查询过期记录数量
     * @param table
     * @param expiredDate
     * @param conn
     * @param stmt
     * @return
     * @throws SQLException
     */
    private int queryExpiredCount(String table,String expiredDate,Statement stmt) throws SQLException {
        String sql="SELECT COUNT (*) as cnt FROM "+table+" WHERE CREATEDTIME < to_date('"+expiredDate+"','yyyy-MM-dd') and rownum<10 ";

        ResultSet rs = stmt.executeQuery(sql);

        while (rs.next()) {
            int count = rs.getInt("cnt");
            return count;
        }

        return 0;
    }

    /**
     * change seconds to DayHourMinuteSecond format
     * @param stratMs
     * @param endMs
     * @return
     */
    private static String sec2DHMS(long stratMs,long endMs) {
        String retval = null;
        long secondCount=(endMs-stratMs)/1000;

        long days = secondCount / (60 * 60 * 24);
        long hours = (secondCount % (60 * 60 * 24)) / (60 * 60);
        long minutes = (secondCount % (60 * 60)) / 60;
        long seconds = secondCount % 60;

        if (days > 0) {
            retval = days + "d" + hours + "h" + minutes + "m" + seconds + "s";
        } else if (hours > 0) {
            retval = hours + "h" + minutes + "m" + seconds + "s";
        } else if (minutes > 0) {
            retval = minutes + "m" + seconds + "s";
        } else {
            retval = seconds + "s";
        }

        return retval;
    }
}

这个例子也证明了多线程相对于单线程的优势。

--END-- 2019年11月17日11:24:17

【Oracle/Java】批量删除16张十万数据的表单线程耗时45秒多线程耗时38秒的更多相关文章

在Oracle中快速创建一张百万级别的表，一张十万级别的表并修改两表中1%的数据全部运行时间66秒
万以下小表做性能优化没有多大意义,因此我需要创建大表: 创建大表有三种方法,一种是insert into table selec..connect by.的方式,它最快但是数据要么是连续值,要么是随机 ...
mybatis（二）接口编程、动态sql 、批量删除、动态更新、连表查询
原理等不在赘述,这里主要通过代码展现. 在mybatis(一)基础上,新建一个dao包,并在里面编写接口,然后再在xml文件中引入接口路径,其他不变,在运用阶段将比原始方法更节约时间,因为不用再去手动 ...
oracle数据库，怎么给已有数据的表添加自增字段
场景:数据仓库,ODI为使用Oracle Incremental Update IKM,需要对一事实表增加主键. 思想:基于老表创建新表,添加自增字段(序列+触发器实现),把老数据导入新表,删除老表, ...
php批量删除数据库下指定前缀的表
如何用php批量删除数据库下所有前缀为prefix_的表. 例子,统一删除前缀为“prefix_”的表. <?php //设置数据库连接信息.数据库服务器地址,数据库用户名,数据密码 mysql ...
Mysql批量删除和修改某个前缀的表
1.批量删除某个前缀的表名,首先选出这些个表. select concat( 'drop table ', table_name, ';' ) from information_schema.tabl ...
【Oracle/Java】给十六张表各插入十万条数据单线程耗时半小时多线程耗时一刻钟
测试机Oracle版本: SQL> select * from v$version; BANNER ----------------------------------------------- ...
【Oracle/Java】向三张表各插入百万数据，共用时18分3秒，平均每张表6分钟
三张表DDL如下: CREATE TABLE tb01 ( "ID" ,) not null primary key, "NAME" NVARCHAR2() n ...
Java 批量删除Word中的空白段落
1. 测试文档.期望达到的目标文档效果用于测试的Word文档如下所示,包含的空白段落影响文章整体布局及美观性: 目标文档效果: 2. 辅助工具 2.1 使用类库:Free Spire.Doc for ...
java 批量插入10万条数据
for (int i = 0; i < 100000; i++) { dbHelper.insert("INSERT aaa(name) Values ('1')"); } ...

随机推荐

mongodb副本集和分片存储理论整理
目录理论概述一.各种集群简述二.原理主从复制 Mongodb副本集理论概述一.各种集群简述 mongodb有三种集群搭建方式: 分片:sharding.指为处理大量数据,将数据分开存储,不 ...
Android笔记（十七） Android中的Service
定义和用途 Service是Android的四大组件之一,一直在后台运行,没有用户界面.Service组件通常用于为其他组件提供后台服务或者监控其他组件的运行状态,例如播放音乐.记录地理位置,监听用户 ...
TLS1.3 认证和秘钥建立握手环节的分析
1.ClientHello 中的参数 ClientHello---{ Random_C .extension } 在 extension中的扩展中包含 ( supported_version ...
Linux命令——pr
参考:Linux命令——column 前言接触这个命令的初衷是我想把一个很长的单列输出设置成多列输出,奈何column的分列输出机制太智障,直到我发现了pr 参数 pr -# 输出指定的列数. -t ...
Nginx服务rewrite模块功能说明网站自动跳转功能
实现域名地址信息跳转,用于做伪静态地址 www.impkk.com/oldboy?edu.html 动态地址 www.impkk.com/oldboy-edu.html 伪静态地址 rewrite ^ ...
POJ 2155 Matrix[树状数组+差分]
原题链接:https://vjudge.net/problem/POJ-2155 题目大意给定 n* n 矩阵A,其元素为0或1. A [i][j] 表示第i行和第j列中的数字.最初全为0. 我们有 ...
[51Nod 1237] 最大公约数之和 (杜教筛+莫比乌斯反演)
题目描述求∑i=1n∑j=1n(i,j) mod (1e9+7)n<=1010\sum_{i=1}^n\sum_{j=1}^n(i,j)~mod~(1e9+7)\\n<=10^{10}i ...
Java链接Redis时出现 “ERR Client sent AUTH, but no password is set”
Java链接Redis时出现 “ERR Client sent AUTH, but no password is set” 异常的原因及解决办法. [错误提示] redis.clients.jedis ...
007_项目制作拍摄视频篇之_《基于ARM与ZigBee的实验室签到系统》
研究的背景和意义: 随着社会生活节奏的加快,科技日新月异,信息更新迅速,人们之间的交流也变得越来越频繁,社会群体乃至政府之间的交流也朝着轻松.快速.容易管理和控制的方向发展,这种信息交流方式已经逐步得 ...
Postgresql 日志相关
目录日志种类作用总结配置文件中与日志相关的配置日志种类 PostgreSQL有3种日志 pg_log(数据库运行日志) 内容可读默认关闭的,需要设置参数启动pg_xlog(WAL 日志,即 ...

【Oracle/Java】批量删除16张十万数据的表 单线程耗时45秒 多线程耗时38秒

【Oracle/Java】批量删除16张十万数据的表 单线程耗时45秒 多线程耗时38秒的更多相关文章

随机推荐

热门专题

【Oracle/Java】批量删除16张十万数据的表单线程耗时45秒多线程耗时38秒

【Oracle/Java】批量删除16张十万数据的表单线程耗时45秒多线程耗时38秒的更多相关文章