【Oracle/Java】批量删除16张十万数据的表 单线程耗时45秒 多线程耗时38秒
昨天做了插入的单线程多线程比较，今天做个删除的。
单线程批量删除类代码：
package com.hy.delete.singlethread;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import org.apache.log4j.Logger;
import com.hy.DBParam;
/**
* 多表单线程删除
* Single thread table deleter
* @author 逆火
*
* 2019年11月17日 上午8:42:41
*/
public class BatchDeleter {
private static Logger log = Logger.getLogger(BatchDeleter.class);
// Commit size
private static final int commitSize=10000;
private String[] tablenames= { "TestTB01",
"TestTB02",
"TestTB03",
"TestTB04",
"TestTB05",
"TestTB06",
"TestTB07",
"TestTB08",
"TestTB09",
"TestTB10",
"TestTB11",
"TestTB12",
"TestTB13",
"TestTB14",
"TestTB15",
"TestTB16",
};
/**
* 批量插入
*/
public void batchDelete(String expiredDate) {
Connection conn = null;
Statement stmt = null;
try{
Class.forName(DBParam.Driver).newInstance();
conn = DriverManager.getConnection(DBParam.DbUrl, DBParam.User, DBParam.Pswd);
stmt = conn.createStatement();
System.out.println("Begin to access "+DBParam.DbUrl+" as "+DBParam.User+"...");
int index=1;
for(String table:tablenames) {
int total=delete(index,table,expiredDate,conn,stmt);
log.info("#"+index+" "+total+" records deleted from table:'"+table+"'.");
index++;
}
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
stmt.close();
conn.close();
} catch (SQLException e) {
log.error("Can't close stmt/conn because of " + e.getMessage());
}
}
}
/**
* change seconds to DayHourMinuteSecond format
* @param stratMs
* @param endMs
* @return
*/
private static String sec2DHMS(long stratMs,long endMs) {
String retval = null;
long secondCount=(endMs-stratMs)/1000;
long days = secondCount / (60 * 60 * 24);
long hours = (secondCount % (60 * 60 * 24)) / (60 * 60);
long minutes = (secondCount % (60 * 60)) / 60;
long seconds = secondCount % 60;
if (days > 0) {
retval = days + "d" + hours + "h" + minutes + "m" + seconds + "s";
} else if (hours > 0) {
retval = hours + "h" + minutes + "m" + seconds + "s";
} else if (minutes > 0) {
retval = minutes + "m" + seconds + "s";
} else {
retval = seconds + "s";
}
return retval;
}
/**
* 按日期删一张表的记录
* @param tableIndex
* @param table
* @param expiredDate
* @param conn
* @param stmt
* @throws SQLException
*/
private int delete(int tableIndex,String table,String expiredDate,Connection conn,Statement stmt) throws SQLException {
int totalDeleted=0;
int expiredCount=0;
do {
String sql="delete from "+table+" WHERE CREATEDTIME < to_date('"+expiredDate+"','yyyy-MM-dd') and rownum<'"+commitSize+"' ";
int deleted=stmt.executeUpdate(sql);
//log.info("#"+tableIndex+" "+deleted+" records deleted from table:'"+table+"'.");
totalDeleted+=deleted;
expiredCount=queryExpiredCount(table,expiredDate,stmt);
}while(expiredCount>0);
return totalDeleted;
}
/**
* 查询过期记录数量
* @param table
* @param expiredDate
* @param conn
* @param stmt
* @return
* @throws SQLException
*/
private int queryExpiredCount(String table,String expiredDate,Statement stmt) throws SQLException {
String sql="SELECT COUNT (*) as cnt FROM "+table+" WHERE CREATEDTIME < to_date('"+expiredDate+"','yyyy-MM-dd') and rownum<10 ";
ResultSet rs = stmt.executeQuery(sql);
while (rs.next()) {
int count = rs.getInt("cnt");
return count;
}
return 0;
}
public static void main(String[] args) {
long startTime = System.currentTimeMillis();
BatchDeleter bd=new BatchDeleter();
bd.batchDelete("2019-07-17");
long endTime = System.currentTimeMillis();
log.info("Time elapsed:" + sec2DHMS(startTime,endTime) );
}
}
输出：
Begin to access jdbc:oracle:thin:@127.0.0.1:1521:orcl as ufo... 2019-11-17 09:40:44,951 INFO[main]-#1 83077 records deleted from table:'TestTB01'. 2019-11-17 09:40:46,165 INFO[main]-#2 83077 records deleted from table:'TestTB02'. 2019-11-17 09:40:49,874 INFO[main]-#3 83077 records deleted from table:'TestTB03'. 2019-11-17 09:40:53,271 INFO[main]-#4 83077 records deleted from table:'TestTB04'. 2019-11-17 09:40:56,728 INFO[main]-#5 83077 records deleted from table:'TestTB05'. 2019-11-17 09:40:59,420 INFO[main]-#6 83077 records deleted from table:'TestTB06'. 2019-11-17 09:41:02,766 INFO[main]-#7 83077 records deleted from table:'TestTB07'. 2019-11-17 09:41:05,983 INFO[main]-#8 83077 records deleted from table:'TestTB08'. 2019-11-17 09:41:08,704 INFO[main]-#9 83077 records deleted from table:'TestTB09'. 2019-11-17 09:41:11,827 INFO[main]-#10 83077 records deleted from table:'TestTB10'. 2019-11-17 09:41:14,895 INFO[main]-#11 83077 records deleted from table:'TestTB11'. 2019-11-17 09:41:18,155 INFO[main]-#12 83077 records deleted from table:'TestTB12'. 2019-11-17 09:41:20,941 INFO[main]-#13 83077 records deleted from table:'TestTB13'. 2019-11-17 09:41:23,992 INFO[main]-#14 83077 records deleted from table:'TestTB14'. 2019-11-17 09:41:25,462 INFO[main]-#15 83077 records deleted from table:'TestTB15'. 2019-11-17 09:41:28,066 INFO[main]-#16 83077 records deleted from table:'TestTB16'. 2019-11-17 09:41:28,068 INFO[main]-Time elapsed:45s
多线程批量删除管理者类：
package com.hy.delete.multithread;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.List;
import org.apache.log4j.Logger;
/**
* 批量删表管理者
* @author 逆火
*
* 2019年11月17日 上午9:54:46
*/
public class DeleteManager {
private static Logger log = Logger.getLogger(DeleteManager.class);
private String[] tablenames= { "TestTB01",
"TestTB02",
"TestTB03",
"TestTB04",
"TestTB05",
"TestTB06",
"TestTB07",
"TestTB08",
"TestTB09",
"TestTB10",
"TestTB11",
"TestTB12",
"TestTB13",
"TestTB14",
"TestTB15",
"TestTB16",
};
private long startTime;// Start time
private List<DeleteJobInfo> jobInfos;// delete table informations
/**
* 批量插入
*/
public void batchDelete(String expiredDate) {
startTime=System.currentTimeMillis();
jobInfos=new ArrayList<DeleteJobInfo>();
int index=1;
for(String table:tablenames) {
DeleteThread dt=new DeleteThread(index,table,expiredDate,this);
dt.start();
index++;
}
}
/**
* Thread report manager "job done."
* @param tbSN
* @param tableName
* @param timeElasped
*/
public void reportFinished(String tbSN,String tableName,String timeElasped,String deleted) {
jobInfos.add(new DeleteJobInfo(tbSN,tableName,timeElasped,deleted));
if(jobInfos.size()==tablenames.length) {
long endTime = System.currentTimeMillis();
log.info(">>> Delete jobs finished.( time elapsed： " + sec2DHMS(startTime,endTime)+") <<<");
log.info("------------ Details ------------");
log.info("#,table,deleted,time elapsed");
for(DeleteJobInfo jobInfo:jobInfos) {
String raw="{0},{1},{2}";
Object[] arr={jobInfo.tbSn,jobInfo.tableName,jobInfo.deleted,jobInfo.timeElapsed};
String line=MessageFormat.format(raw, arr);
log.info(line);
}
log.info("------------ Details ------------");
}else {
log.info(jobInfos.size()+" deleters completed their jobs.");
}
}
/**
* 成员内部类，用来做数据结构
* 参考：https://blog.csdn.net/weixin_42762133/article/details/82890555
* @author 逆火
*
* 2019年11月17日 上午9:22:04
*/
protected static final class DeleteJobInfo{
String tbSn;// 表序号
String tableName;// 表名
String timeElapsed;// 耗时
String deleted;// count of deleted records
public DeleteJobInfo(String tbSn,String tableName,String timeElapsed,String deleted) {
this.tbSn=tbSn;
this.tableName=tableName;
this.timeElapsed=timeElapsed;
this.deleted=deleted;
}
}
/**
* change seconds to DayHourMinuteSecond format
* @param stratMs
* @param endMs
* @return
*/
private static String sec2DHMS(long stratMs,long endMs) {
String retval = null;
long secondCount=(endMs-stratMs)/1000;
long days = secondCount / (60 * 60 * 24);
long hours = (secondCount % (60 * 60 * 24)) / (60 * 60);
long minutes = (secondCount % (60 * 60)) / 60;
long seconds = secondCount % 60;
if (days > 0) {
retval = days + "d" + hours + "h" + minutes + "m" + seconds + "s";
} else if (hours > 0) {
retval = hours + "h" + minutes + "m" + seconds + "s";
} else if (minutes > 0) {
retval = minutes + "m" + seconds + "s";
} else {
retval = seconds + "s";
}
return retval;
}
public static void main(String[] args) {
DeleteManager dm=new DeleteManager();
dm.batchDelete("2019-07-17");
}
}
多线程删除删除者类：
package com.hy.delete.multithread;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import org.apache.log4j.Logger;
import com.hy.DBParam;
public class DeleteThread extends Thread {
private static Logger log = Logger.getLogger(DeleteThread.class);
// Commit size
private static final int commitSize=10000;
private int tableIndex;// table serial number
private String table;// table will be deleted
private String expiredDate;// expired date
private DeleteManager menager; // reference to manager
public DeleteThread(int tableIndex,String table,String expiredDate,DeleteManager menager) {
this.tableIndex=tableIndex;
this.table=table;
this.expiredDate=expiredDate;
this.menager=menager;
}
public void run() {
Connection conn = null;
Statement stmt = null;
try{
long startTime = System.currentTimeMillis();
Class.forName(DBParam.Driver).newInstance();
conn = DriverManager.getConnection(DBParam.DbUrl, DBParam.User, DBParam.Pswd);
stmt = conn.createStatement();
log.info("Begin to access "+DBParam.DbUrl+" as "+DBParam.User+"...");
int total=delete(tableIndex,table,expiredDate,conn,stmt);
log.info("#"+tableIndex+" "+total+" records deleted from table:'"+table+"'.");
long endTime = System.currentTimeMillis();
String timeElasped=sec2DHMS(startTime,endTime);
menager.reportFinished(String.valueOf(tableIndex), table, timeElasped,String.valueOf(total));
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
stmt.close();
conn.close();
} catch (SQLException e) {
log.error("Can't close stmt/conn because of " + e.getMessage());
}
}
}
/**
* 按日期删一张表的记录
* @param tableIndex
* @param table
* @param expiredDate
* @param conn
* @param stmt
* @throws SQLException
*/
private int delete(int tableIndex,String table,String expiredDate,Connection conn,Statement stmt) throws SQLException {
int totalDeleted=0;
int expiredCount=0;
do {
String sql="delete from "+table+" WHERE CREATEDTIME < to_date('"+expiredDate+"','yyyy-MM-dd') and rownum<'"+commitSize+"' ";
int deleted=stmt.executeUpdate(sql);
//log.info("#"+tableIndex+" "+deleted+" records deleted from table:'"+table+"'.");
totalDeleted+=deleted;
expiredCount=queryExpiredCount(table,expiredDate,stmt);
}while(expiredCount>0);
return totalDeleted;
}
/**
* 查询过期记录数量
* @param table
* @param expiredDate
* @param conn
* @param stmt
* @return
* @throws SQLException
*/
private int queryExpiredCount(String table,String expiredDate,Statement stmt) throws SQLException {
String sql="SELECT COUNT (*) as cnt FROM "+table+" WHERE CREATEDTIME < to_date('"+expiredDate+"','yyyy-MM-dd') and rownum<10 ";
ResultSet rs = stmt.executeQuery(sql);
while (rs.next()) {
int count = rs.getInt("cnt");
return count;
}
return 0;
}
/**
* change seconds to DayHourMinuteSecond format
* @param stratMs
* @param endMs
* @return
*/
private static String sec2DHMS(long stratMs,long endMs) {
String retval = null;
long secondCount=(endMs-stratMs)/1000;
long days = secondCount / (60 * 60 * 24);
long hours = (secondCount % (60 * 60 * 24)) / (60 * 60);
long minutes = (secondCount % (60 * 60)) / 60;
long seconds = secondCount % 60;
if (days > 0) {
retval = days + "d" + hours + "h" + minutes + "m" + seconds + "s";
} else if (hours > 0) {
retval = hours + "h" + minutes + "m" + seconds + "s";
} else if (minutes > 0) {
retval = minutes + "m" + seconds + "s";
} else {
retval = seconds + "s";
}
return retval;
}
}
这个例子也证明了多线程相对于单线程的优势。
--END-- 2019年11月17日11:24:17
【Oracle/Java】批量删除16张十万数据的表 单线程耗时45秒 多线程耗时38秒的更多相关文章
- 在Oracle中快速创建一张百万级别的表，一张十万级别的表 并修改两表中1%的数据 全部运行时间66秒
万以下小表做性能优化没有多大意义,因此我需要创建大表: 创建大表有三种方法,一种是insert into table selec..connect by.的方式,它最快但是数据要么是连续值,要么是随机 ...
- mybatis（二）接口编程 、动态sql 、批量删除 、动态更新、连表查询
原理等不在赘述,这里主要通过代码展现. 在mybatis(一)基础上,新建一个dao包,并在里面编写接口,然后再在xml文件中引入接口路径,其他不变,在运用阶段将比原始方法更节约时间,因为不用再去手动 ...
- oracle数据库，怎么给已有数据的表添加自增字段
场景:数据仓库,ODI为使用Oracle Incremental Update IKM,需要对一事实表增加主键. 思想:基于老表创建新表,添加自增字段(序列+触发器实现),把老数据导入新表,删除老表, ...
- php批量删除数据库下指定前缀的表
如何用php批量删除数据库下所有前缀为prefix_的表. 例子,统一删除前缀为“prefix_”的表. <?php //设置数据库连接信息.数据库服务器地址,数据库用户名,数据密码 mysql ...
- Mysql批量删除和修改某个前缀的表
1.批量删除某个前缀的表名,首先选出这些个表. select concat( 'drop table ', table_name, ';' ) from information_schema.tabl ...
- 【Oracle/Java】给十六张表各插入十万条数据 单线程耗时半小时 多线程耗时一刻钟
测试机Oracle版本: SQL> select * from v$version; BANNER ----------------------------------------------- ...
- 【Oracle/Java】向三张表各插入百万数据，共用时18分3秒，平均每张表6分钟
三张表DDL如下: CREATE TABLE tb01 ( "ID" ,) not null primary key, "NAME" NVARCHAR2() n ...
- Java 批量删除Word中的空白段落
1. 测试文档.期望达到的目标文档效果 用于测试的Word文档如下所示,包含的空白段落影响文章整体布局及美观性: 目标文档效果: 2. 辅助工具 2.1 使用类库:Free Spire.Doc for ...
- java 批量插入10万条数据
for (int i = 0; i < 100000; i++) { dbHelper.insert("INSERT aaa(name) Values ('1')"); } ...
随机推荐
- MVC-Cache-1.输出缓存(Cache:[1].输出缓存2.应用程序缓存)
缓存前提概念: 1.使用缓存的目的就是为提供网站性能,减轻对数据库的压力,提高访问的速度. 2.如果使用缓存不当,比不使用缓存造成的影响更恶劣(缓存数据的更新不及时.缓存过多等). 3..net MV ...
- python之random、time与sys模块
一.random模块 import random # float型 print(random.random()) #取0-1之间的随机小数 print(random.uniform(n,m)) #取 ...
- CSS之属相相关
一.padding与margin padding:就是内边距的意思,它是边框到内容之间的距离 另外padding的区域是有背景颜色的.并且背景颜色和内容的颜色一样.也就是说background-col ...
- 【OF框架】在部署中使用 Windows身份认证
准备 了解Windows身份认证相关知识及原理 框架开发项目完成,并完成发布包,完成在IIS中部署. 框架支持对Windows身份认证的实现,仅需要通过配置节进行配置即可切换,可以在部署的时候配置即可 ...
- PAT基础级-钻石段位样卷2-7-3 大笨钟 (10 分)
微博上有个自称“大笨钟V”的家伙,每天敲钟催促码农们爱惜身体早点睡觉.不过由于笨钟自己作息也不是很规律,所以敲钟并不定时.一般敲钟的点数是根据敲钟时间而定的,如果正好在某个整点敲,那么“当”数就等于那 ...
- 如何为Spring Boot应用程序配置端口
[转]https://www.javaroad.cn/questions/11162 1 个月前 1.1通过属性文件更新 . /src/main/resources/application.prope ...
- js依赖mui.css生成图片验证码
js依赖mui.css生成图片验证码 相关css和js引入路径 https://cdnjs.cloudflare.com/ajax/libs/mui/3.7.1/css/mui.css https:/ ...
- linux网络编程之posix线程（二）
继续接着上次的posix线程来学习: 回顾一下创建线程的函数: pthread_att_t属性变量是需要进行初始化才能够用的,一定初始化了属性变量,它就包含了线程的多种属性的值,那到底有哪些属性了,下 ...
- nexus私服搭建及信息配置
nexus私服搭建及信息配置 下载 登录nexus官网下载nexus安装包https://help.sonatype.com/repomanager2/download/download-archiv ...
- 25.C# 异步调用Web服务
1.创建Web服务 1.1VS新建ASP.Net空Web应用程序 1.2添加Web服务新建项 1.3添加GetWeather方法和相关类 using System; using System.Coll ...