spring Batch实现数据库大数据量读写

spring Batch实现数据库大数据量读写

博客分类：

• spring

springBatchquartz定时调度批处理 

1. data-source-context.xml

1. <?xml version="1.0" encoding="UTF-8"?>
2. <beans xmlns="http://www.springframework.org/schema/beans"
3. xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:context="http://www.springframework.org/schema/context"
4. xmlns:tx="http://www.springframework.org/schema/tx"
5. xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
6. http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.0.xsd
7. http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-3.0.xsd">
8. <!-- 1) USE ANNOTATIONS TO IDENTIFY AND WIRE SPRING BEANS. -->
9. <context:component-scan base-package="net.etongbao.vasp.ac" />
10. <!-- 2) DATASOURCE, TRANSACTION MANAGER AND JDBC TEMPLATE -->
11. <bean id="dataSource" class="com.mchange.v2.c3p0.ComboPooledDataSource"
12. destroy-method="close" abstract="false" scope="singleton">
13. <!-- oracle.jdbc.driver.oracleDriver -->
14. <property name="driverClass" value="oracle.jdbc.OracleDriver" />
15. <property name="jdbcUrl" value="jdbc:oracle:thin:@192.168.1.23:1521:orcl01" />
16. <property name="user" value="USR_DEV01" />
17. <property name="password" value="2AF0829C" />
18. <property name="checkoutTimeout" value="30000" />
19. <property name="maxIdleTime" value="120" />
20. <property name="maxPoolSize" value="100" />
21. <property name="minPoolSize" value="2" />
22. <property name="initialPoolSize" value="2" />
23. <property name="maxStatements" value="0" />
24. <property name="maxStatementsPerConnection" value="0" />
25. <property name="idleConnectionTestPeriod" value="30" />
26. </bean>
27. <bean id="jdbcTemplate" class="org.springframework.jdbc.core.JdbcTemplate">
28. <property name="dataSource" ref="dataSource" />
29. </bean>
30. <bean id="transactionManager"
31. class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
32. <property name="dataSource" ref="dataSource" />
33. </bean>
34. <tx:annotation-driven transaction-manager="transactionManager" />
35. </beans>

2. quartz-context.xml       commit-interval="10000"每次批量数据的条数，数值越大效率越高，可在此处添加事物处理，

每次回滚数就是commit-interval数

1. <?xml version="1.0" encoding="UTF-8"?>
2. <beans xmlns="http://www.springframework.org/schema/beans"
3. xmlns:batch="http://www.springframework.org/schema/batch"
4. xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:context="http://www.springframework.org/schema/context"
5. xmlns:tx="http://www.springframework.org/schema/tx"
6. xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.0.xsd
7. http://www.springframework.org/schema/batch http://www.springframework.org/schema/batch/spring-batch-2.1.xsd
8. http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.0.xsd
9. http://www.springframework.org/schema/tx http://www.springframework.org/schema/tx/spring-tx-3.0.xsd">
10. <import resource="data-source-context.xml"/>
11. <!--   JOB REPOSITORY - WE USE IN-MEMORY REPOSITORY FOR OUR EXAMPLE -->
12. <bean id="jobRepository"
13. class="org.springframework.batch.core.repository.support.MapJobRepositoryFactoryBean">
14. <property name="transactionManager" ref="transactionManager" />
15. </bean>
16. <!-- batch config -->
17. <bean id="jobLauncher"
18. class="org.springframework.batch.core.launch.support.SimpleJobLauncher">
19. <property name="jobRepository" ref="jobRepository" />
20. </bean>
21. <!--  FINALLY OUR JOB DEFINITION. THIS IS A 1 STEP JOB -->
22. <batch:job id="ledgerJob">
23. <batch:listeners>
24. <batch:listener ref="appJobExecutionListener" />
25. </batch:listeners>
26. <batch:step id="step1">

1. <span style="white-space: pre;">    </span>   <batch:tasklet transaction-manager="transactionManager">
2. <batch:tasklet>
3. <batch:listeners>
4. <batch:listener ref="itemFailureLoggerListener" />
5. </batch:listeners>
6. <batch:chunk reader="ledgerReader" writer="ledgerWriter"
7. commit-interval="10000" /> <!-- 1万条进行一次commit -->
8. </batch:tasklet>

1. </batch:tasklet>
2. </batch:step>
3. </batch:job>
4. <!--  READER -->
5. <bean id="ledgerReader"
6. class="org.springframework.batch.item.database.JdbcCursorItemReader">
7. <property name="dataSource" ref="dataSource" />
8. <property name="sql" value="select * from ledger" />
9. <property name="rowMapper" ref="ledgerRowMapper" />
10. </bean>
11. <!-- Spring Batch Job同一个job instance，成功执行后是不允许重新执行的【失败后是否允许重跑，可通过配置Job的restartable参数来控制，默认是true】，如果需要重新执行，可以变通处理，
12. 添加一个JobParameters构建类,以当前时间作为参数，保证其他参数相同的情况下却是不同的job instance -->
13. <bean id="jobParameterBulider" class="org.springframework.batch.core.JobParametersBuilder" />
14. <!-- 定时任务 开始 -->
15. <bean id="ledgerJobDetail" class="org.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean">
16. <property name="targetObject">
17. <!-- 定时执行的类 -->
18. <ref bean="quartzLedgerJob" />
19. </property>
20. <property name="targetMethod">
21. <!-- 定时执行的类方法 -->
22. <value>execute</value>
23. </property>
24. </bean>
25. <bean id="ledgerCronTrigger" class="org.springframework.scheduling.quartz.CronTriggerBean" >
26. <!-- 这里不可以直接在属性jobDetail中引用taskJob，因为他要求的是一个jobDetail类型的对象，所以我们得通过MethodInvokingJobDetailFactoryBean来转一下 -->
27. <property name="jobDetail" >
28. <ref bean="ledgerJobDetail" />
29. </property>
30. <!--在每天下午18点到下午18:59期间的每1分钟触发  -->
31. <!--在每天上午10点40分准时触发  -->
32. <property name="cronExpression" >
33. <!-- <value>0 * 15 * * ?</value> -->
34. <value>0 45 10 * * ? * </value>
35. </property>
36. </bean>
37. <!-- 触发器工厂，将所有的定时任务都注入工厂-->
38. <bean class="org.springframework.scheduling.quartz.SchedulerFactoryBean">
39. <!-- 添加触发器 -->
40. <property name="triggers">
41. <list>
42. <!-- 将上面定义的测试定时任务注入（可以定义多个定时任务，同时注入）-->
43. <ref local="ledgerCronTrigger" />
44. </list>
45. </property>
46. </bean>
47. <!-- 定时任务 结束 -->
48. lt;/beans>

3.定时调度job类 QuartzLedgerJob.java

package net.etongbao.vasp.ac.quartz;

1. import java.util.Date;
2. import org.slf4j.Logger;
3. import org.slf4j.LoggerFactory;
4. import org.springframework.batch.core.Job;
5. import org.springframework.batch.core.JobParametersBuilder;
6. import org.springframework.batch.core.launch.JobLauncher;
7. import org.springframework.beans.factory.annotation.Autowired;
8. import org.springframework.stereotype.Component;
9. import org.springframework.util.StopWatch;
10. /**
11. * 定时调度类
12. * @author Fu Wei
13. *
14. */
15. @Component("quartzLedgerJob")
16. public class QuartzLedgerJob {
17. private static final Logger LOG = LoggerFactory.getLogger(QuartzLedgerJob.class);
18. @Autowired
19. private JobLauncher jobLauncher;
20. @Autowired
21. private Job ledgerJob;
22. @Autowired
23. JobParametersBuilder jobParameterBulider;
24. private static long counter = 0l;
25. /**
26. * 执行业务方法
27. * @throws Exception
28. */
29. public void execute() throws Exception {
30. LOG.debug("start...");
31. StopWatch sw = new StopWatch();
32. sw.start();
33. /*
34. * Spring Batch Job同一个job instance，成功执行后是不允许重新执行的【失败后是否允许重跑，
35. * 可通过配置Job的restartable参数来控制，默认是true】，如果需要重新执行，可以变通处理，
36. * 添加一个JobParameters构建类,以当前时间作为参数，保证其他参数相同的情况下却是不同的job instance
37. */
38. jobParameterBulider.addDate("date", new Date());
39. jobLauncher.run(ledgerJob, jobParameterBulider.toJobParameters());
40. sw.stop();
41. LOG.debug("Time elapsed:{},Execute quartz ledgerJob:{}", sw.prettyPrint(), ++counter);
42. }
43. }

4.程序启动类 StartQuartz.java

package net.etongbao.vasp.ac.quartz;

1. import java.io.FileNotFoundException;
2. import org.springframework.context.support.ClassPathXmlApplicationContext;
3. /**
4. * 启动定时调度
5. * @author Fu Wei
6. *
7. */
8. public class StartQuartz {
9. public static void main(String[] args) throws FileNotFoundException {
10. new ClassPathXmlApplicationContext("/net/etongbao/vasp/ac/resources/quartz-context.xml");
11. }
12. }

5.pojo类 Ledger.java

1. package net.etongbao.vasp.ac.pojo;
2. import java.io.Serializable;
3. import java.util.Date;
4. public class Ledger implements Serializable {
5. private int id;
6. private Date receiptDate;
7. private String memberName;
8. private String checkNumber;
9. private Date checkDate;
10. private String paymentType;
11. private double depositAmount;
12. private double paymentAmount;
13. private String comments;
14. public Ledger() {
15. super();
16. }
17. public Ledger(int id, Date receiptDate, String memberName, String checkNumber, Date checkDate, String paymentType,
18. double depositAmount, double paymentAmount, String comments) {
19. super();
20. this.id = id;
21. this.receiptDate = receiptDate;
22. this.memberName = memberName;
23. this.checkNumber = checkNumber;
24. this.checkDate = checkDate;
25. this.paymentType = paymentType;
26. this.depositAmount = depositAmount;
27. this.paymentAmount = paymentAmount;
28. this.comments = comments;
29. }
30. public int getId() {
31. return id;
32. }
33. public void setId(int id) {
34. this.id = id;
35. }
36. public Date getReceiptDate() {
37. return receiptDate;
38. }
39. public void setReceiptDate(Date receiptDate) {
40. this.receiptDate = receiptDate;
41. }
42. public String getMemberName() {
43. return memberName;
44. }
45. public void setMemberName(String memberName) {
46. this.memberName = memberName;
47. }
48. public String getCheckNumber() {
49. return checkNumber;
50. }
51. public void setCheckNumber(String checkNumber) {
52. this.checkNumber = checkNumber;
53. }
54. public Date getCheckDate() {
55. return checkDate;
56. }
57. public void setCheckDate(Date checkDate) {
58. this.checkDate = checkDate;
59. }
60. public String getPaymentType() {
61. return paymentType;
62. }
63. public void setPaymentType(String paymentType) {
64. this.paymentType = paymentType;
65. }
66. public double getDepositAmount() {
67. return depositAmount;
68. }
69. public void setDepositAmount(double depositAmount) {
70. this.depositAmount = depositAmount;
71. }
72. public double getPaymentAmount() {
73. return paymentAmount;
74. }
75. public void setPaymentAmount(double paymentAmount) {
76. this.paymentAmount = paymentAmount;
77. }
78. public String getComments() {
79. return comments;
80. }
81. public void setComments(String comments) {
82. this.comments = comments;
83. }
84. }

6. LedgerDaoImpl.java

package net.etongbao.vasp.ac.dao.impl;

1. import java.sql.PreparedStatement;
2. import java.sql.SQLException;
3. import net.etongbao.vasp.ac.dao.LedgerDao;
4. import net.etongbao.vasp.ac.pojo.Ledger;
5. import org.springframework.beans.factory.annotation.Autowired;
6. import org.springframework.jdbc.core.JdbcTemplate;
7. import org.springframework.jdbc.core.PreparedStatementSetter;
8. import org.springframework.stereotype.Repository;
9. /**
10. * ledger数据操作类
11. *
12. * @author Fu Wei
13. *
14. */
15. @Repository
16. public class LedgerDaoImpl implements LedgerDao {
17. private static final String SAVE_SQL = "insert into ledger_temp (rcv_dt, mbr_nm, chk_nbr, chk_dt, pymt_typ, dpst_amt, pymt_amt, comments) values(?,?,?,?,?,?,?,?)";
18. @Autowired
19. private JdbcTemplate jdbcTemplate;
20. @Override
21. public void save(final Ledger item) {
22. jdbcTemplate.update(SAVE_SQL, new PreparedStatementSetter() {
23. public void setValues(PreparedStatement stmt) throws SQLException {
24. stmt.setDate(1, new java.sql.Date(item.getReceiptDate().getTime()));
25. stmt.setString(2, item.getMemberName());
26. stmt.setString(3, item.getCheckNumber());
27. stmt.setDate(4, new java.sql.Date(item.getCheckDate().getTime()));
28. stmt.setString(5, item.getPaymentType());
29. stmt.setDouble(6, item.getDepositAmount());
30. stmt.setDouble(7, item.getPaymentAmount());
31. stmt.setString(8, item.getComments());
32. }
33. });
34. }
35. }

7.接口 LedgerDao .java

1. package net.etongbao.vasp.ac.dao;
2. import net.etongbao.vasp.ac.pojo.Ledger;
3. public interface LedgerDao {
4. public void save(final Ledger item) ;
5. }

8. JdbcTemplete 需要的LedgerRowMapper.java

package net.etongbao.vasp.ac.batch.writer;

1. import java.sql.ResultSet;
2. import java.sql.SQLException;
3. import net.etongbao.vasp.ac.pojo.Ledger;
4. import org.springframework.jdbc.core.RowMapper;
5. import org.springframework.stereotype.Component;
6. /**
7. * ledger行的映射类
8. * @author Administrator
9. *
10. */
11. @Component("ledgerRowMapper")
12. public class LedgerRowMapper implements RowMapper {
13. public Object mapRow(ResultSet rs, int rowNum) throws SQLException {
14. Ledger ledger = new Ledger();
15. ledger.setId(rs.getInt("id"));
16. ledger.setReceiptDate(rs.getDate("rcv_dt"));
17. ledger.setMemberName(rs.getString("mbr_nm"));
18. ledger.setCheckNumber(rs.getString("chk_nbr"));
19. ledger.setCheckDate(rs.getDate("chk_dt"));
20. ledger.setPaymentType(rs.getString("pymt_typ"));
21. ledger.setDepositAmount(rs.getDouble("dpst_amt"));
22. ledger.setPaymentAmount(rs.getDouble("pymt_amt"));
23. ledger.setComments(rs.getString("comments"));
24. return ledger;
25. }
26. }

9.关键类LedgerWriter.java ，写入数据，负责数据的添加

1. package net.etongbao.vasp.ac.batch.writer;
2. import java.util.List;
3. import net.etongbao.vasp.ac.dao.LedgerDao;
4. import net.etongbao.vasp.ac.pojo.Ledger;
5. import org.springframework.batch.item.ItemWriter;
6. import org.springframework.beans.factory.annotation.Autowired;
7. import org.springframework.stereotype.Component;
8. /**
9. * ledger写入数据
10. *
11. * @author Fu Wei
12. *
13. */
14. @Component("ledgerWriter")
15. public class LedgerWriter implements ItemWriter<Ledger> {
16. @Autowired
17. private LedgerDao ledgerDao;
18. /**
19. * 写入数据
20. *
21. * @param ledgers
22. */
23. public void write(List<? extends Ledger> ledgers) throws Exception {
24. for (Ledger ledger : ledgers) {
25. ledgerDao.save(ledger);
26. }
27. }
28. }

classPath:

<?xml version="1.0" encoding="UTF-8"?>

1. <classpath>
2. <classpathentry kind="src" path="src"/>
3. <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/jrockit-jdk1.6.0_24-R28.1.3-4.0.1"/>
4. <classpathentry kind="lib" path="lib/aopalliance-1.0.jar"/>
5. <classpathentry kind="lib" path="lib/c3p0-0.9.1.2.jar"/>
6. <classpathentry kind="lib" path="lib/commons-collections-3.2.1.jar"/>
7. <classpathentry kind="lib" path="lib/commons-lang-2.3.jar"/>
8. <classpathentry kind="lib" path="lib/commons-logging-1.1.1.jar"/>
9. <classpathentry kind="lib" path="lib/etb-log4j-1.2.16.jar"/>
10. <classpathentry kind="lib" path="lib/etb-slf4j-api-1.5.8.jar"/>
11. <classpathentry kind="lib" path="lib/etb-slf4j-log4j12-1.5.8.jar"/>
12. <classpathentry kind="lib" path="lib/ojdbc6.jar"/>
13. <classpathentry kind="lib" path="lib/org.springframework.aop-3.0.5.RELEASE.jar"/>
14. <classpathentry kind="lib" path="lib/org.springframework.asm-3.0.5.RELEASE.jar"/>
15. <classpathentry kind="lib" path="lib/org.springframework.aspects-3.0.5.RELEASE.jar"/>
16. <classpathentry kind="lib" path="lib/org.springframework.beans-3.0.5.RELEASE.jar"/>
17. <classpathentry kind="lib" path="lib/org.springframework.context-3.0.5.RELEASE.jar"/>
18. <classpathentry kind="lib" path="lib/org.springframework.context.support-3.0.5.RELEASE.jar"/>
19. <classpathentry kind="lib" path="lib/org.springframework.core-3.0.5.RELEASE.jar"/>
20. <classpathentry kind="lib" path="lib/org.springframework.expression-3.0.5.RELEASE.jar"/>
21. <classpathentry kind="lib" path="lib/org.springframework.instrument-3.0.5.RELEASE.jar"/>
22. <classpathentry kind="lib" path="lib/org.springframework.instrument.tomcat-3.0.5.RELEASE.jar"/>
23. <classpathentry kind="lib" path="lib/org.springframework.jdbc-3.0.5.RELEASE.jar"/>
24. <classpathentry kind="lib" path="lib/org.springframework.jms-3.0.5.RELEASE.jar"/>
25. <classpathentry kind="lib" path="lib/org.springframework.orm-3.0.5.RELEASE.jar"/>
26. <classpathentry kind="lib" path="lib/org.springframework.oxm-3.0.5.RELEASE.jar"/>
27. <classpathentry kind="lib" path="lib/org.springframework.test-3.0.5.RELEASE.jar"/>
28. <classpathentry kind="lib" path="lib/org.springframework.transaction-3.0.5.RELEASE.jar"/>
29. <classpathentry kind="lib" path="lib/quartz-all-1.6.5.jar"/>
30. <classpathentry kind="lib" path="lib/spring-batch-core-2.1.6.RELEASE.jar"/>
31. <classpathentry kind="lib" path="lib/spring-batch-infrastructure-2.1.6.RELEASE.jar"/>
32. <classpathentry kind="lib" path="lib/spring-batch-test-2.1.6.RELEASE.jar"/>
33. <classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
34. <classpathentry kind="output" path="bin"/>
35. </classpath>

总结: 测试数据8万多条，响应时间3分多钟。

关键在于quartz-context.xml 中<bean id="ledgerReader"

class="org.springframework.batch.item.database.JdbcCursorItemReader">

<property name="dataSource" ref="dataSource" />

<property name="sql" value="select * from ledger" />

<property name="rowMapper" ref="ledgerRowMapper" />

</bean> 负责读取数据 ，在程序执行时一次性抓取全部数据后在批量的交给LedgerWriter进行写操作。当然也可以使用分页读取JdbcPagingItemReader，但要分页数量与写入数量要大写相同，还可以对分页出来的数据进行添加悲观锁

LedgerWriter.java 负责写入数据，每次写入1000条。