Trident-MySQL
使用事物TridentTopology 持久化数据到MySQL 1、构建拓扑JDBCTopology类
package storm.trident.mysql; import java.util.Arrays;
import java.util.Map; import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.trident.TridentState;
import org.apache.storm.trident.TridentTopology;
import org.apache.storm.trident.operation.BaseFunction;
import org.apache.storm.trident.operation.CombinerAggregator;
import org.apache.storm.trident.operation.TridentCollector;
import org.apache.storm.trident.spout.IBatchSpout;
import org.apache.storm.trident.state.StateType;
import org.apache.storm.trident.testing.FixedBatchSpout;
import org.apache.storm.trident.testing.MemoryMapState;
import org.apache.storm.trident.tuple.TridentTuple;
import org.apache.storm.tuple.Fields;
import org.apache.storm.tuple.Values; /**
* 事物Trident-MySQL Topology
* @author mengyao
*
*/
@SuppressWarnings("all")
public class JDBCTopology { public static void main(String[] args) {
TridentTopology topology = new TridentTopology(); //Spout数据源
FixedBatchSpout spout = new FixedBatchSpout(new Fields("tels"), 7,
new Values("189111 3"),
new Values("135111 7"),
new Values("189111 2"),
new Values("158111 5"),
new Values("159111 6"),
new Values("159111 3"),
new Values("158111 5")
);
spout.setCycle(false); //State持久化配置属性
JDBCStateConfig config = new JDBCStateConfig();
config.setDriver("com.mysql.jdbc.Driver");
config.setUrl("jdbc:mysql://localhost:3306/test");
config.setUsername("root");
config.setPassword("123456");
config.setBatchSize(10);
config.setCacheSize(10);
config.setType(StateType.TRANSACTIONAL);
config.setCols("tel");
config.setColVals("sum");
config.setTable("tbl_tel"); topology.newStream("spout", spout)
.each(new Fields("tels"), new KeyValueFun(), new Fields("tel", "money"))
.groupBy(new Fields("tel"))
.persistentAggregate(JDBCState.getFactory(config), new Fields("money"), new SumCombinerAgg(), new Fields("sum")); LocalCluster cluster = new LocalCluster();
cluster.submitTopology("test1", new Config(), topology.build());
} } @SuppressWarnings("all")
class KeyValueFun extends BaseFunction {
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
String record = tuple.getString(0);
collector.emit(new Values(record.split("\t")[0], record.split("\t")[1]));
}
} @SuppressWarnings("all")
class SumCombinerAgg implements CombinerAggregator<Long> {
@Override
public Long init(TridentTuple tuple) {
return Long.parseLong(tuple.getString(0));
}
@Override
public Long combine(Long val1, Long val2) {
Long val = val1+val2;
System.out.println(val);
return val;
}
@Override
public Long zero() {
return 0L;
}
} 2、构建基于IBackingMap的JDBCState类
package storm.trident.mysql; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map; import org.apache.storm.task.IMetricsContext;
import org.apache.storm.trident.state.OpaqueValue;
import org.apache.storm.trident.state.State;
import org.apache.storm.trident.state.StateFactory;
import org.apache.storm.trident.state.StateType;
import org.apache.storm.trident.state.TransactionalValue;
import org.apache.storm.trident.state.map.CachedMap;
import org.apache.storm.trident.state.map.IBackingMap;
import org.apache.storm.trident.state.map.NonTransactionalMap;
import org.apache.storm.trident.state.map.OpaqueMap;
import org.apache.storm.trident.state.map.TransactionalMap; @SuppressWarnings("all")
public class JDBCState<T> implements IBackingMap<T> { private static JDBCStateConfig config; JDBCState(JDBCStateConfig config){
this.config = config;
} @Override
public List<T> multiGet(List<List<Object>> keys) {
StringBuilder sqlBuilder = new StringBuilder("SELECT ").append(config.getCols())
.append(","+config.getColVals())
.append(",txid")
.append(" FROM "+config.getTable())
.append(" WHERE ")
.append(config.getCols())
.append("='"); JDBCUtil jdbcUtil = new JDBCUtil(config.getDriver(),config.getUrl(),config.getUsername(),config.getPassword()); List<Object> result = new ArrayList<Object>();
Map<String, Object> map = null;
for (List<Object> list : keys) {
Object key = list.get(0);
map = jdbcUtil.queryForMap(sqlBuilder.toString()+key+"'");
System.out.println(sqlBuilder.toString()+key+"'"+" 【"+map);
Bean itemBean = (Bean)map.get(key);
long txid=0L;
long val=0L;
if (itemBean!=null) {
val=itemBean.getSum();
txid=itemBean.getTxid();
}
if (config.getType()==StateType.OPAQUE) {
result.add(new OpaqueValue(txid, val));
} else if (config.getType()==StateType.TRANSACTIONAL) {
result.add(new TransactionalValue(txid, val));
} else {
result.add(val);
}
}
return (List<T>) result;
} @Override
public void multiPut(List<List<Object>> keys, List<T> vals) {
//构建新增SQL
StringBuilder sqlBuilder = new StringBuilder("INSERT INTO ").append(config.getTable())
.append("("+config.getCols())
.append(","+config.getColVals())
.append(",txid")
.append(",time")
.append(") VALUES ");
for (int i = 0; i < keys.size(); i++) {
List<Object> key = keys.get(i);
if (config.getType()==StateType.TRANSACTIONAL) {
TransactionalValue val = (TransactionalValue)vals.get(i);
sqlBuilder.append("(");
sqlBuilder.append(key.get(0));
sqlBuilder.append(",");
sqlBuilder.append(val.getVal());
sqlBuilder.append(",");
sqlBuilder.append(val.getTxid());
sqlBuilder.append(",NOW()");
sqlBuilder.append("),");
}
}
sqlBuilder.setLength(sqlBuilder.length()-1);
System.out.println(sqlBuilder.toString());
//新增数据
JDBCUtil jdbcUtil = new JDBCUtil(config.getDriver(),config.getUrl(),config.getUsername(),config.getPassword());
jdbcUtil.insert(sqlBuilder.toString());
} public static Factory getFactory(JDBCStateConfig config) {
return new Factory(config);
} static class Factory implements StateFactory {
private static JDBCStateConfig config;
public Factory(JDBCStateConfig config) {
this.config = config;
}
@Override
public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) {
final CachedMap map = new CachedMap(new JDBCState(config), config.getCacheSize());
System.out.println(config);
if(config.getType()==StateType.OPAQUE) {
return OpaqueMap.build(map);
} else if(config.getType()==StateType.TRANSACTIONAL){
return TransactionalMap.build(map);
}else {
return NonTransactionalMap.build(map);
}
}
} } 3、构建基于IBackingMap的JDBCStateConfig配置类
package storm.trident.mysql; import java.util.List; import org.apache.storm.trident.state.StateType; @SuppressWarnings("all")
public class JDBCStateConfig { private String url;
private String driver;
private String username;
private String password;
private String table;
private int batchSize;
private String cols;
private String colVals;
private int cacheSize = 100;
private StateType type = StateType.OPAQUE; public String getUrl() {
return url;
} public void setUrl(String url) {
this.url = url;
} public String getDriver() {
return driver;
} public void setDriver(String driver) {
this.driver = driver;
} public String getUsername() {
return username;
} public void setUsername(String username) {
this.username = username;
} public String getPassword() {
return password;
} public void setPassword(String password) {
this.password = password;
} public String getTable() {
return table;
} public void setTable(String table) {
this.table = table;
} public int getBatchSize() {
return batchSize;
} public void setBatchSize(int batchSize) {
this.batchSize = batchSize;
} public String getCols() {
return cols;
} public void setCols(String cols) {
this.cols = cols;
} public String getColVals() {
return colVals;
} public void setColVals(String colVals) {
this.colVals = colVals;
} public int getCacheSize() {
return cacheSize;
} public void setCacheSize(int cacheSize) {
this.cacheSize = cacheSize;
} public StateType getType() {
return type;
} public void setType(StateType type) {
this.type = type;
} @Override
public String toString() {
return "Test2StateConfig [url=" + url + ", driver=" + driver + ", username=" + username + ", password="
+ password + ", table=" + table + ", batchSize=" + batchSize + ", cols=" + cols
+ ", colVals=" + colVals + ", cacheSize=" + cacheSize + ", type=" + type + "]";
} } 4、构建JDBC工具类和实体Bean
package storm.trident.mysql; import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map; public class JDBCUtil { private String driver;
private String url;
private String username;
private String password;
private Connection connection;
private PreparedStatement ps;
private ResultSet rs; public JDBCUtil(String driver, String url, String username, String password) {
this.driver = driver;
this.url = url;
this.username = username;
this.password = password;
init();
} void init(){
try {
Class.forName(driver);
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
} public boolean insert(String sql) {
int state = 0;
try {
connection = DriverManager.getConnection(url, username, password);
ps = connection.prepareStatement(sql);
state = ps.executeUpdate();
} catch (SQLException e) {
e.printStackTrace();
} finally {
try {
ps.close();
connection.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
if (state>0) {
return true;
}
return false;
} public Map<String, Object> queryForMap(String sql) {
Map<String, Object> result = new HashMap<String, Object>();
try {
connection = DriverManager.getConnection(url, username, password);
ps = connection.prepareStatement(sql);
rs = ps.executeQuery();
if(rs.next()){
Bean iteBean=new Bean(rs.getString("tel"), rs.getLong("sum"), rs.getLong("txid"), null);
result.put(rs.getString("tel"), iteBean);
}
} catch (SQLException e) {
e.printStackTrace();
} finally {
try {
ps.close();
connection.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
return result;
} public String getDriver() {
return driver;
} public void setDriver(String driver) {
this.driver = driver;
} public String getUrl() {
return url;
} public void setUrl(String url) {
this.url = url;
} public String getUsername() {
return username;
} public void setUsername(String username) {
this.username = username;
} public String getPassword() {
return password;
} public void setPassword(String password) {
this.password = password;
} } package storm.trident.mysql; public class Bean { private String tel;
private long sum;
private long txid;
private String time; public Bean(){
} public Bean(String tel, long sum, long txid, String time) {
super();
this.tel = tel;
this.sum = sum;
this.txid = txid;
this.time = time;
} public String getTel() {
return tel;
} public void setTel(String tel) {
this.tel = tel;
} public long getSum() {
return sum;
} public void setSum(long sum) {
this.sum = sum;
} public long getTxid() {
return txid;
} public void setTxid(long txid) {
this.txid = txid;
} public String getTime() {
return time;
} public void setTime(String time) {
this.time = time;
} @Override
public String toString() {
return "Bean [tel=" + tel + ", sum=" + sum + ", txid=" + txid + ", time=" + time + "]";
} }
Trident-MySQL的更多相关文章
- Phantomjs+Nodejs+Mysql数据抓取(2.抓取图片)
概要 这篇博客是在上一篇博客Phantomjs+Nodejs+Mysql数据抓取(1.抓取数据) http://blog.csdn.net/jokerkon/article/details/50868 ...
- Android+PHP服务器+MySQL实现安卓端的登录
时隔已久的一个任务,今天终于可以画上一个句号了.心情是万分的激动,虽然这份小成就来的有点迟但还是按捺不住心情的澎湃.下面我就先上几张图片来展示一下我的成绩 android源代码: 首先最重要的一件事是 ...
- ASP.NET实现二维码 ASP.Net上传文件 SQL基础语法 C# 动态创建数据库三(MySQL) Net Core 实现谷歌翻译ApI 免费版 C#发布和调试WebService ajax调用WebService实现数据库操作 C# 实体类转json数据过滤掉字段为null的字段
ASP.NET实现二维码 using System;using System.Collections.Generic;using System.Drawing;using System.Linq;us ...
- 猫眼电影和电影天堂数据csv和mysql存储
字符串常用方法 # 去掉左右空格 'hello world'.strip() # 'hello world' # 按指定字符切割 'hello world'.split(' ') # ['hello' ...
- Storm-jdbc-2讲 高级API及Trident
之前对Storm集成JDBC写了一个简单的demo,最近深度研究了下,代码如下 首先,先写一个抽象类,便于减少代码的重复性: import com.google.common.collect.List ...
- Hadoop 中利用 mapreduce 读写 mysql 数据
Hadoop 中利用 mapreduce 读写 mysql 数据 有时候我们在项目中会遇到输入结果集很大,但是输出结果很小,比如一些 pv.uv 数据,然后为了实时查询的需求,或者一些 OLAP ...
- mysql每秒最多能插入多少条数据 ? 死磕性能压测
前段时间搞优化,最后瓶颈发现都在数据库单点上. 问DBA,给我的写入答案是在1W(机械硬盘)左右. 联想起前几天infoQ上一篇文章说他们最好的硬件写入速度在2W后也无法提高(SSD硬盘) 但这东西感 ...
- LINUX篇,设置MYSQL远程访问实用版
每次设置root和远程访问都容易出现问题, 总结了个通用方法, 关键在于实用 step1: # mysql -u root mysql mysql> Grant all privileges o ...
- nodejs进阶(6)—连接MySQL数据库
1. 建库连库 连接MySQL数据库需要安装支持 npm install mysql 我们需要提前安装按mysql sever端 建一个数据库mydb1 mysql> CREATE DATABA ...
随机推荐
- .net版ckeditor配置水印功能(转)
本文简单讲解ckfinder控件给上图片加水印效果. 1.将ckfinder/plugins/watermark/bin/Debug目录下的CKFinder_Watermark.dll和CKFinde ...
- Spring 实例化bean的方式
实例化bean的方式有三种: 1.用构造器来实例化 2.使用静态工厂方法实例化 3.使用实例工厂方法实例化 当采用构造器来创建bean实例时,Spring对class并没有特殊的要求, 我们通常使用的 ...
- 【转】WF4.0实战系列索引
转自:http://www.cnblogs.com/zhuqil/archive/2010/07/05/wf4-in-action-index.html 此系列的工作流文件案例比较多点,实用性好. W ...
- C++函数二义性问题,我怎么感觉编译器有偷懒嫌疑!!!
瞎扯一段,讲得不一定对.纯属学习! struct BB{ void a(){ cout << "bb's a()\n"; }}; struct B1 : public ...
- PHP中的&传值引用的问题,在foreach循环的结果能帮解释下输出的结果原理是什么?
PHP中的&传值引用的问题,在foreach循环的结果能帮解释下输出的结果原理是什么? 代码如下: <?php $arr = array('one','two','three'); fo ...
- java之两个字符串的比较
compareTo() 的返回值是int, 它是先比较对应字符的大小(ASCII码顺序)1.如果字符串相等返回值02.如果第一个字符和参数的第一个字符不等,结束比较,返回他们之间的差值(ascii码值 ...
- jQuery 源码基本框架
抽丝剥茧, 7000+ 行的 jQuery 源码基本可以概括为以下的伪代码 (function (window, undefined) { //将 document 封装成 jQuery 对象并缓存 ...
- CPU与外设传送数据方式
7.2 CPU与外设之间数据传送的方式 在微型计算机系统中,CPU与外设之间的数据传送方式主要有程序传送方式.中断传送方式和直接存储器存取(DMA)传送方式,分别介绍如下. 7.2.1 程序传 ...
- TatukGIS-TGIS_Editor.CreateShape
procedure CreateShape(const _layer: TObject; const _ptg: TGIS_Point3D; const _type: TGIS_ShapeType; ...
- Android app自动化测试之Python+Appium环境搭建
1.安装JDK (1)JDK安装时会有两次,一次是jdk,第二次是jre. (2)环境变量配置: 添加JAVA_HOME变量, 值:Jdk的安装路径 添加CLASSPATH变量,值: .;%JAVA_ ...