Trident中使用HBase进行状态管理

1.使用的类

　　

2.使用HBaseMapState

　　

3.使用状态管理

　　使用的状态管理还要看Spout

　　StateFactory factory1 = HBaseMapState.opaque(opts1);

4.服务

　　需要Kafka，zookeeper

　　然后Hbase，前提需要Hadoop

　　

5.主驱动类

 package com.jun.tridentWithHbase;

 import backtype.storm.Config;
 import backtype.storm.LocalCluster;
 import backtype.storm.StormSubmitter;
 import backtype.storm.generated.AlreadyAliveException;
 import backtype.storm.generated.InvalidTopologyException;
 import backtype.storm.tuple.Fields;
 import backtype.storm.tuple.Values;
 import org.apache.storm.hbase.trident.state.HBaseMapState;
 import storm.trident.Stream;
 import storm.trident.TridentState;
 import storm.trident.TridentTopology;
 import storm.trident.operation.builtin.Count;
 import storm.trident.operation.builtin.Sum;
 import storm.trident.state.OpaqueValue;
 import storm.trident.state.StateFactory;
 import storm.trident.testing.FixedBatchSpout;
 import storm.trident.testing.MemoryMapState;

 public class TridentDemo {
     public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
         TridentTopology tridentTopology=new TridentTopology();
         //模拟数据
         Fields field=new Fields("log","flag");
         FixedBatchSpout spout=new FixedBatchSpout(field,5,
             new Values("168.214.187.214 - - [1481953616092] \"GET /view.php HTTP/1.1\" 200 0 \"http://cn.bing.com/search?q=spark mllib\" \"Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1\" \"-\"","A"),
             new Values("168.187.202.202 - - [1481953537038] \"GET /IBEIfeng.gif?order_id=1063&orderTime=1481953537038&memberId=4000012340500607&productInfos=10005-2099.48-B-1|10004-1886.62-A-2|10001-961.99-A-1&orderAmt=6834.70 HTTP/1.1\" 200 0 \"-\" \"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2;Tident/6.0)\" \"-\"","A"),
             new Values("61.30.167.187 - - [1481953539039] \"GET /IBEIfeng.gif?order_id=1064&orderTime=1481953539039&memberId=4000930409959999&productInfos=10007-3329.13-B-1|10009-2607.71-B-1|10002-390.62-A-1|10006-411.00-B-2&orderAmt=7149.46 HTTP/1.1\" 200 0 \"-\" \"Mozilla/5.0 (Linux; Android 4.2.1; Galaxy Nexus Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19\" \"-\"","A"),
             new Values("30.29.132.190 - - [1481953544042] \"GET /IBEIfeng.gif?order_id=1065&orderTime=1481953544043&memberId=1234568970080798&productInfos=10005-2099.48-B-1|10001-3242.40-C-2|10006-411.00-B-1&orderAmt=8995.28 HTTP/1.1\" 200 0 \"-\" \"Mozilla/5.0 (iPhone; CPU iPhone OS 7_)_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11B511 Safari/9537.53\" \"-\"","B"),
             new Values("222.190.187.201 - - [1481953578068] \"GET /IBEIfeng.gif?order_id=1066&orderTime=1481953578068&memberId=3488586887970809&productInfos=10005-2099.48-B-1|10001-2774.16-C-2&orderAmt=7647.80 HTTP/1.1\" 200 0 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1\" \"-\"","B"),
             new Values("72.202.43.53 - - [1481953579069] \"GET /IBEIfeng.gif?order_id=1067&orderTime=1481953579069&memberId=2084859896989877&productInfos=10007-3329.13-B-1|10001-961.99-A-2&orderAmt=5253.10 HTTP/1.1\" 200 0 \"-\" \"Mozilla/5.0 (Linux; Android 4.2.1; Galaxy Nexus Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19\" \"-\"","B")
         );
         //多次循环
         spout.setCycle(true);
         //流处理
         Stream stream=tridentTopology.newStream("orderAnalyse",spout)
                 //过滤
             .each(new Fields("log"),new ValidLogFilter())
                 //解析
             .each(new Fields("log"), new LogParserFunction(),new Fields("orderId","orderTime","orderAmtStr","memberId"))
                 //投影
             .project(new Fields("orderId","orderTime","orderAmtStr","memberId"))
                 //时间解析
             .each(new Fields("orderTime"),new DateTransFormerFunction(),new Fields("day","hour","minter"))
          ;
         //分流
         //1.基于minter统计订单数量，分组统计
         TridentState state=stream.groupBy(new Fields("minter"))
                 //全局聚合，使用内存存储状态信息
                 .persistentAggregate(new MemoryMapState.Factory(),new Count(),new Fields("orderNumByMinter"));
 //        state.newValuesStream().each(new Fields("minter","orderNumByMinter"),new PrintFilter());

         //2.另一个流,基于分钟的订单金额，局部聚合
         Stream partitionStream=stream.each(new Fields("orderAmtStr"),new TransforAmtToDoubleFunction(),new Fields("orderAmt"))
             .groupBy(new Fields("minter"))
                     //局部聚合
                 .chainedAgg()    //聚合链
             .partitionAggregate(new Fields("orderAmt"),new LocalSum(),new Fields("orderAmtSumOfLocal"))
                 .chainEnd();      //聚合链

         //*******************************使用Hbase存储***********************
             //产生Factory，而不是内存MemoryMapState.Factory()
         HBaseMapState.Options<OpaqueValue> opts1 = new HBaseMapState.Options<>();
         opts1.tableName ="orderAnalyse";       //表名称
         opts1.columnFamily = "cf";              // 列簇名称
         opts1.qualifier = "totalOrderAmt";
         StateFactory factory1 = HBaseMapState.opaque(opts1);

         //*******************************************************************

         //做一次全局聚合
         TridentState partitionState=partitionStream.groupBy(new Fields("minter"))
                 //全局聚合
                 .persistentAggregate(factory1,new Fields("orderAmtSumOfLocal"),new Sum(),new Fields("totalOrderAmt"));
         partitionState.newValuesStream().each(new Fields("minter","totalOrderAmt"),new PrintFilter());

         //提交
         Config config=new Config();
         if(args==null || args.length<=0){
             LocalCluster localCluster=new LocalCluster();
             localCluster.submitTopology("tridentDemo",config,tridentTopology.build());
         }else {
             config.setNumWorkers(2);
             StormSubmitter.submitTopology(args[0],config,tridentTopology.build());
         }
     }
 }

6.HBase中效果