03_Storm编程上手-wordcount

1. Storm编程模型概要

消息源spout, 继承BaseRichSpout类 或 实现IRichSpout接口
1）BaseRichSpout类相对比较简单，需要覆写的方法较少，满足基本业务需求
2）IRichSpout接口，需要实现较多的框架定义好的抽象方法，部分方法和BaseRichSpout重叠，通常用于比较复杂的业务数据接入需求

重要方法
open方法： 初始化动作（传入参数，传入上下文），是spout第一个执行的方法，主类中的拓扑配置实例conf会将相关参数作为实参传入给open
nextTuple方法： 数据接入，消息tuple发射 （spout中最重要的方法）
ack方法：传入参数是object,本质是id,标识唯一一个tuple, 实现该tuple被spout成功处理后的逻辑，storm消息保证的API接口
fail方法：与ack方法对应，实现tuple处理失败后的逻辑, storm消息保证的API接口
declareOutputFields方法：声明输出字段，storm中的每个tuple都要求设置输出字段

处理单元bolt: 继承BaseBasicBolt类，或者实现IRichBolt接口

重要方法
*prepare方法：worker初始化，传递参数，声明上下文
*execute方法：接收1个tuple, 实现bolt处理逻辑，发射新tuple(最重要）
*clearup方法：worker关闭前调用，资源释放等，但不能保证该方法一定会被调用（worker进程出现问题等情况）
*declareOutputFields方法：声明输出tuple的每个字段，storm中的每个tuple都要求设置输出字段

2. wordcount代码架构

从拓扑的角度来看各个类间的关系

各个类主要功能

1. spout中随机发送内置的语句作为消息源
2. 第一组bolt进行语句切分，分离出单词，然后发送给下游bolt
3. 第二组bolt，基于字段分组策略，订阅（接收）切分好的单词，完成单词统计并将结果发送给下游
4. 最后1个bolt, 将结果打印到console

3. RandomSentenceSpout

import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
import backtype.storm.utils.Utils;
import java.util.Map;
import java.util.Random;
/**
 *
 * @Author: shayzhang
 * @Email: shayzhang@sina.com
 * @Blog: cn.blogs.com/shay-zhangjin
 * @Describe: select 1 random sentence and send to next-level bolt using field streaming groups
 *
 */
public class RandomSentenceSpout extends BaseRichSpout {
    SpoutOutputCollector  spcollector; //for emit tuple
    Random random;

    //spout initialize, spout执行的第一个方法,主要是从conf自动传入spout发送数据的SpoutOutputCollector
    public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
        spcollector = spoutOutputCollector;
        random = new Random();
    }
    //generate tuple and emit
    public void nextTuple() {
        //每2秒发送一次tuple
        Utils.sleep(2000);
        String[] sentences = new String[]{
                "And if the golden sun",
                "four score and seven years ago",
                "storm hadoop hbase spark",
                "storm is a real-time processing frame-work",
                "you make my night bright"
        };
        // select random sentence
        String msg = sentences[random.nextInt(sentences.length)];
        // emit tuple, doesn't assign message id
        spcollector.emit(new Values(msg.trim().toLowerCase()) );
        // ack tuple, successfully sent
    }

    //每个tuple都要说明字段
    public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        outputFieldsDeclarer.declare(new Fields("sentence") );
    }
}

4. WordNormalizserBolt

import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.IRichBolt;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
import java.util.Map;

/**
 *
 * @author: shayzhang
 * @email: shayzhang@sina.com
 * @blog: cn.blogs.com/shay-zhangjin
 * @description: split sentence into words and emit
 *
 */
public class WordNormalizerBolt implements IRichBolt {
    private OutputCollector wncollector;

    //bolt执行的第一个方法，主类中conf对象，将相关数据推送过来,重点是获取Bolt发送数据使用的OutputCollector
    public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
        wncollector = outputCollector;
    }

    public void execute(Tuple tuple) {
        //获取整个句子
        String sentence = tuple.getString(0);
        //分离出单词
        String words[] = sentence.split(" ");
        //发送每个单词
        for(String word: words){
            wncollector.emit(new Values(word));
            wncollector.ack(tuple);
        }
    }
　　//Bolt退出会执行，但不能保证cleanup方法一定会被执行到（节点故障等）
    public void cleanup() {

    }
    //字段说明
    public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        outputFieldsDeclarer.declare(new Fields("word")); //当下游bolt通过fieldsGrouping方式获取数据时尤为重要，要保障上下游使用的Fields能够对的上
    }

    public Map<String, Object> getComponentConfiguration() {
        return null;
    }
}

5. WordCountBolt

import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.IRichBolt;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;
import java.util.HashMap;
import java.util.Map;

/**
 *
 * @author: shayzhang
 * @email: shayzhang@sina.com
 * @blog: cn.blogs.com/shay-zhangjin
 * @description: count word number, emit top N
 *
 */
public class WordCountBolt implements IRichBolt{
    OutputCollector collector;  //for emit
    Map<String,Integer> counters; //for word count

    //init bolt, bolt执行的第一个方法，主类中的config对象自动进行参数传递
    public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
        collector = outputCollector;
        counters = new HashMap<String, Integer>();

    }

    //统计每个单词的次数，并排序
    public void execute(Tuple tuple) {
        String word = tuple.getString(0);

        //处理收到的每一个word
        if ( !counters.containsKey(word)){
            counters.put(word,1);
        }else{
            counters.put(word, counters.get(word)+1);
        }

        //实时输出 <word, count>
        Integer length = counters.keySet().size();
        Integer count = 0;

        for(String key:counters.keySet()){
            word = "[" + new String(key) + ": " + String.valueOf(counters.get(key)) + "]";
            collector.emit(new Values(word));
            collector.ack(tuple);
            count++;
            if(count==length){
                break;
            }

        }//execute

    }//execute

    public void cleanup() {

    }

    public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        outputFieldsDeclarer.declare(new Fields("wordcount"));
    }

    public Map<String, Object> getComponentConfiguration() {
        return null;
    }
}

6. PrintBolt

import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.BasicOutputCollector;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseBasicBolt;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Tuple;
import java.util.Map;

public class PrintBolt extends BaseBasicBolt{

    OutputCollector collector;

    public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
        collector = outputCollector;
    }

    public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
        try {
            String msg = tuple.getString(0);
            if (msg != null) {
                System.out.println(msg);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        outputFieldsDeclarer.declare(new Fields("result"));
    }
}

7. 主类：构建拓扑，根据是否传入参数决定运行本地模式、集群模式

import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.StormSubmitter;
import backtype.storm.generated.AlreadyAliveException;
import backtype.storm.generated.InvalidTopologyException;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.tuple.Fields;
import backtype.storm.utils.Utils;

public class WordCountTopology {

    public static void main(String args[]) throws AlreadyAliveException, InvalidTopologyException {
        //Topology配置
        Config config = new Config();

        //Topology构造器
        TopologyBuilder builder = new TopologyBuilder();

        //使用builder构建Topology
        //spout, 2个executor
        builder.setSpout("RandomSentence", new RandomSentenceSpout(), 2);

        //bolt, 2个executor, 从id为RandomSentence的spout接收tuple, shuffle方式
        builder.setBolt("WordNormalizer", new WordNormalizerBolt(), 2).shuffleGrouping("RandomSentence");  //从id为“RandomSentence”的spout获取数据

        //bolt, 2个executor,从id为wordNormalizer的bolt接收tuple, fields方式: 根据‘word’字段, 进行单词接收
        builder.setBolt("wordcount", new WordCountBolt(), 2).fieldsGrouping("WordNormalizer", new Fields("word")); //从id为WordNormalizser的bolt获取数据，并且按照word域进行分发

        //bolt, 1个executor, 收集所有结果并打印, global方式接收所有id为wordcount的bolt发出的信息
        builder.setBolt("print", new PrintBolt(), 1).shuffleGrouping("wordcount"); //从id为wordcount的bolt获取数据

        //设置debug, true则打印节点间交换的所有信息
        config.setDebug(false);

        //通过是否有参数，来控制启动本地模式，集群模式
        if(args!=null && args.length>0){
            //有提交参数，执行集群模式
            config.setNumWorkers(1);  //1个worker进程
            try {
                //提交拓扑到storm集群, 拓扑名通过提交参数给定(args[0])
                StormSubmitter.submitTopology(args[0], config, builder.createTopology());
            } catch (AlreadyAliveException e) {
                e.printStackTrace();
            } catch (InvalidTopologyException e) {
                e.printStackTrace();
            }
        }else{
            //无提交参数，执行本地模式，调测用
            //本地模式，通过LocalCluster实例提交拓扑任务
            config.setMaxTaskParallelism(1);
            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("WC_Local", config, builder.createTopology());

            //运行5分钟后,kill该topology任务
            Utils.sleep(300000);
            cluster.killTopology("WC_Local");
            cluster.shutdown();

        }

    }//main
}//WordCountTopology

8. 本地模式运行

在IDEA中选择主类，直接运行即可， 从IDE终端可以看到代码是否正常运行