Storm完整例子

1. import backtype.storm.spout.SpoutOutputCollector;
2. import backtype.storm.task.TopologyContext;
3. import backtype.storm.topology.base.BaseRichSpout;
4. import backtype.storm.utils.Utils;
5. import backtype.storm.Config;
6. import backtype.storm.LocalCluster;
7. import backtype.storm.StormSubmitter;
8. import backtype.storm.task.ShellBolt;
9. import backtype.storm.topology.BasicOutputCollector;
10. import backtype.storm.topology.IRichBolt;
11. import backtype.storm.topology.OutputFieldsDeclarer;
12. import backtype.storm.topology.TopologyBuilder;
13. import backtype.storm.topology.base.BaseBasicBolt;
14. import backtype.storm.tuple.Fields;
15. import backtype.storm.tuple.Tuple;
16. import backtype.storm.tuple.Values;
17. import java.util.*;
18. //import java.util.HashMap;
19. //import java.util.Map;
20. //import java.util.Random;
21. //import java.util.StringTokenizer;
22. /*
23. ** WordCountTopolopgyAllInJava类（单词计数）
24. */
25. public class  WordCountTopolopgyAllInJava{
26. // 定义一个喷头，用于产生数据。该类继承自BaseRichSpout
27. public static class RandomSentenceSpout extends BaseRichSpout {
28. SpoutOutputCollector _collector;
29. Random _rand;
30. @Override
31. public void open(Map conf, TopologyContext context, SpoutOutputCollector collector){
32. _collector = collector;
33. _rand = new Random();
34. }
35. @Override
36. public void nextTuple(){
37. // 睡眠一段时间后再产生一个数据
38. Utils.sleep(100);
39. // 句子数组
40. String[] sentences = new String[]{ "the cow jumped over the moon", "an apple a day keeps the doctor away",
41. "four score and seven years ago", "snow white and the seven dwarfs", "i am at two with nature" };
42. // 随机选择一个句子
43. String sentence = sentences[_rand.nextInt(sentences.length)];
44. // 发射该句子给Bolt
45. _collector.emit(new Values(sentence));
46. }
47. // 确认函数
48. @Override
49. public void ack(Object id){
50. }
51. // 处理失败的时候调用
52. @Override
53. public void fail(Object id){
54. }
55. @Override
56. public void declareOutputFields(OutputFieldsDeclarer declarer){
57. // 定义一个字段word
58. declarer.declare(new Fields("word"));
59. }
60. }
61. // 定义个Bolt，用于将句子切分为单词
62. public static class SplitSentence extends BaseBasicBolt{
63. @Override
64. public void execute(Tuple tuple, BasicOutputCollector collector){
65. // 接收到一个句子
66. String sentence = tuple.getString(0);
67. // 把句子切割为单词
68. StringTokenizer iter = new StringTokenizer(sentence);
69. // 发送每一个单词
70. while(iter.hasMoreElements()){
71. collector.emit(new Values(iter.nextToken()));
72. }
73. }
74. @Override
75. public void declareOutputFields(OutputFieldsDeclarer declarer){
76. // 定义一个字段
77. declarer.declare(new Fields("word"));
78. }
79. }
80. // 定义一个Bolt，用于单词计数
81. public static class WordCount extends BaseBasicBolt {
82. Map<String, Integer> counts = new HashMap<String, Integer>();
83. @Override
84. public void execute(Tuple tuple, BasicOutputCollector collector){
85. // 接收一个单词
86. String word = tuple.getString(0);
87. // 获取该单词对应的计数
88. Integer count = counts.get(word);
89. if(count == null)
90. count = 0;
91. // 计数增加
92. count++;
93. // 将单词和对应的计数加入map中
94. counts.put(word,count);
95. System.out.println("hello word!");
96. System.out.println(word +"  "+count);
97. // 发送单词和计数（分别对应字段word和count）
98. collector.emit(new Values(word, count));
99. }
100. @Override
101. public void declareOutputFields(OutputFieldsDeclarer declarer){
102. // 定义两个字段word和count
103. declarer.declare(new Fields("word","count"));
104. }
105. }
106. public static void main(String[] args) throws Exception
107. {
108. // 创建一个拓扑
109. TopologyBuilder builder = new TopologyBuilder();
110. // 设置Spout，这个Spout的名字叫做"Spout"，设置并行度为5
111. builder.setSpout("Spout", new RandomSentenceSpout(), 5);
112. // 设置slot——“split”，并行度为8，它的数据来源是spout的
113. builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
114. // 设置slot——“count”,你并行度为12，它的数据来源是split的word字段
115. builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));
116. Config conf = new Config();
117. conf.setDebug(false);
118. //if(args != null && args.length > 0){
119. //if(false){
120. //  conf.setNumWorkers(3);
121. //  StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
122. //}else{
123. conf.setMaxTaskParallelism(3);
124. // 本地集群
125. LocalCluster cluster = new LocalCluster();
126. // 提交拓扑（该拓扑的名字叫word-count）
127. cluster.submitTopology("word-count", conf, builder.createTopology() );
128. Thread.sleep(10000);
129. //  cluster.shutdown();
130. //}
131. }
132. }

使用maven编译该项目： mvn clean package

运行：storm jar word-count-1.0.jar WordCountTopolopgyAllInJava

结果如下：

hello word!
moon    811
hello word!
an      829
hello word!
apple   829
hello word!
a       829
hello word!
keeps   829
hello word!
day     829
hello word!
score   800
hello word!

pom.xml文件定义如下

1. <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2. xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3. <modelVersion>4.0.0</modelVersion>
4. <groupId>storm-yqj</groupId>
5. <artifactId>word-count</artifactId>
6. <version>1.0</version>
7. <packaging>jar</packaging>
8. <name>word-count</name>
9. <url>http://maven.apache.org</url>
10. <properties>
11. <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
12. </properties>
13. <dependencies>
14. <dependency>
15. <groupId>junit</groupId>
16. <artifactId>junit</artifactId>
17. <version>3.8.1</version>
18. <scope>test</scope>
19. </dependency>
20. <dependency>
21. <groupId>org.testng</groupId>
22. <artifactId>testng</artifactId>
23. <version>6.8.5</version>
24. <scope>test</scope>
25. </dependency>
26. <dependency>
27. <groupId>org.mockito</groupId>
28. <artifactId>mockito-all</artifactId>
29. <version>1.9.0</version>
30. <scope>test</scope>
31. </dependency>
32. <dependency>
33. <groupId>org.easytesting</groupId>
34. <artifactId>fest-assert-core</artifactId>
35. <version>2.0M8</version>
36. <scope>test</scope>
37. </dependency>
38. <dependency>
39. <groupId>org.jmock</groupId>
40. <artifactId>jmock</artifactId>
41. <version>2.6.0</version>
42. <scope>test</scope>
43. </dependency>
44. <dependency>
45. <groupId>org.apache.storm</groupId>
46. <artifactId>storm-core</artifactId>
47. <version>0.9.1-incubating</version>
48. </dependency>
49. <dependency>
50. <groupId>commons-collections</groupId>
51. <artifactId>commons-collections</artifactId>
52. <version>3.2.1</version>
53. </dependency>
54. <dependency>
55. <groupId>com.google.guava</groupId>
56. <artifactId>guava</artifactId>
57. <version>15.0</version>
58. </dependency>
59. </dependencies>
60. <build>
61. <resources>
62. <resource>
63. <directory>${basedir}/multilang</directory>
64. </resource>
65. </resources>
66. <plugins>
67. <plugin>
68. <artifactId>maven-assembly-plugin</artifactId>
69. <configuration>
70. <descriptorRefs>
71. <descriptorRef>jar-with-dependencies</descriptorRef>
72. </descriptorRefs>
73. <archive>
74. <manifest>
75. <mainClass></mainClass>
76. </manifest>
77. </archive>
78. </configuration>
79. <executions>
80. <execution>
81. <id>make-assembly</id>
82. <phase>package</phase>
83. <goals>
84. <goal>single</goal>
85. </goals>
86. </execution>
87. </executions>
88. </plugin>
89. <plugin>
90. <groupId>com.theoryinpractise</groupId>
91. <artifactId>clojure-maven-plugin</artifactId>
92. <version>1.3.12</version>
93. <extensions>true</extensions>
94. <configuration>
95. <sourceDirectories>
96. <sourceDirectory>src/clj</sourceDirectory>
97. </sourceDirectories>
98. </configuration>
99. <executions>
100. <execution>
101. <id>compile</id>
102. <phase>compile</phase>
103. <goals>
104. <goal>compile</goal>
105. </goals>
106. </execution>
107. <execution>
108. <id>test</id>
109. <phase>test</phase>
110. <goals>
111. <goal>test</goal>
112. </goals>
113. </execution>
114. </executions>
115. </plugin>
116. <plugin>
117. <groupId>org.codehaus.mojo</groupId>
118. <artifactId>exec-maven-plugin</artifactId>
119. <version>1.2.1</version>
120. <executions>
121. <execution>
122. <goals>
123. <goal>exec</goal>
124. </goals>
125. </execution>
126. </executions>
127. <configuration>
128. <executable>java</executable>
129. <includeProjectDependencies>true</includeProjectDependencies>
130. <includePluginDependencies>false</includePluginDependencies>
131. <classpathScope>compile</classpathScope>
132. <mainClass>${storm.topology}</mainClass>
133. </configuration>
134. </plugin>
135. <plugin>
136. <groupId>org.apache.maven.plugins</groupId>
137. <artifactId>maven-compiler-plugin</artifactId>
138. <configuration>
139. <source>1.6</source>
140. <target>1.6</target>
141. </configuration>
142. </plugin>
143. </plugins>
144. </build>
145. </project>