MapReduce编程：词频统计

首先在项目的src文件中需要加入以下文件，log4j的内容为：

log4j.rootLogger=INFO, stdout

log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n

log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

代码如下：

 package org.apache.hadoop.examples;

     import java.io.IOException;
     import java.util.Iterator;
     import java.util.StringTokenizer;
     import org.apache.hadoop.conf.Configuration;
     import org.apache.hadoop.fs.Path;
     import org.apache.hadoop.io.IntWritable;
     import org.apache.hadoop.io.Text;
     import org.apache.hadoop.mapreduce.Job;
     import org.apache.hadoop.mapreduce.Mapper;
     import org.apache.hadoop.mapreduce.Reducer;
     import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
     import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
     import org.apache.hadoop.util.GenericOptionsParser;

     public class WordCount {
         public WordCount() {
         }

         //main函数，MapReduce程序运行的入口
         public static void main(String[] args) throws Exception {
             Configuration conf = new Configuration();   //指定HDFS相关的参数

             //String[] otherArgs = (new GenericOptionsParser(conf, args)).getRemainingArgs();
             String[] otherArgs = new String[]{"input","output"};
             if(otherArgs.length < 2) {
                 System.err.println("Usage: wordcount <in> [<in>...] <out>");
                 System.exit(2);
             }

             //通过Job类设置Hadoop程序运行时的环境变量
             Job job = Job.getInstance(conf, "word count");  //设置环境参数
             job.setJarByClass(WordCount.class);  //设置整个程序的类名
             job.setMapperClass(WordCount.TokenizerMapper.class); //添加Mapper类
             job.setCombinerClass(WordCount.IntSumReducer.class);
             job.setReducerClass(WordCount.IntSumReducer.class); //添加Reducer类
             job.setOutputKeyClass(Text.class);  //设置输出类型，因为输出的形式是<单词，个数>，所以这里用Text，类似于Java的String，但还是有些区别
             job.setOutputValueClass(IntWritable.class);  //设置输出类型，类似于Java的Int

             for(int i = 0; i < otherArgs.length - 1; ++i) {
                 FileInputFormat.addInputPath(job, new Path(otherArgs[i]));    //设置输入文件
             }

             FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));   //设置输出文件
             System.exit(job.waitForCompletion(true)?0:1);  //提交作业
         }

         //Reduce处理逻辑
         public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
             private IntWritable result = new IntWritable();

             public IntSumReducer() {
             }

             public void reduce(Text key, Iterable<IntWritable> values, Reducer<Text, IntWritable, Text, IntWritable>.Context context) throws IOException, InterruptedException {
                 int sum = 0;

                 IntWritable val;
                 for(Iterator i$ = values.iterator(); i$.hasNext(); sum += val.get()) {
                     val = (IntWritable)i$.next();
                 }

                 this.result.set(sum);
                 context.write(key, this.result);
             }
         }

         //Map处理逻辑
         public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {
             private static final IntWritable one = new IntWritable(1);
             private Text word = new Text();

             public TokenizerMapper() {
             }

             public void map(Object key, Text value, Mapper<Object, Text, Text, IntWritable>.Context context) throws IOException, InterruptedException {
                 StringTokenizer itr = new StringTokenizer(value.toString());   //分词器

                 while(itr.hasMoreTokens()) {
                     this.word.set(itr.nextToken());
                     context.write(this.word, one);  //输出键值对
                     //这里也可以直接写成context.write(new Text(word), new IntWritable(1));
                 }

             }
         }
     }