第一个MapReduce程序

计算文件中每个单词的频数

wordcount 程序调用 wordmap 和 wordreduce 程序。

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

 public class wordcount {

     /**
      * @param args
      */
     public static void main(String[] args) throws Exception {
         // TODO Auto-generated method stub

         Configuration conf = new Configuration();
         Job job = new Job(conf,"wordcount");
         job.setJarByClass(wordcount.class);

         job.setMapperClass(wordmap.class);
         job.setReducerClass(wordreduce.class);

         job.setInputFormatClass(TextInputFormat.class);
         job.setOutputFormatClass(TextOutputFormat.class);

         FileInputFormat.addInputPath(job,new Path(args[0]));
         FileOutputFormat.setOutputPath(job, new Path(args[1]));

         job.setOutputKeyClass(Text.class);
         job.setOutputValueClass(IntWritable.class);

         job.waitForCompletion(true);

     }

 }

wordmap 程序的输入为<key,value>（key是当前输入的行数，value对应的是行的内容），然后对此行的内容进行切词，每切下一个词就将其组织成<word,1>的形式，word表示文本内容，1代表出现了一次。

 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Mapper;

 public class wordmap extends Mapper<LongWritable, Text, Text, IntWritable> {

     private static final IntWritable one = new IntWritable(1);
     protected void map(
             LongWritable key,
             Text value,
             org.apache.hadoop.mapreduce.Mapper<LongWritable, Text, Text, IntWritable>.Context context)
             throws java.io.IOException, InterruptedException {

         String line = value.toString();
         String[] words = line.split(" ");
         for(String word : words){
             context.write(new Text(word), one);

         }

     };

 }

wordreduce 程序会接受到<word,{1,1,1,1……}>形式的数据，也就是特定单词及其出现的次数，其中 "1" 表示 word 出现的频数，所以每接收一个<word,{1,1,1,1……}>，就会在 word 的频数加 1 ，最后组织成<word,sum>的形式直接输出。

 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Reducer;

 public class wordreduce extends Reducer<Text, IntWritable, Text, IntWritable> {

     protected void reduce(
             Text key,
             java.lang.Iterable<IntWritable> values,
             org.apache.hadoop.mapreduce.Reducer<Text, IntWritable, Text, IntWritable>.Context context)
             throws java.io.IOException, InterruptedException {

         int sum = 0;
         for(IntWritable count : values){
             sum+= count.get();

         }
         context.write(key, new IntWritable(sum));
     };

 }