使用hadoop统计多个文本中每个单词数目

程序源码

 import java.io.IOException;
 import java.util.StringTokenizer;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

 public class WordCount {
     public static class WordCountMap extends
             Mapper<LongWritable, Text, Text, IntWritable> {
         private final IntWritable one = new IntWritable(1);//输出的值  1
         private Text word = new Text();//输出的键 单词

         public void map(LongWritable key, Text value, Context context)
                 throws IOException, InterruptedException {//处理经过  TextInputFormat  产生的  <k1,v1>，然后产生 <k2,v2>
             String line = value.toString();//读取文本中
             StringTokenizer token = new StringTokenizer(line);//按照空格对单词进行切割
             while (token.hasMoreTokens()) {
                 word.set(token.nextToken());//读取到的单词作为键值
                 context.write(word, one);//以  单词,1的中间形式交给reduce处理
             }
         }
     }

     public static class WordCountReduce extends
             Reducer<Text, IntWritable, Text, IntWritable> {
         public void reduce(Text key, Iterable<IntWritable> values,
                 Context context) throws IOException, InterruptedException {
             int sum = 0;
             for (IntWritable val : values) {
                 sum += val.get();
             }
             context.write(key, new IntWritable(sum));
         }
     }

     public static void main(String[] args) throws Exception {
         Configuration conf = new Configuration();
         Job job = new Job(conf);
         job.setJarByClass(WordCount.class);
         job.setJobName("wordcount");
         job.setOutputKeyClass(Text.class);
         job.setOutputValueClass(IntWritable.class);
         job.setMapperClass(WordCountMap.class);
         job.setReducerClass(WordCountReduce.class);
         job.setInputFormatClass(TextInputFormat.class);//生成可供Map处理的键值对
         job.setOutputFormatClass(TextOutputFormat.class);
         FileInputFormat.addInputPath(job, new Path(args[0]));
         FileOutputFormat.setOutputPath(job, new Path(args[1]));
         job.waitForCompletion(true);
     }
 }

1 编译源码

javac -classpath /opt/hadoop-1.2.1/hadoop-core-1.2.1.jar:/opt/hadoop-1.2.1/lib/commons-cli-1.2.jar -d ./word_count_class/ WordCount.java
将源码编译成class文件并放在当前文件夹下的word_count_class目录，当然，首先需要创建该目录

2 将源码打成jar包

进入源码目录

jar -cvf wordcount.jar  *

3 上传输入文件

先在hadoop中为本次任务创建一个输入文件存放目录

hadoop fs -mkdir input_wordcount

将input目录下的所有文本文件上传到hadoop中的input_wordcount目录下

hadoop fs -put input/* input_wordcount/

注意：不能在运行前穿创建输出文件夹

4 上传jar并执行

hadoop jar word_count_class/wordcount.jar input_wordcount output_wordcount

5 查看计算结果

程序输出目录

hadoop fs -ls output_wordcount

程序输出内容

hadoop fs -cat output_wordcount/part-r-00000

---

---

版本二：自己实际操作中的程序

Map程序

 package com.zln.chapter03;

 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.MapReduceBase;
 import org.apache.hadoop.mapred.Mapper;
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reporter;

 import java.io.IOException;
 import java.util.StringTokenizer;

 /**
  * Created by sherry on 15-7-12.
  */
 public class WordCountMap extends MapReduceBase implements Mapper<LongWritable,Text,Text,IntWritable> {
     private final static IntWritable one = new IntWritable(1);//每个单词 +1
     private Text word = new Text();

     @Override
     public void map(LongWritable longWritable, Text text, OutputCollector<Text, IntWritable> outputCollector, Reporter reporter) throws IOException {
         String line = text.toString();
         StringTokenizer tokenizer = new StringTokenizer(line);//分割出单词
         while (tokenizer.hasMoreTokens()){
             word.set(tokenizer.nextToken());
             outputCollector.collect(word,one);
         }
     }
 }

Reduce程序

 package com.zln.chapter03;

 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.MapReduceBase;
 import org.apache.hadoop.mapred.OutputCollector;
 import org.apache.hadoop.mapred.Reducer;
 import org.apache.hadoop.mapred.Reporter;

 import java.io.IOException;
 import java.util.Iterator;

 /**
  * Created by sherry on 15-7-12.
  */
 public class WordCountReduce extends MapReduceBase implements Reducer<Text,IntWritable,Text,IntWritable> {
     @Override
     public void reduce(Text text, Iterator<IntWritable> iterator, OutputCollector<Text, IntWritable> outputCollector, Reporter reporter) throws IOException {
         int sum = 0;
         while (iterator.hasNext()){
             sum += iterator.next().get();
         }
         outputCollector.collect(text,new IntWritable(sum));
     }
 }

主函数

 package com.zln.chapter03;

 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.*;

 import java.io.IOException;

 /**
  * Created by sherry on 15-7-12.
  */
 public class WordCount {
     public static void main(String[] args) throws IOException {
         JobConf conf = new JobConf(WordCount.class);
         conf.setJobName("wordCount");

         //设置输出格式
         conf.setOutputKeyClass(Text.class);
         conf.setOutputValueClass(IntWritable.class);

         //设置MapReduce类
         conf.setMapperClass(WordCountMap.class);
         conf.setReducerClass(WordCountReduce.class);

         //设置处理输入类
         conf.setInputFormat(TextInputFormat.class);
         //设置处理输出类
         conf.setOutputFormat(TextOutputFormat.class);

         FileInputFormat.setInputPaths(conf, new Path(args[0]));
         FileOutputFormat.setOutputPath(conf, new Path(args[1]));

         JobClient.runJob(conf);
     }
 }

准备输入文件

file1

Hello Word By Word
Hello Word By zln

file2

Hello Hadoop
Hello GoodBye

放在同一个目录下：/home/sherry/IdeaProjects/Hadoop/WordCount/输入文件准备

编译class打成一个jar包

我使用IDEA进行编译。注意不要忘记指定main函数

上传输入文件

root@sherry:/opt/hadoop-1.2.# hadoop fs -mkdir /user/root/zln/WordCount/InputFiles

root@sherry:/opt/hadoop-1.2.# hadoop fs -put /home/sherry/IdeaProjects/Hadoop/WordCount/输入文件准备/* /user/root/zln/WordCount/InputFiles

上传jar并执行

root@sherry:/opt/hadoop-1.2.# hadoop jar /home/sherry/IdeaProjects/Hadoop/out/artifacts/WordCount_jar/WordCount.jar /user/root/zln/WordCount/InputFiles /user/root/zln/WordCount/OutputFiles

查看执行结果

root@sherry:/opt/hadoop-1.2.# hadoop fs -ls /user/root/zln/WordCount/OutputFiles
root@sherry:/opt/hadoop-1.2.# hadoop fs -text /user/root/zln/WordCount/OutputFiles/part-

---

---

版本三：使用新版本的API对Map  Reduce  main函数进行重写

Map

 package com.zln.chapter03;

 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Mapper;

 import java.io.IOException;
 import java.util.StringTokenizer;

 /**
  * Created by sherry on 15-7-12.
  */
 public class WordCountMap extends Mapper<LongWritable,Text,Text,IntWritable> {
     private final static IntWritable one = new IntWritable(1);//每个单词 +1
     private Text word = new Text();

     @Override
     protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
         String line = value.toString();
         StringTokenizer tokenizer = new StringTokenizer(line);//分割出单词
         while (tokenizer.hasMoreTokens()){
             word.set(tokenizer.nextToken());
             context.write(word,one);
         }
     }

 }

Reduce

 package com.zln.chapter03;

 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Reducer;

 import java.io.IOException;

 /**
  * Created by sherry on 15-7-12.
  */
 public class WordCountReduce extends Reducer<Text,IntWritable,Text,IntWritable> {

     @Override
     protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
         int sum = 0;
         for (IntWritable intWritable:values){
             sum += intWritable.get();
         }
         context.write(key,new IntWritable(sum));
     }
 }

Main

 package com.zln.chapter03;

 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;

 /**
  * Created by sherry on 15-7-12.
  */
 public class WordCount extends Configured implements Tool{

     public int run(String[] args) throws Exception {
         Job job = new Job(getConf());
         job.setJarByClass(WordCount.class);
         job.setJobName("WordCount");

         job.setOutputKeyClass(Text.class);
         job.setOutputValueClass(IntWritable.class);

         job.setMapperClass(WordCountMap.class);
         job.setReducerClass(WordCountReduce.class);

         job.setInputFormatClass(TextInputFormat.class);
         job.setOutputFormatClass(TextOutputFormat.class);

         FileInputFormat.setInputPaths(job,new Path(args[0]));
         FileOutputFormat.setOutputPath(job,new Path(args[1]));

         boolean success = job.waitForCompletion(true);
         return success?0:1;
     }

     public static void main(String[] args) throws Exception {
         int ret = ToolRunner.run(new WordCount(),args);
         System.exit(ret);
     }
 }