027_编写MapReduce的模板类Mapper、Reducer和Driver

模板类编写好后写MapReduce程序，的模板类编写好以后只需要改参数就行了，代码如下：

 package org.dragon.hadoop.mr.module;

 import java.io.IOException;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
 import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;

 /**
  *
  * ###########################################
  * ############     MapReduce 模板类        ##########
  * ###########################################
  *
  * @author ZhuXY
  * @time 2016-3-13 下午10:21:06
  *
  */
 public class ModuleMapReduce extends Configured implements Tool {

     /**
      * Mapper Class
      */
     public static class ModuleMapper extends
             Mapper<LongWritable, Text, LongWritable, Text> {

         @Override
         protected void setup(Context context) throws IOException,
                 InterruptedException {
             super.setup(context);
         }

         @Override
         protected void map(LongWritable key, Text value, Context context)
                 throws IOException, InterruptedException {
             super.map(key, value, context);
         }

         @Override
         protected void cleanup(Context context) throws IOException,
                 InterruptedException {
             super.cleanup(context);
         }
     }

     /**
      * Reducer Class
      */
     public static class ModuleReducer extends
             Reducer<LongWritable, Text, LongWritable, Text> {

         @Override
         protected void setup(Context context) throws IOException,
                 InterruptedException {
             // TODO Auto-generated method stub
             super.setup(context);
         }

         @Override
         protected void reduce(LongWritable key, Iterable<Text> values,
                 Context context) throws IOException, InterruptedException {
             // TODO Auto-generated method stub
             super.reduce(key, values, context);
         }

         @Override
         protected void cleanup(Context context) throws IOException,
                 InterruptedException {
             // TODO Auto-generated method stub
             super.cleanup(context);
         }

     }

     /**
      * Driver Class
      */

     //    专门抽取一个方法出来用于设置
     public Job parseInputAndOutput(Tool tool,Configuration conf,String[] args) throws IOException
     {
         if (args.length!=2) {
             System.err.printf("Usage:%s [generic options] <input> <output>\n", tool.getClass().getSimpleName());
             ToolRunner.printGenericCommandUsage(System.err);
             return null;
         }

         //创建job，并设置配置信息和job名称
         Job job=new Job(conf, ModuleMapReduce.class.getSimpleName());

         //设置job的运行类
         // step 3：set job
         // 1) set run jar class
         job.setJarByClass(tool.getClass());

         // 14) job output path
         FileOutputFormat.setOutputPath(job, new Path(args[1]));

         return job;
     }

     @Override
     public int run(String[] args) throws Exception {

         // step 1：get conf
         Configuration conf = new Configuration();

         // step 2：create job
         Job job = parseInputAndOutput(this, conf, args);

         // 2) set input format
         job.setInputFormatClass(TextInputFormat.class); // 可省

         // 3) set input path
         FileInputFormat.addInputPath(job, new Path(args[0]));

         // 4) set mapper class
         job.setMapperClass(ModuleMapper.class); // 可省

         // 5)set map input key/value class
         job.setMapOutputKeyClass(LongWritable.class); // 可省
         job.setMapOutputValueClass(Text.class); // 可省

         // 6) set partitioner class
         job.setPartitionerClass(HashPartitioner.class); // 可省

         // 7) set reducer number
         job.setNumReduceTasks(1);// default 1 //可省

         // 8)set sort comparator class
         //job.setSortComparatorClass(LongWritable.Comparator.class); // 可省

         // 9) set group comparator class
         //job.setGroupingComparatorClass(LongWritable.Comparator.class); // 可省

         // 10) set combiner class
         // job.setCombinerClass(null);默认是null，但是此处不能写 //可省

         // 11) set reducer class
         job.setReducerClass(ModuleReducer.class); // 可省

         // 12) set output format
         job.setOutputFormatClass(TextOutputFormat.class); // 可省

         // 13) job output key/value class
         job.setOutputKeyClass(LongWritable.class); // 可省
         job.setOutputValueClass(Text.class); // 可省

         // step 4: submit job
         boolean isSuccess = job.waitForCompletion(true);

         // step 5: return status
         return isSuccess ? 0 : 1;
     }

     public static void main(String[] args) throws Exception {

         args = new String[] {
                 "hdfs://hadoop-master.dragon.org:9000/wc/mininput/",
                 "hdfs://hadoop-master.dragon.org:9000/wc/minoutput"
                 };

         //run mapreduce
         int status=ToolRunner.run(new ModuleMapReduce(), args);

         //exit
         System.exit(status);
     }
 }

View Module Code

模板使用步骤：

1） 改名称（MapReduce类的名称、Mapper类的名称、Reducer类的名称）

2） 依据实际的业务逻辑修改Mapper类和Reducer类的Key/Value输入输出参数的类型

3） 修改驱动Driver部分的Job的参数设置（Mapper类和Reducer类的输出类型）

4） 在Mapper类中编写实际的业务逻辑（setup()、map()、cleanup()）

5） 在Reducer类中编写实际的业务逻辑（setup()、map()、cleanup()）

6） 检查并修改驱动Driver代码（模板类中的run()方法）

7） 设置输入输出路径，进行MR测试。

使用ModuleMapReduce编写wordcount程序

 package org.dragon.hadoop.mr.module;

 import java.io.IOException;
 import java.util.StringTokenizer;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
 import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;

 /**
  *
  * ###########################################
  * ############        MapReduce 模板类     ##########
  * ###########################################
  *
  * @author ZhuXY
  * @time 2016-3-13 下午10:21:06
  *
  */
 public class WordcountByModuleMapReduce extends Configured implements Tool {

     /**
      * Mapper Class
      */
     public static class WordcountMapper extends
             Mapper<LongWritable, Text, Text, LongWritable> {

         @Override
         protected void setup(Context context) throws IOException,
                 InterruptedException {
             super.setup(context);
         }

         private Text word = new Text();
         private final static LongWritable one = new LongWritable(1);

         @Override
         protected void map(LongWritable key, Text value, Context context)
                 throws IOException, InterruptedException {

             // 获取每行数据的值
             String lineValue = value.toString();

             // 进行分割
             StringTokenizer stringTokenizer = new StringTokenizer(lineValue);

             // 遍历
             while (stringTokenizer.hasMoreElements()) {

                 // 获取每个值
                 String worldValue = stringTokenizer.nextToken();

                 // 设置map, 输入的key值
                 word.set(worldValue);
                 context.write(word, one); // 如果出现就出现一次，存在每行出现几次，这时候键的值一样，多个键值对
             }
         }

         @Override
         protected void cleanup(Context context) throws IOException,
                 InterruptedException {
             super.cleanup(context);
         }
     }

     /**
      * Reducer Class
      */
     public static class WordcountReducer extends
             Reducer<Text, LongWritable, Text, LongWritable> {
         private LongWritable resultLongWritable = new LongWritable();

         @Override
         protected void setup(Context context) throws IOException,
                 InterruptedException {
             // TODO Auto-generated method stub
             super.setup(context);
         }

         @Override
         protected void reduce(Text key, Iterable<LongWritable> values,
                 Context context) throws IOException, InterruptedException {
             int sum = 0;
             // 循环遍历Interable
             for (LongWritable value : values) {
                 // 累加
                 sum += value.get();
             }

             // 设置总次数
             resultLongWritable.set(sum);
             context.write(key, resultLongWritable);
         }

         @Override
         protected void cleanup(Context context) throws IOException,
                 InterruptedException {
             // TODO Auto-generated method stub
             super.cleanup(context);
         }

     }

     /**
      * Driver Class
      */

     // 专门抽取一个方法出来用于设置
     public Job parseInputAndOutput(Tool tool, Configuration conf, String[] args)
             throws IOException {
         if (args.length != 2) {
             System.err.printf("Usage:%s [generic options] <input> <output>\n",
                     tool.getClass().getSimpleName());
             ToolRunner.printGenericCommandUsage(System.err);
             return null;
         }

         // 创建job，并设置配置信息和job名称
         Job job = new Job(conf,
                 WordcountByModuleMapReduce.class.getSimpleName());

         // 设置job的运行类
         // step 3：set job
         // 1) set run jar class
         job.setJarByClass(tool.getClass());

         // 14) job output path
         FileOutputFormat.setOutputPath(job, new Path(args[1]));

         return job;
     }

     @Override
     public int run(String[] args) throws Exception {

         // step 1：get conf
         Configuration conf = new Configuration();

         // step 2：create job
         Job job = parseInputAndOutput(this, conf, args);

         // 2) set input format
         job.setInputFormatClass(TextInputFormat.class); // 可省

         // 3) set input path
         FileInputFormat.addInputPath(job, new Path(args[0]));

         // 4) set mapper class
         job.setMapperClass(WordcountMapper.class); // 可省

         // 5)set map input key/value class
         job.setMapOutputKeyClass(Text.class); // 可省
         job.setMapOutputValueClass(LongWritable.class); // 可省

         // 6) set partitioner class
         job.setPartitionerClass(HashPartitioner.class); // 可省

         // 7) set reducer number
         job.setNumReduceTasks(1);// default 1 //可省

         // 8)set sort comparator class
         // job.setSortComparatorClass(LongWritable.Comparator.class); // 可省

         // 9) set group comparator class
         // job.setGroupingComparatorClass(LongWritable.Comparator.class); // 可省

         // 10) set combiner class
         // job.setCombinerClass(null);默认是null，但是此处不能写 //可省

         // 11) set reducer class
         job.setReducerClass(WordcountReducer.class); // 可省

         // 12) set output format
         job.setOutputFormatClass(TextOutputFormat.class); // 可省

         // 13) job output key/value class
         job.setOutputKeyClass(Text.class); // 可省
         job.setOutputValueClass(LongWritable.class); // 可省

         // step 4: submit job
         boolean isSuccess = job.waitForCompletion(true);

         // step 5: return status
         return isSuccess ? 0 : 1;
     }

     public static void main(String[] args) throws Exception {

         args = new String[] {
                 "hdfs://hadoop-master.dragon.org:9000/wc/mininput/",
                 "hdfs://hadoop-master.dragon.org:9000/wc/minoutput" };

         // run mapreduce
         int status = ToolRunner.run(new WordcountByModuleMapReduce(), args);

         // exit
         System.exit(status);
     }
 }

View WordcountByModuleMapReduce Code