MapReduce之Reduce Join

一 介绍

Reduce Join其主要思想如下：
 在map阶段，map函数同时读取两个文件File1和File2，为了区分两种来源的key/value数据对，对每条数据打一个标签（tag）， 比如：tag=0表示来自文件File1，tag=2表示来自文件File2。即：map阶段的主要任务是对不同文件中的数据打标签。在reduce阶段，reduce函数获取key相同的来自File1和File2文件的value list， 然后对于同一个key，对File1和File2中的数据进行join（笛卡尔乘积)，即：reduce阶段进行实际的连接操作。

在这个例子中我们假设有两个数据文件如下：
存储客户信息的文件：customers.csv

1,stephaie leung,555-555-5555
2,edward kim,123-456-7890
3,jose madriz,281-330-8004
4,david storkk,408-55-0000

存储订单信息的文件：orders.csv

3,A,12.95,02-Jun-2008
1,B,88.25,20-May-2008
2,C,32.00,30-Nov-2007
3,D,25.02,22-Jan-2009

要求最终的输出结果为：

1,Stephanie Leung,555-555-5555,B,88.25,20-May-2008
2,Edward Kim,123-456-7890,C,32.00,30-Nov-2007
3,Jose Madriz,281-330-8004,A,12.95,02-Jun-2008
3,Jose Madriz,281-330-8004,D,25.02,22-Jan-2009

二 代码部分

自定义数据类型：用于对不同文件数据打标签

 package mapreduce.reducejoin;

 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import org.apache.hadoop.io.Writable;

 public class DataJoinWritable implements Writable {

     // mark ,customer / order
     private String tag;

     // info
     private String data;

     public DataJoinWritable() {

     }

     public DataJoinWritable(String tag, String data) {
         this.set(tag, data);
     }

     public void set(String tag, String data) {
         this.setTag(tag);
         this.setData(data);
     }

     public String getTag() {
         return tag;
     }

     public void setTag(String tag) {
         this.tag = tag;
     }

     public String getData() {
         return data;
     }

     public void setData(String data) {
         this.data = data;
     }

     public void write(DataOutput out) throws IOException {
         out.writeUTF(this.getTag());
         out.writeUTF(this.getData());
     }

     public void readFields(DataInput in) throws IOException {
         this.setTag(in.readUTF());
         this.setData(in.readUTF());
     }

     @Override
     public int hashCode() {
         final int prime = 31;
         int result = 1;
         result = prime * result + ((data == null) ? 0 : data.hashCode());
         result = prime * result + ((tag == null) ? 0 : tag.hashCode());
         return result;
     }

     @Override
     public boolean equals(Object obj) {
         if (this == obj)
             return true;
         if (obj == null)
             return false;
         if (getClass() != obj.getClass())
             return false;
         DataJoinWritable other = (DataJoinWritable) obj;
         if (data == null) {
             if (other.data != null)
                 return false;
         } else if (!data.equals(other.data))
             return false;
         if (tag == null) {
             if (other.tag != null)
                 return false;
         } else if (!tag.equals(other.tag))
             return false;
         return true;
     }

     @Override
     public String toString() {
         return tag + "," + data;
     }
 }

MapReduce代码部分

 package mapreduce.reducejoin;

 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;

 public class DataJoinMapReduce extends Configured implements Tool {

     // step 1: Mapper
     public static class DataJoinMapper extends
             Mapper<LongWritable, Text, LongWritable, DataJoinWritable> {

         // map output key
         private LongWritable mapOutputKey = new LongWritable();

         // map output value
         private DataJoinWritable mapOutputValue = new DataJoinWritable();

         @Override
         public void setup(Context context) throws IOException,
                 InterruptedException {
         }

         @Override
         public void map(LongWritable key, Text value, Context context)
                 throws IOException, InterruptedException {

             // line value
             String lineValue = value.toString();

             // split
             String[] vals = lineValue.split(",");

             int length = vals.length;

             if ((3 != length) && (4 != length)) {
                 return;
             }

             // get cid
             Long cid = Long.valueOf(vals[0]);

             // get name
             String name = vals[1];

             // set customer
             if (3 == length) {
                 String phone = vals[2];

                 // set
                 mapOutputKey.set(cid);
                 mapOutputValue.set("customer", name + "," + phone);
             }

             // set order
             if (4 == length) {
                 String price = vals[2];
                 String date = vals[3];

                 // set
                 mapOutputKey.set(cid);
                 mapOutputValue.set("order", name + "," + price + "," + date);
             }

             // output
             context.write(mapOutputKey, mapOutputValue);

         }

         @Override
         public void cleanup(Context context) throws IOException,
                 InterruptedException {
         }
     }

     // step 2: Reducer
     public static class DataJoinReducer extends
             Reducer<LongWritable, DataJoinWritable, NullWritable, Text> {

         private Text outputValue = new Text();

         @Override
         protected void setup(Context context) throws IOException,
                 InterruptedException {
         }

         @Override
         protected void reduce(LongWritable key,
                 Iterable<DataJoinWritable> values, Context context)
                 throws IOException, InterruptedException {
             String customerInfo = null;
             List<String> orderList = new ArrayList<String>();

             for (DataJoinWritable value : values) {
                 if ("customer".equals(value.getTag())) {
                     customerInfo = value.getData();
                 } else if ("order".equals(value.getTag())) {
                     orderList.add(value.getData());
                 }
             }

             // output
             for (String order : orderList) {

                 // ser outout value
                 outputValue.set(key.get() + "," + customerInfo + "," + order);

                 // output
                 context.write(NullWritable.get(), outputValue);
             }
         }

         @Override
         protected void cleanup(Context context) throws IOException,
                 InterruptedException {
         }
     }

     /**
      * Execute the command with the given arguments.
      *
      * @param args
      *            command specific arguments.
      * @return exit code.
      * @throws Exception
      */

     // step 3: Driver
     public int run(String[] args) throws Exception {

         Configuration configuration = this.getConf();

         // set job
         Job job = Job.getInstance(configuration, this.getClass().getSimpleName());
         job.setJarByClass(DataJoinMapReduce.class);

         // input
         Path inpath = new Path(args[0]);
         FileInputFormat.addInputPath(job, inpath);

         // output
         Path outPath = new Path(args[1]);
         FileOutputFormat.setOutputPath(job, outPath);

         // Mapper
         job.setMapperClass(DataJoinMapper.class);
         job.setMapOutputKeyClass(LongWritable.class);
         job.setMapOutputValueClass(DataJoinWritable.class);

         // Reducer
         job.setReducerClass(DataJoinReducer.class);
         job.setOutputKeyClass(NullWritable.class);
         job.setOutputValueClass(Text.class);

         // submit job -> YARN
         boolean isSuccess = job.waitForCompletion(true);
         return isSuccess ? 0 : 1;

     }

     public static void main(String[] args) throws Exception {

         Configuration configuration = new Configuration();

         args = new String[] {
                 "hdfs://beifeng01:8020/user/beifeng01/mapreduce/input/reducejoin",
                 "hdfs://beifeng01:8020/user/beifeng01/mapreduce/output" };

         // run job
         int status = ToolRunner.run(configuration, new DataJoinMapReduce(),
                 args);

         // exit program
         System.exit(status);
     }
 }

执行代码后查询结果

[hadoop@beifeng01 hadoop-2.5.0-cdh5.3.6]$ bin/hdfs dfs -text /user/beifeng01/mapreduce/output/p*
1,stephaie leung,555-555-5555,B,88.25,20-May-2008
2,edward kim,123-456-7890,C,32.00,30-Nov-2007
3,jose madriz,281-330-8004,D,25.02,22-Jan-2009
3,jose madriz,281-330-8004,A,12.95,02-Jun-2008