Hadoop学习笔记： MapReduce Java编程简介

概述

本文主要基于Hadoop 1.0.0后推出的新Java API为例介绍MapReduce的Java编程模型。新旧API主要区别在于新API(org.apache.hadoop.mapreduce)将原来的旧API(org.apache.hadoop.mapred)中的接口转换为了抽象类。

MapReduce编程主要将程序运行过程分为两个阶段：Map阶段和Reduce阶段。其中Map阶段由若干Map task组成，主要由InputFormat, Mapper, Partitioner等类完成工作。Reduce阶段由若干Reduce task组成，主要由Reducer, OutputFormat等类完成工作。

Java API

InputFormat:

InputFormat是一个抽象类，所有输入格式都继承于这个类，例如读取数据库的DBInputFormat，读取普通文件的FileInputFormat等。MapReduce程序依靠InputFormat类/子类完成如下工作：

1. 确定所有输入文件，将其划分为逻辑上的InputSplits分片。每个分片分别交给一个Mapper处理。

2. 提供对象RecordReader，提取分片中的内容。

public abstract class InputFormat<K, V> {
  public abstract List<InputSplit> getSplits(JobContext context
                               ) throws IOException, InterruptedException;
  public abstract
    RecordReader<K,V> createRecordReader(InputSplit split,
                                         TaskAttemptContext context
                                        ) throws IOException,
                                                 InterruptedException;
}

除非有特殊指明，TextInputFormat为默认类，该类是FileInputFormat的一个子类。使用TextInputFormat为输入格式时，Key为每一行内容在文件中的偏移量，Value为每一行的实际内容。InputFormat类的层次如下：

InputSplit:

InputSplit是一个抽象类，定义了如下三个方法。getLength()用来获取分片大小，以支持对分片进行排序。getLocation()用来获取分片的位置列表。从代码也可以看出，实际上Input split并不存放数据，只是存放了实际文件的位置信息。

public abstract class InputSplit {
  public abstract long getLength() throws IOException, InterruptedException;
  public abstract String[] getLocations() throws IOException, InterruptedException;
  public SplitLocationInfo[] getLocationInfo() throws IOException {
    return null;
  }
}

RecordReader

RecordReader将InputSplit解析成<K,V>对，作为Mapper的输入。

public abstract class RecordReader<KEYIN, VALUEIN> implements Closeable {
  public abstract void initialize(InputSplit split,
                                  TaskAttemptContext context
                                  ) throws IOException, InterruptedException;
  public abstract
  boolean nextKeyValue() throws IOException, InterruptedException;
  public abstract
  KEYIN getCurrentKey() throws IOException, InterruptedException;
  public abstract
  VALUEIN getCurrentValue() throws IOException, InterruptedException;
  public abstract float getProgress() throws IOException, InterruptedException;
  public abstract void close() throws IOException;
}

自定义InputFormat代码实例：

package InputFormat;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;

public class URLWritable implements Writable{

    private Text url;
    public URLWritable(){
        url = new Text();
    }
    public URLWritable(Text url){
        this.url = url;
    }

    public Text getUrl() {
        return url;
    }
    public void setUrl(Text url) {
        this.url = url;
    }

    @Override
    public void readFields(DataInput in) throws IOException {
        url.set(in.readUTF());
    }

    @Override
    public void write(DataOutput out) throws IOException {
        out.writeUTF(url.toString());
    }
}

package InputFormat;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.*;

public class URLRecordReader extends RecordReader<Text, URLWritable>{

    private KeyValueLineRecordReader lineReader;
    private Text key;
    private URLWritable value;

    public URLRecordReader(Configuration conf) throws IOException{
        super();
        this.lineReader = new KeyValueLineRecordReader(conf);
        this.key = new Text();
        this.value = new URLWritable();
    }

    @Override
    public void close() throws IOException {
        lineReader.close();
    }

    @Override
    public Text getCurrentKey() throws IOException, InterruptedException {
        key = lineReader.getCurrentKey();
        return key;
    }

    @Override
    public URLWritable getCurrentValue() throws IOException, InterruptedException {
        value.setUrl(lineReader.getCurrentValue());
        return value;
    }

    @Override
    public float getProgress() throws IOException, InterruptedException {
        return lineReader.getProgress();
    }

    @Override
    public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException, InterruptedException {
        lineReader.initialize(genericSplit, context);
    }

    @Override
    public boolean nextKeyValue() throws IOException, InterruptedException {
        return lineReader.nextKeyValue();
    }
}

package InputFormat;

import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

public class URLInputFormat extends FileInputFormat<Text, URLWritable>{

    @Override
    public RecordReader<Text, URLWritable> createRecordReader(InputSplit genericSplit, TaskAttemptContext context)
            throws IOException, InterruptedException {
        context.setStatus(genericSplit.toString());
        return new URLRecordReader(context.getConfiguration());
    }
}

package InputFormat;

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

public class InputFormatTest extends Configured implements Tool{

    public static class MapperTest extends Mapper<Text, URLWritable, Text, Text>{
        public void map(Text key, URLWritable value, Context context)
                throws IOException, InterruptedException {
            context.write(key, value.getUrl());
        }
    }

    public static class ReducerTest extends Reducer<Text, Text, Text, Text>{
        public void reduce(Text key, Iterable<Text> values, Context context)
                throws IOException, InterruptedException {
            String out="";
            for(Text val : values) {
                out+=val.toString()+"|";
            }
            context.write(key, new Text(out));
        }
    }

    public static void main(String[] args) {
        try {
            int returnCode = ToolRunner.run(new InputFormatTest(), args);
            System.exit(returnCode);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    @Override
    public int run(String[] arg0) throws Exception {
        Configuration conf = new Configuration();
        conf.set("fs.default.name","hdfs://localhost:9001");
//        String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
//        if (otherArgs.length < 2) {
//            System.err.println("Usage: Test_1 <in> [<in>...] <out>");
//            System.exit(2);
//        }
//
        Job job = Job.getInstance(conf, "InputFormatTest");

        job.setJarByClass(InputFormatTest.class);
        job.setMapperClass(MapperTest.class);

        job.setCombinerClass(ReducerTest.class);
        job.setReducerClass(ReducerTest.class);

        job.setInputFormatClass(URLInputFormat.class); // 设置文件输入格式
        job.setOutputFormatClass(TextOutputFormat.class);// 使用默认的output格格式
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);
        FileInputFormat.addInputPath(job, new Path("/home/test_input/"));
        FileOutputFormat.setOutputPath(job, new Path("/home/test_output"));

//        for (int i = 0; i < otherArgs.length - 1; ++i) {
//            FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
//        }
        //FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));
        job.waitForCompletion(true);
        return job.isSuccessful() ? 0 : 1;
    }

}

Mapper

MapReduce框架对于每一个从InputFormat产生的InputSplit，都生成一个map task来进行处理，因此分片数也就等于map task的数量。Mapper类包含如下四个方法，setup方法会在map方法执行前被调用一次，而cleanup在map结束时执行一次，默认不做任何事。run方法也就是每个map task调用的方法。map task具体的业务逻辑我们一般通过重写Mapper子类的map方法来实现。

public class Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> {
  public abstract class Context
    implements MapContext<KEYIN,VALUEIN,KEYOUT,VALUEOUT> {
  }
  protected void setup(Context context
                       ) throws IOException, InterruptedException {
    // NOTHING
  }
  protected void map(KEYIN key, VALUEIN value,
                     Context context) throws IOException, InterruptedException {
    context.write((KEYOUT) key, (VALUEOUT) value);
  }
  protected void cleanup(Context context
                         ) throws IOException, InterruptedException {
    // NOTHING
  }
  public void run(Context context) throws IOException, InterruptedException {
    setup(context);
    try {
      while (context.nextKeyValue()) {
        map(context.getCurrentKey(), context.getCurrentValue(), context);
      }
    } finally {
      cleanup(context);
    }
  }
}

Partitioner

由Mapper生成的中间键值对将由Partitioner来决定交由哪一个Reducer进行处理，partition的数量也就等于reducer的数量。Hadoop目前提供了四种不同的Partitioner: BinaryPartitioner, HashPartitioner, KeyFieldBasedPartitioner, TotalOrderPartitioner，默认为HashPartitioner。我们也可以通过继承Partitioner抽象类来实现自己特殊的Partitoner逻辑。

public abstract class Partitioner<KEY, VALUE> {
  public abstract int getPartition(KEY key, VALUE value, int numPartitions);
}

HashPartitioner代码:

public class HashPartitioner<K, V> extends Partitioner<K, V> {
  public int getPartition(K key, V value,
                          int numReduceTasks) {
    return (key.hashCode() & Integer.MAX_VALUE) % numReduceTasks;
  }
}

Reducer

Reducer中的方法类似于Mapper，一般我们通过重写Reducer子类的reduce方法来实现具体业务逻辑。

public class Reducer<KEYIN,VALUEIN,KEYOUT,VALUEOUT> {
  public abstract class Context
    implements ReduceContext<KEYIN,VALUEIN,KEYOUT,VALUEOUT> {
  }
  protected void setup(Context context
                       ) throws IOException, InterruptedException {
    // NOTHING
  }
  protected void reduce(KEYIN key, Iterable<VALUEIN> values, Context context
                        ) throws IOException, InterruptedException {
    for(VALUEIN value: values) {
      context.write((KEYOUT) key, (VALUEOUT) value);
    }
  }
  protected void cleanup(Context context
                         ) throws IOException, InterruptedException {
    // NOTHING
  }
  public void run(Context context) throws IOException, InterruptedException {
    setup(context);
    try {
      while (context.nextKey()) {
        reduce(context.getCurrentKey(), context.getValues(), context);
        // If a back up store is used, reset it
        Iterator<VALUEIN> iter = context.getValues().iterator();
        if(iter instanceof ReduceContext.ValueIterator) {
          ((ReduceContext.ValueIterator<VALUEIN>)iter).resetBackupStore();
        }
      }
    } finally {
      cleanup(context);
    }
  }
}

OutputFormat

MapReduce程序依靠OutputFormat类/子类完成如下工作：

1. 验证任务的输出，比如输出目录是否存在等。

2. 提供对象RecordWriter，将内容写到文件系统上。

Hadoop默认的是TextOutputFormat，我们也可以自定义自己特殊的outputFormat，代码可以参考之前给出的自定义InputFormat。

public abstract class OutputFormat<K, V> {
  public abstract RecordWriter<K, V>
    getRecordWriter(TaskAttemptContext context
                    ) throws IOException, InterruptedException;
  public abstract void checkOutputSpecs(JobContext context
                                        ) throws IOException,
                                                 InterruptedException;
  public abstract
  OutputCommitter getOutputCommitter(TaskAttemptContext context
                                     ) throws IOException, InterruptedException;
}

MapReduce Java程序框架

public class Test extends Configured implements Tool{

    public static class MapperTest extends Mapper<Text, Text, Text, Text>{
        public void map(Text key, Text value, Context context)
                throws IOException, InterruptedException {
                        //map logic
            context.write(key, value);
        }
    }

    public static class ReducerTest extends Reducer<Text, Text, Text, Text>{
        public void reduce(Text key, Iterable<Text> values, Context context)
                throws IOException, InterruptedException {
            String out="";
            //reduce logic
            context.write(key, value);
        }
    }

    public static void main(String[] args) {
        try {
            int returnCode = ToolRunner.run(new InputFormatTest(), args);
            System.exit(returnCode);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    @Override
    public int run(String[] arg0) throws Exception {
        Configuration conf = new Configuration();
        String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
        if (otherArgs.length < 2) {
            System.err.println("Usage: Test_1 <in> [<in>...] <out>");
            System.exit(2);
        }

        Job job = Job.getInstance(conf, "Test");

        job.setJarByClass(InputFormatTest.class);
        job.setMapperClass(MapperTest.class);

        job.setCombinerClass(ReducerTest.class);
        job.setReducerClass(ReducerTest.class);

        job.setInputFormatClass(TextInputFormat.class); // 设置文件输入格式
        job.setOutputFormatClass(TextOutputFormat.class);// 使用默认的output格格式
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Text.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        for (int i = 0; i < otherArgs.length - 1; ++i) {
            FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
        }
        FileOutputFormat.setOutputPath(job, new Path(otherArgs[otherArgs.length - 1]));
        job.waitForCompletion(true);
        return job.isSuccessful() ? 0 : 1;
    }
}       

MapReduce Java doc：http://hadoop.apache.org/docs/current/api/

MapReduce Shuffle详解：http://langyu.iteye.com/blog/992916