MapReduce编程小结

　　（1）key-value到map端比较容易，每个分片都会交由一个MapTask，而每个分片由InputFormat（一般是FileInputFormat）决定（一般是64M），

　　　　  每个MapTask会调用N次map函数，具体是多少次map函数呢？

　　　　  由job.setInputFormatClass(?)中？决定，默认是TextInputFormat.class，TextInputFormat是以一行为解析对象，一行对应一个map函数的调用。

　　（2）key-value在reduce端比较复杂，第二参数是Iterable<?>对象，涉及<key,list{value1,value2...}>,它对应一次reduce函数的调用，

　　　　  也就是说，一次reduce函数调用将会处理一个key，多个value，

　　（3）而这个<key,list{value1,value2...}>输入是如何来的呢？

　　　　mapreduce框架自带了预定义key（Text、LongWritable等）的排序，

　　　　将来自不同MapTask的相同的key加以聚合，变为<key,list{value1,value2...}>作为reduce函数的输入。

　　（4）说了MapTask个数有分片决定，那ReduceTask将由什么决定呢？

　　　　 每个map函数执行后都会调用一次getPartition函数(默认是HashPartitioner类的)来获取分区号，最终写入磁盘文件带有分区号这条尾巴，以便reduce端的拉取，

　　　　 而getPartition函数中最重要的参数numReduceTasks将由job.setNumReduceTasks决定，默认值为1，

　　　　 故若不设置此参数很多情况下getPartition函数会返回0，也就对应一个ReduceTask。

　　（5）说完了分区，再来说分组。分区是在map端确定，相对于每个map函数，而分组却放到了reduce端，相对于多个MapTask，组属于区。

　　　　分组会影响什么呢？

　　（6）当map端的输出key是自定义NewK2时，且自定义了compareTo，使用分组后，

　　　    将使用分组类MyGroupingComparator的compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2)进行sort，

　　　　 得到<key,list{value1,value2...}>。

　　附上一个例子：

　　

package examples; 

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;

public class GroupApp {
	static final String INPUT_PATH = "hdfs://192.168.2.100:9000/hello";
	static final String OUTPUT_PATH = "hdfs://192.168.2.100:9000/out";

	public static void main(String[] args) throws Exception {
		Configuration conf = new Configuration();
		final FileSystem fileSystem = FileSystem.get(new URI(INPUT_PATH), conf);
		final Path outPath = new Path(OUTPUT_PATH);
		if(fileSystem.exists(outPath)) {
			fileSystem.delete(outPath, true);
		}

		final Job job = new Job(conf, GroupApp.class.getSimpleName());
		job.setJarByClass(GroupApp.class);
		FileInputFormat.setInputPaths(job, INPUT_PATH);

		job.setInputFormatClass(TextInputFormat.class);

		job.setMapperClass(MyMapper.class);
		job.setMapOutputKeyClass(NewK2.class);
		job.setMapOutputValueClass(LongWritable.class);

		job.setPartitionerClass(MyPartitoner.class);
		job.setNumReduceTasks(3);

		job.setGroupingComparatorClass(MyGroupingComparator.class);

		job.setReducerClass(MyReducer.class);

		job.setOutputKeyClass(LongWritable.class);
		job.setOutputValueClass(LongWritable.class);

		FileOutputFormat.setOutputPath(job, outPath);

		job.waitForCompletion(true);
	}

	static class MyPartitoner extends HashPartitioner<NewK2, LongWritable> {
		  public int getPartition(NewK2 key, LongWritable value, int numReduceTasks) {
			  System.out.println("the getPartition() is called...");
			  if(key.first == 1) {
				  return 0 % numReduceTasks;
			  }
			  else if(key.first == 2) {
				  return 1 % numReduceTasks;
			  }
			  else {
				  return 2 % numReduceTasks;
			  }
		  }
	}

	static class NewK2 implements WritableComparable<NewK2> {
		Long first = 0L;
		Long second = 0L;

		public NewK2(){}

		public NewK2(long first, long second) {
			this.first = first;
			this.second = second;
		}

		public void write(DataOutput out) throws IOException {
			out.writeLong(first);
			out.writeLong(second);
		}

		public void readFields(DataInput in) throws IOException {
			first = in.readLong();
			second = in.readLong();
		}

		public int compareTo(NewK2 o) {
			System.out.println("the compareTo() is called...");

			final long minus = this.first - o.first;
			if(minus != 0) {
				return (int)minus;
			}
			return (int) (this.second - o.second);
		}
	}

	static class MyGroupingComparator implements RawComparator<NewK2> {
		public int compare(NewK2 o1, NewK2 o2) {
	//		System.out.println("the compare() is called...");
			return (int) (o1.first - o2.first);
		}

		public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
			System.out.println("the compare() is called...");
			return WritableComparator.compareBytes(b1, s1, 8, b2, s2, 8);
		}
	}

	static class MyMapper extends Mapper<LongWritable, Text, NewK2, LongWritable> {
		protected void map(LongWritable k1, Text v1, Context ctx) throws IOException, InterruptedException {
			final String[] splited = v1.toString().split("\t");

			System.out.println("the map() is called...");

			NewK2 k2 = new NewK2(Integer.parseInt(splited[0]), Integer.parseInt(splited[1]));
			LongWritable v2 = new LongWritable(Long.parseLong((splited[1])));
			ctx.write(k2, v2);
//			System.out.println("the real map output...");
//			System.out.println("<"+k2.first+","+v2+">");
		}
	}
	static class MyReducer extends Reducer<NewK2, LongWritable, LongWritable, LongWritable> {
		long v3 = 0;
		protected void reduce(NewK2 k2, Iterable<LongWritable> v2s, Context ctx) throws IOException, InterruptedException {
			System.out.println("the reduce() is called...");
			for(LongWritable secend : v2s) {
				v3 = secend.get();
				System.out.println("<"+k2.first+","+k2.second+">, "+v3+"");
			}
			System.out.println("--------------------------------------------");
			System.out.println("the real reduce output...");
			System.out.println("<"+k2.first+","+v3+">");
			ctx.write(new LongWritable(k2.first), new LongWritable(v3));
			System.out.println("--------------------------------------------");
		}
	}
}