琐碎-hadoop1.X和2.X的区别

---

1.  jobtracker做了分离，分成了resourceManager和nodemanager；

2.  MR变成了和HBase和Hive等一样的yarn上面的一个应用；

3.  1.x的默认块大小为64M，2.x的默认块大小为128M；

4.  在2.x中除了datanode要向namenode报告status，nodemanager也要向ResourceManager报告status

5. MR API差别

旧的WordCount

 package org.apache.hadoop.mapred;

 ... ...

 public class WordCount extends Configured implements Tool {

   public static class MapClass extends MapReduceBase
     implements Mapper<LongWritable, Text, Text, IntWritable> {

     ... ...

     public void map(LongWritable key, Text value,
                     OutputCollector<Text, IntWritable> output,
                     Reporter reporter) throws IOException {
       ... ...
     }
   }

   public static class Reduce extends MapReduceBase
     implements Reducer<Text, IntWritable, Text, IntWritable> {

     public void reduce(Text key, Iterator<IntWritable> values,
                        OutputCollector<Text, IntWritable> output,
                        Reporter reporter) throws IOException {
      ... ...
     }
   }

   static int printUsage() {
     System.out.println("wordcount [-m <maps>] [-r <reduces>] <input> <output>");
     ToolRunner.printGenericCommandUsage(System.out);
     return -1;
   }

   public int run(String[] args) throws Exception {
     ... ...
     return 0;
   }
   public static void main(String[] args) throws Exception {
     int res = ToolRunner.run(new Configuration(), new WordCount(), args);
     System.exit(res);
   }

 }

新的WordCount

 package org.apache.hadoop.examples;

 ... ...

 public class WordCount {

   public static class TokenizerMapper
        extends Mapper<Object, Text, Text, IntWritable>{

     ... ... 

     public void map(Object key, Text value, Context context
                     ) throws IOException, InterruptedException {
       ... ...
     }
   }

   public static class IntSumReducer
        extends Reducer<Text,IntWritable,Text,IntWritable> {
     private IntWritable result = new IntWritable();

     public void reduce(Text key, Iterable<IntWritable> values,
                        Context context
                        ) throws IOException, InterruptedException {
       ... ...
     }
   }

   public static void main(String[] args) throws Exception {
     ... ...
     System.exit(job.waitForCompletion(true) ? 0 : 1);
   }
 }

6.

---