MapReduce Unit Test

　　以前用java写MR程序总不习惯写单元测试，就是查错也只是在小规模数据上跑一下程序。昨天工作时，遇到一个bug，查了好久也查出来。估计是业务逻辑上的错误。后来没办法，只好写了个单元测试，一步步跟踪，瞬间找到问题所在。所以说，工作中还是要勤快些。

 import static org.junit.Assert.assertEquals;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mrunit.mapreduce.MapDriver;
 import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
 import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
 import org.apache.hadoop.mrunit.types.Pair;
 import org.junit.Before;
 import org.junit.Test;
 import com.wanda.predict.GenerateCustomerNatureFeature.NatureFeatureMappper;
 import com.wanda.predict.GenerateCustomerNatureFeature.NatureReducer;
 import com.wanda.predict.pojo.Settings;

 /**
  * MapReduce 单元测试的模板 , 依赖于junit环境(junit.jar)， mrunit.jar , mockito.jar
  *
  */
 public class MapperReducerUnitTest {
     // 一些设置，与正常的mr程序一样，不过这里主要是加载一些信息。性能优化之类的就不要在单元测试里设置了。
     Configuration conf = new Configuration();
     //Map.class 的测试驱动类
     MapDriver<LongWritable, Text, Text, Text> mapDriver;
     //Reduce.class 的测试驱动类
     ReduceDriver<Text, Text, Text, Text> reduceDriver;
     //Map.calss 、 Reduce.class转接到一起的流程测试驱动
     MapReduceDriver<LongWritable, Text, Text, Text, Text, Text> mapReduceDriver;

     @Before
     public void setUp() {

         //测试mapreduce
         NatureFeatureMappper mapper = new NatureFeatureMappper();
         NatureReducer reducer = new NatureReducer();
         //添加要测试的map类
         mapDriver = MapDriver.newMapDriver(mapper);
         //添加要测试的reduce类
         reduceDriver = ReduceDriver.newReduceDriver(reducer);
         //添加map类和reduce类
         mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);

         //测试配置参数
         conf.setInt(Settings.TestDataSize.getName(), 1);
         conf.setInt(Settings.TrainDataSize.getName(), 6);
         //driver之间是独立的，谁用到谁就设置conf
         reduceDriver.setConfiguration(conf);
         mapReduceDriver.setConfiguration(conf);
     }

     @Test
     public void testMapper() throws IOException {
         mapDriver.withInput(new LongWritable(), new Text("map的输入"));
         mapDriver.withOutput(new Text("期望的key"), new Text("期望的value"));

         //打印实际结果
         List<Pair<Text , Text>> result = mapDriver.run();
         for(Pair<Text , Text> kv : result){
             System.out.println("mapper : " + kv.getFirst());
             System.out.println("mapper : " + kv.getSecond());
         }
         //进行case测试，对比输入输出结果
         mapDriver.runTest();
     }

     @Test
     public void testReducer() throws IOException {
         List<Text> values = new ArrayList<Text>();
         values.add(new Text("输入"));
         reduceDriver.withInput(new Text("输入"), values);
         reduceDriver.withOutput(new Text("期望的输出"), new Text("期望的输出"));
         reduceDriver.runTest();
     }

     @Test
     public void testMapperReducer() throws IOException {
         mapReduceDriver.withInput(new LongWritable(), new Text("输入"));
         mapReduceDriver.withOutput(new Text("期望的输出"), new Text("期望的输出"));
         //打印实际结果
         List<Pair<Text, Text>> list =  mapReduceDriver.run();
         System.out.println("mapreducedriver size:" + list.size());
         for(Pair<Text , Text> lst : list){
             System.out.println(lst.getFirst());
             System.out.println(lst.getSecond());
         }
         //进行case测试，对比输入输出结果
         mapReduceDriver.runTest();
     }

     @Test
     public void testMapperCount() throws IOException {
         mapDriver.withInput(new LongWritable(), new Text("输入"));
         mapDriver.withOutput(new Text("期望的输出"), new Text("期望的输出"));
         mapDriver.runTest();
         //判断 map中的counter值是否与期望的相同
         assertEquals("Expected 1 counter increment", 1, mapDriver.getCounters().findCounter("data", "suc").getValue());
     }
 }