MapReduce自定义InputFormat,RecordReader

MapReduce默认的InputFormat是TextInputFormat,且key是偏移量,value是文本,自定义InputFormat需要实现FileInputFormat,并重写createRecorder方法,如果需要还可以重写isSplitable()来设置是否切片,重写了createRecordReader还需要自定义RecordReader,InputFormat规定了key,value是什么,而RecordReader则是具体的读取逻辑,下面的例子是合并小文件,最终输出的k是文件路径,v是文件二进制字节

1.InputFormat

 /**
  * 自定义InputFormat规定读取文件的k,v
  * @author tele
  *
  */
 public class MyInputFormat extends FileInputFormat<NullWritable,BytesWritable>{
     /**
      * 设置不切片,把小文件作为一个整体
      */
     @Override
     protected boolean isSplitable(JobContext context, Path filename) {
         return false;
     }

     @Override
     public RecordReader<NullWritable,BytesWritable> createRecordReader(InputSplit split, TaskAttemptContext context)
             throws IOException, InterruptedException {
         MyRecordReader recordReader = new MyRecordReader();
         recordReader.initialize(split, context);
         return recordReader;
     }
 }

2.RecordReader

 /**
  * recordreader用于读取文件内容,输出文件内容即可,文件路径信息保存在split中
  * @author tele
  *
  */
 public class MyRecordReader extends RecordReader<NullWritable,BytesWritable> {
     FileSplit split;
     BytesWritable value = new BytesWritable();
     boolean flag = false;
     Configuration conf;
     int count = ;

     /**
      * 初始化
      */
     @Override
     public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {
         this.split = (FileSplit) split;
         conf = context.getConfiguration();    conf = context.getConfiguration();
     }

     /**
      * 业务逻辑处理,这个方法用来判断是否还有文件内容需要读取,会进入两次,第一次读取内容存入value中,返回true,第二次调用返回false
      * 只要返回true,就会调用getCurrentKey().getCurrentValue()把内容返回给map
      *
      */
     @Override
     public boolean nextKeyValue() throws IOException, InterruptedException {
         count++;
         if(!flag) {
             //获取fs
             FileSystem fs = FileSystem.get(conf);
             //开启流
             Path path = this.split.getPath();
             FSDataInputStream fsDataInputStream = fs.open(path);
             long length = this.split.getLength();
             byte[] buf = new byte[(int) length];

             //读取
             IOUtils.readFully(fsDataInputStream, buf, ,buf.length);
             value.set(buf, , buf.length);

             //关闭流
             IOUtils.closeStream(fsDataInputStream);
             flag = true;
         }else {
             flag = false;
         }
         return flag;
     }

     @Override
     public NullWritable getCurrentKey() throws IOException, InterruptedException {
         return NullWritable.get();
     }

     @Override
     public BytesWritable getCurrentValue() throws IOException, InterruptedException {
         return value;
     }

     @Override
     public float getProgress() throws IOException, InterruptedException {
         return flag?:;
     }

     @Override
     public void close() throws IOException {

     }
 }

3.Mapper

 /**
  * 把结果输出到SequenceFileOutPutFormat中,输出的key是文件路径,value为文件内容
  * @author tele
  *
  */
 public class InputformatMapper extends Mapper<NullWritable, BytesWritable, Text,BytesWritable/*Text*/> {
     Text k = new Text();      

     @Override
     protected void map(NullWritable key, BytesWritable value,
             Mapper<NullWritable, BytesWritable, Text, BytesWritable/*Text*/>.Context context)
             throws IOException, InterruptedException {
         FileSplit split = (FileSplit) context.getInputSplit();
         Path path = split.getPath();

         k.set(path.toString());

     /*    String result = new String(value.getBytes(),0,value.getLength());
         context.write(k,new Text(result));*/

         context.write(k, value);
     }
 }

4.Driver(由于输出的是字节,需要指定OutputFormat为SequenceFileOutputFormat)

 /**
  * 驱动
  * @author tele
  *
  */
 public class InputformatDriver {
     public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
         //1.获得job实例
         Configuration conf = new Configuration();
         Job job = Job.getInstance(conf);

         //2.关联class
         job.setJarByClass(InputformatDriver.class);
         job.setMapperClass(InputformatMapper.class);

         //4.设置format
         job.setInputFormatClass(MyInputFormat.class);
         //使用SequenceFileOutputFormat作为输出格式
         job.setOutputFormatClass(SequenceFileOutputFormat.class);

         //5.数据类型
         job.setOutputKeyClass(Text.class);
         job.setOutputValueClass(BytesWritable.class);

     //    job.setOutputValueClass(Text.class);

         //6.设置输入与输出路径
         FileInputFormat.setInputPaths(job,new Path(args[]));
         FileOutputFormat.setOutputPath(job,new Path(args[]));

         //7.提交
         boolean result = job.waitForCompletion(true);
         System.exit(result?:);
     }
 }