MapReduce自定义InputFormat和OutputFormat

一、自定义InputFormat

需求：将多个小文件合并为SequenceFile（存储了多个小文件）

存储格式：文件路径+文件的内容

        c:/a.txt I love Beijing

        c:/b.txt I love China

        inputFormat(自定义加上路径)

1.Mapper类

package com.css.inputformat;

import java.io.IOException;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.BytesWritable;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.lib.input.FileSplit;

public class SequenceFileMapper extends Mapper<NullWritable, BytesWritable, Text, BytesWritable>{

    Text k = new Text();

    @Override

    protected void setup(Context context)

            throws IOException, InterruptedException {

        // 1.拿到切片信息

        FileSplit split = (FileSplit) context.getInputSplit();

        // 2.路径

        Path path = split.getPath();

        // 3.即带路径又带名称

        k.set(path.toString());

    }

    @Override

    protected void map(NullWritable key, BytesWritable value,Context context)

                    throws IOException, InterruptedException {

        // 输出

        context.write(k, value);

    }

}

2.Reducer类

package com.css.inputformat;

import java.io.IOException;

import org.apache.hadoop.io.BytesWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Reducer;

public class SequenceFileReducer extends Reducer<Text, BytesWritable, Text, BytesWritable>{

    @Override

    protected void reduce(Text key, Iterable<BytesWritable> values,

            Context context) throws IOException, InterruptedException {

        for (BytesWritable v : values) {

            context.write(key, v);

        }

    }

}

3.自定义InputFormat类

package com.css.inputformat;

import java.io.IOException;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.BytesWritable;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.mapreduce.InputSplit;

import org.apache.hadoop.mapreduce.JobContext;

import org.apache.hadoop.mapreduce.RecordReader;

import org.apache.hadoop.mapreduce.TaskAttemptContext;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

/**

 * 1.创建自定义Inputformat

 */

public class FuncFileInputFormat extends FileInputFormat<NullWritable, BytesWritable>{

    @Override

    protected boolean isSplitable(JobContext context, Path filename) {

        // 不切原来的文件

        return false;

    }

    @Override

    public RecordReader<NullWritable, BytesWritable> createRecordReader(InputSplit split, TaskAttemptContext context)

            throws IOException, InterruptedException {

        FuncRecordReader RecordReader = new FuncRecordReader();

        return RecordReader;

    }

}

4.自定义RecordReader类

package com.css.inputformat;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FSDataInputStream;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.BytesWritable;

import org.apache.hadoop.io.IOUtils;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.mapreduce.InputSplit;

import org.apache.hadoop.mapreduce.RecordReader;

import org.apache.hadoop.mapreduce.TaskAttemptContext;

import org.apache.hadoop.mapreduce.lib.input.FileSplit;

/**

 * 2.编写RecordReader

 */

public class FuncRecordReader extends RecordReader<NullWritable, BytesWritable>{

    boolean isProcess = false;

    FileSplit split;

    Configuration conf;

    BytesWritable value = new BytesWritable();

    // 初始化

    @Override

    public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException {

        // 初始化文件切片

        this.split = (FileSplit) split;

        // 初始化配置信息

        conf = context.getConfiguration();

    }

    @Override

    public boolean nextKeyValue() {

        if (!isProcess) {

            // 1.根据切片的长度来创建缓冲区

            byte[] buf = new byte[(int)split.getLength()];

            FSDataInputStream fis = null;

            FileSystem fs = null;

            try {

                // 2.获取路径

                Path path = split.getPath();

                // 3.根据路径获取文件系统

                fs = path.getFileSystem(conf);

                // 4.拿到输出流

                fis = fs.open(path);

                // 5.数据拷贝

                IOUtils.readFully(fis, buf, 0, buf.length);

                // 6.拷贝缓存到最终的输出

                value.set(buf, 0, buf.length);;

            } catch (Exception e) {

                e.printStackTrace();

            } finally {

                IOUtils.closeStream(fis);

                IOUtils.closeStream(fs);

            }

            isProcess = true;

            return true;

        }

        return false;

    }

    @Override

    public NullWritable getCurrentKey() throws IOException, InterruptedException {

        return NullWritable.get();

    }

    @Override

    public BytesWritable getCurrentValue() throws IOException, InterruptedException {

        return value;

    }

    @Override

    public float getProgress() throws IOException, InterruptedException {

        return 0;

    }

    @Override

    public void close() throws IOException {

    }

}

5.Driver类

package com.css.inputformat;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.BytesWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;

public class SequenceDriver {

    public static void main(String[] args) throws IllegalArgumentException, IOException, ClassNotFoundException, InterruptedException {

        // 1.获取job信息

        Configuration conf = new Configuration();

        Job job = Job.getInstance(conf);

        // 2.获取jar包

        job.setJarByClass(SequenceDriver.class);

        // 3.获取自定义的mapper与reducer类

        job.setMapperClass(SequenceFileMapper.class);

        job.setReducerClass(SequenceFileReducer.class);

        // 4.设置自定义读取方式

        job.setInputFormatClass(FuncFileInputFormat.class);

        // 5.设置默认的输出方式

        job.setOutputFormatClass(SequenceFileOutputFormat.class);

        // 6.设置map输出的数据类型

        job.setMapOutputKeyClass(Text.class);

        job.setMapOutputValueClass(BytesWritable.class);

        // 7.设置reduce输出的数据类型（最终的数据类型）

        job.setOutputKeyClass(Text.class);

        job.setOutputValueClass(BytesWritable.class);

        // 8.设置输入存在的路径与处理后的结果路径

        FileInputFormat.setInputPaths(job, new Path("c:/in1027/"));

        FileOutputFormat.setOutputPath(job, new Path("c:/out1027/"));

        // 9.提交任务

        boolean rs = job.waitForCompletion(true);

        System.out.println(rs ? 0 : 1);

    }

}

6.输入小文件

（1）a.txt

I love Beijing

（2）b.txt

I love China

（3）c.txt

Bejing is the capital of China

7.输出文件part-r-00000

SEQorg.apache.hadoop.io.Text"org.apache.hadoop.io.BytesWritable      嫜瑻z萶2

?擎?  (   file:/c:/in1027/a.txt   I love Beijing   &   file:/c:/in1027/b.txt   I love China   8   file:/c:/in1027/c.txt   Bejing is the capital of China

二、自定义OutputFormat

需求:过滤日志文件

    把包含main的放在一个文件中 d:/main.logs

    把不包含main的放在另外一个文件 d:/other.logs

1.Mapper类

package com.css.outputformat;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;

public class FileMapper extends Mapper<LongWritable, Text, Text, NullWritable>{

    @Override

    protected void map(LongWritable key, Text value, Context context)

            throws IOException, InterruptedException {

        // 输出

        context.write(value, NullWritable.get());

    }

}

2.Reducer类

package com.css.outputformat;

import java.io.IOException;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Reducer;

public class FileReducer extends Reducer<Text, NullWritable, Text, NullWritable>{

    @Override

    protected void reduce(Text key, Iterable<NullWritable> values,

            Context context) throws IOException, InterruptedException {

        // 数据换行

        String k = key.toString() + "\n";

        context.write(new Text(k), NullWritable.get());

    }

}

3.自定义OutputFormat类

package com.css.outputformat;

import java.io.IOException;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.RecordWriter;

import org.apache.hadoop.mapreduce.TaskAttemptContext;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class FuncFileOutputFormat extends FileOutputFormat<Text, NullWritable>{

    @Override

    public RecordWriter<Text, NullWritable> getRecordWriter(TaskAttemptContext job)

            throws IOException, InterruptedException {

        FileRecordWriter fileRecordWriter = new FileRecordWriter(job);

        return fileRecordWriter;

    }

}

4.自定义RecordWriter类

package com.css.outputformat;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FSDataOutputStream;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.RecordWriter;

import org.apache.hadoop.mapreduce.TaskAttemptContext;

public class FileRecordWriter extends RecordWriter<Text, NullWritable>{

    Configuration conf = null;

    FSDataOutputStream mainlog = null;

    FSDataOutputStream otherlog = null;

    // 1.定义数据输出路径

    public FileRecordWriter(TaskAttemptContext job) throws IOException{

        // 获取配置信息

        conf = job.getConfiguration();

        // 获取文件系统

        FileSystem fs = FileSystem.get(conf);

        // 定义输出路径

        mainlog = fs.create(new Path("c:/outputmain/main.logs")); // part-r-00000

        otherlog = fs.create(new Path("c:/outputother/other.logs"));

    }

    // 2.数据输出

    @Override

    public void write(Text key, NullWritable value) throws IOException, InterruptedException {

        // 判断的话根据key

        if (key.toString().contains("main")) {

            // 写出到文件

            mainlog.write(key.getBytes());

        }else {

            otherlog.write(key.getBytes());

        }

    }

    // 3.关闭资源

    @Override

    public void close(TaskAttemptContext context) throws IOException, InterruptedException {

        if (null != mainlog) {

            mainlog.close();

        }

        if (null != otherlog) {

            otherlog.close();

        }

    }

}

5.Driver类

package com.css.outputformat;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class FileDriver {

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {

        // 1.获取job信息

        Configuration conf = new Configuration();

        Job job = Job.getInstance(conf);

        // 2.获取jar包

        job.setJarByClass(FileDriver.class);

        // 3.获取自定义的mapper与reducer类

        job.setMapperClass(FileMapper.class);

        job.setReducerClass(FileReducer.class);

        // 4.设置map输出的数据类型

        job.setMapOutputKeyClass(Text.class);

        job.setMapOutputValueClass(NullWritable.class);

        // 5.设置reduce输出的数据类型（最终的数据类型）

        job.setOutputKeyClass(Text.class);

        job.setOutputValueClass(NullWritable.class);

        // 6.设置自定义outputFormat

        job.setOutputFormatClass(FuncFileOutputFormat.class);

        // 7.设置输入存在的路径与处理后的结果路径

        FileInputFormat.setInputPaths(job, new Path("c:/in1029/"));

        FileOutputFormat.setOutputPath(job, new Path("c:/out1029/"));

        // 8.提交任务

        boolean rs = job.waitForCompletion(true);

        System.out.println(rs ? 0 : 1);

    }

}

6.输入文件data.logs

http://www.baidu.com

http://taobao.com

http://jd.com

http://it.com

http://main.js

http://qq.com

http://main.com.cn

7.输出文件

（1）main.logs

http://main.com.cn

 http://main.js

（2）other.logs

http://it.com

 http://jd.com

 http://qq.com

 http://taobao.com

 http://www.baidu.com

MapReduce自定义InputFormat和OutputFormat的更多相关文章

【Hadoop离线基础总结】MapReduce自定义InputFormat和OutputFormat案例
MapReduce自定义InputFormat和OutputFormat案例自定义InputFormat 合并小文件需求无论hdfs还是mapreduce,存放小文件会占用元数据信息,白白浪费内 ...
自定义InputFormat和OutputFormat案例
一.自定义InputFormat InputFormat是输入流,在前面的例子中使用的是文件输入输出流FileInputFormat和FileOutputFormat,而FileInputFormat ...
自定义inputformat和outputformat
1. 自定义inputFormat 1.1 需求无论hdfs还是mapreduce,对于小文件都有损效率,实践中,又难免面临处理大量小文件的场景,此时,就需要有相应解决方案 1.2 分析小文件的优 ...
MapReduce自定义InputFormat,RecordReader
MapReduce默认的InputFormat是TextInputFormat,且key是偏移量,value是文本,自定义InputFormat需要实现FileInputFormat,并重写creat ...
【转自】自定义InputFormat、OutputFormat
转自:http://www.cnblogs.com/xiaolong1032/p/4529534.html 一:自定义实现InputFormat *数据源来自于内存*1.InputFormat是用于处 ...
MapReduce之自定义InputFormat
在企业开发中,Hadoop框架自带的InputFormat类型不能满足所有应用场景,需要自定义InputFormat来解决实际问题. 自定义InputFormat步骤如下: (1)自定义一个类继承Fi ...
Hadoop中常用的InputFormat、OutputFormat（转）
Hadoop中的Map Reduce框架依赖InputFormat提供数据,依赖OutputFormat输出数据,每一个Map Reduce程序都离不开它们.Hadoop提供了一系列InputForm ...
Hadoop权威指南: InputFormat,RecordReader,OutputFormat和RecordWriter
InputFormat和RecordReader Hadoop提出了InputFormat的概念 org.apache.hadoop.mapreduce包里的InputFormat抽象类提供了如下列代 ...
Hadoop案例（六）小文件处理（自定义InputFormat）
小文件处理(自定义InputFormat) 1.需求分析无论hdfs还是mapreduce,对于小文件都有损效率,实践中,又难免面临处理大量小文件的场景,此时,就需要有相应解决方案.将多个小文件合并 ...

随机推荐

Oracle字符串分割Split（超简单一条sql解决）
) FROM renyuan where name ='张三' 解决如下问题我现在有一个字段是存:,,3的,而它对应另一张值集表中.eg; 课程人员表 renyuan id name Course ...
noip2014滚粗记
滚粗了..伤心. day0:和baba一起去,但是整天都是下雨啊好不爽,鞋子都湿了啊好不爽,注定是要滚粗?在火车站等了1h后上动车走人...在此期间我还天真的认为火车站的wifi可以被我给破解然后上网 ...
maven公布jar、aar、war等到中央库（Central Repository）的步骤
步骤一:注冊账号.申请ticket. 注冊在这里:https://issues.sonatype.org 申请ticket:创建一个issue.注意这里要选OSSRH,且是PROJECT而不是TASK ...
1QT在线帮助文档
http://www.kuqin.com/qtdocument/classes.html
修改CFileDialog的标题
CFileDialog f(TRUE); f.m_ofn.lpstrTitle = "我的标题"; f.DoModal(); 设置标题! CFileDialog ...
权限模块_使用权限_实现主页面的效果_显示左侧菜单&只显示有权限的菜单项
权限模块__使用权限__实现主页面的效果 HomeAction.java public class HomeAction extends ActionSupport { public String i ...
GSAP 官方文档（结贴）
好久没写GSAP的教程的(其实我也不懂哈哈),国内也没什么人用,不对动画要求特别高的话,其实也没必要用GSAP,现在工作上没用到这个东西,也懒得写了,所以有问题的话去找一下greensock的官方文档 ...
Spring中bean的作用范围
singleton作用域: Spring的scope的默认值是singleton Spring 只会为每一个bean创建一个实例,并保持bean的引用. <bean id="bean的 ...
【BZOJ3442】学习小组费用流
[BZOJ3442]学习小组 Description [背景] 坑校准备鼓励学生参加学习小组. [描述] 共有n个学生,m个学习小组,每个学生有一定的喜好,只愿意参加其中的一些学习小组,但是校领导为学 ...
SSH电力项目九--运行监控首页显示
需求:在首页显示出设备运行情况,并去掉<br>换行符,每隔十分钟刷新一次页面. ElecMenuAction.java 首先注入运行监控service public class ElecM ...

MapReduce自定义InputFormat和OutputFormat

MapReduce自定义InputFormat和OutputFormat的更多相关文章

随机推荐

热门专题