MapReduce输出文件名更改

1、默认情况下生成的文件名是part-r-00000格式，想要自定义生成输出文件名可以使用org.apache.hadoop.mapreduce.lib.output.MultipleOutputs类用来写出

2、MultipleOutputs类需要在Reduce的setup()方法初始化，最好在cleanup()中关闭

3、这个时候还会生产成part-r-000000这种文件，发现是里面是空的，需要 LazyOutputFormat.setOutputFormatClass(job, TextOutputFormat.class);

4、MultipleOutputs类的write()方法有几个重载的函数

write(KEYOUT key, VALUEOUT value, String baseOutputPath)
如果baseOutputPath带/，那么输出路径就是baseOutputPath + -r-00000，比如baseOutpuPath="/PUFA/" + key.toString()，输出文件路径就是/PUFA/0000000001-r-000000
不带/，输出路径就是FileOutputFormat.setOutputPath(job, outPutPath)下面，比如baseOutputPath=key.toString()，ouputPath=/trx，输出文件的路径就是/trx/0000000001-r-000000

write方法如果是带namedOutput参数的，需要在运行主类上面指定namedOutput，
MultipleOutputs.addNamedOutput(job, "PFBANK", TextOutputFormat.class, JournalTrxDataSet.class, NullWritable.class);
MultipleOutputs.addNamedOutput(job, "ZSBANK", TextOutputFormat.class, JournalTrxDataSet.class, NullWritable.class);

write(String namedOutput, K key, V value, String baseOutputPath) 

write(String namedOutput, K key, V value) 

这两种况和上面方法差不多，就是通过namedOutput对一组reduce处理的结果输出到不同的文件夹中，如果没有baseOutputPath,会输出到FileOutputFormat.setOutputPath()目录

  if (Integer.parseInt(key.toString()) >= 500000){
       mos.write("PFBANK", journalTrxDataSet, NullWritable.get(), "/PUFA/" + key.toString());
  }else if(Integer.parseInt(key.toString()) < 500000){
       mos.write("ZSBANK", journalTrxDataSet, NullWritable.get(), "/ZHAOSHANG/" + key.toString());
  }

注意：有的时候会出现_SUCCESS文件和reduce输出的文件不在同一个目录，这是因为FileOutputFormat.setOutputPaht()和MultipleOutputs类的write()方法设置的baseOutputPath不一样所致，_SUCCESS文件始终在FileOutputFormat.setOutputPaht()设定的路径上

有的时候会报一些莫名其妙的错的话，可能是LazyOutputFormat.setOutputFormatClass()和MultipleOutputs.addNamedOutput()的formatclass参数有关

最后附上完整代码

当时我们的需求是，需要分析统计银行各个终端的交易情况，当时我们数据量也不太多，领导说尽可能简单点做，当时统计纬度有两种，一种是按照机器，一种是按照分行，所以直接使用了同一个mapreduce来完成，当然，下面代码只是统计机器的，分行的维度还没写上。当时mr程序是独立的一个模块，在数据采集完成后，会直接使用sh或cmd命令调用这个jar包，文件名当成启动参数传递过来。文件需要自定义

package com.xhy.xgg.mapreduce;

import com.xhy.xgg.common.HDFSCommand;
import com.xhy.xgg.common.enums.StatisticDemensionEnums;
import com.xhy.xgg.entity.JournalTrxDataSet;
import lombok.extern.slf4j.Slf4j;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.SpringApplication;
import org.springframework.stereotype.Service;

import javax.annotation.PostConstruct;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;

@Slf4j
@Service
public class MapReduceTest {

    @Value("${data-set.output-path}")
    private String dataCollectionOutputPath;
    @Value("${data-collection.output-path}")
    private String dataCollectionIutputPath;

    @Value("${data-collection.hdfsURI}")
    private String hdfsURI;

    @Value("${mr-run-mode.value}")
    private String mrRunMode;

    @PostConstruct
    public void executeMRJob(StatisticDemensionEnums statisticDemensionEnums)
            throws IOException, ClassNotFoundException, InterruptedException {

        log.info(">> executeMRJob start execute");
        Configuration conf = new Configuration();
        conf.set("dfs.blocksize", "67108864");
        conf.set("dfs.replication", "1");
        conf.set("mapreduce.framework.name", mrRunMode);
        conf.set("fs.defaultFS", hdfsURI);
        Job job;
        job = Job.getInstance(conf, "JournalDataProcessService");
        job.setJarByClass(com.xhy.xgg.mapreduce.TerminalJournalDataService.class);
        job.getConfiguration().set("statisticDemensionEnums", String.valueOf(statisticDemensionEnums.getIndex()));
        job.setMapperClass(com.xhy.xgg.mapreduce.MapReduceTest.JournalDataMapper.class);
        job.setMapOutputKeyClass(Text.class);

        job.setMapOutputValueClass(JournalTrxDataSet.class);
        job.setReducerClass(com.xhy.xgg.mapreduce.MapReduceTest.JournalDataReducer.class);
        job.setOutputKeyClass(JournalTrxDataSet.class);
        job.setOutputValueClass(NullWritable.class);
        job.setPartitionerClass(com.xhy.xgg.mapreduce.MapReduceTest.JournalDataPartitioner.class);
        // job.setSortComparatorClass(JournalTrxDataComparator.class);
        job.setNumReduceTasks(3);
        Path inputPath = new Path(dataCollectionIutputPath + File.separator + "esbTrx" + File.separator
                + hdfsFileName);

        log.info("-- executeMRJob inputPath = {}", inputPath.toString());

        FileInputFormat.addInputPath(job, inputPath);

        String outPutPathData = "";

        outPutPathData = dataCollectionOutputPath + "_" + new SimpleDateFormat("yyyyMMdd").format(new Date());
        try {
            if (HDFSCommand.exists(conf, hdfsURI, outPutPathData)) {
                HDFSCommand.delete(conf, hdfsURI, outPutPathData);
            }
        } catch (Exception e) {
            // TODO Auto-generated catch block
            log.error("<< executeMRJob exception, message {}", e.getMessage());
        }
        Path outPutPath = new Path(outPutPathData);
        FileOutputFormat.setOutputPath(job, outPutPath);
        MultipleOutputs.addNamedOutput(job, "PFBANK", TextOutputFormat.class, JournalTrxDataSet.class, NullWritable.class);
        MultipleOutputs.addNamedOutput(job, "ZSBANK", TextOutputFormat.class, JournalTrxDataSet.class, NullWritable.class);

        LazyOutputFormat.setOutputFormatClass(job, TextOutputFormat.class);

        System.exit(job.waitForCompletion(true) ? 0 : 1);
    }

    public void start() throws Exception {
        executeMRJob(StatisticDemensionEnums.TERMINAL);
    }

    private String hdfsFileName = "journal_test.txt";

    private static class JournalDataReducer extends Reducer<Text, JournalTrxDataSet, JournalTrxDataSet, NullWritable> {

        private String statisticDemensionEnums = null;
        JournalTrxDataSet journalTrxDataSet = new JournalTrxDataSet();

        private MultipleOutputs mos;

        @Override
        protected void setup(Context context) throws IOException, InterruptedException {
            super.setup(context);
            mos = new MultipleOutputs<>(context);
            statisticDemensionEnums = context.getConfiguration().get("statisticDemensionEnums");

        }

        @Override
        protected void reduce(Text key, Iterable<JournalTrxDataSet> values,
                              Reducer<Text, JournalTrxDataSet, JournalTrxDataSet, NullWritable>.Context context)
                throws IOException, InterruptedException {

            String branchId = "";
            String branchName = "";
            String trxType = "";
            double trxAmt = 0.0;
            int trxCount = 0;

            for (JournalTrxDataSet rw : values) {
                branchId = rw.getBranchId();
                branchName = rw.getBranchName();
                trxType = rw.getTrxType();
                trxAmt += rw.getAmount();
                trxCount += rw.getCount();

            }
            journalTrxDataSet.Set(key.toString(), branchId, branchName, trxType, trxAmt, trxCount);
            if (Integer.parseInt(key.toString()) >= 500000) {
                mos.write("PFBANK", journalTrxDataSet, NullWritable.get(), "/PUFA/" + key.toString());
            } else if (Integer.parseInt(key.toString()) < 500000) {
                mos.write("ZSBANK", journalTrxDataSet, NullWritable.get(), "/ZHAOSHANG/" + key.toString());
            }
            //mos.write(journalTrxDataSet, NullWritable.get(), "/PUFA/"+  key.toString());
            //mos.write(journalTrxDataSet, NullWritable.get(), key.toString());
        }

        @Override
        protected void cleanup(Context context) throws IOException, InterruptedException {
            super.cleanup(context);
            mos.close();
        }

    }

    private static class JournalDataPartitioner extends Partitioner<Text, JournalTrxDataSet> {
        @Override
        public int getPartition(Text key, JournalTrxDataSet value, int arg2) {

            if ("706010101".equals(value.getBranchId())) {
                return 0;
            } else if ("706010106".equals(value.getBranchId())) {
                return 1;
            }
            return 2;
        }

    }

    private static class JournalDataMapper extends Mapper<Object, Text, Text, JournalTrxDataSet> {

        private String statisticDemensionEnums = null;

        JournalTrxDataSet journalTrxDataSet = new JournalTrxDataSet();

        @Override
        protected void setup(Context context) throws IOException, InterruptedException {
            super.setup(context);
            // 上面传递过来需要map reduce的纬度，一个按照终端来统计，一个按照分行来统计
            statisticDemensionEnums = context.getConfiguration().get("statisticDemensionEnums");

        }

        @Override
        protected void map(Object key, Text value, Mapper<Object, Text, Text, JournalTrxDataSet>.Context context)
                throws IOException, InterruptedException {

            String strContent = value.toString();
            String result[] = strContent.split("\\|");
            String terminalId = result[0]; // terminal id
            String branchId = result[1]; // branch id
            String branchName = result[2]; // branch id
            double amount = Double.parseDouble(result[4]); // transaction amount
            String trxType = result[5];
            journalTrxDataSet.Set(terminalId, branchId, branchName, trxType, amount, 1);
            context.write(new Text(terminalId), journalTrxDataSet);
        }

    }

    private static class JournalTrxDataComparator extends WritableComparator {

        protected JournalTrxDataComparator() {
            super(JournalTrxDataSet.class, true);
        }

        @Override
        public int compare(WritableComparable w1, WritableComparable w2) {

            JournalTrxDataSet j1 = (JournalTrxDataSet) w1;
            JournalTrxDataSet j2 = (JournalTrxDataSet) w2;
            int resultCompare = 0;
            /*
             * if (j1.getAmount() == j2.getAmount()) { resultCompare = 0; } else if
             * (j1.getAmount() < j2.getAmount()) { resultCompare = -1; } else {
             * resultCompare = 1; }
             */
            return resultCompare;// return -1，0，1
        }

        public static void main(String[] args) {

            SpringApplication.run(com.xhy.xgg.mapreduce.TerminalJournalDataService.class, args);
        }

    }
}