MapReduce编程练习（三），按要求不同文件名输出结果

问题：按要求文件名输出结果，比如这里我要求对一个输入文件中的WARN，INFO，ERROR，的信息项进行分析，并分别输入到对应的以WARN，INFO。ERROR和OTHER开头的结果文件中，其中结果文件包含对应的相关信息。

输入文件：

　　　　输入文件为hadoop的一些logs日志信息文件，比如：

示例程序：

package com.map.splitFile;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.regex.Pattern;  

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 

public class SplitFilesToResult extends Configured{  

    @SuppressWarnings("deprecation")
	public static void main(String[] args) {
        String in = "/SplitFilesToResult/input";
        String out = "/SplitFilesToResult/output";  

        Job job;
        try {
            //删除hdfs目录
        	SplitFilesToResult wc2 = new SplitFilesToResult();
            wc2.removeDir(out);  

            job = new Job(new Configuration(), "wordcount Job");
            job.setOutputKeyClass(Text.class);
            job.setOutputValueClass(Text.class);
            job.setMapperClass(mapperString.class);
            job.setReducerClass(reduceStatistics.class);  

            //定义附加的输出文件
            MultipleOutputs.addNamedOutput(job,"INFO",TextOutputFormat.class,Text.class,Text.class);
            MultipleOutputs.addNamedOutput(job,"ERROR",TextOutputFormat.class,Text.class,Text.class);
            MultipleOutputs.addNamedOutput(job,"WARN",TextOutputFormat.class,Text.class,Text.class);
            MultipleOutputs.addNamedOutput(job,"OTHER",TextOutputFormat.class,Text.class,Text.class);   

            FileInputFormat.addInputPath(job, new Path(in));
            FileOutputFormat.setOutputPath(job, new Path(out));
            job.waitForCompletion(true);  

            FileSystem fs  = FileSystem.get(new URI("hdfs://localhost:9000"), new Configuration());
            fs.delete(new Path("/SplitFilesToResult/output/part-r-00000")); 

        } catch (IOException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }  

    @SuppressWarnings("deprecation")
	public void removeDir(String filePath) throws IOException, URISyntaxException{
        String url = "hdfs://localhost:9000";
        FileSystem fs  = FileSystem.get(new URI(url), new Configuration());
        fs.delete(new Path(filePath));
    }
}  

/**
 * 重写maptask使用的map方法
 * @author nange
 *
 */
class mapperString extends Mapper<LongWritable, Text, Text, Text>{
    //设置正则表达式的编译表达形式
    public static Pattern PATTERN = Pattern.compile(" ");
    @Override
    protected void map(LongWritable key, Text value, Context context)
            throws IOException, InterruptedException {  

        String[] words = PATTERN.split(value.toString());
        System.out.println("********" + value.toString());
       if(words.length >= 2){
    	   if(words.length == 2){
        	   context.write(new Text("ERROR"), new Text(value.toString()));
    	   }else if(words[0].equals("at")){
        	   context.write(new Text("ERROR"), new Text(value.toString()));
           }else{
    	       context.write(new Text(words[2]), new Text(value.toString()));
    	   }
       }else
           context.write(new Text("OTHER"), new Text(value.toString()));

    }
}  

/**
 * 对单词做统计
 * @author nange
 *
 */
class reduceStatistics extends Reducer<Text, Text, Text, Text>{  

    //将结果输出到多个文件或多个文件夹
    private MultipleOutputs<Text,Text> mos;
    //创建MultipleOutputs对象
    protected void setup(Context context) throws IOException,InterruptedException {
        mos = new MultipleOutputs<Text, Text>(context);
     }  

    @Override
    protected void reduce(Text key, Iterable<Text> values, Context context)
            throws IOException, InterruptedException {
    	for(Text t: values){
            //使用MultipleOutputs对象输出数据
            if(key.toString().equals("INFO")){
                mos.write("INFO", "", t);
            }else if(key.toString().equals("ERROR")){
                mos.write("ERROR", "", t);
            }else if(key.toString().equals("WARN")){
                //输出到hadoop/hadoopfile-r-00000文件
                mos.write("WARN", "", t, "WARN");
            }else{
            	mos.write("OTHER", "", t);
            }
    	}

    }  

    //关闭MultipleOutputs对象
    protected void cleanup(Context context) throws IOException,InterruptedException {
        mos.close();
    }
}