hadoop学习笔记之倒排索引

开发工具：eclipse

目标：对下面文档phone_numbers进行倒排索引：

13599999999 10086
13899999999 120
13944444444 13800138000
13722222222 13800138000
18800000000 120
13722222222 10086
18944444444 10086

代码：

 import java.io.IOException;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.io.*;
 import org.apache.hadoop.mapreduce.*;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

 public class Test_1 extends Configured implements Tool
 {
     enum Counter
     {
         LINESKIP, // error lines
     }

     public static class Map extends Mapper<LongWritable, Text, Text, Text>
     {
         public void map(LongWritable key, Text value, Context context)throws IOException, InterruptedException
         {
             String line = value.toString(); // read original data

             try
             {
                 // process data
                 String[] lineSplit = line.split(" ");
                 String anum = lineSplit[0];
                 String bnum = lineSplit[1];

                 context.write(new Text(bnum), new Text(anum)); // map output
             }
             catch(java.lang.ArrayIndexOutOfBoundsException e)
             {
                 context.getCounter(Counter.LINESKIP).increment(1);
                 return;
             }

         }
     }
     public static class Reduce extends Reducer<Text, Text, Text, Text>
     {
         public void reduce(Text key, Iterable<Text>values, Context context)throws IOException, InterruptedException
         {
             String valueString;
             String out = "";

             for (Text value : values)
             {
                 valueString = value.toString();
                 out += valueString + "|";
             }

             context.write(key, new Text(out));    // reduce output
         }
     }
     public int run(String[] args)throws Exception
     {
         Configuration conf = getConf();

         Job job = new Job(conf, "Test_1");  // task name
         job.setJarByClass(Test_1.class);    // specified task

         FileInputFormat.addInputPath(job, new Path(args[0]));  // input path
         FileOutputFormat.setOutputPath(job, new Path(args[1])); // output path

         job.setMapperClass(Map.class);
         job.setReducerClass(Reduce.class);
         job.setOutputFormatClass(TextOutputFormat.class);
         job.setOutputKeyClass(Text.class);
         job.setOutputValueClass(Text.class);

         job.waitForCompletion(true);

         return job.isSuccessful() ? 0 : 1;
     }

     public static void main(String[] args)throws Exception
     {
         int res = ToolRunner.run(new Configuration(), new Test_1(), args);
         System.exit(res);
     }
 }

运行结果：