MapReduce中的倒排索引

0.倒排索引资料:

http://blog.csdn.net/pzasdq/article/details/51442856

1.三个日志源文件:

a.txt

hello tom
hello jerry
hello tom

b.txt

hello jerry
hello jerry
tom jerry

c.txt

hello jerry
hello tom

希望统计出来的结果如下:

hello   a.txt->3 b.txt->2 c.txt->2
jerry   b.txt->3 a.txt->1 c.txt->1
tom     a.txt->2 b.txt->1 c.txt->1

2.上代码:

 import java.io.IOException;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.FileSplit;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

 public class InverseIndex {

     public static class IndexMapper extends Mapper<LongWritable, Text, Text, Text>{
         private Text k = new Text();
         private Text v = new Text();
         @Override
         protected void map(LongWritable key, Text value,Mapper<LongWritable, Text, Text, Text>.Context context)
                 throws IOException, InterruptedException {
             String line = value.toString();
             String [] words = line.split(" ");
             FileSplit inputSplit = (FileSplit)context.getInputSplit();//返回mapper读取的是哪个切片split
             //path=hdfs://itcast:9000/ii/a.txt
             //k2,v2 为 hello->a.txt     {1,1,1}
             String path = inputSplit.getPath().toString();
             for (String word : words) {
                 k.set(word + "->" + path);
                 v.set("1");
                 context.write(k, v);
             }
         }
     }

     public static class IndexCombiner extends Reducer<Text, Text, Text, Text>{
         private Text k = new Text();
         private Text v = new Text();
         @Override
         protected void reduce(Text key, Iterable<Text> values,Reducer<Text, Text, Text, Text>.Context context)
                 throws IOException, InterruptedException {
             //k2,v2 为hello->a.txt {1,1,1}   ----->  k3,v3为 hello,a.txt->3
             int counter = 0;
             for(Text text :values){
                 counter += Integer.parseInt(text.toString());
             }
             String[] wordAndPath = key.toString().split("->");
             String word = wordAndPath[0];
             String path = wordAndPath[1];
             k.set(word);
             v.set(path+"->"+counter);
             context.write(k,v);
         }
     }

     public static class IndexReducer extends Reducer<Text, Text, Text, Text>{
         private Text v = new Text();
         @Override
         protected void reduce(Text key, Iterable<Text> values,Reducer<Text, Text, Text, Text>.Context context)
                 throws IOException, InterruptedException {
             //Reducer这里 是把所有key相同的搞到一块了,这个地方对应的values为Iterable也证实这一点.
             //不同的Map根据k2 到达Reducer 把k2相同的汇聚到一起...对应的k2对应的v2组成一个集合.
             //从combiner过来的k和v为   hello,a.txt->3  经过reducer变成
             String result = "";
             for(Text t:values){
                 result += t.toString() + "\t";
             }
             v.set(result);
             context.write(key,v);
         }
     }

     public static void main(String[] args) throws Exception {
         Configuration conf = new Configuration();
         Job job = Job.getInstance(conf);
         job.setJarByClass(InverseIndex.class);

         job.setMapperClass(IndexMapper.class);
         job.setMapOutputKeyClass(Text.class);
         job.setMapOutputValueClass(Text.class);

         job.setCombinerClass(IndexCombiner.class);
         FileInputFormat.setInputPaths(job, new Path(args[0]));

         job.setReducerClass(IndexReducer.class);
         job.setOutputKeyClass(Text.class);
         job.setOutputValueClass(Text.class);

         FileOutputFormat.setOutputPath(job, new Path(args[1]));
         System.exit(job.waitForCompletion(true) ? 0 : 1);//0是正常推出以 1是异常退出.
     }
 }

3.打成jar包,通过命令执行

hadoop jar /root/itcastmr.jar itcastmr.inverseindex.InverseIndex /user/root/InverseIndex /InverseIndexResult

查看结果文件: