MapReduce 简单的全文搜索

上一个已经实现了反向索引，那么为什么不尝试下全文搜索呢。例如有了

Hello     file3.txt:1;
MapReduce     file3.txt:2;fil1.txt:1;fil2.txt:1;
bye     file3.txt:1; 
is     fil1.txt:1;fil2.txt:2;
powerful     fil2.txt:1;
simple     fil2.txt:1;fil1.txt:1;

那么我要找MapReduce is simple，那么就有file1 和file2有。基本的思想就是先按照MapReduce is simple一个个在索引上查找，例如

MapReduce 3,1,2

is 1,2

simple 2,1

接着以file作为key，word作为value输出

1 MapReduce is simple

2 MapReduce is simple

3 MapReduce

接下来在Reduce中对各个value的单词名进行统计，如果超过3个，那就说明有匹配的了。

 

这里主要的技术是map，Reduce如何获得命令行参数。在主类中可以通过

String[] pathargs= new GenericOptionsParser(conf, args).getRemainingArgs();

来获得参数，但是如何向map和reduce传参呢，这里有三种方法，只看了一种，因为感觉够用了。

我们通过在主类中的配置实例写参数conf.set(key,value)这里的key，value都是String。要记住一点，这个语句一定要在jog.getInstance(conf)之前，否则都实例化了一个job了还怎么配置呢。接着在map或者reduce中通过

Configuration  conf=context.getConfiguration()来获得主类的配置文件。接着就可以conf.get(key)了。

代码具体如下：

public class Find {

 

                 public static void main(String[] args) throws Exception {

                                Configuration conf = new Configuration();

                                String[] pathargs= new GenericOptionsParser(conf, args).getRemainingArgs();

                                 if(pathargs.length <2){

                                                System. err.println(pathargs.length );

                                                System. exit(2);

                                }

                                

                                conf.set( "argsnum",Integer.toString(pathargs. length));

                                 for(int i=2;i<pathargs.length;i++){  

                                                conf.set( "args"+i,pathargs[i]);

                                                System. out.println(pathargs[i]);

                                }

                                

                                Job job = Job. getInstance(conf, "JobName");

                                job.setJarByClass(org.apache.hadoop.examples10.Find. class);

                                 // TODO: specify a mapper

                                job.setMapperClass(MyMapper. class);

                                 // TODO: specify a reducer

                                job.setReducerClass(MyReducer. class);

 

                                 // TODO: specify output types

                                job.setOutputKeyClass(Text. class);

                                job.setOutputValueClass(Text. class);

 

                                 // TODO: specify input and output DIRECTORIES (not files)

                                FileInputFormat. setInputPaths(job, new Path(pathargs[0]));

                                FileOutputFormat. setOutputPath(job, new Path(pathargs[1]));

                                

 

                                

                                 if (!job.waitForCompletion(true))

                                                 return;

                }

 

}

 

 

public class MyMapper extends Mapper<LongWritable, Text, Text, Text> {

                 //String[] content={"MapReduce","is","simple"};

                 public void map(LongWritable ikey, Text ivalue, Context context)

                                                 throws IOException, InterruptedException {

                                Configuration conf=context.getConfiguration();

                                 int argsnum=Integer.parseInt(conf.get( "argsnum"));

                                 //int argsnum=conf.get(" argsnum");

                                 int i=0;

                                ArrayList<String> content= new ArrayList<String>();

                                 for(i=2;i<argsnum;i++ ){

                                                 //System.out.println(conf.get("args"+i));

                                                content.add(conf.get( "args"+i));

                                                                     

                                }                              

                                String line=ivalue.toString();

                                String key=line.split( "         " )[0];

                                String value=line.split( "      " )[1];

                                StringTokenizer st= new StringTokenizer(value,";" );

                                 for(i=0;i<content.size();i++){

                                                 if(content.get(i).compareTo(key)==0){

                                                                ArrayList<String> filelist=new ArrayList<String>();

                                                                 while(st.hasMoreTokens()){

                                                                                String file=st.nextToken();

                                                                                file=file.split( ":")[0];

                                                                                filelist.add(file);

                                                                }

                                                                 for(int j=0;j<filelist.size();j++){

                                                                                context.write( new Text(filelist.get(j)),new Text(key));

                                                                }

                                                }

                                }

                }

 

}

 

 

public class MyReducer extends Reducer<Text, Text, Text, Text> {

 

                 public void reduce(Text _key, Iterable<Text> values, Context context)

                                                 throws IOException, InterruptedException {

                                Configuration conf=context.getConfiguration();

                                 int argsnum=Integer.parseInt(conf.get( "argsnum"));

                                 // process values

                                 int sum=0;

                                String filename= new String();

                                

                                 for(int i =2;i<argsnum; i++ ){

                                                 //System.out.println(conf.get("args"+i));

                                                filename+=(conf.get( "args"+i));

                                                filename+= " ";       

                                }

                                

                                 for (Text val : values) {

                                                sum++;

                                }

                                 if(sum>=argsnum-2){

                                                context.write( new Text(filename),_key);

                                }

                }

 

}