InAction-MR的topK

---

本来只是想拿搜狗的数据练练手的，却无意踏足MR的topK问题。经过几番波折，虽然现在看起来很简单，但是摸爬滚打中也学到了不少

---

数据是搜狗实验室下的搜索日志，格式大概为：

 00:00:00    2982199073774412    [360安全卫士]    8 3    download.it.com.cn/softweb/software/firewall/antivirus/20067/17938.html
 00:00:00    07594220010824798    [哄抢救灾物资]    1 1    news.21cn.com/social/daqian/2008/05/29/4777194_1.shtml
 00:00:00    5228056822071097    [75810部队]    14 5    www.greatoo.com/greatoo_cn/list.asp?link_id=276&title=%BE%DE%C2%D6%D0%C2%CE%C5
 00:00:00    6140463203615646    [绳艺]    62 36    www.jd-cd.com/jd_opus/xx/200607/706.html
 00:00:00    8561366108033201    [汶川地震原因]    3 2    www.big38.net/
 00:00:00    23908140386148713    [莫衷一是的意思]    1 2    www.chinabaike.com/article/81/82/110/2007/2007020724490.html
 00:00:00    1797943298449139    [星梦缘全集在线观看]    8 5    www.6wei.net/dianshiju/????\xa1\xe9|????do=index
 00:00:00    00717725924582846    [闪字吧]    1 2    www.shanziba.com/

我只是要搜索词，其他的不管，然后通过MR计算出搜索量最高的前N个词（N自定义）

整体项目结构为：

先来个类处理根据日志格式拿出搜索词

SEA.java

 package org.admln.topK;

 /**
  * @author admln
  *
  */
 public class SEA {

     private String seaWord;

     private boolean isValid;

     public static SEA parser(String line) {
         SEA sea = new SEA();
         String str = line.split("\t")[2];
         if(str.length()<3) {
             sea.setValid(false);
         }else {
             sea.setValid(true);
             sea.setSeaWord(str.substring(1, str.length()-1));
         }
         return sea;
     }

     public String getSeaWord() {
         return seaWord;
     }

     public void setSeaWord(String seaWord) {
         this.seaWord = seaWord;
     }

     public boolean isValid() {
         return isValid;
     }

     public void setValid(boolean isValid) {
         this.isValid = isValid;
     }

 }

然后就是MR

 package org.admln.topK;

 import java.io.IOException;
 import java.util.Collections;
 import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TreeMap;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

 /**
  * @author admln
  *
  */
 public class TopK {

     public static class topKMapper extends
             Mapper<Object, Text, Text, IntWritable> {
         Text word = new Text();
         IntWritable ONE = new IntWritable(1);

         @Override
         public void map(Object key, Text value, Context context)
                 throws IOException, InterruptedException {
             SEA sea = SEA.parser(value.toString());
             if (sea.isValid()) {
                 word.set(sea.getSeaWord());
                 context.write(word, ONE);
             }
         }
     }

     public static class topKReducer extends
             Reducer<Text, IntWritable, Text, IntWritable> {
         int sum;
         int max;
         private static TreeMap<Integer,String> tree = new TreeMap<Integer,String>(Collections.reverseOrder());

         public void reduce(Text key, Iterable<IntWritable> values,
                 Context context) {
             sum = 0;
             max = context.getConfiguration().getInt("topK", 10);
             for (IntWritable val : values) {
                 sum += val.get();
             }
             tree.put(Integer.valueOf(sum), key.toString());
             if (tree.size() > max) {
                 tree.remove(tree.lastKey());
             }

         }

         @Override
         protected void cleanup(Context context) throws IOException, InterruptedException {
             Set<Entry<Integer, String>> set = tree.entrySet();
             for (Entry<Integer, String> entry : set) {
                 context.write(new Text(entry.getValue()), new IntWritable(entry.getKey()));
             }
         }
     }

     public static void main(String[] args) throws Exception {
         Path input = new Path("hdfs://hadoop:8020/input/topK/");
         Path output = new Path("hdfs://hadoop:8020/output/topK/");

         Configuration conf = new Configuration();

         conf.setInt("topK", Integer.valueOf(args[1]));

         Job job = new Job(conf, "topK");

         job.setJarByClass(TopK.class);

         job.setMapperClass(topKMapper.class);
         job.setReducerClass(topKReducer.class);

         job.setOutputKeyClass(Text.class);
         job.setOutputValueClass(IntWritable.class);

         FileInputFormat.addInputPath(job, input);
         FileOutputFormat.setOutputPath(job, output);

         System.exit(job.waitForCompletion(true) ? 0 : 1);

     }

 }

然后上传数据（注意文件格式要从gb2312改成utf-8的。因为hadoop全部是utf-8编码的。如果不转码最后结果中文就是乱码）

本机调试或者上传到hadoop上运行

机器环境是centos6.4、hadoop是2.2.0、JDK是1.7

运行结果：

---

重要知识点：

　　1.TreeMap，虽然是Java的知识，还是普及了一下；

　　2.cleanup，这个复写API的执行时间要知道。

---

源码：http://pan.baidu.com/s/1i3y0rwL

---