MapReduce -- 统计天气信息

示例

数据：

-- ::    34c
-- ::    36c
-- ::    32c
-- ::    37c
-- ::    23c
-- ::    45c
-- ::    50c
-- ::    33c
-- ::    41c
-- ::    27c
-- ::    45c
-- ::    46c
-- ::    47c

要求：

将每年每月中的气温排名前三的数据找出来

实现：

1.每一年用一个reduce任务处理;

2.创建自定义数据类型，存储 [年-月-日-温度];

2.自己实现排序函数 根据 [年-月-温度] 降序排列，也可以在定义数据类型中进行排序;

3.自己实现分组函数，对 [年-月] 分组，reduce的key是分组结果，根据相同的分组值，统计reduce的value值，只统计三个值就可以，因为已经实现了自排序函数。

注意点：

1.自定义数据类型的使用;

2.自定义排序类的使用;

3.自定义分组类的使用，分组类对那些数据进行分组;

4.自定义分区类，分区类与reduce job个数的关系;

示例代码：

RunJob.java

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

 import java.io.IOException;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
 import java.util.Date;

 /**
  * weather 统计天气信息
  *
  * 数据：
  *  1999-10-01 14:21:02    34c
  *  1999-11-02 13:01:02    30c
  *
  * 要求：
  * 将每年的每月中气温排名前三的数据找出来
  *
  * 实现：
  * 1.每一年用一个reduce任务处理;
  * 2.创建自定义数据类型，存储 [年-月-日-温度];
  * 2.自己实现排序函数 根据 [年-月-温度] 降序排列，也可以在定义数据类型中进行排序;
  * 3.自己实现分组函数，对 [年-月] 分组，reduce的key是分组结果，根据相同的分组值，统计reduce的value值，只统计三个值就可以，因为已经实现了自排序函数。
  *
  * Created by Edward on 2016/7/11.
  */
 public class RunJob {

     public static void main(String[] args)
     {
         //access hdfs's user
         System.setProperty("HADOOP_USER_NAME","root");

         Configuration conf = new Configuration();
         conf.set("fs.defaultFS", "hdfs://node1:8020");

         try {
             FileSystem fs = FileSystem.get(conf);

             Job job = Job.getInstance(conf);
             job.setJarByClass(RunJob.class);
             job.setMapperClass(MyMapper.class);
             job.setReducerClass(MyReducer.class);

             //需要指定 map out 的 key 和 value
             job.setOutputKeyClass(InfoWritable.class);
             job.setOutputValueClass(Text.class);

             //设置分区 继承 HashPartitioner
             job.setPartitionerClass(YearPartition.class);
             //根据年份创建指定数量的reduce task
             job.setNumReduceTasks(3);

             //设置排序 继承 WritableComparator
             //job.setSortComparatorClass(SortComparator.class);

             //设置分组 继承 WritableComparator 对reduce中的key进行分组
             job.setGroupingComparatorClass(GroupComparator.class);

             FileInputFormat.addInputPath(job, new Path("/test/weather/input"));

             Path path = new Path("/test/weather/output");
             if(fs.exists(path))//如果目录存在，则删除目录
             {
                 fs.delete(path,true);
             }
             FileOutputFormat.setOutputPath(job, path);

             boolean b = job.waitForCompletion(true);
             if(b)
             {
                 System.out.println("OK");
             }

         } catch (Exception e) {
             e.printStackTrace();
         }
     }

     public static class MyMapper extends Mapper<LongWritable, Text, InfoWritable, Text > {

         private static SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

         @Override
         protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
             String[] str = value.toString().split("\t");

             try {
                 Date date = sdf.parse(str[0]);
                 Calendar c = Calendar.getInstance();
                 c.setTime(date);
                 int year = c.get(Calendar.YEAR);
                 int month = c.get(Calendar.MONTH)+1;
                 int day = c.get(Calendar.DAY_OF_MONTH);

                 double temperature = Double.parseDouble(str[1].substring(0,str[1].length()-1));

                 InfoWritable info = new InfoWritable();
                 info.setYear(year);
                 info.setMonth(month);
                 info.setDay(day);
                 info.setTemperature(temperature);

                 context.write(info, value);

             } catch (ParseException e) {
                 e.printStackTrace();
             }
         }
     }

     public static class MyReducer extends Reducer<InfoWritable, Text, Text, NullWritable> {
         @Override
         protected void reduce(InfoWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
             int i=0;
             for(Text t: values)
             {
                 i++;
                 if(i>3)
                     break;
                 context.write(t, NullWritable.get());
             }
         }
     }
 }

InfoWritable.java

 import org.apache.hadoop.io.WritableComparable;

 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;

 /**
  * 自定义数据类型 继承 WritableComparable
  * 【年-月-日-温度】
  * Created by Edward on 2016/7/11.
  */
 public class InfoWritable implements WritableComparable<InfoWritable> {

     private int year;
     private int month;
     private int day;
     private double temperature;

     public void setYear(int year) {
         this.year = year;
     }

     public void setMonth(int month) {
         this.month = month;
     }

     public void setDay(int day) {
         this.day = day;
     }

     public void setTemperature(double temperature) {
         this.temperature = temperature;
     }

     public int getYear() {
         return year;
     }

     public int getMonth() {
         return month;
     }

     public int getDay() {
         return day;
     }

     public double getTemperature() {
         return temperature;
     }

     /**
      *
      * 对象比较，对温度进行倒序排序
      */
     @Override
     public int compareTo(InfoWritable o) {

         int result = Integer.compare(this.getYear(),o.getYear());
         if(result == 0)
         {
             result = Integer.compare(this.getMonth(),o.getMonth());
             if(result == 0)
             {
                 return -Double.compare(this.getTemperature(), o.getTemperature());
             }
             else
                 return result;
         }
         else
             return result;

         //return this==o?0:-1;
     }

     @Override
     public void write(DataOutput dataOutput) throws IOException {
         dataOutput.writeInt(this.year);
         dataOutput.writeInt(this.month);
         dataOutput.writeInt(this.day);
         dataOutput.writeDouble(this.temperature);
     }

     @Override
     public void readFields(DataInput dataInput) throws IOException {
         this.year = dataInput.readInt();
         this.month = dataInput.readInt();
         this.day = dataInput.readInt();
         this.temperature = dataInput.readDouble();
     }
 }

YearPartition.java

 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.lib.partition.HashPartitioner;

 /**
  *
  * 创建分区，通过key中的year来创建分区
  *
  * Created by Edward on 2016/7/11.
  */
 public class YearPartition extends HashPartitioner <InfoWritable, Text>{
     @Override
     public int getPartition(InfoWritable key, Text value, int numReduceTasks) {
         return key.getYear()%numReduceTasks;
     }
 }

GroupComparator.java

 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.io.WritableComparator;

 /**
  * 创建分组类，继承WritableComparator
  * 【年-月】
  * Created by Edward on 2016/7/11.
  */
 public class GroupComparator extends WritableComparator {

     GroupComparator()
     {
         super(InfoWritable.class, true);
     }

     @Override
     public int compare(WritableComparable a, WritableComparable b) {
         InfoWritable ia = (InfoWritable)a;
         InfoWritable ib = (InfoWritable)b;

         int result = Integer.compare(ia.getYear(),ib.getYear());
         if(result == 0)
         {
             return Integer.compare(ia.getMonth(),ib.getMonth());
         }
         else
             return result;
     }
 }

SortComparator.java

 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.io.WritableComparator;

 /**
  * 排序类，继承WritableComparator
  * 排序规则【年-月-温度】 温度降序
  * Created by Edward on 2016/7/11.
  */
 public class SortComparator extends WritableComparator {

     /**
      * 调用父类的构造函数
      */
     SortComparator()
     {
         super(InfoWritable.class, true);
     }

     /**
      * 比较两个对象的大小，使用降序排列
      * @param a
      * @param b
      * @return
      */
     @Override
     public int compare(WritableComparable a, WritableComparable b) {

         InfoWritable ia = (InfoWritable)a;
         InfoWritable ib = (InfoWritable)b;

         int result = Integer.compare(ia.getYear(),ib.getYear());
         if(result == 0)
         {
             result = Integer.compare(ia.getMonth(),ib.getMonth());
             if(result == 0)
             {
                 return -Double.compare(ia.getTemperature(), ib.getTemperature());
             }
             else
                 return result;
         }
         else
             return result;
     }
 }