Mapreduce 订单分组案例

程序执行流程如下:

map()-->getPartition()分区--->write()(序列化,每一行都顺序执行这三个方法)--->readFields()---->compareTo()排序---->readFields()--->分组compare--->reduce()

对于每一行的内容,依次执行map()-->getPartition()---->write()(序列化,会在指定的输出目录下生成temporary目录),全部序列化完成之后执行readFields(),之后进行排序,全部排序完毕后才会再次执行readFields(),

之后进行分组,分出一组,执行一次reduce()写出数据

1.OrderBean(分了区之后就不需要对OrderId进行排序了,因为同一分区OrderId相同)

 package mapreduce.order;

 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import org.apache.hadoop.io.WritableComparable;

 public class OrderBean implements WritableComparable<OrderBean>{
     private String orderId;
     private Double price;

     public String getOrderId() {
         return orderId;
     }

     public void setOrderId(String orderId) {
         this.orderId = orderId;
     }

     public Double getPrice() {
         return price;
     }

     public void setPrice(Double price) {
         this.price = price;
     }

     public OrderBean() {
         super();
     }

     /**
      * 序列化
      * @param out
      * @throws IOException
      */
     @Override
     public void write(DataOutput out) throws IOException {
         out.writeUTF(orderId);
         out.writeDouble(price);
     }

     /**
      * 反序列化
      * @param in
      * @throws IOException
      */
     @Override
     public void readFields(DataInput in) throws IOException {
         orderId = in.readUTF();
         price   = in.readDouble();
     }

     @Override
     public String toString() {
         return orderId + "\t" + price;
     }

     /**
      * 二次排序(如果进行了分区,可以直接对price排序,不再需要二次排序)
      * @param o
      * @return
      */
     @Override
     public int compareTo(OrderBean o) {
         //先比较orderId(字符串)
     //    int result = this.getOrderId().compareTo(o.getOrderId());

         //相同再比较price
     //    if(result == 0) {
             //降序,数字的compareTo() -1表示小于
         int    result = this.getPrice() > o.getPrice()?-:;
     //    }
         return result;
     }
 }

2.OrderMapper

 package mapreduce.order;

 import java.io.IOException;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Mapper;

 public class OrderMapper extends Mapper<LongWritable, Text, OrderBean, NullWritable> {
     OrderBean order = new OrderBean();

     @Override
     protected void map(LongWritable key, Text value,
             Mapper<LongWritable, Text, OrderBean, NullWritable>.Context context)
             throws IOException, InterruptedException {

         // 1.读取一行
         String line = value.toString();

         // 2.切割并封装对象
         String[] split = line.split("\t");
         order.setOrderId(split[]);
         order.setPrice(Double.parseDouble(split[]));

         // 3.输出
         context.write(order, NullWritable.get());
     }
 }

3.OrderPartioner

 package mapreduce.order;

 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.mapreduce.Partitioner;

 /**
  * 分区
  * @author tele
  *
  */
 public class OrderPartioner extends Partitioner<OrderBean, NullWritable> {
     /**
      * 默认是根据orderBean的hashcode进行分区
      * 现在更改为orderBean对象的orderId进行分区,这样相同orderId的即可进入同一分区
      */
     @Override
     public int getPartition(OrderBean key, NullWritable value, int numReduceTasks) {
         return (key.getOrderId().hashCode() & Integer.MAX_VALUE) % numReduceTasks;
     }
 }

4.OrderGroupComparator

 package mapreduce.order;

 import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.io.WritableComparator;

 /**
  * reduce分组的方法
  *
  * @author tele
  *
  */
 public class OrderGroupComparator extends WritableComparator {
     /**
      * 确保类型转换成功,能够创建实例,如果是false,创建的实例为null
      */
     protected OrderGroupComparator() {
         super(OrderBean.class, true);
     }

     /**
      * reduce默认是根据key进行分组,即此处的orderBean,重写之后
      * 根据orderBean的orderId进行分组,相同的orderId的orderBean将会分到同一组
      */
     @Override
     public int compare(WritableComparable a, WritableComparable b) {
         OrderBean bean1 = (OrderBean) a;
         OrderBean bean2 = (OrderBean) b;
         return bean1.getOrderId().compareTo(bean2.getOrderId());
     }
 }

5.OrderReducer(reduce()每次读入一组)

 package mapreduce.order;

 import java.io.IOException;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.mapreduce.Reducer;

 public class OrderReducer extends Reducer<OrderBean, NullWritable, OrderBean, NullWritable> {
     @Override
     protected void reduce(OrderBean key, Iterable<NullWritable> value,
             Reducer<OrderBean, NullWritable, OrderBean, NullWritable>.Context context)
             throws IOException, InterruptedException {
         context.write(key, NullWritable.get());
     }
 }

6.OrderDriver(必须设置分组类)

 package mapreduce.order;

 import java.io.IOException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

 /**
  *
  * @author tele
  *
  */
 public class OrderDriver {
     public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
         // 1.获得job实例
         Configuration conf = new Configuration();
         Job job = Job.getInstance(conf);

         // 2.设置driverclass
         job.setJarByClass(OrderDriver.class);
         // mapclass
         job.setMapperClass(OrderMapper.class);
         // reduceclass
         job.setReducerClass(OrderReducer.class);

         // 3.map输入输出数据类型
         job.setMapOutputKeyClass(OrderBean.class);
         job.setMapOutputValueClass(NullWritable.class);

         // 4.最终输出数据类型
         job.setOutputKeyClass(OrderBean.class);
         job.setOutputValueClass(NullWritable.class);

         // 7.设置分区类
         job.setPartitionerClass(OrderPartioner.class);
         job.setNumReduceTasks();

         // 8.设置分组类
         job.setGroupingComparatorClass(OrderGroupComparator.class);

         // 5.输入与输出路径
         FileInputFormat.setInputPaths(job, new Path(args[]));
         FileOutputFormat.setOutputPath(job, new Path(args[]));

         // 6.提交与退出
         boolean result = job.waitForCompletion(true);
         System.exit(result ?  : );

     }
 }