02Hadoop二次排序2

案例：

数据：

邮编   |     日期     |金额

ILMN,2013-12-05,97.65
GOOD,2013-12-09,1078.14
IBM,2013-12-09,177.46
ILMN,2013-12-09,101.33
ILMN,2013-12-06,99.25,
GOOD,2013-12-06,1069.87
IBM,2013-12-06,177.67
GOOD,2013-12-05,1057.34
GOOD,2013-12-05,10.23
GOOD,2013-12-05,11.43
GOOD,2013-12-05,17.34

要求：把同一个邮编的放在一起，然后根据日期和金额降序排列。

效果如下：

思路：在map阶段，构造的key（CompositeKey）是：（邮编，日期）；value（NaturalValue）是（日期，价格）。然后key继承

WritableComparable，实现比较函数这样就可以保证一份数据出来是分区且区内有序的。

然后在shuffle过程中，指定一个key比较器(CompositeKeyComparator),使得在聚合过程后，对key按照先邮编，再时间，最后金额的顺序排序，key-value是键值对，key按照我们的意愿排好序了，

value也就排好了。

总的来说：降序什么的都是CompositeKeyComparator来决定的。

代码结构：

（1）key:组合键

 package com.book.test1;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;

 import org.apache.hadoop.io.DoubleWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
 /**
  * 这个的作用就是要数据在分区里面有序
  */
 /**
  * 定义组合键:就是可以把自己要比较的字段写入
  * @author Sxq
  *
  */
 //必须要时间这个WritableComparable这个类
 public class CompositeKey implements Writable, WritableComparable<CompositeKey> {

     // 股票的名字
     private Text stockSymbol;
     // 日期
     private LongWritable timestamp;
     private DoubleWritable price;

     public DoubleWritable getPrice() {
         return price;
     }
     public void setPrice(DoubleWritable price) {
         this.price = price;
     }
     public CompositeKey()
     {

     }
     public CompositeKey(Text _stockSymbol, LongWritable _timestamp,DoubleWritable _price) {
         this.stockSymbol = _stockSymbol;
         this.timestamp = _timestamp;
         this.price=_price;
     }

     public Text getStockSymbol() {
         return stockSymbol;
     }

     public void setStockSymbol(Text stockSymbol) {
         this.stockSymbol = stockSymbol;
     }

     public LongWritable getTimestamp() {
         return timestamp;
     }

     public void setTimestamp(LongWritable timestamp) {
         this.timestamp = timestamp;
     }

     //读出
     public void readFields(DataInput input) throws IOException {
         String value1=input.readUTF();
         long value2=input.readLong();
       this.stockSymbol=new  Text( value1);
       this.timestamp=  new LongWritable(value2);
       this.price=new DoubleWritable(input.readDouble());
     }

    //写入

     //@Override
     public void write(DataOutput output) throws IOException {
         output.writeUTF(this.stockSymbol.toString());
         output.writeLong(this.timestamp.get());
         output.writeDouble(this.price.get());
     }

     public int compareTo(CompositeKey other) {

            int comparator=this.stockSymbol.compareTo(other.stockSymbol);
             if(comparator==0)
             {
                 comparator=this.timestamp.compareTo(other.timestamp);
             }

         //升序
         //return comparator;

     return -comparator;
     }

     @Override
     public String toString() {
         return "CompositeKey [stockSymbol=" + stockSymbol + ", timestamp=" + timestamp + "]";
     }

 }

(2)key对应的value：

package com.book.test1;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.Writable;

public class NaturalValue implements Writable {
private long timestamp;
private double privce;

public long getTimestamp() {
    return timestamp;
}

public void setTimestamp(long timestamp) {
    this.timestamp = timestamp;
}

public double getPrivce() {
    return privce;
}

public void setPrivce(double privce) {
    this.privce = privce;
}

public void readFields(DataInput input) throws IOException {
    this.timestamp=input.readLong();
    this.privce=input.readDouble();

}

public void write(DataOutput output) throws IOException {

    output.writeLong(this.timestamp);
    output.writeDouble(this.privce);

}

}

（3）分区器：

NaturalKeyPartitioner

package com.book.test1;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Partitioner;
/**
 * 分区：按照邮编分，把邮编相同的放在一起
 * @author Sxq
 */

public class NaturalKeyPartitioner extends Partitioner<CompositeKey, NaturalValue> {

    @Override
    public int getPartition(CompositeKey key, NaturalValue value, int numPartitions) {
        return Math.abs((int)(key.getStockSymbol().hashCode())%numPartitions);
    }

}

（4）把key排序的比较器：在shuffle过程中用到的

package com.book.test1;

import javax.print.attribute.standard.MediaSize.Other;

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;

/**
 * 这个类的作用是把组合键排序，使得组合键也有顺序
 * @author Sxq
 *
 */
public class CompositeKeyComparator extends WritableComparator {

    public CompositeKeyComparator() {
         super(CompositeKey.class,true);
    }

    @Override
    public int compare(WritableComparable a, WritableComparable b) {
        CompositeKey ck1 = (CompositeKey) a;
        CompositeKey ck2 = (CompositeKey) b;
        int comparison = ck1.getStockSymbol().compareTo(ck2.getStockSymbol());
        //如果邮编相同，则根据日期进一步处理。
        if (comparison == 0) {

            int comparison2=ck1.getTimestamp().compareTo(ck2.getTimestamp());
            // 如果日期相同，则需要根据价格进一步处理
            if (comparison2==0) {
                //按照价格降序
                return ck1.getPrice().compareTo(ck2.getPrice())>0?-1:1;

            } else {
                //日期不同，就按照日期降序
                return ck1.getTimestamp().compareTo(ck2.getTimestamp())>0?-1:1;
            }
            }
        else {
            return comparison;
        }
    }
    static {
        WritableComparator.define(CompositeKey.class, new CompositeKeyComparator());
    }

}

（5）reduce的分区器：

CompositeGroupingComparator

package com.book.test1;

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;

/**
 * 分组：就是在reduce阶段分到一个组；
 * 就是邮编相同的放在一个组里面
 * @author Sxq
 *
 */
public class CompositeGroupingComparator extends WritableComparator{

     public CompositeGroupingComparator() {

     super(CompositeKey.class,true);
     }

    @Override
    public int compare(WritableComparable a, WritableComparable b) {
      CompositeKey v1=(CompositeKey)a;
      CompositeKey v2=(CompositeKey)b;

      return v1.getStockSymbol().compareTo(v2.getStockSymbol());

    }

}

（6）驱动类：

package com.book.test1;

import java.io.IOException;
import java.util.Date;
import java.util.Iterator;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class Cmain {
    static class Map1 extends Mapper<LongWritable, Text, CompositeKey, NaturalValue> {
        @Override
        protected void map(LongWritable key, Text value,
                Mapper<LongWritable, Text, CompositeKey, NaturalValue>.Context context)
                throws IOException, InterruptedException {
            String line = value.toString().trim();
            String[] lines = line.split(",");
            Date date = DateUtil.getDate(lines[1]);
            //long timestamp = date.getTime();

            long timestamp=UtilsCmain.DataTranform(lines[1]);
            CompositeKey compositeKey = new CompositeKey();
            NaturalValue naturalValue = new NaturalValue();
            naturalValue.setPrivce(Double.valueOf(lines[2]));
            naturalValue.setTimestamp(timestamp);
            compositeKey.setStockSymbol(new Text(lines[0]));
            compositeKey.setPrice(new DoubleWritable(Double.valueOf(lines[2])));
            compositeKey.setTimestamp(new LongWritable(timestamp));
            context.write(compositeKey, naturalValue);
        }

    }

    static class reduce1 extends Reducer<CompositeKey, NaturalValue, Text, Text> {
        @Override
        protected void reduce(CompositeKey key, Iterable<NaturalValue> vlaue,
                Reducer<CompositeKey, NaturalValue, Text, Text>.Context context) throws IOException, InterruptedException {

            Iterator<NaturalValue> iterator = vlaue.iterator();
            StringBuffer stringBuffer = new StringBuffer();
            while (iterator.hasNext()) {
                NaturalValue naturalValue=iterator.next();
                stringBuffer.append("(");
                stringBuffer.append(naturalValue.getTimestamp());
                stringBuffer.append(","+naturalValue.getPrivce()+")");
            }

            context.write(new Text(key.getStockSymbol()), new Text(stringBuffer.toString()));
        }
    }

    public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);
        job.setJarByClass(Cmain.class);

        job.setMapperClass(Map1.class);
        job.setReducerClass(reduce1.class);

        job.setMapOutputKeyClass(CompositeKey.class);
        job.setMapOutputValueClass(NaturalValue.class);

        job.setOutputValueClass(Text.class);
        job.setOutputKeyClass(Text.class);

        job.setSortComparatorClass(CompositeKeyComparator.class);
        // 在Reduce端设置分组，使得同一个邮编的在同一个组
        job.setGroupingComparatorClass(CompositeGroupingComparator.class);
        // 设置分区
        job.setPartitionerClass(NaturalKeyPartitioner.class);

        // 指定输入的数据的目录
        FileInputFormat.setInputPaths(job, new Path("/Users/mac/Desktop/stock.txt"));

        FileOutputFormat.setOutputPath(job, new Path("/Users/mac/Desktop/flowresort"));

        boolean result = job.waitForCompletion(true);
        System.exit(result ? 0 : 1);

    }

}

（7）工具类：将2012-12-09转为20121209这种形式：

package com.book.test1;

public class UtilsCmain {
     /**
      * 时间
      */
     public static  long  DataTranform(String vaule)
    {
        String[] args=vaule.split("-");
        String datatime=args[0]+args[1]+args[2];

         return Long.valueOf(datatime);

    }

}

运行结果：