直接附代码,说明都在源码里了。

 package com.hadoop.totalsort;

 import java.io.IOException;
 import java.util.ArrayList;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.FileInputFormat;
 import org.apache.hadoop.mapred.FileSplit;
 import org.apache.hadoop.mapred.InputSplit;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.LineRecordReader;
 import org.apache.hadoop.mapred.RecordReader;
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.util.IndexedSortable;
 import org.apache.hadoop.util.QuickSort;

 public class SamplerInputFormat extends FileInputFormat<Text, Text> {  

     static final String PARTITION_FILENAME = "_partition.lst";
     static final String SAMPLE_SIZE = "terasort.partitions.sample";
     private static JobConf lastConf = null;
     private static InputSplit[] lastResult = null;  

     static class TextSampler implements IndexedSortable {  

         public ArrayList<Text> records = new ArrayList<Text>();  

         public int compare(int arg0, int arg1) {
             Text right = records.get(arg0);
             Text left = records.get(arg1);  

             return right.compareTo(left);
         }  

         public void swap(int arg0, int arg1) {
             Text right = records.get(arg0);
             Text left = records.get(arg1);  

             records.set(arg0, left);
             records.set(arg1, right);
         }  

         public void addKey(Text key) {
             records.add(new Text(key));
         }  

         //将采集出来的key数据排序
         public Text[] createPartitions(int numPartitions) {
             int numRecords = records.size();
             if (numPartitions > numRecords) {
                 throw new IllegalArgumentException("Requested more partitions than input keys (" + numPartitions +
                         " > " + numRecords + ")");
             }
             new QuickSort().sort(this, 0, records.size());
             float stepSize = numRecords / (float) numPartitions;  //采集的时候应该是采了100条记录,从10个分片查找的,此处再取numPartitions-1条
             Text[] result = new Text[numPartitions - 1];
             for (int i = 1; i < numPartitions; ++i) {
                 result[i - 1] = records.get(Math.round(stepSize * i));
             }
             return result;
         }  

     }  

     public static void writePartitionFile(JobConf conf, Path partFile) throws IOException {
         //前段代码从分片中采集数据,通过sampler.addKey存入TextSampler中的records数组
         SamplerInputFormat inputFormat = new SamplerInputFormat();
         TextSampler sampler = new TextSampler();
         Text key = new Text();
         Text value = new Text();  

         int partitions = conf.getNumReduceTasks(); // Reducer任务的个数
         long sampleSize = conf.getLong(SAMPLE_SIZE, 100); // 采集数据-键值对的个数
         InputSplit[] splits = inputFormat.getSplits(conf, conf.getNumMapTasks());// 获得数据分片
         int samples = Math.min(10, splits.length);// 采集分片的个数   ,采集10个分片
         long recordsPerSample = sampleSize / samples;// 每个分片采集的键值对个数
         int sampleStep = splits.length / samples; // 采集分片的步长   ,总的分片个数/要采集的分片个数
         long records = 0;  

         for (int i = 0; i < samples; i++) {  //1...10分片数
             RecordReader<Text, Text> reader = inputFormat.getRecordReader(splits[sampleStep * i], conf, null);
             while (reader.next(key, value)) {
                 sampler.addKey(key);   //将key值增加到sampler的records数组
                 records += 1;
                 if ((i + 1) * recordsPerSample <= records) {  //目的是均匀采集各分片的条数,比如采集到第5个分片,那么记录条数应该小于5个分片应该的条数
                     break;
                 }
             }
         }
         FileSystem outFs = partFile.getFileSystem(conf);
         if (outFs.exists(partFile)) {
             outFs.delete(partFile, false);
         }
         SequenceFile.Writer writer = SequenceFile.createWriter(outFs, conf, partFile, Text.class, NullWritable.class);
         NullWritable nullValue = NullWritable.get();
         for (Text split : sampler.createPartitions(partitions)) {  //调用createPartitions方法,排序采集出来的数据,并取partitions条
             writer.append(split, nullValue);
         }
         writer.close();  

     }  

     static class TeraRecordReader implements RecordReader<Text, Text> {  

         private LineRecordReader in;
         private LongWritable junk = new LongWritable();
         private Text line = new Text();
         private static int KEY_LENGTH = 10;  

         public TeraRecordReader(Configuration job, FileSplit split) throws IOException {
             in = new LineRecordReader(job, split);
         }  

         public void close() throws IOException {
             in.close();
         }  

         public Text createKey() {
             // TODO Auto-generated method stub
             return new Text();
         }  

         public Text createValue() {
             return new Text();
         }  

         public long getPos() throws IOException {
             // TODO Auto-generated method stub
             return in.getPos();
         }  

         public float getProgress() throws IOException {
             // TODO Auto-generated method stub
             return in.getProgress();
         }  

         public boolean next(Text arg0, Text arg1) throws IOException {
             if (in.next(junk, line)) {   //调用父类方法,将value值赋给key
                // if (line.getLength() < KEY_LENGTH) {
                     arg0.set(line);
                     arg1.clear();
 //                } else {
 //                    byte[] bytes = line.getBytes(); // 默认知道读取要比较值的前10个字节 作为key
 //                                                    // 后面的字节作为value;
 //                    arg0.set(bytes, 0, KEY_LENGTH);
 //                    arg1.set(bytes, KEY_LENGTH, line.getLength() - KEY_LENGTH);
 //                }
                 return true;
             } else {
                 return false;
             }
         }  

     }  

     @Override
     public InputSplit[] getSplits(JobConf conf, int splits) throws IOException {
         if (conf == lastConf) {
             return lastResult;
         }
         lastConf = conf;
         lastResult = super.getSplits(lastConf, splits);
         return lastResult;  

     }  

     public org.apache.hadoop.mapred.RecordReader<Text, Text> getRecordReader(InputSplit arg0, JobConf arg1,
             Reporter arg2) throws IOException {
         return new TeraRecordReader(arg1, (FileSplit) arg0);
     }  

 }  

转载自:http://www.open-open.com/lib/view/open1381329062408.html

mapreduce实现全局排序的更多相关文章

  1. MapReduce TotalOrderPartitioner 全局排序

    我们知道Mapreduce框架在feed数据给reducer之前会对map output key排序,这种排序机制保证了每一个reducer局部有序,hadoop 默认的partitioner是Has ...

  2. 三种方法实现Hadoop(MapReduce)全局排序(1)

    我们可能会有些需求要求MapReduce的输出全局有序,这里说的有序是指Key全局有序.但是我们知道,MapReduce默认只是保证同一个分区内的Key是有序的,但是不保证全局有序.基于此,本文提供三 ...

  3. Mapreduce的排序(全局排序、分区加排序、Combiner优化)

    一.MR排序的分类 1.部分排序:MR会根据自己输出记录的KV对数据进行排序,保证输出到每一个文件内存都是经过排序的: 2.全局排序: 3.辅助排序:再第一次排序后经过分区再排序一次: 4.二次排序: ...

  4. 大数据mapreduce全局排序top-N之python实现

    a.txt.b.txt文件如下: a.txt hadoop hadoop hadoop hadoop hadoop hadoop hadoop hadoop hadoop hadoop hadoop ...

  5. MapReduce怎么优雅地实现全局排序

    思考 想到全局排序,是否第一想到的是,从map端收集数据,shuffle到reduce来,设置一个reduce,再对reduce中的数据排序,显然这样和单机器并没有什么区别,要知道mapreduce框 ...

  6. Hadoop对文本文件的快速全局排序

    一.背景 Hadoop中实现了用于全局排序的InputSampler类和TotalOrderPartitioner类,调用示例是org.apache.hadoop.examples.Sort. 但是当 ...

  7. 一起学Hadoop——TotalOrderPartitioner类实现全局排序

    Hadoop排序,从大的范围来说有两种排序,一种是按照key排序,一种是按照value排序.如果按照value排序,只需在map函数中将key和value对调,然后在reduce函数中在对调回去.从小 ...

  8. MapReduce分区和排序

    一.排序 排序: 需求:根据用户每月使用的流量按照使用的流量多少排序 接口-->WritableCompareable 排序操作在hadoop中属于默认的行为.默认按照字典殊勋排序. 排序的分类 ...

  9. Hadoop学习笔记—11.MapReduce中的排序和分组

    一.写在之前的 1.1 回顾Map阶段四大步骤 首先,我们回顾一下在MapReduce中,排序和分组在哪里被执行: 从上图中可以清楚地看出,在Step1.4也就是第四步中,需要对不同分区中的数据进行排 ...

随机推荐

  1. oc 基础知识

    一.枚举 结构体 typedef enum{ sexMan, sexWoman }Sex;   tydedef struct{    int year;    int month;    int da ...

  2. Linux Terminal命令

    Linux Terminal命令 1.Ctrl + a 回到命令行の「行首/head」. 2.Ctrl + e 回到命令行の「行尾/tail」, ctrl + end. 3.Ctrl + w 後向/b ...

  3. 306573704 Char型和string型字符串比较整理(转)

    1.赋值 char赋值: char ch1[] = "give me"; char ch2[] = "a cup"; strcpy(ch1,ch2); cout ...

  4. free -m

    free -m total used free shared buffers cached Mem: 7760 1572   6187          0              9       ...

  5. ajax、json一些整理(3)

    写上面那些都是因为对ajax不熟悉 从w3c抄写JS原生ajax的东西补充一些基础 XMLHttpRequest 是 AJAX 的基础. XMLHttpRequest 对象 所有现代浏览器均支持 XM ...

  6. 配置 Struts2 Hello World

    http://javaweb.group.iteye.com/group/wiki/1505-struts2-under-helloworld---how-to-make-the-first-of-t ...

  7. Angular ng-repeat

    <tr ng-repeat="(key,item) in tableData"> <td class="check hidden-xs"> ...

  8. C++ dll调用-动态(显式)

    C++ dll调用-动态(显式) 废话不说上代码, dll 头文件 j_test.h #pragma once extern "C"_declspec(dllexport) voi ...

  9. 纯JavaScript实现一些小功能

    题目链接:http://wenku.baidu.com/link?url=7Gbarr5q9X6h1QFRVAsHmfPp1xXagG209mvrJqBogseb4WLeRqbVKwxQieoh8SL ...

  10. strstr函数与strcmp函数

    1.strstr函数主要完成在一个字串中寻找另外一个字串 函数实现工程如下:摘自http://baike.baidu.com/link?url=RwrzOxs0w68j02J2uQs5u1A56bEN ...