原生MapReduce开发样例

一、需求

data:  将相同名字合并为一个，并计算出平均数

tom
小明
jerry
2哈
tom
tom
小明

二、编码

1.导入jar包

2.编码

2.1Map编写

package com.wzy.studentscore;

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

/**
* @author：吴兆跃
* @version 创建时间：2018年6月5日 下午5:58:55
* 类说明：
*/
public class ScoreMap extends Mapper<LongWritable, Text, Text, IntWritable>{
    @Override
    public void map(LongWritable key, Text value, Context context)
            throws IOException,InterruptedException{

        String line = value.toString(); //一行的数据
        StringTokenizer tokenizerArticle = new StringTokenizer(line, "\n");

        System.out.println("key: "+key);
        System.out.println("value-line: "+line);
        System.out.println("count: "+tokenizerArticle.countTokens());

        while(tokenizerArticle.hasMoreTokens()){
            String token = tokenizerArticle.nextToken();
            System.out.println("token: "+token);

            StringTokenizer tokenizerLine = new StringTokenizer(token);
            String strName = tokenizerLine.nextToken(); // 得到name
            String strScore = tokenizerLine.nextToken(); // 得到分数

            Text name = new Text(strName);
            int scoreInt = Integer.parseInt(strScore);
            context.write(name, new IntWritable(scoreInt));

        }
        System.out.println("context: "+context.toString());
    }

}

2.2Reduce编写

package com.wzy.studentscore;

import java.io.IOException;
import java.util.Iterator;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

/**
* @author：吴兆跃
* @version 创建时间：2018年6月5日 下午6:50:28
* 类说明：
*/
public class ScoreReduce extends Reducer<Text, IntWritable, Text, IntWritable>{

    @Override
    public void reduce(Text key, Iterable<IntWritable> values, Context context)
        throws IOException, InterruptedException{

        int sum = ;
        int count = ;
        Iterator<IntWritable> iterator = values.iterator();
        while(iterator.hasNext()){
            sum += iterator.next().get(); //求和
            count++;
        }
        int average = (int)sum / count;  //求平均数
        context.write(key, new IntWritable(average));
    }

}

2.3运行类编写

package com.wzy.studentscore;

import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

/**
* @author：吴兆跃
* @version 创建时间：2018年6月5日 下午6:59:29
* 类说明：
*/
public class ScoreProcess extends Configured implements Tool{

    public static void main(String[] args) throws Exception {
        int ret = ToolRunner.run(new ScoreProcess(), new String[]{"input","output"});
        System.exit(ret);
    }

    @Override
    public int run(String[] args) throws Exception {
        Job job = new Job(getConf());
        job.setJarByClass(ScoreProcess.class);
        job.setJobName("score_process");

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        job.setMapperClass(ScoreMap.class);
        job.setCombinerClass(ScoreReduce.class);
        job.setReducerClass(ScoreReduce.class);

        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);

        FileInputFormat.setInputPaths(job, new Path(args[]));
        FileOutputFormat.setOutputPath(job, new Path(args[]));

        boolean success = job.waitForCompletion(true);
        return success ?  : ;
    }

}

3.打包

三、调试

1. java本地运行

root@master:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess# ls
input  part  scoreProcess.jar
root@master:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess# java -jar scoreProcess.jar 
Jun ,  :: AM org.apache.hadoop.util.NativeCodeLoader <clinit>
WARNING: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Jun ,  :: AM org.apache.hadoop.mapreduce.lib.input.FileInputFormat listStatus
INFO: Total input paths to process :
Jun ,  :: AM org.apache.hadoop.io.compress.snappy.LoadSnappy <clinit>
WARNING: Snappy native library not loaded
Jun ,  :: AM org.apache.hadoop.mapred.JobClient monitorAndPrintJob
INFO: Running job: job_local1903623691_0001
Jun ,  :: AM org.apache.hadoop.mapred.LocalJobRunner$Job$MapTaskRunnable run
INFO: Starting task: attempt_local1903623691_0001_m_000000_0
Jun ,  :: AM org.apache.hadoop.mapred.LocalJobRunner$Job run
INFO: Waiting for map tasks
Jun ,  :: AM org.apache.hadoop.util.ProcessTree isSetsidSupported
INFO: setsid exited with exit code
Jun ,  :: AM org.apache.hadoop.mapred.Task initialize
INFO:  Using ResourceCalculatorPlugin : org.apache.hadoop.util.LinuxResourceCalculatorPlugin@5ddf714a
Jun ,  :: AM org.apache.hadoop.mapred.MapTask runNewMapper
INFO: Processing split: file:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess/input/data:+
Jun ,  :: AM org.apache.hadoop.mapred.MapTask$MapOutputBuffer <init>
INFO: io.sort.mb =
Jun ,  :: AM org.apache.hadoop.mapred.MapTask$MapOutputBuffer <init>
INFO: data buffer = /
Jun ,  :: AM org.apache.hadoop.mapred.MapTask$MapOutputBuffer <init>
INFO: record buffer = /
key:
value-line: tom
count:
token: tom
context: org.apache.hadoop.mapreduce.Mapper$Context@41b9bff9
key:
value-line: 小明
count:
token: 小明
context: org.apache.hadoop.mapreduce.Mapper$Context@41b9bff9
key:
value-line: jerry
count:
token: jerry
context: org.apache.hadoop.mapreduce.Mapper$Context@41b9bff9
key:
value-line: 哈2
count:
token: 哈2
context: org.apache.hadoop.mapreduce.Mapper$Context@41b9bff9
key:
value-line: tom
count:
token: tom
context: org.apache.hadoop.mapreduce.Mapper$Context@41b9bff9
key:
value-line: tom
count:
token: tom
context: org.apache.hadoop.mapreduce.Mapper$Context@41b9bff9
key:
value-line: 小明
count:
token: 小明
context: org.apache.hadoop.mapreduce.Mapper$Context@41b9bff9

root@master:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess# ls
input  output  part  scoreProcess.jar
root@master:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess# cd output/
root@master:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess/output# ls
part-r-  _SUCCESS
root@master:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess/output# cat part-r-
jerry
tom
哈2
小明    

2. 在hadoop hdfs上运行

2.1 data文件上传到hdfs

root@master:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess# ../../bin/hadoop fs -mkdir /user
root@master:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess# ../../bin/hadoop fs -mkdir /user/root
root@master:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess# ../../bin/hadoop fs -mkdir /user/root/input
root@master:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess# ../../bin/hadoop fs -put input/data /user/root/input
root@master:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess# ../../bin/hadoop fs -ls /user/root/input
Found  items
-rw-r--r--    root supergroup          -- : /user/root/input/data

2.2 运行

root@master:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess# ../../bin/hadoop jar scoreProcess.jar
// :: INFO input.FileInputFormat: Total input paths to process :
// :: INFO mapred.JobClient: Running job: job_201806060358_0002
// :: INFO mapred.JobClient:  map % reduce %
// :: INFO mapred.JobClient:  map % reduce %
// :: INFO mapred.JobClient:  map % reduce %
// :: INFO mapred.JobClient: Job complete: job_201806060358_0002
// :: INFO mapred.JobClient: Counters:
// :: INFO mapred.JobClient:   Map-Reduce Framework
// :: INFO mapred.JobClient:     Combine output records=
// :: INFO mapred.JobClient:     Spilled Records=
// :: INFO mapred.JobClient:     Reduce input records=
// :: INFO mapred.JobClient:     Reduce output records=
// :: INFO mapred.JobClient:     Map input records=
// :: INFO mapred.JobClient:     Map output records=
// :: INFO mapred.JobClient:     Map output bytes=
// :: INFO mapred.JobClient:     Reduce shuffle bytes=
// :: INFO mapred.JobClient:     Combine input records=
// :: INFO mapred.JobClient:     Reduce input groups=
// :: INFO mapred.JobClient:   FileSystemCounters
// :: INFO mapred.JobClient:     HDFS_BYTES_READ=
// :: INFO mapred.JobClient:     FILE_BYTES_WRITTEN=
// :: INFO mapred.JobClient:     FILE_BYTES_READ=
// :: INFO mapred.JobClient:     HDFS_BYTES_WRITTEN=
// :: INFO mapred.JobClient:   Job Counters
// :: INFO mapred.JobClient:     Launched map tasks=
// :: INFO mapred.JobClient:     Launched reduce tasks=
// :: INFO mapred.JobClient:     Data-local map tasks=

2.3 查看结果

root@master:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess# ../../bin/hadoop fs -ls /user/root/output/
Found  items
drwxr-xr-x   - root supergroup           -- : /user/root/output/_logs
-rw-r--r--    root supergroup          -- : /user/root/output/part-r-00000

root@master:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess# ../../bin/hadoop fs -get /user/root/output/part-r- part
root@master:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess# ls
input  output  part  scoreProcess.jar
root@master:/home/wzy/software/hadoop-0.20./testfile/ScoreProcess# cat part
jerry
tom
2哈
小明