Hadoop入门经典:WordCount
转:http://blog.csdn.net/jediael_lu/article/details/38705371
以下程序在hadoop1.2.1上测试成功。
本例先将源代码呈现,然后详细说明执行步骤,最后对源代码及执行过程进行分析。
一、源代码
package org.jediael.hadoopdemo.wordcount; import java.io.IOException;
import java.util.StringTokenizer; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; public class WordCount { public static class WordCountMap extends
Mapper<LongWritable, Text, Text, IntWritable> { private final IntWritable one = new IntWritable(1);
private Text word = new Text(); public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
StringTokenizer token = new StringTokenizer(line);
while (token.hasMoreTokens()) {
word.set(token.nextToken());
context.write(word, one);
}
}
} public static class WordCountReduce extends
Reducer<Text, IntWritable, Text, IntWritable> { public void reduce(Text key, Iterable<IntWritable> values,
Context context) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
context.write(key, new IntWritable(sum));
}
} public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = new Job(conf);
job.setJarByClass(WordCount.class);
job.setJobName("wordcount"); job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class); job.setMapperClass(WordCountMap.class);
job.setReducerClass(WordCountReduce.class); job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class); FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1])); job.waitForCompletion(true);
}
}
二、执行程序
1、从eclipse从导出至wordcount.jar,并上传至hadoop服务器,本例中,将程序上传至/home/jediael/project。
2、安装hadoop伪分布模式,可参考Hadoop1.2.1伪分布模式安装指南,本实例将运行在hadoop的伪公布环境中。
3、在HDFS中创建目录wcinput,用作输入目录,并将需要分析的文件复制到目录下。
[root@jediael conf]# hadoop fs -mkdir wcinput
[root@jediael conf]# hadoop fs -copyFromLocal * wcinput
[root@jediael conf]# hadoop fs -ls wcinput
Found 26 items
-rw-r--r-- 1 root supergroup 1524 2014-08-20 12:29 /user/root/wcinput/automaton-urlfilter.txt
-rw-r--r-- 1 root supergroup 1311 2014-08-20 12:29 /user/root/wcinput/configuration.xsl
-rw-r--r-- 1 root supergroup 131090 2014-08-20 12:29 /user/root/wcinput/domain-suffixes.xml
-rw-r--r-- 1 root supergroup 4649 2014-08-20 12:29 /user/root/wcinput/domain-suffixes.xsd
-rw-r--r-- 1 root supergroup 824 2014-08-20 12:29 /user/root/wcinput/domain-urlfilter.txt
-rw-r--r-- 1 root supergroup 3368 2014-08-20 12:29 /user/root/wcinput/gora-accumulo-mapping.xml
-rw-r--r-- 1 root supergroup 3279 2014-08-20 12:29 /user/root/wcinput/gora-cassandra-mapping.xml
-rw-r--r-- 1 root supergroup 3447 2014-08-20 12:29 /user/root/wcinput/gora-hbase-mapping.xml
-rw-r--r-- 1 root supergroup 2677 2014-08-20 12:29 /user/root/wcinput/gora-sql-mapping.xml
-rw-r--r-- 1 root supergroup 2993 2014-08-20 12:29 /user/root/wcinput/gora.properties
-rw-r--r-- 1 root supergroup 983 2014-08-20 12:29 /user/root/wcinput/hbase-site.xml
-rw-r--r-- 1 root supergroup 3096 2014-08-20 12:29 /user/root/wcinput/httpclient-auth.xml
-rw-r--r-- 1 root supergroup 3948 2014-08-20 12:29 /user/root/wcinput/log4j.properties
-rw-r--r-- 1 root supergroup 511 2014-08-20 12:29 /user/root/wcinput/nutch-conf.xsl
-rw-r--r-- 1 root supergroup 42610 2014-08-20 12:29 /user/root/wcinput/nutch-default.xml
-rw-r--r-- 1 root supergroup 753 2014-08-20 12:29 /user/root/wcinput/nutch-site.xml
-rw-r--r-- 1 root supergroup 347 2014-08-20 12:29 /user/root/wcinput/parse-plugins.dtd
-rw-r--r-- 1 root supergroup 3016 2014-08-20 12:29 /user/root/wcinput/parse-plugins.xml
-rw-r--r-- 1 root supergroup 857 2014-08-20 12:29 /user/root/wcinput/prefix-urlfilter.txt
-rw-r--r-- 1 root supergroup 2484 2014-08-20 12:29 /user/root/wcinput/regex-normalize.xml
-rw-r--r-- 1 root supergroup 1736 2014-08-20 12:29 /user/root/wcinput/regex-urlfilter.txt
-rw-r--r-- 1 root supergroup 18969 2014-08-20 12:29 /user/root/wcinput/schema-solr4.xml
-rw-r--r-- 1 root supergroup 6020 2014-08-20 12:29 /user/root/wcinput/schema.xml
-rw-r--r-- 1 root supergroup 1766 2014-08-20 12:29 /user/root/wcinput/solrindex-mapping.xml
-rw-r--r-- 1 root supergroup 1044 2014-08-20 12:29 /user/root/wcinput/subcollections.xml
-rw-r--r-- 1 root supergroup 1411 2014-08-20 12:29 /user/root/wcinput/suffix-urlfilter.txt
4、运行程序
[root@jediael project]# hadoop org.jediael.hadoopdemo.wordcount.WordCount wcinput wcoutput3
14/08/20 12:50:25 WARN mapred.JobClient: Use GenericOptionsParser for parsing the arguments. Applications should implement Tool for the same.
14/08/20 12:50:26 INFO input.FileInputFormat: Total input paths to process : 26
14/08/20 12:50:26 INFO util.NativeCodeLoader: Loaded the native-hadoop library
14/08/20 12:50:26 WARN snappy.LoadSnappy: Snappy native library not loaded
14/08/20 12:50:26 INFO mapred.JobClient: Running job: job_201408191134_0005
14/08/20 12:50:27 INFO mapred.JobClient: map 0% reduce 0%
14/08/20 12:50:38 INFO mapred.JobClient: map 3% reduce 0%
14/08/20 12:50:39 INFO mapred.JobClient: map 7% reduce 0%
14/08/20 12:50:50 INFO mapred.JobClient: map 15% reduce 0%
14/08/20 12:50:57 INFO mapred.JobClient: map 19% reduce 0%
14/08/20 12:50:58 INFO mapred.JobClient: map 23% reduce 0%
14/08/20 12:51:00 INFO mapred.JobClient: map 23% reduce 5%
14/08/20 12:51:04 INFO mapred.JobClient: map 30% reduce 5%
14/08/20 12:51:06 INFO mapred.JobClient: map 30% reduce 10%
14/08/20 12:51:11 INFO mapred.JobClient: map 38% reduce 10%
14/08/20 12:51:16 INFO mapred.JobClient: map 38% reduce 11%
14/08/20 12:51:18 INFO mapred.JobClient: map 46% reduce 11%
14/08/20 12:51:19 INFO mapred.JobClient: map 46% reduce 12%
14/08/20 12:51:22 INFO mapred.JobClient: map 46% reduce 15%
14/08/20 12:51:25 INFO mapred.JobClient: map 53% reduce 15%
14/08/20 12:51:31 INFO mapred.JobClient: map 53% reduce 17%
14/08/20 12:51:32 INFO mapred.JobClient: map 61% reduce 17%
14/08/20 12:51:39 INFO mapred.JobClient: map 69% reduce 17%
14/08/20 12:51:40 INFO mapred.JobClient: map 69% reduce 20%
14/08/20 12:51:45 INFO mapred.JobClient: map 73% reduce 20%
14/08/20 12:51:46 INFO mapred.JobClient: map 76% reduce 23%
14/08/20 12:51:52 INFO mapred.JobClient: map 80% reduce 23%
14/08/20 12:51:53 INFO mapred.JobClient: map 84% reduce 23%
14/08/20 12:51:55 INFO mapred.JobClient: map 84% reduce 25%
14/08/20 12:51:59 INFO mapred.JobClient: map 88% reduce 25%
14/08/20 12:52:00 INFO mapred.JobClient: map 92% reduce 25%
14/08/20 12:52:02 INFO mapred.JobClient: map 92% reduce 29%
14/08/20 12:52:06 INFO mapred.JobClient: map 96% reduce 29%
14/08/20 12:52:07 INFO mapred.JobClient: map 100% reduce 29%
14/08/20 12:52:11 INFO mapred.JobClient: map 100% reduce 30%
14/08/20 12:52:15 INFO mapred.JobClient: map 100% reduce 100%
14/08/20 12:52:17 INFO mapred.JobClient: Job complete: job_201408191134_0005
14/08/20 12:52:18 INFO mapred.JobClient: Counters: 29
14/08/20 12:52:18 INFO mapred.JobClient: Job Counters
14/08/20 12:52:18 INFO mapred.JobClient: Launched reduce tasks=1
14/08/20 12:52:18 INFO mapred.JobClient: SLOTS_MILLIS_MAPS=192038
14/08/20 12:52:18 INFO mapred.JobClient: Total time spent by all reduces waiting after reserving slots (ms)=0
14/08/20 12:52:18 INFO mapred.JobClient: Total time spent by all maps waiting after reserving slots (ms)=0
14/08/20 12:52:18 INFO mapred.JobClient: Launched map tasks=26
14/08/20 12:52:18 INFO mapred.JobClient: Data-local map tasks=26
14/08/20 12:52:18 INFO mapred.JobClient: SLOTS_MILLIS_REDUCES=95814
14/08/20 12:52:18 INFO mapred.JobClient: File Output Format Counters
14/08/20 12:52:18 INFO mapred.JobClient: Bytes Written=123950
14/08/20 12:52:18 INFO mapred.JobClient: FileSystemCounters
14/08/20 12:52:18 INFO mapred.JobClient: FILE_BYTES_READ=352500
14/08/20 12:52:18 INFO mapred.JobClient: HDFS_BYTES_READ=247920
14/08/20 12:52:18 INFO mapred.JobClient: FILE_BYTES_WRITTEN=2177502
14/08/20 12:52:18 INFO mapred.JobClient: HDFS_BYTES_WRITTEN=123950
14/08/20 12:52:18 INFO mapred.JobClient: File Input Format Counters
14/08/20 12:52:18 INFO mapred.JobClient: Bytes Read=244713
14/08/20 12:52:18 INFO mapred.JobClient: Map-Reduce Framework
14/08/20 12:52:18 INFO mapred.JobClient: Map output materialized bytes=352650
14/08/20 12:52:18 INFO mapred.JobClient: Map input records=7403
14/08/20 12:52:18 INFO mapred.JobClient: Reduce shuffle bytes=352650
14/08/20 12:52:18 INFO mapred.JobClient: Spilled Records=45210
14/08/20 12:52:18 INFO mapred.JobClient: Map output bytes=307281
14/08/20 12:52:18 INFO mapred.JobClient: Total committed heap usage (bytes)=3398606848
14/08/20 12:52:18 INFO mapred.JobClient: CPU time spent (ms)=14400
14/08/20 12:52:18 INFO mapred.JobClient: Combine input records=0
14/08/20 12:52:18 INFO mapred.JobClient: SPLIT_RAW_BYTES=3207
14/08/20 12:52:18 INFO mapred.JobClient: Reduce input records=22605
14/08/20 12:52:18 INFO mapred.JobClient: Reduce input groups=6749
14/08/20 12:52:18 INFO mapred.JobClient: Combine output records=0
14/08/20 12:52:18 INFO mapred.JobClient: Physical memory (bytes) snapshot=4799041536
14/08/20 12:52:18 INFO mapred.JobClient: Reduce output records=6749
14/08/20 12:52:18 INFO mapred.JobClient: Virtual memory (bytes) snapshot=19545337856
14/08/20 12:52:18 INFO mapred.JobClient: Map output records=22605
5、查看结果
root@jediael project]# hadoop fs -ls wcoutput3
Found 3 items
-rw-r--r-- 1 root supergroup 0 2014-08-20 12:52 /user/root/wcoutput3/_SUCCESS
drwxr-xr-x - root supergroup 0 2014-08-20 12:50 /user/root/wcoutput3/_logs
-rw-r--r-- 1 root supergroup 123950 2014-08-20 12:52 /user/root/wcoutput3/part-r-00000
[root@jediael project]# hadoop fs -cat wcoutput3/part-r-00000
!! 2
!ci.*.*.us 1
!co.*.*.us 1
!town.*.*.us 1
"AS 22
"Accept" 1
"Accept-Language" 1
"License"); 22
"NOW" 1
"WiFi" 1
"Z" 1
"all" 1
"content" 1
"delete 1
"delimiter" 1
三、程序分析
Hadoop入门经典:WordCount的更多相关文章
- Hadoop入门经典:WordCount 分类: A1_HADOOP 2014-08-20 14:43 2514人阅读 评论(0) 收藏
以下程序在hadoop1.2.1上测试成功. 本例先将源代码呈现,然后详细说明执行步骤,最后对源代码及执行过程进行分析. 一.源代码 package org.jediael.hadoopdemo.wo ...
- Hadoop入门程序WordCount的执行过程
首先编写WordCount.java源文件,分别通过map和reduce方法统计文本中每个单词出现的次数,然后按照字母的顺序排列输出, Map过程首先是多个map并行提取多个句子里面的单词然后分别列出 ...
- Hadoop入门实例——WordCount统计单词
首先要说明的是运行Hadoop需要jdk1.6或以上版本,如果你还没有搭建好Hadoop集群,请参考我的另一篇文章: Linux环境搭建Hadoop伪分布模式 马上进入正题. 1.启动Hadoop集群 ...
- 初识Hadoop入门介绍
初识hadoop入门介绍 Hadoop一直是我想学习的技术,正巧最近项目组要做电子商城,我就开始研究Hadoop,虽然最后鉴定Hadoop不适用我们的项目,但是我会继续研究下去,技多不压身. < ...
- hadoop 入门实例【转】
原文链接:http://www.cnblogs.com/xia520pi/archive/2012/06/04/2534533.html 1.数据去重 "数据去重"主要是为了掌握 ...
- 一.hadoop入门须知
目录: 1.hadoop入门须知 2.hadoop环境搭建 3.hadoop mapreduce之WordCount例子 4.idea本地调试hadoop程序 5.hadoop 从mysql中读取数据 ...
- Hadoop Mapreduce 案例 wordcount+统计手机流量使用情况
mapreduce设计思想 概念:它是一个分布式并行计算的应用框架它提供相应简单的api模型,我们只需按照这些模型规则编写程序,即可实现"分布式并行计算"的功能. 案例一:word ...
- 大数据初级笔记二:Hadoop入门之Hadoop集群搭建
Hadoop集群搭建 把环境全部准备好,包括编程环境. JDK安装 版本要求: 强烈建议使用64位的JDK版本,这样的优势在于JVM的能够访问到的最大内存就不受限制,基于后期可能会学习到Spark技术 ...
- 转 Kafka入门经典教程
Kafka入门经典教程 http://www.aboutyun.com/thread-12882-1-1.html 问题导读 1.Kafka独特设计在什么地方?2.Kafka如何搭建及创建topic. ...
随机推荐
- 一个完整的编译器前端-A.1 源语言
这个语言的一个程序由一个块组成,该块中包含可选的声明和语句.语法符号basic表示基本类型. program –> block block –> { decls stmts } dec ...
- derby数据库ql语法
[数据库知识] 主键.唯一键包含索引 主键包含唯一键.索引.非空 唯一键包含索引,可空或非空 数据库需要与执行服务的在同个目录下 唯一键 create table app.tyu ( primaryk ...
- java连接各种数据库代码大全
1.Oracle8/8i/9i数据库(thin模式)Class.forName("oracle.jdbc.driver.OracleDriver").newInstance();S ...
- Change ICON of MFC Application and Dialog
Change ICON of MFC Application and Dialoghttp://www.codeproject.com/Tips/406870/Change-ICON-of-MFC-A ...
- C#实现AES加解密方法
using System; using System.Collections.Generic; using System.Text; using System.Security.Cryptograph ...
- IOS: 使用imageIO获取和修改图片的exif信息
使用imageIO获取和修改图片的exif信息 一幅图片除了包含我们能看见的像素信息,背后还包含了拍摄时间,光圈大小,曝光等信息.UIImage类将这些细节信息都隐藏了起来,只提供我们关心的图片尺寸, ...
- java工具类包
Hutool 提供丰富的java方法,其maven引用 <dependency> <groupId>com.xiaoleilu</groupId> <arti ...
- Bookshelf 2
Bookshelf 2 Time Limit:1000MS Memory Limit:65536KB 64bit IO Format:%I64d & %I64u Submit ...
- CloudPlatform和CloudStack的关系
The Scalr team is at the CloudStack Collab Conf, and this post summarizes a few things we learned. C ...
- Poj 3233 Matrix Power Series(矩阵二分快速幂)
题目链接:http://poj.org/problem?id=3233 解题报告:输入一个边长为n的矩阵A,然后输入一个k,要你求A + A^2 + A^3 + A^4 + A^5.......A^k ...