mapreduce 依赖组合
mport java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob;
import org.apache.hadoop.mapreduce.lib.jobcontrol.JobControl;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
public class Driver {
public static class TokenizerMapper extends
Mapper<Object, Text, Text, IntWritable> {
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
StringTokenizer itr = new StringTokenizer(value.toString());
while (itr.hasMoreTokens()) {
word.set(itr.nextToken());
context.write(word, one);
}
}
}
public static class IntSumReducer extends
Reducer<Text, IntWritable, Text, IntWritable> {
private IntWritable result = new IntWritable();
public void reduce(Text key, Iterable<IntWritable> values,
Context context) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
result.set(sum);
context.write(key, result);
}
}
public static class DependenceMapper extends
Mapper<Object, Text, Text, Text> {
private Text word = new Text();
public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
String []sep=value.toString().split("\t");
word.set(sep[1]+"\t"+sep[0]);
System.out.println(value.toString());
context.write(word,new Text(""));
}
}
public static class DependenceReducer extends
Reducer<Text,Text,Text,Text> {
public void reduce(Text key, Iterable<Text> values,
Context context) throws IOException, InterruptedException {
String[] sep = key.toString().split("\t");
System.out.println( sep[0]+"++++++++="+ sep[1]);
context.write(key,new Text(""));
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args)
.getRemainingArgs();
if (otherArgs.length < 2) {
System.err.println("Usage: wordcount <in> <out>");
System.exit(2);
}
Job job = new Job(conf, "word count");
//加入控制容器
ControlledJob ctrljob1=new ControlledJob(conf);
ctrljob1.setJob(job);
job.setJarByClass(Driver.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(IntSumReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
// job.waitForCompletion(true);
Configuration conf2 = new Configuration();
Job job2 = new Job(conf2, "word count1");
ControlledJob ctrljob2=new ControlledJob(conf);
ctrljob2.setJob(job2);
ctrljob2.addDependingJob(ctrljob1);
job2.setJarByClass(Driver.class);
job2.setMapperClass(DependenceMapper.class);
job2.setReducerClass(DependenceReducer.class);
job2.setOutputKeyClass(Text.class);
job2.setOutputValueClass(Text.class);
FileInputFormat.addInputPath(job2, new Path(otherArgs[1]));
FileOutputFormat.setOutputPath(job2, new Path(otherArgs[2]));
// job2.waitForCompletion(true);
JobControl jobCtrl=new JobControl("myctrl");
//添加到总的JobControl里,进行控制
jobCtrl.addJob(ctrljob1);
jobCtrl.addJob(ctrljob2);
jobCtrl.run();
}
}
mapreduce 依赖组合的更多相关文章
- Python 入门 之 类的三大关系(依赖 / 组合/ 继承关系)
Python 入门 之 类的三大关系(依赖 / 组合/ 继承关系) 在面向对象的中,类与类之间存在三种关系:依赖关系.组合关系.继承关系. 1.依赖关系:将一个类的类名或对象当做参数传递给另一个函数被 ...
- mapreduce 顺序组合
import java.io.IOException;import java.util.StringTokenizer; import org.apache.hadoop.conf.Configura ...
- 8、Situation-Dependent Combination of Long-Term and Session-Based Preferences in Group Recommendations: An Experimental Analysis ----组推荐中基于长期和会话偏好的情景依赖组合
一.摘要: 背景:会话组推荐系统的一个主要挑战是如何适当地利用群组成员之间的交互引起用户偏好,这可能会偏离用户的长期偏好.长期偏好和群组诱导的偏好之间的相对重要性应该根据具体的群组设置而变化. 本文: ...
- Hadoop官方文档翻译——MapReduce Tutorial
MapReduce Tutorial(个人指导) Purpose(目的) Prerequisites(必备条件) Overview(综述) Inputs and Outputs(输入输出) MapRe ...
- [大牛翻译系列]Hadoop(5)MapReduce 排序:次排序(Secondary sort)
4.2 排序(SORT) 在MapReduce中,排序的目的有两个: MapReduce可以通过排序将Map输出的键分组.然后每组键调用一次reduce. 在某些需要排序的特定场景中,用户可以将作业( ...
- 【原创】MapReduce编程系列之二元排序
普通排序实现 普通排序的实现利用了按姓名的排序,调用了默认的对key的HashPartition函数来实现数据的分组.partition操作之后写入磁盘时会对数据进行排序操作(对一个分区内的数据作排序 ...
- Mapreduce执行过程分析(基于Hadoop2.4)——(一)
1 概述 该瞅瞅MapReduce的内部运行原理了,以前只知道个皮毛,再不搞搞,不然怎么死的都不晓得.下文会以2.4版本中的WordCount这个经典例子作为分析的切入点,一步步来看里面到底是个什么情 ...
- 大数据技术 —— MapReduce 简介
本文为senlie原创,转载请保留此地址:http://www.cnblogs.com/senlie/ 1.概要很多计算在概念上很直观,但由于输入数据很大,为了能在合理的时间内完成,这些计算必须分布在 ...
- Mapreduce运行过程分析(基于Hadoop2.4)——(一)
1 概述 该瞅瞅MapReduce的内部执行原理了,曾经仅仅知道个皮毛,再不搞搞,不然怎么死的都不晓得.下文会以2.4版本号中的WordCount这个经典样例作为分析的切入点.一步步来看里面究竟是个什 ...
随机推荐
- C语言 - 预编译
1.#ifdef 实现 与 或#if (defined SIMULATION) && (defined _FMM_LOG)#endif #if (!defined A) || (!de ...
- kissy小记
<script> KISSY.add('demo',function(S ,require, exports, module){ var Node = require('node'); v ...
- kali linux 更新软件源,安装中文输入法,修复Linux与windows引导菜单解决windows引导丢失
1. 更新软件源打开sources.list文件,进行添加更新源:leafpad /etc/apt/sources.list 2. 添加软件源#官方源 deb http://http.kali.org ...
- lucene 简单搜索步骤
1.创建IndexReader实例: Directory dir = FSDirectory.open(new File(indexDir)); IndexReader reader = Direct ...
- sql随机
想从MySQL数据库中随机取一条或者N条记录时,最好把RAND()生成随机数放在JOIN子查询中以提高效率. SELECT id FROM table ORDER BY RAND() LIMIT n; ...
- WebDriver获取table的内容(通过动态获取Table单元格的TagName对其innerHTML值进行获取)
import java.util.ArrayList;import java.util.Iterator;import java.util.LinkedHashMap;import java.util ...
- scanf函数与scanf_s函数
ANSI C中没有scanf_s(),只有scanf(),scanf()在读取时不检查边界,所以可能会造成内存泄露.所以vc++2005/2008中提供了scanf_s(),在最新的VS2013中也提 ...
- Varnish && Varnish Cache
1 1 1 Varnish https://www.varnish-cache.org/intro/index.html#intro Introduction to Varnish The basic ...
- <context:component-scan>
首先看配置文件: <?xml version="1.0" encoding="UTF-8"?> <beans xmlns="http ...
- hdu_5620_KK's Steel(水题)
题目连接:http://acm.hdu.edu.cn/showproblem.php?pid=5620 题意:给你一个n长的钢管,要分的尽可能多,且任意三条不能构成三角形 题解:看hint就知道用递推 ...