【Hadoop】Hadoop MR 如何实现倒排索引算法?
1、概念、方案
2、代码示例
InverseIndexOne
package com.ares.hadoop.mr.inverseindex; import java.io.IOException; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.log4j.Logger; public class InverseIndexOne extends Configured implements Tool { private static final Logger LOGGER = Logger.getLogger(InverseIndexOne.class);
enum Counter {
LINESKIP
} public static class InverseIndexOneMapper
extends Mapper<LongWritable, Text, Text, LongWritable> { private String line;
private final static char separatorA = ' ';
private final static char separatorB = '-';
private String fileName; private Text text = new Text();
private final static LongWritable ONE = new LongWritable(1L); @Override
protected void map(LongWritable key, Text value,
Mapper<LongWritable, Text, Text, LongWritable>.Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
//super.map(key, value, context);
try {
line = value.toString();
String[] fields = StringUtils.split(line, separatorA); FileSplit fileSplit = (FileSplit) context.getInputSplit();
fileName = fileSplit.getPath().getName(); for (int i = ; i < fields.length; i++) {
text.set(fields[i] + separatorB + fileName);
context.write(text, ONE);
}
} catch (Exception e) {
// TODO: handle exception
LOGGER.error(e);
System.out.println(e);
context.getCounter(Counter.LINESKIP).increment();
return;
}
}
} public static class InverseIndexOneReducer
extends Reducer<Text, LongWritable, Text, LongWritable> {
private LongWritable result = new LongWritable(); @Override
protected void reduce(Text key, Iterable<LongWritable> values,
Reducer<Text, LongWritable, Text, LongWritable>.Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
//super.reduce(arg0, arg1, arg2);
long count = ;
for (LongWritable value : values) {
count += value.get();
}
result.set(count);
context.write(key, result);
}
} @Override
public int run(String[] args) throws Exception {
// TODO Auto-generated method stub
//return 0;
String errMsg = "InverseIndexOne: TEST STARTED...";
LOGGER.debug(errMsg);
System.out.println(errMsg); Configuration conf = new Configuration();
//FOR Eclipse JVM Debug
//conf.set("mapreduce.job.jar", "flowsum.jar");
Job job = Job.getInstance(conf); // JOB NAME
job.setJobName("InverseIndexOne"); // JOB MAPPER & REDUCER
job.setJarByClass(InverseIndexOne.class);
job.setMapperClass(InverseIndexOneMapper.class);
job.setReducerClass(InverseIndexOneReducer.class); // JOB PARTITION
//job.setPartitionerClass(FlowGroupPartition.class); // JOB REDUCE TASK NUMBER
//job.setNumReduceTasks(5); // MAP & REDUCE
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
// MAP
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(LongWritable.class); // JOB INPUT & OUTPUT PATH
//FileInputFormat.addInputPath(job, new Path(args[0]));
FileInputFormat.setInputPaths(job, args[]);
Path output = new Path(args[]);
// FileSystem fs = FileSystem.get(conf);
// if (fs.exists(output)) {
// fs.delete(output, true);
// }
FileOutputFormat.setOutputPath(job, output); // VERBOSE OUTPUT
if (job.waitForCompletion(true)) {
errMsg = "InverseIndexOne: TEST SUCCESSFULLY...";
LOGGER.debug(errMsg);
System.out.println(errMsg);
return ;
} else {
errMsg = "InverseIndexOne: TEST FAILED...";
LOGGER.debug(errMsg);
System.out.println(errMsg);
return ;
}
} public static void main(String[] args) throws Exception {
if (args.length != ) {
String errMsg = "InverseIndexOne: ARGUMENTS ERROR";
LOGGER.error(errMsg);
System.out.println(errMsg);
System.exit(-);
} int result = ToolRunner.run(new Configuration(), new InverseIndexOne(), args);
System.exit(result);
}
}
InverseIndexTwo
package com.ares.hadoop.mr.inverseindex; import java.io.IOException; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.log4j.Logger; public class InverseIndexTwo extends Configured implements Tool{
private static final Logger LOGGER = Logger.getLogger(InverseIndexOne.class);
enum Counter {
LINESKIP
} public static class InverseIndexTwoMapper extends
Mapper<LongWritable, Text, Text, Text> { private String line;
private final static char separatorA = '\t';
private final static char separatorB = '-'; private Text textKey = new Text();
private Text textValue = new Text(); @Override
protected void map(LongWritable key, Text value,
Mapper<LongWritable, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
//super.map(key, value, context);
try {
line = value.toString();
String[] fields = StringUtils.split(line, separatorA);
String[] wordAndfileName = StringUtils.split(fields[], separatorB);
long count = Long.parseLong(fields[]);
String word = wordAndfileName[];
String fileName = wordAndfileName[]; textKey.set(word);
textValue.set(fileName + separatorB + count);
context.write(textKey, textValue);
} catch (Exception e) {
// TODO: handle exception
LOGGER.error(e);
System.out.println(e);
context.getCounter(Counter.LINESKIP).increment();
return;
}
}
} public static class InverseIndexTwoReducer extends
Reducer<Text, Text, Text, Text> { private Text textValue = new Text(); @Override
protected void reduce(Text key, Iterable<Text> values,
Reducer<Text, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
//super.reduce(arg0, arg1, arg2);
StringBuilder index = new StringBuilder("");
// for (Text text : values) {
// if (condition) {
//
// }
// index.append(text.toString() + separatorA);
// }
String separatorA = "";
for (Text text : values) {
index.append(separatorA + text.toString());
separatorA = ",";
}
textValue.set(index.toString());
context.write(key, textValue);
}
} @Override
public int run(String[] args) throws Exception {
// TODO Auto-generated method stub
//return 0;
String errMsg = "InverseIndexTwo: TEST STARTED...";
LOGGER.debug(errMsg);
System.out.println(errMsg); Configuration conf = new Configuration();
//FOR Eclipse JVM Debug
//conf.set("mapreduce.job.jar", "flowsum.jar");
Job job = Job.getInstance(conf); // JOB NAME
job.setJobName("InverseIndexTwo"); // JOB MAPPER & REDUCER
job.setJarByClass(InverseIndexTwo.class);
job.setMapperClass(InverseIndexTwoMapper.class);
job.setReducerClass(InverseIndexTwoReducer.class); // JOB PARTITION
//job.setPartitionerClass(FlowGroupPartition.class); // JOB REDUCE TASK NUMBER
//job.setNumReduceTasks(5); // MAP & REDUCE
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
// MAP
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class); // JOB INPUT & OUTPUT PATH
//FileInputFormat.addInputPath(job, new Path(args[0]));
FileInputFormat.setInputPaths(job, args[]);
Path output = new Path(args[]);
// FileSystem fs = FileSystem.get(conf);
// if (fs.exists(output)) {
// fs.delete(output, true);
// }
FileOutputFormat.setOutputPath(job, output); // VERBOSE OUTPUT
if (job.waitForCompletion(true)) {
errMsg = "InverseIndexTwo: TEST SUCCESSFULLY...";
LOGGER.debug(errMsg);
System.out.println(errMsg);
return ;
} else {
errMsg = "InverseIndexTwo: TEST FAILED...";
LOGGER.debug(errMsg);
System.out.println(errMsg);
return ;
}
} public static void main(String[] args) throws Exception {
if (args.length != ) {
String errMsg = "InverseIndexOne: ARGUMENTS ERROR";
LOGGER.error(errMsg);
System.out.println(errMsg);
System.exit(-);
} int result = ToolRunner.run(new Configuration(), new InverseIndexTwo(), args);
System.exit(result);
} }
参考资料:
How to check if processing the last item in an Iterator?:http://stackoverflow.com/questions/9633991/how-to-check-if-processing-the-last-item-in-an-iterator
【Hadoop】Hadoop MR 如何实现倒排索引算法?的更多相关文章
- hadoop修改MR的提交的代码程序的副本数
hadoop修改MR的提交的代码程序的副本数 Under-Replicated Blocks的数量很多,有7万多个.hadoop fsck -blocks 检查发现有很多replica missing ...
- 腾讯公司数据分析岗位的hadoop工作 线性回归 k-means算法 朴素贝叶斯算法 SpringMVC组件 某公司的广告投放系统 KNN算法 社交网络模型 SpringMVC注解方式
腾讯公司数据分析岗位的hadoop工作 线性回归 k-means算法 朴素贝叶斯算法 SpringMVC组件 某公司的广告投放系统 KNN算法 社交网络模型 SpringMVC注解方式 某移动公司实时 ...
- Hadoop【MR开发规范、序列化】
Hadoop[MR开发规范.序列化] 目录 Hadoop[MR开发规范.序列化] 一.MapReduce编程规范 1.Mapper阶段 2.Reducer阶段 3.Driver阶段 二.WordCou ...
- [Hadoop]Hadoop章2 HDFS原理及读写过程
HDFS(Hadoop Distributed File System )Hadoop分布式文件系统. HDFS有很多特点: ① 保存多个副本,且提供容错机制,副本丢失或宕机自动恢复.默认存3份. ② ...
- hadoop hadoop install (1)
vmuser@vmuser-VirtualBox:~$ sudo useradd -m hadoop -s /bin/bash[sudo] vmuser 的密码: vmuser@vmuser-Virt ...
- MR案例:倒排索引
1.map阶段:将单词和URI组成Key值(如“MapReduce :1.txt”),将词频作为value. 利用MR框架自带的Map端排序,将同一文档的相同单词的词频组成列表,传递给Combine过 ...
- Hadoop hadoop 机架感知配置
机架感知脚本 使用python3编写机架感知脚本,报存到topology.py,给予执行权限 import sys import os DEFAULT_RACK="/default-rack ...
- hadoop之 mr输出到hbase
1.注意问题: 1.在开发过程中一定要导入hbase源码中的lib库否则出现如下错误 TableMapReducUtil 找不到什么-- 2.编码: import java.io.IOExceptio ...
- Hadoop案例(四)倒排索引(多job串联)与全局计数器
一. 倒排索引(多job串联) 1. 需求分析 有大量的文本(文档.网页),需要建立搜索索引 xyg pingping xyg ss xyg ss a.txt xyg pingping xyg pin ...
随机推荐
- NOIP2017赛前考试注意事项总结
考前: 考试前把读入优化和库以及对拍文件打好做好准备工作,另外注意放松心态,太紧张了肯定考不好··将自己的注意力集中起来 考场策略: 考试的基本策略是对每于道题先想个20分钟,如果想不出个靠谱的方 ...
- Linux临时增加swap空间
linux临时增加swap空间:step 1: #dd if=/dev/zero of=/home/swap bs=1024 count=500000 注释:of=/home/swap,放置swap的 ...
- 百度之星初赛(A)——T1
小C的倍数问题 Problem Description 根据小学数学的知识,我们知道一个正整数x是3的倍数的条件是x每一位加起来的和是3的倍数.反之,如果一个数每一位加起来是3的倍数,则这个数肯定是3 ...
- odp.net 读写oracle blob字段
DEVELOPER: ODP.NET Serving Winning LOBs: http://www.oracle.com/technetwork/issue-archive/2005/05-nov ...
- 无线网络发射器选址 (NOIP2014)(真·纯模拟)
原题传送门 好吧,如果说D1T1是纯模拟大水题 D2T1就是纯模拟略水题. 这道题首先我们要看一看数据范围.. 0<=n,m<=128 送分也不带这么送的吧.. 二维前缀和,前缀和,二次循 ...
- Java设计模式_创建型模式_单例模式
单例模式的实现: 定义一个类,在类中定义该类的静态变量,再定一个一个获取该类的静态变量的方法. UML图:
- v4l2读取摄像头程序流程解析【转】
转自:https://my.oschina.net/u/1024767/blog/210801 v4l2 操作实际上就是 open() 设备, close() 设备,以及中间过程的 ioctl() 操 ...
- kvm虚拟机最佳实践系列2-创建KVM及KVM优化
创建KVM及KVM优化 把KVM优化与KVM创建放在一起,是因为我们创建的KVM是要用在生产环境中,所以基础优化工作是必备的. 创建KVM 创建系统盘, 大小: 操作系统通常都不到10G,所以系统盘2 ...
- kvm虚拟机最佳实践系列1-kvm宿主机准备
KVM宿主机配置 系统环境:ubuntu16, bond0 业务网口 bond1 管理网口+存储网口 安装KVM环境支持 sudo apt-get install qemu-kvm sudo apt- ...
- 清明小长假之VUE.JS学习测试码
我们放了四天假,刚好借此机会,系统的了解一下VUE.JS. <!DOCTYPE html> <html> <head> <meta charset=" ...