1,代码

package mr;

import java.io.IOException;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.ArrayWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; /**
* 使用ArrayWritable
*/
public class TrafficApp4 { public static void main(String[] args) throws Exception{
Configuration conf = new Configuration();
Job job = Job.getInstance(conf , TrafficApp4.class.getSimpleName());
job.setJarByClass(TrafficApp4.class); FileInputFormat.setInputPaths(job, args[]);
job.setMapperClass(TrafficMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(LongArrayWritable.class); job.setReducerClass(TrafficReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongArrayWritable.class);
FileOutputFormat.setOutputPath(job, new Path(args[])); job.waitForCompletion(true);
} public static class TrafficMapper extends Mapper<LongWritable, Text, Text, LongArrayWritable>{
@Override
protected void map(LongWritable key, Text value,
Mapper<LongWritable, Text, Text, LongArrayWritable>.Context context)
throws IOException, InterruptedException {
String line = value.toString();
String[] splited = line.split("\t");
String phonenumber = splited[]; String upPackNum = splited[];
String downPackNum = splited[];
String upPayLoad = splited[];
String downPayLoad = splited[]; Text k2 = new Text(phonenumber);
LongArrayWritable v2 = new LongArrayWritable(upPackNum, downPackNum, upPayLoad, downPayLoad);
context.write(k2, v2);
}
} public static class TrafficReducer extends Reducer<Text, LongArrayWritable, Text, LongArrayWritable>{
@Override
protected void reduce(Text k2, Iterable<LongArrayWritable> v2s,
Reducer<Text, LongArrayWritable, Text, LongArrayWritable>.Context context)
throws IOException, InterruptedException { long upPackNum = 0L;
long downPackNum = 0L;
long upPayLoad = 0L;
long downPayLoad = 0L;
for (LongArrayWritable v2 : v2s) {
Writable[] values = v2.get();
upPackNum += ((LongWritable)values[]).get();
downPackNum += ((LongWritable)values[]).get();
upPayLoad += ((LongWritable)values[]).get();
downPayLoad += ((LongWritable)values[]).get();
} LongArrayWritable v3 = new LongArrayWritable(upPackNum, downPackNum, upPayLoad, downPayLoad);
context.write(k2, v3);
}
} public static class LongArrayWritable extends ArrayWritable{
/**
* 在调用的时候,首先调用该方法,然后调用set(Writable[])
*/
public LongArrayWritable() {
super(LongWritable.class);
}
/**
* 直接调用该方法即可
* @param values
*/
public LongArrayWritable(LongWritable[] values) {
super(LongWritable.class, values);
}
/**
* 直接调用该方法即可
* @param upPackNum
* @param downPackNum
* @param upPayLoad
* @param downPayLoad
*/
public LongArrayWritable(Long upPackNum, Long downPackNum, Long upPayLoad, Long downPayLoad) {
super(LongWritable.class);
LongWritable[] values = new LongWritable[];
values[] = new LongWritable(upPackNum);
values[] = new LongWritable(downPackNum);
values[] = new LongWritable(upPayLoad);
values[] = new LongWritable(downPayLoad);
super.set(values);
}
/**
* 直接调用该方法即可
* @param upPackNum
* @param downPackNum
* @param upPayLoad
* @param downPayLoad
*/
public LongArrayWritable(String upPackNum, String downPackNum, String upPayLoad, String downPayLoad) {
super(LongWritable.class);
LongWritable[] values = new LongWritable[];
values[] = new LongWritable(Long.parseLong(upPackNum));
values[] = new LongWritable(Long.parseLong(downPackNum));
values[] = new LongWritable(Long.parseLong(upPayLoad));
values[] = new LongWritable(Long.parseLong(downPayLoad));
super.set(values);
} @Override
public String toString() {
String[] array = super.toStrings();
return StringUtils.join(array, "\t");
}
} }

2,ArrayWritable的API

org.apache.hadoop.io
Class ArrayWritable

java.lang.Object
  

org.apache.hadoop.io.ArrayWritable
已实现的接口:
Writable

public class ArrayWritableextends Objectimplements Writable

A Writable for arrays containing instances of a class. The elements of this writable must all be instances of the same class. If this writable will be the input for a Reducer, you will need to create a subclass that sets the value to be of the proper type. For example: public class IntArrayWritable extends ArrayWritable { public IntArrayWritable() { super(IntWritable.class); } }


构造方法摘要
ArrayWritable(Class<? extends Writable> valueClass) 
           
ArrayWritable(Class<? extends Writable> valueClass, Writable[] values) 
           
ArrayWritable(String[] strings) 
           
方法摘要
 Writable[] get() 
           
 Class getValueClass() 
           
 void readFields(DataInput in) 
          Deserialize the fields of this object from in.
 void set(Writable[] values) 
           
 Object toArray() 
           
 String[] toStrings() 
           
 void write(DataOutput out) 
          Serialize the fields of this object to out.
Methods inherited from class java.lang.Object
clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
构造方法详细信息

ArrayWritable

public ArrayWritable(Class<? extends Writable> valueClass)

ArrayWritable

public ArrayWritable(Class<? extends Writable> valueClass,
Writable[] values)

ArrayWritable

public ArrayWritable(String[] strings)
方法详细信息

getValueClass

public Class getValueClass()

toStrings

public String[] toStrings()

toArray

public Object toArray()

set

public void set(Writable[] values)

get

public Writable[] get()

readFields

public void readFields(DataInput in)
throws IOException
Description copied from interface: Writable
Deserialize the fields of this object from in.

For efficiency, implementations should attempt to re-use storage in the existing object where possible.

Specified by:
readFields in interface Writable
Parameters:
in - DataInput to deseriablize this object from.
Throws:
IOException

write

public void write(DataOutput out)
throws IOException
Description copied from interface: Writable
Serialize the fields of this object to out.

Specified by:
write in interface Writable
Parameters:
out - DataOuput to serialize this object into.
Throws:
IOException

 

 

基于MapReduce的手机流量统计分析的更多相关文章

  1. MapReduce的手机流量统计的案例

    程序:(另外一个关于单词计数的总结:http://www.cnblogs.com/DreamDrive/p/5492572.html) import java.io.IOException; impo ...

  2. 基于winpcap的以太网流量分析器(java)

    开发工具 IDE:eclipse -neon JDK:1.8 OS:Win10-64bit 主要功能 1.要求完成一个基于Winpcap的网络流量统计分析系统,具有易用.美观的界面. 2.完成局域网( ...

  3. 023_数量类型练习——Hadoop MapReduce手机流量统计

    1) 分析业务需求:用户使用手机上网,存在流量的消耗.流量包括两部分:其一是上行流量(发送消息流量),其二是下行流量(接收消息的流量).每种流量在网络传输过程中,有两种形式说明:包的大小,流量的大小. ...

  4. 第2节 mapreduce深入学习:8、手机流量汇总求和

    第2节 mapreduce深入学习:8.手机流量汇总求和 例子:MapReduce综合练习之上网流量统计. 数据格式参见资料夹 需求一:统计求和 统计每个手机号的上行流量总和,下行流量总和,上行总流量 ...

  5. MapReduce 经典案例手机流量排序的分析

    在进行流量排序之前,先要明白排序是发生在map阶段,排序之后(排序结束后map阶段才会显示100%完成)才会到reduce阶段(事实上reduce也会排序),.此外排序之前要已经完成了手机流量的统计工 ...

  6. 基于mapreduce的大规模连通图寻找算法

    基于mapreduce的大规模连通图寻找算法 当我们想要知道哪些账号是一个人的时候往往可以通过业务得到两个账号之间有联系,但是这种联系如何传播呢? 问题 已知每个账号之间的联系 如: A B B C ...

  7. 字节数转换为b,kb,mb,gb的方法 通用的手机流量计算方法

    //通用的手机流量计算方法 private String byteToMB(long size){ long kb = 1024; long mb = kb*1024; long gb = mb*10 ...

  8. MapReduce教程(一)基于MapReduce框架开发<转>

    1 MapReduce编程 1.1 MapReduce简介 MapReduce是一种编程模型,用于大规模数据集(大于1TB)的并行运算,用于解决海量数据的计算问题. MapReduce分成了两个部分: ...

  9. 基于MapReduce的贝叶斯网络算法研究参考文献

    原文链接(系列):http://blog.csdn.net/XuanZuoNuo/article/details/10472219 论文: 加速贝叶斯网络:Accelerating Bayesian ...

随机推荐

  1. 程序在Linux下前后台切换

    程序在Linux下前后台切换 一.为什么要使程序在后台执行 背景:SecureCRT远程连接到linux主机,使程序在后台运行有以下好处: (1)本机关机不影响linux主机运行 (2)不影响计算效率 ...

  2. html基础问题总结

    1.reflow 在CSS规范中有一个渲染对象的概念,通常用一个盒子(box, rectangle)来表示.mozilla通过一个叫frame的对象对盒子进行操作.frame主要的动作有三个: 构造f ...

  3. linux开发基本库

    1.ZeroMQ zmq是一个消息队列.可以在进程内.进程间.TCP.多播中,以消息为单位传输数据,而不是socket的字节流.官方主页上有下载.使用.文档,蛮全的. 常用模式有:Request-Re ...

  4. mysql数据库----Pymysql

    本节重点: pymysql下载和使用 sql注入 增.删.改:conn.commit() 查:fetchone.fetchmany.fetchall 一.pymysql的下载和使用 之前我们都是通过M ...

  5. 多版本python import 问题解决方案

    原文http://www.tuicool.com/articles/EnE7nm6 多版本Python共存[支持使用pip安装包] 有时特殊需要会要用到高版本的Python, 但是系统自带的版本又是很 ...

  6. Apache服务器的Options 的 Indexes FollowSymLinks详解

    禁止显示Apache目录列表 - Indexes FollowSymLinks 如何修改目录的配置以禁止显示 Apache 目录列表. 缺省情况下如果你在浏览器输入地址: http://localho ...

  7. 前端将markdown转换成html

    实现过程: 1. npm引入:npm install marked --save 2.在需要的文件(.ts)里import Marked from "marked";   如果.j ...

  8. 开发一个delphi写的桌面图标管理代码

    参加工作了就很少有时间去玩delphi了,这个适合初学者看看,大神勿喷 工具 delhpi7.0 access数据库 原则win下有安装office就可用 当初不太熟悉sqlite所有没用这做数据库. ...

  9. BZOJ 3668:起床困难综合症(贪心)

    分析:按位贪心即可. program sleep; var a,g:..]of longint; n,i,m,ans,t,len,x,y,v:longint; c:char; s:string; e: ...

  10. ECharts饼图制作分析

    ECharts,缩写来自Enterprise Charts,商业级数据图表,一个纯Javascript的图表库,可以流畅的运行在PC和移动设备上,兼容当前绝大部分浏览器(IE6/7/8/9/10/11 ...