大数据学习——mapreduce运营商日志增强

需求

1、对原始json数据进行解析，变成普通文本数据

2、求出每个人评分最高的3部电影

3、求出被评分次数最多的3部电影

数据

https://pan.baidu.com/s/1gPsQXVYSQEZ2OYek4HxK6A

pom.xml

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <groupId>com.cyf</groupId>

    <artifactId>MapReduceCases</artifactId>

    <packaging>jar</packaging>

    <version>1.0</version>

    <properties>

        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

    </properties>

    <dependencies>

        <dependency>

            <groupId>org.apache.hadoop</groupId>

            <artifactId>hadoop-common</artifactId>

            <version>2.6.4</version>

        </dependency>

        <dependency>

            <groupId>org.apache.hadoop</groupId>

            <artifactId>hadoop-hdfs</artifactId>

            <version>2.6.4</version>

        </dependency>

        <dependency>

            <groupId>org.apache.hadoop</groupId>

            <artifactId>hadoop-client</artifactId>

            <version>2.6.4</version>

        </dependency>

        <dependency>

            <groupId>org.apache.hadoop</groupId>

            <artifactId>hadoop-mapreduce-client-core</artifactId>

            <version>2.6.4</version>

        </dependency>

        <dependency>

            <groupId>com.alibaba</groupId>

            <artifactId>fastjson</artifactId>

            <version>1.1.40</version>

        </dependency>

        <dependency>

            <groupId>mysql</groupId>

            <artifactId>mysql-connector-java</artifactId>

            <version>5.1.36</version>

        </dependency>

    </dependencies>

    <build>

        <plugins>

            <plugin>

                <artifactId>maven-assembly-plugin</artifactId>

                <configuration>

                    <appendAssemblyId>false</appendAssemblyId>

                    <descriptorRefs>

                        <descriptorRef>jar-with-dependencies</descriptorRef>

                    </descriptorRefs>

                    <archive>

                        <manifest>

                            <mainClass>cn.itcast.mapreduce.json.JsonToText</mainClass>

                        </manifest>

                    </archive>

                </configuration>

                <executions>

                    <execution>

                        <id>make-assembly</id>

                        <phase>package</phase>

                        <goals>

                            <goal>assembly</goal>

                        </goals>

                    </execution>

                </executions>

            </plugin>

        </plugins>

    </build>

</project>

package cn.itcast.mapreduce.json;

import java.io.DataInput;

import java.io.DataOutput;

import java.io.IOException;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.WritableComparable;

public class OriginBean implements WritableComparable<OriginBean> {

    private Long movie;

    private Long rate;

    private Long timeStamp;

    private Long uid;

    public Long getMovie() {

        return movie;

    }

    public void setMovie(Long movie) {

        this.movie = movie;

    }

    public Long getRate() {

        return rate;

    }

    public void setRate(Long rate) {

        this.rate = rate;

    }

    public Long getTimeStamp() {

        return timeStamp;

    }

    public void setTimeStamp(Long timeStamp) {

        this.timeStamp = timeStamp;

    }

    public Long getUid() {

        return uid;

    }

    public void setUid(Long uid) {

        this.uid = uid;

    }

    public OriginBean(Long movie, Long rate, Long timeStamp, Long uid) {

        this.movie = movie;

        this.rate = rate;

        this.timeStamp = timeStamp;

        this.uid = uid;

    }

    public OriginBean() {

        // TODO Auto-generated constructor stub

    }

    public int compareTo(OriginBean o) {

        return this.movie.compareTo(o.movie);

    }

    public void write(DataOutput out) throws IOException {

        out.writeLong(movie);

        out.writeLong(rate);

        out.writeLong(timeStamp);

        out.writeLong(uid);

    }

    public void readFields(DataInput in) throws IOException {

        this.movie = in.readLong();

        this.rate = in.readLong();

        this.timeStamp = in.readLong();

        this.uid = in.readLong();

    }

    @Override

    public String toString() {

        return this.movie + "\t" + this.rate + "\t" + this.timeStamp + "\t" + this.uid;

    }

}

package cn.itcast.mapreduce.json;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;

import org.codehaus.jackson.map.ObjectMapper;

import com.alibaba.fastjson.JSON;

import com.alibaba.fastjson.JSONObject;

public class JsonToText {

    static class MyMapper extends Mapper<LongWritable, Text, Text, NullWritable> {

        Text k = new Text();

        @Override

        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

            // Bean bean = mapper.readValue(value.toString(), Bean.class);

            JSONObject valueJson = JSON.parseObject(value.toString());

            Long movie = valueJson.getLong("movie");

            OriginBean bean = new OriginBean(movie, valueJson.getLong("rate"), valueJson.getLong("timeStamp"), valueJson.getLong("uid"));

            k.set(bean.toString());

            context.write(k, NullWritable.get());

        }

    }

    public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();

//16777216/1024/1024=16 （62.5M/16）4个切片，启动4个maptask,处理结果4个文件

conf.set("mapreduce.input.fileinputformat.split.maxsize", "16777216"); Job job = Job.getInstance(conf); // job.setJarByClass(JsonToText.class); //告诉框架，我们的程序所在jar包的位置 job.setJar("/root/JsonToText.jar"); job.setMapperClass(MyMapper.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(NullWritable.class); // job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setNumReduceTasks(0); FileInputFormat.setInputPaths(job, new Path("/json/input")); FileOutputFormat.setOutputPath(job, new Path("/json/output")); // FileInputFormat.setInputPaths(job, new Path(args[0])); // FileOutputFormat.setOutputPath(job, new Path(args[1]));  job.waitForCompletion(true); } }

创建文件夹并上传数据

hadoop fs -mkdir -p /json/input

hadoop fs -put rating.json /json/input

运行

hadoop jar JsonToText.jar cn.itcast.mapreduce.json.JsonToText

运行结果

https://pan.baidu.com/s/1ayrpl7w8Dlzpc7TRZIO94w

pom.xml

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <groupId>com.cyf</groupId>

    <artifactId>MapReduceCases</artifactId>

    <packaging>jar</packaging>

    <version>1.0</version>

    <properties>

        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>

    </properties>

    <dependencies>

        <dependency>

            <groupId>org.apache.hadoop</groupId>

            <artifactId>hadoop-common</artifactId>

            <version>2.6.4</version>

        </dependency>

        <dependency>

            <groupId>org.apache.hadoop</groupId>

            <artifactId>hadoop-hdfs</artifactId>

            <version>2.6.4</version>

        </dependency>

        <dependency>

            <groupId>org.apache.hadoop</groupId>

            <artifactId>hadoop-client</artifactId>

            <version>2.6.4</version>

        </dependency>

        <dependency>

            <groupId>org.apache.hadoop</groupId>

            <artifactId>hadoop-mapreduce-client-core</artifactId>

            <version>2.6.4</version>

        </dependency>

        <dependency>

            <groupId>com.alibaba</groupId>

            <artifactId>fastjson</artifactId>

            <version>1.1.40</version>

        </dependency>

        <dependency>

            <groupId>mysql</groupId>

            <artifactId>mysql-connector-java</artifactId>

            <version>5.1.36</version>

        </dependency>

    </dependencies>

    <build>

        <plugins>

            <plugin>

                <artifactId>maven-assembly-plugin</artifactId>

                <configuration>

                    <appendAssemblyId>false</appendAssemblyId>

                    <descriptorRefs>

                        <descriptorRef>jar-with-dependencies</descriptorRef>

                    </descriptorRefs>

                    <archive>

                        <manifest>

                            <mainClass>cn.itcast.mapreduce.json.MovieRateSum</mainClass>

                        </manifest>

                    </archive>

                </configuration>

                <executions>

                    <execution>

                        <id>make-assembly</id>

                        <phase>package</phase>

                        <goals>

                            <goal>assembly</goal>

                        </goals>

                    </execution>

                </executions>

            </plugin>

        </plugins>

    </build>

</project>

package cn.itcast.mapreduce.json;

import java.io.DataInput;

import java.io.DataOutput;

import java.io.IOException;

import org.apache.hadoop.io.WritableComparable;

public class ResultBean implements WritableComparable<ResultBean> {

    private Long movie;

    private Long sumRate;

    public void setSumRate(long sumRate) {

        this.sumRate = sumRate;

    }

    public Long getMovie() {

        return movie;

    }

    public void setMovie(Long movie) {

        this.movie = movie;

    }

    public ResultBean(Long movie, Long sumRate) {

        this.movie = movie;

        this.sumRate = sumRate;

    }

    public ResultBean() {

        // TODO Auto-generated constructor stub

    }

    public int compareTo(ResultBean o) {

        if (this.movie - o.movie != 0) {

            return (int) (this.movie - o.movie);

        }

        return (int) (o.sumRate - this.sumRate);

    }

    public void write(DataOutput out) throws IOException {

        out.writeLong(movie);

        out.writeLong(sumRate);

    }

    public ResultBean(Long sumRate) {

        super();

        this.sumRate = sumRate;

    }

    public void readFields(DataInput in) throws IOException {

        this.movie = in.readLong();

        this.sumRate = in.readLong();

    }

    @Override

    public String toString() {

        //return movie + "\t" + sumRate;

        return movie + "\t" + sumRate;

    }

}

package cn.itcast.mapreduce.json;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.NullWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;

import org.codehaus.jackson.map.ObjectMapper;

import com.alibaba.fastjson.JSON;

import com.alibaba.fastjson.JSONObject;

public class MovieRateSum {

    static class MyMapper extends Mapper<LongWritable, Text, LongWritable, OriginBean> {

        ObjectMapper mapper = new ObjectMapper();

        @Override

        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

            // Bean bean = mapper.readValue(value.toString(), Bean.class);

            JSONObject valueJson = JSON.parseObject(value.toString());

            Long movie = valueJson.getLong("movie");

            OriginBean bean = new OriginBean(movie, valueJson.getLong("rate"), valueJson.getLong("timeStamp"), valueJson.getLong("uid"));

            context.write(new LongWritable(bean.getMovie()), bean);

        }

    }

    static class MyReduce extends Reducer<LongWritable, OriginBean, ResultBean, NullWritable> {

        @Override

        protected void reduce(LongWritable movie, Iterable<OriginBean> beans, Context context) throws IOException, InterruptedException {

            long sum = 0L;

            for (OriginBean bean : beans) {

                sum += bean.getRate();

            }

            ResultBean bean = new ResultBean();

            bean.setMovie(movie.get());

            bean.setSumRate(sum);

            context.write(bean, NullWritable.get());

        }

    }

    public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();

        Job job = Job.getInstance(conf);

//        job.setJarByClass(MovieRateSum.class);

        //告诉框架，我们的程序所在jar包的位置

        job.setJar("/root/MovieRateSum.jar");

        job.setMapperClass(MyMapper.class);

        job.setReducerClass(MyReduce.class);

        job.setMapOutputKeyClass(LongWritable.class);

        job.setMapOutputValueClass(OriginBean.class);

        job.setOutputKeyClass(ResultBean.class);

        job.setOutputValueClass(NullWritable.class);

        job.setOutputFormatClass(SequenceFileOutputFormat.class);

        FileInputFormat.setInputPaths(job, new Path("/json/output"));

        FileOutputFormat.setOutputPath(job, new Path("/json/output-seq"));

        // FileInputFormat.setInputPaths(job, new Path(args[0]));

        // FileOutputFormat.setOutputPath(job, new Path(args[1]));

        job.waitForCompletion(true);

    }

}

大数据学习——mapreduce运营商日志增强的更多相关文章

大数据学习——点击流日志每天都10T，在业务应用服务器上，需要准实时上传至（Hadoop HDFS）上
点击流日志每天都10T,在业务应用服务器上,需要准实时上传至(Hadoop HDFS)上 1需求说明点击流日志每天都10T,在业务应用服务器上,需要准实时上传至(Hadoop HDFS)上 2需求分 ...
大数据学习——mapreduce案例join算法
需求: 用mapreduce实现select order.orderid,order.pdtid,pdts.pdt_name,oder.amount from orderjoin pdtson ord ...
大数据学习——mapreduce学习topN问题
求每一个订单中成交金额最大的那一笔 top1 数据 Order_0000001,Pdt_01,222.8 Order_0000001,Pdt_05,25.8 Order_0000002,Pdt_05 ...
大数据学习——mapreduce共同好友
数据 commonfriends.txt A:B,C,D,F,E,O B:A,C,E,K C:F,A,D,I D:A,E,F,L E:B,C,D,M,L F:A,B,C,D,E,O,M G:A,C,D ...
大数据学习——mapreduce倒排索引
数据 a.txt hello jerry hello tom b.txt allen tom allen jerry allen hello c.txt hello jerry hello tom 1 ...
大数据学习——mapreduce汇总手机号上行流量下行流量总流量
时间戳手机号 MAC地址 ip 域名上行流量包个数下行上行流量下行流量 http状态码 1363157995052 13826544101 5C-0E-8B-C7-F1-E0:CMCC 12 ...
大数据学习——mapreduce程序单词统计
项目结构 pom.xml文件 <?xml version="1.0" encoding="UTF-8"?> <project xmlns=&q ...
大数据学习——MapReduce学习——字符统计WordCount
操作背景 jdk的版本为1.8以上 ubuntu12 hadoop2.5伪分布安装 Hadoop-Eclipse-Plugin 要在 Eclipse 上编译和运行 MapReduce 程序,需要安装 ...
大数据学习系列之七 ----- Hadoop+Spark+Zookeeper+HBase+Hive集群搭建图文详解
引言在之前的大数据学习系列中,搭建了Hadoop+Spark+HBase+Hive 环境以及一些测试.其实要说的话,我开始学习大数据的时候,搭建的就是集群,并不是单机模式和伪分布式.至于为什么先写单 ...

随机推荐

题解报告：hihoCoder #1050 : 树中的最长路
描述上回说到,小Ho得到了一棵二叉树玩具,这个玩具是由小球和木棍连接起来的,而在拆拼它的过程中,小Ho发现他不仅仅可以拼凑成一棵二叉树!还可以拼凑成一棵多叉树——好吧,其实就是更为平常的树而已. 但 ...
125 Valid Palindrome 验证回文字符串
给定一个字符串,确定它是否是回文,只考虑字母数字字符和忽略大小写.例如:"A man, a plan, a canal: Panama" 是回文字符串."race a c ...
debug授权码
www.vfxcx.com 704835b5c54b56426257e0742568fe54
LSP
Liskov Substitution Principle里氏替换原则,OCP作为OO的高层原则,主张使用“抽象(Abstraction)”和“多态(Polymorphism)”将设计中的静态结构改为 ...
canvas基础绘制-绚丽时钟
效果图: 与canvas基础绘制-绚丽倒计时的代码差异: // var endTime = new Date();//const声明变量,不可修改,必须声明时赋值: // endTime.setTim ...
Javaweb学习笔记2—Tomcat和http协议
今天来讲javaweb的第二个阶段学习. 老规矩,首先先用一张思维导图来展现今天的博客内容. ps:我的思维是用的xMind画的,如果你对我的思维导图感兴趣并且想看到你们跟详细的备注信息,请点击下 ...
一个PHP开发APP接口的视频教程
感觉php做接口方面的教程很少,无意中搜到了这个视频教程,希望能给一些人带来帮助http://www.imooc.com/learn/163
ECharts是我接触过的最优秀的可视化工具，也是进步最快的软件，希望它早日成为世界级的开源项目。
ECharts的广泛网址: http://echarts.baidu.com/doc/example.html 零编程玩转图表: http://tushuo.baidu.com/?qq-pf-to=p ...
pylint安装失败的解决方法
原文链接http://www.cnblogs.com/Loonger/p/7815335.html 使用命令pip3 install pylint安装pylint是出现错误.查了一圈也找不到答案.仔细 ...
中间件及tomcat的内存溢出调优
主要是这三个选项的调整需要根据主机的内存配置以及业务量的使用情况调节 -Xmx4g -Xms4g -Xmn2g xmx 与xms一般设置为一样 xmn大致设置为xmx xms的三分之一可以使用 ...

大数据学习——mapreduce运营商日志增强

大数据学习——mapreduce运营商日志增强的更多相关文章

随机推荐

热门专题