Spark2.0基于广播变量broadcast实现实时数据按天统计

package com.gm.hive.SparkHive;

import java.text.SimpleDateFormat;

import java.util.Arrays;

import java.util.Collection;

import java.util.Date;

import java.util.HashMap;

import java.util.List;

import java.util.Map;

import java.util.Properties;

import org.apache.spark.Partition;

import org.apache.kafka.clients.consumer.ConsumerRecord;

import org.apache.kafka.common.serialization.StringDeserializer;

import org.apache.spark.SparkConf;

import org.apache.spark.api.java.JavaPairRDD;

import org.apache.spark.api.java.JavaRDD;

import org.apache.spark.api.java.JavaSparkContext;

import org.apache.spark.api.java.Optional;

import org.apache.spark.api.java.function.Function;

import org.apache.spark.api.java.function.Function2;

import org.apache.spark.api.java.function.VoidFunction;

import org.apache.spark.broadcast.Broadcast;

import org.apache.spark.rdd.RDD;

import org.apache.spark.sql.Dataset;

import org.apache.spark.sql.Row;

import org.apache.spark.sql.SaveMode;

import org.apache.spark.sql.SparkSession;

import org.apache.spark.streaming.Durations;

import org.apache.spark.streaming.api.java.JavaInputDStream;

import org.apache.spark.streaming.api.java.JavaPairDStream;

import org.apache.spark.streaming.api.java.JavaStreamingContext;

import org.apache.spark.streaming.kafka010.ConsumerStrategies;

import org.apache.spark.streaming.kafka010.KafkaUtils;

import org.apache.spark.streaming.kafka010.LocationStrategies;

import scala.Tuple2;

import scala.reflect.ClassManifestFactory;

public class App {

	private static volatile Broadcast<Map<String,Boolean>> bcMap = null;

	public static void main(String[] args) {

		// TODO Auto-generated method stub

		SparkConf conf = new SparkConf().setMaster("local[2]").setAppName(

				"SparkStreaming");

		JavaSparkContext sc = new JavaSparkContext(conf);

		sc.setLogLevel("ERROR");

		sc.setCheckpointDir("./checkpoint");

		JavaStreamingContext ssc = new JavaStreamingContext(sc,

				Durations.seconds(10));

		Date date = new Date();

		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");

		Map<String,Boolean> map = new HashMap<String,Boolean>();

		map.put(sdf.format(date), true);

		bcMap = sc.broadcast(map);//初始广播变量

		// kafka相关参数，必要！缺了会报错

		Map<String, Object> kafkaParams = new HashMap<>();

		kafkaParams.put("bootstrap.servers", "192.168.174.200:9092");

		kafkaParams.put("key.deserializer", StringDeserializer.class);

		kafkaParams.put("value.deserializer", StringDeserializer.class);

		kafkaParams.put("group.id", "newgroup2");

		kafkaParams.put("auto.offset.reset", "latest");

		kafkaParams.put("enable.auto.commit", false);

		Collection<String> topics = Arrays.asList("test");

		JavaInputDStream<ConsumerRecord<String, String>> stream = KafkaUtils

				.createDirectStream(ssc, LocationStrategies.PreferConsistent(),

						ConsumerStrategies.<String, String> Subscribe(topics,

								kafkaParams));

		// 注意这边的stream里的参数本身是个ConsumerRecord对象

		JavaPairDStream<String, Integer> counts = stream

				.flatMap(

						x -> Arrays.asList(x.value().toString().split(" "))

								.iterator())

				.mapToPair(x -> new Tuple2<String, Integer>(x, 1))

				.reduceByKey((x, y) -> x + y);

		//counts.print();

		stream.foreachRDD(rdd -> {

			Map<String,Boolean> map1 = bcMap.value();

			Date newDate = new Date();

			SimpleDateFormat newsdf = new SimpleDateFormat("yyyy-MM-dd");

			String newDay = newsdf.format(newDate);

			if (map1.get(newDay) != null) {//存在当前天

				if (bcMap.value().get(newDay)) {//当前天的值为true，更新为false并更新到广播变量中

					map1.put(newDay, false);

					bcMap = rdd.context().broadcast(map1,ClassManifestFactory.classType(Map.class));

				}

			} else {

				if (bcMap != null) {//不存在当前天，将新的一天添加并更新到广播变量中

					bcMap.unpersist();

				}

				map1.put(newDay, true);

				bcMap = rdd.context().broadcast(map1,ClassManifestFactory.classType(Map.class));

			}

		});

		JavaPairDStream<String, Integer> result = counts

				.updateStateByKey(new Function2<List<Integer>, Optional<Integer>, Optional<Integer>>() {

					private static final long serialVersionUID = 1L;

					@Override

					public Optional<Integer> call(List<Integer> values,

							Optional<Integer> state) throws Exception {

						/**

						 * values:经过分组最后 这个key所对应的value，如：[1,1,1,1,1]

						 * state:这个key在本次之前之前的状态

						 */

						Integer updateValue = 0;

						Date newDate = new Date();

						SimpleDateFormat newsdf = new SimpleDateFormat("yyyy-MM-dd");

						String newDay = newsdf.format(newDate);

						Map<String,Boolean> map1 = bcMap.value();

						if (map1.get(newDay) != null) {

							if(map1.get(newDay)){//新的一天开始，将计算结果更新为0

								for (Integer value : values) {

									updateValue += value;

								}

							} else {//新的一天进行中，已计算过数据，正常运算

								if (state.isPresent()) {

									updateValue = state.get();

								}

								for (Integer value : values) {

									updateValue += value;

								}

							}

						}

						return Optional.of(updateValue);

					}

				});

		//数据库内容

		String url = "jdbc:postgresql://192.168.174.200:5432/postgres?charSet=utf-8";

		Properties connectionProperties = new Properties();

		connectionProperties.put("user","postgres");

		connectionProperties.put("password","postgres");

		connectionProperties.put("driver","org.postgresql.Driver");

		result.print();

		result.foreachRDD(new VoidFunction<JavaPairRDD<String, Integer>>(){

			public void call(JavaPairRDD<String, Integer> rdd)

					throws Exception {

				// TODO Auto-generated method stub

				JavaRDD<ResultRow> rowRDD = rdd.map(new Function<Tuple2<String, Integer>,ResultRow>(){

					public ResultRow call(Tuple2<String, Integer> arg0)

							throws Exception {

						// TODO Auto-generated method stub

						Date newDate = new Date();

						SimpleDateFormat newsdf = new SimpleDateFormat("yyyy-MM-dd");

						String newDay = newsdf.format(newDate);

						ResultRow rr = new ResultRow();

						rr.setTypeid(arg0._1+"_"+newDay);

						rr.setKczs(arg0._2);

						return rr;

					}

				});

				SparkSession spark = SparkSession.builder().config(rdd.context().getConf()).getOrCreate();

				Dataset<Row>  data = spark.createDataFrame(rowRDD, ResultRow.class);

				//将数据通过覆盖的形式保存在数据表中

				data.write().mode(SaveMode.Append).jdbc(url, "kcssqktj", connectionProperties);

			}

		});

		ssc.start();

		try {

			ssc.awaitTermination();

		} catch (InterruptedException e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

		}

		ssc.close();

	}

}

package com.gm.hive.SparkHive;

import java.io.Serializable;

public class ResultRow implements Serializable {

	private static final long serialVersionUID = 6681372116317508248L;

	String typeid;

	int kczs;

	public String getTypeid() {

		return typeid;

	}

	public void setTypeid(String typeid) {

		this.typeid = typeid;

	}

	public int getKczs() {

		return kczs;

	}

	public void setKczs(int kczs) {

		this.kczs = kczs;

	}

}

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

	<modelVersion>4.0.0</modelVersion>

	<groupId>com.test</groupId>

	<artifactId>kcssqktj_spark</artifactId>

	<version>0.0.1-SNAPSHOT</version>

	<properties>

		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

	</properties>

	<dependencies>

		<dependency>

			<groupId>junit</groupId>

			<artifactId>junit</artifactId>

			<version>3.8.1</version>

			<scope>test</scope>

		</dependency>

		<dependency>

			<groupId>org.slf4j</groupId>

			<artifactId>slf4j-log4j12</artifactId>

			<version>1.7.22</version>

		</dependency>

		<dependency>

			<groupId>org.apache.hadoop</groupId>

			<artifactId>hadoop-client</artifactId>

			<version>2.8.0</version>

			<exclusions>

				<exclusion>

					<groupId>javax.servlet</groupId>

					<artifactId>*</artifactId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>org.apache.spark</groupId>

			<artifactId>spark-sql_2.11</artifactId>

			<version>2.0.0</version>

		</dependency>

		<dependency>

			<groupId>org.apache.spark</groupId>

			<artifactId>spark-hive_2.11</artifactId>

			<version>2.0.0</version>

		</dependency>

		<dependency>

			<groupId>org.apache.spark</groupId>

			<artifactId>spark-streaming_2.11</artifactId>

			<version>2.0.0</version>

			<exclusions>

				<exclusion>

					<artifactId>slf4j-log4j12</artifactId>

					<groupId>org.slf4j</groupId>

				</exclusion>

			</exclusions>

		</dependency>

		<dependency>

			<groupId>org.apache.spark</groupId>

			<artifactId>spark-core_2.11</artifactId>

			<version>2.0.0</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/org.apache.hive/hive-jdbc -->

		<dependency>

			<groupId>org.apache.hive</groupId>

			<artifactId>hive-jdbc</artifactId>

			<version>2.1.1</version>

		</dependency>

		<!-- https://mvnrepository.com/artifact/org.apache.hive/hive-exec -->

		<dependency>

			<groupId>org.apache.hive</groupId>

			<artifactId>hive-exec</artifactId>

			<version>2.1.1</version>

		</dependency>

		<dependency>

			<groupId>org.postgresql</groupId>

			<artifactId>postgresql</artifactId>

			<version>9.4-1201-jdbc4</version>

		</dependency>

		<dependency>

			<groupId>org.apache.spark</groupId>

			<artifactId>spark-streaming-kafka-0-10_2.11</artifactId>

			<version>2.0.0</version>

		</dependency>

	</dependencies>

	<build>

		<plugins>

			<plugin>

				<groupId>org.apache.maven.plugins</groupId>

				<artifactId>maven-compiler-plugin</artifactId>

				<configuration>

					<source>1.8</source>

					<target>1.8</target>

				</configuration>

			</plugin>

			<plugin>

				<groupId>org.apache.maven.plugins</groupId>

				<artifactId>maven-shade-plugin</artifactId>

				<configuration>

					<source>1.8</source>

					<target>1.8</target>

				</configuration>

				<executions>

					<execution>

						<phase>package</phase>

						<goals>

							<goal>shade</goal>

						</goals>

						<configuration>

							<shadedArtifactAttached>true</shadedArtifactAttached>

							<shadedClassifierName>allinone</shadedClassifierName>

							<artifactSet>

								<includes>

									<include>*:*</include>

								</includes>

							</artifactSet>

							<filters>

								<filter>

									<artifact>*:*</artifact>

									<excludes>

										<exclude>META-INF/*.SF</exclude>

										<exclude>META-INF/*.DSA</exclude>

										<exclude>META-INF/*.RSA</exclude>

									</excludes>

								</filter>

							</filters>

							<transformers>

								<transformer

									implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">

									<resource>reference.conf</resource>

								</transformer>

								<transformer

									implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">

									<resource>META-INF/spring.handlers</resource>

								</transformer>

								<transformer

									implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">

									<resource>META-INF/spring.schemas</resource>

								</transformer>

								<transformer

									implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">

									<manifestEntries>

										<Main-Class></Main-Class>

									</manifestEntries>

								</transformer>

							</transformers>

						</configuration>

					</execution>

				</executions>

			</plugin>

		</plugins>

	</build>

</project>

Spark2.0基于广播变量broadcast实现实时数据按天统计的更多相关文章

spark中的广播变量broadcast
Spark中的Broadcast处理首先先来看一看broadcast的使用代码: val values = List[Int](1,2,3) val broadcastValues = sparkC ...
Spark 广播变量BroadCast
一. 广播变量广播变量允许程序员将一个只读的变量缓存在每台机器上,而不用在任务之间传递变量.广播变量可被用于有效地给每个节点一个大输入数据集的副本.Spark还尝试使用高效地广播算法来分发变量,进而 ...
初识Flink广播变量broadcast
Broadcast 广播变量:可以理解为是一个公共的共享变量,我们可以把一个dataset 或者不变的缓存对象(例如map list集合对象等)数据集广播出去,然后不同的任务在节点上都能够获取到,并在 ...
基于Redis、Storm的实时数据查询实践
通过算法小组给出的聚合文件,我们需要实现一种业务场景,通过用户的消费地点的商户ID与posId,查询出他所在的商圈,并通过商圈地点查询出与该区域的做活动的商户,并与之进行消息匹配,推送相应活动信息到用 ...
Spark——DataFrames，RDD，DataSets、广播变量与累加器
Spark--DataFrames,RDD,DataSets 一.弹性数据集(RDD) 创建RDD 1.1RDD的宽依赖和窄依赖二.DataFrames 三.DataSets 四.什么时候使用Dat ...
Spark（三）RDD与广播变量、累加器
一.RDD的概述 1.1 什么是RDD RDD(Resilient Distributed Dataset)叫做弹性分布式数据集,是Spark中最基本的数据抽象,它代表一个不可变.可分区.里面的元素可 ...
Spark2.0机器学习系列之3：决策树
概述分类决策树模型是一种描述对实例进行分类的树形结构. 决策树可以看为一个if-then规则集合,具有“互斥完备”性质 .决策树基本上都是采用的是贪心(即非回溯)的算法,自顶向下递归分治构造. 生 ...
Spark学习之路（六）—— 累加器与广播变量
一.简介在Spark中,提供了两种类型的共享变量:累加器(accumulator)与广播变量(broadcast variable): 累加器:用来对信息进行聚合,主要用于累计计数等场景: 广播变量 ...
Spark 系列（六）—— 累加器与广播变量
一.简介在 Spark 中,提供了两种类型的共享变量:累加器 (accumulator) 与广播变量 (broadcast variable): 累加器:用来对信息进行聚合,主要用于累计计数等场景: ...

随机推荐

centos搭建lamp环境参考（根据腾讯云实验室）
1.安装MYSQL 使用 yum 安装 MySQL: yum install mysql-server -y 安装完成后,启动 MySQL 服务: service mysqld restart 设置 ...
老男孩python3.5全栈开发第9期+课件笔记（1-15部全共125天完整无加密）
点击了解更多Python课程>>> 老男孩python3.5全栈开发第9期+课件笔记(1-15部全共125天完整无加密)大小:236G 此课程为老男孩全栈开发最新完结课程,适合零基 ...
kohana附件上传
try { $upload = Uploader::factory('Picture', $_FILES['Filedata'])->execute();}catch (Exception $e ...
[CF1082G]Petya and Graph：最小割
分析发这篇博客的目的就是要让你们知道博主到底有多菜. 类似于[NOI2006]最大获利.(明明就是一模一样好吧!) 不知道怎么了,半秒就想到用网络流,却没想出怎么建图. 连这么简单的题都没做出来,我 ...
动态GI
在Engine/Config 目录中找到ConsoleVariables.ini并打开,在其中加入 r.LightPropagationVolume = 1 ,保存,重启引擎如果场景中有Post P ...
All men are brothers
All men are brothers 牛客多校第九场E 给定n个人,起初互不认识然后m各阶段每个阶段有两个人x.y认识求每个阶段选出四个人互不认识的方式并查集 #include<bi ...
Menu Items are not showing on Action Bar
http://stackoverflow.com/questions/18010072/menu-items-are-not-showing-on-action-bar 版权声明:本文为博主原创文章, ...
微信小程序之登录用户不是该小程序的开发者
当uniapp项目转为小程序项目的时候,预览会显示“您不是登录用户...”,所以没有办法发布,这个时候打开“project.config.json”设置一下appid,与你公众号appid一致就可以
Linux下修改mysql root密码
1.修改MySQL的配置文件(默认为/etc/my.cnf),在[mysqld]下添加一行skip-grant-tables 2.保存配置文件后,重启MySQL服务 service mysqld re ...
PHP 几种常用框架的区别
工作中使用的是TP框架,在工作之余接触了一下Yii框架和Laravel框架,为了更好的了解几种框架,此处总结一下几种框架各自的优势及缺点. 一.CI框架 CI框架是一个简单快速的PHP MVC轻量级框 ...

Spark2.0基于广播变量broadcast实现实时数据按天统计

Spark2.0基于广播变量broadcast实现实时数据按天统计的更多相关文章

随机推荐

热门专题