/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.table.runtime.aggregate import java.lang.Iterable import org.apache.flink.api.common.functions.{MapPartitionFunction, RichGroupReduceFunction}
import org.apache.flink.configuration.Configuration
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.Collector /**
* [[RichGroupReduceFunction]] and [[MapPartitionFunction]] to compute aggregates that do
* not support pre-aggregation for batch(DataSet) queries.
*
* @param genAggregations Code-generated [[GeneratedAggregations]]
*/
class DataSetAggFunction(
private val genAggregations: GeneratedAggregationsFunction)
extends RichGroupReduceFunction[Row, Row]
with MapPartitionFunction[Row, Row]
with Compiler[GeneratedAggregations] with Logging { private var output: Row = _
private var accumulators: Row = _ private var function: GeneratedAggregations = _ override def open(config: Configuration) {
LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
s"Code:\n$genAggregations.code")
val clazz = compile(
getRuntimeContext.getUserCodeClassLoader,
genAggregations.name,
genAggregations.code)
LOG.debug("Instantiating AggregateHelper.")
function = clazz.newInstance() output = function.createOutputRow()
accumulators = function.createAccumulators()
} /**
* Computes a non-pre-aggregated aggregation.
*
* @param records An iterator over all records of the group.
* @param out The collector to hand results to.
*/
override def reduce(records: Iterable[Row], out: Collector[Row]): Unit = { // reset accumulators
function.resetAccumulator(accumulators) val iterator = records.iterator() var record: Row = null
while (iterator.hasNext) {
record = iterator.next() // accumulate
function.accumulate(accumulators, record)
} // set group keys value to final output
function.setForwardedFields(record, output) // set agg results to output
function.setAggregationResults(accumulators, output) out.collect(output)
} /**
* Computes a non-pre-aggregated aggregation and returns a row even if the input is empty.
*
* @param records An iterator over all records of the partition.
* @param out The collector to hand results to.
*/
override def mapPartition(records: Iterable[Row], out: Collector[Row]): Unit = {
reduce(records, out)
} } 、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、、
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.table.runtime.aggregate import java.lang.Iterable import org.apache.flink.api.common.functions.RichGroupReduceFunction
import org.apache.flink.configuration.Configuration
import org.apache.flink.table.codegen.{Compiler, GeneratedAggregationsFunction}
import org.apache.flink.table.util.Logging
import org.apache.flink.types.Row
import org.apache.flink.util.Collector /**
* It wraps the aggregate logic inside of
* [[org.apache.flink.api.java.operators.GroupReduceOperator]]. It is used for tumbling time-window
* on batch.
*
* @param genAggregations Code-generated [[GeneratedAggregations]]
* @param windowSize Tumbling time window size
* @param windowStartPos The relative window-start field position to the last field of output row
* @param windowEndPos The relative window-end field position to the last field of output row
* @param windowRowtimePos The relative window-rowtime field position to the last field of
* output row
* @param keysAndAggregatesArity The total arity of keys and aggregates
*/
class DataSetTumbleTimeWindowAggReduceGroupFunction(
genAggregations: GeneratedAggregationsFunction,
windowSize: Long,
windowStartPos: Option[Int],
windowEndPos: Option[Int],
windowRowtimePos: Option[Int],
keysAndAggregatesArity: Int)
extends RichGroupReduceFunction[Row, Row]
with Compiler[GeneratedAggregations]
with Logging { private var collector: DataSetTimeWindowPropertyCollector = _
protected var aggregateBuffer: Row = new Row(keysAndAggregatesArity + 1) private var output: Row = _
protected var accumulators: Row = _ protected var function: GeneratedAggregations = _ override def open(config: Configuration) {
LOG.debug(s"Compiling AggregateHelper: $genAggregations.name \n\n " +
s"Code:\n$genAggregations.code")
val clazz = compile(
getRuntimeContext.getUserCodeClassLoader,
genAggregations.name,
genAggregations.code)
LOG.debug("Instantiating AggregateHelper.")
function = clazz.newInstance() output = function.createOutputRow()
accumulators = function.createAccumulators()
collector = new DataSetTimeWindowPropertyCollector(
windowStartPos,
windowEndPos,
windowRowtimePos)
} override def reduce(records: Iterable[Row], out: Collector[Row]): Unit = { var last: Row = null
val iterator = records.iterator() // reset accumulator
function.resetAccumulator(accumulators) while (iterator.hasNext) {
val record = iterator.next()
function.mergeAccumulatorsPair(accumulators, record)
last = record
} // set group keys value to final output.
function.setForwardedFields(last, output) // get final aggregate value and set to output.
function.setAggregationResults(accumulators, output) // get window start timestamp
val startTs: Long = last.getField(keysAndAggregatesArity).asInstanceOf[Long] // set collector and window
collector.wrappedCollector = out
collector.windowStart = startTs
collector.windowEnd = startTs + windowSize collector.collect(output)
} }

window函数 resetAccumulator的更多相关文章

  1. DStream-04 Window函数的原理和源码

    DStream 中 window 函数有两种,一种是普通 WindowedDStream,另外一种是针对 window聚合 优化的 ReducedWindowedDStream. Demo objec ...

  2. MySQL 对window函数执行sum函数疑似Bug

    MySQL 对window函数执行sum函数疑似Bug 使用MySql的窗口函数统计数据时,发现一个小的问题,与大家一起探讨下. 环境配置: mysql-installer-community-8.0 ...

  3. 使用streaming window函数统计用户不同时间段平均消费金额等指标

    场景 现在餐厅老板已经不满足仅仅统计历史用户消费金额总数了,他想知道每个用户半年,每个月,每天,或者一小时消费的总额,来店消费的次数以及平均金额. 给出的例子计算的是每5秒,每30秒,每1分钟的用户消 ...

  4. javascript中的this与函数讲解

    前言 javascript中没有块级作用域(es6以前),javascript中作用域分为函数作用域和全局作用域.并且,大家可以认为全局作用域其实就是Window函数的函数作用域,我们编写的js代码, ...

  5. avascript中的this与函数讲解

    徐某某 一个半路出家的野生程序员 javascript中的this与函数讲解 前言 javascript中没有块级作用域(es6以前),javascript中作用域分为函数作用域和全局作用域.并且,大 ...

  6. javascript篇-----函数作用域,函数作用域链和声明提前

    在一些类似C语言的编程语言中,花括号内的每一段代码都具有各自的作用域,而且变量在声明它们的代码段之外是不可见的(也就是我们不能在代码段外直接访问代码段内声明的变量),我们称之为块级作用域,然而,不同于 ...

  7. javascript 函数声明与函数表达式的区别

    先看一段代码 var f = function g() { return 1; }; if (false) { f = function g(){ return 2; }; } alert(g()); ...

  8. javascript + jquery函数大全

    JAVASCRIPT Array 函数   array创建数组 concat()连接两个或更多的数组,并返回结果. join()把数组中所有元素组成字符串. pop()删除并返回数组的最后一个元素 s ...

  9. JavaScript window

    window -- window对象是BOM中所有对象的核心 window,中文"窗口" window对象除了是BOM中所有对象的父对象外,还包含一些窗口控制函数 全局的windo ...

随机推荐

  1. Windows操作系统安全加固

    本文档旨在指导系统管理人员或安全检查人员进行Windows操作系统的安全合规性检查和配置. 1. 账户管理和认证授权 1.1 账户 默认账户安全 禁用Guest账户. 禁用或删除其他无用账户(建议先禁 ...

  2. java正则表达式去除html中所有的标签和特殊HTML字符(以&开头的)

    来源于:https://www.androiddev.net/java%E6%AD%A3%E5%88%99%E8%A1%A8%E8%BE%BE%E5%BC%8F%E5%8E%BB%E9%99%A4ht ...

  3. wordpress搭建博客上传begin主题The themes is locked to another domain

    如题, 在使用wordpress搭建个人博客过程中, 上传begin主题, 出现如下弹框的错误, 而且样式有误. 环境: Los Angeles CentOS 7 x64 nginx+mysql 解决 ...

  4. lua api 官方文档 函数后面的方括号 说明 [-0, +0, –]

    本博客注有“转”字样的为转载文章,其余为本人原创文章,转载请务必注明出处或保存此段.c++/lua/windows逆向交流群:69148232 每一个lua api 函数结尾都给出了这样的说明例如: ...

  5. ios实例开发精品文章推荐(7.23)

    ---------------------------------------------------------------------------------------------------- ...

  6. TensorFlow 基本概念

    一.概述 使用图(graph)来表示计算任务 在会话(Session)的上下文(context)中执行图(graph) 使用tensor表示数据 通过 变量(Variable)维护状态 使用 feed ...

  7. 【Spring】Spring之浅析Spring框架的搭建

    Spring是什么 Spring是一个开源的容器框架,用于配置bean并且维护bean之间关系的.其主要的思想就是IOC(Inversion Of Control,控制反转)或者称作DI(Depend ...

  8. 什么是 SUID, SGID 和 Sticky bit

    在可执行文件中有三种权限,如下: 1. SUID 权限 (Set-user Identification) 2. SGID 权限(Set-group identification) 3. Sticky ...

  9. list、map、数组 转换

    list,set,map,数组间的相互转换1.list转setSet set = new HashSet(new ArrayList()); 2.set转listList list = new Arr ...

  10. [转]使用自定义HttpMessageConverter对返回内容进行加密

    今天上午技术群里的一个人问” 如何在 Spring MVC 中统一对返回的 Json 进行加密?”. 大部分人的第一反应是通过 Spring 拦截器(Interceptor)中的postHandler ...