初始化

Task

List<InputGateDeploymentDescriptor> consumedPartitions = tdd.getInputGates();

// Consumed intermediate result partitions
this.inputGates = new SingleInputGate[consumedPartitions.size()];
this.inputGatesById = new HashMap<IntermediateDataSetID, SingleInputGate>(); for (int i = 0; i < this.inputGates.length; i++) {
SingleInputGate gate = SingleInputGate.create(
taskNameWithSubtaskAndId, jobId, executionId, consumedPartitions.get(i), networkEnvironment,
metricGroup.getIOMetricGroup()); this.inputGates[i] = gate;
inputGatesById.put(gate.getConsumedResultId(), gate);
}

 

初始化networkEnvironment

network.registerTask(this);

 

NetworkEnvironment

registerTask
// Setup the buffer pool for each buffer reader
final SingleInputGate[] inputGates = task.getAllInputGates(); for (SingleInputGate gate : inputGates) {
BufferPool bufferPool = null; try {
bufferPool = networkBufferPool.createBufferPool(gate.getNumberOfInputChannels(), false);
gate.setBufferPool(bufferPool);
}

 

SingleInputGate

create

/**
* Creates an input gate and all of its input channels.
*/
public static SingleInputGate create(
String owningTaskName,
JobID jobId,
ExecutionAttemptID executionId,
InputGateDeploymentDescriptor igdd,
NetworkEnvironment networkEnvironment,
IOMetricGroup metrics) { final IntermediateDataSetID consumedResultId = checkNotNull(igdd.getConsumedResultId()); final int consumedSubpartitionIndex = igdd.getConsumedSubpartitionIndex();
checkArgument(consumedSubpartitionIndex >= 0); final InputChannelDeploymentDescriptor[] icdd = checkNotNull(igdd.getInputChannelDeploymentDescriptors()); final SingleInputGate inputGate = new SingleInputGate( //生成SingleInputGate对象
owningTaskName, jobId, executionId, consumedResultId, consumedSubpartitionIndex,
icdd.length, networkEnvironment.getPartitionStateChecker(), metrics); // Create the input channels. There is one input channel for each consumed partition.
final InputChannel[] inputChannels = new InputChannel[icdd.length]; //初始化InputChannel for (int i = 0; i < inputChannels.length; i++) { final ResultPartitionID partitionId = icdd[i].getConsumedPartitionId();
final ResultPartitionLocation partitionLocation = icdd[i].getConsumedPartitionLocation(); if (partitionLocation.isLocal()) { //local
inputChannels[i] = new LocalInputChannel(inputGate, i, partitionId,
networkEnvironment.getPartitionManager(),
networkEnvironment.getTaskEventDispatcher(),
networkEnvironment.getPartitionRequestInitialAndMaxBackoff(),
metrics
);
}
else if (partitionLocation.isRemote()) { //remote
inputChannels[i] = new RemoteInputChannel(inputGate, i, partitionId,
partitionLocation.getConnectionId(),
networkEnvironment.getConnectionManager(),
networkEnvironment.getPartitionRequestInitialAndMaxBackoff(),
metrics
);
}
else if (partitionLocation.isUnknown()) {
inputChannels[i] = new UnknownInputChannel(inputGate, i, partitionId,
networkEnvironment.getPartitionManager(),
networkEnvironment.getTaskEventDispatcher(),
networkEnvironment.getConnectionManager(),
networkEnvironment.getPartitionRequestInitialAndMaxBackoff(),
metrics
);
}
else {
throw new IllegalStateException("Unexpected partition location.");
} inputGate.setInputChannel(partitionId.getPartitionId(), inputChannels[i]); //将inputChannel设置inputGate
} return inputGate;
}

inputGate的inputChannel,对应于resultPartition的resultSubPartition

 

------------------------------------------------------------------------------------------------------

OneInputStreamTask

if (numberOfInputs > 0) {
InputGate[] inputGates = getEnvironment().getAllInputGates();
inputProcessor = new StreamInputProcessor<IN>(inputGates, inSerializer,
getCheckpointBarrierListener(),
configuration.getCheckpointMode(),
getEnvironment().getIOManager(),
isSerializingTimestamps());

 

StreamInputProcessor

InputGate inputGate = InputGateUtil.createInputGate(inputGates);

if (checkpointMode == CheckpointingMode.EXACTLY_ONCE) {
this.barrierHandler = new BarrierBuffer(inputGate, ioManager);
}
else if (checkpointMode == CheckpointingMode.AT_LEAST_ONCE) {
this.barrierHandler = new BarrierTracker(inputGate);
}

StreamInputProcessor.processInput中

final BufferOrEvent bufferOrEvent = barrierHandler.getNextNonBlocked();
if (bufferOrEvent != null) {
if (bufferOrEvent.isBuffer()) {
currentChannel = bufferOrEvent.getChannelIndex();
currentRecordDeserializer = recordDeserializers[currentChannel]; //SpillingAdaptiveSpanningRecordDeserializer
currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer()); //将buffer set到SpillingAdaptiveSpanningRecordDeserializer
} //后续可以从set到SpillingAdaptiveSpanningRecordDeserializer中反序列化出record
DeserializationResult result = currentRecordDeserializer.getNextRecord(deserializationDelegate);
if (result.isFullRecord()) {
StreamElement recordOrWatermark = deserializationDelegate.getInstance();
final BufferOrEvent bufferOrEvent = barrierHandler.getNextNonBlocked()

 

BarrierBuffer

public BufferOrEvent getNextNonBlocked() throws IOException, InterruptedException {
while (true) {
// process buffered BufferOrEvents before grabbing new ones
BufferOrEvent next;
if (currentBuffered == null) { //如果currentBuffered为空,说明没有unblock的buffer数据,直接从inputGate读取
next = inputGate.getNextBufferOrEvent();
}

 

InputGateUtil.createInputGate

public static InputGate createInputGate(InputGate[] inputGates) {

    if (inputGates.length < 2) {
return inputGates[0];
} else {
return new UnionInputGate(inputGates);
}
}

 

UnionInputGate

/**
* Input gate wrapper to union the input from multiple input gates.
*
* <p> Each input gate has input channels attached from which it reads data. At each input gate, the
* input channels have unique IDs from 0 (inclusive) to the number of input channels (exclusive).
*
* <pre>
* +---+---+ +---+---+---+
* | 0 | 1 | | 0 | 1 | 2 |
* +--------------+--------------+
* | Input gate 0 | Input gate 1 |
* +--------------+--------------+
* </pre>
*
* The union input gate maps these IDs from 0 to the *total* number of input channels across all
* unioned input gates, e.g. the channels of input gate 0 keep their original indexes and the
* channel indexes of input gate 1 are set off by 2 to 2--4.
*
* <pre>
* +---+---++---+---+---+
* | 0 | 1 || 2 | 3 | 4 |
* +--------------------+
* | Union input gate |
* +--------------------+
* </pre>
*
* It is possible to recursively union union input gates.
*/
public class UnionInputGate implements InputGate { /** The input gates to union. */
private final InputGate[] inputGates; private final Set<InputGate> inputGatesWithRemainingData; //没有结束的inputGate /** Data availability listener across all unioned input gates. */
private final InputGateListener inputGateListener; /** The total number of input channels across all unioned input gates. */
private final int totalNumberOfInputChannels; //所有的inputGates的所有channels的数目 /**
* A mapping from input gate to (logical) channel index offset. Valid channel indexes go from 0
* (inclusive) to the total number of input channels (exclusive).
*/
private final Map<InputGate, Integer> inputGateToIndexOffsetMap; //每个inputGate的index的base,比如上面的gate1的base就是2 /** Flag indicating whether partitions have been requested. */
private boolean requestedPartitionsFlag; public UnionInputGate(InputGate... inputGates) { for (InputGate inputGate : inputGates) {
// The offset to use for buffer or event instances received from this input gate.
inputGateToIndexOffsetMap.put(checkNotNull(inputGate), currentNumberOfInputChannels); //当前InputChannels的总数就代表该inputGate的base
inputGatesWithRemainingData.add(inputGate); //加入inputGatesWithRemainingData,表示该inputGate没有结束 currentNumberOfInputChannels += inputGate.getNumberOfInputChannels(); //channel数累加
} this.totalNumberOfInputChannels = currentNumberOfInputChannels; this.inputGateListener = new InputGateListener(inputGates, this); //InputGateListener
}

将多个实际的inputGates,合成一个抽象的inputGate;这样做的目的是为了后面处理方便,把多个输入对后面透明化掉

 

那这样在BarrierBuffer,调用inputGate.getNextBufferOrEvent

其实就是调用,UnionInputGate.getNextBufferOrEvent

@Override
public BufferOrEvent getNextBufferOrEvent() throws IOException, InterruptedException { if (inputGatesWithRemainingData.isEmpty()) { //如果所有的inputgate都已经结束
return null;
} // Make sure to request the partitions, if they have not been requested before.
requestPartitions(); //从相应的resultpartition去request数据 final InputGate inputGate = inputGateListener.getNextInputGateToReadFrom(); //获取一个有数据的inputGate final BufferOrEvent bufferOrEvent = inputGate.getNextBufferOrEvent(); //真正的取数据,SingleInputGate.getNextBufferOrEvent if (bufferOrEvent.isEvent()
&& bufferOrEvent.getEvent().getClass() == EndOfPartitionEvent.class
&& inputGate.isFinished()) { //如果是结束event,则表示该inputGate已经结束 if (!inputGatesWithRemainingData.remove(inputGate)) { //从队列内删除
throw new IllegalStateException("Couldn't find input gate in set of remaining " +
"input gates.");
}
} // Set the channel index to identify the input channel (across all unioned input gates)
final int channelIndexOffset = inputGateToIndexOffsetMap.get(inputGate); //取得改inputgate的baseindex bufferOrEvent.setChannelIndex(channelIndexOffset + bufferOrEvent.getChannelIndex()); //baseindx + 真实的index = union index return bufferOrEvent;
}

 

InputGateListener

/**
* Data availability listener at all unioned input gates.
*
* <p> The listener registers itself at each input gate and is notified for *each incoming
* buffer* at one of the unioned input gates.
*/
private static class InputGateListener implements EventListener<InputGate> { private final UnionInputGate unionInputGate; private final BlockingQueue<InputGate> inputGatesWithData = new LinkedBlockingQueue<InputGate>(); //Cache所有有available buffer的inputGate @Override
public void onEvent(InputGate inputGate) { //SingleInputGate.onAvailableBuffer时被触发
// This method is called from the input channel thread, which can be either the same
// thread as the consuming task thread or a different one.
inputGatesWithData.add(inputGate); //将inputGate加入队列,等待读取 for (int i = 0; i < registeredListeners.size(); i++) {
registeredListeners.get(i).onEvent(unionInputGate);
}
} InputGate getNextInputGateToReadFrom() throws InterruptedException { //从队列头取一个inputGate
return inputGatesWithData.take();
}

 

先看下requestPartitions,如何request resultpartition的?

public void requestPartitions() throws IOException, InterruptedException {
if (!requestedPartitionsFlag) {//只需要做一次
for (InputGate inputGate : inputGates) {
inputGate.requestPartitions();
} requestedPartitionsFlag = true;
}
}

 

SingleInputGate.requestPartitions

public void requestPartitions() throws IOException, InterruptedException {
synchronized (requestLock) {
if (!requestedPartitionsFlag) { //只做一次 for (InputChannel inputChannel : inputChannels.values()) {
inputChannel.requestSubpartition(consumedSubpartitionIndex); //调用inputChannel.requestSubpartition
}
} requestedPartitionsFlag = true;
}
}

 

RemoteInputChannel

@Override
void requestSubpartition(int subpartitionIndex) throws IOException, InterruptedException {
if (partitionRequestClient == null) {
// Create a client and request the partition
partitionRequestClient = connectionManager
.createPartitionRequestClient(connectionId); partitionRequestClient.requestSubpartition(partitionId, subpartitionIndex, this, 0);
}
}

PartitionRequestClient,先创建,这个负责和resultSubPartition通信

requestSubpartition

public ChannelFuture requestSubpartition(
final ResultPartitionID partitionId,
final int subpartitionIndex,
final RemoteInputChannel inputChannel,
int delayMs) throws IOException { partitionRequestHandler.addInputChannel(inputChannel); //将inputChannel加入partitionRequestHandler final PartitionRequest request = new PartitionRequest( //生成request
partitionId, subpartitionIndex, inputChannel.getInputChannelId()); if (delayMs == 0) {
ChannelFuture f = tcpChannel.writeAndFlush(request); //发送request
f.addListener(listener);
return f;
}
else {
final ChannelFuture[] f = new ChannelFuture[1];
tcpChannel.eventLoop().schedule(new Runnable() {
@Override
public void run() {
f[0] = tcpChannel.writeAndFlush(request);
f[0].addListener(listener);
}
}, delayMs, TimeUnit.MILLISECONDS); return f[0];
}
}

 

PartitionRequestClientHandler

@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
try {
if (!bufferListener.hasStagedBufferOrEvent() && stagedMessages.isEmpty()) { //普遍msg
decodeMsg(msg);
}
else {
stagedMessages.add(msg);
}
}
catch (Throwable t) {
notifyAllChannelsOfErrorAndClose(t);
}
}

decodeMsg

private boolean decodeMsg(Object msg) throws Throwable {
final Class<?> msgClazz = msg.getClass(); // ---- Buffer --------------------------------------------------------
if (msgClazz == NettyMessage.BufferResponse.class) {
NettyMessage.BufferResponse bufferOrEvent = (NettyMessage.BufferResponse) msg; RemoteInputChannel inputChannel = inputChannels.get(bufferOrEvent.receiverId); //获取对应的inputChannel return decodeBufferOrEvent(inputChannel, bufferOrEvent);
}

 

decodeBufferOrEvent

private boolean decodeBufferOrEvent(RemoteInputChannel inputChannel, NettyMessage.BufferResponse bufferOrEvent) throws Throwable {
boolean releaseNettyBuffer = true; try {
if (bufferOrEvent.isBuffer()) {
// ---- Buffer ------------------------------------------------
BufferProvider bufferProvider = inputChannel.getBufferProvider(); while (true) {
Buffer buffer = bufferProvider.requestBuffer(); //从channel的bufferProvider中获取buffer if (buffer != null) {
buffer.setSize(bufferOrEvent.getSize());
bufferOrEvent.getNettyBuffer().readBytes(buffer.getNioBuffer()); //将数据写入buffer中 inputChannel.onBuffer(buffer, bufferOrEvent.sequenceNumber); //调用inputChannel.onBuffer return true;
}
else if (bufferListener.waitForBuffer(bufferProvider, bufferOrEvent)) {
releaseNettyBuffer = false; return false;
}
else if (bufferProvider.isDestroyed()) {
return false;
}
}
}
}

 

RemoteInputChannel

public void onBuffer(Buffer buffer, int sequenceNumber) {
boolean success = false; try {
synchronized (receivedBuffers) {
if (!isReleased.get()) {
if (expectedSequenceNumber == sequenceNumber) {
receivedBuffers.add(buffer); //将buffer放入receivedBuffers
expectedSequenceNumber++; notifyAvailableBuffer();//通知有available buffer success = true;
}
}
}
}
}

notifyAvailableBuffer

protected void notifyAvailableBuffer() {
inputGate.onAvailableBuffer(this);
}

 

SingleInputGate

public void onAvailableBuffer(InputChannel channel) {
inputChannelsWithData.add(channel); //inputChannelsWithData中表示该channel有数据需要读
EventListener<InputGate> listener = registeredListener;
if (listener != null) {
listener.onEvent(this); //通知UnionInputGate,该inputGate有data需要读
}
}

 

---------------------------------------------------

SingleInputGate.getNextBufferOrEvent

@Override
public BufferOrEvent getNextBufferOrEvent() throws IOException, InterruptedException { requestPartitions(); InputChannel currentChannel = null;
while (currentChannel == null) { //如果没有有数据的channel,会循环blocking
currentChannel = inputChannelsWithData.poll(2, TimeUnit.SECONDS); //从inputChannelsWithData poll一个有数据的channel
} final Buffer buffer = currentChannel.getNextBuffer(); //读出buffer if (buffer.isBuffer()) {
return new BufferOrEvent(buffer, currentChannel.getChannelIndex());
}
else {
final AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader()); if (event.getClass() == EndOfPartitionEvent.class) {
channelsWithEndOfPartitionEvents.set(currentChannel.getChannelIndex()); if (channelsWithEndOfPartitionEvents.cardinality() == numberOfInputChannels) {
hasReceivedAllEndOfPartitionEvents = true;
} currentChannel.notifySubpartitionConsumed(); currentChannel.releaseAllResources();
} return new BufferOrEvent(event, currentChannel.getChannelIndex());
}
}

 

RemoteInputChannel

Buffer getNextBuffer() throws IOException {
synchronized (receivedBuffers) {
Buffer buffer = receivedBuffers.poll(); numBytesIn.inc(buffer.getSize());
return buffer;
}
}

 

 

 

 

 

 

												

Flink - InputGate的更多相关文章

  1. Flink Internals

    https://cwiki.apache.org/confluence/display/FLINK/Flink+Internals   Memory Management (Batch API) In ...

  2. 追源索骥:透过源码看懂Flink核心框架的执行流程

    li,ol.inline>li{display:inline-block;padding-right:5px;padding-left:5px}dl{margin-bottom:20px}dt, ...

  3. Flink的TaskManager启动(源码分析)

    通过启动脚本已经找到了TaskManager 的启动类org.apache.flink.runtime.taskexecutor.TaskManagerRunner 来看一下它的main方法中 最后被 ...

  4. Flink的Job启动TaskManager端(源码分析)

    前面说到了  Flink的JobManager启动(源码分析)  启动了TaskManager 然后  Flink的Job启动JobManager端(源码分析)  说到JobManager会将转化得到 ...

  5. Flink中接收端反压以及Credit机制 (源码分析)

    先上一张图整体了解Flink中的反压   可以看到每个task都会有自己对应的IG(inputgate)对接上游发送过来的数据和RS(resultPatation)对接往下游发送数据, 整个反压机制通 ...

  6. Flink task之间的数据交换

    Flink中的数据交换是围绕着下面的原则设计的: 1.数据交换的控制流(即,为了启动交换而传递的消息)是由接收者发起的,就像原始的MapReduce一样. 2.用于数据交换的数据流,即通过电缆的实际数 ...

  7. Apache Flink - 架构和拓扑

    Flink结构: flink cli 解析本地环境配置,启动 ApplicationMaster 在 ApplicationMaster 中启动 JobManager 在 ApplicationMas ...

  8. 【转帖】两年Flink迁移之路:从standalone到on yarn,处理能力提升五倍

    两年Flink迁移之路:从standalone到on yarn,处理能力提升五倍 https://segmentfault.com/a/1190000020209179 flink 1.7k 次阅读 ...

  9. Apache Flink 的迁移之路,2 年处理效果提升 5 倍

    一.背景与痛点 在 2017 年上半年以前,TalkingData 的 App Analytics 和 Game Analytics 两个产品,流式框架使用的是自研的 td-etl-framework ...

随机推荐

  1. 不同局域网中同一IP地址的计算机怎么通信的

    1.IP地址在192.--.255之内的是私有地址,即192.168.1.56的电脑a是不能直接与192.168.1.56的电脑b进行通信的.他们需要用到NAT技术,即网络地址转换.2.NAT的作用是 ...

  2. [Big Data - Kafka] Kafka设计解析(三):Kafka High Availability (下)

    Kafka是由LinkedIn开发的一个分布式的消息系统,使用Scala编写,它以可水平扩展和高吞吐率而被广泛使用.目前越来越多的开源分布式处理系统如Cloudera.Apache Storm.Spa ...

  3. Asp.Net 导入Excel自动获取表名

    public static DataSet ReadExcel(string Path, string fileType) { //服务器需要安装驱动 //http://download.micros ...

  4. 基于Docker的redis集群搭建

    Redis集群官方介绍:http://www.redis.cn/topics/cluster-tutorial.html 基于Docker搭建Redis集群 环境:6个节点,三主三从 制作Redis镜 ...

  5. java 实现websocket

    最近了解了下websocket和socket这个东西,说不得不来说下为何要使用 WebSocket ,和为何不用http. 为何需要WebSocket ? HTTP 协议是一种无状态的.无连接的.单向 ...

  6. Go_14:GoLang中 json、map、struct 之间的相互转化

    1. golang 中 json 转 struct <1. 使用 json.Unmarshal 时,结构体的每一项必须是导出项(import field).也就是说结构体的 key 对应的首字母 ...

  7. android 中的一些资源注解,让编译器帮你检查代码

    android 中的一些资源注解,让编译器帮你检查代码 写方便的时候可以用注解来声明一些参数,以明确的指示参数的类型,让代码更安全.我们看到,在android源代码里大量使用了注解.我整理了一些注解如 ...

  8. 设计模式-行为型模式,python 中介者模式

    中介者模式 中介者模式(Mediator Pattern)是用来降低多个对象和类之间的通信复杂性.这种模式提供了一个中介类,该类通常处理不同类之间的通信,并支持松耦合,使代码易于维护.中介者模式属于行 ...

  9. [Algorithm] Polynomial and FFT

    排序:nlogn 二分查找:logn <-- 利用单调性,查n次,每次logn Multiply the following pairs of polynomials using at most ...

  10. WPF自定义路由事件(二)

    WPF中的路由事件 as U know,和以前Windows消息事件区别不再多讲,这篇博文中,将首先回顾下WPF内置的路由事件的用法,然后在此基础上自定义一个路由事件. 1.WPF内置路由事件 WPF ...