https://github.com/apache/spark/tree/master/core/src/main/scala/org/apache/spark/network

https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/network/BlockTransferService.scala

https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/network/BlockDataManager.scala

package org.apache.spark.network

import scala.reflect.ClassTag

import org.apache.spark.network.buffer.ManagedBuffer
import org.apache.spark.storage.{BlockId, StorageLevel} private[spark]
trait BlockDataManager { /**
* Interface to get local block data. Throws an exception if the block cannot be found or
* cannot be read successfully.
*/
def getBlockData(blockId: BlockId): ManagedBuffer /**
* Put the block locally, using the given storage level.
*
* Returns true if the block was stored and false if the put operation failed or the block
* already existed.
*/
def putBlockData(
blockId: BlockId,
data: ManagedBuffer,
level: StorageLevel,
classTag: ClassTag[_]): Boolean /**
* Release locks acquired by [[putBlockData()]] and [[getBlockData()]].
*/
def releaseLock(blockId: BlockId, taskAttemptId: Option[Long]): Unit
}
package org.apache.spark.network

import java.io.Closeable
import java.nio.ByteBuffer import scala.concurrent.{Future, Promise}
import scala.concurrent.duration.Duration
import scala.reflect.ClassTag import org.apache.spark.internal.Logging
import org.apache.spark.network.buffer.{FileSegmentManagedBuffer, ManagedBuffer, NioManagedBuffer}
import org.apache.spark.network.shuffle.{BlockFetchingListener, ShuffleClient, TempFileManager}
import org.apache.spark.storage.{BlockId, StorageLevel}
import org.apache.spark.util.ThreadUtils private[spark]
abstract class BlockTransferService extends ShuffleClient with Closeable with Logging { /**
* Initialize the transfer service by giving it the BlockDataManager that can be used to fetch
* local blocks or put local blocks.
*/
def init(blockDataManager: BlockDataManager): Unit /**
* Tear down the transfer service.
*/
def close(): Unit /**
* Port number the service is listening on, available only after [[init]] is invoked.
*/
def port: Int /**
* Host name the service is listening on, available only after [[init]] is invoked.
*/
def hostName: String /**
* Fetch a sequence of blocks from a remote node asynchronously,
* available only after [[init]] is invoked.
*
* Note that this API takes a sequence so the implementation can batch requests, and does not
* return a future so the underlying implementation can invoke onBlockFetchSuccess as soon as
* the data of a block is fetched, rather than waiting for all blocks to be fetched.
*/
override def fetchBlocks(
host: String,
port: Int,
execId: String,
blockIds: Array[String],
listener: BlockFetchingListener,
tempFileManager: TempFileManager): Unit /**
* Upload a single block to a remote node, available only after [[init]] is invoked.
*/
def uploadBlock(
hostname: String,
port: Int,
execId: String,
blockId: BlockId,
blockData: ManagedBuffer,
level: StorageLevel,
classTag: ClassTag[_]): Future[Unit] /**
* A special case of [[fetchBlocks]], as it fetches only one block and is blocking.
*
* It is also only available after [[init]] is invoked.
*/
def fetchBlockSync(
host: String,
port: Int,
execId: String,
blockId: String,
tempFileManager: TempFileManager): ManagedBuffer = {
// A monitor for the thread to wait on.
val result = Promise[ManagedBuffer]()
fetchBlocks(host, port, execId, Array(blockId),
new BlockFetchingListener {
override def onBlockFetchFailure(blockId: String, exception: Throwable): Unit = {
result.failure(exception)
}
override def onBlockFetchSuccess(blockId: String, data: ManagedBuffer): Unit = {
data match {
case f: FileSegmentManagedBuffer =>
result.success(f)
case _ =>
val ret = ByteBuffer.allocate(data.size.toInt)
ret.put(data.nioByteBuffer())
ret.flip()
result.success(new NioManagedBuffer(ret))
}
}
}, tempFileManager)
ThreadUtils.awaitResult(result.future, Duration.Inf)
} /**
* Upload a single block to a remote node, available only after [[init]] is invoked.
*
* This method is similar to [[uploadBlock]], except this one blocks the thread
* until the upload finishes.
*/
def uploadBlockSync(
hostname: String,
port: Int,
execId: String,
blockId: BlockId,
blockData: ManagedBuffer,
level: StorageLevel,
classTag: ClassTag[_]): Unit = {
val future = uploadBlock(hostname, port, execId, blockId, blockData, level, classTag)
ThreadUtils.awaitResult(future, Duration.Inf)
}
}

spark通信原理的更多相关文章

  1. Spark Shuffle 堆外内存溢出问题与解决(Shuffle通信原理)

    Spark Shuffle 堆外内存溢出问题与解决(Shuffle通信原理) http://xiguada.org/spark-shuffle-direct-buffer-oom/ 问题描述 Spar ...

  2. [Spark内核] 第32课:Spark Worker原理和源码剖析解密:Worker工作流程图、Worker启动Driver源码解密、Worker启动Executor源码解密等

    本課主題 Spark Worker 原理 Worker 启动 Driver 源码鉴赏 Worker 启动 Executor 源码鉴赏 Worker 与 Master 的交互关系 [引言部份:你希望读者 ...

  3. Spark核心技术原理透视一(Spark运行原理)

    在大数据领域,只有深挖数据科学领域,走在学术前沿,才能在底层算法和模型方面走在前面,从而占据领先地位. Spark的这种学术基因,使得它从一开始就在大数据领域建立了一定优势.无论是性能,还是方案的统一 ...

  4. spark核心原理

    spark运行结构图如下: spark基本概念 应用程序(application):用户编写的spark应用程序,包含驱动程序(Driver)和分布在集群中多个节点上运行的Executor代码,在执行 ...

  5. Spark集群基础概念 与 spark架构原理

    一.Spark集群基础概念 将DAG划分为多个stage阶段,遵循以下原则: 1.将尽可能多的窄依赖关系的RDD划为同一个stage阶段. 2.当遇到shuffle操作,就意味着上一个stage阶段结 ...

  6. Spark Worker原理和源码剖析解密:Worker工作流程图、Worker启动Driver源码解密、Worker启动Executor源码解密等

    本课主题 Spark Worker 原理 Worker 启动 Driver 源码鉴赏 Worker 启动 Executor 源码鉴赏 Worker 与 Master 的交互关系 Spark Worke ...

  7. 基于web的IM软件通信原理分析

    关于IM(InstantMessaging)即时通信类软件(如微信,QQ),大多数都是桌面应用程序或者native应用较为流行,而网上关于原生IM或桌面IM软件类的通信原理介绍也较多,此处不再赘述.而 ...

  8. Socket 通信原理(Android客户端和服务器以TCP&&UDP方式互通)

    转载地址:http://blog.csdn.net/mad1989/article/details/9147661 ZERO.前言 有关通信原理内容是在网上或百科整理得到,代码部分为本人所写,如果不当 ...

  9. SSL 通信原理及Tomcat SSL 配置

    SSL 通信原理及Tomcat SSL 双向配置 目录1 参考资料 .................................................................. ...

随机推荐

  1. 【Luogu】P1948电话线(二分SPFA)

    题目链接 二分最长的电话线长度.把所有大于这个长度的边权设成1,小于等于的设成零,然后跑SPFA看dis[n]是否>k.若>k则l=mid+1 否则r=mid-1 放代码 #include ...

  2. Hibernate 笔记 HQL查询 条件查询,聚集函数,子查询,导航查询

    在hibernate中进行多表查询,每个表中各取几个字段,也就是说查询出来的结果集并没有一个实体类与之对应,如何解决这个问题? 解决方案一,按照Object[]数据取出数据,然后自己组bean 解决方 ...

  3. 【2018.9.20】JOI 2017 Final T3「JOIOI 王国 / The Kingdom of JOIOI」

    题目链接 题目描述 JOIOI 王国是一个 $H$ 行 $W$ 列的长方形网格,每个 $1\times 1$ 的子网格都是一个正方形的小区块.为了提高管理效率,我们决定把整个国家划分成两个省 $JOI ...

  4. 转 CListCtrl::InsertColumn、InsertItem、SetItemText;

    将数据写入到CListCtrl 向CListCtrl中写入数据,一般使用3个成员方法: CListCtrl::InsertColumn; CListCtrl::InsertItem; CListCtr ...

  5. Spoj-BGSHOOT

    The problem is about Mr.BG who is a great hunter. Today he has gone to a dense forest for hunting an ...

  6. BZOJ 2973 入门OJ4798 石头游戏

    矩阵递推 #include <iostream> #include <cstdio> #include <cstdlib> #include <cstring ...

  7. 洛谷P3393 逃离僵尸岛

    题目描述 小a住的国家被僵尸侵略了!小a打算逃离到该国唯一的国际空港逃出这个国家. 该国有N个城市,城市之间有道路相连.一共有M条双向道路.保证没有自环和重边. K个城市已经被僵尸控制了,如果贸然闯入 ...

  8. 漫话最小割 part1

    codeforces 724D [n个城市每个城市有一个特产的产出,一个特产的最大需求.当i<j时,城市i可以运最多C个特产到j.求所有城市可以满足最大的需求和] [如果直接最大流建图显然会T. ...

  9. Croc Champ 2013 - Round 2 C. Cube Problem

    问满足a^3 + b^3 + c^3 + n = (a+b+c)^3 的 (a,b,c)的个数 可化简为 n = 3*(a + b) (a + c) (b + c) 于是 n / 3 = (a + b ...

  10. JStorm学习

    一.简介 JStorm是一个分布式实时计算引擎.JStorm是一个类似于Hadoop MapReduce的系统,用户按照指定的接口实现一个任务,然后将这个任务交给JStorm系统,JStorm将这个任 ...