https://github.com/apache/spark/tree/master/core/src/main/scala/org/apache/spark/network

https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/network/BlockTransferService.scala

https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/network/BlockDataManager.scala

package org.apache.spark.network

import scala.reflect.ClassTag

import org.apache.spark.network.buffer.ManagedBuffer
import org.apache.spark.storage.{BlockId, StorageLevel} private[spark]
trait BlockDataManager { /**
* Interface to get local block data. Throws an exception if the block cannot be found or
* cannot be read successfully.
*/
def getBlockData(blockId: BlockId): ManagedBuffer /**
* Put the block locally, using the given storage level.
*
* Returns true if the block was stored and false if the put operation failed or the block
* already existed.
*/
def putBlockData(
blockId: BlockId,
data: ManagedBuffer,
level: StorageLevel,
classTag: ClassTag[_]): Boolean /**
* Release locks acquired by [[putBlockData()]] and [[getBlockData()]].
*/
def releaseLock(blockId: BlockId, taskAttemptId: Option[Long]): Unit
}
package org.apache.spark.network

import java.io.Closeable
import java.nio.ByteBuffer import scala.concurrent.{Future, Promise}
import scala.concurrent.duration.Duration
import scala.reflect.ClassTag import org.apache.spark.internal.Logging
import org.apache.spark.network.buffer.{FileSegmentManagedBuffer, ManagedBuffer, NioManagedBuffer}
import org.apache.spark.network.shuffle.{BlockFetchingListener, ShuffleClient, TempFileManager}
import org.apache.spark.storage.{BlockId, StorageLevel}
import org.apache.spark.util.ThreadUtils private[spark]
abstract class BlockTransferService extends ShuffleClient with Closeable with Logging { /**
* Initialize the transfer service by giving it the BlockDataManager that can be used to fetch
* local blocks or put local blocks.
*/
def init(blockDataManager: BlockDataManager): Unit /**
* Tear down the transfer service.
*/
def close(): Unit /**
* Port number the service is listening on, available only after [[init]] is invoked.
*/
def port: Int /**
* Host name the service is listening on, available only after [[init]] is invoked.
*/
def hostName: String /**
* Fetch a sequence of blocks from a remote node asynchronously,
* available only after [[init]] is invoked.
*
* Note that this API takes a sequence so the implementation can batch requests, and does not
* return a future so the underlying implementation can invoke onBlockFetchSuccess as soon as
* the data of a block is fetched, rather than waiting for all blocks to be fetched.
*/
override def fetchBlocks(
host: String,
port: Int,
execId: String,
blockIds: Array[String],
listener: BlockFetchingListener,
tempFileManager: TempFileManager): Unit /**
* Upload a single block to a remote node, available only after [[init]] is invoked.
*/
def uploadBlock(
hostname: String,
port: Int,
execId: String,
blockId: BlockId,
blockData: ManagedBuffer,
level: StorageLevel,
classTag: ClassTag[_]): Future[Unit] /**
* A special case of [[fetchBlocks]], as it fetches only one block and is blocking.
*
* It is also only available after [[init]] is invoked.
*/
def fetchBlockSync(
host: String,
port: Int,
execId: String,
blockId: String,
tempFileManager: TempFileManager): ManagedBuffer = {
// A monitor for the thread to wait on.
val result = Promise[ManagedBuffer]()
fetchBlocks(host, port, execId, Array(blockId),
new BlockFetchingListener {
override def onBlockFetchFailure(blockId: String, exception: Throwable): Unit = {
result.failure(exception)
}
override def onBlockFetchSuccess(blockId: String, data: ManagedBuffer): Unit = {
data match {
case f: FileSegmentManagedBuffer =>
result.success(f)
case _ =>
val ret = ByteBuffer.allocate(data.size.toInt)
ret.put(data.nioByteBuffer())
ret.flip()
result.success(new NioManagedBuffer(ret))
}
}
}, tempFileManager)
ThreadUtils.awaitResult(result.future, Duration.Inf)
} /**
* Upload a single block to a remote node, available only after [[init]] is invoked.
*
* This method is similar to [[uploadBlock]], except this one blocks the thread
* until the upload finishes.
*/
def uploadBlockSync(
hostname: String,
port: Int,
execId: String,
blockId: BlockId,
blockData: ManagedBuffer,
level: StorageLevel,
classTag: ClassTag[_]): Unit = {
val future = uploadBlock(hostname, port, execId, blockId, blockData, level, classTag)
ThreadUtils.awaitResult(future, Duration.Inf)
}
}

spark通信原理的更多相关文章

  1. Spark Shuffle 堆外内存溢出问题与解决(Shuffle通信原理)

    Spark Shuffle 堆外内存溢出问题与解决(Shuffle通信原理) http://xiguada.org/spark-shuffle-direct-buffer-oom/ 问题描述 Spar ...

  2. [Spark内核] 第32课:Spark Worker原理和源码剖析解密:Worker工作流程图、Worker启动Driver源码解密、Worker启动Executor源码解密等

    本課主題 Spark Worker 原理 Worker 启动 Driver 源码鉴赏 Worker 启动 Executor 源码鉴赏 Worker 与 Master 的交互关系 [引言部份:你希望读者 ...

  3. Spark核心技术原理透视一(Spark运行原理)

    在大数据领域,只有深挖数据科学领域,走在学术前沿,才能在底层算法和模型方面走在前面,从而占据领先地位. Spark的这种学术基因,使得它从一开始就在大数据领域建立了一定优势.无论是性能,还是方案的统一 ...

  4. spark核心原理

    spark运行结构图如下: spark基本概念 应用程序(application):用户编写的spark应用程序,包含驱动程序(Driver)和分布在集群中多个节点上运行的Executor代码,在执行 ...

  5. Spark集群基础概念 与 spark架构原理

    一.Spark集群基础概念 将DAG划分为多个stage阶段,遵循以下原则: 1.将尽可能多的窄依赖关系的RDD划为同一个stage阶段. 2.当遇到shuffle操作,就意味着上一个stage阶段结 ...

  6. Spark Worker原理和源码剖析解密:Worker工作流程图、Worker启动Driver源码解密、Worker启动Executor源码解密等

    本课主题 Spark Worker 原理 Worker 启动 Driver 源码鉴赏 Worker 启动 Executor 源码鉴赏 Worker 与 Master 的交互关系 Spark Worke ...

  7. 基于web的IM软件通信原理分析

    关于IM(InstantMessaging)即时通信类软件(如微信,QQ),大多数都是桌面应用程序或者native应用较为流行,而网上关于原生IM或桌面IM软件类的通信原理介绍也较多,此处不再赘述.而 ...

  8. Socket 通信原理(Android客户端和服务器以TCP&&UDP方式互通)

    转载地址:http://blog.csdn.net/mad1989/article/details/9147661 ZERO.前言 有关通信原理内容是在网上或百科整理得到,代码部分为本人所写,如果不当 ...

  9. SSL 通信原理及Tomcat SSL 配置

    SSL 通信原理及Tomcat SSL 双向配置 目录1 参考资料 .................................................................. ...

随机推荐

  1. offset家族

  2. 在LoadRunner中执行命令行程序之:popen()取代system()

    我想大家应该都知道在LoadRunner可以使用函数system()来调用系统指令,结果同在批处理里执行一样. 但是system()有个缺陷:无法获取命令的返回结果. 也许你可以用`echo comm ...

  3. POJ 2411 Mondriaan's Dream ——状压DP 插头DP

    [题目分析] 用1*2的牌铺满n*m的格子. 刚开始用到动规想写一个n*m*2^m,写了半天才知道会有重复的情况. So Sad. 然后想到数据范围这么小,爆搜好了.于是把每一种状态对应的转移都搜了出 ...

  4. vue项目实战, webpack 配置流程记录

    vue项目实战记录,地址在这 购物车单界面 npm install npm run dev 跑起来可以看到界面效果 这里简单记录一下webpack的编译流程 入口 package.json " ...

  5. 云计算与 OpenStack

    “云计算” 算是近年来最热的词了.现在 IT 行业见面不说这三个字您都不好意思跟人家打招呼. 对于云计算,学术界有各种定义,大家有兴趣可以百度一下. CloudMan 这里主要想从技术的角度谈谈对云计 ...

  6. Java面试题集(五)

    三.开源框架 什么是mybaties? Mybaties是支持普通sql查询,存储过程和高级映射的优秀持久层框架.Mybaties消除了几乎所有的jdbc代码和参数的手工设置以及结果集的检索.Myba ...

  7. R语言入门-画图(二)--heatmap

    一.函数参数: pheatmap参数: treeheight_row #横有多长 treeheight_col #竖有多长 cluster_cols=FLASE #单一方向聚类,也就是只有一边有树状结 ...

  8. laravel 数据库配置

    数据库配置文件为项目根目录下的config/database.php //默认数据库为mysql 'default' => env('DB_CONNECTION', 'mysql'), 'mys ...

  9. formSubmit

    精简代码: <form name='form0001' method="post"> .... <li id="view"><a ...

  10. grafana 安装 和 Nginx 、EL 联调

    https://blog.csdn.net/u010735147/article/details/80943593