org.apache.spark.SparkException: Could not find CoarseGrainedScheduler or it has been stopped.

        at org.apache.spark.rpc.netty.Dispatcher.postMessage(Dispatcher.scala:163)

        at org.apache.spark.rpc.netty.Dispatcher.postOneWayMessage(Dispatcher.scala:133)

        at org.apache.spark.rpc.netty.NettyRpcEnv.send(NettyRpcEnv.scala:192)

        at org.apache.spark.rpc.netty.NettyRpcEndpointRef.send(NettyRpcEnv.scala:516)

        at org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend.reviveOffers(CoarseGrainedSchedulerBackend.scala:356)

        at org.apache.spark.scheduler.TaskSchedulerImpl.executorLost(TaskSchedulerImpl.scala:494)

        at org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend$DriverEndpoint.disableExecutor(CoarseGrainedSchedulerBackend.scala:301)

        at org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnDriverEndpoint$$anonfun$onDisconnected$1.apply(YarnSchedulerBackend.scala:121)

        at org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnDriverEndpoint$$anonfun$onDisconnected$1.apply(YarnSchedulerBackend.scala:120)

        at scala.Option.foreach(Option.scala:236)

        at org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnDriverEndpoint.onDisconnected(YarnSchedulerBackend.scala:120)

        at org.apache.spark.rpc.netty.Inbox$$anonfun$process$1.apply$mcV$sp(Inbox.scala:142)

        at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:204)

        at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:100)

        at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:217)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)

        at java.lang.Thread.run(Thread.java:745)

18/10/14 22:23:26 ERROR netty.Inbox: Ignoring error

org.apache.spark.SparkException: Could not find CoarseGrainedScheduler or it has been stopped.

        at org.apache.spark.rpc.netty.Dispatcher.postMessage(Dispatcher.scala:163)

        at org.apache.spark.rpc.netty.Dispatcher.postOneWayMessage(Dispatcher.scala:133)

        at org.apache.spark.rpc.netty.NettyRpcEnv.send(NettyRpcEnv.scala:192)

        at org.apache.spark.rpc.netty.NettyRpcEndpointRef.send(NettyRpcEnv.scala:516)

        at org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend.reviveOffers(CoarseGrainedSchedulerBackend.scala:356)

        at org.apache.spark.scheduler.TaskSchedulerImpl.executorLost(TaskSchedulerImpl.scala:494)

        at org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend$DriverEndpoint.disableExecutor(CoarseGrainedSchedulerBackend.scala:301)

        at org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnDriverEndpoint$$anonfun$onDisconnected$1.apply(YarnSchedulerBackend.scala:121)

        at org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnDriverEndpoint$$anonfun$onDisconnected$1.apply(YarnSchedulerBackend.scala:120)

        at scala.Option.foreach(Option.scala:236)

        at org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnDriverEndpoint.onDisconnected(YarnSchedulerBackend.scala:120)

        at org.apache.spark.rpc.netty.Inbox$$anonfun$process$1.apply$mcV$sp(Inbox.scala:142)

        at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:204)

        at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:100)

        at org.apache.spark.rpc.netty.Dispatcher$MessageLoop.run(Dispatcher.scala:217)

        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)

        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)

        at java.lang.Thread.run(Thread.java:745)

......

java.util.concurrent.RejectedExecutionException: Task scala.concurrent.impl.CallbackRunnable@708dfce7 rejected from java.util.concurrent.ThreadPoolExecutor@346be0ef[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 224]

        at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)

        at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)

        at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)

        at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:122)

        at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40)

        at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248)

        at scala.concurrent.Promise$class.complete(Promise.scala:55)

        at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:153)

        at scala.concurrent.Future$$anonfun$recover$1.apply(Future.scala:324)

        at scala.concurrent.Future$$anonfun$recover$1.apply(Future.scala:324)

        at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32)

        at org.spark-project.guava.util.concurrent.MoreExecutors$SameThreadExecutorService.execute(MoreExecutors.java:293)

        at scala.concurrent.impl.ExecutionContextImpl$$anon$1.execute(ExecutionContextImpl.scala:133)

        at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40)

        at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248)

        at scala.concurrent.Promise$class.complete(Promise.scala:55)

        at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:153)

        at scala.concurrent.Future$$anonfun$map$1.apply(Future.scala:235)

        at scala.concurrent.Future$$anonfun$map$1.apply(Future.scala:235)

        at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32)

        at scala.concurrent.Future$InternalCallbackExecutor$Batch$$anonfun$run$1.processBatch$1(Future.scala:643)

        at scala.concurrent.Future$InternalCallbackExecutor$Batch$$anonfun$run$1.apply$mcV$sp(Future.scala:658)

        at scala.concurrent.Future$InternalCallbackExecutor$Batch$$anonfun$run$1.apply(Future.scala:635)

        at scala.concurrent.Future$InternalCallbackExecutor$Batch$$anonfun$run$1.apply(Future.scala:635)

        at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72)

        at scala.concurrent.Future$InternalCallbackExecutor$Batch.run(Future.scala:634)

        at scala.concurrent.Future$InternalCallbackExecutor$.scala$concurrent$Future$InternalCallbackExecutor$$unbatchedExecute(Future.scala:694)

        at scala.concurrent.Future$InternalCallbackExecutor$.execute(Future.scala:685)

        at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40)

        at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248)

        at scala.concurrent.Promise$class.tryFailure(Promise.scala:112)

        at scala.concurrent.impl.Promise$DefaultPromise.tryFailure(Promise.scala:153)

        at org.apache.spark.rpc.netty.NettyRpcEnv.org$apache$spark$rpc$netty$NettyRpcEnv$$onFailure$1(NettyRpcEnv.scala:208)

        at org.apache.spark.rpc.netty.NettyRpcEnv$$anonfun$2.apply(NettyRpcEnv.scala:230)

        at org.apache.spark.rpc.netty.NettyRpcEnv$$anonfun$2.apply(NettyRpcEnv.scala:230)

        at org.apache.spark.rpc.netty.RpcOutboxMessage.onFailure(Outbox.scala:71)

        at org.apache.spark.network.client.TransportResponseHandler.failOutstandingRequests(TransportResponseHandler.java:110)

        at org.apache.spark.network.client.TransportResponseHandler.channelUnregistered(TransportResponseHandler.java:124)

        at org.apache.spark.network.server.TransportChannelHandler.channelUnregistered(TransportChannelHandler.java:94)

        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:158)

        at io.netty.channel.AbstractChannelHandlerContext.fireChannelUnregistered(AbstractChannelHandlerContext.java:144)

        at io.netty.channel.ChannelInboundHandlerAdapter.channelUnregistered(ChannelInboundHandlerAdapter.java:53)

        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:158)

        at io.netty.channel.AbstractChannelHandlerContext.fireChannelUnregistered(AbstractChannelHandlerContext.java:144)

        at io.netty.channel.ChannelInboundHandlerAdapter.channelUnregistered(ChannelInboundHandlerAdapter.java:53)

        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:158)

        at io.netty.channel.AbstractChannelHandlerContext.fireChannelUnregistered(AbstractChannelHandlerContext.java:144)

        at io.netty.channel.ChannelInboundHandlerAdapter.channelUnregistered(ChannelInboundHandlerAdapter.java:53)

        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:158)

        at io.netty.channel.AbstractChannelHandlerContext.fireChannelUnregistered(AbstractChannelHandlerContext.java:144)

        at io.netty.channel.DefaultChannelPipeline.fireChannelUnregistered(DefaultChannelPipeline.java:739)

        at io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:659)

        at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:328)

        at io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:627)

        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:362)

解决办法：

  1. 在提交代码的时候添加配置spark-submit  ...  –conf spark.dynamicAllocation.enabled=false
  2. 也可以在代码中通过SparkConf设置：conf.set(“spark.dynamicAllocation.enabled”,”false”)

SparkException: Could not find CoarseGrainedScheduler or it has been stopped.的更多相关文章

  1. org.apache.spark.rpc.RpcTimeout$$anonfun$1.applyOrElse

    跑sparkPis示例程序 [root@node01 bin]# ./spark-submit --master spark://node01:7077 --class org.apache.spar ...

  2. spark streaming 1: SparkContex

    StreamingContext 和SparkContex的用途是差不多的,作为spark stream的入口,提供配置.生成DStream等功能. 总体来看,spark stream包括如下模块: ...

  3. 【原创】大叔问题定位分享（10）提交spark任务偶尔报错 org.apache.spark.SparkException: A master URL must be set in your configuration

    spark 2.1.1 一 问题重现 问题代码示例 object MethodPositionTest { val sparkConf = new SparkConf().setAppName(&qu ...

  4. spark出现task不能序列化错误的解决方法 org.apache.spark.SparkException: Task not serializable

    import org.elasticsearch.cluster.routing.Murmur3HashFunction; import org.elasticsearch.common.math.M ...

  5. Spark运行程序异常信息： org.apache.spark.SparkException: Task not serializable 解决办法

    错误信息: 17/05/20 18:51:39 ERROR JobScheduler: Error running job streaming job 1495277499000 ms.0 org.a ...

  6. SparkException: Master removed our application

    come from https://stackoverflow.com/questions/32245498/sparkexception-master-removed-our-application ...

  7. 错误：Caused by:org.apache.spark.SparkException: Kryo serialization failed: Buffer overflow.Available: 0, required: 21. To avoid this,

    这个是写入Redis时用的序列化器,然后错误提示是超过了大小限制,把配置调大即可. .set("spark.kryoserializer.buffer.max","128 ...

  8. Spark On YARN内存分配

    本文转自:http://blog.javachen.com/2015/06/09/memory-in-spark-on-yarn.html?utm_source=tuicool 此文解决了Spark ...

  9. spark on yarn 内存分配

    Spark On YARN内存分配 本文主要了解Spark On YARN部署模式下的内存分配情况,因为没有深入研究Spark的源代码,所以只能根据日志去看相关的源代码,从而了解“为什么会这样,为什么 ...

随机推荐

  1. Confluence 6 用户目录图例 - 使用 LDAP 授权，在用户第一次登陆时拷贝用户

    上面的图:Confluence 连接到一个 LDAP 目录只用作授权,当用户登录 Confluence 的时候,使用 LDAP 授权并且将用户信息同步到本地路服务器上. https://www.cwi ...

  2. selenium之 chromedriver与chrome版本映射表（更新至v2.33）

    看到网上基本没有最新的chromedriver与chrome的对应关系表,便兴起整理了一份如下,希望对大家有用: chromedriver版本 支持的Chrome版本 v2.33 v60-62 v2. ...

  3. laravel 不理解的call方法

    返回结果: 原来是调用同控制器的这四个方法之一...vendor\zhiyicx\plus-question\src\API2\Controllers\UserQuestionController.p ...

  4. 基于Web的漏洞利用

    1.Nikto 基于Web的漏洞信息扫描 nikto 自动扫描web服务器上没有打补丁的软件,同时同时也检测驻留在服务器上的危险文件,nikto能够识别出特定的问题,检测服务器的配置问题, 检测某台主 ...

  5. Software tips

    1.Microsoft office professional plus 2013秘钥 PD3W7-NVGG7-YKGQX-7CRDG-J2MG7(test success)MTGRC-8BM6H-W ...

  6. MySQL5.7.11版本，报错Cannot proceed because system tables used by Event Scheduler were found damaged at server start

    解决思路: 1. 在MySQL安装目录下执行./mysql_upgrade -uroot -p,此处是为了更新MySQL的系统表,在5.6之前的版本上,更新系统表的命令是mysql_fix_privi ...

  7. 卸载列表信息——Uninstall注册表

    今天用InstallShield打包了一个安装程序,安装顺利完成了,但是当我去控制面板准备卸载时,发现我的程序没有详细的信息,正常的软件信息如下图: 而我的程序没有发布者,大小和版本,也没有图标,于是 ...

  8. 复习reactnative....

    import React, { Component } from 'react'; import { AppRegistry, Text, Image, View, TextInput, Scroll ...

  9. 爬虫基础以及 re,BeatifulSoup,requests模块使用

    爬虫基础以及BeatifulSoup模块使用 爬虫的定义:向网站发起请求,获取资源后分析并提取有用数据的程序 爬虫的流程 发送请求 ---> request 获取响应内容 ---> res ...

  10. GoLang函数参数的传递练习

    春节买的GO方面的书,看了一次.现在撸一些代码,作为练习. // Copyright © 2019 NAME HERE <EMAIL ADDRESS> // // Licensed und ...