cm5.9.2安装spark启动报错解决办法

[root@db02 scala-2.11.5]# spark-shell

Setting default log level to "WARN".

To adjust logging level use sc.setLogLevel(newLevel).

Welcome to
       ____              __
      / __/__  ___ _____/ /__
     _\ \/ _ \/ _ `/ __/  '_/
    /___/ .__/\_,_/_/ /_/\_\   version 1.6.0
       /_/

Using Scala version 2.10.5 (Java HotSpot(TM) 64-Bit Server VM, Java 1.7.0_67)

Type in expressions to have them evaluated.

Type :help for more information.

17/08/26 10:48:23 ERROR spark.SparkContext: Error initializing SparkContext.

java.lang.IllegalArgumentException: Required executor memory (1024+384 MB) is above the max threshold (1024 MB) of this cluster! Please check the values of 'yarn.scheduler.maximum-allocation-mb' and/or 'yarn.nodemanager.resource.memory-mb'.
     at org.apache.spark.deploy.yarn.Client.verifyClusterResources(Client.scala:292)
     at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:139)
     at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:57)
     at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:157)
     at org.apache.spark.SparkContext.<init>(SparkContext.scala:542)
     at org.apache.spark.repl.SparkILoop.createSparkContext(SparkILoop.scala:1022)
     at $line3.$read$$iwC$$iwC.<init>(<console>:15)
     at $line3.$read$$iwC.<init>(<console>:25)
     at $line3.$read.<init>(<console>:27)
     at $line3.$read$.<init>(<console>:31)
     at $line3.$read$.<clinit>(<console>)
     at $line3.$eval$.<init>(<console>:7)
     at $line3.$eval$.<clinit>(<console>)
     at $line3.$eval.$print(<console>)
     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
     at java.lang.reflect.Method.invoke(Method.java:606)
     at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1045)
     at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1326)
     at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:821)
     at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:852)
     at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:800)
     at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:857)
     at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:902)
     at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814)
     at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:125)
     at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:124)
     at org.apache.spark.repl.SparkIMain.beQuietDuring(SparkIMain.scala:305)
     at org.apache.spark.repl.SparkILoopInit$class.initializeSpark(SparkILoopInit.scala:124)
     at org.apache.spark.repl.SparkILoop.initializeSpark(SparkILoop.scala:64)
     at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1$$anonfun$apply$mcZ$sp$5.apply$mcV$sp(SparkILoop.scala:974)
     at org.apache.spark.repl.SparkILoopInit$class.runThunks(SparkILoopInit.scala:160)
     at org.apache.spark.repl.SparkILoop.runThunks(SparkILoop.scala:64)
     at org.apache.spark.repl.SparkILoopInit$class.postInitialization(SparkILoopInit.scala:108)
     at org.apache.spark.repl.SparkILoop.postInitialization(SparkILoop.scala:64)
     at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply$mcZ$sp(SparkILoop.scala:991)
     at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
     at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
     at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
     at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$process(SparkILoop.scala:945)
     at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1064)
     at org.apache.spark.repl.Main$.main(Main.scala:35)
     at org.apache.spark.repl.Main.main(Main.scala)
     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
     at java.lang.reflect.Method.invoke(Method.java:606)
     at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:729)
     at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
     at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
     at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)
     at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

17/08/26 10:48:23 WARN cluster.YarnSchedulerBackend$YarnSchedulerEndpoint: Attempted to request executors before the AM has registered!

17/08/26 10:48:23 ERROR util.Utils: Uncaught exception in thread main

java.lang.NullPointerException
     at org.apache.spark.network.shuffle.ExternalShuffleClient.close(ExternalShuffleClient.java:152)
     at org.apache.spark.storage.BlockManager.stop(BlockManager.scala:1317)
     at org.apache.spark.SparkEnv.stop(SparkEnv.scala:96)
     at org.apache.spark.SparkContext$$anonfun$stop$12.apply$mcV$sp(SparkContext.scala:1768)
     at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1230)
     at org.apache.spark.SparkContext.stop(SparkContext.scala:1767)
     at org.apache.spark.SparkContext.<init>(SparkContext.scala:614)
     at org.apache.spark.repl.SparkILoop.createSparkContext(SparkILoop.scala:1022)
     at $line3.$read$$iwC$$iwC.<init>(<console>:15)
     at $line3.$read$$iwC.<init>(<console>:25)
     at $line3.$read.<init>(<console>:27)
     at $line3.$read$.<init>(<console>:31)
     at $line3.$read$.<clinit>(<console>)
     at $line3.$eval$.<init>(<console>:7)
     at $line3.$eval$.<clinit>(<console>)
     at $line3.$eval.$print(<console>)
     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
     at java.lang.reflect.Method.invoke(Method.java:606)
     at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1045)
     at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1326)
     at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:821)
     at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:852)
     at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:800)
     at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:857)
     at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:902)
     at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814)
     at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:125)
     at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:124)
     at org.apache.spark.repl.SparkIMain.beQuietDuring(SparkIMain.scala:305)
     at org.apache.spark.repl.SparkILoopInit$class.initializeSpark(SparkILoopInit.scala:124)
     at org.apache.spark.repl.SparkILoop.initializeSpark(SparkILoop.scala:64)
     at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1$$anonfun$apply$mcZ$sp$5.apply$mcV$sp(SparkILoop.scala:974)
     at org.apache.spark.repl.SparkILoopInit$class.runThunks(SparkILoopInit.scala:160)
     at org.apache.spark.repl.SparkILoop.runThunks(SparkILoop.scala:64)
     at org.apache.spark.repl.SparkILoopInit$class.postInitialization(SparkILoopInit.scala:108)
     at org.apache.spark.repl.SparkILoop.postInitialization(SparkILoop.scala:64)
     at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply$mcZ$sp(SparkILoop.scala:991)
     at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
     at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
     at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
     at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$process(SparkILoop.scala:945)
     at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1064)
     at org.apache.spark.repl.Main$.main(Main.scala:35)
     at org.apache.spark.repl.Main.main(Main.scala)
     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
     at java.lang.reflect.Method.invoke(Method.java:606)
     at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:729)
     at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
     at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
     at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)
     at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

java.lang.IllegalArgumentException: Required executor memory (1024+384 MB) is above the max threshold (1024 MB) of this cluster! Please check the values of 'yarn.scheduler.maximum-allocation-mb' and/or 'yarn.nodemanager.resource.memory-mb'.
     at org.apache.spark.deploy.yarn.Client.verifyClusterResources(Client.scala:292)
     at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:139)
     at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:57)
     at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:157)
     at org.apache.spark.SparkContext.<init>(SparkContext.scala:542)
     at org.apache.spark.repl.SparkILoop.createSparkContext(SparkILoop.scala:1022)
     at $iwC$$iwC.<init>(<console>:15)
     at $iwC.<init>(<console>:25)
     at <init>(<console>:27)
     at .<init>(<console>:31)
     at .<clinit>(<console>)
     at .<init>(<console>:7)
     at .<clinit>(<console>)
     at $print(<console>)
     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
     at java.lang.reflect.Method.invoke(Method.java:606)
     at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1045)
     at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1326)
     at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:821)
     at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:852)
     at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:800)
     at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:857)
     at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:902)
     at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814)
     at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:125)
     at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:124)
     at org.apache.spark.repl.SparkIMain.beQuietDuring(SparkIMain.scala:305)
     at org.apache.spark.repl.SparkILoopInit$class.initializeSpark(SparkILoopInit.scala:124)
     at org.apache.spark.repl.SparkILoop.initializeSpark(SparkILoop.scala:64)
     at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1$$anonfun$apply$mcZ$sp$5.apply$mcV$sp(SparkILoop.scala:974)
     at org.apache.spark.repl.SparkILoopInit$class.runThunks(SparkILoopInit.scala:160)
     at org.apache.spark.repl.SparkILoop.runThunks(SparkILoop.scala:64)
     at org.apache.spark.repl.SparkILoopInit$class.postInitialization(SparkILoopInit.scala:108)
     at org.apache.spark.repl.SparkILoop.postInitialization(SparkILoop.scala:64)
     at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply$mcZ$sp(SparkILoop.scala:991)
     at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
     at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
     at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
     at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$process(SparkILoop.scala:945)
     at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1064)
     at org.apache.spark.repl.Main$.main(Main.scala:35)
     at org.apache.spark.repl.Main.main(Main.scala)
     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
     at java.lang.reflect.Method.invoke(Method.java:606)
     at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:729)
     at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
     at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
     at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)
     at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

java.lang.NullPointerException
     at org.apache.spark.sql.SQLContext$.createListenerAndUI(SQLContext.scala:1375)
     at org.apache.spark.sql.hive.HiveContext.<init>(HiveContext.scala:101)
     at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
     at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
     at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
     at java.lang.reflect.Constructor.newInstance(Constructor.java:526)
     at org.apache.spark.repl.SparkILoop.createSQLContext(SparkILoop.scala:1033)
     at $iwC$$iwC.<init>(<console>:15)
     at $iwC.<init>(<console>:24)
     at <init>(<console>:26)
     at .<init>(<console>:30)
     at .<clinit>(<console>)
     at .<init>(<console>:7)
     at .<clinit>(<console>)
     at $print(<console>)
     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
     at java.lang.reflect.Method.invoke(Method.java:606)
     at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1045)
     at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1326)
     at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:821)
     at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:852)
     at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:800)
     at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:857)
     at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:902)
     at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814)
     at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:133)
     at org.apache.spark.repl.SparkILoopInit$$anonfun$initializeSpark$1.apply(SparkILoopInit.scala:124)
     at org.apache.spark.repl.SparkIMain.beQuietDuring(SparkIMain.scala:305)
     at org.apache.spark.repl.SparkILoopInit$class.initializeSpark(SparkILoopInit.scala:124)
     at org.apache.spark.repl.SparkILoop.initializeSpark(SparkILoop.scala:64)
     at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1$$anonfun$apply$mcZ$sp$5.apply$mcV$sp(SparkILoop.scala:974)
     at org.apache.spark.repl.SparkILoopInit$class.runThunks(SparkILoopInit.scala:160)
     at org.apache.spark.repl.SparkILoop.runThunks(SparkILoop.scala:64)
     at org.apache.spark.repl.SparkILoopInit$class.postInitialization(SparkILoopInit.scala:108)
     at org.apache.spark.repl.SparkILoop.postInitialization(SparkILoop.scala:64)
     at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply$mcZ$sp(SparkILoop.scala:991)
     at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
     at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
     at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
     at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$process(SparkILoop.scala:945)
     at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1064)
     at org.apache.spark.repl.Main$.main(Main.scala:35)
     at org.apache.spark.repl.Main.main(Main.scala)
     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
     at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
     at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
     at java.lang.reflect.Method.invoke(Method.java:606)
     at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:729)
     at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
     at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
     at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)
     at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

<console>:16: error: not found: value sqlContext
          import sqlContext.implicits._
                 ^

<console>:16: error: not found: value sqlContext
          import sqlContext.sql
                 ^

通过以上日志我们可以看到如下信息：

17/08/26 10:48:23 ERROR spark.SparkContext: Error initializing SparkContext.

java.lang.IllegalArgumentException: Required executor memory (1024+384 MB) is above the max threshold (1024 MB) of this cluster! Please check the values of 'yarn.scheduler.maximum-allocation-mb' and/or 'yarn.nodemanager.resource.memory-mb'.

这里我们调整参数：yarn.scheduler.maximum-allocation-mb' 和 'yarn.nodemanager.resource.memory-mb'的默认值为2g。