spark 2.1.0 集群安装

jdk安装

http://www.cnblogs.com/xiaojf/p/6568426.html

scala2.11 安装

http://www.cnblogs.com/xiaojf/p/6568432.html

hadoop2.7 安装

http://www.cnblogs.com/xiaojf/p/6629351.html

开始spark2.1.0安装

解压

[root@m1 jar]# tar zxvf spark-2.1.-bin-hadoop2..tgz -C ../

重命名

[root@m1 jar]# cd ..
[root@m1 soft]# ll
total
drwxr-xr-x.  root   root    Mar  : hadoop
drwxr-xr-x.   root   root    Mar  : jar
drwxr-xr-x.   root   root    Dec  : jdk
drwxr-xr-x.   root   root    Mar  : kafka
drwxrwxr-x.   root   root    Mar     scala-2.11.
drwxr-xr-x.  xiaojf xiaojf  Dec  : spark-2.1.-bin-hadoop2.
drwxr-xr-x.   root   root    Mar  : tmp
drwxr-xr-x.  root   root    Aug    zookeeper-3.4.
[root@m1 soft]# mv spark-2.1.-bin-hadoop2. spark
[root@m1 soft]# ll
total
drwxr-xr-x.  root   root    Mar  : hadoop
drwxr-xr-x.   root   root    Mar  : jar
drwxr-xr-x.   root   root    Dec  : jdk
drwxr-xr-x.   root   root    Mar  : kafka
drwxrwxr-x.   root   root    Mar     scala-2.11.
drwxr-xr-x.  xiaojf xiaojf  Dec  : spark
drwxr-xr-x.   root   root    Mar  : tmp
drwxr-xr-x.  root   root    Aug    zookeeper-3.4.

配置环境变量

[root@m1 soft]# vi /etc/profile
[root@m1 soft]# source /etc/profile

export SPARK_HOME=/usr/local/soft/spark
export PATH=$SPARK_HOME/bin:$SPARK_HOME/sbin:$PATH

修改配置文件

[root@m1 soft]# cd /usr/local/soft/spark/conf/
[root@m1 conf]# pwd
/usr/local/soft/spark/conf
[root@m1 conf]# ll
total
-rw-r--r--.  xiaojf xiaojf   Dec  : docker.properties.template
-rw-r--r--.  xiaojf xiaojf  Dec  : fairscheduler.xml.template
-rw-r--r--.  xiaojf xiaojf  Dec  : log4j.properties.template
-rw-r--r--.  xiaojf xiaojf  Dec  : metrics.properties.template
-rw-r--r--.  xiaojf xiaojf   Dec  : slaves.template
-rw-r--r--.  xiaojf xiaojf  Dec  : spark-defaults.conf.template
-rwxr-xr-x.  xiaojf xiaojf  Dec  : spark-env.sh.template
[root@m1 conf]# cp log4j.properties.template log4j.properties
[root@m1 conf]# cp slaves.template slaves
[root@m1 conf]# cp spark-defaults.conf.template spark-defaults.conf
[root@m1 conf]# cp spark-env.sh.template spark-env.sh

修改 spark-defaults.conf

[root@m1 conf]# vi spark-defaults.conf

spark.eventLog.enabled           true
spark.eventLog.dir              hdfs://m1:9000/historyserverforSpark
spark.executor.extraJavaOptions  -XX:+PrintGCDetails -Dkey=value -Dnumbers="one two three"
spark.yarn.historyServer.address   m1:
spark.history.fs.logDirectory      hdfs://m1:9000/historyserverforSpark
spark.driver.memory              5g

修改 spark-env.sh

[root@m1 conf]# vi spark-env.sh

export HADOOP_HOME=/usr/local/soft/hadoop
export HADOOP_CONF_DIR=/usr/local/soft/hadoop/etc/hadoop
export JAVA_HOME=/usr/local/soft/jdk
export SCALA_HOME=/usr/local/soft/scala
export SPARK_MASTER_IP=m1
export SPARK_WORKER_MEMORY=1G
export SPARK_EXECUTOR_MEMORY=1G
export SPARK_DRIVER_MEMORY=1G
export SPARK_WORKER_CORES=

修改 slaves

[root@m1 conf]# vi slaves
s1
s2

分发代码到集群节点

[root@m1 soft]# scp -r spark root@s1:/usr/local/soft/
[root@m1 soft]# scp -r spark root@s2:/usr/local/soft/

启动

[root@m1 soft]# cd /usr/local/soft/spark/sbin/
[root@m1 sbin]# ll
total
-rwxr-xr-x.  xiaojf xiaojf  Dec  : slaves.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : spark-config.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : spark-daemon.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : spark-daemons.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : start-all.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : start-history-server.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : start-master.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : start-mesos-dispatcher.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : start-mesos-shuffle-service.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : start-shuffle-service.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : start-slave.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : start-slaves.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : start-thriftserver.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : stop-all.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : stop-history-server.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : stop-master.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : stop-mesos-dispatcher.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : stop-mesos-shuffle-service.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : stop-shuffle-service.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : stop-slave.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : stop-slaves.sh
-rwxr-xr-x.  xiaojf xiaojf  Dec  : stop-thriftserver.sh
[root@m1 sbin]# ./start-all.sh
starting org.apache.spark.deploy.master.Master, logging to /usr/local/soft/spark/logs/spark-root-org.apache.spark.deploy.master.Master--m1.out
s1: starting org.apache.spark.deploy.worker.Worker, logging to /usr/local/soft/spark/logs/spark-root-org.apache.spark.deploy.worker.Worker--s1.out
s2: starting org.apache.spark.deploy.worker.Worker, logging to /usr/local/soft/spark/logs/spark-root-org.apache.spark.deploy.worker.Worker--s2.out

查看进程

[root@m1 sbin]# jps
 Master
 Kafka
 Jps
 QuorumPeerMain

这个时候还没有启动hadoop，所以先启动hadoop，再启动spark

[root@m1 sbin]# /usr/local/soft/hadoop/sbin/start-all.sh
This script is Deprecated. Instead use start-dfs.sh and start-yarn.sh
Starting namenodes on [m1]
m1: starting namenode, logging to /usr/local/soft/hadoop/logs/hadoop-root-namenode-m1.out
s2: starting datanode, logging to /usr/local/soft/hadoop/logs/hadoop-root-datanode-s2.out
s1: starting datanode, logging to /usr/local/soft/hadoop/logs/hadoop-root-datanode-s1.out
Starting secondary namenodes [m1]
m1: starting secondarynamenode, logging to /usr/local/soft/hadoop/logs/hadoop-root-secondarynamenode-m1.out
starting yarn daemons
starting resourcemanager, logging to /usr/local/soft/hadoop/logs/yarn-root-resourcemanager-m1.out
s2: starting nodemanager, logging to /usr/local/soft/hadoop/logs/yarn-root-nodemanager-s2.out
s1: starting nodemanager, logging to /usr/local/soft/hadoop/logs/yarn-root-nodemanager-s1.out
[root@m1 sbin]# jps
 ResourceManager
 Kafka
 SecondaryNameNode
 NameNode
 Jps
 QuorumPeerMain

[root@m1 sbin]# /usr/local/soft/spark/sbin/start-all.sh
starting org.apache.spark.deploy.master.Master, logging to /usr/local/soft/spark/logs/spark-root-org.apache.spark.deploy.master.Master--m1.out
s2: starting org.apache.spark.deploy.worker.Worker, logging to /usr/local/soft/spark/logs/spark-root-org.apache.spark.deploy.worker.Worker--s2.out
s1: starting org.apache.spark.deploy.worker.Worker, logging to /usr/local/soft/spark/logs/spark-root-org.apache.spark.deploy.worker.Worker--s1.out
[root@m1 sbin]# jps
 ResourceManager
 Kafka
 SecondaryNameNode
 Master
 NameNode
 Jps
 QuorumPeerMain

打开spark-shell 测试

[root@m1 sbin]# spark-shell 

完成