hadoop生态搭建（3节点）-10.spark配置

# https://www.scala-lang.org/download/2.12.4.html
# ==================================================================安装 scala

tar -zxvf ~/scala-2.12.4.tgz -C /usr/local
rm –r ~/scala-2.12.4.tgz

# http://archive.apache.org/dist/spark/spark-2.3.0/

# ==================================================================安装 spark

tar -zxf ~/spark-2.3.0-bin-hadoop2.7.tgz -C /usr/local
mv /usr/local/spark-2.3.0-bin-hadoop2.7 /usr/local/spark-2.3.0
rm –r ~/spark-2.3.0-bin-hadoop2.7.tgz

# 环境变量
# ==================================================================node1 node2 node3

vi /etc/profile

# 在export PATH USER LOGNAME MAIL HOSTNAME HISTSIZE HISTCONTROL下添加

export JAVA_HOME=/usr/java/jdk1.8.0_111
export ZOOKEEPER_HOME=/usr/local/zookeeper-3.4.12
export HADOOP_HOME=/usr/local/hadoop/hadoop-2.7.6
export MYSQL_HOME=/usr/local/mysql
export HBASE_HOME=/usr/local/hbase-1.2.4
export HIVE_HOME=/usr/local/hive-2.1.1
export SCALA_HOME=/usr/local/scala-2.12.4
export KAFKA_HOME=/usr/local/kafka_2.12-0.10.2.1
export FLUME_HOME=/usr/local/flume-1.8.0
export SPARK_HOME=/usr/local/spark-2.3.0

export PATH=$PATH:$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$ZOOKEEPER_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$MYSQL_HOME/bin:$HBASE_HOME/bin:$HIVE_HOME/bin:$SCALA_HOME/bin:$KAFKA_HOME/bin:$FLUME_HOME/bin:$SPARK_HOME/bin:$SPARK_HOME/sbin
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

export HADOOP_INSTALL=$HADOOP_HOME
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native

# ==================================================================node1

# 使环境变量生效
source /etc/profile

# 查看配置结果
echo $SPARK_HOME

# ==================================================================node1

cp $SPARK_HOME/conf/docker.properties.template $SPARK_HOME/conf/docker.properties
vi $SPARK_HOME/conf/docker.properties

spark.mesos.executor.home: /usr/local/spark-2.3.0

cp $SPARK_HOME/conf/fairscheduler.xml.template $SPARK_HOME/conf/fairscheduler.xml
cp $SPARK_HOME/conf/log4j.properties.template $SPARK_HOME/conf/log4j.properties
cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties

cp $SPARK_HOME/conf/slaves.template $SPARK_HOME/conf/slaves
vi $SPARK_HOME/conf/slaves

node1
node2
node3

cp $SPARK_HOME/conf/spark-defaults.conf.template $SPARK_HOME/conf/spark-defaults.conf
vi $SPARK_HOME/conf/spark-defaults.conf

spark.eventLog.enabled           true
spark.eventLog.dir               hdfs://appcluster/spark/eventslog
# 监控页面需要监控的目录，需要先启用和指定事件日志目录，配合上面两项使用
spark.history.fs.logDirectory    hdfs://appcluster/spark
spark.eventLog.compress          true

# 如果想 YARN ResourceManager 访问 Spark History Server ，则添加一行：
# spark.yarn.historyServer.address http://node1:19888

cp $SPARK_HOME/conf/spark-env.sh.template $SPARK_HOME/conf/spark-env.sh
vi $SPARK_HOME/conf/spark-env.sh

export SPARK_MASTER_PORT=7077        #提交任务的端口，默认是7077
export SPARK_MASTER_WEBUI_PORT=8070  #masster节点的webui端口 默认8080改为8070
export SPARK_WORKER_CORES=1          #每个worker从节点能够支配的core的个数
export SPARK_WORKER_MEMORY=1g        #每个worker从节点能够支配的内存数
export SPARK_WORKER_PORT=7078        #每个worker从节点的端口（可选配置）
export SPARK_WORKER_WEBUI_PORT=8071  #每个worker从节点的wwebui端口（可选配置）
export SPARK_WORKER_INSTANCES=1      #每个worker从节点的实例（可选配置）

export JAVA_HOME=/usr/java/jdk1.8.0_111
export SCALA_HOME=/usr/local/scala-2.12.4
export HADOOP_HOME=/usr/local/hadoop-2.7.6
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export YARN_CONF_DIR=$HADOOP_HOME/etc/Hadoop
export SPARK_PID_DIR=/usr/local/spark-2.3.0/pids
export SPARK_LOCAL_DIR=/usr/local/spark-2.3.0/tmp
export LD_LIBRARY_PATH=$HADOOP_HOME/lib/native
export SPARK_DAEMON_JAVA_OPTS="-Dspark.deploy.recoveryMode=ZOOKEEPER -Dspark.deploy.zookeeper.url=node1:2181,node2:2181,node3:2181 -Dspark.deploy.zookeeper.dir=/spark"

vi $SPARK_HOME/sbin/start-master.sh

SPARK_MASTER_WEBUI_PORT=8070

cp $HADOOP_HOME/etc/hadoop/hdfs-site.xml $SPARK_HOME/conf/

vi $HADOOP_HOME/etc/hadoop/log4j.properties

log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR

scp -r $HADOOP_HOME/etc/hadoop/log4j.properties node2:$HADOOP_HOME/etc/hadoop/
scp -r $HADOOP_HOME/etc/hadoop/log4j.properties node3:$HADOOP_HOME/etc/hadoop/

# ==================================================================node1

scp -r $SPARK_HOME node2:/usr/local/
scp -r $SPARK_HOME node3:/usr/local/

# ==================================================================node2 node3

# 使环境变量生效
source /etc/profile

# 查看配置结果
echo $FLUME_HOME

# 启动

# ==================================================================node1 node2 node3
# 先启动zookeeper 和 hdfs
zkServer.sh start
zkServer.sh status

# ==================================================================node1
zkCli.sh
create /spark ''

$HADOOP_HOME/sbin/start-all.sh

$HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc

# ==================================================================node2
$HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc
$HADOOP_HOME/sbin/yarn-daemon.sh start resourcemanager

# 启动spark

# ==================================================================node1
$SPARK_HOME/sbin/start-master.sh

$SPARK_HOME/sbin/start-slaves.sh

# ==================================================================node2
$SPARK_HOME/sbin/start-master.sh

# ==================================================================node1
# 获取安全模式的状态:
hdfs dfsadmin -safemode get

# 安全模式打开
# hdfs dfsadmin -safemode enter

# 安全模式关闭
# hdfs dfsadmin -safemode leave

hdfs dfs -mkdir -p /spark/eventslog

$SPARK_HOME/bin/spark-shell

# http://node1:4040
# http://node1:8070

> :quit

# test

# 需保证hdfs上该目录不存在
# hdfs dfs -mkdir -p /spark/output
# hdfs dfs -rmr /spark/output

vi ~/sparkdata.txt

hello man
what are you doing now
my running
hello
kevin
hi man

hdfs dfs -mkdir -p /usr/file/input

hdfs dfs -put ~/sparkdata.txt /usr/file/input
hdfs dfs -ls /usr/file/input

val file1 = sc.textFile("file:///root/sparkdata.txt")
val count1=file1.flatMap(line => line.split(" ")).map(word => (word,1)).reduceByKey(_+_)
count1.saveAsTextFile("hdfs://node1:8020/spark/output1")

val file=sc.textFile("hdfs://appcluster/usr/file/input/sparkdata.txt")
val count=file.flatMap(line => line.split(" ")).map(word => (word,1)).reduceByKey(_+_)
count.saveAsTextFile("hdfs://node1:8020/spark/output")

hdfs dfs -ls /spark/output

hdfs dfs -cat /spark/output/part-00000

# stop已经启动的进程

# ==================================================================node1
$SPARK_HOME/sbin/stop-slaves.sh

$SPARK_HOME/sbin/stop-master.sh

$HADOOP_HOME/sbin/stop-all.sh

# ==================================================================node1 node2 node3
# 停止 zookeeper
zkServer.sh stop

# ==================================================================node2
$HADOOP_HOME/sbin/yarn-daemon.sh stop resourcemanager
$HADOOP_HOME/sbin/hadoop-daemon.sh stop zkfc

# ==================================================================node1
$HADOOP_HOME/sbin/hadoop-daemon.sh stop zkfc

shutdown -h now
# 快照 spark