1.[start-all.sh]

#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# # Start all spark daemons.    #启动所有守护进程
# Starts the master on this node.  #在这个节点上启动master
# Starts a worker on each node specified in conf/slaves #在conf/slaves文件下指定的节点上启动worker进程 if [ -z "${SPARK_HOME}" ]; then    #判断spark环境变量在不在
export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"  #如果在的话,就把环境变量导入
fi # Load the Spark configuration    #加载配置文件
. "${SPARK_HOME}/sbin/spark-config.sh"  #调用spark-config.sh脚本 # Start Master
"${SPARK_HOME}/sbin"/start-master.sh  #启动master脚本进程 # Start Workers
"${SPARK_HOME}/sbin"/start-slaves.sh  #启动slaves脚本进程

2.[start-master.sh]

#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# # Starts the master on the machine this script is executed on.  #在这个机器上执行脚本,启动master进程 if [ -z "${SPARK_HOME}" ]; then    #首先配置环境变量
export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
fi # NOTE: This exact class name is matched downstream by SparkSubmit.
# Any changes need to be reflected there.
CLASS="org.apache.spark.deploy.master.Master" if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
echo "Usage: ./sbin/start-master.sh [options]"
pattern="Usage:"
pattern+="\|Using Spark's default log4j profile:"
pattern+="\|Registered signal handlers for" "${SPARK_HOME}"/bin/spark-class $CLASS --help >& | grep -v "$pattern" >&
exit
fi ORIGINAL_ARGS="$@" . "${SPARK_HOME}/sbin/spark-config.sh" . "${SPARK_HOME}/bin/load-spark-env.sh" if [ "$SPARK_MASTER_PORT" = "" ]; then
SPARK_MASTER_PORT=
fi if [ "$SPARK_MASTER_HOST" = "" ]; then
SPARK_MASTER_HOST=`hostname -f`
fi if [ "$SPARK_MASTER_WEBUI_PORT" = "" ]; then
SPARK_MASTER_WEBUI_PORT=
fi "${SPARK_HOME}/sbin"/spark-daemon.sh start $CLASS \
--host $SPARK_MASTER_HOST --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT \
$ORIGINAL_ARGS

spark复习笔记(4):spark脚本分析的更多相关文章

  1. spark复习笔记(2)

    之前工作的时候经常用,隔了段时间,现在学校要用学的东西也忘了,翻翻书谢谢博客吧. 1.什么是spark? Spark是一种快速.通用.可扩展的大数据分析引擎,2009年诞生于加州大学伯克利分校AMPL ...

  2. spark学习笔记总结-spark入门资料精化

    Spark学习笔记 Spark简介 spark 可以很容易和yarn结合,直接调用HDFS.Hbase上面的数据,和hadoop结合.配置很容易. spark发展迅猛,框架比hadoop更加灵活实用. ...

  3. Spark学习笔记-使用Spark History Server

    在运行Spark应用程序的时候,driver会提供一个webUI给出应用程序的运行信息,但是该webUI随着应用程序的完成而关闭端口,也就是 说,Spark应用程序运行完后,将无法查看应用程序的历史记 ...

  4. Spark 学习笔记之 Spark history Server 搭建

    在hdfs上建立文件夹/directory hadoop fs -mkdir /directory 进入conf目录  spark-env.sh 增加以下配置 export SPARK_HISTORY ...

  5. spark复习笔记(5):API分析

    0.spark是基于hadoop的mr模型,扩展了MR,高效实用MR模型,内存型集群计算,提高了app处理速度. 1.特点:(1)在内存中存储中间结果 (2)支持多种语言:java scala pyt ...

  6. spark复习笔记(4):RDD变换

    一.RDD变换 1.返回执行新的rdd的指针,在rdd之间创建依赖关系.每个rdd都有一个计算函数和指向父rdd的指针 Spark是惰性的,因此除非调用某个转换或动作,否则不会执行任何操作,否则将触发 ...

  7. spark复习笔记(1)

    使用spark实现work count ---------------------------------------------------- (1)用sc.textFile(" &quo ...

  8. spark复习笔记(7):sparkstreaming

    一.介绍 1.sparkStreaming是核心模块Spark API的扩展,具有可伸缩,高吞吐量以及容错的实时数据流处理等.数据可以从许多来源(如Kafka,Flume,Kinesis或TCP套接字 ...

  9. spark复习笔记(7):sparkSQL

    一.saprkSQL模块,使用类sql的方式访问Hadoop,实现mr计算,底层使用的是rdd 1.hive //hadoop  mr  sql 2.phenoix //hbase上构建sql的交互过 ...

随机推荐

  1. 51nod1730 涂边

    题目描述 题解 八级sb题 显然可以想到状压 枚举当前的宽度\(I\),设\(f[s]\)表示在当前的宽度下选的竖边的状态为s 再设\(g[s1][s2]\)表示状态s1转移到s2的方案数,枚举中间横 ...

  2. mybatis添加数据时返回主键 insert 返回主键值

    insert 返回主键值 useGeneratedKeys=“true” parameterType=“USer” keyProperty=“id”, <insert id="inse ...

  3. django快速搭建blog

    python版本:3.5.4: Django版本:2.0 创建项目 创建mysite项目和 blog应用: django-admin startproject mysite # 创建mysite项目 ...

  4. Java的基本使用

    1.如何运行一个Java源码 打开文本编辑器,输入以下代码: public class Hello { public static void main(String[] args) { System. ...

  5. 2014ACM-ICPC广州站题解(摘自闭幕式)

    第39届ACM-ICPC亚洲区广州站题解 Ltysky摘抄自闭幕式题目分析 Problem A 满足px+qy=c的点(x,y)在一条直线上,而c的值由直线的截距确定,所以最大化c,就要在糖果(x,y ...

  6. uploadify上传插件参数的一些设置

    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/ ...

  7. EventBus和Otto第三方构架

    代码 添加依赖:implementation 'org.greenrobot:eventbus:3.0.0'1注册并声明订阅者,然后发布事件最后解除注册 @Override protected voi ...

  8. MySQL数据库:RESET MASTER、RESET SLAVE、MASTER_INFO、RELAY_LOG_INFO

    MySQL数据库:RESET MASTER.RESET SLAVE.MASTER_INFO.RELAY_LOG_INFO RESET MASTER 删除所有index file中记录的所有binlog ...

  9. mongoexport导出记录到csv文件

    root@service:~# mongoexport -d prod -c employees -f _id,platform,phone --csv -o /opt/employees.csv 2 ...

  10. c# 用DotNetZip来解压/压缩文件

    //https://archive.codeplex.com/?p=dotnetzip //最新在Nuget 下载DotNetZip using Ionic.Zip; private void but ...