Spark-shell启动脚本解读

#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

#
# Shell script for starting the Spark Shell REPL

#判断是否为cygwin
cygwin=false
case "`uname`" in
CYGWIN*) cygwin=true;;
esac

# Enter posix mode for bash
set -o posix

## Global script variables

#进入到spark的安装目录
FWDIR="$(cd `dirname $0`/..; pwd)"

#定义帮助信息的方法
#调用spark-submit的帮助信息，只是把submit以下帮助信息过滤掉
#   Usage: spark-submit [options] <app jar | python file> [app arguments]
#   Usage: spark-submit --kill [submission ID] --master [spark://...]
#   Usage: spark-submit --status [submission ID] --master [spark://...]

function usage() {
echo "Usage: ./bin/spark-shell [options]"
$FWDIR/bin/spark-submit --help >& | grep -v Usage >&
exit
}

if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
usage
fi

#引用utils.sh脚本，脚本的功能为整理脚本参数、判断部分参数的合法性，给以下两个变量赋值
#SUBMISSION_OPTS:
#SUBMISSION_OPTS参数包括：
#      K-V形式的有： --master | --deploy-mode | --class | --name | --jars | --py-files | --files | \
#                   --conf | --properties-file | --driver-memory | --driver-java-options | \
#                   --driver-library-path | --driver-class-path | --executor-memory | --driver-cores | \
#                   --total-executor-cores | --executor-cores | --queue | --num-executors | --archives
#      非K-V形式的有
#                   --verbose | -v | --supervise
#      KV形式的需要对个数进行判断
#
#APPLICATION_OPTS参数包括除SUBMISSION_OPTS之外的参数

source $FWDIR/bin/utils.sh

#定义帮助信息方法的变量
SUBMIT_USAGE_FUNCTION=usage
#调用utils.sh脚本中的gatherSparkSubmitOpts方法。对参数进行整理
gatherSparkSubmitOpts "$@"

#主函数，调用spark-submit --class org.apache.spark.repl.Main方法

function main() {
if $cygwin; then
# Workaround for issue involving JLine and Cygwin
# (see http://sourceforge.net/p/jline/bugs/40/).
# If you're using the Mintty terminal emulator in Cygwin, may need to set the
# "Backspace sends ^H" setting in "Keys" section of the Mintty options
# (see https://github.com/sbt/sbt/issues/562).
stty -icanon min  -echo > /dev/null >&
export SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS -Djline.terminal=unix"
$FWDIR/bin/spark-submit --class org.apache.spark.repl.Main "${SUBMISSION_OPTS[@]}" spark-shell "${APPLICATION_OPTS[@]}"
stty icanon echo > /dev/null >&
else
export SPARK_SUBMIT_OPTS
$FWDIR/bin/spark-submit --class org.apache.spark.repl.Main "${SUBMISSION_OPTS[@]}" spark-shell "${APPLICATION_OPTS[@]}"
fi
}

# Copy restore-TTY-on-exit functions from Scala script so spark-shell exits properly even in
# binary distribution of Spark where Scala is not installed
exit_status=
saved_stty=""

# restore stty settings (echo in particular)
function restoreSttySettings() {
stty $saved_stty
saved_stty=""
}

function onExit() {
if [[ "$saved_stty" != "" ]]; then
restoreSttySettings
fi
exit $exit_status
}

# to reenable echo if we are interrupted before completing.
trap onExit INT

# save terminal settings
saved_stty=$(stty -g >/dev/null)
# clear on error so we don't later try to restore them
if [[ ! $? ]]; then
saved_stty=""
fi

main "$@"

# record the exit status lest it be overwritten:
# then reenable echo and propagate the code.
exit_status=$?
onExit

utils.sh脚本内容：

 #!/usr/bin/env bash

 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
 # The ASF licenses this file to You under the Apache License, Version 2.0
 # (the "License"); you may not use this file except in compliance with
 # the License.  You may obtain a copy of the License at
 #
 #    http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #

 # Gather all all spark-submit options into SUBMISSION_OPTS
 function gatherSparkSubmitOpts() {

   if [ -z "$SUBMIT_USAGE_FUNCTION" ]; then
     echo "Function for printing usage of $0 is not set." >&
     echo "Please set usage function to shell variable 'SUBMIT_USAGE_FUNCTION' in $0" >&
     exit
   fi

   # NOTE: If you add or remove spark-sumbmit options,
   # modify NOT ONLY this script but also SparkSubmitArgument.scala
   SUBMISSION_OPTS=()
   APPLICATION_OPTS=()
   while (($#)); do
     case "$1" in
       --master | --deploy-mode | --class | --name | --jars | --py-files | --files | \
       --conf | --properties-file | --driver-memory | --driver-java-options | \
       --driver-library-path | --driver-class-path | --executor-memory | --driver-cores | \
       --total-executor-cores | --executor-cores | --queue | --num-executors | --archives)
         if [[ $# -lt  ]]; then
           "$SUBMIT_USAGE_FUNCTION"
           exit ;
         fi
         SUBMISSION_OPTS+=("$1"); shift
         SUBMISSION_OPTS+=("$1"); shift
         ;;

       --verbose | -v | --supervise)
         SUBMISSION_OPTS+=("$1"); shift
         ;;

       *)
         APPLICATION_OPTS+=("$1"); shift
         ;;
     esac
   done

   export SUBMISSION_OPTS
   export APPLICATION_OPTS
 }