【Hadoop】伪分布式环境搭建、验证

Hadoop伪分布式环境搭建：

自动部署脚本：

#!/bin/bash
set -eux

export APP_PATH=/opt/applications
export APP_NAME=Ares

# 安装apt依赖包
apt-get update -y \
    && apt-get install supervisor -y \
    && apt-get install python-dev python-pip libmysqlclient-dev -y

# 安装pip、python依赖
pip install --upgrade pip \
    && pip install -r ./build-depends/pip-requirements/requirements.txt

# 安装JDK
tar -xzvf ./build-depends/jdk-package/jdk-7u60-linux-x64.tar.gz \
    && ln -s jdk1..0_60/ jdk

# 配置JAVA环境变量
echo -e '\n' >> /etc/profile
echo '# !!!No Modification, This Section is Auto Generated by '${APP_NAME} >> /etc/profile
echo 'export JAVA_HOME='${APP_PATH}/${APP_NAME}/jdk >> /etc/profile
echo 'export JRE_HOME=${JAVA_HOME}/jre' >> /etc/profile
echo 'export CLASSPATH=.:${JAVA_HOME}/jre/lib/rt.jar:${JAVA_HOME}/lib/dt.jar:${JAVA_HOME}/lib/tools.jar' >> /etc/profile
echo 'export PATH=${PATH}:${JAVA_HOME}/bin:${JRE_HOME}/bin' >> /etc/profile
source /etc/profile && java -version

# 安装Hadoop
tar -xzvf ./build-depends/hadoop-package/hadoop-2.5..tar.gz \
    && ln -s hadoop-2.5. hadoop
# hadoop-env.sh配置JAVA_HOME
mv ./hadoop/etc/hadoop/hadoop-env.sh ./hadoop/etc/hadoop/hadoop-env.sh.bak \
    && cp -rf ./build-depends/hadoop-conf/hadoop-env.sh ./hadoop/etc/hadoop/ \
    && sed -i "25a export JAVA_HOME=${APP_PATH}/${APP_NAME}/jdk" ./hadoop/etc/hadoop/hadoop-env.sh
# core-site.xml配置
mv ./hadoop/etc/hadoop/core-site.xml ./hadoop/etc/hadoop/core-site.xml.bak \
    && python ./build-utils/configueUpdate/templateInvoke.py ./build-depends/hadoop-conf/core-site.xml ./hadoop/etc/hadoop/core-site.xml
# hdfs-site.xml配置
mv ./hadoop/etc/hadoop/hdfs-site.xml ./hadoop/etc/hadoop/hdfs-site.xml.bak \
    && python ./build-utils/configueUpdate/templateInvoke.py ./build-depends/hadoop-conf/hdfs-site.xml ./hadoop/etc/hadoop/hdfs-site.xml
# mapred-site.xml配置
python ./build-utils/configueUpdate/templateInvoke.py ./build-depends/hadoop-conf/mapred-site.xml.template ./hadoop/etc/hadoop/mapred-site.xml
# yarn-site.xml配置
mv ./hadoop/etc/hadoop/yarn-site.xml ./hadoop/etc/hadoop/yarn-site.xml.bak \
    && python ./build-utils/configueUpdate/templateInvoke.py ./build-depends/hadoop-conf/yarn-site.xml ./hadoop/etc/hadoop/yarn-site.xml
# slaves, 即DataNode配置
mv ./hadoop/etc/hadoop/slaves ./hadoop/etc/hadoop/slaves.bak
DataNodeList=(`echo ${DataNodeList} | tr ";" "\n"`)
for DataNode in ${DataNodeList}; do
    echo ${DataNode} >> ./hadoop/etc/hadoop/slaves
done

# 配置Hadoop环境变量
echo -e '\n' >> /etc/profile
echo '# !!!No Modification, This Section is Auto Generated by '${APP_NAME} >> /etc/profile
echo 'export HADOOP_HOME='${APP_PATH}/${APP_NAME}/hadoop >> /etc/profile
echo 'export PATH=${PATH}:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin' >> /etc/profile
source /etc/profile && hadoop version

# Namenode格式化
# hadoop namenode -format -force
hdfs namenode -format -force

# 启动hdfs、yarn
stop-dfs.sh && start-dfs.sh && jps
stop-yarn.sh && start-yarn.sh && jps

# hdfs测试
# hadoop fs -put ./build-depends/jdk-package/jdk-7u60-linux-x64.tar.gz hdfs://HADOOP-NODE1:9000/
hdfs dfs -put ./build-depends/jdk-package/jdk-7u60-linux-x64.tar.gz hdfs://HADOOP-NODE1:9000/
# hadoop fs -get hdfs://HADOOP-NODE1:9000/jdk-7u60-linux-x64.tar.gz .
hdfs dfs -get hdfs://HADOOP-NODE1:9000/jdk-7u60-linux-x64.tar.gz .
rm -rf jdk-7u60-linux-x64.tar.gz

# mapred测试
hadoop jar ./hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.5..jar pi  

# word-count测试
touch word-count.txt    \
    && echo "hello world" >> word-count.txt \
    && echo "hello tom" >> word-count.txt \
    && echo "hello jim" >> word-count.txt \
    && echo "hello kitty" >> word-count.txt \
    && echo "hello baby" >> word-count.txt
# hadoop fs -put word-count.txt hdfs://HADOOP-NODE1:9000/
# hadoop fs -rm hdfs://HADOOP-NODE1:9000/word-count.txt
hadoop fs -mkdir hdfs://HADOOP-NODE1:9000/word-count
hadoop fs -mkdir hdfs://HADOOP-NODE1:9000/word-count/input
# hadoop fs -mkdir hdfs://HADOOP-NODE1:9000/word-count/output
# hadoop fs -rmdir hdfs://HADOOP-NODE1:9000/word-count/output
hadoop fs -put word-count.txt hdfs://HADOOP-NODE1:9000/word-count/input
hadoop jar ./hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.5..jar wordcount hdfs://HADOOP-NODE1:9000/word-count/input hdfs://HADOOP-NODE1:9000/word-count/output
hadoop fs -ls hdfs://HADOOP-NODE1:9000/word-count/output
hadoop fs -cat hdfs://HADOOP-NODE1:9000/word-count/output/part-r-00000

# supervisord 配置文件
#cp ${APP_PATH}/supervisor.conf.d/*.conf /etc/supervisor/conf.d/

# start supervisord nodaemon
# /usr/bin/supervisord --nodaemon
#/usr/bin/supervisord

运行脚本：

# 此处描述应用运行命令使用方法.
export APP_PATH=/opt/applications
export APP_NAME=Ares
export APP_Version=2.5.

# 单节点-伪分布式
#HOSTNAME           IP              HDFS                                YARN
#HADOOP-NODE1       10.20.0.11      NameNode/SNameNode/DataNode         NodeManager/ResourceManager
export NameNode_HOST=HADOOP-NODE1
export NameNode_RPCPort=
export NameNode_HTTP_PORT=
export SNameNode_HOST=HADOOP-NODE1
export SNameNode_HTTP_PORT=
export SNameNode_HTTPS_PORT=
export HDFS_Replication=
export YARN_RSC_MGR_HOST=HADOOP-NODE1
export YARN_RSC_MGR_HTTP_PORT=
export YARN_RSC_MGR_HTTPS_PORT=
export DataNodeList='HADOOP-NODE1'

mkdir -p ${APP_PATH}/${APP_NAME} \
    && mv ${APP_NAME}-${APP_Version}.zip ${APP_PATH}/${APP_NAME}/ \
    && cd ${APP_PATH}/${APP_NAME}/ \
    && unzip ${APP_NAME}-${APP_Version}.zip \
    && chmod a+x run.sh \
    && ./run.sh

ssh免密码登录过程：