安装hadoop

生成yum源

cd /var/ftp/pub/cdh/5

createrepo --update .

从节点

yum clean all

配置yum库

/etc/yum.repos.d

# cat /etc/yum.repos.d/cloudera-cdh.repo
[hadoop]
name=hadoop
baseurl=ftp://192.168.34.135/pub/cdh/5/
enabled=1
gpgcheck=0

安装JDK

yum install jdk

echo "export JAVA_HOME=/usr/java/latest" >> /root/.bash_profile

echo "export PATH=\$JAVA_HOME/bin:\$PATH" >> /root/.bash_profile

安装namenode

yum install hadoop hadoop-hdfs hadoop-client hadoop-doc hadoop-debuginfo hadoop-hdfs-namenode

安装webhdfs

yum install hadoop-httpfs

安装secondary namenode

yum install hadoop-hdfs-secondarynamenode

安装datanode

yum install hadoop hadoop-hdfs hadoop-client hadoop-doc hadoop-debuginfo hadoop-hdfs-datanode

配置HDFS

配置文件路径

/etc/hadoop/conf

core-site.xml

<property>
<name>hadoop.tmp.dir</name>
<value>/home/yimr/var/filesystem</value>
</property>

<property>
<name>fs.default.name</name>
<value>hdfs://yi01:9000</value>
</property>

hdfs-site.xml

<property>
<name>dfs.replication</name>
<value>2</value>
</property>

<property>
<name>dfs.permissions.superusergroup</name>
<value>yimr</value>
</property>

配置Secondary Namenode

hdfs-site.xml

<property>
<name>dfs.secondary.http.address</name>
<value>yi01:50090</value>
</property>

配置WebHdfs

core-site.xml

<property>
<name>hadoop.proxyuser.httpfs.hosts</name>
<value>*</value>
</property>

<property>
<name>hadoop.proxyuser.httpfs.groups</name>
<value>*</value>
</property>

启动webhdfs

service hadoop-httpfs start

启动HDFS

同步配置到所有节点

rsync -av /etc/hadoop/conf/ 192.168.34.130:/etc/hadoop/conf/

格式化namenode

mkdir -p /home/yimr/var/filesystem

chown -R hdfs.hdfs /home/yimr/var/filesystem

sudo -u hdfs hadoop namenode -format

每个节点启动hdfs

for x in `ls /etc/init.d/|grep  hadoop-hdfs` ; do service $x start ; done

service hadoop-hdfs-datanode start
service hadoop-hdfs-namenode start

安装Yarn

安装resource manager

yum install hadoop-yarn hadoop-yarn-resourcemanager

安装node manager

yum install hadoop-yarn hadoop-yarn-nodemanager hadoop-mapreduce

安装history server

yum install hadoop-mapreduce-historyserver hadoop-yarn-proxyserver

配置Yarn

mapred-site.xml

<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>

yarn-site.xml

<property>
<description>The hostname of the RM.</description>
<name>yarn.resourcemanager.hostname</name>
<value>yi01</value>
</property>

<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>

配置history server

mapred-site.xml

<property>
<name>yarn.app.mapreduce.am.staging-dir</name>
<value>/user</value>
</property>

启动服务

启动history server

sudo -u hdfs hadoop fs -mkdir -p /user

sudo -u hdfs hadoop fs -chmod 777 /user

sudo -u hdfs hadoop fs -mkdir -p /user/history

sudo -u hdfs hadoop fs -chmod -R 1777 /user/history

sudo -u hdfs hadoop fs -chown mapred:hadoop /user/history

/etc/init.d/hadoop-mapreduce-historyserver start

启动yarn

for x in `ls /etc/init.d/|grep hadoop-yarn` ; do service $x start ; done

service hadoop-yarn-nodemanager start

HDFS界面

yi00:50070

Yarn界面

yi00:8088

history界面

yi00:19888

测试