install Hadoop
Installing Java
Hadoop runs on both Unix and Windows operating systems, and requires Java to be
installed. For a production installation, you should select a combination of operating
system, Java, and Hadoop that has been certified by the vendor of the Hadoop distribution
you are using. There is also a page on the Hadoop wiki that lists combinations
that community members have run with success.
#Master 192.168.1.201 jdk、hadoop NameNode、DFSZKFailoverController(zkfc)、ResourceManager、JournalNode
#Slave1 192.168.1.202 jdk、hadoop NameNode、DFSZKFailoverController(zkfc)、ResourceManager
、JournalNode
#Slave2 192.168.1.203 jdk、hadoop、zookeeper DataNode、NodeManager、QuorumPeerMain
#Slave3 192.168.1.203 jdk、hadoop、zookeeper DataNode、NodeManager、QuorumPeerMain
#Slave4 192.168.1.204 jdk、hadoop、zookeeper DataNode、NodeManager、QuorumPeerMain
##############虚拟机集中配置本机环境######################
sed -i '/^SELINUX=/cSELINUX=disabled' /etc/selinux/config
chkconfig iptables off
service iptables stop
rm /etc/udev/rules.d/70-persistent-net.rules
#del MAC form /etc/sysconfig/network-scripts/ifcfg-eth0
tar -zxvf jdk-8u74-linux-x64.tar.gz -C /usr/java
rm /usr/java/latest -rf
ln -s /usr/java/jdk1.8.0_74 latest
echo "export JAVA_HOME=/usr/java/latest" >> /etc/profile
echo "export HADOOP_HOME=/usr/local/hadoop" >> /etc/profile
echo "export HBASE_HOME=/usr/local/hbase" >> /etc/profile
echo "export ZOOKEEPER_HOME=/usr/local/zookeeper" >> /etc/profile
echo "export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/sbin:$HADOOP_HOME/bin:$ZOOKEEPER_HOME/bin:$HBASE_HOME/bin:" >> /etc/profile
source /etc/profile
cd && tar -zxvf hadoop-2.6.4.tar.gz -C /usr/local/
mv /usr/local/hadoop-2.6.4 /usr/local/hadoop
echo "slave1" >> /usr/local/hadoop/etc/hadoop/slaves
echo "slave2" >> /usr/local/hadoop/etc/hadoop/slaves
echo "slave3" >> /usr/local/hadoop/etc/hadoop/slaves
##############虚拟机集中配置本机环境######################
##############虚拟机分散配置克隆环境########################
sed -i '/^HOSTNAME=/cHOSTNAME=master' /etc/sysconfig/network
sed -i '/^HOSTNAME=/cHOSTNAME=slave1' /etc/sysconfig/network
sed -i '/^HOSTNAME=/cHOSTNAME=slave2' /etc/sysconfig/network
sed -i '/^HOSTNAME=/cHOSTNAME=slave3' /etc/sysconfig/network
echo "192.168.26.130 master" >> /etc/hosts
echo "192.168.26.163 slave1" >> /etc/hosts
echo "192.168.26.164 slave2" >> /etc/hosts
echo "192.168.26.165 slave3" >> /etc/hosts
cd && mkdir .ssh
ssh-keygen -N '' -t rsa -q -f id_rsa
ssh-copy-id master
ssh-copy-id slave1
ssh-copy-id slave2
ssh-copy-id slave3
##############虚拟机分散配置克隆环境########################
#######################配置Zookeeper####################################################
tar -zxvf zookeeper-3.4.8.tar.gz -C /usr/local/
mv /usr/local/zookeeper-3.4.8/ /usr/local/zookeeper
cd /usr/local/zookeeper/conf/
cp zoo_sample.cfg zoo.cfg
mkdir /usr/local/zookeeper/data -p
sed -i '/^dataDir/cdataDir=/usr/local/zookeeper/data' /usr/local/zookeeper/conf/zoo.cfg
echo "server.0=slave1:2288:3388" >> /usr/local/zookeeper/conf/zoo.cfg
echo "server.1=slave2:2288:3388" >> /usr/local/zookeeper/conf/zoo.cfg
echo "server.2=slave3:2288:3388" >> /usr/local/zookeeper/conf/zoo.cfg
echo 0 > /usr/local/zookeeper/data/myid
#echo 1 > /usr/local/zookeeper/data/myid
#echo 2 > /usr/local/zookeeper/data/myid
#######################配置Zookeeper####################################################
#######################配置HADOOP#######################################################
#hadoo-env.sh
cd /usr/local/hadoop/etc/hadoop
sed -i '/^export JAVA_HOME/cexport JAVA_HOME=/usr/java/latest' hadoop-env.sh
#core-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<!-- 指定hdfs的nameservice为masters -->
<property>
<name>fs.defaultFS</name>
<value>hdfs://masters</value>
</property>
<!-- 指定hadoop临时目录 -->
<property>
<name>hadoop.tmp.dir</name>
<value>/usr/local/hadoop/tmp</value>
</property>
<!-- 指定zookeeper地址 -->
<property>
<name>ha.zookeeper.quorum</name>
<value>slave1:2181,slave2:2181,slave3:2181</value>
</property>
</configuration>
#hdfs-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<!--指定hdfs的nameservice为masters，需要和core-site.xml中的保持一致 -->
<property>
<name>dfs.nameservices</name>
<value>masters</value>
</property>
<!-- Master下面有两个NameNode，分别是Master，Slave1 -->
<property>
<name>dfs.ha.namenodes.masters</name>
<value>master,slave1</value>
</property>
<!-- Master的RPC通信地址 -->
<property>
<name>dfs.namenode.rpc-address.masters.master</name>
<value>master:9000</value>
</property>
<!-- Master的http通信地址 -->
<property>
<name>dfs.namenode.http-address.masters.master</name>
<value>master:50070</value>
</property>
<!-- Slave1的RPC通信地址 -->
<property>
<name>dfs.namenode.rpc-address.masters.slave1</name>
<value>slave1:9000</value>
</property>
<!-- Slave1的http通信地址 -->
<property>
<name>dfs.namenode.http-address.masters.slave1</name>
<value>slave1:50070</value>
</property>
<!-- 指定NameNode的元数据在JournalNode上的存放位置 -->
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://slave1:8485;slave2:8485;slave3:8485/masters</value>
</property>
<!-- 指定JournalNode在本地磁盘存放数据的位置 -->
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/usr/local/hadoop/journal</value>
</property>
<!-- 开启NameNode失败自动切换 -->
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<!-- 配置失败自动切换实现方式 -->
<property>
<name>dfs.client.failover.proxy.provider.masters</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<!-- 配置隔离机制方法，多个机制用换行分割，即每个机制暂用一行-->
<property>
<name>dfs.ha.fencing.methods</name>
<value>
sshfence
</value>
</property>
<!-- 使用sshfence隔离机制时需要ssh免登陆 -->
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/root/.ssh/id_rsa</value>
</property>
<!-- 配置sshfence隔离机制超时时间 -->
<property>
<name>dfs.ha.fencing.ssh.connect-timeout</name>
<value>30000</value>
</property>
</configuration>
mapred-site.xml
<configuration>
<!-- 指定mr框架为yarn方式 -->
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>
<configuration>
<!-- 开启RM高可靠 -->
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
<!-- 指定RM的cluster id -->
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>RM_HA_ID</value>
</property>
<!-- 指定RM的名字 -->
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<!-- 分别指定RM的地址 -->
<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>master</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>slave1</value>
</property>
<property>
<name>yarn.resourcemanager.recovery.enabled</name>
<value>true</value>
</property>
<property>
<name>yarn.resourcemanager.store.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
</property>
<!-- 指定zk集群地址 -->
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>slave1:2181,slave2:2181,slave3:2181</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
</configuration>
scp -r /usr/local/hadoop/etc/hadoop slave1:/usr/local/hadoop/etc/hadoop
scp -r /usr/local/hadoop/etc/hadoop slave2:/usr/local/hadoop/etc/hadoop
scp -r /usr/local/hadoop/etc/hadoop slave3:/usr/local/hadoop/etc/hadoop
#######################配置HADOOP#######################################################
##############配置hbase#########################hbase-**
tar -zxvf hbase-** /usr/local/hbase
##############配置hbase#########################hbase-**
##############COMMAND##############################
cd /usr/local/hadoop/sbin
hadoop-daemon.sh start journalnode
cd /usr/local/zookeeper/bin/
zkServer.sh start
zkServer.sh status
hdfs namenode -format
scp -r /usr/local/hadoop/tmp slave1:/usr/local/hadoop/
hdfs zkfc -formatZK
start-dfs.sh
start-yarn.sh
yarn-daemon.sh start resourcemanager
hadoop fs -put /etc/profile /profile
hadoop fs -ls /
hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.4.1.jar wordcount /profile /out
start-hbase.sh
##############COMMAND#########################
install Hadoop的更多相关文章
- Hadoop学习日志- install hadoop
资料来源 : http://www.tutorialspoint.com/hadoop/hadoop_enviornment_setup.htm Hadoop 安装 创建新用户 $ su passwo ...
- install hadoop on xubuntu
0. install xubuntu we recommend to set username as "hadoop" after installation, set user & ...
- mac osx 系统 brew install hadoop 安装指南
mac osx 系统 brew install hadoop 安装指南 brew install hadoop 配置 core-site.xml:配置hdfs文件地址(记得chmod 对应文件夹 ...
- How to install Hadoop
1.How to install Hadoop 3.0.0 http://blog.sina.com.cn/s/blog_4a1f59bf01010kx3.html 2.How to install ...
- [Spark] 00 - Install Hadoop & Spark
Hadoop安装 Java环境配置 安装课程:安装配置 配置手册:Hadoop安装教程_单机/伪分布式配置_Hadoop2.6.0/Ubuntu14.04[依照步骤完成配置] jsk安装使用的链接中第 ...
- Steps to Install Hadoop on CentOS/RHEL 6---reference
http://tecadmin.net/steps-to-install-hadoop-on-centosrhel-6/# The Apache Hadoop software library is ...
- yum install hadoop related client
yum list avaliable hadoop\* yum list installed yum repolist repo is in /etc/yum.repos.d yum install ...
- How to install Hadoop Cluster
https://dwbi.org/etl/bigdata/183-setup-hadoop-cluster https://www.linode.com/docs/databases/hadoop/h ...
- Install hadoop on windows(non-virtual machine, such cygwin)
DownloadBefore starting make sure you have this two softwares Hadoop 2.7.1 Java – Jdk 1.7+ Extract d ...
随机推荐
- PHP往mysql数据库中写入中文失败
该类问题解决办法就是 在建立数据库连接之后,将该连接的编码方式改为中文. 代码如下: $linkID=@mysql_connect("localhost","root&q ...
- [蟒蛇菜谱] Python封装shell命令
# -*- coding: utf-8 -*- import os import subprocess import signal import pwd import sys class MockLo ...
- iOS中block的用法 以及和函数用法的区别
ios中block的用法和函数的用法大致相同 但是block的用法的灵活性更高: 不带参数的block: void ^(MyBlock)() = ^{}; 调用的时候 MyBlock(); 带参数的 ...
- What is the difference between the ways to implement inheritance in javascript.
see also : http://www.w3school.com.cn/js/pro_js_inheritance_implementing.asp http://davidshariff.com ...
- 使用 UnrealPak.exe 创建 Pak文件方法
看来各位摸UE4 的基佬们,也是被DLC搞得不要不要的呢,其实热更新PAK是很简单就可以实现的,虽然当时我也是弄了快一个月. 下面贴一段以前在 Runtime 状态下 Mount Pak的代码,希望能 ...
- github上下载的文件如何在本地运行
最近想学习自己写插件,所以先下载大神写的插件膜拜下,下载下来以后,直接双击运行报错,大神怎么会犯这么低级的错误咧!所以在网上查资料,找到了解决方法. (1)上传到github上面的插件都很规范,文件根 ...
- Python学习感悟
学习任何新知识,我自己偏向于在实践中学习,这样效率更高:如果只是学习基本概念,会很枯燥. 现在的问题是:不是所有的新知识都可以在实践中学习!
- ReportViewer改变图表类型
/// <summary> /// 切换成柱状图 /// </summary> /// <param name="sender"&g ...
- 快速上手RaphaelJS--Instant RaphaelJS Starter翻译（二）
(目前发现一些文章被盗用的情况,我们将在每篇文章前面添加原文地址,本文源地址:http://www.cnblogs.com/idealer3d/p/Instant_RaphaelJS_Starter2 ...
- web前后端框架整理
nodejs <——> Express <———> bootstrap <———> angularJS <————> jquery <—————& ...