Hadoop分布式集群部署

环境准备

IP	HOSTNAME	SYSTEM
192.168.131.129	hadoop-master	CentOS 7.6
192.168.131.135	hadoop-slave1	CentOS 7.6
192.168.131.137	hadoop-slave2	CentOS 7.6

[root@localhost ~]# cat /etc/redhat-release

CentOS Linux release 7.6.1810 (Core)

[root@localhost ~]# sestatus

SELinux status:                 disabled

[root@localhost ~]# systemctl status firewalld.service

● firewalld.service - firewalld - dynamic firewall daemon

   Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled)

   Active: inactive (dead)

     Docs: man:firewalld(1)

[root@localhost ~]# hostnamectl set-hostname --static hadoop-master

[root@localhost ~]# hostnamectl set-hostname --static hadoop-slave1

[root@localhost ~]# hostnamectl set-hostname --static hadoop-slave2

配置免密

[root@hadoop-master ~]# cat >> /etc/hosts <<EOF

192.168.131.129 hadoop-master

192.168.131.135 hadoop-slave1

192.168.131.137 hadoop-slave2

EOF

[root@hadoop-master ~]# ssh-keygen

[root@hadoop-master ~]# ssh-copy-id hadoop-slave1

[root@hadoop-master ~]# ssh-copy-id hadoop-slave2

[root@hadoop-master ~]# scp /etc/hosts hadoop-slave1:/etc/hosts

hosts                                                              100%  248   151.9KB/s   00:00

[root@hadoop-master ~]# scp /etc/hosts hadoop-slave2:/etc/hosts

hosts                                                              100%  248   220.9KB/s   00:00

配置java环境

[root@hadoop-master ~]# tar xf jdk-8u211-linux-x64.tar.gz -C /usr/local/

[root@hadoop-master ~]# ln -s /usr/local/jdk1.8.0_211/ /usr/local/java

[root@hadoop-master ~]# ll /usr/local/

total 0

drwxr-xr-x. 2 root root   6 Apr 11  2018 bin

drwxr-xr-x. 2 root root   6 Apr 11  2018 etc

drwxr-xr-x. 2 root root   6 Apr 11  2018 games

drwxr-xr-x. 2 root root   6 Apr 11  2018 include

lrwxrwxrwx  1 root root  24 Sep  7 14:49 java -> /usr/local/jdk1.8.0_211/

drwxr-xr-x  7   10  143 245 Apr  2  2019 jdk1.8.0_211

drwxr-xr-x. 2 root root   6 Apr 11  2018 lib

drwxr-xr-x. 2 root root   6 Apr 11  2018 lib64

drwxr-xr-x. 2 root root   6 Apr 11  2018 libexec

drwxr-xr-x. 2 root root   6 Apr 11  2018 sbin

drwxr-xr-x. 5 root root  49 Mar 30  2019 share

drwxr-xr-x. 2 root root   6 Apr 11  2018 src

[root@hadoop-master ~]# vim /etc/profile

export JAVA_HOME=/usr/local/java/

export PATH=$JAVA_HOME/bin:$PATH

export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

[root@hadoop-master ~]# source /etc/profile

[root@hadoop-master ~]# java -version

java version "1.8.0_211"

Java(TM) SE Runtime Environment (build 1.8.0_211-b12)

Java HotSpot(TM) 64-Bit Server VM (build 25.211-b12, mixed mode)

部署hadoop

[root@hadoop-master ~]# wget https://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-2.7.7/hadoop-2.7.7.tar.gz

[root@hadoop-master ~]# tar xf hadoop-2.7.7.tar.gz -C /opt/

[root@hadoop-master ~]# vim /etc/profile

export HADOOP_HOME=/opt/hadoop-2.7.7

export PATH=$HADOOP_HOME/bin:$PATH

[root@hadoop-master ~]# source /etc/profile

[root@hadoop-master ~]# cd /opt/hadoop-2.7.7/

[root@hadoop-master hadoop-2.7.7]# ls

bin  etc  include  lib  libexec  LICENSE.txt  NOTICE.txt  README.txt  sbin  share

[root@hadoop-master hadoop-2.7.7]# cd etc/hadoop/

[root@hadoop-master hadoop]# ll

total 152

-rw-r--r-- 1 1000 ftp  4436 Jul 19  2018 capacity-scheduler.xml

-rw-r--r-- 1 1000 ftp  1335 Jul 19  2018 configuration.xsl

-rw-r--r-- 1 1000 ftp   318 Jul 19  2018 container-executor.cfg

-rw-r--r-- 1 1000 ftp   774 Jul 19  2018 core-site.xml

-rw-r--r-- 1 1000 ftp  3670 Jul 19  2018 hadoop-env.cmd

-rw-r--r-- 1 1000 ftp  4224 Jul 19  2018 hadoop-env.sh

-rw-r--r-- 1 1000 ftp  2598 Jul 19  2018 hadoop-metrics2.properties

-rw-r--r-- 1 1000 ftp  2490 Jul 19  2018 hadoop-metrics.properties

-rw-r--r-- 1 1000 ftp  9683 Jul 19  2018 hadoop-policy.xml

-rw-r--r-- 1 1000 ftp   775 Jul 19  2018 hdfs-site.xml

-rw-r--r-- 1 1000 ftp  1449 Jul 19  2018 httpfs-env.sh

-rw-r--r-- 1 1000 ftp  1657 Jul 19  2018 httpfs-log4j.properties

-rw-r--r-- 1 1000 ftp    21 Jul 19  2018 httpfs-signature.secret

-rw-r--r-- 1 1000 ftp   620 Jul 19  2018 httpfs-site.xml

-rw-r--r-- 1 1000 ftp  3518 Jul 19  2018 kms-acls.xml

-rw-r--r-- 1 1000 ftp  1527 Jul 19  2018 kms-env.sh

-rw-r--r-- 1 1000 ftp  1631 Jul 19  2018 kms-log4j.properties

-rw-r--r-- 1 1000 ftp  5540 Jul 19  2018 kms-site.xml

-rw-r--r-- 1 1000 ftp 11801 Jul 19  2018 log4j.properties

-rw-r--r-- 1 1000 ftp   951 Jul 19  2018 mapred-env.cmd

-rw-r--r-- 1 1000 ftp  1383 Jul 19  2018 mapred-env.sh

-rw-r--r-- 1 1000 ftp  4113 Jul 19  2018 mapred-queues.xml.template

-rw-r--r-- 1 1000 ftp   758 Jul 19  2018 mapred-site.xml.template

-rw-r--r-- 1 1000 ftp    10 Jul 19  2018 slaves

-rw-r--r-- 1 1000 ftp  2316 Jul 19  2018 ssl-client.xml.example

-rw-r--r-- 1 1000 ftp  2697 Jul 19  2018 ssl-server.xml.example

-rw-r--r-- 1 1000 ftp  2250 Jul 19  2018 yarn-env.cmd

-rw-r--r-- 1 1000 ftp  4567 Jul 19  2018 yarn-env.sh

-rw-r--r-- 1 1000 ftp   690 Jul 19  2018 yarn-site.xml

修改hadoop-env.sh和yarn-env.sh中JAVA_HOME参数

[root@hadoop-master hadoop]# sed -i 's#export JAVA_HOME=.*#export JAVA_HOME='/usr/local/java'#g' hadoop-env.sh

[root@hadoop-master hadoop]# sed -i 's#.*export JAVA_HOME=.*#export JAVA_HOME='/usr/local/java'#g' yarn-env.sh

[root@hadoop-master hadoop]# grep "export JAVA_HOME" hadoop-env.sh

export JAVA_HOME=/usr/local/java

[root@hadoop-master hadoop]# grep "export JAVA_HOME" yarn-env.sh

export JAVA_HOME=/usr/local/java

编辑core-site.xml

[root@hadoop-master hadoop]# vim core-site.xml

<configuration>

   <property>

      <name>fs.default.name</name>

      <value>hdfs://192.168.131.129:9000</value>

   </property>

   <property>

      <name>hadoop.tmp.dir</name>

      <value>/opt/hadoop/tmp</value>

   </property>

</configuration>

编辑hdfs-site.xml

[root@hadoop-master hadoop]# vim hdfs-site.xml

<configuration>

   <property>

      <name>dfs.replication</name>

      <value>2</value>

   </property>

   <property>

      <name>dfs.namenode.name.dir</name>

      <value>file:/opt/hadoop/dfs/name</value>

   </property>

   <property>

      <name>dfs.datanode.data.dir</name>

      <value>file:/opt/hadoop/dfs/data</value>

   </property>

</configuration>

编辑mapred-site.xml

[root@hadoop-master hadoop]# cp mapred-site.xml.template mapred-site.xml

[root@hadoop-master hadoop]# vim mapred-site.xml

<configuration>

   <property>

      <name>mapreduce.framework.name</name>

      <value>yarn</value>

   </property>

   <property>

      <name>mapreduce.jobhistory.address</name>

      <value>192.168.131.129:10020</value>

   </property>

   <property>

      <name>mapreduce.jobhistory.webapp.address</name>

      <value>192.168.131.129:19888</value>

   </property>

</configuration>

编辑yarn-site.xml

[root@hadoop-master hadoop]# vim yarn-site.xml

<configuration>

<!-- Site specific YARN configuration properties -->

   <property>

      <name>yarn.nodemanager.aux-services</name>

      <value>mapreduce_shuffle</value>

   </property>

   <property>

      <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>

      <value>org.apache.hadoop.mapred.ShuffleHandler</value>

    </property>

   <property>

      <name>yarn.resourcemanager.address</name>

      <value>192.168.131.129:8032</value>

   </property>

   <property>

      <name>yarn.resourcemanager.scheduler.address</name>

      <value>192.168.131.129:8030</value>

   </property>

   <property>

      <name>yarn.resourcemanager.resource-tracker.address</name>

      <value>192.168.131.129:8031</value>

   </property>

   <property>

      <name>yarn.resourcemanager.admin.address</name>

      <value>192.168.131.129:8033</value>

   </property>

   <property>

      <name>yarn.resourcemanager.webapp.address</name>

      <value>192.168.131.129:8088</value>

   </property>

</configuration>

编辑slaves

[root@hadoop-master hadoop]# vim slaves

hadoop-slave1

hadoop-slave2

把配置分发到各个节点

[root@hadoop-master opt]# scp -r hadoop-2.7.7/ hadoop-slave1:/opt/

[root@hadoop-master opt]# scp -r hadoop-2.7.7/ hadoop-slave2:/opt/

在主节点启动hadoop

初始化namenode

[root@hadoop-master ~]# hdfs namenode -format

启动HDFS

[root@hadoop-master ~]# cd /opt/hadoop-2.7.7/sbin/

[root@hadoop-master sbin]# ./start-dfs.sh

启动YARN

[root@hadoop-master sbin]# ./start-yarn.sh

测试

浏览器访问

YARN WEB地址：http://192.168.131.129:8088/

HDFS WEB地址：http://192.168.131.129:50070/

Hadoop分布式集群部署的更多相关文章

Hadoop分布式集群部署(单namenode节点)
Hadoop分布式集群部署系统系统环境: OS: CentOS 6.8 内存:2G CPU:1核 Software:jdk-8u151-linux-x64.rpm hadoop-2.7.4.tar. ...
Hadoop教程(五)Hadoop分布式集群部署安装
Hadoop教程(五)Hadoop分布式集群部署安装 1 Hadoop分布式集群部署安装在hadoop2.0中通常由两个NameNode组成,一个处于active状态,还有一个处于standby状态 ...
hadoop分布式集群部署①
Linux系统的安装和配置.(在VM虚拟机上) 一:安装虚拟机VMware Workstation 14 Pro 以上,虚拟机软件安装完成. 二:创建虚拟机. 三:安装CentOS系统 (1)上面步 ...
大数据系列之Hadoop分布式集群部署
本节目的:搭建Hadoop分布式集群环境环境准备 LZ用OS X系统 ,安装两台Linux虚拟机,Linux系统用的是CentOS6.5:Master Ip:10.211.55.3 ,Slave ...
Hadoop实战：Hadoop分布式集群部署（一）
一.系统参数优化配置 1.1 系统内核参数优化配置修改文件/etc/sysctl.conf,使用sysctl -p命令即时生效. 1 2 3 4 5 6 7 8 9 10 11 12 13 14 ...
一脸懵逼学习Hadoop分布式集群HA模式部署（七台机器跑集群）
1)集群规划:主机名 IP 安装的软件运行的进程master 192.168.199.130 jdk.hadoop ...
Hadoop(HA)分布式集群部署
Hadoop(HA)分布式集群部署和单节点namenode部署其实一样,只是配置文件的不同罢了. 这篇就讲解hadoop双namenode的部署,实现高可用. 系统环境: OS: CentOS 6.8 ...
超快速使用docker在本地搭建hadoop分布式集群
超快速使用docker在本地搭建hadoop分布式集群超快速使用docker在本地搭建hadoop分布式集群学习hadoop集群环境搭建是hadoop入门的必经之路.搭建分布式集群通常有两个办法: ...
基于Hadoop分布式集群YARN模式下的TensorFlowOnSpark平台搭建
1. 介绍在过去几年中,神经网络已经有了很壮观的进展,现在他们几乎已经是图像识别和自动翻译领域中最强者[1].为了从海量数据中获得洞察力,需要部署分布式深度学习.现有的DL框架通常需要为深度学习设置 ...

随机推荐

CentOS6.5-Hadoop2.7.3安装hive-2.1.1
注:图片如果损坏,点击文章链接:https://www.toutiao.com/i6627723801960382979/ 系统版本,Hadoop已安装完成 Mysql安装完成准备的解压包安装zi ...
Clickhouse的MergeTree表引擎存储结构
MergeTree存储的文件结构一张数据表被分成几个data part,每个data part对应文件系统中的一个目录.通过以下SQL可以查询data parts的信息. select table, ...
【SpringCloud技术专题】「Gateway网关系列」（3）微服务网关服务的Gateway全流程开发实践指南（2.2.X）
开发指南须知本次实践主要在版本:2.2.0.BUILD-SNAPSHOT上进行构建,这个项目提供了构建在Spring生态系统之上API网关. Spring Cloud Gateway的介绍 Spri ...
论文解读二代GCN《Convolutional Neural Networks on Graphs with Fast Localized Spectral Filtering》
Paper Information Title:Convolutional Neural Networks on Graphs with Fast Localized Spectral Filteri ...
Winfrom统一单例窗口
//调用方式 var frm = new MyForm().Instance(); public static class ExFrm { static Dictionary<string, F ...
WSL删除子系统后无法重装
问题 WSL卸载后安装error 解决办法 UWP应用卸载后没有删除目录下的文件 C:\Users\wwwfe\AppData\Local\Packages路径下删除就可以了再次安装会卡顿很久,可能 ...
manjaro20夜灯夜间模式开关
Linux深入探索01-stty与键盘信号
----- 最近更新[2021-12-20]----- 一.简介最初的 Unix 设定假定人们使用终端连接主机计算机.30多年过去后,情况依然如此,即便是在自己的PC机上运行Unix.多年以来,终端 ...
IoC容器-Bean管理XML方式（注入内部bean和级联赋值）
注入属性-内部bean和级联赋值 (1)一对多关系:部分和员工一个部门有多个员工,一个员工属于一个部门部门是一,员工是多 (2)在实体类之间表示一对多关系 (3)在spring配置文件中进行配置 ...
linux下查看开放的端口
Nmap是一款针对大型网络的端口扫描工具,它也适用于单机扫描,它支持很多扫描,也同时支持性能和可靠性统计. [root@localhost ~]# yum install namp [root@loc ...

Hadoop分布式集群部署

环境准备

配置免密

配置java环境

部署hadoop

修改hadoop-env.sh和yarn-env.sh中JAVA_HOME参数

编辑core-site.xml

编辑hdfs-site.xml

编辑mapred-site.xml

编辑yarn-site.xml

编辑slaves

把配置分发到各个节点

在主节点启动hadoop

测试

Hadoop分布式集群部署的更多相关文章

随机推荐

热门专题