Hadoop分布式集群部署

环境准备

IP	HOSTNAME	SYSTEM
192.168.131.129	hadoop-master	CentOS 7.6
192.168.131.135	hadoop-slave1	CentOS 7.6
192.168.131.137	hadoop-slave2	CentOS 7.6

[root@localhost ~]# cat /etc/redhat-release

CentOS Linux release 7.6.1810 (Core)

[root@localhost ~]# sestatus

SELinux status:                 disabled

[root@localhost ~]# systemctl status firewalld.service

● firewalld.service - firewalld - dynamic firewall daemon

   Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled)

   Active: inactive (dead)

     Docs: man:firewalld(1)

[root@localhost ~]# hostnamectl set-hostname --static hadoop-master

[root@localhost ~]# hostnamectl set-hostname --static hadoop-slave1

[root@localhost ~]# hostnamectl set-hostname --static hadoop-slave2

配置免密

[root@hadoop-master ~]# cat >> /etc/hosts <<EOF

192.168.131.129 hadoop-master

192.168.131.135 hadoop-slave1

192.168.131.137 hadoop-slave2

EOF

[root@hadoop-master ~]# ssh-keygen

[root@hadoop-master ~]# ssh-copy-id hadoop-slave1

[root@hadoop-master ~]# ssh-copy-id hadoop-slave2

[root@hadoop-master ~]# scp /etc/hosts hadoop-slave1:/etc/hosts

hosts                                                              100%  248   151.9KB/s   00:00

[root@hadoop-master ~]# scp /etc/hosts hadoop-slave2:/etc/hosts

hosts                                                              100%  248   220.9KB/s   00:00

配置java环境

[root@hadoop-master ~]# tar xf jdk-8u211-linux-x64.tar.gz -C /usr/local/

[root@hadoop-master ~]# ln -s /usr/local/jdk1.8.0_211/ /usr/local/java

[root@hadoop-master ~]# ll /usr/local/

total 0

drwxr-xr-x. 2 root root   6 Apr 11  2018 bin

drwxr-xr-x. 2 root root   6 Apr 11  2018 etc

drwxr-xr-x. 2 root root   6 Apr 11  2018 games

drwxr-xr-x. 2 root root   6 Apr 11  2018 include

lrwxrwxrwx  1 root root  24 Sep  7 14:49 java -> /usr/local/jdk1.8.0_211/

drwxr-xr-x  7   10  143 245 Apr  2  2019 jdk1.8.0_211

drwxr-xr-x. 2 root root   6 Apr 11  2018 lib

drwxr-xr-x. 2 root root   6 Apr 11  2018 lib64

drwxr-xr-x. 2 root root   6 Apr 11  2018 libexec

drwxr-xr-x. 2 root root   6 Apr 11  2018 sbin

drwxr-xr-x. 5 root root  49 Mar 30  2019 share

drwxr-xr-x. 2 root root   6 Apr 11  2018 src

[root@hadoop-master ~]# vim /etc/profile

export JAVA_HOME=/usr/local/java/

export PATH=$JAVA_HOME/bin:$PATH

export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

[root@hadoop-master ~]# source /etc/profile

[root@hadoop-master ~]# java -version

java version "1.8.0_211"

Java(TM) SE Runtime Environment (build 1.8.0_211-b12)

Java HotSpot(TM) 64-Bit Server VM (build 25.211-b12, mixed mode)

部署hadoop

[root@hadoop-master ~]# wget https://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-2.7.7/hadoop-2.7.7.tar.gz

[root@hadoop-master ~]# tar xf hadoop-2.7.7.tar.gz -C /opt/

[root@hadoop-master ~]# vim /etc/profile

export HADOOP_HOME=/opt/hadoop-2.7.7

export PATH=$HADOOP_HOME/bin:$PATH

[root@hadoop-master ~]# source /etc/profile

[root@hadoop-master ~]# cd /opt/hadoop-2.7.7/

[root@hadoop-master hadoop-2.7.7]# ls

bin  etc  include  lib  libexec  LICENSE.txt  NOTICE.txt  README.txt  sbin  share

[root@hadoop-master hadoop-2.7.7]# cd etc/hadoop/

[root@hadoop-master hadoop]# ll

total 152

-rw-r--r-- 1 1000 ftp  4436 Jul 19  2018 capacity-scheduler.xml

-rw-r--r-- 1 1000 ftp  1335 Jul 19  2018 configuration.xsl

-rw-r--r-- 1 1000 ftp   318 Jul 19  2018 container-executor.cfg

-rw-r--r-- 1 1000 ftp   774 Jul 19  2018 core-site.xml

-rw-r--r-- 1 1000 ftp  3670 Jul 19  2018 hadoop-env.cmd

-rw-r--r-- 1 1000 ftp  4224 Jul 19  2018 hadoop-env.sh

-rw-r--r-- 1 1000 ftp  2598 Jul 19  2018 hadoop-metrics2.properties

-rw-r--r-- 1 1000 ftp  2490 Jul 19  2018 hadoop-metrics.properties

-rw-r--r-- 1 1000 ftp  9683 Jul 19  2018 hadoop-policy.xml

-rw-r--r-- 1 1000 ftp   775 Jul 19  2018 hdfs-site.xml

-rw-r--r-- 1 1000 ftp  1449 Jul 19  2018 httpfs-env.sh

-rw-r--r-- 1 1000 ftp  1657 Jul 19  2018 httpfs-log4j.properties

-rw-r--r-- 1 1000 ftp    21 Jul 19  2018 httpfs-signature.secret

-rw-r--r-- 1 1000 ftp   620 Jul 19  2018 httpfs-site.xml

-rw-r--r-- 1 1000 ftp  3518 Jul 19  2018 kms-acls.xml

-rw-r--r-- 1 1000 ftp  1527 Jul 19  2018 kms-env.sh

-rw-r--r-- 1 1000 ftp  1631 Jul 19  2018 kms-log4j.properties

-rw-r--r-- 1 1000 ftp  5540 Jul 19  2018 kms-site.xml

-rw-r--r-- 1 1000 ftp 11801 Jul 19  2018 log4j.properties

-rw-r--r-- 1 1000 ftp   951 Jul 19  2018 mapred-env.cmd

-rw-r--r-- 1 1000 ftp  1383 Jul 19  2018 mapred-env.sh

-rw-r--r-- 1 1000 ftp  4113 Jul 19  2018 mapred-queues.xml.template

-rw-r--r-- 1 1000 ftp   758 Jul 19  2018 mapred-site.xml.template

-rw-r--r-- 1 1000 ftp    10 Jul 19  2018 slaves

-rw-r--r-- 1 1000 ftp  2316 Jul 19  2018 ssl-client.xml.example

-rw-r--r-- 1 1000 ftp  2697 Jul 19  2018 ssl-server.xml.example

-rw-r--r-- 1 1000 ftp  2250 Jul 19  2018 yarn-env.cmd

-rw-r--r-- 1 1000 ftp  4567 Jul 19  2018 yarn-env.sh

-rw-r--r-- 1 1000 ftp   690 Jul 19  2018 yarn-site.xml

修改hadoop-env.sh和yarn-env.sh中JAVA_HOME参数

[root@hadoop-master hadoop]# sed -i 's#export JAVA_HOME=.*#export JAVA_HOME='/usr/local/java'#g' hadoop-env.sh

[root@hadoop-master hadoop]# sed -i 's#.*export JAVA_HOME=.*#export JAVA_HOME='/usr/local/java'#g' yarn-env.sh

[root@hadoop-master hadoop]# grep "export JAVA_HOME" hadoop-env.sh

export JAVA_HOME=/usr/local/java

[root@hadoop-master hadoop]# grep "export JAVA_HOME" yarn-env.sh

export JAVA_HOME=/usr/local/java

编辑core-site.xml

[root@hadoop-master hadoop]# vim core-site.xml

<configuration>

   <property>

      <name>fs.default.name</name>

      <value>hdfs://192.168.131.129:9000</value>

   </property>

   <property>

      <name>hadoop.tmp.dir</name>

      <value>/opt/hadoop/tmp</value>

   </property>

</configuration>

编辑hdfs-site.xml

[root@hadoop-master hadoop]# vim hdfs-site.xml

<configuration>

   <property>

      <name>dfs.replication</name>

      <value>2</value>

   </property>

   <property>

      <name>dfs.namenode.name.dir</name>

      <value>file:/opt/hadoop/dfs/name</value>

   </property>

   <property>

      <name>dfs.datanode.data.dir</name>

      <value>file:/opt/hadoop/dfs/data</value>

   </property>

</configuration>

编辑mapred-site.xml

[root@hadoop-master hadoop]# cp mapred-site.xml.template mapred-site.xml

[root@hadoop-master hadoop]# vim mapred-site.xml

<configuration>

   <property>

      <name>mapreduce.framework.name</name>

      <value>yarn</value>

   </property>

   <property>

      <name>mapreduce.jobhistory.address</name>

      <value>192.168.131.129:10020</value>

   </property>

   <property>

      <name>mapreduce.jobhistory.webapp.address</name>

      <value>192.168.131.129:19888</value>

   </property>

</configuration>

编辑yarn-site.xml

[root@hadoop-master hadoop]# vim yarn-site.xml

<configuration>

<!-- Site specific YARN configuration properties -->

   <property>

      <name>yarn.nodemanager.aux-services</name>

      <value>mapreduce_shuffle</value>

   </property>

   <property>

      <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>

      <value>org.apache.hadoop.mapred.ShuffleHandler</value>

    </property>

   <property>

      <name>yarn.resourcemanager.address</name>

      <value>192.168.131.129:8032</value>

   </property>

   <property>

      <name>yarn.resourcemanager.scheduler.address</name>

      <value>192.168.131.129:8030</value>

   </property>

   <property>

      <name>yarn.resourcemanager.resource-tracker.address</name>

      <value>192.168.131.129:8031</value>

   </property>

   <property>

      <name>yarn.resourcemanager.admin.address</name>

      <value>192.168.131.129:8033</value>

   </property>

   <property>

      <name>yarn.resourcemanager.webapp.address</name>

      <value>192.168.131.129:8088</value>

   </property>

</configuration>

编辑slaves

[root@hadoop-master hadoop]# vim slaves

hadoop-slave1

hadoop-slave2

把配置分发到各个节点

[root@hadoop-master opt]# scp -r hadoop-2.7.7/ hadoop-slave1:/opt/

[root@hadoop-master opt]# scp -r hadoop-2.7.7/ hadoop-slave2:/opt/

在主节点启动hadoop

初始化namenode

[root@hadoop-master ~]# hdfs namenode -format

启动HDFS

[root@hadoop-master ~]# cd /opt/hadoop-2.7.7/sbin/

[root@hadoop-master sbin]# ./start-dfs.sh

启动YARN

[root@hadoop-master sbin]# ./start-yarn.sh

测试

浏览器访问

YARN WEB地址：http://192.168.131.129:8088/

HDFS WEB地址：http://192.168.131.129:50070/

Hadoop分布式集群部署的更多相关文章

Hadoop分布式集群部署(单namenode节点)
Hadoop分布式集群部署系统系统环境: OS: CentOS 6.8 内存:2G CPU:1核 Software:jdk-8u151-linux-x64.rpm hadoop-2.7.4.tar. ...
Hadoop教程(五)Hadoop分布式集群部署安装
Hadoop教程(五)Hadoop分布式集群部署安装 1 Hadoop分布式集群部署安装在hadoop2.0中通常由两个NameNode组成,一个处于active状态,还有一个处于standby状态 ...
hadoop分布式集群部署①
Linux系统的安装和配置.(在VM虚拟机上) 一:安装虚拟机VMware Workstation 14 Pro 以上,虚拟机软件安装完成. 二:创建虚拟机. 三:安装CentOS系统 (1)上面步 ...
大数据系列之Hadoop分布式集群部署
本节目的:搭建Hadoop分布式集群环境环境准备 LZ用OS X系统 ,安装两台Linux虚拟机,Linux系统用的是CentOS6.5:Master Ip:10.211.55.3 ,Slave ...
Hadoop实战：Hadoop分布式集群部署（一）
一.系统参数优化配置 1.1 系统内核参数优化配置修改文件/etc/sysctl.conf,使用sysctl -p命令即时生效. 1 2 3 4 5 6 7 8 9 10 11 12 13 14 ...
一脸懵逼学习Hadoop分布式集群HA模式部署（七台机器跑集群）
1)集群规划:主机名 IP 安装的软件运行的进程master 192.168.199.130 jdk.hadoop ...
Hadoop(HA)分布式集群部署
Hadoop(HA)分布式集群部署和单节点namenode部署其实一样,只是配置文件的不同罢了. 这篇就讲解hadoop双namenode的部署,实现高可用. 系统环境: OS: CentOS 6.8 ...
超快速使用docker在本地搭建hadoop分布式集群
超快速使用docker在本地搭建hadoop分布式集群超快速使用docker在本地搭建hadoop分布式集群学习hadoop集群环境搭建是hadoop入门的必经之路.搭建分布式集群通常有两个办法: ...
基于Hadoop分布式集群YARN模式下的TensorFlowOnSpark平台搭建
1. 介绍在过去几年中,神经网络已经有了很壮观的进展,现在他们几乎已经是图像识别和自动翻译领域中最强者[1].为了从海量数据中获得洞察力,需要部署分布式深度学习.现有的DL框架通常需要为深度学习设置 ...

随机推荐

Jekyll + NexT + GitHub Pages 主题深度优化
前言笔者在用 Jekyll 搭建个人博客时踩了很多的坑,最后发现了一款不错的主题 jekyll-theme-next,但网上关于 Jekyll 版的 Next 主题优化教程少之又少,于是就决定自己写 ...
Java 集合详解 | 一篇文章解决Java 三大集合
更好阅读体验:Java 集合详解 | 一篇文章搞定Java 三大集合好看的皮囊像是一个个容器,有趣的灵魂像是容器里的数据.接下来讲解Java集合数据容器. 文章篇幅有点长,还请耐心阅读.如只是为了解 ...
Solon 开发，一、注入或手动获取配置
Solon 开发一.注入或手动获取配置二.注入或手动获取Bean 三.构建一个Bean的三种方式四.Bean 扫描的三种方式五.切面与环绕拦截六.提取Bean的函数进行定制开发七.自定义注 ...
今日学习——蓝桥杯 2019年 C语言 B组
1.手淦(亲身体验,,,没啥大用,最终还是代码) 2.代码(下面是我看其他博主代码答案能看的懂的....具体的可以直接去下面的网址看) https://blog.csdn.net/qq_4452491 ...
【记录一个问题】android下opencl中的event.getProfilingInfo()测速时间并不准确
使用了类似的代码来做android下opencl的时间测试: cl::CommandQueue queue(context, devices[0], CL_QUEUE_PROFILING_ENABLE ...
manjaro20初始配置
输入法配置安装完以后需要注销或重启,然后配置fctx 注意如果刚开始 export GTK_IM_MODULE=fcitx export QT_IM_MODULE=fcitx export XMOD ...
gin中绑定表单数据至自定义结构体
package main import "github.com/gin-gonic/gin" type StructA struct { FieldA string `form:& ...
多线程-创建线程第二种方式-实现Runnable接口-细节和好处
1 package multithread2; 2 3 /* 4 * 创建线程的第一种方法:继承Thread类 5 * 6 * 创建线程的第二种方式:实现Runnable接口 7 * 8 * 1,定义 ...
Java高级语法之反射
Java高级语法之反射什么是反射 java.lang包提供java语言程序设计的基础类,在lang包下存在一个子包:reflect,与反射相关的APIs均在此处: 官方对reflect包的介绍如下: ...
Java多线程专题4: 锁的实现基础 AQS
合集目录 Java多线程专题4: 锁的实现基础 AQS 对 AQS(AbstractQueuedSynchronizer)的理解 Provides a framework for implementi ...

Hadoop分布式集群部署

环境准备

配置免密

配置java环境

部署hadoop

修改hadoop-env.sh和yarn-env.sh中JAVA_HOME参数

编辑core-site.xml

编辑hdfs-site.xml

编辑mapred-site.xml

编辑yarn-site.xml

编辑slaves

把配置分发到各个节点

在主节点启动hadoop

测试

Hadoop分布式集群部署的更多相关文章

随机推荐

热门专题