Hadoop分布式集群部署

环境准备

IP	HOSTNAME	SYSTEM
192.168.131.129	hadoop-master	CentOS 7.6
192.168.131.135	hadoop-slave1	CentOS 7.6
192.168.131.137	hadoop-slave2	CentOS 7.6

[root@localhost ~]# cat /etc/redhat-release

CentOS Linux release 7.6.1810 (Core)

[root@localhost ~]# sestatus

SELinux status:                 disabled

[root@localhost ~]# systemctl status firewalld.service

● firewalld.service - firewalld - dynamic firewall daemon

   Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled)

   Active: inactive (dead)

     Docs: man:firewalld(1)

[root@localhost ~]# hostnamectl set-hostname --static hadoop-master

[root@localhost ~]# hostnamectl set-hostname --static hadoop-slave1

[root@localhost ~]# hostnamectl set-hostname --static hadoop-slave2

配置免密

[root@hadoop-master ~]# cat >> /etc/hosts <<EOF

192.168.131.129 hadoop-master

192.168.131.135 hadoop-slave1

192.168.131.137 hadoop-slave2

EOF

[root@hadoop-master ~]# ssh-keygen

[root@hadoop-master ~]# ssh-copy-id hadoop-slave1

[root@hadoop-master ~]# ssh-copy-id hadoop-slave2

[root@hadoop-master ~]# scp /etc/hosts hadoop-slave1:/etc/hosts

hosts                                                              100%  248   151.9KB/s   00:00

[root@hadoop-master ~]# scp /etc/hosts hadoop-slave2:/etc/hosts

hosts                                                              100%  248   220.9KB/s   00:00

配置java环境

[root@hadoop-master ~]# tar xf jdk-8u211-linux-x64.tar.gz -C /usr/local/

[root@hadoop-master ~]# ln -s /usr/local/jdk1.8.0_211/ /usr/local/java

[root@hadoop-master ~]# ll /usr/local/

total 0

drwxr-xr-x. 2 root root   6 Apr 11  2018 bin

drwxr-xr-x. 2 root root   6 Apr 11  2018 etc

drwxr-xr-x. 2 root root   6 Apr 11  2018 games

drwxr-xr-x. 2 root root   6 Apr 11  2018 include

lrwxrwxrwx  1 root root  24 Sep  7 14:49 java -> /usr/local/jdk1.8.0_211/

drwxr-xr-x  7   10  143 245 Apr  2  2019 jdk1.8.0_211

drwxr-xr-x. 2 root root   6 Apr 11  2018 lib

drwxr-xr-x. 2 root root   6 Apr 11  2018 lib64

drwxr-xr-x. 2 root root   6 Apr 11  2018 libexec

drwxr-xr-x. 2 root root   6 Apr 11  2018 sbin

drwxr-xr-x. 5 root root  49 Mar 30  2019 share

drwxr-xr-x. 2 root root   6 Apr 11  2018 src

[root@hadoop-master ~]# vim /etc/profile

export JAVA_HOME=/usr/local/java/

export PATH=$JAVA_HOME/bin:$PATH

export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

[root@hadoop-master ~]# source /etc/profile

[root@hadoop-master ~]# java -version

java version "1.8.0_211"

Java(TM) SE Runtime Environment (build 1.8.0_211-b12)

Java HotSpot(TM) 64-Bit Server VM (build 25.211-b12, mixed mode)

部署hadoop

[root@hadoop-master ~]# wget https://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-2.7.7/hadoop-2.7.7.tar.gz

[root@hadoop-master ~]# tar xf hadoop-2.7.7.tar.gz -C /opt/

[root@hadoop-master ~]# vim /etc/profile

export HADOOP_HOME=/opt/hadoop-2.7.7

export PATH=$HADOOP_HOME/bin:$PATH

[root@hadoop-master ~]# source /etc/profile

[root@hadoop-master ~]# cd /opt/hadoop-2.7.7/

[root@hadoop-master hadoop-2.7.7]# ls

bin  etc  include  lib  libexec  LICENSE.txt  NOTICE.txt  README.txt  sbin  share

[root@hadoop-master hadoop-2.7.7]# cd etc/hadoop/

[root@hadoop-master hadoop]# ll

total 152

-rw-r--r-- 1 1000 ftp  4436 Jul 19  2018 capacity-scheduler.xml

-rw-r--r-- 1 1000 ftp  1335 Jul 19  2018 configuration.xsl

-rw-r--r-- 1 1000 ftp   318 Jul 19  2018 container-executor.cfg

-rw-r--r-- 1 1000 ftp   774 Jul 19  2018 core-site.xml

-rw-r--r-- 1 1000 ftp  3670 Jul 19  2018 hadoop-env.cmd

-rw-r--r-- 1 1000 ftp  4224 Jul 19  2018 hadoop-env.sh

-rw-r--r-- 1 1000 ftp  2598 Jul 19  2018 hadoop-metrics2.properties

-rw-r--r-- 1 1000 ftp  2490 Jul 19  2018 hadoop-metrics.properties

-rw-r--r-- 1 1000 ftp  9683 Jul 19  2018 hadoop-policy.xml

-rw-r--r-- 1 1000 ftp   775 Jul 19  2018 hdfs-site.xml

-rw-r--r-- 1 1000 ftp  1449 Jul 19  2018 httpfs-env.sh

-rw-r--r-- 1 1000 ftp  1657 Jul 19  2018 httpfs-log4j.properties

-rw-r--r-- 1 1000 ftp    21 Jul 19  2018 httpfs-signature.secret

-rw-r--r-- 1 1000 ftp   620 Jul 19  2018 httpfs-site.xml

-rw-r--r-- 1 1000 ftp  3518 Jul 19  2018 kms-acls.xml

-rw-r--r-- 1 1000 ftp  1527 Jul 19  2018 kms-env.sh

-rw-r--r-- 1 1000 ftp  1631 Jul 19  2018 kms-log4j.properties

-rw-r--r-- 1 1000 ftp  5540 Jul 19  2018 kms-site.xml

-rw-r--r-- 1 1000 ftp 11801 Jul 19  2018 log4j.properties

-rw-r--r-- 1 1000 ftp   951 Jul 19  2018 mapred-env.cmd

-rw-r--r-- 1 1000 ftp  1383 Jul 19  2018 mapred-env.sh

-rw-r--r-- 1 1000 ftp  4113 Jul 19  2018 mapred-queues.xml.template

-rw-r--r-- 1 1000 ftp   758 Jul 19  2018 mapred-site.xml.template

-rw-r--r-- 1 1000 ftp    10 Jul 19  2018 slaves

-rw-r--r-- 1 1000 ftp  2316 Jul 19  2018 ssl-client.xml.example

-rw-r--r-- 1 1000 ftp  2697 Jul 19  2018 ssl-server.xml.example

-rw-r--r-- 1 1000 ftp  2250 Jul 19  2018 yarn-env.cmd

-rw-r--r-- 1 1000 ftp  4567 Jul 19  2018 yarn-env.sh

-rw-r--r-- 1 1000 ftp   690 Jul 19  2018 yarn-site.xml

修改hadoop-env.sh和yarn-env.sh中JAVA_HOME参数

[root@hadoop-master hadoop]# sed -i 's#export JAVA_HOME=.*#export JAVA_HOME='/usr/local/java'#g' hadoop-env.sh

[root@hadoop-master hadoop]# sed -i 's#.*export JAVA_HOME=.*#export JAVA_HOME='/usr/local/java'#g' yarn-env.sh

[root@hadoop-master hadoop]# grep "export JAVA_HOME" hadoop-env.sh

export JAVA_HOME=/usr/local/java

[root@hadoop-master hadoop]# grep "export JAVA_HOME" yarn-env.sh

export JAVA_HOME=/usr/local/java

编辑core-site.xml

[root@hadoop-master hadoop]# vim core-site.xml

<configuration>

   <property>

      <name>fs.default.name</name>

      <value>hdfs://192.168.131.129:9000</value>

   </property>

   <property>

      <name>hadoop.tmp.dir</name>

      <value>/opt/hadoop/tmp</value>

   </property>

</configuration>

编辑hdfs-site.xml

[root@hadoop-master hadoop]# vim hdfs-site.xml

<configuration>

   <property>

      <name>dfs.replication</name>

      <value>2</value>

   </property>

   <property>

      <name>dfs.namenode.name.dir</name>

      <value>file:/opt/hadoop/dfs/name</value>

   </property>

   <property>

      <name>dfs.datanode.data.dir</name>

      <value>file:/opt/hadoop/dfs/data</value>

   </property>

</configuration>

编辑mapred-site.xml

[root@hadoop-master hadoop]# cp mapred-site.xml.template mapred-site.xml

[root@hadoop-master hadoop]# vim mapred-site.xml

<configuration>

   <property>

      <name>mapreduce.framework.name</name>

      <value>yarn</value>

   </property>

   <property>

      <name>mapreduce.jobhistory.address</name>

      <value>192.168.131.129:10020</value>

   </property>

   <property>

      <name>mapreduce.jobhistory.webapp.address</name>

      <value>192.168.131.129:19888</value>

   </property>

</configuration>

编辑yarn-site.xml

[root@hadoop-master hadoop]# vim yarn-site.xml

<configuration>

<!-- Site specific YARN configuration properties -->

   <property>

      <name>yarn.nodemanager.aux-services</name>

      <value>mapreduce_shuffle</value>

   </property>

   <property>

      <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>

      <value>org.apache.hadoop.mapred.ShuffleHandler</value>

    </property>

   <property>

      <name>yarn.resourcemanager.address</name>

      <value>192.168.131.129:8032</value>

   </property>

   <property>

      <name>yarn.resourcemanager.scheduler.address</name>

      <value>192.168.131.129:8030</value>

   </property>

   <property>

      <name>yarn.resourcemanager.resource-tracker.address</name>

      <value>192.168.131.129:8031</value>

   </property>

   <property>

      <name>yarn.resourcemanager.admin.address</name>

      <value>192.168.131.129:8033</value>

   </property>

   <property>

      <name>yarn.resourcemanager.webapp.address</name>

      <value>192.168.131.129:8088</value>

   </property>

</configuration>

编辑slaves

[root@hadoop-master hadoop]# vim slaves

hadoop-slave1

hadoop-slave2

把配置分发到各个节点

[root@hadoop-master opt]# scp -r hadoop-2.7.7/ hadoop-slave1:/opt/

[root@hadoop-master opt]# scp -r hadoop-2.7.7/ hadoop-slave2:/opt/

在主节点启动hadoop

初始化namenode

[root@hadoop-master ~]# hdfs namenode -format

启动HDFS

[root@hadoop-master ~]# cd /opt/hadoop-2.7.7/sbin/

[root@hadoop-master sbin]# ./start-dfs.sh

启动YARN

[root@hadoop-master sbin]# ./start-yarn.sh

测试

浏览器访问

YARN WEB地址：http://192.168.131.129:8088/

HDFS WEB地址：http://192.168.131.129:50070/

Hadoop分布式集群部署的更多相关文章

Hadoop分布式集群部署(单namenode节点)
Hadoop分布式集群部署系统系统环境: OS: CentOS 6.8 内存:2G CPU:1核 Software:jdk-8u151-linux-x64.rpm hadoop-2.7.4.tar. ...
Hadoop教程(五)Hadoop分布式集群部署安装
Hadoop教程(五)Hadoop分布式集群部署安装 1 Hadoop分布式集群部署安装在hadoop2.0中通常由两个NameNode组成,一个处于active状态,还有一个处于standby状态 ...
hadoop分布式集群部署①
Linux系统的安装和配置.(在VM虚拟机上) 一:安装虚拟机VMware Workstation 14 Pro 以上,虚拟机软件安装完成. 二:创建虚拟机. 三:安装CentOS系统 (1)上面步 ...
大数据系列之Hadoop分布式集群部署
本节目的:搭建Hadoop分布式集群环境环境准备 LZ用OS X系统 ,安装两台Linux虚拟机,Linux系统用的是CentOS6.5:Master Ip:10.211.55.3 ,Slave ...
Hadoop实战：Hadoop分布式集群部署（一）
一.系统参数优化配置 1.1 系统内核参数优化配置修改文件/etc/sysctl.conf,使用sysctl -p命令即时生效. 1 2 3 4 5 6 7 8 9 10 11 12 13 14 ...
一脸懵逼学习Hadoop分布式集群HA模式部署（七台机器跑集群）
1)集群规划:主机名 IP 安装的软件运行的进程master 192.168.199.130 jdk.hadoop ...
Hadoop(HA)分布式集群部署
Hadoop(HA)分布式集群部署和单节点namenode部署其实一样,只是配置文件的不同罢了. 这篇就讲解hadoop双namenode的部署,实现高可用. 系统环境: OS: CentOS 6.8 ...
超快速使用docker在本地搭建hadoop分布式集群
超快速使用docker在本地搭建hadoop分布式集群超快速使用docker在本地搭建hadoop分布式集群学习hadoop集群环境搭建是hadoop入门的必经之路.搭建分布式集群通常有两个办法: ...
基于Hadoop分布式集群YARN模式下的TensorFlowOnSpark平台搭建
1. 介绍在过去几年中,神经网络已经有了很壮观的进展,现在他们几乎已经是图像识别和自动翻译领域中最强者[1].为了从海量数据中获得洞察力,需要部署分布式深度学习.现有的DL框架通常需要为深度学习设置 ...

随机推荐

Pandas系列（十七）-EDA（pandas-profiling）
对于探索性数据分析来说,做数据分析前需要先看一下数据的总体概况,pandas_profiling工具可以快速预览数据. 安装 pip install pandas-profiling 使用 impor ...
在Linux系统(centos7)中，安装VScode，并在VScode上编写HTML网页
[实验目的] 在Linux系统中,搭建编写HTML网页的环境.在VS code官网上,下载VS code安装程序,进行安装.在VS code软件中编写HTML页面,并正确运行. [实验步骤] 1) ...
winform创建桌面快捷方式
//引用IWshRuntimeLibrary COM组件-Windows Script Host Object Model /// <summary> /// 创建快捷方式的类 /// & ...
[ARM汇编]常用ARM汇编指令
【记录一个问题】android下的ucontext协程，因为使用栈上的对象，导致cv::Mat被莫名析构
工作的流程是这样:某个协程在栈上创建task对象,在task对象内有需要返回的cv::Mat. 然后把task放到另一个线程上去执行,然后切换到别的协程,等到工作线程执行完task后,再唤醒协程. 这 ...
ArrayList实现类
特点数组结构实现,查询快,增删慢运行效率高,线程不安全可重复常用方法 Modifier and Type Method and Description boolean add(E e) 将指定 ...
java匿名内部类概述
1 package face_09; 2 /* 3 * 匿名内部类.就是内部类的简写格式. 4 * 必须有前提: 5 * 内部类必须继承或者实现一个外部类或者接口. 6 * 匿名内部类:其实就是一个匿 ...
jdk11+安装（win）
jdk11+安装(win) 官网下载官网下载地址:https://adoptopenjdk.net/index.html 选择合适的版本安装运行下载的 MSI 包下一步选择安装位置,下一步 ...
学习Java第14天
今天成功安装了MySQL Visual Studio Code 准备试着学习HTML+CSS了明天开始认识HTML和CSS学会软件的使用今天安装调试这些东西属实费了点劲,可能电脑配置较低吧,还有点 ...
jsp 中 include指令用法， <%@ include file="..."%> 和 <jsp:include page="..." flush="true" />的区别？
原文链接https://blog.csdn.net/u012187452/article/details/51779052 1. 什么是jsp 文件? 个人理解. jsp 是一个容器,可以将我们编写 ...

Hadoop分布式集群部署

环境准备

配置免密

配置java环境

部署hadoop

修改hadoop-env.sh和yarn-env.sh中JAVA_HOME参数

编辑core-site.xml

编辑hdfs-site.xml

编辑mapred-site.xml

编辑yarn-site.xml

编辑slaves

把配置分发到各个节点

在主节点启动hadoop

测试

Hadoop分布式集群部署的更多相关文章

随机推荐

热门专题