hadoop实战之分布式模式
环境
192.168.1.101 host101
192.168.1.102 host102
1.安装配置host101
[root@host101 ~]# cat /etc/hosts |grep 192
192.168.1.101 host101
192.168.1.102 host102
[root@host101 ~]# rpm -ivh jdk-8u91-linux-x64.rpm
[root@host101 ~]# tar -zxvf hadoop-2.6.4.tar.gz
[root@host101 ~]# mv hadoop-2.6.4 /usr/local/hadoop
[root@host101 ~]# cd /usr/local/hadoop/
[root@host101 hadoop]# vim etc/hadoop/hadoop-env.sh
export JAVA_HOME=/usr/java/latest
export HADOOP_PREFIX=/usr/local/hadoop
[root@host101 hadoop]# vim etc/hadoop/slaves
host101
host102
[root@host101 hadoop]# vim etc/hadoop/core-site.xml
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://host101:9000</value>
</property>
</configuration> [root@host101 hadoop]# mkdir -p /hadoop/
[root@host101 hadoop]# vim etc/hadoop/hdfs-site.xml
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>/hadoop/name/</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>/hadoop/data/</value>
</property>
</configuration>
[root@host101 hadoop]# vim mapred-site.xml
<configuration>
<property>
<name>mapred.job.tracker</name>
<value>host101:9001</value>
</property>
</configuration>
[root@host101 ~]# ssh-keygen
[root@host101 ~]# ssh-copy-id host101
[root@host101 ~]# ssh-copy-id host102
2.安装配置host102
[root@host102 ~]# scp host101:/root/hadoop-2.6.4.tar.gz .
[root@host102 ~]# scp host101:/root/jdk-8u91-linux-x64.rpm . [root@host102 ~]# rpm -ivh jdk-8u91-linux-x64.rpm
[root@host102 ~]# tar -zxvf hadoop-2.6.4.tar.gz
[root@host102 ~]# mv hadoop-2.6.4 /usr/local/hadoop
[root@host102 ~]# ssh-keygen
[root@host102 ~]# ssh-copy-id host101
[root@host102 ~]# ssh-copy-id host102
[root@host102 etc]# cd /usr/local/hadoop/etc/hadoop/
[root@host102 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/mapred-site.xml .
[root@host102 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/slaves .
[root@host102 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/hdfs-site.xml .
[root@host102 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/hadoop-env.sh .
[root@host102 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/core-site.xml .
3.启动hadoop集群
[root@host101 hadoop]# sbin/start-all.sh
This script is Deprecated. Instead use start-dfs.sh and start-yarn.sh
Starting namenodes on [host101]
host101: starting namenode, logging to /usr/local/hadoop/logs/hadoop-root-namenode-host101.out
host101: starting datanode, logging to /usr/local/hadoop/logs/hadoop-root-datanode-host101.out
host102: starting datanode, logging to /usr/local/hadoop/logs/hadoop-root-datanode-host102.out
Starting secondary namenodes [0.0.0.0]
0.0.0.0: starting secondarynamenode, logging to /usr/local/hadoop/logs/hadoop-root-secondarynamenode-host101.out
starting yarn daemons
starting resourcemanager, logging to /usr/local/hadoop/logs/yarn-root-resourcemanager-host101.out
host101: starting nodemanager, logging to /usr/local/hadoop/logs/yarn-root-nodemanager-host101.out
host102: starting nodemanager, logging to /usr/local/hadoop/logs/yarn-root-nodemanager-host102.out [root@host101 hadoop]# bin/hdfs dfs -mkdir /eric
[root@host101 hadoop]# bin/hdfs dfs -ls /
Found 1 items
drwxr-xr-x - root supergroup 0 2016-07-06 12:09 /eric
[root@host101 hadoop]# bin/hadoop dfsadmin -report
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it. Configured Capacity: 37576769536 (35.00 GB)
Present Capacity: 29447094272 (27.42 GB)
DFS Remaining: 29447086080 (27.42 GB)
DFS Used: 8192 (8 KB)
DFS Used%: 0.00%
Under replicated blocks: 0
Blocks with corrupt replicas: 0
Missing blocks: 0 -------------------------------------------------
Live datanodes (2): Name: 192.168.1.101:50010 (host101)
Hostname: host101
Decommission Status : Normal
Configured Capacity: 18788384768 (17.50 GB)
DFS Used: 4096 (4 KB)
Non DFS Used: 3870842880 (3.61 GB)
DFS Remaining: 14917537792 (13.89 GB)
DFS Used%: 0.00%
DFS Remaining%: 79.40%
Configured Cache Capacity: 0 (0 B)
Cache Used: 0 (0 B)
Cache Remaining: 0 (0 B)
Cache Used%: 100.00%
Cache Remaining%: 0.00%
Xceivers: 1
Last contact: Wed Jul 06 12:10:07 CST 2016 Name: 192.168.1.102:50010 (host102)
Hostname: host102
Decommission Status : Normal
Configured Capacity: 18788384768 (17.50 GB)
DFS Used: 4096 (4 KB)
Non DFS Used: 4258832384 (3.97 GB)
DFS Remaining: 14529548288 (13.53 GB)
DFS Used%: 0.00%
DFS Remaining%: 77.33%
Configured Cache Capacity: 0 (0 B)
Cache Used: 0 (0 B)
Cache Remaining: 0 (0 B)
Cache Used%: 100.00%
Cache Remaining%: 0.00%
Xceivers: 1
Last contact: Wed Jul 06 12:10:07 CST 2016
[root@host101 hadoop]# jps
3920 DataNode
3811 NameNode
4056 SecondaryNameNode
4299 Jps
4. 测试集群
NameNode http://192.168.1.101:50070/dfshealth.html
ResourceManager http://192.168.1.101:8088/cluster
http://192.168.1.101:8042/node [root@host101 hadoop]# bin/hadoop fs -mkdir /eric/input
[root@host101 hadoop]# bin/hadoop fs -copyFromLocal etc/hadoop/*.xml /eric/input
[root@host101 hadoop]# bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.4.jar grep /eric/input /eric/output 'dfs[a-z.]+'
[root@host101 hadoop]# bin/hadoop fs -ls /eric/output/
Found 2 items
-rw-r--r-- 1 root supergroup 0 2016-07-06 12:38 /eric/output/_SUCCESS
-rw-r--r-- 1 root supergroup 77 2016-07-06 12:38 /eric/output/part-r-00000
[root@host101 hadoop]# bin/hadoop fs -cat /eric/output/part-r-00000
1 dfsadmin
1 dfs.replication
1 dfs.namenode.name.dir
1 dfs.datanode.data.dir
[root@host101 hadoop]# sbin/stop-all.sh
This script is Deprecated. Instead use stop-dfs.sh and stop-yarn.sh
Stopping namenodes on [host101]
host101: stopping namenode
host101: stopping datanode
host102: stopping datanode
Stopping secondary namenodes [0.0.0.0]
0.0.0.0: stopping secondarynamenode
stopping yarn daemons
stopping resourcemanager
host101: stopping nodemanager
host102: no nodemanager to stop
no proxyserver to stop
5. 动态添加节点
[root@host101 hadoop]# echo "192.168.1.161 host161" >> /etc/hosts
[root@host102 hadoop]# echo "192.168.1.161 host161" >> /etc/hosts
[root@host101 hadoop]# ssh-copy-id host161
[root@host102 hadoop]# ssh-copy-id host161
[root@host161 ~]# ssh-copy-id host161
[root@host161 ~]# ssh-copy-id host101
[root@host161 ~]# ssh-copy-id host102
[root@host102 ~]# scp host101:/root/hadoop-2.6.4.tar.gz .
[root@host102 ~]# scp host101:/root/jdk-8u91-linux-x64.rpm .
[root@host102 ~]# rpm -ivh jdk-8u91-linux-x64.rpm
[root@host102 ~]# tar -zxvf hadoop-2.6.4.tar.gz
[root@host102 ~]# mv hadoop-2.6.4 /usr/local/hadoop
[root@host101 hadoop]# echo 'host161' >> etc/hadoop/slaves
[root@host102 hadoop]# echo 'host161' >> etc/hadoop/slaves
[root@host161 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/mapred-site.xml .
[root@host161 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/slaves .
[root@host161 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/hdfs-site.xml .
[root@host161 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/hadoop-env.sh .
[root@host161 hadoop]# scp host101:/usr/local/hadoop/etc/hadoop/core-site.xml .
[root@host161 hadoop]# sbin/hadoop-daemon.sh start datanode
starting datanode, logging to /usr/local/hadoop/logs/hadoop-root-datanode-host161.out [root@host101 hadoop]# bin/hadoop dfsadmin -report
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it. Configured Capacity: 56365154304 (52.49 GB)
Present Capacity: 44354347008 (41.31 GB)
DFS Remaining: 44192788480 (41.16 GB)
DFS Used: 161558528 (154.07 MB)
DFS Used%: 0.36%
Under replicated blocks: 0
Blocks with corrupt replicas: 0
Missing blocks: 0 -------------------------------------------------
Live datanodes (3): Name: 192.168.1.101:50010 (host101)
Hostname: host101
Decommission Status : Normal
Configured Capacity: 18788384768 (17.50 GB)
DFS Used: 161546240 (154.06 MB)
Non DFS Used: 3873861632 (3.61 GB)
DFS Remaining: 14752976896 (13.74 GB)
DFS Used%: 0.86%
DFS Remaining%: 78.52%
Configured Cache Capacity: 0 (0 B)
Cache Used: 0 (0 B)
Cache Remaining: 0 (0 B)
Cache Used%: 100.00%
Cache Remaining%: 0.00%
Xceivers: 1
Last contact: Wed Jul 06 16:02:19 CST 2016 Name: 192.168.1.161:50010 (host161)
Hostname: host161
Decommission Status : Normal
Configured Capacity: 18788384768 (17.50 GB)
DFS Used: 4096 (4 KB)
Non DFS Used: 3877494784 (3.61 GB)
DFS Remaining: 14910885888 (13.89 GB)
DFS Used%: 0.00%
DFS Remaining%: 79.36%
Configured Cache Capacity: 0 (0 B)
Cache Used: 0 (0 B)
Cache Remaining: 0 (0 B)
Cache Used%: 100.00%
Cache Remaining%: 0.00%
Xceivers: 1
Last contact: Wed Jul 06 16:02:20 CST 2016 Name: 192.168.1.102:50010 (host102)
Hostname: host102
Decommission Status : Normal
Configured Capacity: 18788384768 (17.50 GB)
DFS Used: 8192 (8 KB)
Non DFS Used: 4259450880 (3.97 GB)
DFS Remaining: 14528925696 (13.53 GB)
DFS Used%: 0.00%
DFS Remaining%: 77.33%
Configured Cache Capacity: 0 (0 B)
Cache Used: 0 (0 B)
Cache Remaining: 0 (0 B)
Cache Used%: 100.00%
Cache Remaining%: 0.00%
Xceivers: 1
Last contact: Wed Jul 06 16:02:19 CST 2016
hadoop实战之分布式模式的更多相关文章
- Hadoop基础-完全分布式模式部署yarn日志聚集功能
Hadoop基础-完全分布式模式部署yarn日志聚集功能 作者:尹正杰 版权声明:原创作品,谢绝转载!否则将追究法律责任. 其实我们不用配置也可以在服务器后台通过命令行的形式查看相应的日志,但为了更方 ...
- Hadoop三种安装模式：单机模式，伪分布式，真正分布式
Hadoop三种安装模式:单机模式,伪分布式,真正分布式 一 单机模式standalone单 机模式是Hadoop的默认模式.当首次解压Hadoop的源码包时,Hadoop无法了解硬件安装环境,便保守 ...
- 王家林的“云计算分布式大数据Hadoop实战高手之路---从零开始”的第十一讲Hadoop图文训练课程：MapReduce的原理机制和流程图剖析
这一讲我们主要剖析MapReduce的原理机制和流程. “云计算分布式大数据Hadoop实战高手之路”之完整发布目录 云计算分布式大数据实战技术Hadoop交流群:312494188,每天都会在群中发 ...
- 云计算分布式大数据Hadoop实战高手之路第七讲Hadoop图文训练课程：通过HDFS的心跳来测试replication具体的工作机制和流程
这一讲主要深入使用HDFS命令行工具操作Hadoop分布式集群,主要是通过实验的配置hdfs-site.xml文件的心跳来测试replication具体的工作和流程. 通过HDFS的心跳来测试repl ...
- 云计算分布式大数据Hadoop实战高手之路第八讲Hadoop图文训练课程：Hadoop文件系统的操作实战
本讲通过实验的方式讲解Hadoop文件系统的操作. “云计算分布式大数据Hadoop实战高手之路”之完整发布目录 云计算分布式大数据实战技术Hadoop交流群:312494188,每天都会在群中发布云 ...
- Hadoop伪分布式模式部署
Hadoop的安装有三种执行模式: 单机模式(Local (Standalone) Mode):Hadoop的默认模式,0配置.Hadoop执行在一个Java进程中.使用本地文件系统.不使用HDFS, ...
- hadoop的安装和配置（三）完全分布式模式
博主会用三篇文章为大家详细说明hadoop的三种模式: 本地模式 伪分布模式 完全分布模式 完全分布式模式: 前面已经说了本地模式和伪分布模式,这两种在hadoop的应用中并不用于实际,因为几乎没人会 ...
- 使用docker搭建hadoop环境，并配置伪分布式模式
docker 1.下载docker镜像 docker pull registry.cn-hangzhou.aliyuncs.com/kaibb/hadoop:latest 注:此镜像为阿里云个人上传镜 ...
- Hadoop Single Node Setup（hadoop本地模式和伪分布式模式安装-官方文档翻译 2.7.3）
Purpose(目标) This document describes how to set up and configure a single-node Hadoop installation so ...
随机推荐
- 樱花漫地集于我心，蝶舞纷飞祈愿相随---总结 顕出：void-sampling 显示：void-sampling
知识点: 分支语句,四则运算,(￣y▽￣)~* 实验过程中遇到的问题及解决方法: 忘记换行,忘记代码,输入法切换(´∀｀*) 暂时还得照书写,*★,°*:.☆\(￣▽￣)/$:*.°★* 实验心得体会 ...
- 类库、委托、is as运算符、泛型集合
类库: 说白了,就是让别人调用你写的方法,并且不让别人看到你是怎么实现的. 如果有功能你不会做,需要别人帮忙,那么你的同事可以帮你写好一个类,然后你来调用这个类中的方法,完成你的项目. 1.C#源代码 ...
- PDF 补丁丁 0.4.2.950 测试版发布：按文件夹合并生成单独的PDF文件
新的测试版实现了将文件夹的内容合并为单独的PDF文件的功能.以下图为例讲解操作步骤. 点击工具栏的“合并文件”按钮,打开合并功能. 设M盘下有“test”和“test2”文件夹,里面包含了需要合并的文 ...
- Windows下搭建IOS开发环境（一）
原文:http://blog.csdn.net/shangyuan21/article/details/18153605 我们都知道开发iPhone等ios平台的移动应用时需要使用Mac本,但是Mac ...
- 使用Maven构建RichFaces 4.x项目
使用Maven构建RichFaces 4.x项目 目录 开始之前 第一步 - 创建Maven项目 第二布 - 添加依赖文件 第三步 - 配置RichFaces 第四步 - 创建显示页面 开始之前 本文 ...
- C#微信公众号接口开发实例-高级接口-申请带参数的二维码
最近公司涉及到微信绑定用户,做了高级接口-申请带参数的二维码,总结了下微信开发接口.微信接口开发都是除了消息用的xml 回复基本上都是用json的形式传递信息(post/get),开发的方法基本都是一 ...
- qml 封装技巧-利用数据来进行适配
Text属于用的频率比较高而且需要定义的地方又比较多的地方,看一下如下的把Text封装成Label进行使用. 使用的例子: Label { id: titleLabel anchors { left: ...
- 常用排序算法的Java实现 - 1
学习编程语言时, 我们会接触到许多排序算法, 这里总结了一下常见的排序算法. 不定期更新. * 其实在Java中存在如Collections.sort()这样的方法来自动为我们排序, 不过学习排序算法 ...
- highcharts异步获取数据
页面异步代码 $(function () { var chart_validatestatics; $(document).ready(function () { var options_valida ...
- Jquery EasyUI Tree .net实例
图片: 针对tree: 数据库: CREATE TABLE [dbo].[SystemModel]( [Id] [,) NOT NULL, [Name] [nvarchar]() NULL, [Fat ...