cat /etc/ecm/hadoop-conf/fair-scheduler.xml

<?xml version="1.0"?>

<allocations>

<aclSubmitApps>*</aclSubmitApps>

<weight>2</weight>

<minResources>10000 mb, 10vcores</minResources>

<maxChildResources>34000 mb,24 vcores</maxChildResources>

<maxRunningApps>50</maxRunningApps>

<maxAMShare>1</maxAMShare>

<maxResources>400000 mb, 200vcores</maxResources> #限制队列最大使用资源

<aclAdministerApps>*</aclAdministerApps>

<schedulingPolicy>fair</schedulingPolicy>

<queue name="default">

<aclSubmitApps>*</aclSubmitApps>

<minResources>10000 mb, 10vcores</minResources>

<aclAdministerApps>*</aclAdministerApps>

<weight>1</weight>

<maxRunningApps>10</maxRunningApps>

<maxAMShare>0.5</maxAMShare>

<maxResources>200000 mb, 100vcores</maxResources>

</queue>

<queue name="collects">

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

<weight>8</weight>

<maxAMShare>0.8</maxAMShare>

<minResources>50 mb, 2vcores</minResources>

<maxResources>400000 mb, 200vcores</maxResources>

<maxRunningApps>50</maxRunningApps>

</queue>

<queue name="data_bi">

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

<weight>4</weight>

<minResources>100 mb, 1vcores</minResources>

<maxResources>30000 mb, 50vcores</maxResources>

<maxRunningApps>5</maxRunningApps>

</queue>

<queue name="opay_collects">

<weight>20</weight>

<minResources>10 mb, 1vcores</minResources>

<maxResources>400000 mb, 200vcores</maxResources>

<maxRunningApps>20</maxRunningApps>

<maxAMShare>0.5</maxAMShare>

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

</queue>

<queue name="opos_collects">

<weight>5</weight>

<minResources>10 mb, 1vcores</minResources>

<maxResources>80000 mb, 50vcores</maxResources>

<maxRunningApps>10</maxRunningApps>

</queue>

<queue name="users" type="parent">

<weight>5</weight>

<minResources>10 mb, 1vcores</minResources>

<maxResources>10000 mb, 150vcores</maxResources>

<maxRunningApps>30</maxRunningApps>

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

</queue>

<queue name="airflow">

<schedulingPolicy>fair</schedulingPolicy>

<aclSubmitApps>*</aclSubmitApps>

<aclAdministerApps>*</aclAdministerApps>

<weight>8</weight>

<minResources>10 mb, 2vcores</minResources>

<maxResources>200000 mb, 150vcores</maxResources>

<maxRunningApps>30</maxRunningApps>

</queue>

<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>

<userMaxAppsDefault>50</userMaxAppsDefault>

<queueMaxAppsDefault>50</queueMaxAppsDefault>

<queueMaxAMShareDefault>0.5</queueMaxAMShareDefault>

<defaultFairSharePreemptionThreshold>0.5</defaultFairSharePreemptionThreshold>

<queueMaxResourcesDefault>34000 mb,24vcores</queueMaxResourcesDefault>

<defaultFairSharePreemptionTimeout>9223372036854775807</defaultFairSharePreemptionTimeout>

<defaultMinSharePreemptionTimeout>9223372036854775807</defaultMinSharePreemptionTimeout>

</allocations>

#新的xml, 不带root限制: 放emr的yarn-配置-fair-scheduler

<?xml version="1.0" encoding="utf-8"?>
<allocations>
<queue name="root">
<queue name="default">
<aclSubmitApps>*</aclSubmitApps>
<minResources>10000 mb, 10vcores</minResources>
<aclAdministerApps>*</aclAdministerApps>
<weight>1</weight>
<maxRunningApps>10</maxRunningApps>
<maxAMShare>0.5</maxAMShare>
<maxResources>200000 mb, 100vcores</maxResources>
</queue>
<queue name="collects">
<schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>*</aclAdministerApps>
<weight>8</weight>
<maxAMShare>0.8</maxAMShare>
<minResources>50 mb, 2vcores</minResources>
<maxResources>400000 mb, 200vcores</maxResources>
<maxRunningApps>50</maxRunningApps>
</queue>
<queue name="data_bi">
<schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>*</aclAdministerApps>
<weight>4</weight>
<minResources>100 mb, 1vcores</minResources>
<maxResources>30000 mb, 50vcores</maxResources>
<maxRunningApps>5</maxRunningApps>
</queue>
<queue name="opay_collects">
<weight>20</weight>
<minResources>10 mb, 1vcores</minResources>
<maxResources>400000 mb, 200vcores</maxResources>
<maxRunningApps>20</maxRunningApps>
<maxAMShare>0.5</maxAMShare>
<schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>*</aclAdministerApps>
</queue>
<queue name="opos_collects">
<weight>5</weight>
<minResources>10 mb, 1vcores</minResources>
<maxResources>80000 mb, 50vcores</maxResources>
<maxRunningApps>10</maxRunningApps>
</queue>
<queue name="users" type="parent">
<weight>5</weight>
<minResources>10 mb, 1vcores</minResources>
<maxResources>10000 mb, 150vcores</maxResources>
<maxRunningApps>30</maxRunningApps>
<schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>*</aclAdministerApps>
</queue>
<queue name="airflow">
<schedulingPolicy>fair</schedulingPolicy>
<aclSubmitApps>*</aclSubmitApps>
<aclAdministerApps>*</aclAdministerApps>
<weight>8</weight>
<minResources>10 mb, 2vcores</minResources>
<maxResources>200000 mb, 150vcores</maxResources>
<maxRunningApps>30</maxRunningApps>
</queue>
</queue>
<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>
<userMaxAppsDefault>50</userMaxAppsDefault>
<queueMaxAppsDefault>50</queueMaxAppsDefault>
<queueMaxAMShareDefault>0.5</queueMaxAMShareDefault>
<defaultFairSharePreemptionThreshold>0.5</defaultFairSharePreemptionThreshold>
<queueMaxResourcesDefault>34000 mb,24vcores</queueMaxResourcesDefault>
<defaultFairSharePreemptionTimeout>9223372036854775807</defaultFairSharePreemptionTimeout>
<defaultMinSharePreemptionTimeout>9223372036854775807</defaultMinSharePreemptionTimeout>
</allocations>

EMR的fair-scheduler.xml的更多相关文章

  1. Fair Scheduler 队列设置经验总结

    Fair Scheduler 队列设置经验总结 由于公司的hadoop集群的计算资源不是很充足,需要开启yarn资源队列的资源抢占.在使用过程中,才明白资源抢占的一些特点.在这里总结一下. 只有一个队 ...

  2. 三：Fair Scheduler 公平调度器

    参考资料: http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/FairScheduler.html http://h ...

  3. Fair Scheduler中的Delay Schedule分析

    延迟调度的主要目的是提高数据本地性(data locality),减少数据在网络中的传输.对于那些输入数据不在本地的MapTask,调度器将会延迟调度他们,而把slot分配给那些具备本地性的MapTa ...

  4. Hadoop学习之--Fair Scheduler作业调度分析

    Fair Scheduler调度器同步心跳分配任务的过程简单来讲会经历以下环节: 1. 对map/reduce是否已经达到资源上限的循环判断 2. 对pool队列根据Fair算法排序 3.然后循环po ...

  5. YARN的Fair Scheduler和Capacity Scheduler

    关于Scheduler YARN有四种调度机制:Fair Schedule,Capacity Schedule,FIFO以及Priority: 其中Fair Scheduler是资源池机制,进入到里面 ...

  6. Hadoop的三种调度器FIFO、Capacity Scheduler、Fair Scheduler（转载）

    目前Hadoop有三种比较流行的资源调度器:FIFO .Capacity Scheduler.Fair Scheduler.目前Hadoop2.7默认使用的是Capacity Scheduler容量调 ...

  7. fair scheduler配置

    <property>    <name>yarn.resourcemanager.scheduler.class</name>    <value>or ...

  8. Yarn参数优化(Fair Scheduler版本)

    YARN 自从hadoop2.0之后, 我们可以使用apache yarn 来对集群资源进行管理.yarn把可以把资源(内存,CPU)以Container的方式进行划分隔离.YARN会管理集群中所有机 ...

  9. Linux 2.6 完全公平调度算法CFS（Completely Fair Scheduler） 分析

    转会http://www.ibm.com/developerworks/cn/linux/l-completely-fair-scheduler/index.html? ca=drs-cn-0125 ...

  10. 利用yarn capacity scheduler在EMR集群上实现大集群的多租户的集群资源隔离和quota限制

    转自:https://m.aliyun.com/yunqi/articles/79700 背景 使用过hadoop的人基本都会考虑集群里面资源的调度和优先级的问题,假设你现在所在的公司有一个大hado ...

随机推荐

  1. 2019-2020 ICPC, NERC, Southern and Volga Russian Regional Contest

    目录 Contest Info Solutions A. Berstagram B. The Feast and the Bus C. Trip to Saint Petersburg E. The ...

  2. CNN模型合集 | 1 LeNet

    1.1 LeNet的设计思想 1998年LeCun提出,经典结构,3层,五脏俱全(卷积层.Pooling层.FC网络.Sigmod层),对标传统神经网络.主要设计贡献 局部感受野(local rece ...

  3. ROS手动编写服务端和客户端service demo(C++)

    service demo 原理和 topic 通信方式很像 点击打开链接,因此 1.srv : 进入 service_demo 创建 srv 文件夹,创建 Greeting.srv,将以下代码插入: ...

  4. ajax案例_校验用户名

    目录 ajax案例_校验用户名 代码下载 需求 流程 搭建环境 开发代码 1_jsp 1_servlet 1_service.dao 2_servlet 2_jsp 测试后,功能实现,完结撒花 aja ...

  5. linux经常用的命令

    常用 安装包   centos   yum    /   Ubuntu  Debian  apt-get clear :清空终端       [cmd 下是cls] vi/vim  编辑器    详情 ...

  6. zabbix (三) 所需磁盘容量计算

    一.zabbix 磁盘容量大小 zabbix的数据库大小取决于NVPS(new values per second)大小,NVPS从总体上反应了处理速度,与监控项的数目,监控项的类型,取值间隔,His ...

  7. 说出Servlet的生命周期，并说出Servlet和CGI的区别。

    说出Servlet的生命周期,并说出Servlet和CGI的区别. 山治ZHrx5 | 浏览 1377 次 推荐于2016-09-16 22:39:19 最佳答案 Servlet的生命周期分为5个阶段 ...

  8. chtMultiRegionSimpleFoam求解器的热源不在边界上【翻译】

    翻译自:CFD-online 帖子地址:http://www.cfd-online.com/Forums/openfoam-solving/126777-chtmultiregionsimplefoa ...

  9. Wireshark 序

    1. Foreword 前言 2. Who should read this document? 谁适合读该文档? 3. Acknowledgements 致谢 4. About this docum ...

  10. web目录

    Proj app controllers jobs models view user xxx.html init.go conf message public img js css html