#!/usr/bin/env python
import optparse
from pprint import pprint
import logging
import sys
import math
import ast''' Reserved for OS + DN + NM,  Map: Memory => Reservation '''
reservedStack = { 4:1, 8:2, 16:2, 24:4, 48:6, 64:8, 72:8, 96:12, 128:24, 256:32, 512:64}
''' Reserved for HBase. Map: Memory => Reservation '''reservedHBase = {4:1, 8:1, 16:2, 24:4, 48:8, 64:8, 72:8, 96:16, 128:24, 256:32, 512:64}
GB = 1024def getMinContainerSize(memory):if (memory <= 4):return 256elif (memory <= 8):return 512elif (memory <= 24):return 1024else:return 2048passdef getReservedStackMemory(memory):if (reservedStack.has_key(memory)):return reservedStack[memory]if (memory <= 4):ret = 1elif (memory >= 512):ret = 64else:ret = 1return retdef getReservedHBaseMem(memory):if (reservedHBase.has_key(memory)):return reservedHBase[memory]if (memory <= 4):ret = 1elif (memory >= 512):ret = 64else:ret = 2return retdef main():log = logging.getLogger(__name__)out_hdlr = logging.StreamHandler(sys.stdout)out_hdlr.setFormatter(logging.Formatter(' %(message)s'))out_hdlr.setLevel(logging.INFO)log.addHandler(out_hdlr)log.setLevel(logging.INFO)parser = optparse.OptionParser()memory = 0cores = 0disks = 0hbaseEnabled = Trueparser.add_option('-c', '--cores', default = 16,help = 'Number of cores on each host')parser.add_option('-m', '--memory', default = 64, help = 'Amount of Memory on each host in GB')parser.add_option('-d', '--disks', default = 4, help = 'Number of disks on each host')parser.add_option('-k', '--hbase', default = "True",help = 'True if HBase is installed, False is not')(options, args) = parser.parse_args()cores = int (options.cores)memory = int (options.memory)disks = int (options.disks)hbaseEnabled = ast.literal_eval(options.hbase)log.info("Using cores=" +  str(cores) + " memory=" + str(memory) + "GB" +" disks=" + str(disks) + " hbase=" + str(hbaseEnabled))minContainerSize = getMinContainerSize(memory)reservedStackMemory = getReservedStackMemory(memory)reservedHBaseMemory = 0if (hbaseEnabled):reservedHBaseMemory = getReservedHBaseMem(memory)reservedMem = reservedStackMemory + reservedHBaseMemoryusableMem = memory - reservedMemmemory -= (reservedMem)if (memory < 2):memory = 2reservedMem = max(0, memory - reservedMem)memory *= GBcontainers = int (min(2 * cores,min(math.ceil(1.8 * float(disks)),memory/minContainerSize)))if (containers <= 2):containers = 3log.info("Profile: cores=" + str(cores) + " memory=" + str(memory) + "MB"+ " reserved=" + str(reservedMem) + "GB" + " usableMem="+ str(usableMem) + "GB" + " disks=" + str(disks))container_ram =  abs(memory/containers)if (container_ram > GB):container_ram = int(math.floor(container_ram / 512)) * 512log.info("Num Container=" + str(containers))log.info("Container Ram=" + str(container_ram) + "MB")log.info("Used Ram=" + str(int (containers*container_ram/float(GB))) + "GB")log.info("Unused Ram=" + str(reservedMem) + "GB")log.info("yarn.scheduler.minimum-allocation-mb=" + str(container_ram))log.info("yarn.scheduler.maximum-allocation-mb=" + str(containers*container_ram))log.info("yarn.nodemanager.resource.memory-mb=" + str(containers*container_ram))map_memory = container_ramreduce_memory = 2*container_ram if (container_ram <= 2048) else container_ramam_memory = max(map_memory, reduce_memory)log.info("mapreduce.map.memory.mb=" + str(map_memory))log.info("mapreduce.map.java.opts=-Xmx" + str(int(0.8 * map_memory)) +"m")log.info("mapreduce.reduce.memory.mb=" + str(reduce_memory))log.info("mapreduce.reduce.java.opts=-Xmx" + str(int(0.8 * reduce_memory)) + "m")log.info("yarn.app.mapreduce.am.resource.mb=" + str(am_memory))log.info("yarn.app.mapreduce.am.command-opts=-Xmx" + str(int(0.8*am_memory)) + "m")log.info("mapreduce.task.io.sort.mb=" + str(int(0.4 * map_memory)))passif __name__ == '__main__':try:main()except(KeyboardInterrupt, EOFError):print("\nAborting ... Keyboard Interrupt.")sys.exit(1)

python yarn-utils.py -c 32 -m 128 -d 7 -k False

 Using cores=32 memory=128GB disks=7 hbase=FalseProfile: cores=32 memory=106496MB reserved=24GB usableMem=104GB disks=7Num Container=13Container Ram=8192MBUsed Ram=104GBUnused Ram=24GByarn.scheduler.minimum-allocation-mb=8192yarn.scheduler.maximum-allocation-mb=106496yarn.nodemanager.resource.memory-mb=106496mapreduce.map.memory.mb=8192mapreduce.map.java.opts=-Xmx6553mmapreduce.reduce.memory.mb=8192mapreduce.reduce.java.opts=-Xmx6553myarn.app.mapreduce.am.resource.mb=8192yarn.app.mapreduce.am.command-opts=-Xmx6553mmapreduce.task.io.sort.mb=3276

<property><name>yarn.nodemanager.resource.memory-mb</name><value>106496</value></property><property><name>yarn.scheduler.minimum-allocation-mb</name><value>2048</value></property><property><name>yarn.scheduler.maximum-allocation-mb</name><value>106496</value></property><property><name>yarn.app.mapreduce.am.resource.mb</name><value>4096</value></property><property><name>yarn.app.mapreduce.am.command-opts</name><value>-Xmx3276m</value></property>

  <property><name>yarn.nodemanager.resource.cpu-vcores</name><value>31</value></property><property><name>yarn.scheduler.maximum-allocation-vcores</name><value>124</value></property>

原创文章，转载请注明： 转载自JavaChen Blog，作者：JavaChen
本文链接地址：http://blog.javachen.com/2015/06/05/yarn-memory-and-cpu-configuration.html
本文基于署名2.5中国大陆许可协议发布，欢迎转载、演绎或用于商业目的，但是必须保留本文署名和文章链接。 如您有任何疑问或者授权方面的协商，请邮件联系我。

YARN的内存和CPU配置