001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one 003 * or more contributor license agreements. See the NOTICE file 004 * distributed with this work for additional information 005 * regarding copyright ownership. The ASF licenses this file 006 * to you under the Apache License, Version 2.0 (the 007 * "License"); you may not use this file except in compliance 008 * with the License. You may obtain a copy of the License at 009 * 010 * http://www.apache.org/licenses/LICENSE-2.0 011 * 012 * Unless required by applicable law or agreed to in writing, software 013 * distributed under the License is distributed on an "AS IS" BASIS, 014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 015 * See the License for the specific language governing permissions and 016 * limitations under the License. 017 */ 018 019 package org.apache.hadoop.yarn.conf; 020 021 import java.net.InetAddress; 022 import java.net.InetSocketAddress; 023 import java.net.UnknownHostException; 024 import java.util.Arrays; 025 026 import org.apache.hadoop.classification.InterfaceAudience.Public; 027 import org.apache.hadoop.classification.InterfaceStability.Evolving; 028 import org.apache.hadoop.conf.Configuration; 029 import org.apache.hadoop.net.NetUtils; 030 import org.apache.hadoop.util.StringUtils; 031 import org.apache.hadoop.yarn.api.ApplicationConstants; 032 033 import com.google.common.base.Joiner; 034 035 @Public 036 @Evolving 037 public class YarnConfiguration extends Configuration { 038 039 private static final Joiner JOINER = Joiner.on(""); 040 041 private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; 042 private static final String YARN_SITE_XML_FILE = "yarn-site.xml"; 043 044 static { 045 Configuration.addDefaultResource(YARN_DEFAULT_XML_FILE); 046 Configuration.addDefaultResource(YARN_SITE_XML_FILE); 047 } 048 049 //Configurations 050 051 public static final String YARN_PREFIX = "yarn."; 052 053 /** Delay before deleting resource to ease debugging of NM issues */ 054 public static final String DEBUG_NM_DELETE_DELAY_SEC = 055 YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec"; 056 057 //////////////////////////////// 058 // IPC Configs 059 //////////////////////////////// 060 public static final String IPC_PREFIX = YARN_PREFIX + "ipc."; 061 062 /** Factory to create client IPC classes.*/ 063 public static final String IPC_CLIENT_FACTORY_CLASS = 064 IPC_PREFIX + "client.factory.class"; 065 public static final String DEFAULT_IPC_CLIENT_FACTORY_CLASS = 066 "org.apache.hadoop.yarn.factories.impl.pb.RpcClientFactoryPBImpl"; 067 068 /** Factory to create server IPC classes.*/ 069 public static final String IPC_SERVER_FACTORY_CLASS = 070 IPC_PREFIX + "server.factory.class"; 071 public static final String DEFAULT_IPC_SERVER_FACTORY_CLASS = 072 "org.apache.hadoop.yarn.factories.impl.pb.RpcServerFactoryPBImpl"; 073 074 /** Factory to create serializeable records.*/ 075 public static final String IPC_RECORD_FACTORY_CLASS = 076 IPC_PREFIX + "record.factory.class"; 077 public static final String DEFAULT_IPC_RECORD_FACTORY_CLASS = 078 "org.apache.hadoop.yarn.factories.impl.pb.RecordFactoryPBImpl"; 079 080 /** RPC class implementation*/ 081 public static final String IPC_RPC_IMPL = 082 IPC_PREFIX + "rpc.class"; 083 public static final String DEFAULT_IPC_RPC_IMPL = 084 "org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC"; 085 086 //////////////////////////////// 087 // Resource Manager Configs 088 //////////////////////////////// 089 public static final String RM_PREFIX = "yarn.resourcemanager."; 090 091 /** The address of the applications manager interface in the RM.*/ 092 public static final String RM_ADDRESS = 093 RM_PREFIX + "address"; 094 public static final int DEFAULT_RM_PORT = 8032; 095 public static final String DEFAULT_RM_ADDRESS = 096 "0.0.0.0:" + DEFAULT_RM_PORT; 097 098 /** The number of threads used to handle applications manager requests.*/ 099 public static final String RM_CLIENT_THREAD_COUNT = 100 RM_PREFIX + "client.thread-count"; 101 public static final int DEFAULT_RM_CLIENT_THREAD_COUNT = 50; 102 103 /** The Kerberos principal for the resource manager.*/ 104 public static final String RM_PRINCIPAL = 105 RM_PREFIX + "principal"; 106 107 /** The address of the scheduler interface.*/ 108 public static final String RM_SCHEDULER_ADDRESS = 109 RM_PREFIX + "scheduler.address"; 110 public static final int DEFAULT_RM_SCHEDULER_PORT = 8030; 111 public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:" + 112 DEFAULT_RM_SCHEDULER_PORT; 113 114 /** Miniumum request grant-able by the RM scheduler. */ 115 public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 116 YARN_PREFIX + "scheduler.minimum-allocation-mb"; 117 public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 1024; 118 public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 119 YARN_PREFIX + "scheduler.minimum-allocation-vcores"; 120 public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 1; 121 122 /** Maximum request grant-able by the RM scheduler. */ 123 public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 124 YARN_PREFIX + "scheduler.maximum-allocation-mb"; 125 public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 8192; 126 public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 127 YARN_PREFIX + "scheduler.maximum-allocation-vcores"; 128 public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 4; 129 130 /** Number of threads to handle scheduler interface.*/ 131 public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT = 132 RM_PREFIX + "scheduler.client.thread-count"; 133 public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 50; 134 135 /** 136 * Enable periodic monitor threads. 137 * @see #RM_SCHEDULER_MONITOR_POLICIES 138 */ 139 public static final String RM_SCHEDULER_ENABLE_MONITORS = 140 RM_PREFIX + "scheduler.monitor.enable"; 141 public static final boolean DEFAULT_RM_SCHEDULER_ENABLE_MONITORS = false; 142 143 /** List of SchedulingEditPolicy classes affecting the scheduler. */ 144 public static final String RM_SCHEDULER_MONITOR_POLICIES = 145 RM_PREFIX + "scheduler.monitor.policies"; 146 147 /** The address of the RM web application.*/ 148 public static final String RM_WEBAPP_ADDRESS = 149 RM_PREFIX + "webapp.address"; 150 151 public static final int DEFAULT_RM_WEBAPP_PORT = 8088; 152 public static final String DEFAULT_RM_WEBAPP_ADDRESS = "0.0.0.0:" + 153 DEFAULT_RM_WEBAPP_PORT; 154 155 public static final String RM_RESOURCE_TRACKER_ADDRESS = 156 RM_PREFIX + "resource-tracker.address"; 157 public static final int DEFAULT_RM_RESOURCE_TRACKER_PORT = 8031; 158 public static final String DEFAULT_RM_RESOURCE_TRACKER_ADDRESS = 159 "0.0.0.0:" + DEFAULT_RM_RESOURCE_TRACKER_PORT; 160 161 /** The expiry interval for application master reporting.*/ 162 public static final String RM_AM_EXPIRY_INTERVAL_MS = 163 YARN_PREFIX + "am.liveness-monitor.expiry-interval-ms"; 164 public static final int DEFAULT_RM_AM_EXPIRY_INTERVAL_MS = 600000; 165 166 /** How long to wait until a node manager is considered dead.*/ 167 public static final String RM_NM_EXPIRY_INTERVAL_MS = 168 YARN_PREFIX + "nm.liveness-monitor.expiry-interval-ms"; 169 public static final int DEFAULT_RM_NM_EXPIRY_INTERVAL_MS = 600000; 170 171 /** Are acls enabled.*/ 172 public static final String YARN_ACL_ENABLE = 173 YARN_PREFIX + "acl.enable"; 174 public static final boolean DEFAULT_YARN_ACL_ENABLE = false; 175 176 /** ACL of who can be admin of YARN cluster.*/ 177 public static final String YARN_ADMIN_ACL = 178 YARN_PREFIX + "admin.acl"; 179 public static final String DEFAULT_YARN_ADMIN_ACL = "*"; 180 181 /** ACL used in case none is found. Allows nothing. */ 182 public static final String DEFAULT_YARN_APP_ACL = " "; 183 184 /** The address of the RM admin interface.*/ 185 public static final String RM_ADMIN_ADDRESS = 186 RM_PREFIX + "admin.address"; 187 public static final int DEFAULT_RM_ADMIN_PORT = 8033; 188 public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:" + 189 DEFAULT_RM_ADMIN_PORT; 190 191 /**Number of threads used to handle RM admin interface.*/ 192 public static final String RM_ADMIN_CLIENT_THREAD_COUNT = 193 RM_PREFIX + "admin.client.thread-count"; 194 public static final int DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT = 1; 195 196 /** 197 * The maximum number of application attempts. 198 * It's a global setting for all application masters. 199 */ 200 public static final String RM_AM_MAX_ATTEMPTS = 201 RM_PREFIX + "am.max-attempts"; 202 public static final int DEFAULT_RM_AM_MAX_ATTEMPTS = 2; 203 204 /** The keytab for the resource manager.*/ 205 public static final String RM_KEYTAB = 206 RM_PREFIX + "keytab"; 207 208 /** How long to wait until a container is considered dead.*/ 209 public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 210 RM_PREFIX + "rm.container-allocation.expiry-interval-ms"; 211 public static final int DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 600000; 212 213 /** Path to file with nodes to include.*/ 214 public static final String RM_NODES_INCLUDE_FILE_PATH = 215 RM_PREFIX + "nodes.include-path"; 216 public static final String DEFAULT_RM_NODES_INCLUDE_FILE_PATH = ""; 217 218 /** Path to file with nodes to exclude.*/ 219 public static final String RM_NODES_EXCLUDE_FILE_PATH = 220 RM_PREFIX + "nodes.exclude-path"; 221 public static final String DEFAULT_RM_NODES_EXCLUDE_FILE_PATH = ""; 222 223 /** Number of threads to handle resource tracker calls.*/ 224 public static final String RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 225 RM_PREFIX + "resource-tracker.client.thread-count"; 226 public static final int DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 50; 227 228 /** The class to use as the resource scheduler.*/ 229 public static final String RM_SCHEDULER = 230 RM_PREFIX + "scheduler.class"; 231 232 public static final String DEFAULT_RM_SCHEDULER = 233 "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"; 234 235 /** RM set next Heartbeat interval for NM */ 236 public static final String RM_NM_HEARTBEAT_INTERVAL_MS = 237 RM_PREFIX + "nodemanagers.heartbeat-interval-ms"; 238 public static final long DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS = 1000; 239 240 //Delegation token related keys 241 public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY = 242 RM_PREFIX + "delegation.key.update-interval"; 243 public static final long DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 244 24*60*60*1000; // 1 day 245 public static final String DELEGATION_TOKEN_RENEW_INTERVAL_KEY = 246 RM_PREFIX + "delegation.token.renew-interval"; 247 public static final long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 248 24*60*60*1000; // 1 day 249 public static final String DELEGATION_TOKEN_MAX_LIFETIME_KEY = 250 RM_PREFIX + "delegation.token.max-lifetime"; 251 public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 252 7*24*60*60*1000; // 7 days 253 254 public static final String RECOVERY_ENABLED = RM_PREFIX + "recovery.enabled"; 255 public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false; 256 257 /** The class to use as the persistent store.*/ 258 public static final String RM_STORE = RM_PREFIX + "store.class"; 259 260 /** URI for FileSystemRMStateStore */ 261 public static final String FS_RM_STATE_STORE_URI = 262 RM_PREFIX + "fs.state-store.uri"; 263 264 /** The maximum number of completed applications RM keeps. */ 265 public static final String RM_MAX_COMPLETED_APPLICATIONS = 266 RM_PREFIX + "max-completed-applications"; 267 public static final int DEFAULT_RM_MAX_COMPLETED_APPLICATIONS = 10000; 268 269 /** Default application name */ 270 public static final String DEFAULT_APPLICATION_NAME = "N/A"; 271 272 /** Default application type */ 273 public static final String DEFAULT_APPLICATION_TYPE = "YARN"; 274 275 /** Default application type length */ 276 public static final int APPLICATION_TYPE_LENGTH = 20; 277 278 /** Default queue name */ 279 public static final String DEFAULT_QUEUE_NAME = "default"; 280 281 /** 282 * Buckets (in minutes) for the number of apps running in each queue. 283 */ 284 public static final String RM_METRICS_RUNTIME_BUCKETS = 285 RM_PREFIX + "metrics.runtime.buckets"; 286 287 /** 288 * Default sizes of the runtime metric buckets in minutes. 289 */ 290 public static final String DEFAULT_RM_METRICS_RUNTIME_BUCKETS = 291 "60,300,1440"; 292 293 public static final String RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX 294 + "am-rm-tokens.master-key-rolling-interval-secs"; 295 296 public static final long DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 297 24 * 60 * 60; 298 299 public static final String RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 300 RM_PREFIX + "container-tokens.master-key-rolling-interval-secs"; 301 302 public static final long DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 303 24 * 60 * 60; 304 305 public static final String RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 306 RM_PREFIX + "nm-tokens.master-key-rolling-interval-secs"; 307 308 public static final long DEFAULT_RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 309 24 * 60 * 60; 310 //////////////////////////////// 311 // Node Manager Configs 312 //////////////////////////////// 313 314 /** Prefix for all node manager configs.*/ 315 public static final String NM_PREFIX = "yarn.nodemanager."; 316 317 /** Environment variables that will be sent to containers.*/ 318 public static final String NM_ADMIN_USER_ENV = NM_PREFIX + "admin-env"; 319 public static final String DEFAULT_NM_ADMIN_USER_ENV = "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX"; 320 321 /** Environment variables that containers may override rather than use NodeManager's default.*/ 322 public static final String NM_ENV_WHITELIST = NM_PREFIX + "env-whitelist"; 323 public static final String DEFAULT_NM_ENV_WHITELIST = StringUtils.join(",", 324 Arrays.asList(ApplicationConstants.Environment.JAVA_HOME.key(), 325 ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(), 326 ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(), 327 ApplicationConstants.Environment.HADOOP_CONF_DIR.key(), 328 ApplicationConstants.Environment.HADOOP_YARN_HOME.key())); 329 330 /** address of node manager IPC.*/ 331 public static final String NM_ADDRESS = NM_PREFIX + "address"; 332 public static final int DEFAULT_NM_PORT = 0; 333 public static final String DEFAULT_NM_ADDRESS = "0.0.0.0:" 334 + DEFAULT_NM_PORT; 335 336 /** who will execute(launch) the containers.*/ 337 public static final String NM_CONTAINER_EXECUTOR = 338 NM_PREFIX + "container-executor.class"; 339 340 /** 341 * Adjustment to make to the container os scheduling priority. 342 * The valid values for this could vary depending on the platform. 343 * On Linux, higher values mean run the containers at a less 344 * favorable priority than the NM. 345 * The value specified is an int. 346 */ 347 public static final String NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = 348 NM_PREFIX + "container-executor.os.sched.priority.adjustment"; 349 public static final int DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = 0; 350 351 /** Number of threads container manager uses.*/ 352 public static final String NM_CONTAINER_MGR_THREAD_COUNT = 353 NM_PREFIX + "container-manager.thread-count"; 354 public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 20; 355 356 /** Number of threads used in cleanup.*/ 357 public static final String NM_DELETE_THREAD_COUNT = 358 NM_PREFIX + "delete.thread-count"; 359 public static final int DEFAULT_NM_DELETE_THREAD_COUNT = 4; 360 361 /** Keytab for NM.*/ 362 public static final String NM_KEYTAB = NM_PREFIX + "keytab"; 363 364 /**List of directories to store localized files in.*/ 365 public static final String NM_LOCAL_DIRS = NM_PREFIX + "local-dirs"; 366 public static final String DEFAULT_NM_LOCAL_DIRS = "/tmp/nm-local-dir"; 367 368 /** 369 * Number of files in each localized directories 370 * Avoid tuning this too low. 371 */ 372 public static final String NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = 373 NM_PREFIX + "local-cache.max-files-per-directory"; 374 public static final int DEFAULT_NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = 8192; 375 376 /** Address where the localizer IPC is.*/ 377 public static final String NM_LOCALIZER_ADDRESS = 378 NM_PREFIX + "localizer.address"; 379 public static final int DEFAULT_NM_LOCALIZER_PORT = 8040; 380 public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:" + 381 DEFAULT_NM_LOCALIZER_PORT; 382 383 /** Interval in between cache cleanups.*/ 384 public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = 385 NM_PREFIX + "localizer.cache.cleanup.interval-ms"; 386 public static final long DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = 387 10 * 60 * 1000; 388 389 /** Target size of localizer cache in MB, per local directory.*/ 390 public static final String NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 391 NM_PREFIX + "localizer.cache.target-size-mb"; 392 public static final long DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 10 * 1024; 393 394 /** Number of threads to handle localization requests.*/ 395 public static final String NM_LOCALIZER_CLIENT_THREAD_COUNT = 396 NM_PREFIX + "localizer.client.thread-count"; 397 public static final int DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT = 5; 398 399 /** Number of threads to use for localization fetching.*/ 400 public static final String NM_LOCALIZER_FETCH_THREAD_COUNT = 401 NM_PREFIX + "localizer.fetch.thread-count"; 402 public static final int DEFAULT_NM_LOCALIZER_FETCH_THREAD_COUNT = 4; 403 404 /** Where to store container logs.*/ 405 public static final String NM_LOG_DIRS = NM_PREFIX + "log-dirs"; 406 public static final String DEFAULT_NM_LOG_DIRS = "/tmp/logs"; 407 408 /** Interval at which the delayed token removal thread runs */ 409 public static final String RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS = 410 RM_PREFIX + "delayed.delegation-token.removal-interval-ms"; 411 public static final long DEFAULT_RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS = 412 30000l; 413 414 /** Whether to enable log aggregation */ 415 public static final String LOG_AGGREGATION_ENABLED = YARN_PREFIX 416 + "log-aggregation-enable"; 417 public static final boolean DEFAULT_LOG_AGGREGATION_ENABLED = false; 418 419 /** 420 * How long to wait before deleting aggregated logs, -1 disables. 421 * Be careful set this too small and you will spam the name node. 422 */ 423 public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX 424 + "log-aggregation.retain-seconds"; 425 public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1; 426 427 /** 428 * How long to wait between aggregated log retention checks. If set to 429 * a value <= 0 then the value is computed as one-tenth of the log retention 430 * setting. Be careful set this too small and you will spam the name node. 431 */ 432 public static final String LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = 433 YARN_PREFIX + "log-aggregation.retain-check-interval-seconds"; 434 public static final long DEFAULT_LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = -1; 435 436 /** 437 * Number of seconds to retain logs on the NodeManager. Only applicable if Log 438 * aggregation is disabled 439 */ 440 public static final String NM_LOG_RETAIN_SECONDS = NM_PREFIX 441 + "log.retain-seconds"; 442 443 /** 444 * Number of threads used in log cleanup. Only applicable if Log aggregation 445 * is disabled 446 */ 447 public static final String NM_LOG_DELETION_THREADS_COUNT = 448 NM_PREFIX + "log.deletion-threads-count"; 449 public static final int DEFAULT_NM_LOG_DELETE_THREAD_COUNT = 4; 450 451 /** Where to aggregate logs to.*/ 452 public static final String NM_REMOTE_APP_LOG_DIR = 453 NM_PREFIX + "remote-app-log-dir"; 454 public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR = "/tmp/logs"; 455 456 /** 457 * The remote log dir will be created at 458 * NM_REMOTE_APP_LOG_DIR/${user}/NM_REMOTE_APP_LOG_DIR_SUFFIX/${appId} 459 */ 460 public static final String NM_REMOTE_APP_LOG_DIR_SUFFIX = 461 NM_PREFIX + "remote-app-log-dir-suffix"; 462 public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX="logs"; 463 464 public static final String YARN_LOG_SERVER_URL = 465 YARN_PREFIX + "log.server.url"; 466 467 public static final String YARN_TRACKING_URL_GENERATOR = 468 YARN_PREFIX + "tracking.url.generator"; 469 470 /** Amount of memory in GB that can be allocated for containers.*/ 471 public static final String NM_PMEM_MB = NM_PREFIX + "resource.memory-mb"; 472 public static final int DEFAULT_NM_PMEM_MB = 8 * 1024; 473 474 /** Specifies whether physical memory check is enabled. */ 475 public static final String NM_PMEM_CHECK_ENABLED = NM_PREFIX 476 + "pmem-check-enabled"; 477 public static final boolean DEFAULT_NM_PMEM_CHECK_ENABLED = true; 478 479 /** Specifies whether physical memory check is enabled. */ 480 public static final String NM_VMEM_CHECK_ENABLED = NM_PREFIX 481 + "vmem-check-enabled"; 482 public static final boolean DEFAULT_NM_VMEM_CHECK_ENABLED = true; 483 484 /** Conversion ratio for physical memory to virtual memory. */ 485 public static final String NM_VMEM_PMEM_RATIO = 486 NM_PREFIX + "vmem-pmem-ratio"; 487 public static final float DEFAULT_NM_VMEM_PMEM_RATIO = 2.1f; 488 489 /** Number of Virtual CPU Cores which can be allocated for containers.*/ 490 public static final String NM_VCORES = NM_PREFIX + "resource.cpu-vcores"; 491 public static final int DEFAULT_NM_VCORES = 8; 492 493 /** NM Webapp address.**/ 494 public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address"; 495 public static final int DEFAULT_NM_WEBAPP_PORT = 8042; 496 public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:" + 497 DEFAULT_NM_WEBAPP_PORT; 498 499 /** How often to monitor containers.*/ 500 public final static String NM_CONTAINER_MON_INTERVAL_MS = 501 NM_PREFIX + "container-monitor.interval-ms"; 502 public final static int DEFAULT_NM_CONTAINER_MON_INTERVAL_MS = 3000; 503 504 /** Class that calculates containers current resource utilization.*/ 505 public static final String NM_CONTAINER_MON_RESOURCE_CALCULATOR = 506 NM_PREFIX + "container-monitor.resource-calculator.class"; 507 /** Class that calculates process tree resource utilization.*/ 508 public static final String NM_CONTAINER_MON_PROCESS_TREE = 509 NM_PREFIX + "container-monitor.process-tree.class"; 510 511 /** 512 * Enable/Disable disks' health checker. Default is true. 513 * An expert level configuration property. 514 */ 515 public static final String NM_DISK_HEALTH_CHECK_ENABLE = 516 NM_PREFIX + "disk-health-checker.enable"; 517 /** Frequency of running disks' health checker.*/ 518 public static final String NM_DISK_HEALTH_CHECK_INTERVAL_MS = 519 NM_PREFIX + "disk-health-checker.interval-ms"; 520 /** By default, disks' health is checked every 2 minutes. */ 521 public static final long DEFAULT_NM_DISK_HEALTH_CHECK_INTERVAL_MS = 522 2 * 60 * 1000; 523 524 /** 525 * The minimum fraction of number of disks to be healthy for the nodemanager 526 * to launch new containers. This applies to nm-local-dirs and nm-log-dirs. 527 */ 528 public static final String NM_MIN_HEALTHY_DISKS_FRACTION = 529 NM_PREFIX + "disk-health-checker.min-healthy-disks"; 530 /** 531 * By default, at least 5% of disks are to be healthy to say that the node 532 * is healthy in terms of disks. 533 */ 534 public static final float DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION 535 = 0.25F; 536 537 /** Frequency of running node health script.*/ 538 public static final String NM_HEALTH_CHECK_INTERVAL_MS = 539 NM_PREFIX + "health-checker.interval-ms"; 540 public static final long DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS = 10 * 60 * 1000; 541 542 /** Health check script time out period.*/ 543 public static final String NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = 544 NM_PREFIX + "health-checker.script.timeout-ms"; 545 public static final long DEFAULT_NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = 546 2 * DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS; 547 548 /** The health check script to run.*/ 549 public static final String NM_HEALTH_CHECK_SCRIPT_PATH = 550 NM_PREFIX + "health-checker.script.path"; 551 552 /** The arguments to pass to the health check script.*/ 553 public static final String NM_HEALTH_CHECK_SCRIPT_OPTS = 554 NM_PREFIX + "health-checker.script.opts"; 555 556 /** The path to the Linux container executor.*/ 557 public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH = 558 NM_PREFIX + "linux-container-executor.path"; 559 560 /** 561 * The UNIX group that the linux-container-executor should run as. 562 * This is intended to be set as part of container-executor.cfg. 563 */ 564 public static final String NM_LINUX_CONTAINER_GROUP = 565 NM_PREFIX + "linux-container-executor.group"; 566 567 /** The type of resource enforcement to use with the 568 * linux container executor. 569 */ 570 public static final String NM_LINUX_CONTAINER_RESOURCES_HANDLER = 571 NM_PREFIX + "linux-container-executor.resources-handler.class"; 572 573 /** The path the linux container executor should use for cgroups */ 574 public static final String NM_LINUX_CONTAINER_CGROUPS_HIERARCHY = 575 NM_PREFIX + "linux-container-executor.cgroups.hierarchy"; 576 577 /** Whether the linux container executor should mount cgroups if not found */ 578 public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT = 579 NM_PREFIX + "linux-container-executor.cgroups.mount"; 580 581 /** Where the linux container executor should mount cgroups if not found */ 582 public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH = 583 NM_PREFIX + "linux-container-executor.cgroups.mount-path"; 584 585 /** T-file compression types used to compress aggregated logs.*/ 586 public static final String NM_LOG_AGG_COMPRESSION_TYPE = 587 NM_PREFIX + "log-aggregation.compression-type"; 588 public static final String DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE = "none"; 589 590 /** The kerberos principal for the node manager.*/ 591 public static final String NM_PRINCIPAL = 592 NM_PREFIX + "principal"; 593 594 public static final String NM_AUX_SERVICES = 595 NM_PREFIX + "aux-services"; 596 597 public static final String NM_AUX_SERVICE_FMT = 598 NM_PREFIX + "aux-services.%s.class"; 599 600 public static final String NM_USER_HOME_DIR = 601 NM_PREFIX + "user-home-dir"; 602 603 public static final String DEFAULT_NM_USER_HOME_DIR= "/home/"; 604 605 //////////////////////////////// 606 // Web Proxy Configs 607 //////////////////////////////// 608 public static final String PROXY_PREFIX = "yarn.web-proxy."; 609 610 /** The kerberos principal for the proxy.*/ 611 public static final String PROXY_PRINCIPAL = 612 PROXY_PREFIX + "principal"; 613 614 /** Keytab for Proxy.*/ 615 public static final String PROXY_KEYTAB = PROXY_PREFIX + "keytab"; 616 617 /** The address for the web proxy.*/ 618 public static final String PROXY_ADDRESS = 619 PROXY_PREFIX + "address"; 620 621 /** 622 * YARN Service Level Authorization 623 */ 624 public static final String 625 YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL = 626 "security.resourcetracker.protocol.acl"; 627 public static final String 628 YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL = 629 "security.applicationclient.protocol.acl"; 630 public static final String 631 YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL = 632 "security.resourcemanager-administration.protocol.acl"; 633 public static final String 634 YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL = 635 "security.applicationmaster.protocol.acl"; 636 637 public static final String 638 YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL = 639 "security.containermanagement.protocol.acl"; 640 public static final String 641 YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER = 642 "security.resourcelocalizer.protocol.acl"; 643 644 /** No. of milliseconds to wait between sending a SIGTERM and SIGKILL 645 * to a running container */ 646 public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS = 647 NM_PREFIX + "sleep-delay-before-sigkill.ms"; 648 public static final long DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS = 649 250; 650 651 /** Max time to wait for a process to come up when trying to cleanup 652 * container resources */ 653 public static final String NM_PROCESS_KILL_WAIT_MS = 654 NM_PREFIX + "process-kill-wait.ms"; 655 public static final long DEFAULT_NM_PROCESS_KILL_WAIT_MS = 656 2000; 657 658 /** Max time to wait to establish a connection to RM */ 659 public static final String RESOURCEMANAGER_CONNECT_MAX_WAIT_MS = 660 RM_PREFIX + "connect.max-wait.ms"; 661 public static final int DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS = 662 15 * 60 * 1000; 663 664 /** Time interval between each attempt to connect to RM */ 665 public static final String RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS = 666 RM_PREFIX + "connect.retry-interval.ms"; 667 public static final long DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS 668 = 30 * 1000; 669 670 /** 671 * CLASSPATH for YARN applications. A comma-separated list of CLASSPATH 672 * entries 673 */ 674 public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX 675 + "application.classpath"; 676 677 /** 678 * Default CLASSPATH for YARN applications. A comma-separated list of 679 * CLASSPATH entries 680 */ 681 public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = { 682 ApplicationConstants.Environment.HADOOP_CONF_DIR.$(), 683 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$() 684 + "/share/hadoop/common/*", 685 ApplicationConstants.Environment.HADOOP_COMMON_HOME.$() 686 + "/share/hadoop/common/lib/*", 687 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$() 688 + "/share/hadoop/hdfs/*", 689 ApplicationConstants.Environment.HADOOP_HDFS_HOME.$() 690 + "/share/hadoop/hdfs/lib/*", 691 ApplicationConstants.Environment.HADOOP_YARN_HOME.$() 692 + "/share/hadoop/yarn/*", 693 ApplicationConstants.Environment.HADOOP_YARN_HOME.$() 694 + "/share/hadoop/yarn/lib/*" }; 695 696 /** Container temp directory */ 697 public static final String DEFAULT_CONTAINER_TEMP_DIR = "./tmp"; 698 699 public static final String IS_MINI_YARN_CLUSTER = YARN_PREFIX 700 + "is.minicluster"; 701 702 /** Whether to use fixed ports with the minicluster. */ 703 public static final String YARN_MINICLUSTER_FIXED_PORTS = YARN_PREFIX 704 + "minicluster.fixed.ports"; 705 706 /** 707 * Default is false to be able to run tests concurrently without port 708 * conflicts. 709 */ 710 public static boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false; 711 712 /** 713 * Whether users are explicitly trying to control resource monitoring 714 * configuration for the MiniYARNCluster. Disabled by default. 715 */ 716 public static final String YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING = 717 YARN_PREFIX + "minicluster.control-resource-monitoring"; 718 public static final boolean 719 DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING = false; 720 721 /** The log directory for the containers */ 722 public static final String YARN_APP_CONTAINER_LOG_DIR = 723 YARN_PREFIX + "app.container.log.dir"; 724 725 public static final String YARN_APP_CONTAINER_LOG_SIZE = 726 YARN_PREFIX + "app.container.log.filesize"; 727 728 //////////////////////////////// 729 // Other Configs 730 //////////////////////////////// 731 732 /** 733 * The interval of the yarn client's querying application state after 734 * application submission. The unit is millisecond. 735 */ 736 public static final String YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS = 737 YARN_PREFIX + "client.app-submission.poll-interval"; 738 public static final long DEFAULT_YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS = 739 1000; 740 741 /** 742 * Max number of threads in NMClientAsync to process container management 743 * events 744 */ 745 public static final String NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE = 746 YARN_PREFIX + "client.nodemanager-client-async.thread-pool-max-size"; 747 public static final int DEFAULT_NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE = 500; 748 749 /** 750 * Maximum number of proxy connections for node manager. It should always be 751 * more than 1. NMClient and MRAppMaster will use this to cache connection 752 * with node manager. There will be at max one connection per node manager. 753 * Ex. configuring it to a value of 5 will make sure that client will at 754 * max have 5 connections cached with 5 different node managers. These 755 * connections will be timed out if idle for more than system wide idle 756 * timeout period. The token if used for authentication then it will be used 757 * only at connection creation time. If new token is received then earlier 758 * connection should be closed in order to use newer token. 759 * Note: {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE} 760 * are related to each other. 761 */ 762 public static final String NM_CLIENT_MAX_NM_PROXIES = 763 YARN_PREFIX + "client.max-nodemanagers-proxies"; 764 public static final int DEFAULT_NM_CLIENT_MAX_NM_PROXIES = 500; 765 766 public YarnConfiguration() { 767 super(); 768 } 769 770 public YarnConfiguration(Configuration conf) { 771 super(conf); 772 if (! (conf instanceof YarnConfiguration)) { 773 this.reloadConfiguration(); 774 } 775 } 776 777 public static String getProxyHostAndPort(Configuration conf) { 778 String addr = conf.get(PROXY_ADDRESS); 779 if(addr == null || addr.isEmpty()) { 780 addr = getRMWebAppHostAndPort(conf); 781 } 782 return addr; 783 } 784 785 public static String getRMWebAppHostAndPort(Configuration conf) { 786 InetSocketAddress address = conf.getSocketAddr( 787 YarnConfiguration.RM_WEBAPP_ADDRESS, 788 YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, 789 YarnConfiguration.DEFAULT_RM_WEBAPP_PORT); 790 address = NetUtils.getConnectAddress(address); 791 StringBuffer sb = new StringBuffer(); 792 InetAddress resolved = address.getAddress(); 793 if (resolved == null || resolved.isAnyLocalAddress() || 794 resolved.isLoopbackAddress()) { 795 String lh = address.getHostName(); 796 try { 797 lh = InetAddress.getLocalHost().getCanonicalHostName(); 798 } catch (UnknownHostException e) { 799 //Ignore and fallback. 800 } 801 sb.append(lh); 802 } else { 803 sb.append(address.getHostName()); 804 } 805 sb.append(":").append(address.getPort()); 806 return sb.toString(); 807 } 808 809 public static String getRMWebAppURL(Configuration conf) { 810 return JOINER.join("http://", getRMWebAppHostAndPort(conf)); 811 } 812 813 }