001    /**
002    * Licensed to the Apache Software Foundation (ASF) under one
003    * or more contributor license agreements.  See the NOTICE file
004    * distributed with this work for additional information
005    * regarding copyright ownership.  The ASF licenses this file
006    * to you under the Apache License, Version 2.0 (the
007    * "License"); you may not use this file except in compliance
008    * with the License.  You may obtain a copy of the License at
009    *
010    *     http://www.apache.org/licenses/LICENSE-2.0
011    *
012    * Unless required by applicable law or agreed to in writing, software
013    * distributed under the License is distributed on an "AS IS" BASIS,
014    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015    * See the License for the specific language governing permissions and
016    * limitations under the License.
017    */
018    
019    package org.apache.hadoop.yarn.conf;
020    
021    import java.net.InetAddress;
022    import java.net.InetSocketAddress;
023    import java.net.UnknownHostException;
024    import java.util.Arrays;
025    
026    import org.apache.hadoop.classification.InterfaceAudience.Public;
027    import org.apache.hadoop.classification.InterfaceStability.Evolving;
028    import org.apache.hadoop.conf.Configuration;
029    import org.apache.hadoop.net.NetUtils;
030    import org.apache.hadoop.util.StringUtils;
031    import org.apache.hadoop.yarn.api.ApplicationConstants;
032    
033    import com.google.common.base.Joiner;
034    
035    @Public
036    @Evolving
037    public class YarnConfiguration extends Configuration {
038    
039      private static final Joiner JOINER = Joiner.on("");
040    
041      private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml";
042      private static final String YARN_SITE_XML_FILE = "yarn-site.xml";
043    
044      static {
045        Configuration.addDefaultResource(YARN_DEFAULT_XML_FILE);
046        Configuration.addDefaultResource(YARN_SITE_XML_FILE);
047      }
048    
049      //Configurations
050    
051      public static final String YARN_PREFIX = "yarn.";
052    
053      /** Delay before deleting resource to ease debugging of NM issues */
054      public static final String DEBUG_NM_DELETE_DELAY_SEC =
055        YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec";
056      
057      ////////////////////////////////
058      // IPC Configs
059      ////////////////////////////////
060      public static final String IPC_PREFIX = YARN_PREFIX + "ipc.";
061    
062      /** Factory to create client IPC classes.*/
063      public static final String IPC_CLIENT_FACTORY_CLASS = 
064        IPC_PREFIX + "client.factory.class";
065      public static final String DEFAULT_IPC_CLIENT_FACTORY_CLASS = 
066          "org.apache.hadoop.yarn.factories.impl.pb.RpcClientFactoryPBImpl";
067    
068      /** Factory to create server IPC classes.*/
069      public static final String IPC_SERVER_FACTORY_CLASS = 
070        IPC_PREFIX + "server.factory.class";
071      public static final String DEFAULT_IPC_SERVER_FACTORY_CLASS = 
072          "org.apache.hadoop.yarn.factories.impl.pb.RpcServerFactoryPBImpl";
073    
074      /** Factory to create serializeable records.*/
075      public static final String IPC_RECORD_FACTORY_CLASS = 
076        IPC_PREFIX + "record.factory.class";
077      public static final String DEFAULT_IPC_RECORD_FACTORY_CLASS = 
078          "org.apache.hadoop.yarn.factories.impl.pb.RecordFactoryPBImpl";
079    
080      /** RPC class implementation*/
081      public static final String IPC_RPC_IMPL =
082        IPC_PREFIX + "rpc.class";
083      public static final String DEFAULT_IPC_RPC_IMPL = 
084        "org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC";
085      
086      ////////////////////////////////
087      // Resource Manager Configs
088      ////////////////////////////////
089      public static final String RM_PREFIX = "yarn.resourcemanager.";
090      
091      /** The address of the applications manager interface in the RM.*/
092      public static final String RM_ADDRESS = 
093        RM_PREFIX + "address";
094      public static final int DEFAULT_RM_PORT = 8032;
095      public static final String DEFAULT_RM_ADDRESS =
096        "0.0.0.0:" + DEFAULT_RM_PORT;
097    
098      /** The number of threads used to handle applications manager requests.*/
099      public static final String RM_CLIENT_THREAD_COUNT =
100        RM_PREFIX + "client.thread-count";
101      public static final int DEFAULT_RM_CLIENT_THREAD_COUNT = 50;
102    
103      /** The Kerberos principal for the resource manager.*/
104      public static final String RM_PRINCIPAL =
105        RM_PREFIX + "principal";
106      
107      /** The address of the scheduler interface.*/
108      public static final String RM_SCHEDULER_ADDRESS = 
109        RM_PREFIX + "scheduler.address";
110      public static final int DEFAULT_RM_SCHEDULER_PORT = 8030;
111      public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:" +
112        DEFAULT_RM_SCHEDULER_PORT;
113    
114      /** Miniumum request grant-able by the RM scheduler. */
115      public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_MB =
116        YARN_PREFIX + "scheduler.minimum-allocation-mb";
117      public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 1024;
118      public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES =
119          YARN_PREFIX + "scheduler.minimum-allocation-vcores";
120        public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 1;
121    
122      /** Maximum request grant-able by the RM scheduler. */
123      public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_MB =
124        YARN_PREFIX + "scheduler.maximum-allocation-mb";
125      public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 8192;
126      public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES =
127          YARN_PREFIX + "scheduler.maximum-allocation-vcores";
128      public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 4;
129    
130      /** Number of threads to handle scheduler interface.*/
131      public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT =
132        RM_PREFIX + "scheduler.client.thread-count";
133      public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 50;
134    
135      /**
136       * Enable periodic monitor threads.
137       * @see #RM_SCHEDULER_MONITOR_POLICIES
138       */
139      public static final String RM_SCHEDULER_ENABLE_MONITORS =
140        RM_PREFIX + "scheduler.monitor.enable";
141      public static final boolean DEFAULT_RM_SCHEDULER_ENABLE_MONITORS = false;
142    
143      /** List of SchedulingEditPolicy classes affecting the scheduler. */
144      public static final String RM_SCHEDULER_MONITOR_POLICIES =
145        RM_PREFIX + "scheduler.monitor.policies";
146    
147      /** The address of the RM web application.*/
148      public static final String RM_WEBAPP_ADDRESS = 
149        RM_PREFIX + "webapp.address";
150    
151      public static final int DEFAULT_RM_WEBAPP_PORT = 8088;
152      public static final String DEFAULT_RM_WEBAPP_ADDRESS = "0.0.0.0:" +
153        DEFAULT_RM_WEBAPP_PORT;
154      
155      public static final String RM_RESOURCE_TRACKER_ADDRESS =
156        RM_PREFIX + "resource-tracker.address";
157      public static final int DEFAULT_RM_RESOURCE_TRACKER_PORT = 8031;
158      public static final String DEFAULT_RM_RESOURCE_TRACKER_ADDRESS =
159        "0.0.0.0:" + DEFAULT_RM_RESOURCE_TRACKER_PORT;
160    
161      /** The expiry interval for application master reporting.*/
162      public static final String RM_AM_EXPIRY_INTERVAL_MS = 
163        YARN_PREFIX  + "am.liveness-monitor.expiry-interval-ms";
164      public static final int DEFAULT_RM_AM_EXPIRY_INTERVAL_MS = 600000;
165    
166      /** How long to wait until a node manager is considered dead.*/
167      public static final String RM_NM_EXPIRY_INTERVAL_MS = 
168        YARN_PREFIX + "nm.liveness-monitor.expiry-interval-ms";
169      public static final int DEFAULT_RM_NM_EXPIRY_INTERVAL_MS = 600000;
170    
171      /** Are acls enabled.*/
172      public static final String YARN_ACL_ENABLE = 
173        YARN_PREFIX + "acl.enable";
174      public static final boolean DEFAULT_YARN_ACL_ENABLE = false;
175      
176      /** ACL of who can be admin of YARN cluster.*/
177      public static final String YARN_ADMIN_ACL = 
178        YARN_PREFIX + "admin.acl";
179      public static final String DEFAULT_YARN_ADMIN_ACL = "*";
180      
181      /** ACL used in case none is found. Allows nothing. */
182      public static final String DEFAULT_YARN_APP_ACL = " ";
183    
184      /** The address of the RM admin interface.*/
185      public static final String RM_ADMIN_ADDRESS = 
186        RM_PREFIX + "admin.address";
187      public static final int DEFAULT_RM_ADMIN_PORT = 8033;
188      public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:" +
189          DEFAULT_RM_ADMIN_PORT;
190      
191      /**Number of threads used to handle RM admin interface.*/
192      public static final String RM_ADMIN_CLIENT_THREAD_COUNT =
193        RM_PREFIX + "admin.client.thread-count";
194      public static final int DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT = 1;
195      
196      /**
197       * The maximum number of application attempts.
198       * It's a global setting for all application masters.
199       */
200      public static final String RM_AM_MAX_ATTEMPTS =
201        RM_PREFIX + "am.max-attempts";
202      public static final int DEFAULT_RM_AM_MAX_ATTEMPTS = 2;
203      
204      /** The keytab for the resource manager.*/
205      public static final String RM_KEYTAB = 
206        RM_PREFIX + "keytab";
207    
208      /** How long to wait until a container is considered dead.*/
209      public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 
210        RM_PREFIX + "rm.container-allocation.expiry-interval-ms";
211      public static final int DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 600000;
212      
213      /** Path to file with nodes to include.*/
214      public static final String RM_NODES_INCLUDE_FILE_PATH = 
215        RM_PREFIX + "nodes.include-path";
216      public static final String DEFAULT_RM_NODES_INCLUDE_FILE_PATH = "";
217      
218      /** Path to file with nodes to exclude.*/
219      public static final String RM_NODES_EXCLUDE_FILE_PATH = 
220        RM_PREFIX + "nodes.exclude-path";
221      public static final String DEFAULT_RM_NODES_EXCLUDE_FILE_PATH = "";
222      
223      /** Number of threads to handle resource tracker calls.*/
224      public static final String RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT =
225        RM_PREFIX + "resource-tracker.client.thread-count";
226      public static final int DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 50;
227      
228      /** The class to use as the resource scheduler.*/
229      public static final String RM_SCHEDULER = 
230        RM_PREFIX + "scheduler.class";
231     
232      public static final String DEFAULT_RM_SCHEDULER = 
233          "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler";
234    
235      /** RM set next Heartbeat interval for NM */
236      public static final String RM_NM_HEARTBEAT_INTERVAL_MS =
237          RM_PREFIX + "nodemanagers.heartbeat-interval-ms";
238      public static final long DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS = 1000;
239    
240      //Delegation token related keys
241      public static final String  DELEGATION_KEY_UPDATE_INTERVAL_KEY = 
242        RM_PREFIX + "delegation.key.update-interval";
243      public static final long    DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 
244        24*60*60*1000; // 1 day
245      public static final String  DELEGATION_TOKEN_RENEW_INTERVAL_KEY = 
246        RM_PREFIX + "delegation.token.renew-interval";
247      public static final long    DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 
248        24*60*60*1000;  // 1 day
249      public static final String  DELEGATION_TOKEN_MAX_LIFETIME_KEY = 
250         RM_PREFIX + "delegation.token.max-lifetime";
251      public static final long    DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 
252        7*24*60*60*1000; // 7 days
253      
254      public static final String RECOVERY_ENABLED = RM_PREFIX + "recovery.enabled";
255      public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false;
256      
257      /** The class to use as the persistent store.*/
258      public static final String RM_STORE = RM_PREFIX + "store.class";
259      
260      /** URI for FileSystemRMStateStore */
261      public static final String FS_RM_STATE_STORE_URI =
262                                               RM_PREFIX + "fs.state-store.uri";
263    
264      /** The maximum number of completed applications RM keeps. */ 
265      public static final String RM_MAX_COMPLETED_APPLICATIONS =
266        RM_PREFIX + "max-completed-applications";
267      public static final int DEFAULT_RM_MAX_COMPLETED_APPLICATIONS = 10000;
268      
269      /** Default application name */
270      public static final String DEFAULT_APPLICATION_NAME = "N/A";
271    
272      /** Default application type */
273      public static final String DEFAULT_APPLICATION_TYPE = "YARN";
274    
275      /** Default application type length */
276      public static final int APPLICATION_TYPE_LENGTH = 20;
277      
278      /** Default queue name */
279      public static final String DEFAULT_QUEUE_NAME = "default";
280    
281      /**
282       * Buckets (in minutes) for the number of apps running in each queue.
283       */
284      public static final String RM_METRICS_RUNTIME_BUCKETS =
285        RM_PREFIX + "metrics.runtime.buckets";
286    
287      /**
288       * Default sizes of the runtime metric buckets in minutes.
289       */
290      public static final String DEFAULT_RM_METRICS_RUNTIME_BUCKETS = 
291        "60,300,1440";
292    
293      public static final String RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX
294          + "am-rm-tokens.master-key-rolling-interval-secs";
295    
296      public static final long DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
297          24 * 60 * 60;
298    
299      public static final String RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
300          RM_PREFIX + "container-tokens.master-key-rolling-interval-secs";
301    
302      public static final long DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
303          24 * 60 * 60;
304    
305      public static final String RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
306          RM_PREFIX + "nm-tokens.master-key-rolling-interval-secs";
307      
308      public static final long DEFAULT_RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
309          24 * 60 * 60;
310      ////////////////////////////////
311      // Node Manager Configs
312      ////////////////////////////////
313      
314      /** Prefix for all node manager configs.*/
315      public static final String NM_PREFIX = "yarn.nodemanager.";
316    
317      /** Environment variables that will be sent to containers.*/
318      public static final String NM_ADMIN_USER_ENV = NM_PREFIX + "admin-env";
319      public static final String DEFAULT_NM_ADMIN_USER_ENV = "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX";
320    
321      /** Environment variables that containers may override rather than use NodeManager's default.*/
322      public static final String NM_ENV_WHITELIST = NM_PREFIX + "env-whitelist";
323      public static final String DEFAULT_NM_ENV_WHITELIST = StringUtils.join(",",
324        Arrays.asList(ApplicationConstants.Environment.JAVA_HOME.key(),
325          ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(),
326          ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(),
327          ApplicationConstants.Environment.HADOOP_CONF_DIR.key(),
328          ApplicationConstants.Environment.HADOOP_YARN_HOME.key()));
329      
330      /** address of node manager IPC.*/
331      public static final String NM_ADDRESS = NM_PREFIX + "address";
332      public static final int DEFAULT_NM_PORT = 0;
333      public static final String DEFAULT_NM_ADDRESS = "0.0.0.0:"
334          + DEFAULT_NM_PORT;
335      
336      /** who will execute(launch) the containers.*/
337      public static final String NM_CONTAINER_EXECUTOR = 
338        NM_PREFIX + "container-executor.class";
339    
340      /**  
341       * Adjustment to make to the container os scheduling priority.
342       * The valid values for this could vary depending on the platform.
343       * On Linux, higher values mean run the containers at a less 
344       * favorable priority than the NM. 
345       * The value specified is an int.
346       */
347      public static final String NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = 
348        NM_PREFIX + "container-executor.os.sched.priority.adjustment";
349      public static final int DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = 0;
350      
351      /** Number of threads container manager uses.*/
352      public static final String NM_CONTAINER_MGR_THREAD_COUNT =
353        NM_PREFIX + "container-manager.thread-count";
354      public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 20;
355      
356      /** Number of threads used in cleanup.*/
357      public static final String NM_DELETE_THREAD_COUNT = 
358        NM_PREFIX +  "delete.thread-count";
359      public static final int DEFAULT_NM_DELETE_THREAD_COUNT = 4;
360      
361      /** Keytab for NM.*/
362      public static final String NM_KEYTAB = NM_PREFIX + "keytab";
363      
364      /**List of directories to store localized files in.*/
365      public static final String NM_LOCAL_DIRS = NM_PREFIX + "local-dirs";
366      public static final String DEFAULT_NM_LOCAL_DIRS = "/tmp/nm-local-dir";
367    
368      /**
369       * Number of files in each localized directories
370       * Avoid tuning this too low. 
371       */
372      public static final String NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY =
373        NM_PREFIX + "local-cache.max-files-per-directory";
374      public static final int DEFAULT_NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = 8192;
375    
376      /** Address where the localizer IPC is.*/
377      public static final String NM_LOCALIZER_ADDRESS =
378        NM_PREFIX + "localizer.address";
379      public static final int DEFAULT_NM_LOCALIZER_PORT = 8040;
380      public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:" +
381        DEFAULT_NM_LOCALIZER_PORT;
382      
383      /** Interval in between cache cleanups.*/
384      public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS =
385        NM_PREFIX + "localizer.cache.cleanup.interval-ms";
386      public static final long DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = 
387        10 * 60 * 1000;
388      
389      /** Target size of localizer cache in MB, per local directory.*/
390      public static final String NM_LOCALIZER_CACHE_TARGET_SIZE_MB =
391        NM_PREFIX + "localizer.cache.target-size-mb";
392      public static final long DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 10 * 1024;
393      
394      /** Number of threads to handle localization requests.*/
395      public static final String NM_LOCALIZER_CLIENT_THREAD_COUNT =
396        NM_PREFIX + "localizer.client.thread-count";
397      public static final int DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT = 5;
398      
399      /** Number of threads to use for localization fetching.*/
400      public static final String NM_LOCALIZER_FETCH_THREAD_COUNT = 
401        NM_PREFIX + "localizer.fetch.thread-count";
402      public static final int DEFAULT_NM_LOCALIZER_FETCH_THREAD_COUNT = 4;
403    
404      /** Where to store container logs.*/
405      public static final String NM_LOG_DIRS = NM_PREFIX + "log-dirs";
406      public static final String DEFAULT_NM_LOG_DIRS = "/tmp/logs";
407    
408      /** Interval at which the delayed token removal thread runs */
409      public static final String RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS =
410          RM_PREFIX + "delayed.delegation-token.removal-interval-ms";
411      public static final long DEFAULT_RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS =
412          30000l;
413    
414      /** Whether to enable log aggregation */
415      public static final String LOG_AGGREGATION_ENABLED = YARN_PREFIX
416          + "log-aggregation-enable";
417      public static final boolean DEFAULT_LOG_AGGREGATION_ENABLED = false;
418      
419      /** 
420       * How long to wait before deleting aggregated logs, -1 disables.
421       * Be careful set this too small and you will spam the name node.
422       */
423      public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX
424          + "log-aggregation.retain-seconds";
425      public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1;
426      
427      /**
428       * How long to wait between aggregated log retention checks. If set to
429       * a value <= 0 then the value is computed as one-tenth of the log retention
430       * setting. Be careful set this too small and you will spam the name node.
431       */
432      public static final String LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS =
433          YARN_PREFIX + "log-aggregation.retain-check-interval-seconds";
434      public static final long DEFAULT_LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = -1;
435    
436      /**
437       * Number of seconds to retain logs on the NodeManager. Only applicable if Log
438       * aggregation is disabled
439       */
440      public static final String NM_LOG_RETAIN_SECONDS = NM_PREFIX
441          + "log.retain-seconds";
442    
443      /**
444       * Number of threads used in log cleanup. Only applicable if Log aggregation
445       * is disabled
446       */
447      public static final String NM_LOG_DELETION_THREADS_COUNT = 
448        NM_PREFIX +  "log.deletion-threads-count";
449      public static final int DEFAULT_NM_LOG_DELETE_THREAD_COUNT = 4;
450    
451      /** Where to aggregate logs to.*/
452      public static final String NM_REMOTE_APP_LOG_DIR = 
453        NM_PREFIX + "remote-app-log-dir";
454      public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR = "/tmp/logs";
455    
456      /**
457       * The remote log dir will be created at
458       * NM_REMOTE_APP_LOG_DIR/${user}/NM_REMOTE_APP_LOG_DIR_SUFFIX/${appId}
459       */
460      public static final String NM_REMOTE_APP_LOG_DIR_SUFFIX = 
461        NM_PREFIX + "remote-app-log-dir-suffix";
462      public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX="logs";
463    
464      public static final String YARN_LOG_SERVER_URL =
465        YARN_PREFIX + "log.server.url";
466      
467      public static final String YARN_TRACKING_URL_GENERATOR = 
468          YARN_PREFIX + "tracking.url.generator";
469    
470      /** Amount of memory in GB that can be allocated for containers.*/
471      public static final String NM_PMEM_MB = NM_PREFIX + "resource.memory-mb";
472      public static final int DEFAULT_NM_PMEM_MB = 8 * 1024;
473    
474      /** Specifies whether physical memory check is enabled. */
475      public static final String NM_PMEM_CHECK_ENABLED = NM_PREFIX
476          + "pmem-check-enabled";
477      public static final boolean DEFAULT_NM_PMEM_CHECK_ENABLED = true;
478    
479      /** Specifies whether physical memory check is enabled. */
480      public static final String NM_VMEM_CHECK_ENABLED = NM_PREFIX
481          + "vmem-check-enabled";
482      public static final boolean DEFAULT_NM_VMEM_CHECK_ENABLED = true;
483    
484      /** Conversion ratio for physical memory to virtual memory. */
485      public static final String NM_VMEM_PMEM_RATIO =
486        NM_PREFIX + "vmem-pmem-ratio";
487      public static final float DEFAULT_NM_VMEM_PMEM_RATIO = 2.1f;
488      
489      /** Number of Virtual CPU Cores which can be allocated for containers.*/
490      public static final String NM_VCORES = NM_PREFIX + "resource.cpu-vcores";
491      public static final int DEFAULT_NM_VCORES = 8;
492      
493      /** NM Webapp address.**/
494      public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address";
495      public static final int DEFAULT_NM_WEBAPP_PORT = 8042;
496      public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:" +
497        DEFAULT_NM_WEBAPP_PORT;
498      
499      /** How often to monitor containers.*/
500      public final static String NM_CONTAINER_MON_INTERVAL_MS =
501        NM_PREFIX + "container-monitor.interval-ms";
502      public final static int DEFAULT_NM_CONTAINER_MON_INTERVAL_MS = 3000;
503    
504      /** Class that calculates containers current resource utilization.*/
505      public static final String NM_CONTAINER_MON_RESOURCE_CALCULATOR =
506        NM_PREFIX + "container-monitor.resource-calculator.class";
507      /** Class that calculates process tree resource utilization.*/
508      public static final String NM_CONTAINER_MON_PROCESS_TREE =
509        NM_PREFIX + "container-monitor.process-tree.class";
510    
511      /**
512       * Enable/Disable disks' health checker. Default is true.
513       * An expert level configuration property.
514       */
515      public static final String NM_DISK_HEALTH_CHECK_ENABLE =
516        NM_PREFIX + "disk-health-checker.enable";
517      /** Frequency of running disks' health checker.*/
518      public static final String NM_DISK_HEALTH_CHECK_INTERVAL_MS =
519        NM_PREFIX + "disk-health-checker.interval-ms";
520      /** By default, disks' health is checked every 2 minutes. */
521      public static final long DEFAULT_NM_DISK_HEALTH_CHECK_INTERVAL_MS =
522        2 * 60 * 1000;
523    
524      /**
525       * The minimum fraction of number of disks to be healthy for the nodemanager
526       * to launch new containers. This applies to nm-local-dirs and nm-log-dirs.
527       */
528      public static final String NM_MIN_HEALTHY_DISKS_FRACTION =
529        NM_PREFIX + "disk-health-checker.min-healthy-disks";
530      /**
531       * By default, at least 5% of disks are to be healthy to say that the node
532       * is healthy in terms of disks.
533       */
534      public static final float DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION
535        = 0.25F;
536    
537      /** Frequency of running node health script.*/
538      public static final String NM_HEALTH_CHECK_INTERVAL_MS = 
539        NM_PREFIX + "health-checker.interval-ms";
540      public static final long DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS = 10 * 60 * 1000;
541    
542      /** Health check script time out period.*/  
543      public static final String NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = 
544        NM_PREFIX + "health-checker.script.timeout-ms";
545      public static final long DEFAULT_NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = 
546        2 * DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS;
547      
548      /** The health check script to run.*/
549      public static final String NM_HEALTH_CHECK_SCRIPT_PATH = 
550        NM_PREFIX + "health-checker.script.path";
551      
552      /** The arguments to pass to the health check script.*/
553      public static final String NM_HEALTH_CHECK_SCRIPT_OPTS = 
554        NM_PREFIX + "health-checker.script.opts";
555      
556      /** The path to the Linux container executor.*/
557      public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH =
558        NM_PREFIX + "linux-container-executor.path";
559      
560      /** 
561       * The UNIX group that the linux-container-executor should run as.
562       * This is intended to be set as part of container-executor.cfg. 
563       */
564      public static final String NM_LINUX_CONTAINER_GROUP =
565        NM_PREFIX + "linux-container-executor.group";
566      
567      /** The type of resource enforcement to use with the
568       *  linux container executor.
569       */
570      public static final String NM_LINUX_CONTAINER_RESOURCES_HANDLER = 
571      NM_PREFIX + "linux-container-executor.resources-handler.class";
572      
573      /** The path the linux container executor should use for cgroups */
574      public static final String NM_LINUX_CONTAINER_CGROUPS_HIERARCHY =
575        NM_PREFIX + "linux-container-executor.cgroups.hierarchy";
576      
577      /** Whether the linux container executor should mount cgroups if not found */
578      public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT =
579        NM_PREFIX + "linux-container-executor.cgroups.mount";
580      
581      /** Where the linux container executor should mount cgroups if not found */
582      public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH =
583        NM_PREFIX + "linux-container-executor.cgroups.mount-path";
584      
585      /** T-file compression types used to compress aggregated logs.*/
586      public static final String NM_LOG_AGG_COMPRESSION_TYPE = 
587        NM_PREFIX + "log-aggregation.compression-type";
588      public static final String DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE = "none";
589      
590      /** The kerberos principal for the node manager.*/
591      public static final String NM_PRINCIPAL =
592        NM_PREFIX + "principal";
593      
594      public static final String NM_AUX_SERVICES = 
595        NM_PREFIX + "aux-services";
596      
597      public static final String NM_AUX_SERVICE_FMT =
598        NM_PREFIX + "aux-services.%s.class";
599    
600      public static final String NM_USER_HOME_DIR =
601          NM_PREFIX + "user-home-dir";
602    
603      public static final String DEFAULT_NM_USER_HOME_DIR= "/home/";
604    
605      ////////////////////////////////
606      // Web Proxy Configs
607      ////////////////////////////////
608      public static final String PROXY_PREFIX = "yarn.web-proxy.";
609      
610      /** The kerberos principal for the proxy.*/
611      public static final String PROXY_PRINCIPAL =
612        PROXY_PREFIX + "principal";
613      
614      /** Keytab for Proxy.*/
615      public static final String PROXY_KEYTAB = PROXY_PREFIX + "keytab";
616      
617      /** The address for the web proxy.*/
618      public static final String PROXY_ADDRESS =
619        PROXY_PREFIX + "address";
620      
621      /**
622       * YARN Service Level Authorization
623       */
624      public static final String 
625      YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL =
626          "security.resourcetracker.protocol.acl";
627      public static final String 
628      YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL =
629          "security.applicationclient.protocol.acl";
630      public static final String 
631      YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL =
632          "security.resourcemanager-administration.protocol.acl";
633      public static final String 
634      YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL =
635          "security.applicationmaster.protocol.acl";
636    
637      public static final String 
638      YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL =
639          "security.containermanagement.protocol.acl";
640      public static final String 
641      YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER =
642          "security.resourcelocalizer.protocol.acl";
643    
644      /** No. of milliseconds to wait between sending a SIGTERM and SIGKILL
645       * to a running container */
646      public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS =
647          NM_PREFIX + "sleep-delay-before-sigkill.ms";
648      public static final long DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS =
649          250;
650    
651      /** Max time to wait for a process to come up when trying to cleanup
652       * container resources */
653      public static final String NM_PROCESS_KILL_WAIT_MS =
654          NM_PREFIX + "process-kill-wait.ms";
655      public static final long DEFAULT_NM_PROCESS_KILL_WAIT_MS =
656          2000;
657    
658      /** Max time to wait to establish a connection to RM */
659      public static final String RESOURCEMANAGER_CONNECT_MAX_WAIT_MS =
660          RM_PREFIX + "connect.max-wait.ms";
661      public static final int DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS =
662          15 * 60 * 1000;
663    
664      /** Time interval between each attempt to connect to RM */
665      public static final String RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS =
666          RM_PREFIX + "connect.retry-interval.ms";
667      public static final long DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS
668          = 30 * 1000;
669    
670      /**
671       * CLASSPATH for YARN applications. A comma-separated list of CLASSPATH
672       * entries
673       */
674      public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX
675          + "application.classpath";
676    
677      /**
678       * Default CLASSPATH for YARN applications. A comma-separated list of
679       * CLASSPATH entries
680       */
681      public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = {
682          ApplicationConstants.Environment.HADOOP_CONF_DIR.$(),
683          ApplicationConstants.Environment.HADOOP_COMMON_HOME.$()
684              + "/share/hadoop/common/*",
685          ApplicationConstants.Environment.HADOOP_COMMON_HOME.$()
686              + "/share/hadoop/common/lib/*",
687          ApplicationConstants.Environment.HADOOP_HDFS_HOME.$()
688              + "/share/hadoop/hdfs/*",
689          ApplicationConstants.Environment.HADOOP_HDFS_HOME.$()
690              + "/share/hadoop/hdfs/lib/*",
691          ApplicationConstants.Environment.HADOOP_YARN_HOME.$()
692              + "/share/hadoop/yarn/*",
693          ApplicationConstants.Environment.HADOOP_YARN_HOME.$()
694              + "/share/hadoop/yarn/lib/*" };
695    
696      /** Container temp directory */
697      public static final String DEFAULT_CONTAINER_TEMP_DIR = "./tmp";
698    
699      public static final String IS_MINI_YARN_CLUSTER = YARN_PREFIX
700          + "is.minicluster";
701    
702      /** Whether to use fixed ports with the minicluster. */
703      public static final String YARN_MINICLUSTER_FIXED_PORTS = YARN_PREFIX
704          + "minicluster.fixed.ports";
705    
706      /**
707       * Default is false to be able to run tests concurrently without port
708       * conflicts.
709       */
710      public static boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false;
711    
712      /**
713       * Whether users are explicitly trying to control resource monitoring
714       * configuration for the MiniYARNCluster. Disabled by default.
715       */
716      public static final String YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING =
717          YARN_PREFIX + "minicluster.control-resource-monitoring";
718      public static final boolean
719          DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING = false;
720    
721      /** The log directory for the containers */
722      public static final String YARN_APP_CONTAINER_LOG_DIR =
723          YARN_PREFIX + "app.container.log.dir";
724    
725      public static final String YARN_APP_CONTAINER_LOG_SIZE =
726          YARN_PREFIX + "app.container.log.filesize";
727    
728      ////////////////////////////////
729      // Other Configs
730      ////////////////////////////////
731    
732      /**
733       * The interval of the yarn client's querying application state after
734       * application submission. The unit is millisecond.
735       */
736      public static final String YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS =
737          YARN_PREFIX + "client.app-submission.poll-interval";
738      public static final long DEFAULT_YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS =
739          1000;
740    
741      /**
742       * Max number of threads in NMClientAsync to process container management
743       * events
744       */
745      public static final String NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE =
746          YARN_PREFIX + "client.nodemanager-client-async.thread-pool-max-size";
747      public static final int DEFAULT_NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE = 500;
748    
749      /**
750       * Maximum number of proxy connections for node manager. It should always be
751       * more than 1. NMClient and MRAppMaster will use this to cache connection
752       * with node manager. There will be at max one connection per node manager.
753       * Ex. configuring it to a value of 5 will make sure that client will at
754       * max have 5 connections cached with 5 different node managers. These
755       * connections will be timed out if idle for more than system wide idle
756       * timeout period. The token if used for authentication then it will be used
757       * only at connection creation time. If new token is received then earlier
758       * connection should be closed in order to use newer token.
759       * Note: {@link YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE}
760       * are related to each other.
761       */
762      public static final String NM_CLIENT_MAX_NM_PROXIES =
763          YARN_PREFIX + "client.max-nodemanagers-proxies";
764      public static final int DEFAULT_NM_CLIENT_MAX_NM_PROXIES = 500;
765      
766      public YarnConfiguration() {
767        super();
768      }
769      
770      public YarnConfiguration(Configuration conf) {
771        super(conf);
772        if (! (conf instanceof YarnConfiguration)) {
773          this.reloadConfiguration();
774        }
775      }
776    
777      public static String getProxyHostAndPort(Configuration conf) {
778        String addr = conf.get(PROXY_ADDRESS);
779        if(addr == null || addr.isEmpty()) {
780          addr = getRMWebAppHostAndPort(conf);
781        }
782        return addr;
783      }
784      
785      public static String getRMWebAppHostAndPort(Configuration conf) {
786        InetSocketAddress address = conf.getSocketAddr(
787            YarnConfiguration.RM_WEBAPP_ADDRESS,
788            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
789            YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
790        address = NetUtils.getConnectAddress(address);
791        StringBuffer sb = new StringBuffer();
792        InetAddress resolved = address.getAddress();
793        if (resolved == null || resolved.isAnyLocalAddress() || 
794            resolved.isLoopbackAddress()) {
795          String lh = address.getHostName();
796          try {
797            lh = InetAddress.getLocalHost().getCanonicalHostName();
798          } catch (UnknownHostException e) {
799            //Ignore and fallback.
800          }
801          sb.append(lh);
802        } else {
803          sb.append(address.getHostName());
804        }
805        sb.append(":").append(address.getPort());
806        return sb.toString();
807      }
808      
809      public static String getRMWebAppURL(Configuration conf) {
810        return JOINER.join("http://", getRMWebAppHostAndPort(conf));
811      }
812      
813    }