1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 package org.apache.hadoop.mapreduce; 19 20 import org.apache.hadoop.classification.InterfaceAudience; 21 import org.apache.hadoop.classification.InterfaceAudience.Public; 22 import org.apache.hadoop.classification.InterfaceStability; 23 import org.apache.hadoop.classification.InterfaceStability.Unstable; 24 import org.apache.hadoop.util.Shell; 25 import org.apache.hadoop.yarn.util.Apps; 26 27 @InterfaceAudience.Private 28 @InterfaceStability.Evolving 29 public interface MRJobConfig { 30 31 // Put all of the attribute names in here so that Job and JobContext are 32 // consistent. 33 public static final String INPUT_FORMAT_CLASS_ATTR = "mapreduce.job.inputformat.class"; 34 35 public static final String MAP_CLASS_ATTR = "mapreduce.job.map.class"; 36 37 public static final String MAP_OUTPUT_COLLECTOR_CLASS_ATTR 38 = "mapreduce.job.map.output.collector.class"; 39 40 public static final String COMBINE_CLASS_ATTR = "mapreduce.job.combine.class"; 41 42 public static final String REDUCE_CLASS_ATTR = "mapreduce.job.reduce.class"; 43 44 public static final String OUTPUT_FORMAT_CLASS_ATTR = "mapreduce.job.outputformat.class"; 45 46 public static final String PARTITIONER_CLASS_ATTR = "mapreduce.job.partitioner.class"; 47 48 public static final String SETUP_CLEANUP_NEEDED = "mapreduce.job.committer.setup.cleanup.needed"; 49 50 public static final String TASK_CLEANUP_NEEDED = "mapreduce.job.committer.task.cleanup.needed"; 51 52 public static final String JAR = "mapreduce.job.jar"; 53 54 public static final String ID = "mapreduce.job.id"; 55 56 public static final String JOB_NAME = "mapreduce.job.name"; 57 58 public static final String JAR_UNPACK_PATTERN = "mapreduce.job.jar.unpack.pattern"; 59 60 public static final String USER_NAME = "mapreduce.job.user.name"; 61 62 public static final String PRIORITY = "mapreduce.job.priority"; 63 64 public static final String QUEUE_NAME = "mapreduce.job.queuename"; 65 66 public static final String RESERVATION_ID = "mapreduce.job.reservation.id"; 67 68 public static final String JOB_TAGS = "mapreduce.job.tags"; 69 70 public static final String JVM_NUMTASKS_TORUN = "mapreduce.job.jvm.numtasks"; 71 72 public static final String SPLIT_FILE = "mapreduce.job.splitfile"; 73 74 public static final String SPLIT_METAINFO_MAXSIZE = "mapreduce.job.split.metainfo.maxsize"; 75 public static final long DEFAULT_SPLIT_METAINFO_MAXSIZE = 10000000L; 76 77 public static final String NUM_MAPS = "mapreduce.job.maps"; 78 79 public static final String MAX_TASK_FAILURES_PER_TRACKER = "mapreduce.job.maxtaskfailures.per.tracker"; 80 81 public static final String COMPLETED_MAPS_FOR_REDUCE_SLOWSTART = "mapreduce.job.reduce.slowstart.completedmaps"; 82 83 public static final String NUM_REDUCES = "mapreduce.job.reduces"; 84 85 public static final String SKIP_RECORDS = "mapreduce.job.skiprecords"; 86 87 public static final String SKIP_OUTDIR = "mapreduce.job.skip.outdir"; 88 89 // SPECULATIVE_SLOWNODE_THRESHOLD is obsolete and will be deleted in the future 90 @Deprecated 91 public static final String SPECULATIVE_SLOWNODE_THRESHOLD = "mapreduce.job.speculative.slownodethreshold"; 92 93 public static final String SPECULATIVE_SLOWTASK_THRESHOLD = "mapreduce.job.speculative.slowtaskthreshold"; 94 95 // SPECULATIVECAP is obsolete and will be deleted in the future 96 @Deprecated 97 public static final String SPECULATIVECAP = "mapreduce.job.speculative.speculativecap"; 98 99 public static final String SPECULATIVECAP_RUNNING_TASKS = 100 "mapreduce.job.speculative.speculative-cap-running-tasks"; 101 public static final double DEFAULT_SPECULATIVECAP_RUNNING_TASKS = 102 0.1; 103 104 public static final String SPECULATIVECAP_TOTAL_TASKS = 105 "mapreduce.job.speculative.speculative-cap-total-tasks"; 106 public static final double DEFAULT_SPECULATIVECAP_TOTAL_TASKS = 107 0.01; 108 109 public static final String SPECULATIVE_MINIMUM_ALLOWED_TASKS = 110 "mapreduce.job.speculative.minimum-allowed-tasks"; 111 public static final int DEFAULT_SPECULATIVE_MINIMUM_ALLOWED_TASKS = 112 10; 113 114 public static final String SPECULATIVE_RETRY_AFTER_NO_SPECULATE = 115 "mapreduce.job.speculative.retry-after-no-speculate"; 116 public static final long DEFAULT_SPECULATIVE_RETRY_AFTER_NO_SPECULATE = 117 1000L; 118 119 public static final String SPECULATIVE_RETRY_AFTER_SPECULATE = 120 "mapreduce.job.speculative.retry-after-speculate"; 121 public static final long DEFAULT_SPECULATIVE_RETRY_AFTER_SPECULATE = 122 15000L; 123 124 public static final String JOB_LOCAL_DIR = "mapreduce.job.local.dir"; 125 126 public static final String OUTPUT_KEY_CLASS = "mapreduce.job.output.key.class"; 127 128 public static final String OUTPUT_VALUE_CLASS = "mapreduce.job.output.value.class"; 129 130 public static final String KEY_COMPARATOR = "mapreduce.job.output.key.comparator.class"; 131 132 public static final String COMBINER_GROUP_COMPARATOR_CLASS = "mapreduce.job.combiner.group.comparator.class"; 133 134 public static final String GROUP_COMPARATOR_CLASS = "mapreduce.job.output.group.comparator.class"; 135 136 public static final String WORKING_DIR = "mapreduce.job.working.dir"; 137 138 public static final String CLASSPATH_ARCHIVES = "mapreduce.job.classpath.archives"; 139 140 public static final String CLASSPATH_FILES = "mapreduce.job.classpath.files"; 141 142 public static final String CACHE_FILES = "mapreduce.job.cache.files"; 143 144 public static final String CACHE_ARCHIVES = "mapreduce.job.cache.archives"; 145 146 public static final String CACHE_FILES_SIZES = "mapreduce.job.cache.files.filesizes"; // internal use only 147 148 public static final String CACHE_ARCHIVES_SIZES = "mapreduce.job.cache.archives.filesizes"; // ditto 149 150 public static final String CACHE_LOCALFILES = "mapreduce.job.cache.local.files"; 151 152 public static final String CACHE_LOCALARCHIVES = "mapreduce.job.cache.local.archives"; 153 154 public static final String CACHE_FILE_TIMESTAMPS = "mapreduce.job.cache.files.timestamps"; 155 156 public static final String CACHE_ARCHIVES_TIMESTAMPS = "mapreduce.job.cache.archives.timestamps"; 157 158 public static final String CACHE_FILE_VISIBILITIES = "mapreduce.job.cache.files.visibilities"; 159 160 public static final String CACHE_ARCHIVES_VISIBILITIES = "mapreduce.job.cache.archives.visibilities"; 161 162 /** 163 * @deprecated Symlinks are always on and cannot be disabled. 164 */ 165 @Deprecated 166 public static final String CACHE_SYMLINK = "mapreduce.job.cache.symlink.create"; 167 168 public static final String USER_LOG_RETAIN_HOURS = "mapreduce.job.userlog.retain.hours"; 169 170 public static final String MAPREDUCE_JOB_USER_CLASSPATH_FIRST = "mapreduce.job.user.classpath.first"; 171 172 public static final String MAPREDUCE_JOB_CLASSLOADER = "mapreduce.job.classloader"; 173 174 /** 175 * A comma-separated list of services that function as ShuffleProvider aux-services 176 * (in addition to the built-in ShuffleHandler). 177 * These services can serve shuffle requests from reducetasks. 178 */ 179 public static final String MAPREDUCE_JOB_SHUFFLE_PROVIDER_SERVICES = "mapreduce.job.shuffle.provider.services"; 180 181 public static final String MAPREDUCE_JOB_CLASSLOADER_SYSTEM_CLASSES = "mapreduce.job.classloader.system.classes"; 182 183 public static final String IO_SORT_FACTOR = "mapreduce.task.io.sort.factor"; 184 185 public static final String IO_SORT_MB = "mapreduce.task.io.sort.mb"; 186 187 public static final String INDEX_CACHE_MEMORY_LIMIT = "mapreduce.task.index.cache.limit.bytes"; 188 189 public static final String PRESERVE_FAILED_TASK_FILES = "mapreduce.task.files.preserve.failedtasks"; 190 191 public static final String PRESERVE_FILES_PATTERN = "mapreduce.task.files.preserve.filepattern"; 192 193 public static final String TASK_DEBUGOUT_LINES = "mapreduce.task.debugout.lines"; 194 195 public static final String RECORDS_BEFORE_PROGRESS = "mapreduce.task.merge.progress.records"; 196 197 public static final String SKIP_START_ATTEMPTS = "mapreduce.task.skip.start.attempts"; 198 199 public static final String TASK_ATTEMPT_ID = "mapreduce.task.attempt.id"; 200 201 public static final String TASK_ISMAP = "mapreduce.task.ismap"; 202 public static final boolean DEFAULT_TASK_ISMAP = true; 203 204 public static final String TASK_PARTITION = "mapreduce.task.partition"; 205 206 public static final String TASK_PROFILE = "mapreduce.task.profile"; 207 208 public static final String TASK_PROFILE_PARAMS = "mapreduce.task.profile.params"; 209 210 public static final String DEFAULT_TASK_PROFILE_PARAMS = 211 "-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y," 212 + "verbose=n,file=%s"; 213 214 public static final String NUM_MAP_PROFILES = "mapreduce.task.profile.maps"; 215 216 public static final String NUM_REDUCE_PROFILES = "mapreduce.task.profile.reduces"; 217 218 public static final String TASK_MAP_PROFILE_PARAMS = "mapreduce.task.profile.map.params"; 219 220 public static final String TASK_REDUCE_PROFILE_PARAMS = "mapreduce.task.profile.reduce.params"; 221 222 public static final String TASK_TIMEOUT = "mapreduce.task.timeout"; 223 224 public static final String TASK_TIMEOUT_CHECK_INTERVAL_MS = "mapreduce.task.timeout.check-interval-ms"; 225 226 public static final String TASK_ID = "mapreduce.task.id"; 227 228 public static final String TASK_OUTPUT_DIR = "mapreduce.task.output.dir"; 229 230 public static final String TASK_USERLOG_LIMIT = "mapreduce.task.userlog.limit.kb"; 231 232 public static final String MAP_SORT_SPILL_PERCENT = "mapreduce.map.sort.spill.percent"; 233 234 public static final String MAP_INPUT_FILE = "mapreduce.map.input.file"; 235 236 public static final String MAP_INPUT_PATH = "mapreduce.map.input.length"; 237 238 public static final String MAP_INPUT_START = "mapreduce.map.input.start"; 239 240 public static final String MAP_MEMORY_MB = "mapreduce.map.memory.mb"; 241 public static final int DEFAULT_MAP_MEMORY_MB = 1024; 242 243 public static final String MAP_CPU_VCORES = "mapreduce.map.cpu.vcores"; 244 public static final int DEFAULT_MAP_CPU_VCORES = 1; 245 246 public static final String MAP_ENV = "mapreduce.map.env"; 247 248 public static final String MAP_JAVA_OPTS = "mapreduce.map.java.opts"; 249 250 public static final String MAP_MAX_ATTEMPTS = "mapreduce.map.maxattempts"; 251 252 public static final String MAP_DEBUG_SCRIPT = "mapreduce.map.debug.script"; 253 254 public static final String MAP_SPECULATIVE = "mapreduce.map.speculative"; 255 256 public static final String MAP_FAILURES_MAX_PERCENT = "mapreduce.map.failures.maxpercent"; 257 258 public static final String MAP_SKIP_INCR_PROC_COUNT = "mapreduce.map.skip.proc-count.auto-incr"; 259 260 public static final String MAP_SKIP_MAX_RECORDS = "mapreduce.map.skip.maxrecords"; 261 262 public static final String MAP_COMBINE_MIN_SPILLS = "mapreduce.map.combine.minspills"; 263 264 public static final String MAP_OUTPUT_COMPRESS = "mapreduce.map.output.compress"; 265 266 public static final String MAP_OUTPUT_COMPRESS_CODEC = "mapreduce.map.output.compress.codec"; 267 268 public static final String MAP_OUTPUT_KEY_CLASS = "mapreduce.map.output.key.class"; 269 270 public static final String MAP_OUTPUT_VALUE_CLASS = "mapreduce.map.output.value.class"; 271 272 public static final String MAP_OUTPUT_KEY_FIELD_SEPERATOR = "mapreduce.map.output.key.field.separator"; 273 274 public static final String MAP_LOG_LEVEL = "mapreduce.map.log.level"; 275 276 public static final String REDUCE_LOG_LEVEL = "mapreduce.reduce.log.level"; 277 278 public static final String DEFAULT_LOG_LEVEL = "INFO"; 279 280 public static final String REDUCE_MERGE_INMEM_THRESHOLD = "mapreduce.reduce.merge.inmem.threshold"; 281 282 public static final String REDUCE_INPUT_BUFFER_PERCENT = "mapreduce.reduce.input.buffer.percent"; 283 284 public static final String REDUCE_MARKRESET_BUFFER_PERCENT = "mapreduce.reduce.markreset.buffer.percent"; 285 286 public static final String REDUCE_MARKRESET_BUFFER_SIZE = "mapreduce.reduce.markreset.buffer.size"; 287 288 public static final String REDUCE_MEMORY_MB = "mapreduce.reduce.memory.mb"; 289 public static final int DEFAULT_REDUCE_MEMORY_MB = 1024; 290 291 public static final String REDUCE_CPU_VCORES = "mapreduce.reduce.cpu.vcores"; 292 public static final int DEFAULT_REDUCE_CPU_VCORES = 1; 293 294 public static final String REDUCE_MEMORY_TOTAL_BYTES = "mapreduce.reduce.memory.totalbytes"; 295 296 public static final String SHUFFLE_INPUT_BUFFER_PERCENT = "mapreduce.reduce.shuffle.input.buffer.percent"; 297 public static final float DEFAULT_SHUFFLE_INPUT_BUFFER_PERCENT = 0.70f; 298 299 public static final String SHUFFLE_MEMORY_LIMIT_PERCENT 300 = "mapreduce.reduce.shuffle.memory.limit.percent"; 301 302 public static final String SHUFFLE_MERGE_PERCENT = "mapreduce.reduce.shuffle.merge.percent"; 303 304 public static final String REDUCE_FAILURES_MAXPERCENT = "mapreduce.reduce.failures.maxpercent"; 305 306 public static final String REDUCE_ENV = "mapreduce.reduce.env"; 307 308 public static final String REDUCE_JAVA_OPTS = "mapreduce.reduce.java.opts"; 309 310 public static final String MAPREDUCE_JOB_DIR = "mapreduce.job.dir"; 311 312 public static final String REDUCE_MAX_ATTEMPTS = "mapreduce.reduce.maxattempts"; 313 314 public static final String SHUFFLE_PARALLEL_COPIES = "mapreduce.reduce.shuffle.parallelcopies"; 315 316 public static final String REDUCE_DEBUG_SCRIPT = "mapreduce.reduce.debug.script"; 317 318 public static final String REDUCE_SPECULATIVE = "mapreduce.reduce.speculative"; 319 320 public static final String SHUFFLE_CONNECT_TIMEOUT = "mapreduce.reduce.shuffle.connect.timeout"; 321 322 public static final String SHUFFLE_READ_TIMEOUT = "mapreduce.reduce.shuffle.read.timeout"; 323 324 public static final String SHUFFLE_FETCH_FAILURES = "mapreduce.reduce.shuffle.maxfetchfailures"; 325 public static final String MAX_ALLOWED_FETCH_FAILURES_FRACTION = "mapreduce.reduce.shuffle.max-fetch-failures-fraction"; 326 public static final float DEFAULT_MAX_ALLOWED_FETCH_FAILURES_FRACTION = 0.5f; 327 328 public static final String MAX_FETCH_FAILURES_NOTIFICATIONS = "mapreduce.reduce.shuffle.max-fetch-failures-notifications"; 329 public static final int DEFAULT_MAX_FETCH_FAILURES_NOTIFICATIONS = 3; 330 331 public static final String SHUFFLE_FETCH_RETRY_INTERVAL_MS = "mapreduce.reduce.shuffle.fetch.retry.interval-ms"; 332 /** Default interval that fetcher retry to fetch during NM restart.*/ 333 public final static int DEFAULT_SHUFFLE_FETCH_RETRY_INTERVAL_MS = 1000; 334 335 public static final String SHUFFLE_FETCH_RETRY_TIMEOUT_MS = "mapreduce.reduce.shuffle.fetch.retry.timeout-ms"; 336 337 public static final String SHUFFLE_FETCH_RETRY_ENABLED = "mapreduce.reduce.shuffle.fetch.retry.enabled"; 338 339 public static final String SHUFFLE_NOTIFY_READERROR = "mapreduce.reduce.shuffle.notify.readerror"; 340 341 public static final String MAX_SHUFFLE_FETCH_RETRY_DELAY = "mapreduce.reduce.shuffle.retry-delay.max.ms"; 342 public static final long DEFAULT_MAX_SHUFFLE_FETCH_RETRY_DELAY = 60000; 343 344 public static final String MAX_SHUFFLE_FETCH_HOST_FAILURES = "mapreduce.reduce.shuffle.max-host-failures"; 345 public static final int DEFAULT_MAX_SHUFFLE_FETCH_HOST_FAILURES = 5; 346 347 public static final String REDUCE_SKIP_INCR_PROC_COUNT = "mapreduce.reduce.skip.proc-count.auto-incr"; 348 349 public static final String REDUCE_SKIP_MAXGROUPS = "mapreduce.reduce.skip.maxgroups"; 350 351 public static final String REDUCE_MEMTOMEM_THRESHOLD = "mapreduce.reduce.merge.memtomem.threshold"; 352 353 public static final String REDUCE_MEMTOMEM_ENABLED = "mapreduce.reduce.merge.memtomem.enabled"; 354 355 public static final String COMBINE_RECORDS_BEFORE_PROGRESS = "mapreduce.task.combine.progress.records"; 356 357 public static final String JOB_NAMENODES = "mapreduce.job.hdfs-servers"; 358 359 public static final String JOB_JOBTRACKER_ID = "mapreduce.job.kerberos.jtprinicipal"; 360 361 public static final String JOB_CANCEL_DELEGATION_TOKEN = "mapreduce.job.complete.cancel.delegation.tokens"; 362 363 public static final String JOB_ACL_VIEW_JOB = "mapreduce.job.acl-view-job"; 364 365 public static final String DEFAULT_JOB_ACL_VIEW_JOB = " "; 366 367 public static final String JOB_ACL_MODIFY_JOB = "mapreduce.job.acl-modify-job"; 368 369 public static final String DEFAULT_JOB_ACL_MODIFY_JOB = " "; 370 371 public static final String JOB_RUNNING_MAP_LIMIT = 372 "mapreduce.job.running.map.limit"; 373 public static final int DEFAULT_JOB_RUNNING_MAP_LIMIT = 0; 374 375 public static final String JOB_RUNNING_REDUCE_LIMIT = 376 "mapreduce.job.running.reduce.limit"; 377 public static final int DEFAULT_JOB_RUNNING_REDUCE_LIMIT = 0; 378 379 /* config for tracking the local file where all the credentials for the job 380 * credentials. 381 */ 382 public static final String MAPREDUCE_JOB_CREDENTIALS_BINARY = 383 "mapreduce.job.credentials.binary"; 384 385 /* Configs for tracking ids of tokens used by a job */ 386 public static final String JOB_TOKEN_TRACKING_IDS_ENABLED = 387 "mapreduce.job.token.tracking.ids.enabled"; 388 public static final boolean DEFAULT_JOB_TOKEN_TRACKING_IDS_ENABLED = false; 389 public static final String JOB_TOKEN_TRACKING_IDS = 390 "mapreduce.job.token.tracking.ids"; 391 392 public static final String JOB_SUBMITHOST = 393 "mapreduce.job.submithostname"; 394 public static final String JOB_SUBMITHOSTADDR = 395 "mapreduce.job.submithostaddress"; 396 397 public static final String COUNTERS_MAX_KEY = "mapreduce.job.counters.max"; 398 public static final int COUNTERS_MAX_DEFAULT = 120; 399 400 public static final String COUNTER_GROUP_NAME_MAX_KEY = "mapreduce.job.counters.group.name.max"; 401 public static final int COUNTER_GROUP_NAME_MAX_DEFAULT = 128; 402 403 public static final String COUNTER_NAME_MAX_KEY = "mapreduce.job.counters.counter.name.max"; 404 public static final int COUNTER_NAME_MAX_DEFAULT = 64; 405 406 public static final String COUNTER_GROUPS_MAX_KEY = "mapreduce.job.counters.groups.max"; 407 public static final int COUNTER_GROUPS_MAX_DEFAULT = 50; 408 public static final String JOB_UBERTASK_ENABLE = 409 "mapreduce.job.ubertask.enable"; 410 public static final String JOB_UBERTASK_MAXMAPS = 411 "mapreduce.job.ubertask.maxmaps"; 412 public static final String JOB_UBERTASK_MAXREDUCES = 413 "mapreduce.job.ubertask.maxreduces"; 414 public static final String JOB_UBERTASK_MAXBYTES = 415 "mapreduce.job.ubertask.maxbytes"; 416 417 public static final String MAPREDUCE_JOB_EMIT_TIMELINE_DATA = 418 "mapreduce.job.emit-timeline-data"; 419 public static final boolean DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA = 420 false; 421 422 public static final String MR_PREFIX = "yarn.app.mapreduce."; 423 424 public static final String MR_AM_PREFIX = MR_PREFIX + "am."; 425 426 /** The number of client retries to the AM - before reconnecting to the RM 427 * to fetch Application State. 428 */ 429 public static final String MR_CLIENT_TO_AM_IPC_MAX_RETRIES = 430 MR_PREFIX + "client-am.ipc.max-retries"; 431 public static final int DEFAULT_MR_CLIENT_TO_AM_IPC_MAX_RETRIES = 3; 432 433 /** The number of client retries on socket timeouts to the AM - before 434 * reconnecting to the RM to fetch Application Status. 435 */ 436 public static final String MR_CLIENT_TO_AM_IPC_MAX_RETRIES_ON_TIMEOUTS = 437 MR_PREFIX + "client-am.ipc.max-retries-on-timeouts"; 438 public static final int 439 DEFAULT_MR_CLIENT_TO_AM_IPC_MAX_RETRIES_ON_TIMEOUTS = 3; 440 441 /** 442 * The number of client retries to the RM/HS before throwing exception. 443 */ 444 public static final String MR_CLIENT_MAX_RETRIES = 445 MR_PREFIX + "client.max-retries"; 446 public static final int DEFAULT_MR_CLIENT_MAX_RETRIES = 3; 447 448 /** 449 * How many times to retry jobclient calls (via getjob) 450 */ 451 public static final String MR_CLIENT_JOB_MAX_RETRIES = 452 MR_PREFIX + "client.job.max-retries"; 453 public static final int DEFAULT_MR_CLIENT_JOB_MAX_RETRIES = 0; 454 455 /** 456 * How long to wait between jobclient retries on failure 457 */ 458 public static final String MR_CLIENT_JOB_RETRY_INTERVAL = 459 MR_PREFIX + "client.job.retry-interval"; 460 public static final long DEFAULT_MR_CLIENT_JOB_RETRY_INTERVAL = 461 2000; 462 463 /** The staging directory for map reduce.*/ 464 public static final String MR_AM_STAGING_DIR = 465 MR_AM_PREFIX+"staging-dir"; 466 public static final String DEFAULT_MR_AM_STAGING_DIR = 467 "/tmp/hadoop-yarn/staging"; 468 469 /** The amount of memory the MR app master needs.*/ 470 public static final String MR_AM_VMEM_MB = 471 MR_AM_PREFIX+"resource.mb"; 472 public static final int DEFAULT_MR_AM_VMEM_MB = 1536; 473 474 /** The number of virtual cores the MR app master needs.*/ 475 public static final String MR_AM_CPU_VCORES = 476 MR_AM_PREFIX+"resource.cpu-vcores"; 477 public static final int DEFAULT_MR_AM_CPU_VCORES = 1; 478 479 /** Command line arguments passed to the MR app master.*/ 480 public static final String MR_AM_COMMAND_OPTS = 481 MR_AM_PREFIX+"command-opts"; 482 public static final String DEFAULT_MR_AM_COMMAND_OPTS = "-Xmx1024m"; 483 484 /** Admin command opts passed to the MR app master.*/ 485 public static final String MR_AM_ADMIN_COMMAND_OPTS = 486 MR_AM_PREFIX+"admin-command-opts"; 487 public static final String DEFAULT_MR_AM_ADMIN_COMMAND_OPTS = ""; 488 489 /** Root Logging level passed to the MR app master.*/ 490 public static final String MR_AM_LOG_LEVEL = 491 MR_AM_PREFIX+"log.level"; 492 public static final String DEFAULT_MR_AM_LOG_LEVEL = "INFO"; 493 494 public static final String MR_AM_LOG_KB = 495 MR_AM_PREFIX + "container.log.limit.kb"; 496 public static final int DEFAULT_MR_AM_LOG_KB = 0; // don't roll 497 498 public static final String MR_AM_LOG_BACKUPS = 499 MR_AM_PREFIX + "container.log.backups"; 500 public static final int DEFAULT_MR_AM_LOG_BACKUPS = 0; 501 502 /**The number of splits when reporting progress in MR*/ 503 public static final String MR_AM_NUM_PROGRESS_SPLITS = 504 MR_AM_PREFIX+"num-progress-splits"; 505 public static final int DEFAULT_MR_AM_NUM_PROGRESS_SPLITS = 12; 506 507 /** 508 * Upper limit on the number of threads user to launch containers in the app 509 * master. Expect level config, you shouldn't be needing it in most cases. 510 */ 511 public static final String MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT = 512 MR_AM_PREFIX+"containerlauncher.thread-count-limit"; 513 514 public static final int DEFAULT_MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT = 515 500; 516 517 /** 518 * The initial size of thread pool to launch containers in the app master 519 */ 520 public static final String MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE = 521 MR_AM_PREFIX+"containerlauncher.threadpool-initial-size"; 522 public static final int DEFAULT_MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE = 523 10; 524 525 /** Number of threads to handle job client RPC requests.*/ 526 public static final String MR_AM_JOB_CLIENT_THREAD_COUNT = 527 MR_AM_PREFIX + "job.client.thread-count"; 528 public static final int DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT = 1; 529 530 /** 531 * Range of ports that the MapReduce AM can use when binding. Leave blank 532 * if you want all possible ports. 533 */ 534 public static final String MR_AM_JOB_CLIENT_PORT_RANGE = 535 MR_AM_PREFIX + "job.client.port-range"; 536 537 /** Enable blacklisting of nodes in the job.*/ 538 public static final String MR_AM_JOB_NODE_BLACKLISTING_ENABLE = 539 MR_AM_PREFIX + "job.node-blacklisting.enable"; 540 541 /** Ignore blacklisting if a certain percentage of nodes have been blacklisted */ 542 public static final String MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERECENT = 543 MR_AM_PREFIX + "job.node-blacklisting.ignore-threshold-node-percent"; 544 public static final int DEFAULT_MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERCENT = 545 33; 546 547 /** Enable job recovery.*/ 548 public static final String MR_AM_JOB_RECOVERY_ENABLE = 549 MR_AM_PREFIX + "job.recovery.enable"; 550 public static final boolean MR_AM_JOB_RECOVERY_ENABLE_DEFAULT = true; 551 552 /** 553 * Limit on the number of reducers that can be preempted to ensure that at 554 * least one map task can run if it needs to. Percentage between 0.0 and 1.0 555 */ 556 public static final String MR_AM_JOB_REDUCE_PREEMPTION_LIMIT = 557 MR_AM_PREFIX + "job.reduce.preemption.limit"; 558 public static final float DEFAULT_MR_AM_JOB_REDUCE_PREEMPTION_LIMIT = 0.5f; 559 560 /** AM ACL disabled. **/ 561 public static final String JOB_AM_ACCESS_DISABLED = 562 "mapreduce.job.am-access-disabled"; 563 public static final boolean DEFAULT_JOB_AM_ACCESS_DISABLED = false; 564 565 /** 566 * Limit reduces starting until a certain percentage of maps have finished. 567 * Percentage between 0.0 and 1.0 568 */ 569 public static final String MR_AM_JOB_REDUCE_RAMPUP_UP_LIMIT = 570 MR_AM_PREFIX + "job.reduce.rampup.limit"; 571 public static final float DEFAULT_MR_AM_JOB_REDUCE_RAMP_UP_LIMIT = 0.5f; 572 573 /** The class that should be used for speculative execution calculations.*/ 574 public static final String MR_AM_JOB_SPECULATOR = 575 MR_AM_PREFIX + "job.speculator.class"; 576 577 /** Class used to estimate task resource needs.*/ 578 public static final String MR_AM_TASK_ESTIMATOR = 579 MR_AM_PREFIX + "job.task.estimator.class"; 580 581 /** The lambda value in the smoothing function of the task estimator.*/ 582 public static final String MR_AM_TASK_ESTIMATOR_SMOOTH_LAMBDA_MS = 583 MR_AM_PREFIX 584 + "job.task.estimator.exponential.smooth.lambda-ms"; 585 586 public static final long DEFAULT_MR_AM_TASK_ESTIMATOR_SMOOTH_LAMBDA_MS = 587 1000L * 60; 588 589 /** true if the smoothing rate should be exponential.*/ 590 public static final String MR_AM_TASK_ESTIMATOR_EXPONENTIAL_RATE_ENABLE = 591 MR_AM_PREFIX + "job.task.estimator.exponential.smooth.rate"; 592 593 /** The number of threads used to handle task RPC calls.*/ 594 public static final String MR_AM_TASK_LISTENER_THREAD_COUNT = 595 MR_AM_PREFIX + "job.task.listener.thread-count"; 596 public static final int DEFAULT_MR_AM_TASK_LISTENER_THREAD_COUNT = 30; 597 598 /** How often the AM should send heartbeats to the RM.*/ 599 public static final String MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS = 600 MR_AM_PREFIX + "scheduler.heartbeat.interval-ms"; 601 public static final int DEFAULT_MR_AM_TO_RM_HEARTBEAT_INTERVAL_MS = 1000; 602 603 /** 604 * If contact with RM is lost, the AM will wait MR_AM_TO_RM_WAIT_INTERVAL_MS 605 * milliseconds before aborting. During this interval, AM will still try 606 * to contact the RM. 607 */ 608 public static final String MR_AM_TO_RM_WAIT_INTERVAL_MS = 609 MR_AM_PREFIX + "scheduler.connection.wait.interval-ms"; 610 public static final int DEFAULT_MR_AM_TO_RM_WAIT_INTERVAL_MS = 360000; 611 612 /** 613 * How long to wait in milliseconds for the output committer to cancel 614 * an operation when the job is being killed 615 */ 616 public static final String MR_AM_COMMITTER_CANCEL_TIMEOUT_MS = 617 MR_AM_PREFIX + "job.committer.cancel-timeout"; 618 public static final int DEFAULT_MR_AM_COMMITTER_CANCEL_TIMEOUT_MS = 619 60 * 1000; 620 621 /** 622 * Defines a time window in milliseconds for output committer operations. 623 * If contact with the RM has occurred within this window then commit 624 * operations are allowed, otherwise the AM will not allow output committer 625 * operations until contact with the RM has been re-established. 626 */ 627 public static final String MR_AM_COMMIT_WINDOW_MS = 628 MR_AM_PREFIX + "job.committer.commit-window"; 629 public static final int DEFAULT_MR_AM_COMMIT_WINDOW_MS = 10 * 1000; 630 631 /** 632 * Boolean. Create the base dirs in the JobHistoryEventHandler 633 * Set to false for multi-user clusters. This is an internal config that 634 * is set by the MR framework and read by it too. 635 */ 636 public static final String MR_AM_CREATE_JH_INTERMEDIATE_BASE_DIR = 637 MR_AM_PREFIX + "create-intermediate-jh-base-dir"; 638 639 public static final String MR_AM_HISTORY_MAX_UNFLUSHED_COMPLETE_EVENTS = 640 MR_AM_PREFIX + "history.max-unflushed-events"; 641 public static final int DEFAULT_MR_AM_HISTORY_MAX_UNFLUSHED_COMPLETE_EVENTS = 642 200; 643 644 public static final String MR_AM_HISTORY_JOB_COMPLETE_UNFLUSHED_MULTIPLIER = 645 MR_AM_PREFIX + "history.job-complete-unflushed-multiplier"; 646 public static final int DEFAULT_MR_AM_HISTORY_JOB_COMPLETE_UNFLUSHED_MULTIPLIER = 647 30; 648 649 public static final String MR_AM_HISTORY_COMPLETE_EVENT_FLUSH_TIMEOUT_MS = 650 MR_AM_PREFIX + "history.complete-event-flush-timeout"; 651 public static final long DEFAULT_MR_AM_HISTORY_COMPLETE_EVENT_FLUSH_TIMEOUT_MS = 652 30 * 1000l; 653 654 public static final String MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD = 655 MR_AM_PREFIX + "history.use-batched-flush.queue-size.threshold"; 656 public static final int DEFAULT_MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD = 657 50; 658 659 public static final String MR_AM_HARD_KILL_TIMEOUT_MS = 660 MR_AM_PREFIX + "hard-kill-timeout-ms"; 661 public static final long DEFAULT_MR_AM_HARD_KILL_TIMEOUT_MS = 662 10 * 1000l; 663 664 /** 665 * The threshold in terms of seconds after which an unsatisfied mapper request 666 * triggers reducer preemption to free space. Default 0 implies that the reduces 667 * should be preempted immediately after allocation if there is currently no 668 * room for newly allocated mappers. 669 */ 670 public static final String MR_JOB_REDUCER_PREEMPT_DELAY_SEC = 671 "mapreduce.job.reducer.preempt.delay.sec"; 672 public static final int DEFAULT_MR_JOB_REDUCER_PREEMPT_DELAY_SEC = 0; 673 674 public static final String MR_AM_ENV = 675 MR_AM_PREFIX + "env"; 676 677 public static final String MR_AM_ADMIN_USER_ENV = 678 MR_AM_PREFIX + "admin.user.env"; 679 680 public static final String MR_AM_PROFILE = MR_AM_PREFIX + "profile"; 681 public static final boolean DEFAULT_MR_AM_PROFILE = false; 682 public static final String MR_AM_PROFILE_PARAMS = MR_AM_PREFIX 683 + "profile.params"; 684 685 public static final String MAPRED_MAP_ADMIN_JAVA_OPTS = 686 "mapreduce.admin.map.child.java.opts"; 687 688 public static final String MAPRED_REDUCE_ADMIN_JAVA_OPTS = 689 "mapreduce.admin.reduce.child.java.opts"; 690 691 public static final String DEFAULT_MAPRED_ADMIN_JAVA_OPTS = 692 "-Djava.net.preferIPv4Stack=true " + 693 "-Dhadoop.metrics.log.level=WARN "; 694 695 public static final String MAPRED_ADMIN_USER_SHELL = 696 "mapreduce.admin.user.shell"; 697 698 public static final String DEFAULT_SHELL = "/usr/local/bin/bash"; 699 700 public static final String MAPRED_ADMIN_USER_ENV = 701 "mapreduce.admin.user.env"; 702 703 public final String DEFAULT_MAPRED_ADMIN_USER_ENV = 704 Shell.WINDOWS ? 705 "PATH=%PATH%;%HADOOP_COMMON_HOME%\\bin": 706 "LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native"; 707 708 public static final String WORKDIR = "work"; 709 710 public static final String OUTPUT = "output"; 711 712 public static final String HADOOP_WORK_DIR = "HADOOP_WORK_DIR"; 713 714 // Environment variables used by Pipes. (TODO: these 715 // do not appear to be used by current pipes source code!) 716 public static final String STDOUT_LOGFILE_ENV = "STDOUT_LOGFILE_ENV"; 717 public static final String STDERR_LOGFILE_ENV = "STDERR_LOGFILE_ENV"; 718 719 // This should be the directory where splits file gets localized on the node 720 // running ApplicationMaster. 721 public static final String JOB_SUBMIT_DIR = "jobSubmitDir"; 722 723 // This should be the name of the localized job-configuration file on the node 724 // running ApplicationMaster and Task 725 public static final String JOB_CONF_FILE = "job.xml"; 726 727 // This should be the name of the localized job-jar file on the node running 728 // individual containers/tasks. 729 public static final String JOB_JAR = "job.jar"; 730 731 public static final String JOB_SPLIT = "job.split"; 732 733 public static final String JOB_SPLIT_METAINFO = "job.splitmetainfo"; 734 735 public static final String APPLICATION_MASTER_CLASS = 736 "org.apache.hadoop.mapreduce.v2.app.MRAppMaster"; 737 738 public static final String MAPREDUCE_V2_CHILD_CLASS = 739 "org.apache.hadoop.mapred.YarnChild"; 740 741 public static final String APPLICATION_ATTEMPT_ID = 742 "mapreduce.job.application.attempt.id"; 743 744 /** 745 * Job end notification. 746 */ 747 public static final String MR_JOB_END_NOTIFICATION_URL = 748 "mapreduce.job.end-notification.url"; 749 750 public static final String MR_JOB_END_NOTIFICATION_PROXY = 751 "mapreduce.job.end-notification.proxy"; 752 753 public static final String MR_JOB_END_NOTIFICATION_TIMEOUT = 754 "mapreduce.job.end-notification.timeout"; 755 756 public static final String MR_JOB_END_RETRY_ATTEMPTS = 757 "mapreduce.job.end-notification.retry.attempts"; 758 759 public static final String MR_JOB_END_RETRY_INTERVAL = 760 "mapreduce.job.end-notification.retry.interval"; 761 762 public static final String MR_JOB_END_NOTIFICATION_MAX_ATTEMPTS = 763 "mapreduce.job.end-notification.max.attempts"; 764 765 public static final String MR_JOB_END_NOTIFICATION_MAX_RETRY_INTERVAL = 766 "mapreduce.job.end-notification.max.retry.interval"; 767 768 public static final int DEFAULT_MR_JOB_END_NOTIFICATION_TIMEOUT = 769 5000; 770 771 /* 772 * MR AM Service Authorization 773 */ 774 public static final String 775 MR_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL = 776 "security.job.task.protocol.acl"; 777 public static final String 778 MR_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT = 779 "security.job.client.protocol.acl"; 780 781 /** 782 * CLASSPATH for all YARN MapReduce applications. 783 */ 784 public static final String MAPREDUCE_APPLICATION_CLASSPATH = 785 "mapreduce.application.classpath"; 786 787 public static final String MAPREDUCE_JOB_LOG4J_PROPERTIES_FILE = 788 "mapreduce.job.log4j-properties-file"; 789 790 /** 791 * Path to MapReduce framework archive 792 */ 793 public static final String MAPREDUCE_APPLICATION_FRAMEWORK_PATH = 794 "mapreduce.application.framework.path"; 795 796 /** 797 * Default CLASSPATH for all YARN MapReduce applications constructed with 798 * platform-agnostic syntax. 799 */ 800 @Public 801 @Unstable 802 public final String DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH = Apps 803 .crossPlatformify("HADOOP_MAPRED_HOME") 804 + "/share/hadoop/mapreduce/*," 805 + Apps.crossPlatformify("HADOOP_MAPRED_HOME") 806 + "/share/hadoop/mapreduce/lib/*"; 807 808 /** 809 * Default platform-specific CLASSPATH for all YARN MapReduce applications 810 * constructed based on client OS syntax. 811 * <p> 812 * Note: Use {@link DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH} 813 * for cross-platform practice i.e. submit an application from a Windows 814 * client to a Linux/Unix server or vice versa. 815 * </p> 816 */ 817 public final String DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH = 818 Shell.WINDOWS ? "%HADOOP_MAPRED_HOME%\\share\\hadoop\\mapreduce\\*," 819 + "%HADOOP_MAPRED_HOME%\\share\\hadoop\\mapreduce\\lib\\*" 820 : "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*," 821 + "$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*"; 822 823 public static final String WORKFLOW_ID = "mapreduce.workflow.id"; 824 825 public static final String TASK_LOG_BACKUPS = 826 MR_PREFIX + "task.container.log.backups"; 827 public static final int DEFAULT_TASK_LOG_BACKUPS = 0; // don't roll 828 829 public static final String REDUCE_SEPARATE_SHUFFLE_LOG = 830 MR_PREFIX + "shuffle.log.separate"; 831 public static final boolean DEFAULT_REDUCE_SEPARATE_SHUFFLE_LOG = true; 832 833 public static final String SHUFFLE_LOG_BACKUPS = 834 MR_PREFIX + "shuffle.log.backups"; 835 public static final int DEFAULT_SHUFFLE_LOG_BACKUPS = 0; // don't roll 836 837 public static final String SHUFFLE_LOG_KB = 838 MR_PREFIX + "shuffle.log.limit.kb"; 839 public static final long DEFAULT_SHUFFLE_LOG_KB = 0L; 840 841 public static final String WORKFLOW_NAME = "mapreduce.workflow.name"; 842 843 public static final String WORKFLOW_NODE_NAME = 844 "mapreduce.workflow.node.name"; 845 846 public static final String WORKFLOW_ADJACENCY_PREFIX_STRING = 847 "mapreduce.workflow.adjacency."; 848 849 public static final String WORKFLOW_ADJACENCY_PREFIX_PATTERN = 850 "^mapreduce\\.workflow\\.adjacency\\..+"; 851 852 public static final String WORKFLOW_TAGS = "mapreduce.workflow.tags"; 853 854 /** 855 * The maximum number of application attempts. 856 * It is a application-specific setting. 857 */ 858 public static final String MR_AM_MAX_ATTEMPTS = "mapreduce.am.max-attempts"; 859 860 public static final int DEFAULT_MR_AM_MAX_ATTEMPTS = 2; 861 862 public static final String MR_APPLICATION_TYPE = "MAPREDUCE"; 863 864 public static final String MR_ENCRYPTED_INTERMEDIATE_DATA = 865 "mapreduce.job.encrypted-intermediate-data"; 866 public static final boolean DEFAULT_MR_ENCRYPTED_INTERMEDIATE_DATA = false; 867 868 public static final String MR_ENCRYPTED_INTERMEDIATE_DATA_KEY_SIZE_BITS = 869 "mapreduce.job.encrypted-intermediate-data-key-size-bits"; 870 public static final int DEFAULT_MR_ENCRYPTED_INTERMEDIATE_DATA_KEY_SIZE_BITS = 871 128; 872 873 public static final String MR_ENCRYPTED_INTERMEDIATE_DATA_BUFFER_KB = 874 "mapreduce.job.encrypted-intermediate-data.buffer.kb"; 875 public static final int DEFAULT_MR_ENCRYPTED_INTERMEDIATE_DATA_BUFFER_KB = 876 128; 877 878 } 879