1 /* 2 * Licensed to the Apache Software Foundation (ASF) under one or more 3 * contributor license agreements. See the NOTICE file distributed with 4 * this work for additional information regarding copyright ownership. 5 * The ASF licenses this file to You under the Apache License, Version 2.0 6 * (the "License"); you may not use this file except in compliance with 7 * the License. You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18 package org.apache.spark.launcher; 19 20 import java.io.IOException; 21 import java.util.ArrayList; 22 import java.util.List; 23 import java.util.Map; 24 25 import static org.apache.spark.launcher.CommandBuilderUtils.*; 26 27 /** 28 * Command builder for internal Spark classes. 29 * <p> 30 * This class handles building the command to launch all internal Spark classes except for 31 * SparkSubmit (which is handled by {@link SparkSubmitCommandBuilder} class. 32 */ 33 class SparkClassCommandBuilder extends AbstractCommandBuilder { 34 35 private final String className; 36 private final List<String> classArgs; 37 SparkClassCommandBuilder(String className, List<String> classArgs)38 SparkClassCommandBuilder(String className, List<String> classArgs) { 39 this.className = className; 40 this.classArgs = classArgs; 41 } 42 43 @Override buildCommand(Map<String, String> env)44 public List<String> buildCommand(Map<String, String> env) 45 throws IOException, IllegalArgumentException { 46 List<String> javaOptsKeys = new ArrayList<>(); 47 String memKey = null; 48 String extraClassPath = null; 49 50 // Master, Worker, HistoryServer, ExternalShuffleService, MesosClusterDispatcher use 51 // SPARK_DAEMON_JAVA_OPTS (and specific opts) + SPARK_DAEMON_MEMORY. 52 if (className.equals("org.apache.spark.deploy.master.Master")) { 53 javaOptsKeys.add("SPARK_DAEMON_JAVA_OPTS"); 54 javaOptsKeys.add("SPARK_MASTER_OPTS"); 55 memKey = "SPARK_DAEMON_MEMORY"; 56 } else if (className.equals("org.apache.spark.deploy.worker.Worker")) { 57 javaOptsKeys.add("SPARK_DAEMON_JAVA_OPTS"); 58 javaOptsKeys.add("SPARK_WORKER_OPTS"); 59 memKey = "SPARK_DAEMON_MEMORY"; 60 } else if (className.equals("org.apache.spark.deploy.history.HistoryServer")) { 61 javaOptsKeys.add("SPARK_DAEMON_JAVA_OPTS"); 62 javaOptsKeys.add("SPARK_HISTORY_OPTS"); 63 memKey = "SPARK_DAEMON_MEMORY"; 64 } else if (className.equals("org.apache.spark.executor.CoarseGrainedExecutorBackend")) { 65 javaOptsKeys.add("SPARK_JAVA_OPTS"); 66 javaOptsKeys.add("SPARK_EXECUTOR_OPTS"); 67 memKey = "SPARK_EXECUTOR_MEMORY"; 68 } else if (className.equals("org.apache.spark.executor.MesosExecutorBackend")) { 69 javaOptsKeys.add("SPARK_EXECUTOR_OPTS"); 70 memKey = "SPARK_EXECUTOR_MEMORY"; 71 } else if (className.equals("org.apache.spark.deploy.mesos.MesosClusterDispatcher")) { 72 javaOptsKeys.add("SPARK_DAEMON_JAVA_OPTS"); 73 } else if (className.equals("org.apache.spark.deploy.ExternalShuffleService") || 74 className.equals("org.apache.spark.deploy.mesos.MesosExternalShuffleService")) { 75 javaOptsKeys.add("SPARK_DAEMON_JAVA_OPTS"); 76 javaOptsKeys.add("SPARK_SHUFFLE_OPTS"); 77 memKey = "SPARK_DAEMON_MEMORY"; 78 } else { 79 javaOptsKeys.add("SPARK_JAVA_OPTS"); 80 memKey = "SPARK_DRIVER_MEMORY"; 81 } 82 83 List<String> cmd = buildJavaCommand(extraClassPath); 84 85 for (String key : javaOptsKeys) { 86 String envValue = System.getenv(key); 87 if (!isEmpty(envValue) && envValue.contains("Xmx")) { 88 String msg = String.format("%s is not allowed to specify max heap(Xmx) memory settings " + 89 "(was %s). Use the corresponding configuration instead.", key, envValue); 90 throw new IllegalArgumentException(msg); 91 } 92 addOptionString(cmd, envValue); 93 } 94 95 String mem = firstNonEmpty(memKey != null ? System.getenv(memKey) : null, DEFAULT_MEM); 96 cmd.add("-Xmx" + mem); 97 addPermGenSizeOpt(cmd); 98 cmd.add(className); 99 cmd.addAll(classArgs); 100 return cmd; 101 } 102 103 } 104