1 /*****************************************************************************\ 2 * slurm_opt.h - definitions for salloc/sbatch/srun option processing 3 ***************************************************************************** 4 * Copyright (C) 2002-2007 The Regents of the University of California. 5 * Copyright (C) 2008-2010 Lawrence Livermore National Security. 6 * Portions Copyright (C) 2010-2017 SchedMD LLC <https://www.schedmd.com> 7 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). 8 * Written by Mark Grondona <grondona1@llnl.gov>, 9 * Christopher J. Morrone <morrone2@llnl.gov>, et. al. 10 * CODE-OCEC-09-009. All rights reserved. 11 * 12 * This file is part of Slurm, a resource management program. 13 * For details, see <https://slurm.schedmd.com/>. 14 * Please also read the included file: DISCLAIMER. 15 * 16 * Slurm is free software; you can redistribute it and/or modify it under 17 * the terms of the GNU General Public License as published by the Free 18 * Software Foundation; either version 2 of the License, or (at your option) 19 * any later version. 20 * 21 * In addition, as a special exception, the copyright holders give permission 22 * to link the code of portions of this program with the OpenSSL library under 23 * certain conditions as described in each individual source file, and 24 * distribute linked combinations including the two. You must obey the GNU 25 * General Public License in all respects for all of the code used other than 26 * OpenSSL. If you modify file(s) with this exception, you may extend this 27 * exception to your version of the file(s), but you are not obligated to do 28 * so. If you do not wish to do so, delete this exception statement from your 29 * version. If you delete this exception statement from all source files in 30 * the program, then also delete it here. 31 * 32 * Slurm is distributed in the hope that it will be useful, but WITHOUT ANY 33 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 34 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 35 * details. 36 * 37 * You should have received a copy of the GNU General Public License along 38 * with Slurm; if not, write to the Free Software Foundation, Inc., 39 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 40 \*****************************************************************************/ 41 42 #ifndef _SLURM_OPT_H_ 43 #define _SLURM_OPT_H_ 44 45 #include <inttypes.h> 46 #include <stdbool.h> 47 #include <time.h> 48 49 #include "config.h" 50 51 #include "slurm/slurm.h" 52 53 #include "src/common/data.h" 54 55 #define DEFAULT_IMMEDIATE 1 56 #define DEFAULT_BELL_DELAY 10 57 #define SRUN_MAX_THREADS 60 58 59 typedef enum {BELL_NEVER, BELL_AFTER_DELAY, BELL_ALWAYS} bell_flag_t; 60 61 /* 62 * getopt_long flags used by salloc/sbatch/srun 63 * integers and *not* valid characters to avoid collision with getopt flags. 64 * 65 * Please keep this list alphabetical on the master branch. 66 * On release branches, you must *not* add to the middle of the list. Doing so 67 * would constitute and ABI breaking change. In such cases, please add to the 68 * end of the list, then alphabetize it on the master branch only. 69 */ 70 71 enum { 72 LONG_OPT_ENUM_START = 0x100, 73 LONG_OPT_ACCEL_BIND, 74 LONG_OPT_ACCTG_FREQ, 75 LONG_OPT_ALLOC_NODELIST, 76 LONG_OPT_BATCH, 77 LONG_OPT_BCAST, 78 LONG_OPT_BELL, 79 LONG_OPT_BLRTS_IMAGE, 80 LONG_OPT_BURST_BUFFER_FILE, 81 LONG_OPT_BURST_BUFFER_SPEC, 82 LONG_OPT_CLUSTER, 83 LONG_OPT_CLUSTER_CONSTRAINT, 84 LONG_OPT_COMMENT, 85 LONG_OPT_COMPRESS, 86 LONG_OPT_CONTIGUOUS, 87 LONG_OPT_CORE, 88 LONG_OPT_CORESPERSOCKET, 89 LONG_OPT_CPU_BIND, 90 LONG_OPT_CPU_FREQ, 91 LONG_OPT_CPUS_PER_GPU, 92 LONG_OPT_DEADLINE, 93 LONG_OPT_DEBUGGER_TEST, 94 LONG_OPT_DELAY_BOOT, 95 LONG_OPT_ENVIRONMENT, /* only for data */ 96 LONG_OPT_EPILOG, 97 LONG_OPT_EXCLUSIVE, 98 LONG_OPT_EXPORT, 99 LONG_OPT_EXPORT_FILE, 100 LONG_OPT_GET_USER_ENV, 101 LONG_OPT_GID, 102 LONG_OPT_GPU_BIND, 103 LONG_OPT_GPU_FREQ, 104 LONG_OPT_GPUS, 105 LONG_OPT_GPUS_PER_NODE, 106 LONG_OPT_GPUS_PER_SOCKET, 107 LONG_OPT_GPUS_PER_TASK, 108 LONG_OPT_GRES, 109 LONG_OPT_GRES_FLAGS, 110 LONG_OPT_HINT, 111 LONG_OPT_IGNORE_PBS, 112 LONG_OPT_JOBID, 113 LONG_OPT_KILL_INV_DEP, 114 LONG_OPT_LAUNCH_CMD, 115 LONG_OPT_LAUNCHER_OPTS, 116 LONG_OPT_LINUX_IMAGE, 117 LONG_OPT_MAIL_TYPE, 118 LONG_OPT_MAIL_USER, 119 LONG_OPT_MCS_LABEL, 120 LONG_OPT_MEM, 121 LONG_OPT_MEM_BIND, 122 LONG_OPT_MEM_PER_CPU, 123 LONG_OPT_MEM_PER_GPU, 124 LONG_OPT_MINCORES, 125 LONG_OPT_MINCPUS, 126 LONG_OPT_MINSOCKETS, 127 LONG_OPT_MINTHREADS, 128 LONG_OPT_MLOADER_IMAGE, 129 LONG_OPT_MPI, 130 LONG_OPT_MSG_TIMEOUT, 131 LONG_OPT_MULTI, 132 LONG_OPT_NETWORK, 133 LONG_OPT_NICE, 134 LONG_OPT_NO_BELL, 135 LONG_OPT_NO_REQUEUE, 136 LONG_OPT_NO_SHELL, 137 LONG_OPT_NTASKSPERCORE, 138 LONG_OPT_NTASKSPERNODE, 139 LONG_OPT_NTASKSPERSOCKET, 140 LONG_OPT_OPEN_MODE, 141 LONG_OPT_HET_GROUP, 142 LONG_OPT_PARSABLE, 143 LONG_OPT_POWER, 144 LONG_OPT_PRIORITY, 145 LONG_OPT_PROFILE, 146 LONG_OPT_PROLOG, 147 LONG_OPT_PROPAGATE, 148 LONG_OPT_PTY, 149 LONG_OPT_QUIT_ON_INTR, 150 LONG_OPT_RAMDISK_IMAGE, 151 LONG_OPT_REBOOT, 152 LONG_OPT_REQUEUE, 153 LONG_OPT_RESERVATION, 154 LONG_OPT_RESV_PORTS, 155 LONG_OPT_SIGNAL, 156 LONG_OPT_SLURMD_DEBUG, 157 LONG_OPT_SOCKETSPERNODE, 158 LONG_OPT_SPREAD_JOB, 159 LONG_OPT_SWITCH_REQ, 160 LONG_OPT_SWITCH_WAIT, 161 LONG_OPT_SWITCHES, 162 LONG_OPT_TASK_EPILOG, 163 LONG_OPT_TASK_PROLOG, 164 LONG_OPT_TEST_ONLY, 165 LONG_OPT_THREAD_SPEC, 166 LONG_OPT_THREADSPERCORE, 167 LONG_OPT_TIME_MIN, 168 LONG_OPT_TMP, 169 LONG_OPT_TRES_PER_JOB, 170 LONG_OPT_UID, 171 LONG_OPT_UMASK, 172 LONG_OPT_USAGE, 173 LONG_OPT_USE_MIN_NODES, 174 LONG_OPT_WAIT_ALL_NODES, 175 LONG_OPT_WCKEY, 176 LONG_OPT_WRAP, 177 LONG_OPT_X11, 178 LONG_OPT_ENUM_END 179 }; 180 181 /* 182 * options only processed by salloc 183 */ 184 typedef struct { 185 bell_flag_t bell; /* --bell, --no-bell */ 186 int kill_command_signal; /* --kill-command */ 187 bool no_shell; /* --no-shell */ 188 uint16_t wait_all_nodes; /* --wait-nodes-ready=val */ 189 } salloc_opt_t; 190 191 /* 192 * options only processed by sbatch 193 */ 194 typedef struct { 195 /* batch script argv and argc, if provided on the command line */ 196 int script_argc; 197 char **script_argv; 198 199 char *array_inx; /* --array */ 200 char *batch_features; /* --batch */ 201 char *export_env; /* --export */ 202 char *export_file; /* --export-file=file */ 203 bool ignore_pbs; /* --ignore-pbs */ 204 int minsockets; /* --minsockets=n */ 205 int mincores; /* --mincores=n */ 206 int minthreads; /* --minthreads=n */ 207 bool parsable; /* --parsable */ 208 char *propagate; /* --propagate[=RLIMIT_CORE,...]*/ 209 uint8_t open_mode; /* --open-mode */ 210 int requeue; /* --requeue and --no-requeue */ 211 bool test_only; /* --test-only */ 212 int umask; /* job umask for PBS */ 213 bool wait; /* --wait */ 214 uint16_t wait_all_nodes; /* --wait-nodes-ready=val */ 215 char *wrap; 216 } sbatch_opt_t; 217 218 /* 219 * options only processed by srun 220 */ 221 typedef struct { 222 int argc; /* length of argv array */ 223 char **argv; /* left over on command line */ 224 225 uint16_t accel_bind_type; /* --accel-bind */ 226 char *alloc_nodelist; /* grabbed from the environment */ 227 char *bcast_file; /* --bcast, copy executable to compute nodes */ 228 bool bcast_flag; /* --bcast, copy executable to compute nodes */ 229 char *cmd_name; /* name of command to execute */ 230 uint16_t compress; /* --compress (for --bcast option) */ 231 bool core_spec_set; /* core_spec explicitly set */ 232 char *cpu_bind; /* binding map for map/mask_cpu */ 233 cpu_bind_type_t cpu_bind_type; /* --cpu-bind */ 234 bool debugger_test; /* --debugger-test */ 235 bool disable_status; /* --disable-status */ 236 char *epilog; /* --epilog */ 237 bool exclusive; /* --exclusive */ 238 char *export_env; /* --export */ 239 uint32_t jobid; /* --jobid */ 240 int32_t kill_bad_exit; /* --kill-on-bad-exit */ 241 bool labelio; /* --label-output */ 242 int32_t max_threads; /* --threads */ 243 int max_wait; /* --wait */ 244 int msg_timeout; /* undocumented */ 245 char *mpi_type; /* --mpi=type */ 246 bool multi_prog; /* multiple programs to execute */ 247 int32_t multi_prog_cmds; /* number of commands in multi prog file */ 248 bool no_alloc; /* --no-allocate */ 249 uint8_t open_mode; /* --open-mode=append|truncate */ 250 char *het_group; /* --het-group */ 251 bitstr_t *het_grp_bits; /* --het-group in bitmap form */ 252 int het_step_cnt; /* Total count of het groups to launch */ 253 bool parallel_debug; /* srun controlled by debugger */ 254 bool preserve_env; /* --preserve-env */ 255 char *prolog; /* --prolog */ 256 char *propagate; /* --propagate[=RLIMIT_CORE,...]*/ 257 bool pty; /* --pty */ 258 bool quit_on_intr; /* --quit-on-interrupt */ 259 int relative; /* --relative */ 260 int resv_port_cnt; /* --resv_ports */ 261 int slurmd_debug; /* --slurmd-debug */ 262 char *task_epilog; /* --task-epilog */ 263 char *task_prolog; /* --task-prolog */ 264 bool test_exec; /* test_exec set */ 265 bool test_only; /* --test-only */ 266 bool unbuffered; /* --unbuffered */ 267 } srun_opt_t; 268 269 typedef struct { 270 bool set; /* Has the option been set */ 271 bool set_by_env; /* Has the option been set by env var */ 272 bool set_by_data; /* Has the option been set by data_t */ 273 } slurm_opt_state_t; 274 275 typedef struct { 276 salloc_opt_t *salloc_opt; 277 sbatch_opt_t *sbatch_opt; 278 srun_opt_t *srun_opt; 279 280 slurm_opt_state_t *state; 281 282 void (*help_func)(void); /* Print --help info */ 283 void (*usage_func)(void); /* Print --usage info */ 284 285 char *burst_buffer; /* --bb */ 286 char *burst_buffer_file; /* --bbf */ 287 char *clusters; /* cluster to run this on. */ 288 uid_t uid; /* local uid */ 289 gid_t gid; /* local gid */ 290 char *chdir; /* --chdir */ 291 int ntasks; /* --ntasks */ 292 bool ntasks_set; /* ntasks explicitly set */ 293 int cpus_per_task; /* --cpus-per-task=n */ 294 bool cpus_set; /* cpus_per_task explicitly set */ 295 int min_nodes; /* --nodes=n */ 296 int max_nodes; /* --nodes=x-n */ 297 bool nodes_set; /* nodes explicitly set */ 298 int sockets_per_node; /* --sockets-per-node=n */ 299 int cores_per_socket; /* --cores-per-socket=n */ 300 uint32_t job_flags; /* --kill_invalid_dep, --gres-flags */ 301 int threads_per_core; /* --threads-per-core=n */ 302 int ntasks_per_node; /* --ntasks-per-node=n */ 303 int ntasks_per_socket; /* --ntasks-per-socket=n */ 304 int ntasks_per_core; /* --ntasks-per-core=n */ 305 char *hint; /* --hint or SLURM_HINT envvar */ 306 mem_bind_type_t mem_bind_type; /* --mem-bind= */ 307 char *mem_bind; /* binding map for map/mask_mem */ 308 bool extra_set; /* extra node info explicitly set */ 309 int time_limit; /* --time, in minutes */ 310 int time_min; /* --min-time, in minutes */ 311 char *partition; /* --partition */ 312 uint32_t profile; /* --profile=[all | none] */ 313 enum task_dist_states distribution; 314 /* --distribution */ 315 uint32_t plane_size; /* lllp distribution -> plane_size for 316 * when -m plane=<# of lllp per 317 * plane> */ 318 char *job_name; /* --job-name */ 319 char *dependency; /* --dependency */ 320 int nice; /* --nice */ 321 uint32_t priority; /* --priority */ 322 char *account; /* --account */ 323 char *comment; /* --comment */ 324 char *qos; /* --qos */ 325 int immediate; /* --immediate */ 326 uint16_t warn_flags; /* --signal=flags:<int>@<time> */ 327 uint16_t warn_signal; /* --signal=flags:<int>@<time> */ 328 uint16_t warn_time; /* --signal=flags:<int>@<time> */ 329 330 bool hold; /* --hold */ 331 bool no_kill; /* --no-kill */ 332 char *acctg_freq; /* --acctg-freq=<type1>=<freq1>,... */ 333 bool overcommit; /* --overcommit */ 334 uint16_t shared; /* --share */ 335 char *licenses; /* --licenses */ 336 char *network; /* --network */ 337 int quiet; 338 int verbose; 339 340 /* constraint options */ 341 int cpus_per_gpu; /* --cpus-per-gpu */ 342 char *gpus; /* --gpus */ 343 char *gpu_bind; /* --gpu_bind */ 344 char *gpu_freq; /* --gpu_freq */ 345 char *gpus_per_node; /* --gpus_per_node */ 346 char *gpus_per_socket; /* --gpus_per_socket */ 347 char *gpus_per_task; /* --gpus_per_task */ 348 349 int pn_min_cpus; /* --mincpus */ 350 uint64_t mem_per_cpu; /* --mem-per-cpu */ 351 uint64_t mem_per_gpu; /* --mem-per-gpu */ 352 uint64_t pn_min_memory; /* --mem */ 353 uint64_t pn_min_tmp_disk; /* --tmp */ 354 char *constraint; /* --constraint */ 355 char *c_constraint; /* --cluster-constraint */ 356 char *gres; /* --gres */ 357 bool contiguous; /* --contiguous */ 358 char *nodefile; /* --nodefile */ 359 char *nodelist; /* --nodelist=node1,node2,... */ 360 char **environment; /* job environment */ 361 char *exclude; /* --exclude=node1,node2,... */ 362 363 bool reboot; /* --reboot */ 364 365 time_t begin; /* --begin */ 366 char *extra; /* unused */ 367 uint16_t mail_type; /* --mail-type */ 368 char *mail_user; /* --mail-user */ 369 int get_user_env_time; /* --get-user-env[=timeout] */ 370 int get_user_env_mode; /* --get-user-env=[S|L] */ 371 char *wckey; /* workload characterization key */ 372 char *reservation; /* --reservation */ 373 int req_switch; /* min number of switches */ 374 int wait4switch; /* max time to wait for min switches */ 375 char **spank_job_env; /* SPANK controlled environment for job 376 * Prolog and Epilog */ 377 int spank_job_env_size; /* size of spank_job_env */ 378 int core_spec; /* --core-spec */ 379 uint32_t cpu_freq_min; /* Minimum cpu frequency */ 380 uint32_t cpu_freq_max; /* Maximum cpu frequency */ 381 uint32_t cpu_freq_gov; /* cpu frequency governor */ 382 uint8_t power; /* power management flags */ 383 char *mcs_label; /* mcs label */ 384 time_t deadline; /* ---deadline */ 385 uint32_t delay_boot; /* --delay-boot */ 386 char *tres_bind; /* derived from gpu_bind */ 387 char *tres_freq; /* derived from gpu_freq */ 388 uint16_t x11; /* --x11 */ 389 char *x11_magic_cookie; /* cookie retrieved from xauth */ 390 char *x11_target; /* target host, or unix socket */ 391 /* if x11_target_port == 0 */ 392 uint16_t x11_target_port; /* target display TCP port on localhost */ 393 394 /* used in both sbatch and srun, here for convenience */ 395 char *efname; /* error file name */ 396 char *ifname; /* input file name */ 397 char *ofname; /* output file name */ 398 } slurm_opt_t; 399 400 extern struct option *slurm_option_table_create(slurm_opt_t *opt, 401 char **opt_string); 402 extern void slurm_option_table_destroy(struct option *optz); 403 404 /* 405 * Process individual argument for the current job component 406 * IN opt - current component 407 * IN optval - argument identifier 408 * IN arg - argument value 409 * IN set_by_env - flag if set by environment (and not cli) 410 * IN early_pass - early vs. late pass for HetJob option inheritance 411 * RET SLURM_SUCCESS or error 412 */ 413 extern int slurm_process_option(slurm_opt_t *opt, int optval, const char *arg, 414 bool set_by_env, bool early_pass); 415 416 /* 417 * Process incoming single component of Job data entry 418 * IN opt - options to populate from job chunk 419 * IN job - data containing job request 420 * IN/OUT errors - data dictionary to populate with detailed errors 421 * RET SLURM_SUCCESS or error 422 */ 423 extern int slurm_process_option_data(slurm_opt_t *opt, int optval, 424 const data_t *arg, data_t *errors); 425 426 /* 427 * Print all options that have been set through slurm_process_option() 428 * in a form suitable for use with the -v flag to salloc/sbatch/srun. 429 */ 430 extern void slurm_print_set_options(slurm_opt_t *opt); 431 432 /* 433 * Reset slurm_opt_t settings for a given pass. 434 */ 435 extern void slurm_reset_all_options(slurm_opt_t *opt, bool first_pass); 436 437 /* 438 * Free all memory associated with opt members 439 * Note: assumes that opt, opt->salloc_opt, opt->sbatch_opt, and 440 * opt->srun_opt should not be xfreed. 441 */ 442 extern void slurm_free_options_members(slurm_opt_t *opt); 443 444 /* 445 * Was the option set by a cli argument? 446 */ 447 extern bool slurm_option_set_by_cli(slurm_opt_t *opt, int optval); 448 449 /* 450 * Was the option set by an env var? 451 */ 452 extern bool slurm_option_set_by_env(slurm_opt_t *opt, int optval); 453 454 /* 455 * Was the option set by an data_t value? 456 */ 457 extern bool slurm_option_set_by_env(slurm_opt_t *opt, int optval); 458 459 /* 460 * Get option value by common option name. 461 */ 462 extern char *slurm_option_get(slurm_opt_t *opt, const char *name); 463 464 /* 465 * Is option set? Discover by common option name. 466 */ 467 extern bool slurm_option_isset(slurm_opt_t *opt, const char *name); 468 469 /* 470 * Replace option value by common option name. 471 */ 472 extern int slurm_option_set(slurm_opt_t *opt, const char *name, 473 const char *value, bool early); 474 475 /* 476 * Reset option by common option name. 477 */ 478 extern bool slurm_option_reset(slurm_opt_t *opt, const char *name); 479 480 /* 481 * Function for iterating through all the common option data structure 482 * and returning (via parameter arguments) the name and value of each 483 * set slurm option. 484 * 485 * IN opt - option data structure being interpreted 486 * OUT name - xmalloc()'d string with the option name 487 * OUT value - xmalloc()'d string with the option value 488 * IN/OUT state - internal state, should be set to 0 for the first call 489 * RETURNS - true if name/value set; false if no more options 490 */ 491 extern bool slurm_option_get_next_set(slurm_opt_t *opt, char **name, 492 char **value, size_t *state); 493 494 /* 495 * Validate that the three memory options (--mem, --mem-per-cpu, --mem-per-gpu) 496 * and their associated environment variables are set mutually exclusively. 497 * 498 * This will fatal() if multiple CLI options are specified simultaneously. 499 * If any of the CLI options are specified, the other options are reset to 500 * clear anything that may have been set through the environment. 501 * Otherwise, if multiple environment variables are set simultaneously, 502 * this will fatal(). 503 */ 504 extern void validate_memory_options(slurm_opt_t *opt); 505 506 #endif /* _SLURM_OPT_H_ */ 507