1 /*****************************************************************************\
2 * reservation.c - resource reservation management
3 *****************************************************************************
4 * Copyright (C) 2009-2010 Lawrence Livermore National Security.
5 * Copyright (C) 2012-2017 SchedMD LLC <https://www.schedmd.com>
6 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
7 * Written by Morris Jette <jette1@llnl.gov> et. al.
8 * CODE-OCEC-09-009. All rights reserved.
9 *
10 * This file is part of Slurm, a resource management program.
11 * For details, see <https://slurm.schedmd.com/>.
12 * Please also read the included file: DISCLAIMER.
13 *
14 * Slurm is free software; you can redistribute it and/or modify it under
15 * the terms of the GNU General Public License as published by the Free
16 * Software Foundation; either version 2 of the License, or (at your option)
17 * any later version.
18 *
19 * In addition, as a special exception, the copyright holders give permission
20 * to link the code of portions of this program with the OpenSSL library under
21 * certain conditions as described in each individual source file, and
22 * distribute linked combinations including the two. You must obey the GNU
23 * General Public License in all respects for all of the code used other than
24 * OpenSSL. If you modify file(s) with this exception, you may extend this
25 * exception to your version of the file(s), but you are not obligated to do
26 * so. If you do not wish to do so, delete this exception statement from your
27 * version. If you delete this exception statement from all source files in
28 * the program, then also delete it here.
29 *
30 * Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
31 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
32 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
33 * details.
34 *
35 * You should have received a copy of the GNU General Public License along
36 * with Slurm; if not, write to the Free Software Foundation, Inc.,
37 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
38 \*****************************************************************************/
39
40 #include "config.h"
41
42 #include <fcntl.h>
43 #include <pthread.h>
44 #include <signal.h>
45 #include <stdlib.h>
46 #include <string.h>
47 #include <sys/stat.h>
48 #include <sys/types.h>
49 #include <time.h>
50 #include <unistd.h>
51
52 #include "slurm/slurm.h"
53 #include "slurm/slurm_errno.h"
54
55 #include "src/common/assoc_mgr.h"
56 #include "src/common/bitstring.h"
57 #include "src/common/fd.h"
58 #include "src/common/hostlist.h"
59 #include "src/common/list.h"
60 #include "src/common/log.h"
61 #include "src/common/macros.h"
62 #include "src/common/node_features.h"
63 #include "src/common/node_select.h"
64 #include "src/common/pack.h"
65 #include "src/common/parse_time.h"
66 #include "src/common/slurm_accounting_storage.h"
67 #include "src/common/slurm_time.h"
68 #include "src/common/uid.h"
69 #include "src/common/xassert.h"
70 #include "src/common/xmalloc.h"
71 #include "src/common/xstring.h"
72
73 #include "src/slurmctld/burst_buffer.h"
74 #include "src/slurmctld/job_scheduler.h"
75 #include "src/slurmctld/licenses.h"
76 #include "src/slurmctld/locks.h"
77 #include "src/slurmctld/node_scheduler.h"
78 #include "src/slurmctld/reservation.h"
79 #include "src/slurmctld/slurmctld.h"
80 #include "src/slurmctld/state_save.h"
81
82 #define _DEBUG 0
83 #define RESV_MAGIC 0x3b82
84
85 /* Permit sufficient time for slurmctld failover or other long delay before
86 * considering a reservation time specification being invalid */
87 #define MAX_RESV_DELAY 600
88
89 #define MAX_RESV_COUNT 9999
90
91 /* No need to change we always pack SLURM_PROTOCOL_VERSION */
92 #define RESV_STATE_VERSION "PROTOCOL_VERSION"
93
94 typedef struct resv_thread_args {
95 char *script;
96 char *resv_name;
97 } resv_thread_args_t;
98
99 time_t last_resv_update = (time_t) 0;
100 List resv_list = (List) NULL;
101 static List prom_resv_list = NULL;
102 uint32_t top_suffix = 0;
103
104 /*
105 * the two following structs enable to build a
106 * planning of a constraint evolution over time
107 * taking into account temporal overlapping
108 */
109 typedef struct constraint_planning {
110 List slot_list;
111 } constraint_planning_t;
112
113 typedef struct constraint_slot {
114 time_t start;
115 time_t end;
116 uint32_t value;
117 } constraint_slot_t;
118 /*
119 * the associated functions are the following
120 */
121 static void _init_constraint_planning(constraint_planning_t* sched);
122 static void _print_constraint_planning(constraint_planning_t* sched);
123 static void _free_constraint_planning(constraint_planning_t* sched);
124 static void _update_constraint_planning(constraint_planning_t* sched,
125 uint32_t value, time_t start,
126 time_t end);
127 static uint32_t _max_constraint_planning(constraint_planning_t* sched,
128 time_t *start, time_t *end);
129
130
131 static void _advance_resv_time(slurmctld_resv_t *resv_ptr);
132 static void _advance_time(time_t *res_time, int day_cnt);
133 static int _build_account_list(char *accounts, int *account_cnt,
134 char ***account_list, bool *account_not);
135 static int _build_uid_list(char *users, int *user_cnt, uid_t **user_list,
136 bool *user_not);
137 static void _clear_job_resv(slurmctld_resv_t *resv_ptr);
138 static slurmctld_resv_t *_copy_resv(slurmctld_resv_t *resv_orig_ptr);
139 static void _del_resv_rec(void *x);
140 static void _dump_resv_req(resv_desc_msg_t *resv_ptr, char *mode);
141 static int _find_resv_id(void *x, void *key);
142 static int _find_resv_ptr(void *x, void *key);
143 static int _find_resv_name(void *x, void *key);
144 static void *_fork_script(void *x);
145 static void _free_script_arg(resv_thread_args_t *args);
146 static int _generate_resv_id(void);
147 static void _generate_resv_name(resv_desc_msg_t *resv_ptr);
148 static int _get_core_resrcs(slurmctld_resv_t *resv_ptr);
149 static uint32_t _get_job_duration(job_record_t *job_ptr, bool reboot);
150 static bool _is_account_valid(char *account);
151 static bool _is_resv_used(slurmctld_resv_t *resv_ptr);
152 static bool _job_overlap(time_t start_time, uint64_t flags,
153 bitstr_t *node_bitmap, char *resv_name);
154 static int _job_resv_check(void *x, void *arg);
155 static List _list_dup(List license_list);
156 static Buf _open_resv_state_file(char **state_file);
157 static void _pack_resv(slurmctld_resv_t *resv_ptr, Buf buffer,
158 bool internal, uint16_t protocol_version);
159 static bitstr_t *_pick_idle_nodes(bitstr_t *avail_nodes,
160 resv_desc_msg_t *resv_desc_ptr,
161 bitstr_t **core_bitmap);
162 static bitstr_t *_pick_idle_xand_nodes(bitstr_t *avail_bitmap,
163 resv_desc_msg_t *resv_desc_ptr,
164 bitstr_t **core_bitmap,
165 List feature_list);
166 static bitstr_t *_pick_idle_node_cnt(bitstr_t *avail_bitmap,
167 resv_desc_msg_t *resv_desc_ptr,
168 uint32_t node_cnt,
169 bitstr_t **core_bitmap);
170 static int _post_resv_create(slurmctld_resv_t *resv_ptr);
171 static int _post_resv_delete(slurmctld_resv_t *resv_ptr);
172 static int _post_resv_update(slurmctld_resv_t *resv_ptr,
173 slurmctld_resv_t *old_resv_ptr);
174 static int _resize_resv(slurmctld_resv_t *resv_ptr, uint32_t node_cnt);
175 static void _restore_resv(slurmctld_resv_t *dest_resv,
176 slurmctld_resv_t *src_resv);
177 static bool _resv_overlap(resv_desc_msg_t *resv_desc_ptr,
178 bitstr_t *node_bitmap,
179 slurmctld_resv_t *this_resv_ptr);
180 static bool _resv_time_overlap(resv_desc_msg_t *resv_desc_ptr,
181 slurmctld_resv_t *resv_ptr);
182 static void _run_script(char *script, slurmctld_resv_t *resv_ptr);
183 static int _select_nodes(resv_desc_msg_t *resv_desc_ptr,
184 part_record_t **part_ptr, bitstr_t **resv_bitmap,
185 bitstr_t **core_bitmap);
186 static int _set_assoc_list(slurmctld_resv_t *resv_ptr);
187 static void _set_core_resrcs(slurmctld_resv_t *resv_ptr);
188 static void _set_tres_cnt(slurmctld_resv_t *resv_ptr,
189 slurmctld_resv_t *old_resv_ptr);
190 static void _set_nodes_flags(slurmctld_resv_t *resv_ptr, time_t now,
191 uint32_t flags, bool reset_all);
192 static int _update_account_list(slurmctld_resv_t *resv_ptr,
193 char *accounts);
194 static int _update_uid_list(slurmctld_resv_t *resv_ptr, char *users);
195 static void _validate_all_reservations(void);
196 static int _valid_job_access_resv(job_record_t *job_ptr,
197 slurmctld_resv_t *resv_ptr);
198 static bool _validate_one_reservation(slurmctld_resv_t *resv_ptr);
199 static void _validate_node_choice(slurmctld_resv_t *resv_ptr);
200
_set_boot_time(slurmctld_resv_t * resv_ptr)201 static void _set_boot_time(slurmctld_resv_t *resv_ptr)
202 {
203 resv_ptr->boot_time = 0;
204 if (!resv_ptr->node_bitmap)
205 return;
206
207 if (node_features_g_overlap(resv_ptr->node_bitmap))
208 resv_ptr->boot_time = node_features_g_boot_time();
209 }
210
211 /* Advance res_time by the specified day count,
212 * account for daylight savings time */
_advance_time(time_t * res_time,int day_cnt)213 static void _advance_time(time_t *res_time, int day_cnt)
214 {
215 time_t save_time = *res_time;
216 struct tm time_tm;
217
218 localtime_r(res_time, &time_tm);
219 time_tm.tm_mday += day_cnt;
220 *res_time = slurm_mktime(&time_tm);
221 if (*res_time == (time_t)(-1)) {
222 error("Could not compute reservation time %lu",
223 (long unsigned int) save_time);
224 *res_time = save_time + (24 * 60 * 60);
225 }
226 }
227
_create_cluster_core_bitmap(bitstr_t ** core_bitmap)228 static void _create_cluster_core_bitmap(bitstr_t **core_bitmap)
229 {
230 if (*core_bitmap)
231 return;
232
233 *core_bitmap = bit_alloc(cr_get_coremap_offset(node_record_count));
234 }
235
_list_dup(List license_list)236 static List _list_dup(List license_list)
237 {
238 ListIterator iter;
239 licenses_t *license_src, *license_dest;
240 List lic_list = (List) NULL;
241
242 if (!license_list)
243 return lic_list;
244
245 lic_list = list_create(license_free_rec);
246 iter = list_iterator_create(license_list);
247 while ((license_src = list_next(iter))) {
248 license_dest = xmalloc(sizeof(licenses_t));
249 license_dest->name = xstrdup(license_src->name);
250 license_dest->used = license_src->used;
251 list_push(lic_list, license_dest);
252 }
253 list_iterator_destroy(iter);
254 return lic_list;
255 }
256
_copy_resv(slurmctld_resv_t * resv_orig_ptr)257 static slurmctld_resv_t *_copy_resv(slurmctld_resv_t *resv_orig_ptr)
258 {
259 slurmctld_resv_t *resv_copy_ptr;
260 int i;
261
262 xassert(resv_orig_ptr->magic == RESV_MAGIC);
263 resv_copy_ptr = xmalloc(sizeof(slurmctld_resv_t));
264 resv_copy_ptr->accounts = xstrdup(resv_orig_ptr->accounts);
265 resv_copy_ptr->boot_time = resv_orig_ptr->boot_time;
266 resv_copy_ptr->burst_buffer = xstrdup(resv_orig_ptr->burst_buffer);
267 resv_copy_ptr->account_cnt = resv_orig_ptr->account_cnt;
268 resv_copy_ptr->account_list = xcalloc(resv_orig_ptr->account_cnt,
269 sizeof(char *));
270 resv_copy_ptr->account_not = resv_orig_ptr->account_not;
271 for (i = 0; i < resv_copy_ptr->account_cnt; i++) {
272 resv_copy_ptr->account_list[i] =
273 xstrdup(resv_orig_ptr->account_list[i]);
274 }
275 resv_copy_ptr->assoc_list = xstrdup(resv_orig_ptr->assoc_list);
276 if (resv_orig_ptr->core_bitmap) {
277 resv_copy_ptr->core_bitmap = bit_copy(resv_orig_ptr->
278 core_bitmap);
279 }
280 resv_copy_ptr->core_cnt = resv_orig_ptr->core_cnt;
281 if (resv_orig_ptr->core_resrcs) {
282 resv_copy_ptr->core_resrcs = copy_job_resources(resv_orig_ptr->
283 core_resrcs);
284 }
285 resv_copy_ptr->duration = resv_orig_ptr->duration;
286 resv_copy_ptr->end_time = resv_orig_ptr->end_time;
287 resv_copy_ptr->features = xstrdup(resv_orig_ptr->features);
288 resv_copy_ptr->flags = resv_orig_ptr->flags;
289 resv_copy_ptr->full_nodes = resv_orig_ptr->full_nodes;
290 resv_copy_ptr->job_pend_cnt = resv_orig_ptr->job_pend_cnt;
291 resv_copy_ptr->job_run_cnt = resv_orig_ptr->job_run_cnt;
292 resv_copy_ptr->licenses = xstrdup(resv_orig_ptr->licenses);
293 resv_copy_ptr->license_list = _list_dup(resv_orig_ptr->
294 license_list);
295 resv_copy_ptr->magic = resv_orig_ptr->magic;
296 resv_copy_ptr->flags_set_node = resv_orig_ptr->flags_set_node;
297 resv_copy_ptr->name = xstrdup(resv_orig_ptr->name);
298 if (resv_orig_ptr->node_bitmap) {
299 resv_copy_ptr->node_bitmap =
300 bit_copy(resv_orig_ptr->node_bitmap);
301 }
302 resv_copy_ptr->node_cnt = resv_orig_ptr->node_cnt;
303 resv_copy_ptr->node_list = xstrdup(resv_orig_ptr->node_list);
304 resv_copy_ptr->partition = xstrdup(resv_orig_ptr->partition);
305 resv_copy_ptr->part_ptr = resv_orig_ptr->part_ptr;
306 resv_copy_ptr->resv_id = resv_orig_ptr->resv_id;
307 resv_copy_ptr->resv_watts = resv_orig_ptr->resv_watts;
308 resv_copy_ptr->start_time = resv_orig_ptr->start_time;
309 resv_copy_ptr->start_time_first = resv_orig_ptr->start_time_first;
310 resv_copy_ptr->start_time_prev = resv_orig_ptr->start_time_prev;
311 resv_copy_ptr->tres_str = xstrdup(resv_orig_ptr->tres_str);
312 resv_copy_ptr->tres_fmt_str = xstrdup(resv_orig_ptr->tres_fmt_str);
313 resv_copy_ptr->users = xstrdup(resv_orig_ptr->users);
314 resv_copy_ptr->user_cnt = resv_orig_ptr->user_cnt;
315 resv_copy_ptr->user_list = xcalloc(resv_orig_ptr->user_cnt,
316 sizeof(uid_t));
317 resv_copy_ptr->user_not = resv_orig_ptr->user_not;
318 for (i = 0; i < resv_copy_ptr->user_cnt; i++)
319 resv_copy_ptr->user_list[i] = resv_orig_ptr->user_list[i];
320
321 return resv_copy_ptr;
322 }
323
324 /* Move the contents of src_resv into dest_resv.
325 * NOTE: This is a destructive function with respect to the contents of
326 * src_resv. The data structure src_resv is suitable only for destruction
327 * after this function is called */
_restore_resv(slurmctld_resv_t * dest_resv,slurmctld_resv_t * src_resv)328 static void _restore_resv(slurmctld_resv_t *dest_resv,
329 slurmctld_resv_t *src_resv)
330 {
331 int i;
332
333 xfree(dest_resv->accounts);
334 dest_resv->accounts = src_resv->accounts;
335 src_resv->accounts = NULL;
336
337 for (i = 0; i < dest_resv->account_cnt; i++)
338 xfree(dest_resv->account_list[i]);
339 xfree(dest_resv->account_list);
340 dest_resv->account_cnt = src_resv->account_cnt;
341 src_resv->account_cnt = 0;
342 dest_resv->account_list = src_resv->account_list;
343 src_resv->account_list = NULL;
344
345 dest_resv->account_not = src_resv->account_not;
346
347 xfree(dest_resv->assoc_list);
348 dest_resv->assoc_list = src_resv->assoc_list;
349 src_resv->assoc_list = NULL;
350
351 dest_resv->boot_time = src_resv->boot_time;
352
353 xfree(dest_resv->burst_buffer);
354 dest_resv->burst_buffer = src_resv->burst_buffer;
355 src_resv->burst_buffer = NULL;
356
357 FREE_NULL_BITMAP(dest_resv->core_bitmap);
358 dest_resv->core_bitmap = src_resv->core_bitmap;
359 src_resv->core_bitmap = NULL;
360
361 dest_resv->core_cnt = src_resv->core_cnt;
362
363 free_job_resources(&dest_resv->core_resrcs);
364 dest_resv->core_resrcs = src_resv->core_resrcs;
365 src_resv->core_resrcs = NULL;
366
367 dest_resv->duration = src_resv->duration;
368 dest_resv->end_time = src_resv->end_time;
369
370 xfree(dest_resv->features);
371 dest_resv->features = src_resv->features;
372 src_resv->features = NULL;
373
374 dest_resv->flags = src_resv->flags;
375 dest_resv->full_nodes = src_resv->full_nodes;
376 dest_resv->job_pend_cnt = src_resv->job_pend_cnt;
377 dest_resv->job_run_cnt = src_resv->job_run_cnt;
378
379 xfree(dest_resv->licenses);
380 dest_resv->licenses = src_resv->licenses;
381 src_resv->licenses = NULL;
382
383 FREE_NULL_LIST(dest_resv->license_list);
384 dest_resv->license_list = src_resv->license_list;
385 src_resv->license_list = NULL;
386
387 dest_resv->magic = src_resv->magic;
388 dest_resv->flags_set_node = src_resv->flags_set_node;
389
390 xfree(dest_resv->name);
391 dest_resv->name = src_resv->name;
392 src_resv->name = NULL;
393
394 FREE_NULL_BITMAP(dest_resv->node_bitmap);
395 dest_resv->node_bitmap = src_resv->node_bitmap;
396 src_resv->node_bitmap = NULL;
397
398 dest_resv->node_cnt = src_resv->node_cnt;
399
400 xfree(dest_resv->node_list);
401 dest_resv->node_list = src_resv->node_list;
402 src_resv->node_list = NULL;
403
404 xfree(dest_resv->partition);
405 dest_resv->partition = src_resv->partition;
406 src_resv->partition = NULL;
407
408 dest_resv->part_ptr = src_resv->part_ptr;
409 dest_resv->resv_id = src_resv->resv_id;
410 dest_resv->resv_watts = src_resv->resv_watts;
411 dest_resv->start_time = src_resv->start_time;
412 dest_resv->start_time_first = src_resv->start_time_first;
413 dest_resv->start_time_prev = src_resv->start_time_prev;
414
415 xfree(dest_resv->tres_str);
416 dest_resv->tres_str = src_resv->tres_str;
417 src_resv->tres_str = NULL;
418
419 xfree(dest_resv->tres_fmt_str);
420 dest_resv->tres_fmt_str = src_resv->tres_fmt_str;
421 src_resv->tres_fmt_str = NULL;
422
423 xfree(dest_resv->users);
424 dest_resv->users = src_resv->users;
425 src_resv->users = NULL;
426
427 dest_resv->user_cnt = src_resv->user_cnt;
428 xfree(dest_resv->user_list);
429 dest_resv->user_list = src_resv->user_list;
430 src_resv->user_list = NULL;
431
432 if (dest_resv->flags & RESERVE_FLAG_PROM)
433 list_append(prom_resv_list, dest_resv);
434 }
435
_del_resv_rec(void * x)436 static void _del_resv_rec(void *x)
437 {
438 int i;
439 slurmctld_resv_t *resv_ptr = (slurmctld_resv_t *) x;
440
441 if (resv_ptr) {
442 /*
443 * If shutting down prom_resv_list is already freed, meaning we
444 * don't need to remove anything from it.
445 */
446 if (prom_resv_list &&
447 (resv_ptr->flags & RESERVE_FLAG_PROM))
448 (void)list_remove_first(
449 prom_resv_list, _find_resv_ptr, resv_ptr);
450
451 xassert(resv_ptr->magic == RESV_MAGIC);
452 resv_ptr->magic = 0;
453 xfree(resv_ptr->accounts);
454 for (i = 0; i < resv_ptr->account_cnt; i++)
455 xfree(resv_ptr->account_list[i]);
456 xfree(resv_ptr->account_list);
457 xfree(resv_ptr->assoc_list);
458 xfree(resv_ptr->burst_buffer);
459 FREE_NULL_BITMAP(resv_ptr->core_bitmap);
460 free_job_resources(&resv_ptr->core_resrcs);
461 xfree(resv_ptr->features);
462 FREE_NULL_LIST(resv_ptr->license_list);
463 xfree(resv_ptr->licenses);
464 xfree(resv_ptr->name);
465 FREE_NULL_BITMAP(resv_ptr->node_bitmap);
466 xfree(resv_ptr->node_list);
467 xfree(resv_ptr->partition);
468 xfree(resv_ptr->tres_str);
469 xfree(resv_ptr->tres_fmt_str);
470 xfree(resv_ptr->users);
471 xfree(resv_ptr->user_list);
472 xfree(resv_ptr);
473 }
474 }
475
_create_resv_lists(bool flush)476 static void _create_resv_lists(bool flush)
477 {
478 if (flush && resv_list) {
479 list_flush(prom_resv_list);
480 list_flush(resv_list);
481 return;
482 }
483
484 if (!resv_list)
485 resv_list = list_create(_del_resv_rec);
486 if (!prom_resv_list)
487 prom_resv_list = list_create(NULL);
488 }
489
_add_resv_to_lists(slurmctld_resv_t * resv_ptr)490 static void _add_resv_to_lists(slurmctld_resv_t *resv_ptr)
491 {
492 xassert(resv_list);
493 xassert(prom_resv_list);
494
495 list_append(resv_list, resv_ptr);
496 if (resv_ptr->flags & RESERVE_FLAG_PROM)
497 list_append(prom_resv_list, resv_ptr);
498 }
499
_queue_prom_resv(void * x,void * key)500 static int _queue_prom_resv(void *x, void *key)
501 {
502 slurmctld_resv_t *resv_ptr = (slurmctld_resv_t *) x;
503 job_queue_req_t *job_queue_req = (job_queue_req_t *) key;
504
505 xassert(resv_ptr->magic == RESV_MAGIC);
506
507 if (!(resv_ptr->flags & RESERVE_FLAG_PROM) ||
508 (_valid_job_access_resv(job_queue_req->job_ptr, resv_ptr) !=
509 SLURM_SUCCESS))
510 return 0;
511
512 job_queue_req->resv_ptr = resv_ptr;
513 job_queue_append_internal(job_queue_req);
514
515 return 0;
516 }
517
_find_resv_id(void * x,void * key)518 static int _find_resv_id(void *x, void *key)
519 {
520 slurmctld_resv_t *resv_ptr = (slurmctld_resv_t *) x;
521 uint32_t *resv_id = (uint32_t *) key;
522
523 xassert(resv_ptr->magic == RESV_MAGIC);
524
525 if (resv_ptr->resv_id != *resv_id)
526 return 0;
527 else
528 return 1; /* match */
529 }
530
_find_resv_ptr(void * x,void * key)531 static int _find_resv_ptr(void *x, void *key)
532 {
533 slurmctld_resv_t *resv_ptr = (slurmctld_resv_t *) x;
534 slurmctld_resv_t *resv_ptr_key = (slurmctld_resv_t *) key;
535
536 xassert(resv_ptr->magic == RESV_MAGIC);
537
538 if (resv_ptr != resv_ptr_key)
539 return 0;
540 else
541 return 1; /* match */
542 }
543
_find_resv_name(void * x,void * key)544 static int _find_resv_name(void *x, void *key)
545 {
546 slurmctld_resv_t *resv_ptr = (slurmctld_resv_t *) x;
547
548 xassert(resv_ptr->magic == RESV_MAGIC);
549
550 if (xstrcmp(resv_ptr->name, (char *) key))
551 return 0;
552 else
553 return 1; /* match */
554 }
555
_dump_resv_req(resv_desc_msg_t * resv_ptr,char * mode)556 static void _dump_resv_req(resv_desc_msg_t *resv_ptr, char *mode)
557 {
558
559 char start_str[32] = "-1", end_str[32] = "-1", *flag_str = NULL;
560 char watts_str[32] = "n/a";
561 char *node_cnt_str = NULL, *core_cnt_str = NULL;
562 int duration, i;
563
564 if (!(slurmctld_conf.debug_flags & DEBUG_FLAG_RESERVATION))
565 return;
566
567 if (resv_ptr->start_time != (time_t) NO_VAL) {
568 slurm_make_time_str(&resv_ptr->start_time,
569 start_str, sizeof(start_str));
570 }
571 if (resv_ptr->end_time != (time_t) NO_VAL) {
572 slurm_make_time_str(&resv_ptr->end_time,
573 end_str, sizeof(end_str));
574 }
575 if (resv_ptr->resv_watts != NO_VAL) {
576 snprintf(watts_str, sizeof(watts_str), "%u",
577 resv_ptr->resv_watts);
578 }
579 if (resv_ptr->flags != NO_VAL64) {
580 reserve_info_t resv_info = {
581 .flags = resv_ptr->flags,
582 .purge_comp_time = resv_ptr->purge_comp_time
583 };
584 flag_str = reservation_flags_string(&resv_info);
585 }
586 if (resv_ptr->duration == NO_VAL)
587 duration = -1;
588 else
589 duration = resv_ptr->duration;
590
591 if (resv_ptr->node_cnt) {
592 for (i = 0; resv_ptr->node_cnt[i]; i++) {
593 if (node_cnt_str) {
594 xstrfmtcat(node_cnt_str, ",%u",
595 resv_ptr->node_cnt[i]);
596 } else {
597 xstrfmtcat(node_cnt_str, "%u",
598 resv_ptr->node_cnt[i]);
599 }
600 }
601 }
602
603 if (resv_ptr->core_cnt) {
604 for (i = 0; resv_ptr->core_cnt[i]; i++) {
605 if (core_cnt_str) {
606 xstrfmtcat(core_cnt_str, ",%u",
607 resv_ptr->core_cnt[i]);
608 } else {
609 xstrfmtcat(core_cnt_str, "%u",
610 resv_ptr->core_cnt[i]);
611 }
612 }
613 }
614
615 info("%s: Name=%s StartTime=%s EndTime=%s Duration=%d "
616 "Flags=%s NodeCnt=%s CoreCnt=%s NodeList=%s Features=%s "
617 "PartitionName=%s Users=%s Accounts=%s Licenses=%s BurstBuffer=%s "
618 "TRES=%s Watts=%s",
619 mode, resv_ptr->name, start_str, end_str, duration,
620 flag_str, node_cnt_str, core_cnt_str, resv_ptr->node_list,
621 resv_ptr->features, resv_ptr->partition,
622 resv_ptr->users, resv_ptr->accounts, resv_ptr->licenses,
623 resv_ptr->burst_buffer, resv_ptr->tres_str, watts_str);
624
625 xfree(flag_str);
626 xfree(node_cnt_str);
627 xfree(core_cnt_str);
628 }
629
_generate_resv_id(void)630 static int _generate_resv_id(void)
631 {
632 int i;
633
634 for (i = 0; i < MAX_RESV_COUNT; i++) {
635 if (top_suffix >= MAX_RESV_COUNT)
636 top_suffix = 1; /* wrap around */
637 else
638 top_suffix++;
639 if (!list_find_first(resv_list, _find_resv_id, &top_suffix))
640 return SLURM_SUCCESS;
641 }
642
643 error("%s: Too many reservations in the system, can't create any more.",
644 __func__);
645
646 return ESLURM_RESERVATION_INVALID;
647 }
648
_generate_resv_name(resv_desc_msg_t * resv_ptr)649 static void _generate_resv_name(resv_desc_msg_t *resv_ptr)
650 {
651 char *key, *name, *sep;
652 int len;
653
654 /* Generate name prefix, based upon the first account
655 * name if provided otherwise first user name */
656 if (resv_ptr->accounts && resv_ptr->accounts[0])
657 key = resv_ptr->accounts;
658 else if (resv_ptr->users && resv_ptr->users[0])
659 key = resv_ptr->users;
660 else
661 key = "resv";
662 if (key[0] == '-')
663 key++;
664 sep = strchr(key, ',');
665 if (sep)
666 len = sep - key;
667 else
668 len = strlen(key);
669
670 name = xstrdup_printf("%.*s_%d", len, key, top_suffix);
671
672 xfree(resv_ptr->name);
673 resv_ptr->name = name;
674 }
675
676 /* Validate an account name */
_is_account_valid(char * account)677 static bool _is_account_valid(char *account)
678 {
679 slurmdb_assoc_rec_t assoc_rec, *assoc_ptr;
680
681 if (!(accounting_enforce & ACCOUNTING_ENFORCE_ASSOCS))
682 return true; /* don't worry about account validity */
683
684 memset(&assoc_rec, 0, sizeof(slurmdb_assoc_rec_t));
685 assoc_rec.uid = NO_VAL;
686 assoc_rec.acct = account;
687
688 if (assoc_mgr_fill_in_assoc(acct_db_conn, &assoc_rec,
689 accounting_enforce, &assoc_ptr, false)) {
690 return false;
691 }
692 return true;
693 }
694
695 /* Since the returned assoc_list is full of pointers from the
696 * assoc_mgr_association_list assoc_mgr_lock_t READ_LOCK on
697 * associations must be set before calling this function and while
698 * handling it after a return.
699 */
_append_assoc_list(List assoc_list,slurmdb_assoc_rec_t * assoc)700 static int _append_assoc_list(List assoc_list, slurmdb_assoc_rec_t *assoc)
701 {
702 int rc = ESLURM_INVALID_ACCOUNT;
703 slurmdb_assoc_rec_t *assoc_ptr = NULL;
704 if (assoc_mgr_fill_in_assoc(
705 acct_db_conn, assoc,
706 accounting_enforce,
707 &assoc_ptr, true)) {
708 if (accounting_enforce & ACCOUNTING_ENFORCE_ASSOCS) {
709 error("No association for user %u and account %s",
710 assoc->uid, assoc->acct);
711 } else {
712 verbose("No association for user %u and account %s",
713 assoc->uid, assoc->acct);
714 rc = SLURM_SUCCESS;
715 }
716
717 }
718 if (assoc_ptr) {
719 list_append(assoc_list, assoc_ptr);
720 rc = SLURM_SUCCESS;
721 }
722
723 return rc;
724 }
725
726 /* Set a association list based upon accounts and users */
_set_assoc_list(slurmctld_resv_t * resv_ptr)727 static int _set_assoc_list(slurmctld_resv_t *resv_ptr)
728 {
729 int rc = SLURM_SUCCESS, i = 0, j = 0;
730 List assoc_list_allow = NULL, assoc_list_deny = NULL, assoc_list;
731 slurmdb_assoc_rec_t assoc, *assoc_ptr = NULL;
732 assoc_mgr_lock_t locks = { .assoc = READ_LOCK, .user = READ_LOCK };
733
734
735 /* no need to do this if we can't ;) */
736 if (!association_based_accounting)
737 return rc;
738
739 assoc_list_allow = list_create(NULL);
740 assoc_list_deny = list_create(NULL);
741
742 memset(&assoc, 0, sizeof(slurmdb_assoc_rec_t));
743 xfree(resv_ptr->assoc_list);
744
745 assoc_mgr_lock(&locks);
746 if (resv_ptr->account_cnt && resv_ptr->user_cnt) {
747 if (!resv_ptr->account_not && !resv_ptr->user_not) {
748 /* Add every association that matches both account
749 * and user */
750 for (i=0; i < resv_ptr->user_cnt; i++) {
751 for (j=0; j < resv_ptr->account_cnt; j++) {
752 memset(&assoc, 0,
753 sizeof(slurmdb_assoc_rec_t));
754 assoc.acct = resv_ptr->account_list[j];
755 assoc.uid = resv_ptr->user_list[i];
756 rc = _append_assoc_list(
757 assoc_list_allow, &assoc);
758 if (rc != SLURM_SUCCESS)
759 goto end_it;
760 }
761 }
762 } else {
763 if (resv_ptr->user_not)
764 assoc_list = assoc_list_deny;
765 else
766 assoc_list = assoc_list_allow;
767 for (i=0; i < resv_ptr->user_cnt; i++) {
768 memset(&assoc, 0,
769 sizeof(slurmdb_assoc_rec_t));
770 assoc.uid = resv_ptr->user_list[i];
771 rc = assoc_mgr_get_user_assocs(
772 acct_db_conn, &assoc,
773 accounting_enforce,
774 assoc_list);
775 if (rc != SLURM_SUCCESS) {
776 error("No associations for UID %u",
777 assoc.uid);
778 rc = ESLURM_INVALID_ACCOUNT;
779 goto end_it;
780 }
781 }
782 if (resv_ptr->account_not)
783 assoc_list = assoc_list_deny;
784 else
785 assoc_list = assoc_list_allow;
786 for (j=0; j < resv_ptr->account_cnt; j++) {
787 memset(&assoc, 0,
788 sizeof(slurmdb_assoc_rec_t));
789 assoc.acct = resv_ptr->account_list[j];
790 assoc.uid = NO_VAL;
791 rc = _append_assoc_list(assoc_list, &assoc);
792 if (rc != SLURM_SUCCESS)
793 goto end_it;
794 }
795 }
796 } else if (resv_ptr->user_cnt) {
797 if (resv_ptr->user_not)
798 assoc_list = assoc_list_deny;
799 else
800 assoc_list = assoc_list_allow;
801 for (i=0; i < resv_ptr->user_cnt; i++) {
802 memset(&assoc, 0, sizeof(slurmdb_assoc_rec_t));
803 assoc.uid = resv_ptr->user_list[i];
804 rc = assoc_mgr_get_user_assocs(
805 acct_db_conn, &assoc,
806 accounting_enforce, assoc_list);
807 if (rc != SLURM_SUCCESS) {
808 error("No associations for UID %u",
809 assoc.uid);
810 rc = ESLURM_INVALID_ACCOUNT;
811 goto end_it;
812 }
813 }
814 } else if (resv_ptr->account_cnt) {
815 if (resv_ptr->account_not)
816 assoc_list = assoc_list_deny;
817 else
818 assoc_list = assoc_list_allow;
819 for (i=0; i < resv_ptr->account_cnt; i++) {
820 memset(&assoc, 0, sizeof(slurmdb_assoc_rec_t));
821 assoc.acct = resv_ptr->account_list[i];
822 assoc.uid = NO_VAL;
823 if ((rc = _append_assoc_list(assoc_list, &assoc))
824 != SLURM_SUCCESS) {
825 goto end_it;
826 }
827 }
828 } else if (accounting_enforce & ACCOUNTING_ENFORCE_ASSOCS) {
829 error("We need at least 1 user or 1 account to "
830 "create a reservtion.");
831 rc = SLURM_ERROR;
832 }
833
834 xfree(resv_ptr->assoc_list); /* clear for modify */
835 if (list_count(assoc_list_allow)) {
836 ListIterator itr = list_iterator_create(assoc_list_allow);
837 while ((assoc_ptr = list_next(itr))) {
838 if (resv_ptr->assoc_list) {
839 xstrfmtcat(resv_ptr->assoc_list, "%u,",
840 assoc_ptr->id);
841 } else {
842 xstrfmtcat(resv_ptr->assoc_list, ",%u,",
843 assoc_ptr->id);
844 }
845 }
846 list_iterator_destroy(itr);
847 }
848 if (list_count(assoc_list_deny)) {
849 ListIterator itr = list_iterator_create(assoc_list_deny);
850 while ((assoc_ptr = list_next(itr))) {
851 if (resv_ptr->assoc_list) {
852 xstrfmtcat(resv_ptr->assoc_list, "-%u,",
853 assoc_ptr->id);
854 } else {
855 xstrfmtcat(resv_ptr->assoc_list, ",-%u,",
856 assoc_ptr->id);
857 }
858 }
859 list_iterator_destroy(itr);
860 }
861 debug("assoc_list:%s", resv_ptr->assoc_list);
862
863 end_it:
864 FREE_NULL_LIST(assoc_list_allow);
865 FREE_NULL_LIST(assoc_list_deny);
866 assoc_mgr_unlock(&locks);
867
868 return rc;
869 }
870
871 /* Post reservation create */
_post_resv_create(slurmctld_resv_t * resv_ptr)872 static int _post_resv_create(slurmctld_resv_t *resv_ptr)
873 {
874 int rc = SLURM_SUCCESS;
875 slurmdb_reservation_rec_t resv;
876 char temp_bit[BUF_SIZE];
877
878 _set_boot_time(resv_ptr);
879
880 if (resv_ptr->flags & RESERVE_FLAG_TIME_FLOAT)
881 return rc;
882
883 memset(&resv, 0, sizeof(slurmdb_reservation_rec_t));
884 resv.assocs = resv_ptr->assoc_list;
885 resv.cluster = slurmctld_conf.cluster_name;
886 resv.tres_str = resv_ptr->tres_str;
887
888 resv.flags = resv_ptr->flags;
889 resv.id = resv_ptr->resv_id;
890 resv.name = resv_ptr->name;
891 resv.nodes = resv_ptr->node_list;
892 if (resv_ptr->node_bitmap) {
893 resv.node_inx = bit_fmt(temp_bit, sizeof(temp_bit),
894 resv_ptr->node_bitmap);
895 }
896 resv.time_end = resv_ptr->end_time;
897 resv.time_start = resv_ptr->start_time;
898 resv.tres_str = resv_ptr->tres_str;
899
900 rc = acct_storage_g_add_reservation(acct_db_conn, &resv);
901
902 return rc;
903 }
904
905 /* Note that a reservation has been deleted */
_post_resv_delete(slurmctld_resv_t * resv_ptr)906 static int _post_resv_delete(slurmctld_resv_t *resv_ptr)
907 {
908 int rc = SLURM_SUCCESS;
909 slurmdb_reservation_rec_t resv;
910 time_t now = time(NULL);
911
912 if (resv_ptr->flags & RESERVE_FLAG_TIME_FLOAT)
913 return rc;
914
915 memset(&resv, 0, sizeof(slurmdb_reservation_rec_t));
916 resv.cluster = slurmctld_conf.cluster_name;
917 resv.id = resv_ptr->resv_id;
918 resv.name = resv_ptr->name;
919 resv.time_end = now;
920 resv.time_start = resv_ptr->start_time;
921 /* This is just a time stamp here to delete if the reservation
922 * hasn't started yet so we don't get trash records in the
923 * database if said database isn't up right now */
924 resv.time_start_prev = now;
925 resv.tres_str = resv_ptr->tres_str;
926 rc = acct_storage_g_remove_reservation(acct_db_conn, &resv);
927
928 return rc;
929 }
930
931 /* Note that a reservation has been updated */
_post_resv_update(slurmctld_resv_t * resv_ptr,slurmctld_resv_t * old_resv_ptr)932 static int _post_resv_update(slurmctld_resv_t *resv_ptr,
933 slurmctld_resv_t *old_resv_ptr)
934 {
935 int rc = SLURM_SUCCESS;
936 bool change = false;
937 slurmdb_reservation_rec_t resv;
938 char temp_bit[BUF_SIZE];
939 time_t now = time(NULL);
940
941 xassert(old_resv_ptr);
942
943 _set_boot_time(resv_ptr);
944
945 if (resv_ptr->flags & RESERVE_FLAG_TIME_FLOAT)
946 return rc;
947
948 memset(&resv, 0, sizeof(slurmdb_reservation_rec_t));
949 resv.cluster = slurmctld_conf.cluster_name;
950 resv.id = resv_ptr->resv_id;
951 resv.time_end = resv_ptr->end_time;
952 resv.assocs = resv_ptr->assoc_list;
953 resv.tres_str = resv_ptr->tres_str;
954 resv.flags = resv_ptr->flags;
955 resv.nodes = resv_ptr->node_list;
956
957 if (xstrcmp(old_resv_ptr->assoc_list, resv_ptr->assoc_list) ||
958 xstrcmp(old_resv_ptr->tres_str, resv_ptr->tres_str) ||
959 (old_resv_ptr->flags != resv_ptr->flags) ||
960 xstrcmp(old_resv_ptr->node_list, resv_ptr->node_list))
961 change = true;
962
963 /* Here if the reservation has started already we need
964 * to mark a new start time for it if certain
965 * variables are needed in accounting. Right now if
966 * the assocs, nodes, flags or cpu count changes we need a
967 * new start time of now. */
968 if ((resv_ptr->start_time < now) && change) {
969 resv_ptr->start_time_prev = resv_ptr->start_time;
970 resv_ptr->start_time = now;
971 }
972
973 /* now set the (maybe new) start_times */
974 resv.time_start = resv_ptr->start_time;
975 resv.time_start_prev = resv_ptr->start_time_prev;
976
977 if (resv.nodes && resv_ptr->node_bitmap) {
978 resv.node_inx = bit_fmt(temp_bit, sizeof(temp_bit),
979 resv_ptr->node_bitmap);
980 }
981
982 rc = acct_storage_g_modify_reservation(acct_db_conn, &resv);
983
984 return rc;
985 }
986
987 /*
988 * Validate a comma delimited list of account names and build an array of
989 * them
990 * IN account - a list of account names
991 * OUT account_cnt - number of accounts in the list
992 * OUT account_list - list of the account names,
993 * CALLER MUST XFREE this plus each individual record
994 * OUT account_not - true of account_list is that of accounts to be blocked
995 * from reservation access
996 * RETURN 0 on success
997 */
_build_account_list(char * accounts,int * account_cnt,char *** account_list,bool * account_not)998 static int _build_account_list(char *accounts, int *account_cnt,
999 char ***account_list, bool *account_not)
1000 {
1001 char *last = NULL, *tmp, *tok;
1002 int ac_cnt = 0, i;
1003 char **ac_list;
1004
1005 *account_cnt = 0;
1006 *account_list = (char **) NULL;
1007 *account_not = false;
1008
1009 if (!accounts)
1010 return ESLURM_INVALID_ACCOUNT;
1011
1012 i = strlen(accounts);
1013 ac_list = xcalloc((i + 2), sizeof(char *));
1014 tmp = xstrdup(accounts);
1015 tok = strtok_r(tmp, ",", &last);
1016 while (tok) {
1017 if (tok[0] == '-') {
1018 if (ac_cnt == 0) {
1019 *account_not = true;
1020 } else if (*account_not != true) {
1021 info("Reservation request has some "
1022 "not/accounts");
1023 goto inval;
1024 }
1025 tok++;
1026 } else if (*account_not != false) {
1027 info("Reservation request has some not/accounts");
1028 goto inval;
1029 }
1030 if (!_is_account_valid(tok)) {
1031 info("Reservation request has invalid account %s",
1032 tok);
1033 goto inval;
1034 }
1035 ac_list[ac_cnt++] = xstrdup(tok);
1036 tok = strtok_r(NULL, ",", &last);
1037 }
1038 *account_cnt = ac_cnt;
1039 *account_list = ac_list;
1040 xfree(tmp);
1041 return SLURM_SUCCESS;
1042
1043 inval: for (i=0; i<ac_cnt; i++)
1044 xfree(ac_list[i]);
1045 xfree(ac_list);
1046 xfree(tmp);
1047 return ESLURM_INVALID_ACCOUNT;
1048 }
1049
1050 /*
1051 * Update a account list for an existing reservation based upon an
1052 * update comma delimited specification of accounts to add (+name),
1053 * remove (-name), or set value of
1054 * IN/OUT resv_ptr - pointer to reservation structure being updated
1055 * IN accounts - a list of account names, to set, add, or remove
1056 * RETURN 0 on success
1057 */
_update_account_list(slurmctld_resv_t * resv_ptr,char * accounts)1058 static int _update_account_list(slurmctld_resv_t *resv_ptr,
1059 char *accounts)
1060 {
1061 char *last = NULL, *ac_cpy, *tok;
1062 int ac_cnt = 0, i, j, k;
1063 int *ac_type, minus_account = 0, plus_account = 0;
1064 char **ac_list;
1065 bool found_it;
1066 bool account_not = false;
1067
1068 if (!accounts)
1069 return ESLURM_INVALID_ACCOUNT;
1070
1071 i = strlen(accounts);
1072 ac_list = xcalloc((i + 2), sizeof(char *));
1073 ac_type = xcalloc((i + 2), sizeof(int));
1074 ac_cpy = xstrdup(accounts);
1075 tok = strtok_r(ac_cpy, ",", &last);
1076 while (tok) {
1077 if (tok[0] == '-') {
1078 ac_type[ac_cnt] = 1; /* minus */
1079 minus_account = 1;
1080 tok++;
1081 } else if (tok[0] == '+') {
1082 ac_type[ac_cnt] = 2; /* plus */
1083 plus_account = 1;
1084 tok++;
1085 } else if (tok[0] == '\0') {
1086 continue;
1087 } else if (plus_account || minus_account) {
1088 info("Reservation account expression invalid %s",
1089 accounts);
1090 goto inval;
1091 } else
1092 ac_type[ac_cnt] = 3; /* set */
1093 if (!_is_account_valid(tok)) {
1094 info("Reservation request has invalid account %s",
1095 tok);
1096 goto inval;
1097 }
1098 ac_list[ac_cnt++] = xstrdup(tok);
1099 tok = strtok_r(NULL, ",", &last);
1100 }
1101
1102 if ((plus_account == 0) && (minus_account == 0)) {
1103 /* Just a reset of account list */
1104 xfree(resv_ptr->accounts);
1105 if (accounts[0] != '\0')
1106 resv_ptr->accounts = xstrdup(accounts);
1107 xfree(resv_ptr->account_list);
1108 resv_ptr->account_list = ac_list;
1109 resv_ptr->account_cnt = ac_cnt;
1110 resv_ptr->account_not = account_not;
1111 xfree(ac_cpy);
1112 xfree(ac_type);
1113 return SLURM_SUCCESS;
1114 }
1115
1116 /* Modification of existing account list */
1117 if ((resv_ptr->account_cnt == 0) && minus_account)
1118 resv_ptr->account_not = true;
1119 if (resv_ptr->account_not) {
1120 /* change minus_account to plus_account (add to NOT list) and
1121 * change plus_account to minus_account (remove from NOT list) */
1122 for (i = 0; i < ac_cnt; i++) {
1123 if (ac_type[i] == 1)
1124 ac_type[i] = 2;
1125 else if (ac_type[i] == 2)
1126 ac_type[i] = 1;
1127 }
1128 if (minus_account && !plus_account) {
1129 minus_account = false;
1130 plus_account = true;
1131 } else if (!minus_account && plus_account) {
1132 minus_account = true;
1133 plus_account = false;
1134 }
1135 }
1136 if (minus_account) {
1137 if (resv_ptr->account_cnt == 0)
1138 goto inval;
1139 for (i=0; i<ac_cnt; i++) {
1140 if (ac_type[i] != 1) /* not minus */
1141 continue;
1142 found_it = false;
1143 for (j=0; j<resv_ptr->account_cnt; j++) {
1144 char *test_name = resv_ptr->account_list[j];
1145 if (test_name[0] == '-')
1146 test_name++;
1147 if (xstrcmp(test_name, ac_list[i]))
1148 continue;
1149 found_it = true;
1150 xfree(resv_ptr->account_list[j]);
1151 resv_ptr->account_cnt--;
1152 for (k=j; k<resv_ptr->account_cnt; k++) {
1153 resv_ptr->account_list[k] =
1154 resv_ptr->account_list[k+1];
1155 }
1156 break;
1157 }
1158 if (!found_it)
1159 goto inval;
1160 }
1161 xfree(resv_ptr->accounts);
1162 for (i=0; i<resv_ptr->account_cnt; i++) {
1163 if (i)
1164 xstrcat(resv_ptr->accounts, ",");
1165 if (resv_ptr->account_not)
1166 xstrcat(resv_ptr->accounts, "-");
1167 xstrcat(resv_ptr->accounts, resv_ptr->account_list[i]);
1168 }
1169 }
1170
1171 if (plus_account) {
1172 for (i=0; i<ac_cnt; i++) {
1173 if (ac_type[i] != 2) /* not plus */
1174 continue;
1175 found_it = false;
1176 for (j=0; j<resv_ptr->account_cnt; j++) {
1177 char *test_name = resv_ptr->account_list[j];
1178 if (test_name[0] == '-')
1179 test_name++;
1180 if (xstrcmp(test_name, ac_list[i]))
1181 continue;
1182 found_it = true;
1183 break;
1184 }
1185 if (found_it)
1186 continue; /* duplicate entry */
1187 xrealloc(resv_ptr->account_list,
1188 sizeof(char *) * (resv_ptr->account_cnt + 1));
1189 resv_ptr->account_list[resv_ptr->account_cnt++] =
1190 xstrdup(ac_list[i]);
1191 }
1192 xfree(resv_ptr->accounts);
1193 for (i=0; i<resv_ptr->account_cnt; i++) {
1194 if (i)
1195 xstrcat(resv_ptr->accounts, ",");
1196 if (resv_ptr->account_not)
1197 xstrcat(resv_ptr->accounts, "-");
1198 xstrcat(resv_ptr->accounts, resv_ptr->account_list[i]);
1199 }
1200 }
1201
1202 for (i=0; i<ac_cnt; i++)
1203 xfree(ac_list[i]);
1204 xfree(ac_list);
1205 xfree(ac_type);
1206 xfree(ac_cpy);
1207 return SLURM_SUCCESS;
1208
1209 inval: for (i=0; i<ac_cnt; i++)
1210 xfree(ac_list[i]);
1211 xfree(ac_list);
1212 xfree(ac_type);
1213 xfree(ac_cpy);
1214 return ESLURM_INVALID_ACCOUNT;
1215 }
1216
1217 /*
1218 * Validate a comma delimited list of user names and build an array of
1219 * their UIDs
1220 * IN users - a list of user names
1221 * OUT user_cnt - number of UIDs in the list
1222 * OUT user_list - list of the user's uid, CALLER MUST XFREE;
1223 * OUT user_not - true of user_list is that of users to be blocked
1224 * from reservation access
1225 * RETURN 0 on success
1226 */
_build_uid_list(char * users,int * user_cnt,uid_t ** user_list,bool * user_not)1227 static int _build_uid_list(char *users, int *user_cnt, uid_t **user_list,
1228 bool *user_not)
1229 {
1230 char *last = NULL, *tmp = NULL, *tok;
1231 int u_cnt = 0, i;
1232 uid_t *u_list, u_tmp;
1233
1234 *user_cnt = 0;
1235 *user_list = (uid_t *) NULL;
1236 *user_not = false;
1237
1238 if (!users)
1239 return ESLURM_USER_ID_MISSING;
1240
1241 i = strlen(users);
1242 u_list = xcalloc((i + 2), sizeof(uid_t));
1243 tmp = xstrdup(users);
1244 tok = strtok_r(tmp, ",", &last);
1245 while (tok) {
1246 if (tok[0] == '-') {
1247 if (u_cnt == 0) {
1248 *user_not = true;
1249 } else if (*user_not != true) {
1250 info("Reservation request has some not/users");
1251 goto inval;
1252 }
1253 tok++;
1254 } else if (*user_not != false) {
1255 info("Reservation request has some not/users");
1256 goto inval;
1257 }
1258 if (uid_from_string (tok, &u_tmp) < 0) {
1259 info("Reservation request has invalid user %s", tok);
1260 goto inval;
1261 }
1262 u_list[u_cnt++] = u_tmp;
1263 tok = strtok_r(NULL, ",", &last);
1264 }
1265 *user_cnt = u_cnt;
1266 *user_list = u_list;
1267 xfree(tmp);
1268 return SLURM_SUCCESS;
1269
1270 inval: xfree(tmp);
1271 xfree(u_list);
1272 return ESLURM_USER_ID_MISSING;
1273 }
1274
1275 /*
1276 * Update a user/uid list for an existing reservation based upon an
1277 * update comma delimited specification of users to add (+name),
1278 * remove (-name), or set value of
1279 * IN/OUT resv_ptr - pointer to reservation structure being updated
1280 * IN users - a list of user names, to set, add, or remove
1281 * RETURN 0 on success
1282 */
_update_uid_list(slurmctld_resv_t * resv_ptr,char * users)1283 static int _update_uid_list(slurmctld_resv_t *resv_ptr, char *users)
1284 {
1285 char *last = NULL, *u_cpy = NULL, *tmp = NULL, *tok;
1286 int u_cnt = 0, i, j, k;
1287 uid_t *u_list, u_tmp;
1288 int *u_type, minus_user = 0, plus_user = 0;
1289 char **u_name;
1290 bool found_it;
1291 bool user_not = false;
1292
1293 if (!users)
1294 return ESLURM_USER_ID_MISSING;
1295
1296 /* Parse the incoming user expression */
1297 i = strlen(users);
1298 u_list = xcalloc((i + 2), sizeof(uid_t));
1299 u_name = xcalloc((i + 2), sizeof(char *));
1300 u_type = xcalloc((i + 2), sizeof(int));
1301 u_cpy = xstrdup(users);
1302 tok = strtok_r(u_cpy, ",", &last);
1303 while (tok) {
1304 if (tok[0] == '-') {
1305 u_type[u_cnt] = 1; /* minus */
1306 minus_user = 1;
1307 tok++;
1308 } else if (tok[0] == '+') {
1309 u_type[u_cnt] = 2; /* plus */
1310 plus_user = 1;
1311 tok++;
1312 } else if (tok[0] == '\0') {
1313 continue;
1314 } else if (plus_user || minus_user) {
1315 info("Reservation user expression invalid %s", users);
1316 goto inval;
1317 } else
1318 u_type[u_cnt] = 3; /* set */
1319
1320 if (uid_from_string (tok, &u_tmp) < 0) {
1321 info("Reservation request has invalid user %s", tok);
1322 goto inval;
1323 }
1324
1325 u_name[u_cnt] = tok;
1326 u_list[u_cnt++] = u_tmp;
1327 tok = strtok_r(NULL, ",", &last);
1328 }
1329
1330 if ((plus_user == 0) && (minus_user == 0)) {
1331 /* Just a reset of user list */
1332 xfree(resv_ptr->users);
1333 xfree(resv_ptr->user_list);
1334 if (users[0] != '\0')
1335 resv_ptr->users = xstrdup(users);
1336 resv_ptr->user_cnt = u_cnt;
1337 resv_ptr->user_list = u_list;
1338 resv_ptr->user_not = user_not;
1339 xfree(u_cpy);
1340 xfree(u_name);
1341 xfree(u_type);
1342 return SLURM_SUCCESS;
1343 }
1344
1345 /* Modification of existing user list */
1346 if ((resv_ptr->user_cnt == 0) && minus_user)
1347 resv_ptr->user_not = true;
1348 if (resv_ptr->user_not) {
1349 /* change minus_user to plus_user (add to NOT list) and
1350 * change plus_user to minus_user (remove from NOT list) */
1351 for (i = 0; i < u_cnt; i++) {
1352 if (u_type[i] == 1)
1353 u_type[i] = 2;
1354 else if (u_type[i] == 2)
1355 u_type[i] = 1;
1356 }
1357 if (minus_user && !plus_user) {
1358 minus_user = false;
1359 plus_user = true;
1360 } else if (!minus_user && plus_user) {
1361 minus_user = true;
1362 plus_user = false;
1363 }
1364 }
1365 if (minus_user) {
1366 for (i=0; i<u_cnt; i++) {
1367 if (u_type[i] != 1) /* not minus */
1368 continue;
1369 found_it = false;
1370 for (j=0; j<resv_ptr->user_cnt; j++) {
1371 if (resv_ptr->user_list[j] != u_list[i])
1372 continue;
1373 found_it = true;
1374 resv_ptr->user_cnt--;
1375 for (k=j; k<resv_ptr->user_cnt; k++) {
1376 resv_ptr->user_list[k] =
1377 resv_ptr->user_list[k+1];
1378 }
1379 break;
1380 }
1381 if (!found_it)
1382 goto inval;
1383
1384 /* Now we need to remove from users string */
1385 k = strlen(u_name[i]);
1386 tmp = resv_ptr->users;
1387 while ((tok = strstr(tmp, u_name[i]))) {
1388 if (((tok != resv_ptr->users) &&
1389 (tok[-1] != ',') && (tok[-1] != '-')) ||
1390 ((tok[k] != '\0') && (tok[k] != ','))) {
1391 tmp = tok + 1;
1392 continue;
1393 }
1394 if (tok[-1] == '-') {
1395 tok--;
1396 k++;
1397 }
1398 if (tok[-1] == ',') {
1399 tok--;
1400 k++;
1401 } else if (tok[k] == ',')
1402 k++;
1403 for (j=0; ; j++) {
1404 tok[j] = tok[j+k];
1405 if (tok[j] == '\0')
1406 break;
1407 }
1408 }
1409 }
1410 if ((resv_ptr->users == NULL) ||
1411 (strlen(resv_ptr->users) == 0)) {
1412 resv_ptr->user_not = 0;
1413 xfree(resv_ptr->users);
1414 }
1415 }
1416
1417 if (plus_user) {
1418 for (i=0; i<u_cnt; i++) {
1419 if (u_type[i] != 2) /* not plus */
1420 continue;
1421 found_it = false;
1422 for (j=0; j<resv_ptr->user_cnt; j++) {
1423 if (resv_ptr->user_list[j] != u_list[i])
1424 continue;
1425 found_it = true;
1426 break;
1427 }
1428 if (found_it)
1429 continue; /* duplicate entry */
1430 if (resv_ptr->users && resv_ptr->users[0])
1431 xstrcat(resv_ptr->users, ",");
1432 if (resv_ptr->user_not)
1433 xstrcat(resv_ptr->users, "-");
1434 xstrcat(resv_ptr->users, u_name[i]);
1435 xrealloc(resv_ptr->user_list,
1436 sizeof(uid_t) * (resv_ptr->user_cnt + 1));
1437 resv_ptr->user_list[resv_ptr->user_cnt++] =
1438 u_list[i];
1439 }
1440 }
1441 xfree(u_cpy);
1442 xfree(u_list);
1443 xfree(u_name);
1444 xfree(u_type);
1445 return SLURM_SUCCESS;
1446
1447 inval: xfree(u_cpy);
1448 xfree(u_list);
1449 xfree(u_name);
1450 xfree(u_type);
1451 return ESLURM_USER_ID_MISSING;
1452 }
1453
1454 /* Given a core_resrcs data structure (which has information only about the
1455 * nodes in that reservation), build a global core_bitmap (which includes
1456 * information about all nodes in the system).
1457 * RET SLURM_SUCCESS or error code */
_get_core_resrcs(slurmctld_resv_t * resv_ptr)1458 static int _get_core_resrcs(slurmctld_resv_t *resv_ptr)
1459 {
1460 int i, i_first, i_last, j, node_inx;
1461 int c, core_offset_local, core_offset_global, core_end;
1462
1463 if (!resv_ptr->core_resrcs || resv_ptr->core_bitmap ||
1464 !resv_ptr->core_resrcs->core_bitmap ||
1465 (bit_ffs(resv_ptr->core_resrcs->core_bitmap) == -1))
1466 return SLURM_SUCCESS;
1467
1468 FREE_NULL_BITMAP(resv_ptr->core_resrcs->node_bitmap);
1469 if (resv_ptr->core_resrcs->nodes &&
1470 (node_name2bitmap(resv_ptr->core_resrcs->nodes, false,
1471 &resv_ptr->core_resrcs->node_bitmap))) {
1472 error("Invalid nodes (%s) for reservation %s",
1473 resv_ptr->core_resrcs->nodes, resv_ptr->name);
1474 return SLURM_ERROR;
1475 } else if (resv_ptr->core_resrcs->nodes == NULL) {
1476 resv_ptr->core_resrcs->node_bitmap =
1477 bit_alloc(node_record_count);
1478 }
1479
1480 i = bit_set_count(resv_ptr->core_resrcs->node_bitmap);
1481 if (resv_ptr->core_resrcs->nhosts != i) {
1482 error("Invalid change in resource allocation node count for "
1483 "reservation %s, %u to %d",
1484 resv_ptr->name, resv_ptr->core_resrcs->nhosts, i);
1485 return SLURM_ERROR;
1486 }
1487
1488 _create_cluster_core_bitmap(&resv_ptr->core_bitmap);
1489 i_first = bit_ffs(resv_ptr->core_resrcs->node_bitmap);
1490 if (i_first >= 0)
1491 i_last = bit_fls(resv_ptr->core_resrcs->node_bitmap);
1492 else
1493 i_last = i_first - 1;
1494 for (i = i_first, node_inx = -1; i <= i_last; i++) {
1495 if (!bit_test(resv_ptr->core_resrcs->node_bitmap, i))
1496 continue;
1497 node_inx++;
1498 core_offset_global = cr_get_coremap_offset(i);
1499 core_end = cr_get_coremap_offset(i + 1);
1500 core_offset_local = get_job_resources_offset(
1501 resv_ptr->core_resrcs, node_inx, 0, 0);
1502 for (c = core_offset_global, j = core_offset_local;
1503 c < core_end; c++, j++) {
1504 if (!bit_test(resv_ptr->core_resrcs->core_bitmap, j))
1505 continue;
1506 bit_set(resv_ptr->core_bitmap, c);
1507 }
1508 }
1509
1510 return SLURM_SUCCESS;
1511 }
1512
1513 /* Build core_resrcs based upon node_bitmap and core_bitmap as needed.
1514 * This translates a global core_bitmap (including all nodes) to a
1515 * core_bitmap for only those nodes in the reservation. This is needed to
1516 * handle nodes being added or removed from the system or their core count
1517 * changing. */
_set_core_resrcs(slurmctld_resv_t * resv_ptr)1518 static void _set_core_resrcs(slurmctld_resv_t *resv_ptr)
1519 {
1520 int i, i_first, i_last, j, node_inx, rc;
1521 int c, core_offset_local, core_offset_global, core_end;
1522
1523 if (!resv_ptr->core_bitmap || resv_ptr->core_resrcs ||
1524 !resv_ptr->node_bitmap ||
1525 ((i_first = bit_ffs(resv_ptr->node_bitmap)) == -1))
1526 return;
1527
1528 resv_ptr->core_resrcs = create_job_resources();
1529 resv_ptr->core_resrcs->nodes = xstrdup(resv_ptr->node_list);
1530 resv_ptr->core_resrcs->node_bitmap = bit_copy(resv_ptr->node_bitmap);
1531 resv_ptr->core_resrcs->nhosts = bit_set_count(resv_ptr->node_bitmap);
1532 rc = build_job_resources(resv_ptr->core_resrcs, node_record_table_ptr);
1533 if (rc != SLURM_SUCCESS) {
1534 free_job_resources(&resv_ptr->core_resrcs);
1535 return;
1536 }
1537 resv_ptr->core_resrcs->cpus = xcalloc(resv_ptr->core_resrcs->nhosts,
1538 sizeof(uint16_t));
1539
1540 core_offset_local = -1;
1541 node_inx = -1;
1542 i_last = bit_fls(resv_ptr->node_bitmap);
1543 for (i = i_first; i <= i_last; i++) {
1544 if (!bit_test(resv_ptr->node_bitmap, i))
1545 continue;
1546 node_inx++;
1547 core_offset_global = cr_get_coremap_offset(i);
1548 core_end = cr_get_coremap_offset(i + 1);
1549 for (c = core_offset_global, j = core_offset_local;
1550 c < core_end; c++, j++) {
1551 core_offset_local++;
1552 if (!bit_test(resv_ptr->core_bitmap, c))
1553 continue;
1554 if (resv_ptr->core_resrcs->core_bitmap)
1555 bit_set(resv_ptr->core_resrcs->core_bitmap,
1556 core_offset_local);
1557 resv_ptr->core_resrcs->cpus[node_inx]++;
1558 }
1559 }
1560 }
1561
1562 /*
1563 * _pack_resv - dump configuration information about a specific reservation
1564 * in machine independent form (for network transmission or state save)
1565 * IN resv_ptr - pointer to reservation for which information is requested
1566 * IN/OUT buffer - buffer in which data is placed, pointers automatically
1567 * updated
1568 * IN internal - true if for internal save state, false for xmit to users
1569 * NOTE: if you make any changes here be sure to make the corresponding
1570 * to _unpack_reserve_info_members() in common/slurm_protocol_pack.c
1571 * plus load_all_resv_state() below.
1572 */
_pack_resv(slurmctld_resv_t * resv_ptr,Buf buffer,bool internal,uint16_t protocol_version)1573 static void _pack_resv(slurmctld_resv_t *resv_ptr, Buf buffer,
1574 bool internal, uint16_t protocol_version)
1575 {
1576 time_t now = time(NULL), start_relative, end_relative;
1577 int i_first, i_last, i;
1578 int offset_start, offset_end;
1579 uint32_t i_cnt;
1580 node_record_t *node_ptr;
1581 job_resources_t *core_resrcs;
1582 char *core_str;
1583
1584 if (resv_ptr->flags & RESERVE_FLAG_TIME_FLOAT)
1585 last_resv_update = now;
1586 if (!internal && (resv_ptr->flags & RESERVE_FLAG_TIME_FLOAT)) {
1587 start_relative = resv_ptr->start_time + now;
1588 if (resv_ptr->duration == INFINITE)
1589 end_relative = start_relative + YEAR_SECONDS;
1590 else if (resv_ptr->duration && (resv_ptr->duration != NO_VAL))
1591 end_relative = start_relative + resv_ptr->duration * 60;
1592 else {
1593 end_relative = resv_ptr->end_time;
1594 if (start_relative > end_relative)
1595 start_relative = end_relative;
1596 }
1597 } else {
1598 start_relative = resv_ptr->start_time_first;
1599 end_relative = resv_ptr->end_time;
1600 }
1601
1602 if (protocol_version >= SLURM_20_02_PROTOCOL_VERSION) {
1603 packstr(resv_ptr->accounts, buffer);
1604 packstr(resv_ptr->burst_buffer, buffer);
1605 pack32(resv_ptr->core_cnt, buffer);
1606 pack_time(end_relative, buffer);
1607 packstr(resv_ptr->features, buffer);
1608 pack64(resv_ptr->flags, buffer);
1609 packstr(resv_ptr->licenses, buffer);
1610 pack32(resv_ptr->max_start_delay, buffer);
1611 packstr(resv_ptr->name, buffer);
1612 pack32(resv_ptr->node_cnt, buffer);
1613 packstr(resv_ptr->node_list, buffer);
1614 packstr(resv_ptr->partition, buffer);
1615 pack32(resv_ptr->purge_comp_time, buffer);
1616 pack32(resv_ptr->resv_watts, buffer);
1617 pack_time(start_relative, buffer);
1618 packstr(resv_ptr->tres_fmt_str, buffer);
1619 packstr(resv_ptr->users, buffer);
1620
1621 if (internal) {
1622 pack8(resv_ptr->account_not, buffer);
1623 packstr(resv_ptr->assoc_list, buffer);
1624 pack32(resv_ptr->boot_time, buffer);
1625 /*
1626 * NOTE: Restoring core_bitmap directly only works if
1627 * the system's node and core counts don't change.
1628 * core_resrcs is used so configuration changes can be
1629 * supported
1630 */
1631 _set_core_resrcs(resv_ptr);
1632 pack_job_resources(resv_ptr->core_resrcs, buffer,
1633 protocol_version);
1634 pack32(resv_ptr->duration, buffer);
1635 pack8(resv_ptr->full_nodes, buffer);
1636 pack32(resv_ptr->resv_id, buffer);
1637 pack_time(resv_ptr->start_time_prev, buffer);
1638 pack_time(resv_ptr->start_time, buffer);
1639 pack_time(resv_ptr->idle_start_time, buffer);
1640 packstr(resv_ptr->tres_str, buffer);
1641 pack8(resv_ptr->user_not, buffer);
1642 } else {
1643 pack_bit_str_hex(resv_ptr->node_bitmap, buffer);
1644 if (!resv_ptr->core_bitmap ||
1645 !resv_ptr->core_resrcs ||
1646 !resv_ptr->core_resrcs->node_bitmap ||
1647 !resv_ptr->core_resrcs->core_bitmap ||
1648 (bit_ffs(resv_ptr->core_bitmap) == -1)) {
1649 pack32((uint32_t) 0, buffer);
1650 } else {
1651 core_resrcs = resv_ptr->core_resrcs;
1652 i_cnt = bit_set_count(core_resrcs->node_bitmap);
1653 pack32(i_cnt, buffer);
1654 i_first = bit_ffs(core_resrcs->node_bitmap);
1655 i_last = bit_fls(core_resrcs->node_bitmap);
1656 for (i = i_first; i <= i_last; i++) {
1657 if (!bit_test(core_resrcs->node_bitmap,
1658 i))
1659 continue;
1660 offset_start = cr_get_coremap_offset(i);
1661 offset_end = cr_get_coremap_offset(i+1);
1662 node_ptr = node_record_table_ptr + i;
1663 packstr(node_ptr->name, buffer);
1664 core_str = bit_fmt_range(
1665 resv_ptr->core_bitmap,
1666 offset_start,
1667 (offset_end - offset_start));
1668 packstr(core_str, buffer);
1669 xfree(core_str);
1670 }
1671 }
1672 }
1673 } else if (protocol_version >= SLURM_MIN_PROTOCOL_VERSION) {
1674 packstr(resv_ptr->accounts, buffer);
1675 packstr(resv_ptr->burst_buffer, buffer);
1676 pack32(resv_ptr->core_cnt, buffer);
1677 pack_time(end_relative, buffer);
1678 packstr(resv_ptr->features, buffer);
1679 pack64(resv_ptr->flags, buffer);
1680 packstr(resv_ptr->licenses, buffer);
1681 packstr(resv_ptr->name, buffer);
1682 pack32(resv_ptr->node_cnt, buffer);
1683 packstr(resv_ptr->node_list, buffer);
1684 packstr(resv_ptr->partition, buffer);
1685 pack32(resv_ptr->resv_watts, buffer);
1686 pack_time(start_relative, buffer);
1687 packstr(resv_ptr->tres_fmt_str, buffer);
1688 packstr(resv_ptr->users, buffer);
1689
1690 if (internal) {
1691 pack8(resv_ptr->account_not, buffer);
1692 packstr(resv_ptr->assoc_list, buffer);
1693 pack32(resv_ptr->boot_time, buffer);
1694 /*
1695 * NOTE: Restoring core_bitmap directly only works if
1696 * the system's node and core counts don't change.
1697 * core_resrcs is used so configuration changes can be
1698 * supported
1699 */
1700 _set_core_resrcs(resv_ptr);
1701 pack_job_resources(resv_ptr->core_resrcs, buffer,
1702 protocol_version);
1703 pack32(resv_ptr->duration, buffer);
1704 pack8(resv_ptr->full_nodes, buffer);
1705 pack32(resv_ptr->resv_id, buffer);
1706 pack_time(resv_ptr->start_time_prev, buffer);
1707 pack_time(resv_ptr->start_time, buffer);
1708 packstr(resv_ptr->tres_str, buffer);
1709 pack8(resv_ptr->user_not, buffer);
1710 } else {
1711 pack_bit_str_hex(resv_ptr->node_bitmap, buffer);
1712 if (!resv_ptr->core_bitmap ||
1713 !resv_ptr->core_resrcs ||
1714 !resv_ptr->core_resrcs->node_bitmap ||
1715 !resv_ptr->core_resrcs->core_bitmap ||
1716 (bit_ffs(resv_ptr->core_bitmap) == -1)) {
1717 pack32((uint32_t) 0, buffer);
1718 } else {
1719 core_resrcs = resv_ptr->core_resrcs;
1720 i_cnt = bit_set_count(core_resrcs->node_bitmap);
1721 pack32(i_cnt, buffer);
1722 i_first = bit_ffs(core_resrcs->node_bitmap);
1723 i_last = bit_fls(core_resrcs->node_bitmap);
1724 for (i = i_first; i <= i_last; i++) {
1725 if (!bit_test(core_resrcs->node_bitmap,
1726 i))
1727 continue;
1728 offset_start = cr_get_coremap_offset(i);
1729 offset_end = cr_get_coremap_offset(i+1);
1730 node_ptr = node_record_table_ptr + i;
1731 packstr(node_ptr->name, buffer);
1732 core_str = bit_fmt_range(
1733 resv_ptr->core_bitmap,
1734 offset_start,
1735 (offset_end - offset_start));
1736 packstr(core_str, buffer);
1737 xfree(core_str);
1738 }
1739 }
1740 }
1741 }
1742 }
1743
_load_reservation_state(Buf buffer,uint16_t protocol_version)1744 slurmctld_resv_t *_load_reservation_state(Buf buffer,
1745 uint16_t protocol_version)
1746 {
1747 slurmctld_resv_t *resv_ptr;
1748 uint32_t uint32_tmp = 0;
1749
1750 resv_ptr = xmalloc(sizeof(slurmctld_resv_t));
1751 xassert(resv_ptr->magic = RESV_MAGIC); /* Sets value */
1752 if (protocol_version >= SLURM_20_02_PROTOCOL_VERSION) {
1753 safe_unpackstr_xmalloc(&resv_ptr->accounts,
1754 &uint32_tmp, buffer);
1755 safe_unpackstr_xmalloc(&resv_ptr->burst_buffer,
1756 &uint32_tmp, buffer);
1757 safe_unpack32(&resv_ptr->core_cnt, buffer);
1758 safe_unpack_time(&resv_ptr->end_time, buffer);
1759 safe_unpackstr_xmalloc(&resv_ptr->features,
1760 &uint32_tmp, buffer);
1761 safe_unpack64(&resv_ptr->flags, buffer);
1762 safe_unpackstr_xmalloc(&resv_ptr->licenses,
1763 &uint32_tmp, buffer);
1764 safe_unpack32(&resv_ptr->max_start_delay, buffer);
1765 safe_unpackstr_xmalloc(&resv_ptr->name, &uint32_tmp, buffer);
1766
1767 safe_unpack32(&resv_ptr->node_cnt, buffer);
1768 safe_unpackstr_xmalloc(&resv_ptr->node_list,
1769 &uint32_tmp, buffer);
1770 safe_unpackstr_xmalloc(&resv_ptr->partition,
1771 &uint32_tmp, buffer);
1772 safe_unpack32(&resv_ptr->purge_comp_time, buffer);
1773 safe_unpack32(&resv_ptr->resv_watts, buffer);
1774 safe_unpack_time(&resv_ptr->start_time_first, buffer);
1775 safe_unpackstr_xmalloc(&resv_ptr->tres_fmt_str,
1776 &uint32_tmp, buffer);
1777 safe_unpackstr_xmalloc(&resv_ptr->users, &uint32_tmp, buffer);
1778
1779 /* Fields saved for internal use only (save state) */
1780 safe_unpack8((uint8_t *)&resv_ptr->account_not, buffer);
1781 safe_unpackstr_xmalloc(&resv_ptr->assoc_list,
1782 &uint32_tmp, buffer);
1783 safe_unpack32(&resv_ptr->boot_time, buffer);
1784 if (unpack_job_resources(&resv_ptr->core_resrcs, buffer,
1785 protocol_version) != SLURM_SUCCESS)
1786 goto unpack_error;
1787 safe_unpack32(&resv_ptr->duration, buffer);
1788 safe_unpack8((uint8_t *)&resv_ptr->full_nodes, buffer);
1789 safe_unpack32(&resv_ptr->resv_id, buffer);
1790 safe_unpack_time(&resv_ptr->start_time_prev, buffer);
1791 safe_unpack_time(&resv_ptr->start_time, buffer);
1792 safe_unpack_time(&resv_ptr->idle_start_time, buffer);
1793 safe_unpackstr_xmalloc(&resv_ptr->tres_str,
1794 &uint32_tmp, buffer);
1795 safe_unpack8((uint8_t *)&resv_ptr->user_not, buffer);
1796 if (!resv_ptr->purge_comp_time)
1797 resv_ptr->purge_comp_time = 300;
1798 } else if (protocol_version >= SLURM_MIN_PROTOCOL_VERSION) {
1799 safe_unpackstr_xmalloc(&resv_ptr->accounts,
1800 &uint32_tmp, buffer);
1801 safe_unpackstr_xmalloc(&resv_ptr->burst_buffer,
1802 &uint32_tmp, buffer);
1803 safe_unpack32(&resv_ptr->core_cnt, buffer);
1804 safe_unpack_time(&resv_ptr->end_time, buffer);
1805 safe_unpackstr_xmalloc(&resv_ptr->features,
1806 &uint32_tmp, buffer);
1807 safe_unpack64(&resv_ptr->flags, buffer);
1808 safe_unpackstr_xmalloc(&resv_ptr->licenses,
1809 &uint32_tmp, buffer);
1810 safe_unpackstr_xmalloc(&resv_ptr->name, &uint32_tmp, buffer);
1811
1812 safe_unpack32(&resv_ptr->node_cnt, buffer);
1813 safe_unpackstr_xmalloc(&resv_ptr->node_list,
1814 &uint32_tmp, buffer);
1815 safe_unpackstr_xmalloc(&resv_ptr->partition,
1816 &uint32_tmp, buffer);
1817 safe_unpack32(&resv_ptr->resv_watts, buffer);
1818 safe_unpack_time(&resv_ptr->start_time_first, buffer);
1819 safe_unpackstr_xmalloc(&resv_ptr->tres_fmt_str,
1820 &uint32_tmp, buffer);
1821 safe_unpackstr_xmalloc(&resv_ptr->users, &uint32_tmp, buffer);
1822
1823 /* Fields saved for internal use only (save state) */
1824 safe_unpack8((uint8_t *)&resv_ptr->account_not, buffer);
1825 safe_unpackstr_xmalloc(&resv_ptr->assoc_list,
1826 &uint32_tmp, buffer);
1827 safe_unpack32(&resv_ptr->boot_time, buffer);
1828 if (unpack_job_resources(&resv_ptr->core_resrcs, buffer,
1829 protocol_version) != SLURM_SUCCESS)
1830 goto unpack_error;
1831 safe_unpack32(&resv_ptr->duration, buffer);
1832 safe_unpack8((uint8_t *)&resv_ptr->full_nodes, buffer);
1833 safe_unpack32(&resv_ptr->resv_id, buffer);
1834 safe_unpack_time(&resv_ptr->start_time_prev, buffer);
1835 safe_unpack_time(&resv_ptr->start_time, buffer);
1836 safe_unpackstr_xmalloc(&resv_ptr->tres_str,
1837 &uint32_tmp, buffer);
1838 safe_unpack8((uint8_t *)&resv_ptr->user_not, buffer);
1839 if (!resv_ptr->purge_comp_time)
1840 resv_ptr->purge_comp_time = 300;
1841 } else
1842 goto unpack_error;
1843
1844 return resv_ptr;
1845
1846 unpack_error:
1847 error("Incomplete reservation state save file");
1848 _del_resv_rec(resv_ptr);
1849 return NULL;
1850 }
1851
1852 /*
1853 * Test if a new/updated reservation request will overlap running jobs
1854 * Ignore jobs already running in that specific reservation
1855 * resv_name IN - Name of existing reservation or NULL
1856 * RET true if overlap
1857 */
_job_overlap(time_t start_time,uint64_t flags,bitstr_t * node_bitmap,char * resv_name)1858 static bool _job_overlap(time_t start_time, uint64_t flags,
1859 bitstr_t *node_bitmap, char *resv_name)
1860 {
1861 ListIterator job_iterator;
1862 job_record_t *job_ptr;
1863 bool overlap = false;
1864
1865 if (!node_bitmap || /* No nodes to test for */
1866 (flags & RESERVE_FLAG_IGN_JOBS)) /* ignore job overlap */
1867 return overlap;
1868 if (flags & RESERVE_FLAG_TIME_FLOAT)
1869 start_time += time(NULL);
1870
1871 job_iterator = list_iterator_create(job_list);
1872 while ((job_ptr = list_next(job_iterator))) {
1873 if (IS_JOB_RUNNING(job_ptr) &&
1874 (job_ptr->end_time > start_time) &&
1875 bit_overlap_any(job_ptr->node_bitmap, node_bitmap) &&
1876 ((resv_name == NULL) ||
1877 (xstrcmp(resv_name, job_ptr->resv_name) != 0))) {
1878 overlap = true;
1879 break;
1880 }
1881 }
1882 list_iterator_destroy(job_iterator);
1883
1884 return overlap;
1885 }
1886
1887 /*
1888 * Test if a new/updated reservation request overlaps an existing
1889 * reservation
1890 * RET true if overlap
1891 */
_resv_overlap(resv_desc_msg_t * resv_desc_ptr,bitstr_t * node_bitmap,slurmctld_resv_t * this_resv_ptr)1892 static bool _resv_overlap(resv_desc_msg_t *resv_desc_ptr,
1893 bitstr_t *node_bitmap,
1894 slurmctld_resv_t *this_resv_ptr)
1895 {
1896 ListIterator iter;
1897 slurmctld_resv_t *resv_ptr;
1898 bool rc = false;
1899
1900 if ((resv_desc_ptr->flags & RESERVE_FLAG_MAINT) ||
1901 (resv_desc_ptr->flags & RESERVE_FLAG_OVERLAP) ||
1902 (!node_bitmap))
1903 return rc;
1904
1905 iter = list_iterator_create(resv_list);
1906
1907 while ((resv_ptr = list_next(iter))) {
1908 if (resv_ptr == this_resv_ptr)
1909 continue; /* skip self */
1910 if (resv_ptr->node_bitmap == NULL)
1911 continue; /* no specific nodes in reservation */
1912 if ((resv_ptr->flags & RESERVE_FLAG_MAINT) ||
1913 (resv_ptr->flags & RESERVE_FLAG_OVERLAP))
1914 continue;
1915 if (!bit_overlap_any(resv_ptr->node_bitmap, node_bitmap))
1916 continue; /* no overlap */
1917 if (!resv_ptr->full_nodes)
1918 continue;
1919 if (_resv_time_overlap(resv_desc_ptr, resv_ptr)) {
1920 rc = true;
1921 break;
1922 }
1923 }
1924 list_iterator_destroy(iter);
1925
1926 return rc;
1927 }
1928
_resv_time_overlap(resv_desc_msg_t * resv_desc_ptr,slurmctld_resv_t * resv_ptr)1929 static bool _resv_time_overlap(resv_desc_msg_t *resv_desc_ptr,
1930 slurmctld_resv_t *resv_ptr)
1931 {
1932 bool rc = false;
1933 int i, j;
1934 time_t s_time1, s_time2, e_time1, e_time2;
1935 time_t start_relative, end_relative;
1936 time_t now = time(NULL);
1937
1938 if (resv_ptr->flags & RESERVE_FLAG_TIME_FLOAT) {
1939 start_relative = resv_ptr->start_time + now;
1940 if (resv_ptr->duration == INFINITE)
1941 end_relative = start_relative +
1942 YEAR_SECONDS;
1943 else if (resv_ptr->duration &&
1944 (resv_ptr->duration != NO_VAL)) {
1945 end_relative = start_relative +
1946 resv_ptr->duration * 60;
1947 } else {
1948 end_relative = resv_ptr->end_time;
1949 if (start_relative > end_relative)
1950 start_relative = end_relative;
1951 }
1952 } else {
1953 start_relative = resv_ptr->start_time;
1954 end_relative = resv_ptr->end_time;
1955 }
1956
1957 for (i=0; ((i<7) && (!rc)); i++) { /* look forward one week */
1958 s_time1 = resv_desc_ptr->start_time;
1959 e_time1 = resv_desc_ptr->end_time;
1960 _advance_time(&s_time1, i);
1961 _advance_time(&e_time1, i);
1962 for (j=0; ((j<7) && (!rc)); j++) {
1963 s_time2 = start_relative;
1964 e_time2 = end_relative;
1965 _advance_time(&s_time2, j);
1966 _advance_time(&e_time2, j);
1967 if ((s_time1 < e_time2) &&
1968 (e_time1 > s_time2)) {
1969 rc = true;
1970 break;
1971 }
1972 if (!(resv_ptr->flags & RESERVE_FLAG_DAILY))
1973 break;
1974 }
1975 if (!(resv_desc_ptr->flags & RESERVE_FLAG_DAILY))
1976 break;
1977 }
1978
1979 return rc;
1980 }
1981 /* Set a reservation's TRES count. Requires that the reservation's
1982 * node_bitmap be set.
1983 * This needs to be done after all other setup is done.
1984 */
_set_tres_cnt(slurmctld_resv_t * resv_ptr,slurmctld_resv_t * old_resv_ptr)1985 static void _set_tres_cnt(slurmctld_resv_t *resv_ptr,
1986 slurmctld_resv_t *old_resv_ptr)
1987 {
1988 int i;
1989 uint64_t cpu_cnt = 0;
1990 node_record_t *node_ptr = node_record_table_ptr;
1991 char start_time[32], end_time[32], tmp_msd[40];
1992 char *name1, *name2, *val1, *val2;
1993 assoc_mgr_lock_t locks = { .tres = READ_LOCK };
1994
1995 if (resv_ptr->full_nodes && resv_ptr->node_bitmap) {
1996 resv_ptr->core_cnt = 0;
1997
1998 for (i=0; i<node_record_count; i++, node_ptr++) {
1999 if (!bit_test(resv_ptr->node_bitmap, i))
2000 continue;
2001 resv_ptr->core_cnt +=
2002 (node_ptr->config_ptr->cores *
2003 node_ptr->config_ptr->sockets);
2004 cpu_cnt += node_ptr->config_ptr->cpus;
2005 }
2006 } else if (resv_ptr->core_bitmap) {
2007 resv_ptr->core_cnt =
2008 bit_set_count(resv_ptr->core_bitmap);
2009
2010 if (resv_ptr->node_bitmap) {
2011 for (i = 0; i < node_record_count; i++, node_ptr++) {
2012 int offset, core;
2013 uint32_t cores, threads;
2014 if (!bit_test(resv_ptr->node_bitmap, i))
2015 continue;
2016
2017 cores = (node_ptr->config_ptr->cores *
2018 node_ptr->config_ptr->sockets);
2019 threads = node_ptr->config_ptr->threads;
2020
2021 offset = cr_get_coremap_offset(i);
2022
2023 for (core = 0; core < cores; core++) {
2024 if (!bit_test(resv_ptr->core_bitmap,
2025 core + offset))
2026 continue;
2027 cpu_cnt += threads;
2028 }
2029 /* info("cpu_cnt is now %"PRIu64" after %s", */
2030 /* cpu_cnt, node_ptr->name); */
2031 }
2032 } else
2033 cpu_cnt = resv_ptr->core_cnt;
2034 }
2035
2036 xfree(resv_ptr->tres_str);
2037 if (cpu_cnt)
2038 xstrfmtcat(resv_ptr->tres_str, "%s%u=%"PRIu64,
2039 resv_ptr->tres_str ? "," : "",
2040 TRES_CPU, cpu_cnt);
2041
2042 if ((name1 = licenses_2_tres_str(resv_ptr->license_list))) {
2043 xstrfmtcat(resv_ptr->tres_str, "%s%s",
2044 resv_ptr->tres_str ? "," : "",
2045 name1);
2046 xfree(name1);
2047 }
2048
2049 if ((name1 = bb_g_xlate_bb_2_tres_str(resv_ptr->burst_buffer))) {
2050 xstrfmtcat(resv_ptr->tres_str, "%s%s",
2051 resv_ptr->tres_str ? "," : "",
2052 name1);
2053 xfree(name1);
2054 }
2055
2056 xfree(resv_ptr->tres_fmt_str);
2057 assoc_mgr_lock(&locks);
2058 resv_ptr->tres_fmt_str = slurmdb_make_tres_string_from_simple(
2059 resv_ptr->tres_str, assoc_mgr_tres_list, NO_VAL,
2060 CONVERT_NUM_UNIT_EXACT, 0, NULL);
2061 assoc_mgr_unlock(&locks);
2062
2063 slurm_make_time_str(&resv_ptr->start_time, start_time,
2064 sizeof(start_time));
2065 slurm_make_time_str(&resv_ptr->end_time, end_time, sizeof(end_time));
2066 if (resv_ptr->accounts) {
2067 name1 = " accounts=";
2068 val1 = resv_ptr->accounts;
2069 } else
2070 name1 = val1 = "";
2071 if (resv_ptr->users) {
2072 name2 = " users=";
2073 val2 = resv_ptr->users;
2074 } else
2075 name2 = val2 = "";
2076
2077 if (resv_ptr->max_start_delay)
2078 secs2time_str(resv_ptr->max_start_delay,
2079 tmp_msd, sizeof(tmp_msd));
2080
2081 sched_info("%s reservation=%s%s%s%s%s nodes=%s cores=%u "
2082 "licenses=%s tres=%s watts=%u start=%s end=%s MaxStartDelay=%s",
2083 old_resv_ptr ? "Updated" : "Created",
2084 resv_ptr->name, name1, val1, name2, val2,
2085 resv_ptr->node_list, resv_ptr->core_cnt, resv_ptr->licenses,
2086 resv_ptr->tres_fmt_str, resv_ptr->resv_watts,
2087 start_time, end_time,
2088 resv_ptr->max_start_delay ? tmp_msd : "");
2089 if (old_resv_ptr)
2090 _post_resv_update(resv_ptr, old_resv_ptr);
2091 else
2092 _post_resv_create(resv_ptr);
2093 }
2094
2095 /*
2096 * _license_validate2 - A variant of license_validate which considers the
2097 * licenses used by overlapping reservations
2098 */
_license_validate2(resv_desc_msg_t * resv_desc_ptr,bool * valid)2099 static List _license_validate2(resv_desc_msg_t *resv_desc_ptr, bool *valid)
2100 {
2101 List license_list, merged_list;
2102 ListIterator iter;
2103 slurmctld_resv_t *resv_ptr;
2104 char *merged_licenses;
2105
2106 license_list = license_validate(resv_desc_ptr->licenses, true, true,
2107 NULL, valid);
2108 if (resv_desc_ptr->licenses == NULL)
2109 return license_list;
2110
2111 merged_licenses = xstrdup(resv_desc_ptr->licenses);
2112 iter = list_iterator_create(resv_list);
2113 while ((resv_ptr = list_next(iter))) {
2114 if ((resv_ptr->licenses == NULL) ||
2115 (resv_ptr->end_time <= resv_desc_ptr->start_time) ||
2116 (resv_ptr->start_time >= resv_desc_ptr->end_time))
2117 continue; /* No overlap */
2118 if (resv_desc_ptr->name &&
2119 !xstrcmp(resv_desc_ptr->name, resv_ptr->name))
2120 continue; /* Modifying this reservation */
2121 xstrcat(merged_licenses, ",");
2122 xstrcat(merged_licenses, resv_ptr->licenses);
2123 }
2124 list_iterator_destroy(iter);
2125 merged_list = license_validate(merged_licenses, true, true, NULL,
2126 valid);
2127 xfree(merged_licenses);
2128 FREE_NULL_LIST(merged_list);
2129 return license_list;
2130 }
2131
2132 /* Create a resource reservation */
create_resv(resv_desc_msg_t * resv_desc_ptr)2133 extern int create_resv(resv_desc_msg_t *resv_desc_ptr)
2134 {
2135 int i, j, rc = SLURM_SUCCESS;
2136 time_t now = time(NULL);
2137 part_record_t *part_ptr = NULL;
2138 bitstr_t *node_bitmap = NULL;
2139 bitstr_t *core_bitmap = NULL;
2140 slurmctld_resv_t *resv_ptr = NULL;
2141 int account_cnt = 0, user_cnt = 0;
2142 char **account_list = NULL;
2143 uid_t *user_list = NULL;
2144 List license_list = (List) NULL;
2145 uint32_t total_node_cnt = 0;
2146 bool account_not = false, user_not = false;
2147
2148 _create_resv_lists(false);
2149 _dump_resv_req(resv_desc_ptr, "create_resv");
2150
2151 if (resv_desc_ptr->flags == NO_VAL64)
2152 resv_desc_ptr->flags = 0;
2153 else {
2154 resv_desc_ptr->flags &= RESERVE_FLAG_MAINT |
2155 RESERVE_FLAG_FLEX |
2156 RESERVE_FLAG_OVERLAP |
2157 RESERVE_FLAG_IGN_JOBS |
2158 RESERVE_FLAG_DAILY |
2159 RESERVE_FLAG_WEEKDAY |
2160 RESERVE_FLAG_WEEKEND |
2161 RESERVE_FLAG_WEEKLY |
2162 RESERVE_FLAG_STATIC |
2163 RESERVE_FLAG_ANY_NODES |
2164 RESERVE_FLAG_PART_NODES |
2165 RESERVE_FLAG_FIRST_CORES |
2166 RESERVE_FLAG_TIME_FLOAT |
2167 RESERVE_FLAG_PURGE_COMP |
2168 RESERVE_FLAG_REPLACE |
2169 RESERVE_FLAG_REPLACE_DOWN |
2170 RESERVE_FLAG_NO_HOLD_JOBS |
2171 RESERVE_FLAG_PROM;
2172 }
2173
2174 /* Validate the request */
2175 if (resv_desc_ptr->start_time != (time_t) NO_VAL) {
2176 if (resv_desc_ptr->flags & RESERVE_FLAG_TIME_FLOAT) {
2177 if (resv_desc_ptr->start_time < now)
2178 resv_desc_ptr->start_time = now;
2179 } else if (resv_desc_ptr->start_time < (now - MAX_RESV_DELAY)) {
2180 info("Reservation request has invalid start time");
2181 rc = ESLURM_INVALID_TIME_VALUE;
2182 goto bad_parse;
2183 }
2184 } else
2185 resv_desc_ptr->start_time = now;
2186
2187 if (resv_desc_ptr->end_time != (time_t) NO_VAL) {
2188 if (resv_desc_ptr->end_time < (now - MAX_RESV_DELAY)) {
2189 info("Reservation request has invalid end time");
2190 rc = ESLURM_INVALID_TIME_VALUE;
2191 goto bad_parse;
2192 }
2193 } else if (resv_desc_ptr->duration == INFINITE) {
2194 resv_desc_ptr->end_time = resv_desc_ptr->start_time +
2195 YEAR_SECONDS;
2196 } else if (resv_desc_ptr->duration) {
2197 resv_desc_ptr->end_time = resv_desc_ptr->start_time +
2198 (resv_desc_ptr->duration * 60);
2199 } else
2200 resv_desc_ptr->end_time = INFINITE;
2201
2202 if ((resv_desc_ptr->flags & RESERVE_FLAG_REPLACE) ||
2203 (resv_desc_ptr->flags & RESERVE_FLAG_REPLACE_DOWN)) {
2204 if (resv_desc_ptr->node_list) {
2205 rc = ESLURM_INVALID_NODE_NAME;
2206 goto bad_parse;
2207 }
2208 if (resv_desc_ptr->core_cnt) {
2209 rc = ESLURM_INVALID_CPU_COUNT;
2210 goto bad_parse;
2211 }
2212 }
2213
2214 if (resv_desc_ptr->partition) {
2215 part_ptr = find_part_record(resv_desc_ptr->partition);
2216 if (!part_ptr) {
2217 info("Reservation request has invalid partition %s",
2218 resv_desc_ptr->partition);
2219 rc = ESLURM_INVALID_PARTITION_NAME;
2220 goto bad_parse;
2221 }
2222 } else if (resv_desc_ptr->flags & RESERVE_FLAG_PART_NODES) {
2223 info("Reservation request with Part_Nodes flag lacks "
2224 "partition specification");
2225 rc = ESLURM_INVALID_PARTITION_NAME;
2226 goto bad_parse;
2227 }
2228
2229 if ((resv_desc_ptr->flags & RESERVE_FLAG_PART_NODES) &&
2230 (xstrcasecmp(resv_desc_ptr->node_list, "ALL"))) {
2231 info("Reservation request with Part_Nodes flag lacks nodelist=ALL specification");
2232 rc = ESLURM_INVALID_NODE_NAME;
2233 goto bad_parse;
2234 }
2235
2236 if ((resv_desc_ptr->accounts == NULL) &&
2237 (resv_desc_ptr->users == NULL)) {
2238 info("Reservation request lacks users or accounts");
2239 rc = ESLURM_INVALID_ACCOUNT;
2240 goto bad_parse;
2241 }
2242 if (resv_desc_ptr->accounts) {
2243 rc = _build_account_list(resv_desc_ptr->accounts,
2244 &account_cnt, &account_list,
2245 &account_not);
2246 if (rc)
2247 goto bad_parse;
2248 }
2249 if (resv_desc_ptr->users) {
2250 rc = _build_uid_list(resv_desc_ptr->users,
2251 &user_cnt, &user_list, &user_not);
2252 if (rc)
2253 goto bad_parse;
2254 }
2255 if (resv_desc_ptr->licenses) {
2256 bool valid = true;
2257 license_list = _license_validate2(resv_desc_ptr, &valid);
2258 if (!valid) {
2259 info("Reservation request has invalid licenses %s",
2260 resv_desc_ptr->licenses);
2261 rc = ESLURM_INVALID_LICENSES;
2262 goto bad_parse;
2263 }
2264 }
2265 if ((resv_desc_ptr->flags & RESERVE_FLAG_TIME_FLOAT) &&
2266 (resv_desc_ptr->flags & (RESERVE_FLAG_DAILY |
2267 RESERVE_FLAG_WEEKDAY |
2268 RESERVE_FLAG_WEEKEND |
2269 RESERVE_FLAG_WEEKLY))) {
2270 info("Reservation request has mutually exclusive flags. Repeating floating reservations are not supported.");
2271 rc = ESLURM_NOT_SUPPORTED;
2272 goto bad_parse;
2273 }
2274
2275 /* Sort the list of node counts in order descending size */
2276 if (resv_desc_ptr->node_cnt) {
2277 for (i = 0; resv_desc_ptr->node_cnt[i]; i++) {
2278 int max_inx = i;
2279 for (j = (i + 1); resv_desc_ptr->node_cnt[j]; j++) {
2280 if (resv_desc_ptr->node_cnt[j] >
2281 resv_desc_ptr->node_cnt[max_inx])
2282 max_inx = j;
2283 }
2284 if (max_inx != i) { /* swap the values */
2285 uint32_t max_val = resv_desc_ptr->
2286 node_cnt[max_inx];
2287 resv_desc_ptr->node_cnt[max_inx] =
2288 resv_desc_ptr->node_cnt[i];
2289 resv_desc_ptr->node_cnt[i] = max_val;
2290 }
2291 }
2292 }
2293
2294 if (resv_desc_ptr->node_list) {
2295 resv_desc_ptr->flags |= RESERVE_FLAG_SPEC_NODES;
2296 if (xstrcasecmp(resv_desc_ptr->node_list, "ALL") == 0) {
2297 if (resv_desc_ptr->partition && part_ptr &&
2298 (resv_desc_ptr->flags & RESERVE_FLAG_PART_NODES)) {
2299 node_bitmap = bit_copy(part_ptr->node_bitmap);
2300 } else {
2301 resv_desc_ptr->flags &=
2302 (~RESERVE_FLAG_PART_NODES);
2303 resv_desc_ptr->flags |= RESERVE_FLAG_ALL_NODES;
2304 node_bitmap = bit_alloc(node_record_count);
2305 bit_nset(node_bitmap, 0,(node_record_count-1));
2306 }
2307 xfree(resv_desc_ptr->node_list);
2308 resv_desc_ptr->node_list =
2309 bitmap2node_name(node_bitmap);
2310 } else {
2311 resv_desc_ptr->flags &= (~RESERVE_FLAG_PART_NODES);
2312 if (node_name2bitmap(resv_desc_ptr->node_list,
2313 false, &node_bitmap)) {
2314 rc = ESLURM_INVALID_NODE_NAME;
2315 goto bad_parse;
2316 }
2317 xfree(resv_desc_ptr->node_list);
2318 resv_desc_ptr->node_list = bitmap2node_name(node_bitmap);
2319 }
2320 if (bit_set_count(node_bitmap) == 0) {
2321 info("Reservation node list is empty");
2322 rc = ESLURM_INVALID_NODE_NAME;
2323 goto bad_parse;
2324 }
2325 if (!(resv_desc_ptr->flags & RESERVE_FLAG_OVERLAP) &&
2326 _resv_overlap(resv_desc_ptr, node_bitmap, NULL)) {
2327 info("Reservation request overlaps another");
2328 rc = ESLURM_RESERVATION_OVERLAP;
2329 goto bad_parse;
2330 }
2331 total_node_cnt = bit_set_count(node_bitmap);
2332 if (!(resv_desc_ptr->flags & RESERVE_FLAG_IGN_JOBS) &&
2333 !resv_desc_ptr->core_cnt) {
2334 uint64_t flags = resv_desc_ptr->flags;
2335
2336 /*
2337 * Need to clear this flag before _job_overlap()
2338 * which would otherwise add the current time
2339 * on to the start_time. start_time for floating
2340 * reservations has already been set to now.
2341 */
2342 flags &= ~RESERVE_FLAG_TIME_FLOAT;
2343
2344 if (_job_overlap(resv_desc_ptr->start_time, flags,
2345 node_bitmap, NULL)) {
2346 info("Reservation request overlaps jobs");
2347 rc = ESLURM_NODES_BUSY;
2348 goto bad_parse;
2349 }
2350 }
2351 /* We do allow to request cores with nodelist */
2352 if (resv_desc_ptr->core_cnt) {
2353 int nodecnt = bit_set_count(node_bitmap);
2354 int nodeinx = 0;
2355 while (nodeinx < nodecnt) {
2356 if (!resv_desc_ptr->core_cnt[nodeinx]) {
2357 info("Core count for reservation node "
2358 "list is not consistent!");
2359 rc = ESLURM_INVALID_CORE_CNT;
2360 goto bad_parse;
2361 }
2362 #if _DEBUG
2363 info("Requesting %d cores for node_list %d",
2364 resv_desc_ptr->core_cnt[nodeinx],
2365 nodeinx);
2366 #endif
2367 nodeinx++;
2368 }
2369 rc = _select_nodes(resv_desc_ptr, &part_ptr,
2370 &node_bitmap, &core_bitmap);
2371 if (rc != SLURM_SUCCESS)
2372 goto bad_parse;
2373 }
2374 } else if (!(resv_desc_ptr->flags & RESERVE_FLAG_ANY_NODES)) {
2375 resv_desc_ptr->flags &= (~RESERVE_FLAG_PART_NODES);
2376
2377 if ((!resv_desc_ptr->node_cnt || !resv_desc_ptr->node_cnt[0]) &&
2378 !resv_desc_ptr->core_cnt) {
2379 info("Reservation request lacks node specification");
2380 rc = ESLURM_INVALID_NODE_NAME;
2381 } else {
2382 rc = _select_nodes(resv_desc_ptr, &part_ptr, &node_bitmap,
2383 &core_bitmap);
2384 }
2385 if (rc != SLURM_SUCCESS) {
2386 goto bad_parse;
2387 }
2388
2389 /* Get count of allocated nodes, on BlueGene systems, this
2390 * might be more than requested */
2391 total_node_cnt = bit_set_count(node_bitmap);
2392 }
2393
2394 if (resv_desc_ptr->core_cnt && !core_bitmap) {
2395 info("Attempt to reserve cores not possible with current "
2396 "configuration");
2397 rc = ESLURM_INVALID_CPU_COUNT;
2398 goto bad_parse;
2399 }
2400
2401 rc = _generate_resv_id();
2402 if (rc != SLURM_SUCCESS)
2403 goto bad_parse;
2404
2405 /* If name == NULL or empty string, then generate a name. */
2406 if (resv_desc_ptr->name && (resv_desc_ptr->name[0] != '\0')) {
2407 resv_ptr = (slurmctld_resv_t *) list_find_first (resv_list,
2408 _find_resv_name, resv_desc_ptr->name);
2409 if (resv_ptr) {
2410 info("Reservation request name duplication (%s)",
2411 resv_desc_ptr->name);
2412 rc = ESLURM_RESERVATION_NAME_DUP;
2413 goto bad_parse;
2414 }
2415 } else {
2416 xfree(resv_desc_ptr->name);
2417 while (1) {
2418 _generate_resv_name(resv_desc_ptr);
2419 resv_ptr = (slurmctld_resv_t *)
2420 list_find_first (resv_list,
2421 _find_resv_name, resv_desc_ptr->name);
2422 if (!resv_ptr)
2423 break;
2424 rc = _generate_resv_id(); /* makes new suffix */
2425 if (rc != SLURM_SUCCESS)
2426 goto bad_parse;
2427 /* Same as previously created name, retry */
2428 }
2429 }
2430
2431 /* Create a new reservation record */
2432 resv_ptr = xmalloc(sizeof(slurmctld_resv_t));
2433 resv_ptr->accounts = resv_desc_ptr->accounts;
2434 resv_desc_ptr->accounts = NULL; /* Nothing left to free */
2435 resv_ptr->account_cnt = account_cnt;
2436 resv_ptr->account_list = account_list;
2437 account_cnt = 0;
2438 account_list = NULL;
2439 resv_ptr->account_not = account_not;
2440 resv_ptr->burst_buffer = resv_desc_ptr->burst_buffer;
2441 resv_desc_ptr->burst_buffer = NULL; /* Nothing left to free */
2442 resv_ptr->duration = resv_desc_ptr->duration;
2443 if (resv_desc_ptr->purge_comp_time != NO_VAL)
2444 resv_ptr->purge_comp_time = resv_desc_ptr->purge_comp_time;
2445 else
2446 resv_ptr->purge_comp_time = 300; /* default to 5 minutes */
2447 resv_ptr->end_time = resv_desc_ptr->end_time;
2448 resv_ptr->features = resv_desc_ptr->features;
2449 resv_desc_ptr->features = NULL; /* Nothing left to free */
2450 resv_ptr->licenses = resv_desc_ptr->licenses;
2451 resv_desc_ptr->licenses = NULL; /* Nothing left to free */
2452 resv_ptr->license_list = license_list;
2453 license_list = NULL;
2454
2455 if (resv_desc_ptr->max_start_delay != NO_VAL)
2456 resv_ptr->max_start_delay = resv_desc_ptr->max_start_delay;
2457
2458 resv_ptr->resv_id = top_suffix;
2459 xassert((resv_ptr->magic = RESV_MAGIC)); /* Sets value */
2460 resv_ptr->name = xstrdup(resv_desc_ptr->name);
2461 resv_ptr->node_cnt = total_node_cnt;
2462 resv_ptr->node_list = resv_desc_ptr->node_list;
2463 resv_desc_ptr->node_list = NULL; /* Nothing left to free */
2464 resv_ptr->node_bitmap = node_bitmap; /* May be unset */
2465 node_bitmap = NULL;
2466 resv_ptr->core_bitmap = core_bitmap; /* May be unset */
2467 core_bitmap = NULL;
2468 resv_ptr->partition = resv_desc_ptr->partition;
2469 resv_desc_ptr->partition = NULL; /* Nothing left to free */
2470 resv_ptr->part_ptr = part_ptr;
2471 resv_ptr->resv_watts = resv_desc_ptr->resv_watts;
2472 resv_ptr->start_time = resv_desc_ptr->start_time;
2473 resv_ptr->start_time_first = resv_ptr->start_time;
2474 resv_ptr->start_time_prev = resv_ptr->start_time;
2475 resv_ptr->flags = resv_desc_ptr->flags;
2476 resv_ptr->users = resv_desc_ptr->users;
2477 resv_ptr->user_cnt = user_cnt;
2478 resv_ptr->user_list = user_list;
2479 user_list = NULL;
2480 resv_ptr->user_not = user_not;
2481 resv_desc_ptr->users = NULL; /* Nothing left to free */
2482
2483 if (!resv_desc_ptr->core_cnt) {
2484 #if _DEBUG
2485 info("reservation using full nodes");
2486 #endif
2487 resv_ptr->full_nodes = 1;
2488 } else {
2489 #if _DEBUG
2490 info("reservation using partial nodes");
2491 #endif
2492 resv_ptr->full_nodes = 0;
2493 }
2494
2495 if ((rc = _set_assoc_list(resv_ptr)) != SLURM_SUCCESS) {
2496 _del_resv_rec(resv_ptr);
2497 goto bad_parse;
2498 }
2499
2500 if (resv_ptr->flags & RESERVE_FLAG_TIME_FLOAT)
2501 resv_ptr->start_time -= now;
2502
2503 _set_tres_cnt(resv_ptr, NULL);
2504
2505 _add_resv_to_lists(resv_ptr);
2506 last_resv_update = now;
2507 schedule_resv_save();
2508
2509 return SLURM_SUCCESS;
2510
2511 bad_parse:
2512 for (i = 0; i < account_cnt; i++)
2513 xfree(account_list[i]);
2514 xfree(account_list);
2515 FREE_NULL_BITMAP(core_bitmap);
2516 FREE_NULL_LIST(license_list);
2517 FREE_NULL_BITMAP(node_bitmap);
2518 xfree(user_list);
2519 return rc;
2520 }
2521
2522 /* Purge all reservation data structures */
resv_fini(void)2523 extern void resv_fini(void)
2524 {
2525 FREE_NULL_LIST(prom_resv_list);
2526 FREE_NULL_LIST(resv_list);
2527 }
2528
2529 /* Update an exiting resource reservation */
update_resv(resv_desc_msg_t * resv_desc_ptr)2530 extern int update_resv(resv_desc_msg_t *resv_desc_ptr)
2531 {
2532 time_t now = time(NULL);
2533 slurmctld_resv_t *resv_backup, *resv_ptr;
2534 resv_desc_msg_t resv_desc;
2535 int error_code = SLURM_SUCCESS, i, rc;
2536
2537 _create_resv_lists(false);
2538 _dump_resv_req(resv_desc_ptr, "update_resv");
2539
2540 /* Find the specified reservation */
2541 if (!resv_desc_ptr->name)
2542 return ESLURM_RESERVATION_INVALID;
2543
2544 resv_ptr = (slurmctld_resv_t *) list_find_first (resv_list,
2545 _find_resv_name, resv_desc_ptr->name);
2546 if (!resv_ptr)
2547 return ESLURM_RESERVATION_INVALID;
2548
2549 /* FIXME: Support more core based reservation updates */
2550 if ((!resv_ptr->full_nodes &&
2551 (resv_desc_ptr->node_cnt || resv_desc_ptr->node_list)) ||
2552 resv_desc_ptr->core_cnt) {
2553 info("Core-based reservation %s can not be updated",
2554 resv_desc_ptr->name);
2555 return ESLURM_CORE_RESERVATION_UPDATE;
2556 }
2557
2558 /* Make backup to restore state in case of failure */
2559 resv_backup = _copy_resv(resv_ptr);
2560
2561 /* Process the request */
2562 if (resv_desc_ptr->flags != NO_VAL64) {
2563 if (resv_desc_ptr->flags & RESERVE_FLAG_FLEX)
2564 resv_ptr->flags |= RESERVE_FLAG_FLEX;
2565 if (resv_desc_ptr->flags & RESERVE_FLAG_NO_FLEX)
2566 resv_ptr->flags &= (~RESERVE_FLAG_FLEX);
2567 if (resv_desc_ptr->flags & RESERVE_FLAG_MAINT)
2568 resv_ptr->flags |= RESERVE_FLAG_MAINT;
2569 if (resv_desc_ptr->flags & RESERVE_FLAG_NO_MAINT)
2570 resv_ptr->flags &= (~RESERVE_FLAG_MAINT);
2571 if (resv_desc_ptr->flags & RESERVE_FLAG_OVERLAP)
2572 resv_ptr->flags |= RESERVE_FLAG_OVERLAP;
2573 if (resv_desc_ptr->flags & RESERVE_FLAG_IGN_JOBS)
2574 resv_ptr->flags |= RESERVE_FLAG_IGN_JOBS;
2575 if (resv_desc_ptr->flags & RESERVE_FLAG_NO_IGN_JOB)
2576 resv_ptr->flags &= (~RESERVE_FLAG_IGN_JOBS);
2577 if (resv_desc_ptr->flags & RESERVE_FLAG_DAILY)
2578 resv_ptr->flags |= RESERVE_FLAG_DAILY;
2579 if (resv_desc_ptr->flags & RESERVE_FLAG_NO_DAILY)
2580 resv_ptr->flags &= (~RESERVE_FLAG_DAILY);
2581 if (resv_desc_ptr->flags & RESERVE_FLAG_WEEKDAY)
2582 resv_ptr->flags |= RESERVE_FLAG_WEEKDAY;
2583 if (resv_desc_ptr->flags & RESERVE_FLAG_NO_WEEKDAY)
2584 resv_ptr->flags &= (~RESERVE_FLAG_WEEKDAY);
2585 if (resv_desc_ptr->flags & RESERVE_FLAG_WEEKEND)
2586 resv_ptr->flags |= RESERVE_FLAG_WEEKEND;
2587 if (resv_desc_ptr->flags & RESERVE_FLAG_NO_WEEKEND)
2588 resv_ptr->flags &= (~RESERVE_FLAG_WEEKEND);
2589 if (resv_desc_ptr->flags & RESERVE_FLAG_WEEKLY)
2590 resv_ptr->flags |= RESERVE_FLAG_WEEKLY;
2591 if (resv_desc_ptr->flags & RESERVE_FLAG_NO_WEEKLY)
2592 resv_ptr->flags &= (~RESERVE_FLAG_WEEKLY);
2593 if (resv_desc_ptr->flags & RESERVE_FLAG_ANY_NODES)
2594 resv_ptr->flags |= RESERVE_FLAG_ANY_NODES;
2595 if (resv_desc_ptr->flags & RESERVE_FLAG_NO_ANY_NODES)
2596 resv_ptr->flags &= (~RESERVE_FLAG_ANY_NODES);
2597 if (resv_desc_ptr->flags & RESERVE_FLAG_STATIC)
2598 resv_ptr->flags |= RESERVE_FLAG_STATIC;
2599 if (resv_desc_ptr->flags & RESERVE_FLAG_NO_STATIC)
2600 resv_ptr->flags &= (~RESERVE_FLAG_STATIC);
2601 if (resv_desc_ptr->flags & RESERVE_FLAG_FIRST_CORES)
2602 resv_ptr->flags |= RESERVE_FLAG_FIRST_CORES;
2603 if ((resv_desc_ptr->flags & RESERVE_FLAG_REPLACE) ||
2604 (resv_desc_ptr->flags & RESERVE_FLAG_REPLACE_DOWN)) {
2605 if ((resv_ptr->flags & RESERVE_FLAG_SPEC_NODES) ||
2606 (resv_ptr->full_nodes == 0)) {
2607 error_code = ESLURM_NOT_SUPPORTED;
2608 goto update_failure;
2609 }
2610 if (resv_desc_ptr->flags & RESERVE_FLAG_REPLACE)
2611 resv_ptr->flags |= RESERVE_FLAG_REPLACE;
2612 else
2613 resv_ptr->flags |= RESERVE_FLAG_REPLACE_DOWN;
2614 }
2615 if (resv_desc_ptr->flags & RESERVE_FLAG_PART_NODES) {
2616 if ((resv_ptr->partition == NULL) &&
2617 (resv_desc_ptr->partition == NULL)) {
2618 info("Reservation %s request can not set "
2619 "Part_Nodes flag without partition",
2620 resv_desc_ptr->name);
2621 error_code = ESLURM_INVALID_PARTITION_NAME;
2622 goto update_failure;
2623 }
2624 if (xstrcasecmp(resv_desc_ptr->node_list, "ALL")) {
2625 info("Reservation %s request can not set Part_Nodes flag without partition and nodes=ALL",
2626 resv_desc_ptr->name);
2627 error_code = ESLURM_INVALID_NODE_NAME;
2628 goto update_failure;
2629 }
2630 resv_ptr->flags |= RESERVE_FLAG_PART_NODES;
2631 /* Explicitly set the node_list to ALL */
2632 xfree(resv_desc_ptr->node_list);
2633 resv_desc_ptr->node_list = xstrdup("ALL");
2634 }
2635 if (resv_desc_ptr->flags & RESERVE_FLAG_NO_PART_NODES)
2636 resv_ptr->flags &= (~RESERVE_FLAG_PART_NODES);
2637 if (resv_desc_ptr->flags & RESERVE_FLAG_TIME_FLOAT) {
2638 info("Reservation %s request to set TIME_FLOAT flag",
2639 resv_desc_ptr->name);
2640 error_code = ESLURM_INVALID_TIME_VALUE;
2641 goto update_failure;
2642 }
2643 if (resv_desc_ptr->flags & RESERVE_FLAG_PURGE_COMP)
2644 resv_ptr->flags |= RESERVE_FLAG_PURGE_COMP;
2645 if (resv_desc_ptr->flags & RESERVE_FLAG_NO_PURGE_COMP) {
2646 resv_ptr->flags &= (~RESERVE_FLAG_PURGE_COMP);
2647 if (resv_desc_ptr->purge_comp_time == NO_VAL)
2648 resv_ptr->purge_comp_time = 300;
2649 }
2650 if (resv_desc_ptr->flags & RESERVE_FLAG_NO_HOLD_JOBS)
2651 resv_ptr->flags |= RESERVE_FLAG_NO_HOLD_JOBS;
2652 if ((resv_desc_ptr->flags & RESERVE_FLAG_PROM) &&
2653 !(resv_ptr->flags & RESERVE_FLAG_PROM)) {
2654 resv_ptr->flags |= RESERVE_FLAG_PROM;
2655 list_append(prom_resv_list, resv_ptr);
2656 }
2657 if (resv_desc_ptr->flags & RESERVE_FLAG_NO_PROM) {
2658 resv_ptr->flags &= (~RESERVE_FLAG_PROM);
2659 (void)list_remove_first(
2660 prom_resv_list, _find_resv_ptr, resv_ptr);
2661 }
2662 }
2663
2664 if (resv_desc_ptr->max_start_delay != NO_VAL)
2665 resv_ptr->max_start_delay = resv_desc_ptr->max_start_delay;
2666
2667 if (resv_desc_ptr->purge_comp_time != NO_VAL)
2668 resv_ptr->purge_comp_time = resv_desc_ptr->purge_comp_time;
2669
2670 if (resv_desc_ptr->partition && (resv_desc_ptr->partition[0] == '\0')) {
2671 /* Clear the partition */
2672 xfree(resv_desc_ptr->partition);
2673 xfree(resv_ptr->partition);
2674 resv_ptr->part_ptr = NULL;
2675 }
2676 if (resv_desc_ptr->partition) {
2677 part_record_t *part_ptr = NULL;
2678 part_ptr = find_part_record(resv_desc_ptr->partition);
2679 if (!part_ptr) {
2680 info("Reservation %s request has invalid partition (%s)",
2681 resv_desc_ptr->name, resv_desc_ptr->partition);
2682 error_code = ESLURM_INVALID_PARTITION_NAME;
2683 goto update_failure;
2684 }
2685 xfree(resv_ptr->partition);
2686 resv_ptr->partition = resv_desc_ptr->partition;
2687 resv_desc_ptr->partition = NULL; /* Nothing left to free */
2688 resv_ptr->part_ptr = part_ptr;
2689 }
2690 if (resv_desc_ptr->resv_watts != NO_VAL)
2691 resv_ptr->resv_watts = resv_desc_ptr->resv_watts;
2692 if (resv_desc_ptr->accounts) {
2693 rc = _update_account_list(resv_ptr, resv_desc_ptr->accounts);
2694 if (rc) {
2695 error_code = rc;
2696 goto update_failure;
2697 }
2698 }
2699 if (resv_desc_ptr->burst_buffer) {
2700 xfree(resv_ptr->burst_buffer);
2701 if (resv_desc_ptr->burst_buffer[0] != '\0') {
2702 resv_ptr->burst_buffer = resv_desc_ptr->burst_buffer;
2703 resv_desc_ptr->burst_buffer = NULL;
2704 }
2705 }
2706 if (resv_desc_ptr->licenses && (resv_desc_ptr->licenses[0] == '\0')) {
2707 if (((resv_desc_ptr->node_cnt != NULL) &&
2708 (resv_desc_ptr->node_cnt[0] == 0)) ||
2709 ((resv_desc_ptr->node_cnt == NULL) &&
2710 (resv_ptr->node_cnt == 0))) {
2711 info("Reservation %s attempt to clear licenses with "
2712 "NodeCount=0", resv_desc_ptr->name);
2713 error_code = ESLURM_INVALID_LICENSES;
2714 goto update_failure;
2715 }
2716 xfree(resv_desc_ptr->licenses); /* clear licenses */
2717 xfree(resv_ptr->licenses);
2718 FREE_NULL_LIST(resv_ptr->license_list);
2719 }
2720
2721 if (resv_desc_ptr->licenses) {
2722 bool valid = true;
2723 List license_list;
2724 license_list = _license_validate2(resv_desc_ptr, &valid);
2725 if (!valid) {
2726 info("Reservation %s invalid license update (%s)",
2727 resv_desc_ptr->name, resv_desc_ptr->licenses);
2728 error_code = ESLURM_INVALID_LICENSES;
2729 goto update_failure;
2730 }
2731 xfree(resv_ptr->licenses);
2732 resv_ptr->licenses = resv_desc_ptr->licenses;
2733 resv_desc_ptr->licenses = NULL; /* Nothing left to free */
2734 FREE_NULL_LIST(resv_ptr->license_list);
2735 resv_ptr->license_list = license_list;
2736 }
2737 if (resv_desc_ptr->features && (resv_desc_ptr->features[0] == '\0')) {
2738 xfree(resv_desc_ptr->features); /* clear features */
2739 xfree(resv_ptr->features);
2740 }
2741 if (resv_desc_ptr->features) {
2742 /* To support in the future, the reservation resources would
2743 * need to be selected again. For now, administrator can
2744 * delete this reservation and create a new one. */
2745 info("Attempt to change features of reservation %s. "
2746 "Delete the reservation and create a new one.",
2747 resv_desc_ptr->name);
2748 error_code = ESLURM_NOT_SUPPORTED;
2749 goto update_failure;
2750 }
2751 if (resv_desc_ptr->users) {
2752 rc = _update_uid_list(resv_ptr, resv_desc_ptr->users);
2753 if (rc) {
2754 error_code = rc;
2755 goto update_failure;
2756 }
2757 }
2758 if ((resv_ptr->users == NULL) && (resv_ptr->accounts == NULL)) {
2759 info("Reservation %s request lacks users or accounts",
2760 resv_desc_ptr->name);
2761 error_code = ESLURM_RESERVATION_EMPTY;
2762 goto update_failure;
2763 }
2764
2765 if (resv_desc_ptr->start_time != (time_t) NO_VAL) {
2766 if (resv_ptr->start_time <= time(NULL)) {
2767 info("%s: reservation already started", __func__);
2768 error_code = ESLURM_RSV_ALREADY_STARTED;
2769 goto update_failure;
2770 }
2771 if (resv_desc_ptr->start_time < (now - 60)) {
2772 info("Reservation %s request has invalid start time",
2773 resv_desc_ptr->name);
2774 error_code = ESLURM_INVALID_TIME_VALUE;
2775 goto update_failure;
2776 }
2777 resv_ptr->start_time_prev = resv_ptr->start_time;
2778 resv_ptr->start_time = resv_desc_ptr->start_time;
2779 resv_ptr->start_time_first = resv_desc_ptr->start_time;
2780 if (resv_ptr->duration != NO_VAL) {
2781 resv_ptr->end_time = resv_ptr->start_time_first +
2782 (resv_ptr->duration * 60);
2783 }
2784 }
2785 if (resv_desc_ptr->end_time != (time_t) NO_VAL) {
2786 if (resv_desc_ptr->end_time < (now - 60)) {
2787 info("Reservation %s request has invalid end time",
2788 resv_desc_ptr->name);
2789 error_code = ESLURM_INVALID_TIME_VALUE;
2790 goto update_failure;
2791 }
2792 resv_ptr->end_time = resv_desc_ptr->end_time;
2793 resv_ptr->duration = NO_VAL;
2794 }
2795
2796 if (resv_desc_ptr->duration == INFINITE) {
2797 resv_ptr->duration = YEAR_SECONDS / 60;
2798 resv_ptr->end_time = resv_ptr->start_time_first + YEAR_SECONDS;
2799 } else if (resv_desc_ptr->duration != NO_VAL) {
2800 if (resv_desc_ptr->flags == NO_VAL64)
2801 resv_ptr->duration = resv_desc_ptr->duration;
2802 else if (resv_desc_ptr->flags & RESERVE_FLAG_DUR_PLUS)
2803 resv_ptr->duration += resv_desc_ptr->duration;
2804 else if (resv_desc_ptr->flags & RESERVE_FLAG_DUR_MINUS) {
2805 if (resv_ptr->duration >= resv_desc_ptr->duration)
2806 resv_ptr->duration -= resv_desc_ptr->duration;
2807 else
2808 resv_ptr->duration = 0;
2809 } else
2810 resv_ptr->duration = resv_desc_ptr->duration;
2811
2812 resv_ptr->end_time = resv_ptr->start_time_first +
2813 (resv_ptr->duration * 60);
2814 /*
2815 * Since duration is a static number we could put the end time
2816 * in the past if the reservation already started and we are
2817 * removing more time than is left.
2818 */
2819 if (resv_ptr->end_time < now)
2820 resv_ptr->end_time = now;
2821 }
2822
2823 if (resv_ptr->start_time >= resv_ptr->end_time) {
2824 info("Reservation %s request has invalid times (start > end)",
2825 resv_desc_ptr->name);
2826 error_code = ESLURM_INVALID_TIME_VALUE;
2827 goto update_failure;
2828 }
2829 if (resv_desc_ptr->node_list &&
2830 (resv_desc_ptr->node_list[0] == '\0')) { /* Clear bitmap */
2831 resv_ptr->flags &= (~RESERVE_FLAG_SPEC_NODES);
2832 resv_ptr->flags &= (~RESERVE_FLAG_ALL_NODES);
2833 xfree(resv_desc_ptr->node_list);
2834 xfree(resv_ptr->node_list);
2835 FREE_NULL_BITMAP(resv_ptr->node_bitmap);
2836 FREE_NULL_BITMAP(resv_ptr->core_bitmap);
2837 free_job_resources(&resv_ptr->core_resrcs);
2838 resv_ptr->node_bitmap = bit_alloc(node_record_count);
2839 if ((resv_desc_ptr->node_cnt == NULL) ||
2840 (resv_desc_ptr->node_cnt[0] == 0)) {
2841 xrealloc(resv_desc_ptr->node_cnt, sizeof(uint32_t) * 2);
2842 resv_desc_ptr->node_cnt[0] = resv_ptr->node_cnt;
2843 resv_desc_ptr->node_cnt[1] = 0;
2844 }
2845 resv_ptr->node_cnt = 0;
2846 }
2847 if (resv_desc_ptr->node_list) { /* Change bitmap last */
2848 bitstr_t *node_bitmap;
2849 resv_ptr->flags |= RESERVE_FLAG_SPEC_NODES;
2850 if (xstrcasecmp(resv_desc_ptr->node_list, "ALL") == 0) {
2851 if ((resv_ptr->partition) &&
2852 (resv_ptr->flags & RESERVE_FLAG_PART_NODES)) {
2853 part_record_t *part_ptr = NULL;
2854 part_ptr = find_part_record(resv_ptr->
2855 partition);
2856 node_bitmap = bit_copy(part_ptr->node_bitmap);
2857 xfree(resv_ptr->node_list);
2858 xfree(resv_desc_ptr->node_list);
2859 resv_ptr->node_list = xstrdup(part_ptr->nodes);
2860 } else {
2861 resv_ptr->flags |= RESERVE_FLAG_ALL_NODES;
2862 node_bitmap = bit_alloc(node_record_count);
2863 bit_nset(node_bitmap, 0,(node_record_count-1));
2864 resv_ptr->flags &= (~RESERVE_FLAG_PART_NODES);
2865 xfree(resv_ptr->node_list);
2866 xfree(resv_desc_ptr->node_list);
2867 resv_ptr->node_list =
2868 bitmap2node_name(node_bitmap);
2869 }
2870 } else {
2871 resv_ptr->flags &= (~RESERVE_FLAG_PART_NODES);
2872 resv_ptr->flags &= (~RESERVE_FLAG_ALL_NODES);
2873 if (node_name2bitmap(resv_desc_ptr->node_list,
2874 false, &node_bitmap)) {
2875 info("Reservation %s request has invalid node name (%s)",
2876 resv_desc_ptr->name,
2877 resv_desc_ptr->node_list);
2878 error_code = ESLURM_INVALID_NODE_NAME;
2879 goto update_failure;
2880 }
2881 xfree(resv_desc_ptr->node_list);
2882 xfree(resv_ptr->node_list);
2883 resv_ptr->node_list = bitmap2node_name(node_bitmap);
2884 }
2885 resv_desc_ptr->node_list = NULL; /* Nothing left to free */
2886 FREE_NULL_BITMAP(resv_ptr->node_bitmap);
2887 FREE_NULL_BITMAP(resv_ptr->core_bitmap);
2888 free_job_resources(&resv_ptr->core_resrcs);
2889 resv_ptr->node_bitmap = node_bitmap;
2890 resv_ptr->node_cnt = bit_set_count(resv_ptr->node_bitmap);
2891 }
2892 if (resv_desc_ptr->node_cnt) {
2893 uint32_t total_node_cnt = 0;
2894 resv_ptr->flags &= (~RESERVE_FLAG_PART_NODES);
2895 resv_ptr->flags &= (~RESERVE_FLAG_ALL_NODES);
2896
2897 for (i = 0; resv_desc_ptr->node_cnt[i]; i++) {
2898 total_node_cnt += resv_desc_ptr->node_cnt[i];
2899 }
2900 rc = _resize_resv(resv_ptr, total_node_cnt);
2901 if (rc) {
2902 error_code = rc;
2903 goto update_failure;
2904 }
2905 /*
2906 * If the reservation was 0 node count before (ANY_NODES) this
2907 * could be NULL, if for some reason someone tried to update the
2908 * node count in this situation we will still not have a
2909 * node_bitmap.
2910 */
2911 if (resv_ptr->node_bitmap)
2912 resv_ptr->node_cnt =
2913 bit_set_count(resv_ptr->node_bitmap);
2914
2915 }
2916 memset(&resv_desc, 0, sizeof(resv_desc_msg_t));
2917 resv_desc.start_time = resv_ptr->start_time;
2918 resv_desc.end_time = resv_ptr->end_time;
2919 resv_desc.flags = resv_ptr->flags;
2920 if (_resv_overlap(&resv_desc, resv_ptr->node_bitmap, resv_ptr)) {
2921 info("Reservation %s request overlaps another",
2922 resv_desc_ptr->name);
2923 error_code = ESLURM_RESERVATION_OVERLAP;
2924 goto update_failure;
2925 }
2926 if (_job_overlap(resv_ptr->start_time, resv_ptr->flags,
2927 resv_ptr->node_bitmap, resv_desc_ptr->name)) {
2928 info("Reservation %s request overlaps jobs",
2929 resv_desc_ptr->name);
2930 error_code = ESLURM_NODES_BUSY;
2931 goto update_failure;
2932 }
2933
2934 /* This needs to be after checks for both account and user changes */
2935 if ((error_code = _set_assoc_list(resv_ptr)) != SLURM_SUCCESS)
2936 goto update_failure;
2937
2938 _set_tres_cnt(resv_ptr, resv_backup);
2939
2940 _del_resv_rec(resv_backup);
2941 (void) set_node_maint_mode(true);
2942 last_resv_update = now;
2943 schedule_resv_save();
2944 return error_code;
2945
2946 update_failure:
2947 /* Restore backup reservation data */
2948 _restore_resv(resv_ptr, resv_backup);
2949 _del_resv_rec(resv_backup);
2950 return error_code;
2951 }
2952
2953 /* Determine if a running or pending job is using a reservation */
_is_resv_used(slurmctld_resv_t * resv_ptr)2954 static bool _is_resv_used(slurmctld_resv_t *resv_ptr)
2955 {
2956 ListIterator job_iterator;
2957 job_record_t *job_ptr;
2958 bool match = false;
2959
2960 job_iterator = list_iterator_create(job_list);
2961 while ((job_ptr = list_next(job_iterator))) {
2962 if ((!IS_JOB_FINISHED(job_ptr)) &&
2963 (job_ptr->resv_id == resv_ptr->resv_id)) {
2964 match = true;
2965 break;
2966 }
2967 }
2968 list_iterator_destroy(job_iterator);
2969
2970 return match;
2971 }
2972
2973 /* Clear the reservation pointers for jobs referencing a defunct reservation */
_clear_job_resv(slurmctld_resv_t * resv_ptr)2974 static void _clear_job_resv(slurmctld_resv_t *resv_ptr)
2975 {
2976 ListIterator job_iterator;
2977 job_record_t *job_ptr;
2978
2979 job_iterator = list_iterator_create(job_list);
2980 while ((job_ptr = list_next(job_iterator))) {
2981 if ((resv_ptr->flags & RESERVE_FLAG_MAINT) &&
2982 (job_ptr->state_reason == WAIT_NODE_NOT_AVAIL) &&
2983 !xstrcmp(job_ptr->state_desc,
2984 "ReqNodeNotAvail, Reserved for maintenance")) {
2985 /*
2986 * In case of cluster maintenance many jobs may get this
2987 * state set. If we wait for scheduler to update
2988 * the reason it may take long time after the
2989 * reservation completion. Instead of that clear it
2990 * when MAIN reservation ends.
2991 */
2992 job_ptr->state_reason = WAIT_NO_REASON;
2993 xfree(job_ptr->state_desc);
2994 }
2995
2996 if (job_ptr->resv_ptr != resv_ptr)
2997 continue;
2998 if (!IS_JOB_FINISHED(job_ptr)) {
2999 info("%pJ linked to defunct reservation %s, clearing that reservation",
3000 job_ptr, job_ptr->resv_name);
3001 }
3002 job_ptr->resv_id = 0;
3003 job_ptr->resv_ptr = NULL;
3004 xfree(job_ptr->resv_name);
3005 if (!(resv_ptr->flags & RESERVE_FLAG_NO_HOLD_JOBS) &&
3006 IS_JOB_PENDING(job_ptr) &&
3007 (job_ptr->state_reason != WAIT_HELD)) {
3008 xfree(job_ptr->state_desc);
3009 job_ptr->state_reason = WAIT_RESV_DELETED;
3010 job_ptr->job_state |= JOB_RESV_DEL_HOLD;
3011 xstrfmtcat(job_ptr->state_desc,
3012 "Reservation %s was deleted",
3013 resv_ptr->name);
3014 debug("%s: Holding %pJ, reservation %s was deleted",
3015 __func__, job_ptr, resv_ptr->name);
3016 job_ptr->priority = 0; /* Hold job */
3017 }
3018 }
3019 list_iterator_destroy(job_iterator);
3020 }
3021
_match_user_assoc(char * assoc_str,List assoc_list,bool deny)3022 static bool _match_user_assoc(char *assoc_str, List assoc_list, bool deny)
3023 {
3024 ListIterator itr;
3025 bool found = 0;
3026 slurmdb_assoc_rec_t *assoc;
3027 char tmp_char[1000];
3028
3029 if (!assoc_str || !assoc_list || !list_count(assoc_list))
3030 return false;
3031
3032 itr = list_iterator_create(assoc_list);
3033 while ((assoc = list_next(itr))) {
3034 while (assoc) {
3035 snprintf(tmp_char, sizeof(tmp_char), ",%s%u,",
3036 deny ? "-" : "", assoc->id);
3037 if (strstr(assoc_str, tmp_char)) {
3038 found = 1;
3039 goto end_it;
3040 }
3041 assoc = assoc->usage->parent_assoc_ptr;
3042 }
3043 }
3044 end_it:
3045 list_iterator_destroy(itr);
3046
3047 return found;
3048 }
3049
3050 /* Delete an exiting resource reservation */
delete_resv(reservation_name_msg_t * resv_desc_ptr)3051 extern int delete_resv(reservation_name_msg_t *resv_desc_ptr)
3052 {
3053 ListIterator iter;
3054 slurmctld_resv_t *resv_ptr;
3055 int rc = SLURM_SUCCESS;
3056 time_t now = time(NULL);
3057
3058 if (slurmctld_conf.debug_flags & DEBUG_FLAG_RESERVATION)
3059 info("delete_resv: Name=%s", resv_desc_ptr->name);
3060
3061 iter = list_iterator_create(resv_list);
3062 while ((resv_ptr = list_next(iter))) {
3063 if (xstrcmp(resv_ptr->name, resv_desc_ptr->name))
3064 continue;
3065 if (_is_resv_used(resv_ptr)) {
3066 rc = ESLURM_RESERVATION_BUSY;
3067 break;
3068 }
3069
3070 if (resv_ptr->flags_set_node) {
3071 resv_ptr->flags_set_node = false;
3072 _set_nodes_flags(resv_ptr, now,
3073 (NODE_STATE_RES | NODE_STATE_MAINT),
3074 false);
3075 last_node_update = now;
3076 }
3077
3078 rc = _post_resv_delete(resv_ptr);
3079 _clear_job_resv(resv_ptr);
3080 list_delete_item(iter);
3081 break;
3082 }
3083 list_iterator_destroy(iter);
3084
3085 if (!resv_ptr) {
3086 info("Reservation %s not found for deletion",
3087 resv_desc_ptr->name);
3088 return ESLURM_RESERVATION_INVALID;
3089 }
3090
3091 (void) set_node_maint_mode(true);
3092 last_resv_update = time(NULL);
3093 schedule_resv_save();
3094 return rc;
3095 }
3096
3097 /* Return pointer to the named reservation or NULL if not found */
find_resv_name(char * resv_name)3098 extern slurmctld_resv_t *find_resv_name(char *resv_name)
3099 {
3100 slurmctld_resv_t *resv_ptr;
3101 resv_ptr = (slurmctld_resv_t *) list_find_first (resv_list,
3102 _find_resv_name, resv_name);
3103 return resv_ptr;
3104 }
3105
3106 /* Dump the reservation records to a buffer */
show_resv(char ** buffer_ptr,int * buffer_size,uid_t uid,uint16_t protocol_version)3107 extern void show_resv(char **buffer_ptr, int *buffer_size, uid_t uid,
3108 uint16_t protocol_version)
3109 {
3110 ListIterator iter;
3111 slurmctld_resv_t *resv_ptr;
3112 uint32_t resv_packed;
3113 int tmp_offset;
3114 Buf buffer;
3115 time_t now = time(NULL);
3116 List assoc_list = NULL;
3117 bool check_permissions = false;
3118 assoc_mgr_lock_t locks = { .assoc = READ_LOCK };
3119
3120 DEF_TIMERS;
3121
3122 START_TIMER;
3123 _create_resv_lists(false);
3124
3125 buffer_ptr[0] = NULL;
3126 *buffer_size = 0;
3127
3128 buffer = init_buf(BUF_SIZE);
3129
3130 /* write header: version and time */
3131 resv_packed = 0;
3132 pack32(resv_packed, buffer);
3133 pack_time(now, buffer);
3134
3135 /* Create this list once since it will not change during this call. */
3136 if ((slurmctld_conf.private_data & PRIVATE_DATA_RESERVATIONS)
3137 && !validate_operator(uid)) {
3138 slurmdb_assoc_rec_t assoc;
3139
3140 check_permissions = true;
3141
3142 memset(&assoc, 0, sizeof(slurmdb_assoc_rec_t));
3143 assoc.uid = uid;
3144
3145 assoc_list = list_create(NULL);
3146
3147 assoc_mgr_lock(&locks);
3148 if (assoc_mgr_get_user_assocs(acct_db_conn, &assoc,
3149 accounting_enforce, assoc_list)
3150 != SLURM_SUCCESS)
3151 goto no_assocs;
3152 }
3153
3154 /* write individual reservation records */
3155 iter = list_iterator_create(resv_list);
3156 while ((resv_ptr = list_next(iter))) {
3157 if (check_permissions) {
3158 /* Determine if we have access */
3159 if ((accounting_enforce & ACCOUNTING_ENFORCE_ASSOCS)
3160 && resv_ptr->assoc_list) {
3161 /* Check to see if the association is
3162 * here or the parent association is
3163 * listed in the valid associations.
3164 */
3165 if (strchr(resv_ptr->assoc_list, '-')) {
3166 if (_match_user_assoc(
3167 resv_ptr->assoc_list,
3168 assoc_list,
3169 true))
3170 continue;
3171 }
3172
3173 if (strstr(resv_ptr->assoc_list, ",1") ||
3174 strstr(resv_ptr->assoc_list, ",2") ||
3175 strstr(resv_ptr->assoc_list, ",3") ||
3176 strstr(resv_ptr->assoc_list, ",4") ||
3177 strstr(resv_ptr->assoc_list, ",5") ||
3178 strstr(resv_ptr->assoc_list, ",6") ||
3179 strstr(resv_ptr->assoc_list, ",7") ||
3180 strstr(resv_ptr->assoc_list, ",8") ||
3181 strstr(resv_ptr->assoc_list, ",9") ||
3182 strstr(resv_ptr->assoc_list, ",0")) {
3183 if (!_match_user_assoc(
3184 resv_ptr->assoc_list,
3185 assoc_list, false))
3186 continue;
3187 }
3188 } else {
3189 int i = 0;
3190 for (i = 0; i < resv_ptr->user_cnt; i++) {
3191 if (resv_ptr->user_list[i] == uid)
3192 break;
3193 }
3194
3195 if (i >= resv_ptr->user_cnt)
3196 continue;
3197 }
3198 }
3199
3200 _pack_resv(resv_ptr, buffer, false, protocol_version);
3201 resv_packed++;
3202 }
3203 list_iterator_destroy(iter);
3204
3205 no_assocs:
3206 if (check_permissions) {
3207 FREE_NULL_LIST(assoc_list);
3208 assoc_mgr_unlock(&locks);
3209 }
3210
3211 /* put the real record count in the message body header */
3212 tmp_offset = get_buf_offset(buffer);
3213 set_buf_offset(buffer, 0);
3214 pack32(resv_packed, buffer);
3215 set_buf_offset(buffer, tmp_offset);
3216
3217 *buffer_size = get_buf_offset(buffer);
3218 buffer_ptr[0] = xfer_buf_data(buffer);
3219 END_TIMER2("show_resv");
3220 }
3221
3222 /* Save the state of all reservations to file */
dump_all_resv_state(void)3223 extern int dump_all_resv_state(void)
3224 {
3225 ListIterator iter;
3226 slurmctld_resv_t *resv_ptr;
3227 int error_code = 0, log_fd;
3228 char *old_file, *new_file, *reg_file;
3229 /* Locks: Read node */
3230 slurmctld_lock_t resv_read_lock =
3231 { READ_LOCK, NO_LOCK, READ_LOCK, NO_LOCK, NO_LOCK };
3232 Buf buffer = init_buf(BUF_SIZE);
3233 DEF_TIMERS;
3234
3235 START_TIMER;
3236 _create_resv_lists(false);
3237
3238 /* write header: time */
3239 packstr(RESV_STATE_VERSION, buffer);
3240 pack16(SLURM_PROTOCOL_VERSION, buffer);
3241 pack_time(time(NULL), buffer);
3242 pack32(top_suffix, buffer);
3243
3244 /* write reservation records to buffer */
3245 lock_slurmctld(resv_read_lock);
3246 iter = list_iterator_create(resv_list);
3247 while ((resv_ptr = list_next(iter)))
3248 _pack_resv(resv_ptr, buffer, true, SLURM_PROTOCOL_VERSION);
3249 list_iterator_destroy(iter);
3250
3251 old_file = xstrdup(slurmctld_conf.state_save_location);
3252 xstrcat(old_file, "/resv_state.old");
3253 reg_file = xstrdup(slurmctld_conf.state_save_location);
3254 xstrcat(reg_file, "/resv_state");
3255 new_file = xstrdup(slurmctld_conf.state_save_location);
3256 xstrcat(new_file, "/resv_state.new");
3257 unlock_slurmctld(resv_read_lock);
3258
3259 /* write the buffer to file */
3260 lock_state_files();
3261 log_fd = creat(new_file, 0600);
3262 if (log_fd < 0) {
3263 error("Can't save state, error creating file %s, %m",
3264 new_file);
3265 error_code = errno;
3266 } else {
3267 int pos = 0, nwrite = get_buf_offset(buffer), amount, rc;
3268 char *data = (char *)get_buf_data(buffer);
3269
3270 while (nwrite > 0) {
3271 amount = write(log_fd, &data[pos], nwrite);
3272 if ((amount < 0) && (errno != EINTR)) {
3273 error("Error writing file %s, %m", new_file);
3274 error_code = errno;
3275 break;
3276 }
3277 nwrite -= amount;
3278 pos += amount;
3279 }
3280 rc = fsync_and_close(log_fd, "reservation");
3281 if (rc && !error_code)
3282 error_code = rc;
3283 }
3284 if (error_code)
3285 (void) unlink(new_file);
3286 else { /* file shuffle */
3287 (void) unlink(old_file);
3288 if (link(reg_file, old_file))
3289 debug4("unable to create link for %s -> %s: %m",
3290 reg_file, old_file);
3291 (void) unlink(reg_file);
3292 if (link(new_file, reg_file))
3293 debug4("unable to create link for %s -> %s: %m",
3294 new_file, reg_file);
3295 (void) unlink(new_file);
3296 }
3297 xfree(old_file);
3298 xfree(reg_file);
3299 xfree(new_file);
3300 unlock_state_files();
3301
3302 free_buf(buffer);
3303 END_TIMER2("dump_all_resv_state");
3304 return 0;
3305 }
3306
3307 /* Validate one reservation record, return true if good */
_validate_one_reservation(slurmctld_resv_t * resv_ptr)3308 static bool _validate_one_reservation(slurmctld_resv_t *resv_ptr)
3309 {
3310 bool account_not = false, user_not = false;
3311 slurmctld_resv_t old_resv_ptr;
3312
3313 if ((resv_ptr->name == NULL) || (resv_ptr->name[0] == '\0')) {
3314 error("Read reservation without name");
3315 return false;
3316 }
3317 if (_get_core_resrcs(resv_ptr) != SLURM_SUCCESS)
3318 return false;
3319 if (resv_ptr->partition) {
3320 part_record_t *part_ptr = NULL;
3321 part_ptr = find_part_record(resv_ptr->partition);
3322 if (!part_ptr) {
3323 error("Reservation %s has invalid partition (%s)",
3324 resv_ptr->name, resv_ptr->partition);
3325 return false;
3326 }
3327 resv_ptr->part_ptr = part_ptr;
3328 }
3329 if (resv_ptr->accounts) {
3330 int account_cnt = 0, i, rc;
3331 char **account_list;
3332 rc = _build_account_list(resv_ptr->accounts,
3333 &account_cnt, &account_list,
3334 &account_not);
3335 if (rc) {
3336 error("Reservation %s has invalid accounts (%s)",
3337 resv_ptr->name, resv_ptr->accounts);
3338 return false;
3339 }
3340 for (i=0; i<resv_ptr->account_cnt; i++)
3341 xfree(resv_ptr->account_list[i]);
3342 xfree(resv_ptr->account_list);
3343 resv_ptr->account_cnt = account_cnt;
3344 resv_ptr->account_list = account_list;
3345 resv_ptr->account_not = account_not;
3346 }
3347 if (resv_ptr->licenses) {
3348 bool valid = true;
3349 FREE_NULL_LIST(resv_ptr->license_list);
3350 resv_ptr->license_list = license_validate(resv_ptr->licenses,
3351 true, true, NULL,
3352 &valid);
3353 if (!valid) {
3354 error("Reservation %s has invalid licenses (%s)",
3355 resv_ptr->name, resv_ptr->licenses);
3356 return false;
3357 }
3358 }
3359 if (resv_ptr->users) {
3360 int rc, user_cnt = 0;
3361 uid_t *user_list = NULL;
3362 rc = _build_uid_list(resv_ptr->users,
3363 &user_cnt, &user_list, &user_not);
3364 if (rc) {
3365 error("Reservation %s has invalid users (%s)",
3366 resv_ptr->name, resv_ptr->users);
3367 return false;
3368 }
3369 xfree(resv_ptr->user_list);
3370 resv_ptr->user_cnt = user_cnt;
3371 resv_ptr->user_list = user_list;
3372 resv_ptr->user_not = user_not;
3373 }
3374
3375 if ((resv_ptr->flags & RESERVE_FLAG_PART_NODES) &&
3376 resv_ptr->part_ptr && resv_ptr->part_ptr->node_bitmap) {
3377 memset(&old_resv_ptr, 0, sizeof(slurmctld_resv_t));
3378 old_resv_ptr.assoc_list = resv_ptr->assoc_list;
3379 old_resv_ptr.flags = resv_ptr->flags;
3380 old_resv_ptr.node_list = resv_ptr->node_list;
3381 resv_ptr->node_list = NULL;
3382 resv_ptr->node_list = xstrdup(resv_ptr->part_ptr->nodes);
3383 FREE_NULL_BITMAP(resv_ptr->node_bitmap);
3384 resv_ptr->node_bitmap = bit_copy(resv_ptr->part_ptr->
3385 node_bitmap);
3386 resv_ptr->node_cnt = bit_set_count(resv_ptr->node_bitmap);
3387 old_resv_ptr.tres_str = resv_ptr->tres_str;
3388 resv_ptr->tres_str = NULL;
3389 _set_tres_cnt(resv_ptr, &old_resv_ptr);
3390 old_resv_ptr.assoc_list = NULL;
3391 xfree(old_resv_ptr.tres_str);
3392 xfree(old_resv_ptr.node_list);
3393 last_resv_update = time(NULL);
3394 } else if (resv_ptr->flags & RESERVE_FLAG_ALL_NODES) {
3395 memset(&old_resv_ptr, 0, sizeof(slurmctld_resv_t));
3396 old_resv_ptr.assoc_list = resv_ptr->assoc_list;
3397 old_resv_ptr.flags = resv_ptr->flags;
3398 old_resv_ptr.node_list = resv_ptr->node_list;
3399 resv_ptr->node_list = NULL;
3400 FREE_NULL_BITMAP(resv_ptr->node_bitmap);
3401 resv_ptr->node_bitmap = bit_alloc(node_record_count);
3402 bit_nset(resv_ptr->node_bitmap, 0, (node_record_count - 1));
3403 resv_ptr->node_list = bitmap2node_name(resv_ptr->node_bitmap);
3404 resv_ptr->node_cnt = bit_set_count(resv_ptr->node_bitmap);
3405 old_resv_ptr.tres_str = resv_ptr->tres_str;
3406 resv_ptr->tres_str = NULL;
3407 _set_tres_cnt(resv_ptr, &old_resv_ptr);
3408 old_resv_ptr.assoc_list = NULL;
3409 xfree(old_resv_ptr.tres_str);
3410 xfree(old_resv_ptr.node_list);
3411 last_resv_update = time(NULL);
3412 } else if (resv_ptr->node_list) { /* Change bitmap last */
3413 /*
3414 * Node bitmap must be recreated in any case, i.e. when
3415 * they grow because adding new nodes to slurm.conf
3416 */
3417 FREE_NULL_BITMAP(resv_ptr->node_bitmap);
3418 if (node_name2bitmap(resv_ptr->node_list, false,
3419 &resv_ptr->node_bitmap)) {
3420 char *new_node_list;
3421 resv_ptr->node_cnt = bit_set_count(
3422 resv_ptr->node_bitmap);
3423 if (!resv_ptr->node_cnt) {
3424 error("Reservation %s has no nodes left, deleting it",
3425 resv_ptr->name);
3426 return false;
3427 }
3428 memset(&old_resv_ptr, 0, sizeof(slurmctld_resv_t));
3429 old_resv_ptr.assoc_list = resv_ptr->assoc_list;
3430 old_resv_ptr.flags = resv_ptr->flags;
3431 old_resv_ptr.node_list = resv_ptr->node_list;
3432 resv_ptr->node_list = NULL;
3433 new_node_list = bitmap2node_name(resv_ptr->node_bitmap);
3434 info("Reservation %s has invalid nodes (%s), shrinking to (%s)",
3435 resv_ptr->name, resv_ptr->node_list,
3436 new_node_list);
3437 resv_ptr->node_list = new_node_list;
3438 new_node_list = NULL;
3439 old_resv_ptr.tres_str = resv_ptr->tres_str;
3440 resv_ptr->tres_str = NULL;
3441 _set_tres_cnt(resv_ptr, &old_resv_ptr);
3442 old_resv_ptr.assoc_list = NULL;
3443 xfree(old_resv_ptr.tres_str);
3444 xfree(old_resv_ptr.node_list);
3445 last_resv_update = time(NULL);
3446 schedule_resv_save();
3447 }
3448 }
3449
3450 return true;
3451 }
3452
3453 /*
3454 * Validate all reservation records, reset bitmaps, etc.
3455 * Purge any invalid reservation.
3456 */
_validate_all_reservations(void)3457 static void _validate_all_reservations(void)
3458 {
3459 ListIterator iter;
3460 slurmctld_resv_t *resv_ptr;
3461 job_record_t *job_ptr;
3462
3463 iter = list_iterator_create(resv_list);
3464 while ((resv_ptr = list_next(iter))) {
3465 if (!_validate_one_reservation(resv_ptr)) {
3466 error("Purging invalid reservation record %s",
3467 resv_ptr->name);
3468 _post_resv_delete(resv_ptr);
3469 _clear_job_resv(resv_ptr);
3470 list_delete_item(iter);
3471 } else {
3472 _set_assoc_list(resv_ptr);
3473 top_suffix = MAX(top_suffix, resv_ptr->resv_id);
3474 }
3475 }
3476 list_iterator_destroy(iter);
3477
3478 /* Validate all job reservation pointers */
3479 iter = list_iterator_create(job_list);
3480 while ((job_ptr = list_next(iter))) {
3481 if (job_ptr->resv_name == NULL)
3482 continue;
3483
3484 if ((job_ptr->resv_ptr == NULL) ||
3485 (job_ptr->resv_ptr->magic != RESV_MAGIC)) {
3486 job_ptr->resv_ptr = (slurmctld_resv_t *)
3487 list_find_first(resv_list,
3488 _find_resv_name,
3489 job_ptr->resv_name);
3490 }
3491 if (!job_ptr->resv_ptr) {
3492 error("%pJ linked to defunct reservation %s",
3493 job_ptr, job_ptr->resv_name);
3494 job_ptr->resv_id = 0;
3495 xfree(job_ptr->resv_name);
3496 }
3497 }
3498 list_iterator_destroy(iter);
3499
3500 }
3501
3502 /*
3503 * Replace DOWN, DRAIN or ALLOCATED nodes for reservations with "replace" flag
3504 */
_resv_node_replace(slurmctld_resv_t * resv_ptr)3505 static void _resv_node_replace(slurmctld_resv_t *resv_ptr)
3506 {
3507 bitstr_t *preserve_bitmap = NULL;
3508 bitstr_t *core_bitmap = NULL, *new_bitmap = NULL, *tmp_bitmap = NULL;
3509 resv_desc_msg_t resv_desc;
3510 int i, add_nodes, new_nodes, preserve_nodes, busy_nodes_needed;
3511 bool log_it = true;
3512
3513 /* Identify nodes which can be preserved in this reservation */
3514 preserve_bitmap = bit_copy(resv_ptr->node_bitmap);
3515 bit_and(preserve_bitmap, avail_node_bitmap);
3516 if (resv_ptr->flags & RESERVE_FLAG_REPLACE)
3517 bit_and(preserve_bitmap, idle_node_bitmap);
3518 preserve_nodes = bit_set_count(preserve_bitmap);
3519
3520 /*
3521 * Try to get replacement nodes, first from idle pool then re-use
3522 * busy nodes in the current reservation as needed
3523 */
3524 add_nodes = resv_ptr->node_cnt - preserve_nodes;
3525 while (add_nodes) {
3526 memset(&resv_desc, 0, sizeof(resv_desc_msg_t));
3527 resv_desc.start_time = resv_ptr->start_time;
3528 resv_desc.end_time = resv_ptr->end_time;
3529 resv_desc.features = resv_ptr->features;
3530 resv_desc.flags = resv_ptr->flags;
3531 if (!resv_ptr->full_nodes) {
3532 resv_desc.core_cnt = xcalloc(2, sizeof(uint32_t));
3533 resv_desc.core_cnt[0] = resv_ptr->core_cnt;
3534 }
3535 resv_desc.node_cnt = xcalloc(2, sizeof(uint32_t));
3536 resv_desc.node_cnt[0] = add_nodes;
3537 i = _select_nodes(&resv_desc, &resv_ptr->part_ptr, &new_bitmap,
3538 &core_bitmap);
3539 xfree(resv_desc.core_cnt);
3540 xfree(resv_desc.node_cnt);
3541 xfree(resv_desc.node_list);
3542 xfree(resv_desc.partition);
3543 if (i == SLURM_SUCCESS) {
3544 new_nodes = bit_set_count(new_bitmap);
3545 busy_nodes_needed = resv_ptr->node_cnt - new_nodes
3546 - preserve_nodes;
3547 if (busy_nodes_needed > 0) {
3548 bit_and_not(resv_ptr->node_bitmap, preserve_bitmap);
3549 tmp_bitmap = bit_pick_cnt(resv_ptr->node_bitmap,
3550 busy_nodes_needed);
3551 bit_and(resv_ptr->node_bitmap, tmp_bitmap);
3552 FREE_NULL_BITMAP(tmp_bitmap);
3553 bit_or(resv_ptr->node_bitmap, preserve_bitmap);
3554 } else {
3555 bit_and(resv_ptr->node_bitmap, preserve_bitmap);
3556 }
3557 bit_or(resv_ptr->node_bitmap, new_bitmap);
3558 FREE_NULL_BITMAP(new_bitmap);
3559 FREE_NULL_BITMAP(resv_ptr->core_bitmap);
3560 resv_ptr->core_bitmap = core_bitmap; /* is NULL */
3561 free_job_resources(&resv_ptr->core_resrcs);
3562 xfree(resv_ptr->node_list);
3563 resv_ptr->node_list = bitmap2node_name(resv_ptr->
3564 node_bitmap);
3565 info("modified reservation %s with replacement "
3566 "nodes, new nodes: %s",
3567 resv_ptr->name, resv_ptr->node_list);
3568 break;
3569 }
3570 add_nodes /= 2; /* Try to get idle nodes as possible */
3571 if (log_it) {
3572 info("unable to replace all allocated nodes in "
3573 "reservation %s at this time", resv_ptr->name);
3574 log_it = false;
3575 }
3576 }
3577 FREE_NULL_BITMAP(preserve_bitmap);
3578 last_resv_update = time(NULL);
3579 schedule_resv_save();
3580 }
3581
3582 /*
3583 * Replace DOWN or DRAINED in an advanced reservation, also replaces nodes
3584 * in use for reservations with the "replace" flag.
3585 */
_validate_node_choice(slurmctld_resv_t * resv_ptr)3586 static void _validate_node_choice(slurmctld_resv_t *resv_ptr)
3587 {
3588 bitstr_t *tmp_bitmap = NULL;
3589 bitstr_t *core_bitmap = NULL;
3590 int i;
3591 resv_desc_msg_t resv_desc;
3592
3593 if ((resv_ptr->node_bitmap == NULL) ||
3594 (!resv_ptr->full_nodes && (resv_ptr->node_cnt > 1)) ||
3595 (resv_ptr->flags & RESERVE_FLAG_SPEC_NODES) ||
3596 (resv_ptr->flags & RESERVE_FLAG_STATIC))
3597 return;
3598
3599 if ((resv_ptr->flags & RESERVE_FLAG_REPLACE) ||
3600 (resv_ptr->flags & RESERVE_FLAG_REPLACE_DOWN)) {
3601 _resv_node_replace(resv_ptr);
3602 return;
3603 }
3604
3605 i = bit_overlap(resv_ptr->node_bitmap, avail_node_bitmap);
3606 if (i == resv_ptr->node_cnt) {
3607 return;
3608 }
3609
3610 /* Reservation includes DOWN, DRAINED/DRAINING, FAILING or
3611 * NO_RESPOND nodes. Generate new request using _select_nodes()
3612 * in attempt to replace these nodes */
3613 memset(&resv_desc, 0, sizeof(resv_desc_msg_t));
3614 resv_desc.start_time = resv_ptr->start_time;
3615 resv_desc.end_time = resv_ptr->end_time;
3616 resv_desc.features = resv_ptr->features;
3617 resv_desc.flags = resv_ptr->flags;
3618 if (!resv_ptr->full_nodes) {
3619 resv_desc.core_cnt = xcalloc(2, sizeof(uint32_t));
3620 resv_desc.core_cnt[0] = resv_ptr->core_cnt;
3621 }
3622 resv_desc.node_cnt = xcalloc(2, sizeof(uint32_t));
3623 resv_desc.node_cnt[0]= resv_ptr->node_cnt - i;
3624 i = _select_nodes(&resv_desc, &resv_ptr->part_ptr, &tmp_bitmap,
3625 &core_bitmap);
3626 xfree(resv_desc.core_cnt);
3627 xfree(resv_desc.node_cnt);
3628 xfree(resv_desc.node_list);
3629 xfree(resv_desc.partition);
3630 if (i == SLURM_SUCCESS) {
3631 bit_and(resv_ptr->node_bitmap, avail_node_bitmap);
3632 bit_or(resv_ptr->node_bitmap, tmp_bitmap);
3633 FREE_NULL_BITMAP(tmp_bitmap);
3634 FREE_NULL_BITMAP(resv_ptr->core_bitmap);
3635 resv_ptr->core_bitmap = core_bitmap;
3636 free_job_resources(&resv_ptr->core_resrcs);
3637 xfree(resv_ptr->node_list);
3638 resv_ptr->node_list = bitmap2node_name(resv_ptr->node_bitmap);
3639 info("modified reservation %s due to unusable nodes, "
3640 "new nodes: %s", resv_ptr->name, resv_ptr->node_list);
3641 } else if (difftime(resv_ptr->start_time, time(NULL)) < 600) {
3642 info("reservation %s contains unusable nodes, "
3643 "can't reallocate now", resv_ptr->name);
3644 } else {
3645 debug("reservation %s contains unusable nodes, "
3646 "can't reallocate now", resv_ptr->name);
3647 }
3648 }
3649
3650 /* Open the reservation state save file, or backup if necessary.
3651 * state_file IN - the name of the state save file used
3652 * RET the file description to read from or error code
3653 */
_open_resv_state_file(char ** state_file)3654 static Buf _open_resv_state_file(char **state_file)
3655 {
3656 Buf buf;
3657
3658 *state_file = xstrdup(slurmctld_conf.state_save_location);
3659 xstrcat(*state_file, "/resv_state");
3660 if (!(buf = create_mmap_buf(*state_file)))
3661 error("Could not open reservation state file %s: %m",
3662 *state_file);
3663 else
3664 return buf;
3665
3666 error("NOTE: Trying backup state save file. Reservations may be lost");
3667 xstrcat(*state_file, ".old");
3668 return create_mmap_buf(*state_file);
3669 }
3670
3671 /*
3672 * Load the reservation state from file, recover on slurmctld restart.
3673 * Reset reservation pointers for all jobs.
3674 * Execute this after loading the configuration file data.
3675 * IN recover - 0 = validate current reservations ONLY if already recovered,
3676 * otherwise recover from disk
3677 * 1+ = recover all reservation state from disk
3678 * RET SLURM_SUCCESS or error code
3679 * NOTE: READ lock_slurmctld config before entry
3680 */
load_all_resv_state(int recover)3681 extern int load_all_resv_state(int recover)
3682 {
3683 char *state_file, *ver_str = NULL;
3684 time_t now;
3685 uint32_t uint32_tmp;
3686 int error_code = 0;
3687 Buf buffer;
3688 slurmctld_resv_t *resv_ptr = NULL;
3689 uint16_t protocol_version = NO_VAL16;
3690
3691 last_resv_update = time(NULL);
3692 if ((recover == 0) && resv_list) {
3693 _validate_all_reservations();
3694 return SLURM_SUCCESS;
3695 }
3696
3697 /* Read state file and validate */
3698 _create_resv_lists(true);
3699
3700 /* read the file */
3701 lock_state_files();
3702 if (!(buffer = _open_resv_state_file(&state_file))) {
3703 info("No reservation state file (%s) to recover",
3704 state_file);
3705 xfree(state_file);
3706 unlock_state_files();
3707 return ENOENT;
3708 }
3709 xfree(state_file);
3710 unlock_state_files();
3711
3712 safe_unpackstr_xmalloc( &ver_str, &uint32_tmp, buffer);
3713 debug3("Version string in resv_state header is %s", ver_str);
3714 if (ver_str && !xstrcmp(ver_str, RESV_STATE_VERSION))
3715 safe_unpack16(&protocol_version, buffer);
3716
3717 if (protocol_version == NO_VAL16) {
3718 if (!ignore_state_errors)
3719 fatal("Can not recover reservation state, data version incompatible, start with '-i' to ignore this. Warning: using -i will lose the data that can't be recovered.");
3720 error("************************************************************");
3721 error("Can not recover reservation state, data version incompatible");
3722 error("************************************************************");
3723 xfree(ver_str);
3724 free_buf(buffer);
3725 schedule_resv_save(); /* Schedule save with new format */
3726 return EFAULT;
3727 }
3728 xfree(ver_str);
3729 safe_unpack_time(&now, buffer);
3730 safe_unpack32(&top_suffix, buffer);
3731
3732 while (remaining_buf(buffer) > 0) {
3733 resv_ptr = _load_reservation_state(buffer, protocol_version);
3734 if (!resv_ptr)
3735 break;
3736
3737 _add_resv_to_lists(resv_ptr);
3738 info("Recovered state of reservation %s", resv_ptr->name);
3739 }
3740
3741 _validate_all_reservations();
3742 info("Recovered state of %d reservations", list_count(resv_list));
3743 free_buf(buffer);
3744 return error_code;
3745
3746 unpack_error:
3747 if (!ignore_state_errors)
3748 fatal("Incomplete reservation data checkpoint file, start with '-i' to ignore this. Warning: using -i will lose the data that can't be recovered.");
3749 error("Incomplete reservation data checkpoint file");
3750 _validate_all_reservations();
3751 info("Recovered state of %d reservations", list_count(resv_list));
3752 free_buf(buffer);
3753 return EFAULT;
3754 }
3755
3756 /*
3757 * Determine if a job request can use the specified reservations
3758 *
3759 * IN/OUT job_ptr - job to validate, set its resv_id
3760 * RET SLURM_SUCCESS or error code (not found or access denied)
3761 */
validate_job_resv(job_record_t * job_ptr)3762 extern int validate_job_resv(job_record_t *job_ptr)
3763 {
3764 slurmctld_resv_t *resv_ptr = NULL;
3765 int rc;
3766
3767 xassert(job_ptr);
3768
3769 if ((job_ptr->resv_name == NULL) || (job_ptr->resv_name[0] == '\0')) {
3770 xfree(job_ptr->resv_name);
3771 job_ptr->resv_id = 0;
3772 job_ptr->resv_ptr = NULL;
3773 return SLURM_SUCCESS;
3774 }
3775
3776 if (!resv_list)
3777 return ESLURM_RESERVATION_INVALID;
3778
3779 /* Find the named reservation */
3780 resv_ptr = (slurmctld_resv_t *) list_find_first (resv_list,
3781 _find_resv_name, job_ptr->resv_name);
3782 rc = _valid_job_access_resv(job_ptr, resv_ptr);
3783 if (rc == SLURM_SUCCESS) {
3784 job_ptr->resv_id = resv_ptr->resv_id;
3785 job_ptr->resv_ptr = resv_ptr;
3786 resv_ptr->idle_start_time = 0;
3787 _validate_node_choice(resv_ptr);
3788 }
3789 return rc;
3790 }
3791
_resize_resv(slurmctld_resv_t * resv_ptr,uint32_t node_cnt)3792 static int _resize_resv(slurmctld_resv_t *resv_ptr, uint32_t node_cnt)
3793 {
3794 bitstr_t *tmp1_bitmap = NULL, *tmp2_bitmap = NULL;
3795 bitstr_t *core_bitmap = NULL;
3796 int delta_node_cnt, i;
3797 resv_desc_msg_t resv_desc;
3798
3799 delta_node_cnt = resv_ptr->node_cnt - node_cnt;
3800 if (delta_node_cnt == 0) /* Already correct node count */
3801 return SLURM_SUCCESS;
3802
3803 if (delta_node_cnt > 0) { /* Must decrease node count */
3804 if (bit_overlap_any(resv_ptr->node_bitmap, idle_node_bitmap)) {
3805 /* Start by eliminating idle nodes from reservation */
3806 tmp1_bitmap = bit_copy(resv_ptr->node_bitmap);
3807 bit_and(tmp1_bitmap, idle_node_bitmap);
3808 i = bit_set_count(tmp1_bitmap);
3809 if (i > delta_node_cnt) {
3810 tmp2_bitmap = bit_pick_cnt(tmp1_bitmap,
3811 delta_node_cnt);
3812 bit_and_not(resv_ptr->node_bitmap, tmp2_bitmap);
3813 FREE_NULL_BITMAP(tmp1_bitmap);
3814 FREE_NULL_BITMAP(tmp2_bitmap);
3815 delta_node_cnt = 0; /* ALL DONE */
3816 } else if (i) {
3817 bit_and_not(resv_ptr->node_bitmap,
3818 idle_node_bitmap);
3819 resv_ptr->node_cnt = bit_set_count(
3820 resv_ptr->node_bitmap);
3821 delta_node_cnt = resv_ptr->node_cnt -
3822 node_cnt;
3823 }
3824 FREE_NULL_BITMAP(tmp1_bitmap);
3825 }
3826 if (delta_node_cnt > 0) {
3827 /* Now eliminate allocated nodes from reservation */
3828 tmp1_bitmap = bit_pick_cnt(resv_ptr->node_bitmap,
3829 node_cnt);
3830 FREE_NULL_BITMAP(resv_ptr->node_bitmap);
3831 resv_ptr->node_bitmap = tmp1_bitmap;
3832 }
3833 xfree(resv_ptr->node_list);
3834 resv_ptr->node_list = bitmap2node_name(resv_ptr->node_bitmap);
3835 resv_ptr->node_cnt = node_cnt;
3836 return SLURM_SUCCESS;
3837 }
3838
3839 /* Ensure if partition exists in reservation otherwise use default */
3840 if (!resv_ptr->part_ptr) {
3841 resv_ptr->part_ptr = default_part_loc;
3842 if (!resv_ptr->part_ptr)
3843 return ESLURM_DEFAULT_PARTITION_NOT_SET;
3844 }
3845
3846 /* Must increase node count. Make this look like new request so
3847 * we can use _select_nodes() for selecting the nodes */
3848 memset(&resv_desc, 0, sizeof(resv_desc_msg_t));
3849 resv_desc.start_time = resv_ptr->start_time;
3850 resv_desc.end_time = resv_ptr->end_time;
3851 resv_desc.features = resv_ptr->features;
3852 resv_desc.flags = resv_ptr->flags;
3853 resv_desc.node_cnt = xcalloc(2, sizeof(uint32_t));
3854 resv_desc.node_cnt[0]= 0 - delta_node_cnt;
3855
3856 /* Exclude self reserved nodes only if reservation contains any nodes */
3857 if (resv_ptr->node_bitmap) {
3858 tmp1_bitmap = bit_copy(resv_ptr->part_ptr->node_bitmap);
3859 bit_and_not(tmp1_bitmap, resv_ptr->node_bitmap);
3860 }
3861
3862 i = _select_nodes(&resv_desc, &resv_ptr->part_ptr, &tmp1_bitmap,
3863 &core_bitmap);
3864 xfree(resv_desc.node_cnt);
3865 xfree(resv_desc.node_list);
3866 xfree(resv_desc.partition);
3867 if (i == SLURM_SUCCESS) {
3868 /*
3869 * If the reservation was 0 node count before (ANY_NODES) this
3870 * could be NULL, if for some reason someone tried to update the
3871 * node count in this situation we will still not have a
3872 * node_bitmap.
3873 */
3874 if (resv_ptr->node_bitmap)
3875 bit_or(resv_ptr->node_bitmap, tmp1_bitmap);
3876 else
3877 resv_ptr->node_bitmap = bit_copy(tmp1_bitmap);
3878 FREE_NULL_BITMAP(tmp1_bitmap);
3879 FREE_NULL_BITMAP(resv_ptr->core_bitmap);
3880 resv_ptr->core_bitmap = core_bitmap;
3881 free_job_resources(&resv_ptr->core_resrcs);
3882 xfree(resv_ptr->node_list);
3883 resv_ptr->node_list = bitmap2node_name(resv_ptr->node_bitmap);
3884 resv_ptr->node_cnt = node_cnt;
3885 }
3886 return i;
3887 }
3888
_have_xand_feature(void * x,void * key)3889 static int _have_xand_feature(void *x, void *key)
3890 {
3891 job_feature_t *feat_ptr = (job_feature_t *) x;
3892
3893 if (feat_ptr->op_code == FEATURE_OP_XAND)
3894 return 1;
3895 return 0;
3896 }
3897
_have_xor_feature(void * x,void * key)3898 static int _have_xor_feature(void *x, void *key)
3899 {
3900 job_feature_t *feat_ptr = (job_feature_t *) x;
3901
3902 if (feat_ptr->op_code == FEATURE_OP_XOR)
3903 return 1;
3904 return 0;
3905 }
3906
3907 /*
3908 * Given a reservation create request, select appropriate nodes for use
3909 * resv_desc_ptr IN - Reservation request, node_list field set on exit
3910 * part_ptr IN/OUT - Desired partition, if NULL then set to default part
3911 * resv_bitmap IN/OUT - nodes to use, if points to NULL then used nodes in
3912 * specified partition. Set to selected nodes on output.
3913 * core_bitmap OUT - cores allocated to reservation
3914 */
_select_nodes(resv_desc_msg_t * resv_desc_ptr,part_record_t ** part_ptr,bitstr_t ** resv_bitmap,bitstr_t ** core_bitmap)3915 static int _select_nodes(resv_desc_msg_t *resv_desc_ptr,
3916 part_record_t **part_ptr, bitstr_t **resv_bitmap,
3917 bitstr_t **core_bitmap)
3918 {
3919 slurmctld_resv_t *resv_ptr;
3920 bitstr_t *node_bitmap;
3921 ListIterator iter;
3922 int i, rc = SLURM_SUCCESS;
3923 time_t now = time(NULL);
3924 bool have_xand = false;
3925
3926 if (*part_ptr == NULL) {
3927 *part_ptr = default_part_loc;
3928 if (*part_ptr == NULL)
3929 return ESLURM_DEFAULT_PARTITION_NOT_SET;
3930 xfree(resv_desc_ptr->partition); /* should be no-op */
3931 resv_desc_ptr->partition = xstrdup((*part_ptr)->name);
3932 }
3933
3934 /* Start with all nodes in the partition */
3935 if (*resv_bitmap) {
3936 node_bitmap = *resv_bitmap;
3937 *resv_bitmap = NULL;
3938 } else
3939 node_bitmap = bit_copy((*part_ptr)->node_bitmap);
3940
3941 /* Don't use nodes already reserved */
3942 if (!(resv_desc_ptr->flags & RESERVE_FLAG_MAINT) &&
3943 !(resv_desc_ptr->flags & RESERVE_FLAG_OVERLAP)) {
3944 iter = list_iterator_create(resv_list);
3945 while ((resv_ptr = list_next(iter))) {
3946 if ((resv_ptr->flags & RESERVE_FLAG_MAINT) ||
3947 (resv_ptr->flags & RESERVE_FLAG_OVERLAP))
3948 continue;
3949 if (resv_ptr->end_time <= now)
3950 _advance_resv_time(resv_ptr);
3951 if (resv_ptr->node_bitmap == NULL)
3952 continue;
3953 if (!_resv_time_overlap(resv_desc_ptr, resv_ptr))
3954 continue;
3955 if (!resv_ptr->core_bitmap && !resv_ptr->full_nodes) {
3956 error("Reservation %s has no core_bitmap and "
3957 "full_nodes is zero", resv_ptr->name);
3958 resv_ptr->full_nodes = 1;
3959 }
3960 if (resv_ptr->full_nodes || !resv_desc_ptr->core_cnt) {
3961 bit_and_not(node_bitmap, resv_ptr->node_bitmap);
3962 } else {
3963 _create_cluster_core_bitmap(core_bitmap);
3964 bit_or(*core_bitmap, resv_ptr->core_bitmap);
3965 }
3966 }
3967 list_iterator_destroy(iter);
3968 }
3969
3970 if (((resv_desc_ptr->flags & RESERVE_FLAG_MAINT) == 0) &&
3971 ((resv_desc_ptr->flags & RESERVE_FLAG_SPEC_NODES) == 0)) {
3972 /* Nodes must be available */
3973 bit_and(node_bitmap, avail_node_bitmap);
3974 }
3975
3976 /* Satisfy feature specification */
3977 if (resv_desc_ptr->features) {
3978 job_record_t *job_ptr;
3979 bool dummy = false;
3980 int total_node_cnt = 0;
3981
3982 /*
3983 * Build dummy job record so that existing job feature logic
3984 * can be re-used. The alternative of passing all of the
3985 * relevant fields to directly the functions rather than a
3986 * job record was explored and found too cumbersome.
3987 */
3988 job_ptr = xmalloc(sizeof(job_record_t));
3989 job_ptr->details = xmalloc(sizeof(struct job_details));
3990 job_ptr->details->features = resv_desc_ptr->features;
3991 /* job_ptr->job_id = 0; */
3992 /* job_ptr->user_id = 0; */
3993 rc = build_feature_list(job_ptr);
3994 if ((rc == SLURM_SUCCESS) &&
3995 list_find_first(job_ptr->details->feature_list,
3996 _have_xor_feature, &dummy)) {
3997 rc = ESLURM_INVALID_FEATURE;
3998 } else {
3999 find_feature_nodes(job_ptr->details->feature_list,
4000 true);
4001 if (resv_desc_ptr->node_cnt) {
4002 for (i = 0; resv_desc_ptr->node_cnt[i]; i++)
4003 total_node_cnt +=
4004 resv_desc_ptr->node_cnt[i];
4005 }
4006 }
4007 if (rc != SLURM_SUCCESS) {
4008 ;
4009 } else if (list_find_first(job_ptr->details->feature_list,
4010 _have_xand_feature, &dummy)) {
4011 /* Accumulate resources by feature type/count */
4012 have_xand = true;
4013 *resv_bitmap = _pick_idle_xand_nodes(node_bitmap,
4014 resv_desc_ptr, core_bitmap,
4015 job_ptr->details->feature_list);
4016 } else {
4017 /*
4018 * Simple AND/OR node filtering.
4019 * First try to use nodes with the feature active.
4020 * If that fails, use nodes with the feature available.
4021 */
4022 bitstr_t *tmp_bitmap;
4023 tmp_bitmap = bit_copy(node_bitmap);
4024 if (!valid_feature_counts(job_ptr, true, node_bitmap,
4025 &dummy)) {
4026 rc = ESLURM_INVALID_FEATURE;
4027 } else if (bit_set_count(node_bitmap) < total_node_cnt){
4028 bit_or(node_bitmap, tmp_bitmap);
4029 if (!valid_feature_counts(job_ptr, false,
4030 node_bitmap, &dummy)){
4031 rc = ESLURM_INVALID_FEATURE;
4032 }
4033 }
4034 FREE_NULL_BITMAP(tmp_bitmap);
4035
4036 if (bit_set_count(node_bitmap) < total_node_cnt)
4037 rc = ESLURM_REQUESTED_NODE_CONFIG_UNAVAILABLE;
4038 }
4039 FREE_NULL_LIST(job_ptr->details->feature_list);
4040 xfree(job_ptr->details);
4041 xfree(job_ptr);
4042 }
4043
4044 if (!have_xand && (rc == SLURM_SUCCESS)) {
4045 *resv_bitmap = _pick_idle_nodes(node_bitmap,
4046 resv_desc_ptr, core_bitmap);
4047 }
4048 FREE_NULL_BITMAP(node_bitmap);
4049 if (*resv_bitmap == NULL) {
4050 if (rc == SLURM_SUCCESS)
4051 rc = ESLURM_NODES_BUSY;
4052 return rc;
4053 }
4054
4055 if (!resv_desc_ptr->node_list)
4056 resv_desc_ptr->node_list = bitmap2node_name(*resv_bitmap);
4057
4058 return SLURM_SUCCESS;
4059 }
4060
_pick_idle_xand_nodes(bitstr_t * avail_bitmap,resv_desc_msg_t * resv_desc_ptr,bitstr_t ** core_bitmap,List feature_list)4061 static bitstr_t *_pick_idle_xand_nodes(bitstr_t *avail_bitmap,
4062 resv_desc_msg_t *resv_desc_ptr,
4063 bitstr_t **core_bitmap,
4064 List feature_list)
4065 {
4066 bitstr_t *ret_bitmap = NULL, *tmp_bitmap = NULL, *tmp_avail_bitmap;
4067 uint32_t *save_core_cnt, *save_node_cnt;
4068 uint32_t new_node_cnt[2] = {0, 0};
4069 job_feature_t *feat_ptr;
4070 ListIterator feat_iter;
4071 int paren = 0;
4072 bool test_active = true;
4073
4074 save_core_cnt = resv_desc_ptr->core_cnt;
4075 resv_desc_ptr->core_cnt = NULL;
4076 save_node_cnt = resv_desc_ptr->node_cnt;
4077 resv_desc_ptr->node_cnt = new_node_cnt;
4078
4079 TRY_AVAIL:
4080 /*
4081 * In the first pass, we try to satisfy the resource requirements using
4082 * currently active features. If that fails, use available features
4083 * and require a reboot to satisfy the request
4084 */
4085 feat_iter = list_iterator_create(feature_list);
4086 while ((feat_ptr = list_next(feat_iter))) {
4087 if (feat_ptr->paren > paren) { /* Start parenthesis */
4088 paren = feat_ptr->paren;
4089 if (test_active)
4090 tmp_bitmap = feat_ptr->node_bitmap_active;
4091 else
4092 tmp_bitmap = feat_ptr->node_bitmap_avail;
4093 continue;
4094 }
4095 if ((feat_ptr->paren == 1) || /* Continue parenthesis */
4096 (feat_ptr->paren < paren)) { /* End of parenthesis */
4097 paren = feat_ptr->paren;
4098 if (test_active) {
4099 bit_and(feat_ptr->node_bitmap_active,
4100 tmp_bitmap);
4101 tmp_bitmap = feat_ptr->node_bitmap_active;
4102 } else {
4103 bit_and(feat_ptr->node_bitmap_avail,
4104 tmp_bitmap);
4105 tmp_bitmap = feat_ptr->node_bitmap_avail;
4106 }
4107 if (feat_ptr->paren == 1)
4108 continue;
4109 }
4110 if (feat_ptr->count)
4111 new_node_cnt[0] = feat_ptr->count;
4112 else
4113 new_node_cnt[0] = 1;
4114 tmp_avail_bitmap = bit_copy(avail_bitmap);
4115 if (ret_bitmap)
4116 bit_and_not(tmp_avail_bitmap, ret_bitmap);
4117 if (test_active)
4118 bit_and(tmp_avail_bitmap, feat_ptr->node_bitmap_active);
4119 else
4120 bit_and(tmp_avail_bitmap, feat_ptr->node_bitmap_avail);
4121 tmp_bitmap = _pick_idle_nodes(tmp_avail_bitmap, resv_desc_ptr,
4122 core_bitmap);
4123 FREE_NULL_BITMAP(tmp_avail_bitmap);
4124 if (!tmp_bitmap) {
4125 FREE_NULL_BITMAP(ret_bitmap);
4126 break;
4127 }
4128 if (ret_bitmap) {
4129 bit_or(ret_bitmap, tmp_bitmap);
4130 FREE_NULL_BITMAP(tmp_bitmap);
4131 } else {
4132 ret_bitmap = tmp_bitmap;
4133 tmp_bitmap = NULL;
4134 }
4135 }
4136 list_iterator_destroy(feat_iter);
4137 if (!ret_bitmap && test_active) {
4138 /* Test failed for active features, test available features */
4139 test_active = false;
4140 goto TRY_AVAIL;
4141 }
4142
4143 resv_desc_ptr->core_cnt = save_core_cnt;
4144 resv_desc_ptr->node_cnt = save_node_cnt;
4145
4146 return ret_bitmap;
4147 }
4148
_pick_idle_nodes(bitstr_t * avail_bitmap,resv_desc_msg_t * resv_desc_ptr,bitstr_t ** core_bitmap)4149 static bitstr_t *_pick_idle_nodes(bitstr_t *avail_bitmap,
4150 resv_desc_msg_t *resv_desc_ptr,
4151 bitstr_t **core_bitmap)
4152 {
4153 int i;
4154 bitstr_t *ret_bitmap = NULL, *tmp_bitmap;
4155 bool resv_debug;
4156
4157 /* Free node_list here, it could be filled in by the select plugin. */
4158 xfree(resv_desc_ptr->node_list);
4159
4160 if (resv_desc_ptr->node_cnt == NULL) {
4161 return _pick_idle_node_cnt(avail_bitmap, resv_desc_ptr, 0,
4162 core_bitmap);
4163 } else if ((resv_desc_ptr->node_cnt[0] == 0) ||
4164 (resv_desc_ptr->node_cnt[1] == 0)) {
4165 return _pick_idle_node_cnt(avail_bitmap, resv_desc_ptr,
4166 resv_desc_ptr->node_cnt[0],
4167 core_bitmap);
4168 }
4169
4170 /* Need to create reservation containing multiple blocks */
4171 resv_debug = slurmctld_conf.debug_flags & DEBUG_FLAG_RESERVATION;
4172 for (i = 0; resv_desc_ptr->node_cnt[i]; i++) {
4173 tmp_bitmap = _pick_idle_node_cnt(avail_bitmap, resv_desc_ptr,
4174 resv_desc_ptr->node_cnt[i],
4175 core_bitmap);
4176 if (tmp_bitmap == NULL) { /* allocation failure */
4177 if (resv_debug) {
4178 info("reservation of %u nodes failed",
4179 resv_desc_ptr->node_cnt[i]);
4180 }
4181 FREE_NULL_BITMAP(ret_bitmap);
4182 return NULL;
4183 }
4184 if (resv_debug) {
4185 char *tmp_name;
4186 tmp_name = bitmap2node_name(tmp_bitmap);
4187 info("reservation of %u nodes, using %s",
4188 resv_desc_ptr->node_cnt[i], tmp_name);
4189 xfree(tmp_name);
4190 }
4191 if (ret_bitmap)
4192 bit_or(ret_bitmap, tmp_bitmap);
4193 else
4194 ret_bitmap = bit_copy(tmp_bitmap);
4195 bit_and_not(avail_bitmap, tmp_bitmap);
4196 FREE_NULL_BITMAP(tmp_bitmap);
4197 }
4198
4199 return ret_bitmap;
4200 }
4201
_check_job_compatibility(job_record_t * job_ptr,bitstr_t * avail_bitmap,bitstr_t ** core_bitmap)4202 static void _check_job_compatibility(job_record_t *job_ptr,
4203 bitstr_t *avail_bitmap,
4204 bitstr_t **core_bitmap)
4205 {
4206 uint32_t total_nodes;
4207 bitstr_t *full_node_bitmap;
4208 int i_core, i_node, res_inx;
4209 int start = 0;
4210 int rep_count = 0;
4211 job_resources_t *job_res = job_ptr->job_resrcs;
4212
4213 if (!job_res->core_bitmap)
4214 return;
4215
4216 total_nodes = bit_set_count(job_res->node_bitmap);
4217
4218 #if _DEBUG
4219 {
4220 char str[200];
4221 bit_fmt(str, sizeof(str), job_res->core_bitmap);
4222 info("Checking %d nodes (of %"PRIu64") for %pJ, core_bitmap:%s core_bitmap_size:%"PRIu64,
4223 total_nodes, bit_size(job_res->node_bitmap),
4224 job_ptr, str, bit_size(job_res->core_bitmap));
4225 }
4226 #endif
4227
4228 full_node_bitmap = bit_copy(job_res->node_bitmap);
4229 _create_cluster_core_bitmap(core_bitmap);
4230
4231 i_node = 0;
4232 res_inx = 0;
4233 while (i_node < total_nodes) {
4234 int cores_in_a_node = (job_res->sockets_per_node[res_inx] *
4235 job_res->cores_per_socket[res_inx]);
4236 int repeat_node_conf = job_res->sock_core_rep_count[rep_count++];
4237 int node_bitmap_inx;
4238
4239 #if _DEBUG
4240 info("Working with %d cores per node. Same node conf repeated "
4241 "%d times (start core offset %d)",
4242 cores_in_a_node, repeat_node_conf, start);
4243 #endif
4244
4245 i_node += repeat_node_conf;
4246 res_inx++;
4247
4248 while (repeat_node_conf--) {
4249 int allocated;
4250 int global_core_start;
4251
4252 node_bitmap_inx = bit_ffs(full_node_bitmap);
4253 if (node_bitmap_inx < 0)
4254 break; /* No more nodes */
4255 global_core_start =
4256 cr_get_coremap_offset(node_bitmap_inx);
4257 allocated = 0;
4258
4259 for (i_core = 0; i_core < cores_in_a_node; i_core++) {
4260 #if _DEBUG
4261 info("i_core: %d, start: %d, allocated: %d",
4262 i_core, start, allocated);
4263 #endif
4264 if (bit_test(job_ptr->job_resrcs->core_bitmap,
4265 i_core + start)) {
4266 allocated++;
4267 bit_set(*core_bitmap,
4268 global_core_start + i_core);
4269 }
4270 }
4271 #if _DEBUG
4272 info("Checking node %d, allocated: %d, "
4273 "cores_in_a_node: %d", node_bitmap_inx,
4274 allocated, cores_in_a_node);
4275 #endif
4276 if (allocated == cores_in_a_node) {
4277 /* We can exclude this node */
4278 #if _DEBUG
4279 info("Excluding node %d", node_bitmap_inx);
4280 #endif
4281 bit_clear(avail_bitmap, node_bitmap_inx);
4282 }
4283 start += cores_in_a_node;
4284 bit_clear(full_node_bitmap, node_bitmap_inx);
4285 }
4286 }
4287 FREE_NULL_BITMAP(full_node_bitmap);
4288 }
4289
_pick_idle_node_cnt(bitstr_t * avail_bitmap,resv_desc_msg_t * resv_desc_ptr,uint32_t node_cnt,bitstr_t ** core_bitmap)4290 static bitstr_t *_pick_idle_node_cnt(bitstr_t *avail_bitmap,
4291 resv_desc_msg_t *resv_desc_ptr,
4292 uint32_t node_cnt, bitstr_t **core_bitmap)
4293 {
4294 ListIterator job_iterator;
4295 job_record_t *job_ptr;
4296 bitstr_t *orig_bitmap, *save_bitmap = NULL;
4297 bitstr_t *ret_bitmap = NULL, *tmp_bitmap;
4298 int total_node_cnt;
4299
4300 total_node_cnt = bit_set_count(avail_bitmap);
4301 if (total_node_cnt < node_cnt) {
4302 verbose("reservation requests more nodes than are available");
4303 return NULL;
4304 } else if ((total_node_cnt == node_cnt) &&
4305 (resv_desc_ptr->flags & RESERVE_FLAG_IGN_JOBS)) {
4306 return select_g_resv_test(resv_desc_ptr, node_cnt,
4307 avail_bitmap, core_bitmap);
4308 } else if ((node_cnt == 0) &&
4309 ((resv_desc_ptr->core_cnt == NULL) ||
4310 (resv_desc_ptr->core_cnt[0] == 0)) &&
4311 (resv_desc_ptr->flags & RESERVE_FLAG_ANY_NODES)) {
4312 return bit_alloc(bit_size(avail_bitmap));
4313 }
4314
4315 orig_bitmap = bit_copy(avail_bitmap);
4316 job_iterator = list_iterator_create(job_list);
4317 while ((job_ptr = list_next(job_iterator))) {
4318 if (!IS_JOB_RUNNING(job_ptr) && !IS_JOB_SUSPENDED(job_ptr))
4319 continue;
4320 if (job_ptr->end_time < resv_desc_ptr->start_time)
4321 continue;
4322
4323 if (!resv_desc_ptr->core_cnt) {
4324 bit_and_not(avail_bitmap, job_ptr->node_bitmap);
4325 } else if (!(resv_desc_ptr->flags & RESERVE_FLAG_IGN_JOBS)) {
4326 /*
4327 * _check_job_compatibility will remove nodes and cores
4328 * from the available bitmaps if those resources are
4329 * being used by jobs. Don't do this if the IGNORE_JOBS
4330 * flag is set.
4331 */
4332 _check_job_compatibility(job_ptr, avail_bitmap,
4333 core_bitmap);
4334 }
4335 }
4336 list_iterator_destroy(job_iterator);
4337
4338 total_node_cnt = bit_set_count(avail_bitmap);
4339 if (total_node_cnt >= node_cnt) {
4340 /*
4341 * NOTE: select_g_resv_test() does NOT preserve avail_bitmap,
4342 * so we do that here and other calls to that function.
4343 */
4344 save_bitmap = bit_copy(avail_bitmap);
4345 ret_bitmap = select_g_resv_test(resv_desc_ptr, node_cnt,
4346 avail_bitmap, core_bitmap);
4347 if (ret_bitmap)
4348 goto fini;
4349 bit_or(avail_bitmap, save_bitmap);
4350 FREE_NULL_BITMAP(save_bitmap);
4351 }
4352
4353 /* Next: Try to reserve nodes that will be allocated to a limited
4354 * number of running jobs. We could sort the jobs by priority, QOS,
4355 * size or other criterion if desired. Right now we just go down
4356 * the unsorted job list. */
4357 if (resv_desc_ptr->flags & RESERVE_FLAG_IGN_JOBS) {
4358 job_iterator = list_iterator_create(job_list);
4359 while ((job_ptr = list_next(job_iterator))) {
4360 if (!IS_JOB_RUNNING(job_ptr) &&
4361 !IS_JOB_SUSPENDED(job_ptr))
4362 continue;
4363 if (job_ptr->end_time < resv_desc_ptr->start_time)
4364 continue;
4365 tmp_bitmap = bit_copy(orig_bitmap);
4366 bit_and(tmp_bitmap, job_ptr->node_bitmap);
4367 if (bit_set_count(tmp_bitmap) > 0)
4368 bit_or(avail_bitmap, tmp_bitmap);
4369 total_node_cnt = bit_set_count(avail_bitmap);
4370 if (total_node_cnt >= node_cnt) {
4371 save_bitmap = bit_copy(avail_bitmap);
4372 ret_bitmap = select_g_resv_test(
4373 resv_desc_ptr, node_cnt,
4374 avail_bitmap, core_bitmap);
4375 if (!ret_bitmap) {
4376 bit_or(avail_bitmap, save_bitmap);
4377 FREE_NULL_BITMAP(save_bitmap);
4378 }
4379 }
4380 FREE_NULL_BITMAP(tmp_bitmap);
4381 if (ret_bitmap)
4382 break;
4383 }
4384 list_iterator_destroy(job_iterator);
4385 }
4386
4387 fini: FREE_NULL_BITMAP(orig_bitmap);
4388 FREE_NULL_BITMAP(save_bitmap);
4389 #if _DEBUG
4390 if (ret_bitmap) {
4391 char str[300];
4392 bit_fmt(str, (sizeof(str) - 1), ret_bitmap);
4393 info("_pick_idle_node_cnt: node bitmap:%s", str);
4394 if (*core_bitmap) {
4395 bit_fmt(str, (sizeof(str) - 1), *core_bitmap);
4396 info("_pick_idle_node_cnt: core bitmap:%s", str);
4397 }
4398 }
4399 #endif
4400 return ret_bitmap;
4401 }
4402
4403 /* Determine if a job has access to a reservation
4404 * RET SLURM_SUCCESS if true, some error code otherwise */
_valid_job_access_resv(job_record_t * job_ptr,slurmctld_resv_t * resv_ptr)4405 static int _valid_job_access_resv(job_record_t *job_ptr,
4406 slurmctld_resv_t *resv_ptr)
4407 {
4408 bool account_good = false, user_good = false;
4409 int i;
4410
4411 if (!resv_ptr) {
4412 info("Reservation name not found (%s)", job_ptr->resv_name);
4413 return ESLURM_RESERVATION_INVALID;
4414 }
4415
4416 if (resv_ptr->flags & RESERVE_FLAG_TIME_FLOAT) {
4417 verbose("%pJ attempting to use reservation %s with floating start time",
4418 job_ptr, resv_ptr->name);
4419 return ESLURM_RESERVATION_ACCESS;
4420 }
4421
4422 /* Determine if we have access */
4423 if (accounting_enforce & ACCOUNTING_ENFORCE_ASSOCS) {
4424 char tmp_char[30];
4425 slurmdb_assoc_rec_t *assoc;
4426 if (!resv_ptr->assoc_list) {
4427 error("Reservation %s has no association list. "
4428 "Checking user/account lists",
4429 resv_ptr->name);
4430 goto no_assocs;
4431 }
4432
4433 if (!job_ptr->assoc_ptr) {
4434 slurmdb_assoc_rec_t assoc_rec;
4435 /* This should never be called, but just to be
4436 * safe we will try to fill it in. */
4437 memset(&assoc_rec, 0,
4438 sizeof(slurmdb_assoc_rec_t));
4439 assoc_rec.id = job_ptr->assoc_id;
4440 if (assoc_mgr_fill_in_assoc(
4441 acct_db_conn, &assoc_rec,
4442 accounting_enforce,
4443 (slurmdb_assoc_rec_t **)
4444 &job_ptr->assoc_ptr, false))
4445 goto end_it;
4446 }
4447
4448 /* Check to see if the association is here or the parent
4449 * association is listed in the valid associations. */
4450 if (strchr(resv_ptr->assoc_list, '-')) {
4451 assoc = job_ptr->assoc_ptr;
4452 while (assoc) {
4453 snprintf(tmp_char, sizeof(tmp_char), ",-%u,",
4454 assoc->id);
4455 if (strstr(resv_ptr->assoc_list, tmp_char))
4456 goto end_it; /* explicitly denied */
4457 assoc = assoc->usage->parent_assoc_ptr;
4458 }
4459 }
4460 if (strstr(resv_ptr->assoc_list, ",1") ||
4461 strstr(resv_ptr->assoc_list, ",2") ||
4462 strstr(resv_ptr->assoc_list, ",3") ||
4463 strstr(resv_ptr->assoc_list, ",4") ||
4464 strstr(resv_ptr->assoc_list, ",5") ||
4465 strstr(resv_ptr->assoc_list, ",6") ||
4466 strstr(resv_ptr->assoc_list, ",7") ||
4467 strstr(resv_ptr->assoc_list, ",8") ||
4468 strstr(resv_ptr->assoc_list, ",9") ||
4469 strstr(resv_ptr->assoc_list, ",0")) {
4470 assoc = job_ptr->assoc_ptr;
4471 while (assoc) {
4472 snprintf(tmp_char, sizeof(tmp_char), ",%u,",
4473 assoc->id);
4474 if (strstr(resv_ptr->assoc_list, tmp_char))
4475 return SLURM_SUCCESS;
4476 assoc = assoc->usage->parent_assoc_ptr;
4477 }
4478 } else {
4479 return SLURM_SUCCESS;
4480 }
4481 } else {
4482 no_assocs: if ((resv_ptr->user_cnt == 0) || resv_ptr->user_not)
4483 user_good = true;
4484 for (i = 0; i < resv_ptr->user_cnt; i++) {
4485 if (job_ptr->user_id == resv_ptr->user_list[i]) {
4486 if (resv_ptr->user_not)
4487 user_good = false;
4488 else
4489 user_good = true;
4490 break;
4491 }
4492 }
4493 if (!user_good)
4494 goto end_it;
4495 if ((resv_ptr->user_cnt != 0) && (resv_ptr->account_cnt == 0))
4496 return SLURM_SUCCESS;
4497
4498 if ((resv_ptr->account_cnt == 0) || resv_ptr->account_not)
4499 account_good = true;
4500 for (i=0; (i<resv_ptr->account_cnt) && job_ptr->account; i++) {
4501 if (resv_ptr->account_list[i] &&
4502 (xstrcmp(job_ptr->account,
4503 resv_ptr->account_list[i]) == 0)) {
4504 if (resv_ptr->account_not)
4505 account_good = false;
4506 else
4507 account_good = true;
4508 break;
4509 }
4510 }
4511 if (!account_good)
4512 goto end_it;
4513 return SLURM_SUCCESS;
4514 }
4515
4516 end_it:
4517 /*
4518 * If we are trying to run in a magnetic reservation
4519 * (the job didn't request it), don't print a security error.
4520 */
4521 if (job_ptr->resv_name)
4522 info("Security violation, uid=%u account=%s attempt to use reservation %s",
4523 job_ptr->user_id, job_ptr->account, resv_ptr->name);
4524
4525 return ESLURM_RESERVATION_ACCESS;
4526 }
4527
4528 /*
4529 * Determine if a job can start now based only upon reservations
4530 *
4531 * IN job_ptr - job to test
4532 * RET SLURM_SUCCESS if runable now, otherwise an error code
4533 */
job_test_resv_now(job_record_t * job_ptr)4534 extern int job_test_resv_now(job_record_t *job_ptr)
4535 {
4536 slurmctld_resv_t * resv_ptr;
4537 time_t now;
4538 int rc;
4539
4540 if (job_ptr->resv_name == NULL)
4541 return SLURM_SUCCESS;
4542
4543 resv_ptr = (slurmctld_resv_t *) list_find_first (resv_list,
4544 _find_resv_name, job_ptr->resv_name);
4545 job_ptr->resv_ptr = resv_ptr;
4546 rc = _valid_job_access_resv(job_ptr, resv_ptr);
4547 if (rc != SLURM_SUCCESS)
4548 return rc;
4549
4550 if (resv_ptr->flags & RESERVE_FLAG_FLEX)
4551 return SLURM_SUCCESS;
4552
4553 now = time(NULL);
4554 if (now < resv_ptr->start_time) {
4555 /* reservation starts later */
4556 return ESLURM_INVALID_TIME_VALUE;
4557 }
4558 if (now > resv_ptr->end_time) {
4559 /* reservation ended earlier */
4560 return ESLURM_RESERVATION_INVALID;
4561 }
4562 if ((resv_ptr->node_cnt == 0) &&
4563 !(resv_ptr->flags & RESERVE_FLAG_ANY_NODES)) {
4564 /* empty reservation treated like it will start later */
4565 return ESLURM_INVALID_TIME_VALUE;
4566 }
4567
4568 return SLURM_SUCCESS;
4569 }
4570
4571 /*
4572 * Note that a job is starting execution. If that job is associated with a
4573 * reservation having the "Replace" flag, then remove that job's nodes from
4574 * the reservation. Additional nodes will be added to the reservation from
4575 * those currently available.
4576 */
job_claim_resv(job_record_t * job_ptr)4577 extern void job_claim_resv(job_record_t *job_ptr)
4578 {
4579 slurmctld_resv_t *resv_ptr;
4580
4581 if (job_ptr->resv_name == NULL)
4582 return;
4583
4584 resv_ptr = (slurmctld_resv_t *) list_find_first (resv_list,
4585 _find_resv_name, job_ptr->resv_name);
4586 if (!resv_ptr ||
4587 (!resv_ptr->full_nodes && (resv_ptr->node_cnt > 1)) ||
4588 !(resv_ptr->flags & RESERVE_FLAG_REPLACE) ||
4589 (resv_ptr->flags & RESERVE_FLAG_SPEC_NODES) ||
4590 (resv_ptr->flags & RESERVE_FLAG_STATIC))
4591 return;
4592
4593 _resv_node_replace(resv_ptr);
4594 }
4595
4596 /*
4597 * Adjust a job's time_limit and end_time as needed to avoid using
4598 * reserved resources. Don't go below job's time_min value.
4599 */
job_time_adj_resv(job_record_t * job_ptr)4600 extern void job_time_adj_resv(job_record_t *job_ptr)
4601 {
4602 ListIterator iter;
4603 slurmctld_resv_t * resv_ptr;
4604 time_t now = time(NULL);
4605 int32_t resv_begin_time;
4606
4607 iter = list_iterator_create(resv_list);
4608 while ((resv_ptr = list_next(iter))) {
4609 if (resv_ptr->end_time <= now)
4610 _advance_resv_time(resv_ptr);
4611 if (job_ptr->resv_ptr == resv_ptr)
4612 continue; /* authorized user of reservation */
4613 if (resv_ptr->start_time <= now)
4614 continue; /* already validated */
4615 if (resv_ptr->start_time >= job_ptr->end_time)
4616 continue; /* reservation starts after job ends */
4617 if (!license_list_overlap(job_ptr->license_list,
4618 resv_ptr->license_list) &&
4619 ((resv_ptr->node_bitmap == NULL) ||
4620 (bit_overlap_any(resv_ptr->node_bitmap,
4621 job_ptr->node_bitmap) == 0)))
4622 continue; /* disjoint resources */
4623 resv_begin_time = difftime(resv_ptr->start_time, now) / 60;
4624 job_ptr->time_limit = MIN(job_ptr->time_limit,resv_begin_time);
4625 }
4626 list_iterator_destroy(iter);
4627 job_ptr->time_limit = MAX(job_ptr->time_limit, job_ptr->time_min);
4628 job_end_time_reset(job_ptr);
4629 }
4630
4631 /*
4632 * For a given license_list, return the total count of licenses of the
4633 * specified name
4634 */
_license_cnt(List license_list,char * lic_name)4635 static int _license_cnt(List license_list, char *lic_name)
4636 {
4637 int lic_cnt = 0;
4638 ListIterator iter;
4639 licenses_t *license_ptr;
4640
4641 if (license_list == NULL)
4642 return lic_cnt;
4643
4644 iter = list_iterator_create(license_list);
4645 while ((license_ptr = list_next(iter))) {
4646 if (xstrcmp(license_ptr->name, lic_name) == 0)
4647 lic_cnt += license_ptr->total;
4648 }
4649 list_iterator_destroy(iter);
4650
4651 return lic_cnt;
4652 }
4653
4654 /*
4655 * get the run time of a job, in seconds
4656 * job_ptr IN - pointer to the job record
4657 * reboot IN - true if node reboot required
4658 */
_get_job_duration(job_record_t * job_ptr,bool reboot)4659 static uint32_t _get_job_duration(job_record_t *job_ptr, bool reboot)
4660 {
4661 uint32_t duration;
4662 uint16_t time_slices = 1;
4663
4664 if (job_ptr->time_limit == INFINITE)
4665 duration = YEAR_SECONDS;
4666 else if (job_ptr->time_limit != NO_VAL)
4667 duration = (job_ptr->time_limit * 60);
4668 else { /* partition time limit */
4669 if (job_ptr->part_ptr->max_time == INFINITE)
4670 duration = YEAR_SECONDS;
4671 else
4672 duration = (job_ptr->part_ptr->max_time * 60);
4673 }
4674 if (job_ptr->part_ptr)
4675 time_slices = job_ptr->part_ptr->max_share & ~SHARED_FORCE;
4676 if ((duration != YEAR_SECONDS) && (time_slices > 1) &&
4677 (slurmctld_conf.preempt_mode & PREEMPT_MODE_GANG)) {
4678 /* FIXME: Ideally we figure out how many jobs are actually
4679 * time-slicing on each node rather than using the maximum
4680 * value. */
4681 duration *= time_slices;
4682 }
4683
4684 /* FIXME: reboot and sending it to this function needs to be removed */
4685 /* if (reboot) */
4686 /* duration += node_features_g_boot_time(); */
4687 return duration;
4688 }
4689
_add_bb_resv(burst_buffer_info_msg_t ** bb_resv,char * plugin,char * type,uint64_t cnt)4690 static void _add_bb_resv(burst_buffer_info_msg_t **bb_resv, char *plugin,
4691 char *type, uint64_t cnt)
4692 {
4693 burst_buffer_info_t *bb_array;
4694 burst_buffer_pool_t *pool_ptr;
4695 int i;
4696
4697 if (*bb_resv == NULL)
4698 *bb_resv = xmalloc(sizeof(burst_buffer_info_msg_t));
4699
4700 for (i = 0, bb_array = (*bb_resv)->burst_buffer_array;
4701 i < (*bb_resv)->record_count; i++) {
4702 if (!xstrcmp(plugin, bb_array->name))
4703 break;
4704 }
4705 if (i >= (*bb_resv)->record_count) {
4706 (*bb_resv)->record_count++;
4707 (*bb_resv)->burst_buffer_array = xrealloc(
4708 (*bb_resv)->burst_buffer_array,
4709 sizeof(burst_buffer_info_t) * (*bb_resv)->record_count);
4710 bb_array = (*bb_resv)->burst_buffer_array +
4711 (*bb_resv)->record_count - 1;
4712 bb_array->name = xstrdup(plugin);
4713 }
4714
4715 if (type == NULL) {
4716 bb_array->used_space += cnt;
4717 return;
4718 }
4719
4720 for (i = 0, pool_ptr = bb_array->pool_ptr; i < bb_array->pool_cnt; i++){
4721 if ((pool_ptr->name == NULL) || !xstrcmp(type, pool_ptr->name))
4722 break;
4723 }
4724 if (i >= bb_array->pool_cnt) {
4725 bb_array->pool_cnt++;
4726 bb_array->pool_ptr = xrealloc(bb_array->pool_ptr,
4727 sizeof(burst_buffer_pool_t) *
4728 bb_array->pool_cnt);
4729 pool_ptr = bb_array->pool_ptr + bb_array->pool_cnt - 1;
4730 pool_ptr->name = xstrdup(type);
4731 }
4732 pool_ptr->used_space += cnt;
4733 }
4734
_update_bb_resv(burst_buffer_info_msg_t ** bb_resv,char * bb_spec)4735 static void _update_bb_resv(burst_buffer_info_msg_t **bb_resv, char *bb_spec)
4736 {
4737 uint64_t cnt, mult;
4738 char *end_ptr = NULL, *unit = NULL;
4739 char *sep, *tmp_spec, *tok, *plugin, *type;
4740
4741 if ((bb_spec == NULL) || (bb_spec[0] == '\0'))
4742 return;
4743
4744 tmp_spec = xstrdup(bb_spec);
4745 tok = strtok_r(tmp_spec, ",", &end_ptr);
4746 while (tok) {
4747 /*
4748 * Translate "cray" to "datawarp" for backwards compatibility.
4749 */
4750 if (!xstrncmp(tok, "cray:", 5)) {
4751 plugin = "datawarp";
4752 tok += 5;
4753 } else if (!xstrncmp(tok, "datawarp:", 9)) {
4754 plugin = "datawarp";
4755 tok +=9;
4756 } else if (!xstrncmp(tok, "generic:", 8)) {
4757 plugin = "generic";
4758 tok += 8;
4759 } else
4760 plugin = NULL;
4761
4762 sep = strchr(tok, ':');
4763 if (sep) {
4764 type = tok;
4765 sep[0] = '\0';
4766 tok = sep + 1;
4767 } else {
4768 type = NULL;
4769 }
4770
4771 cnt = (uint64_t) strtoull(tok, &unit, 10);
4772 if (!xstrcasecmp(unit, "n") ||
4773 !xstrcasecmp(unit, "node") ||
4774 !xstrcasecmp(unit, "nodes")) {
4775 type = "nodes"; /* Cray node spec format */
4776 } else if ((mult = suffix_mult(unit)) != NO_VAL64) {
4777 cnt *= mult;
4778 }
4779
4780 if (cnt)
4781 _add_bb_resv(bb_resv, plugin, type, cnt);
4782 tok = strtok_r(NULL, ",", &end_ptr);
4783 }
4784 xfree(tmp_spec);
4785 }
4786
4787 /*
4788 * Determine how many burst buffer resources the specified job is prevented
4789 * from using due to reservations
4790 *
4791 * IN job_ptr - job to test
4792 * IN when - when the job is expected to start
4793 * IN reboot - true if node reboot required to start job
4794 * RET burst buffer reservation structure, call
4795 * slurm_free_burst_buffer_info_msg() to free
4796 */
job_test_bb_resv(job_record_t * job_ptr,time_t when,bool reboot)4797 extern burst_buffer_info_msg_t *job_test_bb_resv(job_record_t *job_ptr,
4798 time_t when, bool reboot)
4799 {
4800 slurmctld_resv_t * resv_ptr;
4801 time_t job_start_time, job_end_time, now = time(NULL);
4802 time_t job_end_time_use;
4803 burst_buffer_info_msg_t *bb_resv = NULL;
4804 ListIterator iter;
4805
4806 if ((job_ptr->burst_buffer == NULL) ||
4807 (job_ptr->burst_buffer[0] == '\0'))
4808 return bb_resv;
4809
4810 job_start_time = when;
4811 job_end_time = when + _get_job_duration(job_ptr, reboot);
4812 iter = list_iterator_create(resv_list);
4813 while ((resv_ptr = list_next(iter))) {
4814 if (resv_ptr->end_time <= now)
4815 _advance_resv_time(resv_ptr);
4816
4817 if (reboot)
4818 job_end_time_use =
4819 job_end_time + resv_ptr->boot_time;
4820 else
4821 job_end_time_use = job_end_time;
4822
4823 if ((resv_ptr->start_time >= job_end_time_use) ||
4824 (resv_ptr->end_time <= job_start_time))
4825 continue; /* reservation at different time */
4826 if ((resv_ptr->burst_buffer == NULL) ||
4827 (resv_ptr->burst_buffer[0] == '\0'))
4828 continue; /* reservation has no burst buffers */
4829 if (!xstrcmp(job_ptr->resv_name, resv_ptr->name))
4830 continue; /* job can use this reservation */
4831
4832 _update_bb_resv(&bb_resv, resv_ptr->burst_buffer);
4833 }
4834 list_iterator_destroy(iter);
4835
4836 return bb_resv;
4837 }
4838
4839 /*
4840 * Determine how many licenses of the give type the specified job is
4841 * prevented from using due to reservations
4842 *
4843 * IN job_ptr - job to test
4844 * IN lic_name - name of license
4845 * IN when - when the job is expected to start
4846 * IN reboot - true if node reboot required to start job
4847 * RET number of licenses of this type the job is prevented from using
4848 */
job_test_lic_resv(job_record_t * job_ptr,char * lic_name,time_t when,bool reboot)4849 extern int job_test_lic_resv(job_record_t *job_ptr, char *lic_name,
4850 time_t when, bool reboot)
4851 {
4852 slurmctld_resv_t * resv_ptr;
4853 time_t job_start_time, job_end_time, now = time(NULL);
4854 time_t job_end_time_use;
4855 ListIterator iter;
4856 int resv_cnt = 0;
4857
4858 job_start_time = when;
4859 job_end_time = when + _get_job_duration(job_ptr, reboot);
4860 iter = list_iterator_create(resv_list);
4861 while ((resv_ptr = list_next(iter))) {
4862 if (resv_ptr->end_time <= now)
4863 _advance_resv_time(resv_ptr);
4864
4865 if (reboot)
4866 job_end_time_use =
4867 job_end_time + resv_ptr->boot_time;
4868 else
4869 job_end_time_use = job_end_time;
4870
4871 if ((resv_ptr->start_time >= job_end_time_use) ||
4872 (resv_ptr->end_time <= job_start_time))
4873 continue; /* reservation at different time */
4874
4875 if (job_ptr->resv_name &&
4876 (xstrcmp(job_ptr->resv_name, resv_ptr->name) == 0))
4877 continue; /* job can use this reservation */
4878
4879 resv_cnt += _license_cnt(resv_ptr->license_list, lic_name);
4880 }
4881 list_iterator_destroy(iter);
4882
4883 /* info("%pJ blocked from %d licenses of type %s",
4884 job_ptr, resv_cnt, lic_name); */
4885 return resv_cnt;
4886 }
4887
_init_constraint_planning(constraint_planning_t * sched)4888 static void _init_constraint_planning(constraint_planning_t* sched)
4889 {
4890 sched->slot_list = list_create(xfree_ptr);
4891 }
4892
_free_constraint_planning(constraint_planning_t * sched)4893 static void _free_constraint_planning(constraint_planning_t* sched)
4894 {
4895 FREE_NULL_LIST(sched->slot_list);
4896 }
4897
4898 /*
4899 * update the list of slots with the new time delimited constraint
4900 * the new slot may have to be :
4901 * - inserted
4902 * - added to a previously added slot, if it corresponds to the same
4903 * period
4904 * - shrinked if it overlaps temporarily a previously slot
4905 * (in that case it will result in a new slot insertion and an
4906 * already defined slot update with the addition of the value)
4907 * - splitted in two chunks if it is nested in a previously slot
4908 * (in that case it will result in the insertion of new head,
4909 * the update of the previously defined slot, and the iteration
4910 * of the logic with a new slot reduced to the remaining time)
4911 */
_update_constraint_planning(constraint_planning_t * sched,uint32_t value,time_t start,time_t end)4912 static void _update_constraint_planning(constraint_planning_t* sched,
4913 uint32_t value,
4914 time_t start, time_t end)
4915 {
4916 ListIterator iter;
4917 constraint_slot_t *cur_slot, *cstr_slot, *tmp_slot;
4918 bool done = false;
4919
4920 /* create the constraint slot to add */
4921 cstr_slot = xmalloc(sizeof(constraint_slot_t));
4922 cstr_slot->value = value;
4923 cstr_slot->start = start;
4924 cstr_slot->end = end;
4925
4926 /* iterate on the current slot list to identify
4927 * the modifications and do them live */
4928 iter = list_iterator_create(sched->slot_list);
4929 while ((cur_slot = list_next(iter))) {
4930 /* cur_slot is posterior or contiguous, insert cstr,
4931 * mark the state as done and break */
4932 if (cstr_slot->end <= cur_slot->start) {
4933 list_insert(iter,cstr_slot);
4934 done = true;
4935 break;
4936 }
4937 /* cur_slot has the same time period, update it,
4938 * mark the state as done and break */
4939 if (cstr_slot->start == cur_slot->start &&
4940 cstr_slot->end == cur_slot->end) {
4941 cur_slot->value += cstr_slot->value;
4942 xfree(cstr_slot);
4943 done = true;
4944 break;
4945 }
4946 /* cur_slot is anterior or contiguous, continue */
4947 if (cur_slot->end <= cstr_slot->start)
4948 continue;
4949 /* new slot starts after this one */
4950 if (cur_slot->start <= cstr_slot->start) {
4951 /* we may need up to 2 insertions and one update */
4952 if (cur_slot->start < cstr_slot->start) {
4953 tmp_slot = xmalloc(sizeof(constraint_slot_t));
4954 tmp_slot->value = cur_slot->value;
4955 tmp_slot->start = cur_slot->start;
4956 tmp_slot->end = cstr_slot->start;
4957 list_insert(iter,tmp_slot);
4958 cur_slot->start = tmp_slot->end;
4959 }
4960 if (cstr_slot->end < cur_slot->end) {
4961 cstr_slot->value += cur_slot->value;
4962 list_insert(iter,cstr_slot);
4963 cur_slot->start = cstr_slot->end;
4964 } else if (cstr_slot->end > cur_slot->end) {
4965 cur_slot->value += cstr_slot->value;
4966 cstr_slot->start = cur_slot->end;
4967 continue;
4968 } else {
4969 cur_slot->value += cstr_slot->value;
4970 xfree(cstr_slot);
4971 }
4972 done = true;
4973 break;
4974 } else {
4975 /* new slot starts before, and we know that it is
4976 * not contiguous (previously checked) */
4977 tmp_slot = xmalloc(sizeof(constraint_slot_t));
4978 tmp_slot->value = cstr_slot->value;
4979 tmp_slot->start = cstr_slot->start;
4980 tmp_slot->end = cur_slot->start;
4981 list_insert(iter,tmp_slot);
4982 if (cstr_slot->end == cur_slot-> end) {
4983 cur_slot->value += cstr_slot->value;
4984 xfree(cstr_slot);
4985 done = true;
4986 break;
4987 } else if (cstr_slot->end < cur_slot-> end) {
4988 cstr_slot->start = cur_slot->start;
4989 cstr_slot->value += cur_slot->value;
4990 list_insert(iter,cstr_slot);
4991 cur_slot->start = cstr_slot->end;
4992 done = true;
4993 break;
4994 } else {
4995 cur_slot->value += cstr_slot->value;
4996 cstr_slot->start = cur_slot->end;
4997 continue;
4998 }
4999 }
5000 }
5001 list_iterator_destroy(iter);
5002
5003 /* we might still need to add the [updated] constrain slot */
5004 if (!done)
5005 list_append(sched->slot_list, cstr_slot);
5006 }
5007
_max_constraint_planning(constraint_planning_t * sched,time_t * start,time_t * end)5008 static uint32_t _max_constraint_planning(constraint_planning_t* sched,
5009 time_t *start, time_t *end)
5010 {
5011 ListIterator iter;
5012 constraint_slot_t *cur_slot;
5013 uint32_t max = 0;
5014
5015 iter = list_iterator_create(sched->slot_list);
5016 while ((cur_slot = list_next(iter))) {
5017 if (cur_slot->value > max) {
5018 max = cur_slot->value;
5019 *start = cur_slot->start;
5020 *end = cur_slot->end;
5021 }
5022 }
5023 list_iterator_destroy(iter);
5024
5025 return max;
5026 }
5027
_print_constraint_planning(constraint_planning_t * sched)5028 static void _print_constraint_planning(constraint_planning_t* sched)
5029 {
5030 ListIterator iter;
5031 constraint_slot_t *cur_slot;
5032 char start_str[32] = "-1", end_str[32] = "-1";
5033 uint32_t i = 0;
5034
5035 iter = list_iterator_create(sched->slot_list);
5036 while ((cur_slot = list_next(iter))) {
5037 slurm_make_time_str(&cur_slot->start,
5038 start_str, sizeof(start_str));
5039 slurm_make_time_str(&cur_slot->end,
5040 end_str, sizeof(end_str));
5041 debug2("constraint_planning: slot[%u]: %s to %s count=%u",
5042 i, start_str, end_str, cur_slot->value);
5043 i++;
5044 }
5045 list_iterator_destroy(iter);
5046 }
5047
_get_rel_start_end(slurmctld_resv_t * resv_ptr,time_t now,time_t * start_relative,time_t * end_relative)5048 static void _get_rel_start_end(slurmctld_resv_t *resv_ptr, time_t now,
5049 time_t *start_relative, time_t *end_relative)
5050 {
5051 xassert(resv_ptr);
5052 xassert(start_relative);
5053 xassert(end_relative);
5054
5055 if (resv_ptr->flags & RESERVE_FLAG_TIME_FLOAT) {
5056 *start_relative = resv_ptr->start_time + now;
5057 if (resv_ptr->duration == INFINITE)
5058 *end_relative = *start_relative + YEAR_SECONDS;
5059 else if (resv_ptr->duration && (resv_ptr->duration != NO_VAL)) {
5060 *end_relative =
5061 *start_relative + resv_ptr->duration * 60;
5062 } else {
5063 *end_relative = resv_ptr->end_time;
5064 if (*start_relative > *end_relative)
5065 *start_relative = *end_relative;
5066 }
5067 } else {
5068 if (resv_ptr->end_time <= now)
5069 _advance_resv_time(resv_ptr);
5070 *start_relative = resv_ptr->start_time_first;
5071 *end_relative = resv_ptr->end_time;
5072 }
5073 }
5074
5075 /*
5076 * Determine how many watts the specified job is prevented from using
5077 * due to reservations
5078 *
5079 * IN job_ptr - job to test
5080 * IN when - when the job is expected to start
5081 * IN reboot - true if node reboot required to start job
5082 * RET amount of watts the job is prevented from using
5083 */
job_test_watts_resv(job_record_t * job_ptr,time_t when,bool reboot)5084 extern uint32_t job_test_watts_resv(job_record_t *job_ptr, time_t when,
5085 bool reboot)
5086 {
5087 slurmctld_resv_t * resv_ptr;
5088 time_t job_start_time, job_end_time, now = time(NULL);
5089 time_t job_end_time_use;
5090 ListIterator iter;
5091 constraint_planning_t wsched;
5092 time_t start, end;
5093 char start_str[32] = "-1", end_str[32] = "-1";
5094 uint32_t resv_cnt = 0;
5095
5096 _init_constraint_planning(&wsched);
5097
5098 job_start_time = when;
5099 job_end_time = when + _get_job_duration(job_ptr, reboot);
5100 iter = list_iterator_create(resv_list);
5101 while ((resv_ptr = list_next(iter))) {
5102 if (resv_ptr->end_time <= now)
5103 _advance_resv_time(resv_ptr);
5104 if (resv_ptr->resv_watts == NO_VAL ||
5105 resv_ptr->resv_watts == 0)
5106 continue; /* not a power reservation */
5107
5108 if (reboot)
5109 job_end_time_use =
5110 job_end_time + resv_ptr->boot_time;
5111 else
5112 job_end_time_use = job_end_time;
5113
5114 if ((resv_ptr->start_time >= job_end_time_use) ||
5115 (resv_ptr->end_time <= job_start_time))
5116 continue; /* reservation at different time */
5117
5118 if (job_ptr->resv_name &&
5119 (xstrcmp(job_ptr->resv_name, resv_ptr->name) == 0))
5120 continue; /* job can use this reservation */
5121
5122 _update_constraint_planning(&wsched, resv_ptr->resv_watts,
5123 resv_ptr->start_time,
5124 resv_ptr->end_time);
5125 }
5126 list_iterator_destroy(iter);
5127
5128 resv_cnt = _max_constraint_planning(&wsched, &start, &end);
5129 if (slurm_get_debug_flags() & DEBUG_FLAG_RESERVATION) {
5130 _print_constraint_planning(&wsched);
5131 slurm_make_time_str(&start, start_str, sizeof(start_str));
5132 slurm_make_time_str(&end, end_str, sizeof(end_str));
5133 debug2("reservation: max reserved watts=%u (%s to %s)",
5134 resv_cnt, start_str, end_str);
5135 }
5136 _free_constraint_planning(&wsched);
5137
5138 return resv_cnt;
5139 }
5140
5141 /*
5142 * Determine which nodes a job can use based upon reservations
5143 * IN job_ptr - job to test
5144 * IN/OUT when - when we want the job to start (IN)
5145 * when the reservation is available (OUT)
5146 * IN move_time - if true, then permit the start time to advance from
5147 * "when" as needed IF job has no reservervation
5148 * OUT node_bitmap - nodes which the job can use, caller must free unless error
5149 * OUT exc_core_bitmap - cores which the job can NOT use, caller must free
5150 * unless error
5151 * OUT resv_overlap - set to true if the job's run time and available nodes
5152 * overlap with an advanced reservation, indicates that
5153 * resources were removed from availability to the job
5154 * IN reboot - true if node reboot required to start job
5155 * RET SLURM_SUCCESS if runable now
5156 * ESLURM_RESERVATION_ACCESS access to reservation denied
5157 * ESLURM_RESERVATION_INVALID reservation invalid
5158 * ESLURM_INVALID_TIME_VALUE reservation invalid at time "when"
5159 * ESLURM_NODES_BUSY job has no reservation, but required nodes are
5160 * reserved
5161 * ESLURM_RESERVATION_MAINT job has no reservation, but required nodes are
5162 * in maintenance reservation
5163 */
job_test_resv(job_record_t * job_ptr,time_t * when,bool move_time,bitstr_t ** node_bitmap,bitstr_t ** exc_core_bitmap,bool * resv_overlap,bool reboot)5164 extern int job_test_resv(job_record_t *job_ptr, time_t *when,
5165 bool move_time, bitstr_t **node_bitmap,
5166 bitstr_t **exc_core_bitmap, bool *resv_overlap,
5167 bool reboot)
5168 {
5169 slurmctld_resv_t *resv_ptr = NULL, *res2_ptr;
5170 time_t job_start_time, job_end_time, job_end_time_use, lic_resv_time;
5171 time_t start_relative, end_relative;
5172 time_t now = time(NULL);
5173 ListIterator iter;
5174 int i, rc = SLURM_SUCCESS, rc2;
5175
5176 *resv_overlap = false; /* initialize to false */
5177 job_start_time = *when;
5178 job_end_time = *when + _get_job_duration(job_ptr, reboot);
5179 *node_bitmap = (bitstr_t *) NULL;
5180
5181 if (job_ptr->resv_name) {
5182 resv_ptr = (slurmctld_resv_t *) list_find_first (resv_list,
5183 _find_resv_name, job_ptr->resv_name);
5184 job_ptr->resv_ptr = resv_ptr;
5185 rc2 = _valid_job_access_resv(job_ptr, resv_ptr);
5186 if (rc2 != SLURM_SUCCESS)
5187 return rc2;
5188 /*
5189 * Just in case the reservation was altered since last looking
5190 * we want to make sure things are good in the database.
5191 */
5192 if (job_ptr->resv_id != resv_ptr->resv_id) {
5193 job_ptr->resv_id = resv_ptr->resv_id;
5194 /*
5195 * Update the database if not using a promiscuous
5196 * reservation
5197 */
5198 if (!(job_ptr->bit_flags & JOB_PROM))
5199 jobacct_storage_g_job_start(
5200 acct_db_conn, job_ptr);
5201 }
5202 if (resv_ptr->flags & RESERVE_FLAG_FLEX) {
5203 /* Job not bound to reservation nodes or time */
5204 *node_bitmap = bit_alloc(node_record_count);
5205 bit_nset(*node_bitmap, 0, (node_record_count - 1));
5206 } else {
5207 if (resv_ptr->end_time <= now)
5208 _advance_resv_time(resv_ptr);
5209 if (*when < resv_ptr->start_time) {
5210 /* reservation starts later */
5211 *when = resv_ptr->start_time;
5212 return ESLURM_INVALID_TIME_VALUE;
5213 }
5214 if ((resv_ptr->node_cnt == 0) &&
5215 (!(resv_ptr->flags & RESERVE_FLAG_ANY_NODES))) {
5216 /*
5217 * empty reservation treated like it will
5218 * start later
5219 */
5220 *when = now + 600;
5221 return ESLURM_INVALID_TIME_VALUE;
5222 }
5223 if (*when > resv_ptr->end_time) {
5224 /* reservation ended earlier */
5225 *when = resv_ptr->end_time;
5226 if ((now > resv_ptr->end_time) ||
5227 ((job_ptr->details) &&
5228 (job_ptr->details->begin_time >
5229 resv_ptr->end_time))) {
5230 debug("%s: Holding %pJ, expired reservation %s",
5231 __func__, job_ptr, resv_ptr->name);
5232 job_ptr->priority = 0; /* admin hold */
5233 }
5234 return ESLURM_RESERVATION_INVALID;
5235 }
5236 if (job_ptr->details->req_node_bitmap &&
5237 (!(resv_ptr->flags & RESERVE_FLAG_ANY_NODES)) &&
5238 !bit_super_set(job_ptr->details->req_node_bitmap,
5239 resv_ptr->node_bitmap)) {
5240 return ESLURM_RESERVATION_INVALID;
5241 }
5242 if (resv_ptr->flags & RESERVE_FLAG_ANY_NODES) {
5243 *node_bitmap = bit_alloc(node_record_count);
5244 bit_nset(*node_bitmap, 0,
5245 (node_record_count - 1));
5246 } else {
5247 *node_bitmap = bit_copy(resv_ptr->node_bitmap);
5248 }
5249 }
5250
5251 /*
5252 * if there are any overlapping reservations, we need to
5253 * prevent the job from using those nodes (e.g. MAINT nodes)
5254 */
5255 iter = list_iterator_create(resv_list);
5256 while ((res2_ptr = list_next(iter))) {
5257 if (reboot)
5258 job_end_time_use =
5259 job_end_time + res2_ptr->boot_time;
5260 else
5261 job_end_time_use = job_end_time;
5262
5263 _get_rel_start_end(
5264 res2_ptr, now, &start_relative, &end_relative);
5265
5266 if ((resv_ptr->flags & RESERVE_FLAG_MAINT) ||
5267 ((resv_ptr->flags & RESERVE_FLAG_OVERLAP) &&
5268 !(res2_ptr->flags & RESERVE_FLAG_MAINT)) ||
5269 (res2_ptr == resv_ptr) ||
5270 (res2_ptr->node_bitmap == NULL) ||
5271 (start_relative >= job_end_time_use) ||
5272 (end_relative <= job_start_time) ||
5273 (!res2_ptr->full_nodes))
5274 continue;
5275 if (bit_overlap_any(*node_bitmap,
5276 res2_ptr->node_bitmap)) {
5277 if (slurmctld_conf.debug_flags &
5278 DEBUG_FLAG_RESERVATION)
5279 info("%s: reservation %s overlaps %s with %u nodes",
5280 __func__,
5281 resv_ptr->name,
5282 res2_ptr->name,
5283 bit_overlap(*node_bitmap,
5284 res2_ptr->
5285 node_bitmap));
5286 *resv_overlap = true;
5287 bit_and_not(*node_bitmap,res2_ptr->node_bitmap);
5288 }
5289 }
5290 list_iterator_destroy(iter);
5291
5292 if (slurmctld_conf.debug_flags & DEBUG_FLAG_RESERVATION) {
5293 char *nodes = bitmap2node_name(*node_bitmap);
5294 info("%s: %pJ reservation:%s nodes:%s",
5295 __func__, job_ptr, job_ptr->resv_name, nodes);
5296 xfree(nodes);
5297 }
5298
5299 /*
5300 * if reservation is using just partial nodes, this returns
5301 * coremap to exclude
5302 */
5303 if (resv_ptr->core_bitmap && exc_core_bitmap &&
5304 !(resv_ptr->flags & RESERVE_FLAG_FLEX) ) {
5305 *exc_core_bitmap = bit_copy(resv_ptr->core_bitmap);
5306 bit_not(*exc_core_bitmap);
5307 }
5308
5309 return SLURM_SUCCESS;
5310 }
5311
5312 job_ptr->resv_ptr = NULL; /* should be redundant */
5313 *node_bitmap = bit_alloc(node_record_count);
5314 bit_nset(*node_bitmap, 0, (node_record_count - 1));
5315 if (list_count(resv_list) == 0)
5316 return SLURM_SUCCESS;
5317
5318 /*
5319 * Job has no reservation, try to find time when this can
5320 * run and get it's required nodes (if any)
5321 */
5322 for (i = 0; ; i++) {
5323 lic_resv_time = (time_t) 0;
5324
5325 iter = list_iterator_create(resv_list);
5326 while ((resv_ptr = list_next(iter))) {
5327 _get_rel_start_end(
5328 resv_ptr, now, &start_relative, &end_relative);
5329
5330 if (reboot)
5331 job_end_time_use =
5332 job_end_time + resv_ptr->boot_time;
5333 else
5334 job_end_time_use = job_end_time;
5335
5336 if ((resv_ptr->node_bitmap == NULL) ||
5337 (start_relative >= job_end_time_use) ||
5338 (end_relative <= job_start_time))
5339 continue;
5340
5341 /*
5342 * Check if we are able to use this reservation's
5343 * resources even though we didn't request it.
5344 */
5345 if ((job_ptr->warn_time <= resv_ptr->max_start_delay) &&
5346 (job_ptr->warn_flags & KILL_JOB_RESV)) {
5347 continue;
5348 }
5349
5350 if (resv_ptr->flags & RESERVE_FLAG_ALL_NODES ||
5351 ((resv_ptr->flags & RESERVE_FLAG_PART_NODES) &&
5352 job_ptr->part_ptr == resv_ptr->part_ptr) ||
5353 ((resv_ptr->flags & RESERVE_FLAG_MAINT) &&
5354 job_ptr->part_ptr &&
5355 (bit_super_set(job_ptr->part_ptr->node_bitmap,
5356 resv_ptr->node_bitmap)))) {
5357 rc = ESLURM_RESERVATION_MAINT;
5358 if (move_time)
5359 *when = resv_ptr->end_time;
5360 break;
5361 }
5362
5363 if (job_ptr->details->req_node_bitmap &&
5364 bit_overlap_any(job_ptr->details->req_node_bitmap,
5365 resv_ptr->node_bitmap) &&
5366 (!resv_ptr->tres_str ||
5367 job_ptr->details->whole_node == 1)) {
5368 if (move_time)
5369 *when = resv_ptr->end_time;
5370 rc = ESLURM_NODES_BUSY;
5371 break;
5372 }
5373 /*
5374 * FIXME: This only tracks when ANY licenses required
5375 * by the job are freed by any reservation without
5376 * counting them, so the results are not accurate.
5377 */
5378 if (license_list_overlap(job_ptr->license_list,
5379 resv_ptr->license_list)) {
5380 if ((lic_resv_time == (time_t) 0) ||
5381 (lic_resv_time > resv_ptr->end_time))
5382 lic_resv_time = resv_ptr->end_time;
5383 }
5384
5385 if ((resv_ptr->full_nodes) ||
5386 (job_ptr->details->whole_node == 1)) {
5387 #if _DEBUG
5388 info("reservation %s uses full nodes or %pJ will not share nodes",
5389 resv_ptr->name, job_ptr);
5390 #endif
5391 bit_and_not(*node_bitmap, resv_ptr->node_bitmap);
5392 } else {
5393 #if _DEBUG
5394 info("%s: reservation %s uses partial nodes",
5395 __func__, resv_ptr->name);
5396 #endif
5397 if (resv_ptr->core_bitmap == NULL) {
5398 ;
5399 } else if (exc_core_bitmap == NULL) {
5400 error("%s: exc_core_bitmap is NULL",
5401 __func__);
5402 } else if (*exc_core_bitmap == NULL) {
5403 *exc_core_bitmap =
5404 bit_copy(resv_ptr->core_bitmap);
5405 } else {
5406 bit_or(*exc_core_bitmap,
5407 resv_ptr->core_bitmap);
5408 }
5409 }
5410
5411 if(!job_ptr->part_ptr ||
5412 bit_overlap_any(job_ptr->part_ptr->node_bitmap,
5413 resv_ptr->node_bitmap)) {
5414 *resv_overlap = true;
5415 continue;
5416 }
5417 }
5418 list_iterator_destroy(iter);
5419
5420 if ((rc == SLURM_SUCCESS) && move_time) {
5421 if (license_job_test(job_ptr, job_start_time, reboot)
5422 == EAGAIN) {
5423 /*
5424 * Need to postpone for licenses. Time returned
5425 * is best case; first reservation with those
5426 * licenses ends.
5427 */
5428 rc = ESLURM_NODES_BUSY;
5429 *when = lic_resv_time;
5430 }
5431 }
5432 if (rc == SLURM_SUCCESS)
5433 break;
5434 /*
5435 * rc == ESLURM_NODES_BUSY or rc == ESLURM_RESERVATION_MAINT
5436 * above "break"
5437 */
5438 if (move_time && (i < 10)) { /* Retry for later start time */
5439 job_start_time = *when;
5440 job_end_time = *when +
5441 _get_job_duration(job_ptr, reboot);
5442 bit_nset(*node_bitmap, 0, (node_record_count - 1));
5443 rc = SLURM_SUCCESS;
5444 continue;
5445 }
5446 FREE_NULL_BITMAP(*node_bitmap);
5447 break; /* Give up */
5448 }
5449
5450 return rc;
5451 }
5452
5453 /*
5454 * Determine the time of the first reservation to end after some time.
5455 * return zero of no reservation ends after that time.
5456 * IN start_time - look for reservations ending after this time
5457 * RET the reservation end time or zero of none found
5458 */
find_resv_end(time_t start_time)5459 extern time_t find_resv_end(time_t start_time)
5460 {
5461 ListIterator iter;
5462 slurmctld_resv_t *resv_ptr;
5463 time_t end_time = 0;
5464
5465 if (!resv_list)
5466 return end_time;
5467
5468 iter = list_iterator_create(resv_list);
5469 while ((resv_ptr = list_next(iter))) {
5470 if (start_time > resv_ptr->end_time)
5471 continue;
5472 if ((end_time == 0) || (resv_ptr->end_time < end_time))
5473 end_time = resv_ptr->end_time;
5474 }
5475 list_iterator_destroy(iter);
5476 return end_time;
5477 }
5478
5479 /* Test a particular job for valid reservation
5480 * and refill job_run_cnt/job_pend_cnt */
_job_resv_check(void * x,void * arg)5481 static int _job_resv_check(void *x, void *arg)
5482 {
5483 job_record_t *job_ptr = (job_record_t *) x;
5484
5485 if (!job_ptr->resv_ptr)
5486 return SLURM_SUCCESS;
5487
5488 xassert(job_ptr->resv_ptr->magic == RESV_MAGIC);
5489
5490 if (IS_JOB_PENDING(job_ptr))
5491 job_ptr->resv_ptr->job_pend_cnt++;
5492 else if (!IS_JOB_FINISHED(job_ptr))
5493 job_ptr->resv_ptr->job_run_cnt++;
5494
5495 return SLURM_SUCCESS;
5496 }
5497
_set_job_resvid(void * object,void * arg)5498 static int _set_job_resvid(void *object, void *arg)
5499 {
5500 job_record_t *job_ptr = (job_record_t *) object;
5501 slurmctld_resv_t *resv_ptr = (slurmctld_resv_t *)arg;
5502
5503 if ((job_ptr->resv_ptr != resv_ptr) || !IS_JOB_PENDING(job_ptr))
5504 return SLURM_SUCCESS;
5505
5506 if (slurmctld_conf.debug_flags & DEBUG_FLAG_RESERVATION)
5507 info("updating %pJ to correct resv_id (%u->%u) of reoccurring reservation '%s'",
5508 job_ptr, job_ptr->resv_id, resv_ptr->resv_id,
5509 resv_ptr->name);
5510 job_ptr->resv_id = resv_ptr->resv_id;
5511 /* Update the database */
5512 jobacct_storage_g_job_start(acct_db_conn, job_ptr);
5513
5514 return SLURM_SUCCESS;
5515 }
5516
_update_resv_jobs(void * arg)5517 static void *_update_resv_jobs(void *arg)
5518 {
5519 slurmctld_resv_t *resv_ptr;
5520 uint32_t resv_id = *(uint32_t *)arg;
5521 /* get the job write lock and node and config read lock */
5522 slurmctld_lock_t job_write_lock = {
5523 READ_LOCK, WRITE_LOCK, READ_LOCK, NO_LOCK, NO_LOCK };
5524
5525 lock_slurmctld(job_write_lock);
5526 if (!resv_list) {
5527 unlock_slurmctld(job_write_lock);
5528 return SLURM_SUCCESS;
5529 }
5530
5531 resv_ptr = list_find_first(resv_list, _find_resv_id, &resv_id);
5532
5533 if (!resv_ptr) {
5534 unlock_slurmctld(job_write_lock);
5535 return SLURM_SUCCESS;
5536 }
5537
5538 list_for_each(job_list, _set_job_resvid, resv_ptr);
5539 unlock_slurmctld(job_write_lock);
5540
5541 return NULL;
5542 }
5543
5544
5545 /* Advance a expired reservation's time stamps one day or one week
5546 * as appropriate. */
_advance_resv_time(slurmctld_resv_t * resv_ptr)5547 static void _advance_resv_time(slurmctld_resv_t *resv_ptr)
5548 {
5549 time_t now;
5550 struct tm tm;
5551 int day_cnt = 0;
5552
5553 /* Make sure we have node write locks. */
5554 xassert(verify_lock(NODE_LOCK, WRITE_LOCK));
5555
5556 if (resv_ptr->flags & RESERVE_FLAG_TIME_FLOAT)
5557 return; /* Not applicable */
5558
5559 if (resv_ptr->flags & RESERVE_FLAG_DAILY) {
5560 day_cnt = 1;
5561 } else if (resv_ptr->flags & RESERVE_FLAG_WEEKDAY) {
5562 now = time(NULL);
5563 localtime_r(&now, &tm);
5564 if (tm.tm_wday == 5) /* Friday */
5565 day_cnt = 3;
5566 else if (tm.tm_wday == 6) /* Saturday */
5567 day_cnt = 2;
5568 else
5569 day_cnt = 1;
5570 } else if (resv_ptr->flags & RESERVE_FLAG_WEEKEND) {
5571 now = time(NULL);
5572 localtime_r(&now, &tm);
5573 if (tm.tm_wday == 0) /* Sunday */
5574 day_cnt = 6;
5575 else if (tm.tm_wday == 6) /* Saturday */
5576 day_cnt = 1;
5577 else
5578 day_cnt = 6 - tm.tm_wday;
5579 } else if (resv_ptr->flags & RESERVE_FLAG_WEEKLY) {
5580 day_cnt = 7;
5581 }
5582
5583 if (day_cnt) {
5584 /*
5585 * Repeated reservations need a new reservation id. Try to get a
5586 * new one and update the ID if successful.
5587 */
5588 if (_generate_resv_id()) {
5589 error("%s, Recurring reservation %s is being "
5590 "rescheduled but has the same ID.",
5591 __func__, resv_ptr->name);
5592 } else {
5593 resv_ptr->resv_id = top_suffix;
5594 /*
5595 * Update pending jobs for this reservation with the new
5596 * reservation ID out of band.
5597 */
5598 slurm_thread_create_detached(
5599 NULL, _update_resv_jobs, &resv_ptr->resv_id);
5600 }
5601
5602 verbose("Advance reservation %s by %d day(s)", resv_ptr->name,
5603 day_cnt);
5604 resv_ptr->start_time = resv_ptr->start_time_first;
5605 _advance_time(&resv_ptr->start_time, day_cnt);
5606 resv_ptr->start_time_prev = resv_ptr->start_time;
5607 resv_ptr->start_time_first = resv_ptr->start_time;
5608 _advance_time(&resv_ptr->end_time, day_cnt);
5609 _post_resv_create(resv_ptr);
5610 last_resv_update = time(NULL);
5611 schedule_resv_save();
5612 }
5613 }
5614
_free_script_arg(resv_thread_args_t * args)5615 static void _free_script_arg(resv_thread_args_t *args)
5616 {
5617 if (args) {
5618 xfree(args->script);
5619 xfree(args->resv_name);
5620 xfree(args);
5621 }
5622 }
5623
_fork_script(void * x)5624 static void *_fork_script(void *x)
5625 {
5626 resv_thread_args_t *args = (resv_thread_args_t *) x;
5627 char *argv[3], *envp[1];
5628 int status, wait_rc;
5629 pid_t cpid;
5630 uint16_t tm;
5631
5632 argv[0] = args->script;
5633 argv[1] = args->resv_name;
5634 argv[2] = NULL;
5635 envp[0] = NULL;
5636
5637 if ((cpid = fork()) < 0) {
5638 error("_fork_script fork error: %m");
5639 goto fini;
5640 }
5641 if (cpid == 0) {
5642 setpgid(0, 0);
5643 execve(argv[0], argv, envp);
5644 _exit(127);
5645 }
5646
5647 tm = slurm_get_prolog_timeout();
5648 while (1) {
5649 wait_rc = waitpid_timeout(__func__, cpid, &status, tm);
5650 if (wait_rc < 0) {
5651 if (errno == EINTR)
5652 continue;
5653 error("_fork_script waitpid error: %m");
5654 break;
5655 } else if (wait_rc > 0) {
5656 killpg(cpid, SIGKILL); /* kill children too */
5657 break;
5658 }
5659 }
5660 fini: _free_script_arg(args);
5661 return NULL;
5662 }
5663
_run_script(char * script,slurmctld_resv_t * resv_ptr)5664 static void _run_script(char *script, slurmctld_resv_t *resv_ptr)
5665 {
5666 resv_thread_args_t *args;
5667
5668 if (!script || !script[0])
5669 return;
5670 if (access(script, X_OK) < 0) {
5671 error("Invalid ResvProlog or ResvEpilog(%s): %m", script);
5672 return;
5673 }
5674
5675 args = xmalloc(sizeof(resv_thread_args_t));
5676 args->script = xstrdup(script);
5677 args->resv_name = xstrdup(resv_ptr->name);
5678
5679 slurm_thread_create_detached(NULL, _fork_script, args);
5680 }
5681
_resv_list_reset_cnt(void * x,void * arg)5682 static int _resv_list_reset_cnt(void *x, void *arg)
5683 {
5684 slurmctld_resv_t *resv_ptr = (slurmctld_resv_t *) x;
5685
5686 resv_ptr->job_pend_cnt = 0;
5687 resv_ptr->job_run_cnt = 0;
5688
5689 return 0;
5690 }
5691
5692 /* Finish scan of all jobs for valid reservations
5693 *
5694 * Purge vestigial reservation records.
5695 * Advance daily or weekly reservations that are no longer
5696 * being actively used.
5697 */
job_resv_check(void)5698 extern void job_resv_check(void)
5699 {
5700 ListIterator iter;
5701 slurmctld_resv_t *resv_ptr;
5702 time_t now = time(NULL);
5703
5704 if (!resv_list)
5705 return;
5706
5707 list_for_each(resv_list, _resv_list_reset_cnt, NULL);
5708 list_for_each(job_list, _job_resv_check, NULL);
5709
5710 iter = list_iterator_create(resv_list);
5711 while ((resv_ptr = list_next(iter))) {
5712 if (resv_ptr->start_time <= now) {
5713 if (resv_ptr->job_run_cnt || resv_ptr->job_pend_cnt)
5714 resv_ptr->idle_start_time = 0;
5715 else if (!resv_ptr->idle_start_time)
5716 resv_ptr->idle_start_time = now;
5717 }
5718
5719 if ((resv_ptr->flags & RESERVE_FLAG_PURGE_COMP) &&
5720 resv_ptr->idle_start_time &&
5721 (resv_ptr->end_time > now) &&
5722 (resv_ptr->purge_comp_time <=
5723 (now - resv_ptr->idle_start_time))) {
5724 char tmp_pct[40];
5725 secs2time_str(resv_ptr->purge_comp_time,
5726 tmp_pct, sizeof(tmp_pct));
5727 info("Reservation %s has no more jobs for %s, ending it",
5728 resv_ptr->name, tmp_pct);
5729
5730 /*
5731 * Reset time here for reoccurring reservations so we
5732 * don't continually keep running this.
5733 */
5734 resv_ptr->idle_start_time = 0;
5735
5736 (void)_post_resv_delete(resv_ptr);
5737
5738 if (!resv_ptr->run_epilog)
5739 _run_script(slurmctld_conf.resv_epilog,
5740 resv_ptr);
5741
5742 /*
5743 * If we are ending a reoccurring reservation advance
5744 * it, otherwise delete it.
5745 */
5746 if (!(resv_ptr->flags & (RESERVE_FLAG_DAILY |
5747 RESERVE_FLAG_WEEKDAY |
5748 RESERVE_FLAG_WEEKEND |
5749 RESERVE_FLAG_WEEKLY))) {
5750 /*
5751 * Clear resv ptrs on finished jobs still
5752 * pointing to this reservation.
5753 */
5754 _clear_job_resv(resv_ptr);
5755 list_delete_item(iter);
5756 } else {
5757 resv_ptr->run_prolog = false;
5758 resv_ptr->run_epilog = false;
5759 _advance_resv_time(resv_ptr);
5760 }
5761
5762 last_resv_update = now;
5763 schedule_resv_save();
5764 continue;
5765 }
5766 if ((resv_ptr->end_time >= now) ||
5767 (resv_ptr->duration && (resv_ptr->duration != NO_VAL) &&
5768 (resv_ptr->flags & RESERVE_FLAG_TIME_FLOAT))) {
5769 _validate_node_choice(resv_ptr);
5770 continue;
5771 }
5772 if (!resv_ptr->run_prolog || !resv_ptr->run_epilog)
5773 continue;
5774 _advance_resv_time(resv_ptr);
5775 if ((!resv_ptr->job_run_cnt ||
5776 (resv_ptr->flags & RESERVE_FLAG_FLEX)) &&
5777 ((resv_ptr->flags & RESERVE_FLAG_DAILY ) == 0) &&
5778 ((resv_ptr->flags & RESERVE_FLAG_WEEKDAY) == 0) &&
5779 ((resv_ptr->flags & RESERVE_FLAG_WEEKEND) == 0) &&
5780 ((resv_ptr->flags & RESERVE_FLAG_WEEKLY) == 0)) {
5781 if (resv_ptr->job_pend_cnt) {
5782 info("Purging vestigial reservation %s "
5783 "with %u pending jobs",
5784 resv_ptr->name, resv_ptr->job_pend_cnt);
5785 } else {
5786 debug("Purging vestigial reservation %s",
5787 resv_ptr->name);
5788 }
5789 _clear_job_resv(resv_ptr);
5790 list_delete_item(iter);
5791 last_resv_update = now;
5792 schedule_resv_save();
5793 }
5794 }
5795 list_iterator_destroy(iter);
5796 }
5797
5798 /*
5799 * Send all reservations to accounting. Only needed at first registration
5800 */
send_resvs_to_accounting(int db_rc)5801 extern int send_resvs_to_accounting(int db_rc)
5802 {
5803 ListIterator itr = NULL;
5804 slurmctld_resv_t *resv_ptr;
5805 slurmctld_lock_t node_write_lock = {
5806 NO_LOCK, NO_LOCK, WRITE_LOCK, READ_LOCK, NO_LOCK };
5807
5808 if (!resv_list)
5809 return SLURM_SUCCESS;
5810
5811 lock_slurmctld(node_write_lock);
5812
5813 itr = list_iterator_create(resv_list);
5814 while ((resv_ptr = list_next(itr))) {
5815 if (db_rc == ACCOUNTING_FIRST_REG)
5816 _post_resv_create(resv_ptr);
5817 else if (db_rc == ACCOUNTING_NODES_CHANGE_DB) {
5818 /*
5819 * This makes it so we always get the correct node
5820 * indexes in the database.
5821 */
5822 slurmctld_resv_t tmp_resv = {0};
5823 _post_resv_update(resv_ptr, &tmp_resv);
5824 } else {
5825 error("%s: unknown db_rc %d", __func__, db_rc);
5826 break;
5827 }
5828 }
5829 list_iterator_destroy(itr);
5830
5831 unlock_slurmctld(node_write_lock);
5832
5833 return SLURM_SUCCESS;
5834 }
5835
5836 /*
5837 * Set or clear NODE_STATE_MAINT for node_state as needed
5838 * IN reset_all - if true, then re-initialize all node information for all
5839 * reservations, but do not run any prologs or epilogs or count started
5840 * reservations
5841 * RET count of newly started reservations
5842 */
set_node_maint_mode(bool reset_all)5843 extern int set_node_maint_mode(bool reset_all)
5844 {
5845 int i, res_start_cnt = 0;
5846 node_record_t *node_ptr;
5847 uint32_t flags;
5848 ListIterator iter;
5849 slurmctld_resv_t *resv_ptr;
5850 time_t now = time(NULL);
5851
5852 if (!resv_list)
5853 return res_start_cnt;
5854
5855 flags = NODE_STATE_RES;
5856 if (reset_all)
5857 flags |= NODE_STATE_MAINT;
5858 for (i = 0, node_ptr = node_record_table_ptr;
5859 i <= node_record_count; i++, node_ptr++) {
5860 node_ptr->node_state &= (~flags);
5861 }
5862
5863 if (!reset_all) {
5864 /* NODE_STATE_RES already cleared above,
5865 * clear RESERVE_FLAG_MAINT for expired reservations */
5866 iter = list_iterator_create(resv_list);
5867 while ((resv_ptr = list_next(iter))) {
5868 if ((resv_ptr->flags_set_node) &&
5869 (resv_ptr->flags & RESERVE_FLAG_MAINT) &&
5870 ((now < resv_ptr->start_time) ||
5871 (now >= resv_ptr->end_time ))) {
5872 flags = NODE_STATE_MAINT;
5873 resv_ptr->flags_set_node = false;
5874 _set_nodes_flags(resv_ptr, now, flags,
5875 reset_all);
5876 last_node_update = now;
5877 }
5878 }
5879 list_iterator_destroy(iter);
5880 }
5881
5882 /* Set NODE_STATE_RES and possibly NODE_STATE_MAINT for nodes in all
5883 * currently active reservations */
5884 iter = list_iterator_create(resv_list);
5885 while ((resv_ptr = list_next(iter))) {
5886 if ((now >= resv_ptr->start_time) &&
5887 (now < resv_ptr->end_time )) {
5888 flags = NODE_STATE_RES;
5889 if (resv_ptr->flags & RESERVE_FLAG_MAINT)
5890 flags |= NODE_STATE_MAINT;
5891 resv_ptr->flags_set_node = true;
5892 _set_nodes_flags(resv_ptr, now, flags, reset_all);
5893 last_node_update = now;
5894 }
5895
5896 if (reset_all) /* Defer reservation prolog/epilog */
5897 continue;
5898 if ((resv_ptr->start_time <= now) && !resv_ptr->run_prolog) {
5899 res_start_cnt++;
5900 resv_ptr->run_prolog = true;
5901 _run_script(slurmctld_conf.resv_prolog, resv_ptr);
5902 }
5903 if ((resv_ptr->end_time <= now) && !resv_ptr->run_epilog) {
5904 resv_ptr->run_epilog = true;
5905 _run_script(slurmctld_conf.resv_epilog, resv_ptr);
5906 }
5907 }
5908 list_iterator_destroy(iter);
5909
5910 return res_start_cnt;
5911 }
5912
5913 /* checks if node within node_record_table_ptr is in maint reservation */
is_node_in_maint_reservation(int nodenum)5914 extern bool is_node_in_maint_reservation(int nodenum)
5915 {
5916 bool res = false;
5917 ListIterator iter;
5918 slurmctld_resv_t *resv_ptr;
5919 time_t t;
5920
5921 if (nodenum < 0 || nodenum >= node_record_count || !resv_list)
5922 return false;
5923
5924 t = time(NULL);
5925 iter = list_iterator_create(resv_list);
5926 while ((resv_ptr = list_next(iter))) {
5927 if ((resv_ptr->flags & RESERVE_FLAG_MAINT) == 0)
5928 continue;
5929 if (! (t >= resv_ptr->start_time
5930 && t <= resv_ptr->end_time))
5931 continue;
5932 if (bit_test(resv_ptr->node_bitmap, nodenum)) {
5933 res = true;
5934 break;
5935 }
5936 }
5937 list_iterator_destroy(iter);
5938
5939 return res;
5940 }
5941
update_assocs_in_resvs(void)5942 extern void update_assocs_in_resvs(void)
5943 {
5944 slurmctld_resv_t *resv_ptr = NULL;
5945 ListIterator iter = NULL;
5946 slurmctld_lock_t node_write_lock = {
5947 NO_LOCK, NO_LOCK, WRITE_LOCK, READ_LOCK, NO_LOCK };
5948
5949 if (!resv_list) {
5950 error("No reservation list given for updating associations");
5951 return;
5952 }
5953
5954 lock_slurmctld(node_write_lock);
5955
5956 iter = list_iterator_create(resv_list);
5957 while ((resv_ptr = list_next(iter)))
5958 _set_assoc_list(resv_ptr);
5959 list_iterator_destroy(iter);
5960
5961 unlock_slurmctld(node_write_lock);
5962 }
5963
update_part_nodes_in_resv(part_record_t * part_ptr)5964 extern void update_part_nodes_in_resv(part_record_t *part_ptr)
5965 {
5966 ListIterator iter = NULL;
5967 slurmctld_resv_t *resv_ptr = NULL;
5968 xassert(part_ptr);
5969
5970 iter = list_iterator_create(resv_list);
5971 while ((resv_ptr = list_next(iter))) {
5972 if ((resv_ptr->flags & RESERVE_FLAG_PART_NODES) &&
5973 (resv_ptr->partition != NULL) &&
5974 (xstrcmp(resv_ptr->partition, part_ptr->name) == 0)) {
5975 slurmctld_resv_t old_resv_ptr;
5976 memset(&old_resv_ptr, 0, sizeof(slurmctld_resv_t));
5977 old_resv_ptr.assoc_list = resv_ptr->assoc_list;
5978 old_resv_ptr.flags = resv_ptr->flags;
5979 old_resv_ptr.node_list = resv_ptr->node_list;
5980 resv_ptr->node_list = NULL;
5981 FREE_NULL_BITMAP(resv_ptr->node_bitmap);
5982 resv_ptr->node_bitmap = bit_copy(part_ptr->node_bitmap);
5983 resv_ptr->node_cnt = bit_set_count(resv_ptr->
5984 node_bitmap);
5985 resv_ptr->node_list = xstrdup(part_ptr->nodes);
5986 old_resv_ptr.tres_str = resv_ptr->tres_str;
5987 resv_ptr->tres_str = NULL;
5988 _set_tres_cnt(resv_ptr, &old_resv_ptr);
5989 old_resv_ptr.assoc_list = NULL;
5990 xfree(old_resv_ptr.tres_str);
5991 xfree(old_resv_ptr.node_list);
5992 last_resv_update = time(NULL);
5993 _set_boot_time(resv_ptr);
5994 }
5995 }
5996 list_iterator_destroy(iter);
5997 }
5998
job_borrow_from_resv_check(job_record_t * job_ptr,job_record_t * preemptor_ptr)5999 extern bool job_borrow_from_resv_check(job_record_t *job_ptr,
6000 job_record_t *preemptor_ptr)
6001 {
6002 /*
6003 * If this job is running in a reservation, but not belonging to the
6004 * reservation directly.
6005 */
6006 if (preemptor_ptr->resv_ptr &&
6007 preemptor_ptr->resv_ptr->max_start_delay &&
6008 preemptor_ptr->resv_ptr->node_bitmap &&
6009 (job_ptr->warn_flags & KILL_JOB_RESV) &&
6010 job_ptr->node_bitmap &&
6011 bit_overlap_any(job_ptr->node_bitmap,
6012 preemptor_ptr->resv_ptr->node_bitmap))
6013 return true;
6014 return false;
6015 }
6016
_set_nodes_flags(slurmctld_resv_t * resv_ptr,time_t now,uint32_t flags,bool reset_all)6017 static void _set_nodes_flags(slurmctld_resv_t *resv_ptr, time_t now,
6018 uint32_t flags, bool reset_all)
6019 {
6020 int i, i_first, i_last;
6021 node_record_t *node_ptr;
6022 uint32_t old_state;
6023
6024 if (!resv_ptr->node_bitmap) {
6025 if ((resv_ptr->flags & RESERVE_FLAG_ANY_NODES) == 0) {
6026 error("%s: reservation %s lacks a bitmap",
6027 __func__, resv_ptr->name);
6028 }
6029 return;
6030 }
6031
6032 i_first = bit_ffs(resv_ptr->node_bitmap);
6033 if (i_first < 0) {
6034 if ((resv_ptr->flags & RESERVE_FLAG_ANY_NODES) == 0) {
6035 error("%s: reservation %s includes no nodes",
6036 __func__, resv_ptr->name);
6037 }
6038 return;
6039 }
6040 i_last = bit_fls(resv_ptr->node_bitmap);
6041 for (i = i_first; i <= i_last; i++) {
6042 if (!bit_test(resv_ptr->node_bitmap, i))
6043 continue;
6044
6045 node_ptr = node_record_table_ptr + i;
6046 old_state = node_ptr->node_state;
6047 if (resv_ptr->flags_set_node)
6048 node_ptr->node_state |= flags;
6049 else
6050 node_ptr->node_state &= (~flags);
6051 /* mark that this node is now down if maint mode flag changed */
6052 bool state_change = ((old_state ^ node_ptr->node_state)
6053 & NODE_STATE_MAINT) || reset_all;
6054 if (state_change && (IS_NODE_DOWN(node_ptr) ||
6055 IS_NODE_DRAIN(node_ptr) ||
6056 IS_NODE_FAIL(node_ptr))) {
6057 clusteracct_storage_g_node_down(
6058 acct_db_conn,
6059 node_ptr, now, NULL,
6060 slurmctld_conf.slurm_user_id);
6061 }
6062 }
6063 }
6064
job_resv_append_promiscuous(job_queue_req_t * job_queue_req)6065 extern void job_resv_append_promiscuous(job_queue_req_t *job_queue_req)
6066 {
6067 if (!prom_resv_list || !list_count(prom_resv_list))
6068 return;
6069
6070 list_for_each(prom_resv_list, _queue_prom_resv, job_queue_req);
6071 }
6072
job_resv_clear_promiscous_flag(job_record_t * job_ptr)6073 extern void job_resv_clear_promiscous_flag(job_record_t *job_ptr)
6074 {
6075 if (!(job_ptr->bit_flags & JOB_PROM) ||
6076 (job_ptr->job_state & JOB_RUNNING))
6077 return;
6078
6079 xfree(job_ptr->resv_name);
6080 job_ptr->resv_id = 0;
6081 job_ptr->resv_ptr = NULL;
6082 job_ptr->bit_flags &= (~JOB_PROM);
6083 }
6084