1#!/usr/bin/env expect
2############################################################################
3# Purpose: Test of Slurm sstat functionality
4#          sstat a, n, o, p, P and v options.
5############################################################################
6# Copyright (C) 2009 Lawrence Livermore National Security.
7# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
8# Written by Joseph Donaghy <donaghy1@llnl.gov>
9# CODE-OCEC-09-009. All rights reserved.
10#
11# This file is part of Slurm, a resource management program.
12# For details, see <https://slurm.schedmd.com/>.
13# Please also read the included file: DISCLAIMER.
14#
15# Slurm is free software; you can redistribute it and/or modify it under
16# the terms of the GNU General Public License as published by the Free
17# Software Foundation; either version 2 of the License, or (at your option)
18# any later version.
19#
20# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
21# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
22# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
23# details.
24#
25# You should have received a copy of the GNU General Public License along
26# with Slurm; if not, write to the Free Software Foundation, Inc.,
27# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
28############################################################################
29source ./globals
30
31set exit_code   0
32set file_in1    "test$test_id.input1"
33set file_in2    "test$test_id.input2"
34set file_in3    "test$test_id.input3"
35set test_acct   "test$test_id\_acct"
36set timeout     $max_job_delay
37
38proc cleanup {} {
39	global bin_rm file_in1 file_in2 file_in3
40
41	exec $bin_rm -f $file_in1 $file_in2 $file_in3
42}
43
44set ac		AveCPU
45set ap		AvePages
46set ar		AveRSS
47set av		AveVMSize
48set ji		JobID
49set mp		MaxPages
50set mpn		MaxPagesNode
51set mpt		MaxPagesTask
52set mr		MaxRSS
53set mrn		MaxRSSNode
54set mrt		MaxRSSTask
55set mvs		MaxVMSize
56set mvn		MaxVMSizeNode
57set mvt		MaxVMSizeTask
58set mc		MinCPU
59set mn		MinCPUNode
60set mt		MinCPUTask
61set nt		NTasks
62
63#
64# Check accounting config and bail if not found.
65#
66if {[get_config_param "AccountingStorageType"] ne "accounting_storage/slurmdbd"} {
67	skip "This test can't be run without a usable AccountStorageType"
68}
69
70if {[get_config_param "FrontendName"] ne "MISSING"} {
71	skip "This test can't be run on a front-end system"
72}
73
74if {[string compare [get_admin_level] "Administrator"]} {
75	skip "This test can't be run without being an Accounting administrator.\nUse: sacctmgr mod user \$USER set admin=admin"
76}
77
78#
79# Identify the user and his current default account
80#
81set acct_name ""
82set user_name [get_my_user_name]
83
84set s_pid [spawn $sacctmgr show user $user_name]
85expect {
86	-re "$user_name *($re_word_str)" {
87		set acct_name $expect_out(1,string)
88		exp_continue
89	}
90	timeout {
91		slow_kill $s_pid
92		fail "sacctmgr add not responding"
93	}
94	eof {
95		wait
96	}
97}
98
99#
100# Use sacctmgr to add an account
101#
102set aamatches 0
103set sadd_pid [spawn $sacctmgr -i add account $test_acct]
104expect {
105	-re "Adding Account" {
106		incr aamatches
107		exp_continue
108	}
109	-re "Nothing new added" {
110		log_warn "Vestigial account $test_acct found"
111		incr aamatches
112		exp_continue
113	}
114	timeout {
115		slow_kill $sadd_pid
116		log_error "sacctmgr add not responding"
117	}
118	eof {
119		wait
120	}
121}
122if {$aamatches != 1} {
123	log_error "sacctmgr had a problem adding account"
124}
125
126#
127# Add self to this new account
128#
129set sadd_pid [spawn $sacctmgr -i create user name=$user_name account=$test_acct]
130expect {
131	 timeout {
132		slow_kill $sadd_pid
133		fail "sacctmgr add not responding"
134	}
135	eof {
136		wait
137	}
138}
139
140log_user 0
141set real_memory 0
142spawn $scontrol show node
143expect {
144	-re "RealMemory=($number) " {
145		set real_memory $expect_out(1,string)
146#		exp_continue
147	}
148	eof {
149		wait
150	}
151}
152log_user 1
153if {$real_memory < 100} {
154	set job_mem_opt  "--comment=no_mem_limit"
155	set step_mem_opt "-t1"
156} else {
157	set job_mem_opt  "--mem=100"
158	set step_mem_opt "--mem=20"
159}
160
161#
162# Build input script file1
163#
164make_bash_script $file_in1 "
165  $srun $step_mem_opt --gres=craynetwork:0 sleep 60 &
166  $srun $step_mem_opt --gres=craynetwork:0 sleep 60 &
167  $srun $step_mem_opt --gres=craynetwork:0 sleep 60
168"
169#
170# Spawn a job via srun using this account
171# NOTE: --mem option here and in scripts above to permit running more than one
172#       step per job if DefMemPerCPU or DefMemPerNode is configured.
173#
174set job_id3 0
175spawn $sbatch -N1 -t1 -v $job_mem_opt --gres=craynetwork:0 --account=$test_acct --output=/dev/null $file_in1
176expect {
177	-re "Submitted batch job ($number)" {
178		set job_id3 $expect_out(1,string)
179		log_debug "Found JobID to be $job_id3"
180		exp_continue
181	}
182	timeout {
183		fail "sbatch not responding"
184	}
185	eof {
186		wait
187	}
188}
189if {$job_id3 == 0} {
190	log_error "Did not get srun job_id3"
191	set exit_code 1
192} else {
193	set matches 0
194	spawn $scontrol show job $job_id3
195	expect {
196		 -re "Account=$test_acct" {
197			incr matches
198			exp_continue
199		}
200		timeout {
201			fail "scontrol not responding"
202		}
203		eof {
204			wait
205		}
206	}
207	if {$matches != 1} {
208		log_error "srun failed to use specified account"
209		set exit_code 1
210	}
211}
212
213#
214# Wait for job to start running, then status it
215#
216if {[wait_for_job $job_id3 "RUNNING"] != 0} {
217	fail "Waiting for job to start running"
218}
219sleep 5
220################################################################
221#
222# Proc: sstat_job
223#
224# Purpose:  Pass sstat options and test
225#
226# Returns: Number of matches.
227#
228# Input: Switch options not requiring arguments
229#
230################################################################
231
232proc sstat_job { soption job_id step_id } {
233	global sstat number
234	set debug       0
235	set matches     0
236	set not_support 0
237	log_debug "sstat -$soption -p -j $job_id"
238
239	if { $soption == "-allsteps" || $soption == "a" } {
240		spawn $sstat -$soption -p -j $job_id
241		expect {
242			-re "Slurm accounting storage is disabled" {
243				set not_support 1
244				exp_continue
245			}
246			-re "$job_id\\.($number)" {
247				if {$debug} {log_debug "Match $expect_out(1,string)"}
248				incr matches
249				exp_continue
250			}
251			-re "$job_id\\.batch" {
252				if {$debug} {log_debug "Match batch"}
253				incr matches
254				exp_continue
255			}
256			timeout {
257				fail "sstat not responding"
258			}
259			eof {
260				wait
261			}
262		}
263		if {$not_support != 0} {
264			skip "Can not test without accounting enabled"
265		}
266		return $matches
267	}
268
269	if { $soption == "-noheader" || $soption == "n" } {
270		spawn $sstat -$soption -p -j $job_id
271		expect {
272			-re "Slurm accounting storage is disabled" {
273				set not_support 1
274				exp_continue
275			}
276			-re "JobID|MaxVMSize|MaxVMSizeNode|MaxVMSizeTask|	\
277				AveVMSize|MaxRSS|MaxRSSNode|MaxRSSTask|AveRSS|	\
278				MaxPages|MaxPagesNode|MaxPagesTask|AvePages|	\
279				MinCPU|MinCPUNode|MinCPUTask|AveCPU|NTasks" {
280				if {$debug} {log_debug "Match4"}
281				incr matches
282				exp_continue
283			}
284			-re "$job_id" {
285				if {$debug} {log_debug "Match5"}
286				incr matches
287				exp_continue
288			}
289			timeout {
290				fail "sstat not responding"
291			}
292			eof {
293				wait
294			}
295		}
296		if {$not_support != 0} {
297			skip "Can not test without accounting enabled"
298		}
299		return $matches
300	}
301
302	if { $soption == "-parsable" || $soption == "p" } {
303		spawn $sstat -$soption -p -j $job_id.$step_id
304		expect {
305			-re "Slurm accounting storage is disabled" {
306				set not_support 1
307				exp_continue
308			}
309			-re "JobID\\|MaxVMSize\\|MaxVMSizeNode\\|MaxVMSizeTask\\|AveVMSize\\|MaxRSS\\|" {
310				if {$debug} {log_debug "Match6"}
311				incr matches
312				exp_continue
313			}
314			-re "MaxRSSNode\\|MaxRSSTask\\|AveRSS\\|MaxPages\\|MaxPagesNode\\|MaxPagesTask\\|" {
315				if {$debug} {log_debug "Match7"}
316				incr matches
317				exp_continue
318			}
319			-re "AvePages\\|MinCPU\\|MinCPUNode\\|MinCPUTask\\|AveCPU\\|NTasks\\|" {
320				if {$debug} {log_debug "Match8"}
321				incr matches
322				exp_continue
323			}
324			-re "$job_id\\.$step_id\\|" {
325				if {$debug} {log_debug "Match9"}
326				incr matches
327				exp_continue
328			}
329			timeout {
330				fail "sstat not responding"
331			}
332			eof {
333				wait
334			}
335		}
336		if {$not_support != 0} {
337			skip "Can not test without accounting enabled"
338		}
339		return $matches
340	}
341
342	if { $soption == "-parsable2" || $soption == "P" } {
343		spawn $sstat -$soption -j $job_id.$step_id
344		expect {
345			-re "Slurm accounting storage is disabled" {
346				set not_support 1
347				exp_continue
348			}
349			-re "JobID\\|MaxVMSize\\|MaxVMSizeNode\\|MaxVMSizeTask\\|AveVMSize\\|MaxRSS\\|" {
350				if {$debug} {log_debug "Match10"}
351				incr matches
352				exp_continue
353			}
354			-re "MaxRSSNode\\|MaxRSSTask\\|AveRSS\\|MaxPages\\|MaxPagesNode\\|MaxPagesTask\\|" {
355				if {$debug} {log_debug "Match11"}
356				incr matches
357				exp_continue
358			}
359			-re "AvePages\\|MinCPU\\|MinCPUNode\\|MinCPUTask\\|AveCPU\\|NTasks" {
360				if {$debug} {log_debug "Match12"}
361				incr matches
362				exp_continue
363			}
364			-re "$job_id\\.$step_id\\|" {
365				if {$debug} {log_debug "Match13"}
366				incr matches
367				exp_continue
368			}
369			timeout {
370				fail "sstat not responding"
371			}
372			eof {
373				wait
374			}
375		}
376		if {$not_support != 0} {
377			skip "Can not test without accounting enabled"
378		}
379		return $matches
380	}
381
382	if { $soption == "-verbose" || $soption == "v" } {
383		spawn $sstat -$soption -p -j $job_id
384		expect {
385			-re "Slurm accounting storage is disabled" {
386				set not_support 1
387				exp_continue
388			}
389			-re "JobID.MaxVMSize.MaxVMSizeNode.MaxVMSizeTask.AveVMSize.MaxRSS" {
390				if {$debug} {log_debug "Match14"}
391				incr matches
392				exp_continue
393			}
394			-re "MaxRSSNode.MaxRSSTask.AveRSS.MaxPages.MaxPagesNode.MaxPagesTask" {
395				if {$debug} {log_debug "Match15"}
396				incr matches
397				exp_continue
398			}
399			-re "AvePages.MinCPU.MinCPUNode.MinCPUTask.AveCPU.NTasks" {
400				if {$debug} {log_debug "Match16"}
401				incr matches
402				exp_continue
403			}
404			-re "$job_id" {
405				if {$debug} {log_debug "Match17"}
406				incr matches
407				exp_continue
408			}
409			timeout {
410				fail "sstat not responding"
411			}
412			eof {
413				wait
414			}
415		}
416		if {$not_support != 0} {
417			skip "Can not test without accounting enabled"
418		}
419		return $matches
420	}
421}
422
423################################################################
424#
425# Proc: sstat_vargs
426#
427# Purpose:  Pass sstat options with arguments and test
428#
429# Returns: Number of matches.
430#
431# Input: Switch options with argument
432#
433################################################################
434
435proc sstat_vargs { soption vargs job_id} {
436	global sstat
437	set debug	0
438	set matches     0
439	set not_support 0
440	log_debug "sstat -$soption $vargs -p -j $job_id"
441
442	if { $soption == "o" || $soption == "-format" } {
443		spawn $sstat -$soption $vargs -p -j $job_id
444		expect {
445			-re "Slurm accounting storage is disabled" {
446				set not_support 1
447				exp_continue
448			}
449			-re "AveCPU.AvePages.AveRSS.AveVMSize" {
450				if {$debug} {log_debug "Match18"}
451				incr matches
452				exp_continue
453			}
454			-re "JobID.MaxPages.MaxPagesNode.MaxPagesTask" {
455				if {$debug} {log_debug "Match19"}
456				incr matches
457				exp_continue
458			}
459			-re "MaxRSS.MaxRSSNode.MaxRSSTask.MaxVMSize" {
460				if {$debug} {log_debug "Match20"}
461				incr matches
462				exp_continue
463			}
464			-re "MaxVMSizeNode.MaxVMSizeTask.MinCPU.MinCPUNode" {
465				if {$debug} {log_debug "Match21"}
466				incr matches
467				exp_continue
468			}
469			-re "MinCPUTask.NTasks" {
470				if {$debug} {log_debug "Match22"}
471				incr matches
472				exp_continue
473			}
474			-re "$job_id" {
475				if {$debug} {log_debug "Match23"}
476				incr matches
477				exp_continue
478			}
479			timeout {
480				fail "sstat not responding"
481			}
482			eof {
483				wait
484			}
485		}
486		if {$not_support != 0} {
487			skip "Can not test without accounting enabled"
488		}
489		return $matches
490	}
491}
492################################################################
493set matches [sstat_job a $job_id3 ""]
494if {$matches != 4} {
495	log_error "sstat -a failed ($matches != 3)"
496	set exit_code 1
497}
498
499set matches [sstat_job -allsteps $job_id3 ""]
500if {$matches != 4} {
501	log_error "sstat --allsteps failed ($matches != 3)"
502	set exit_code 1
503}
504
505set matches [sstat_job n $job_id3 ""]
506if {$matches != 1} {
507	log_error "sstat -n failed ($matches != 1)"
508	set exit_code 1
509}
510
511set matches [sstat_job -noheader $job_id3 ""]
512if {$matches != 1} {
513	log_error "sstat --noheader failed ($matches != 1)"
514	set exit_code 1
515}
516
517set matches [sstat_job p $job_id3 0]
518if {$matches != 4} {
519	log_error "sstat -p failed ($matches != 4)"
520	set exit_code 1
521}
522
523set matches [sstat_job -parsable $job_id3 0]
524if {$matches != 4} {
525	log_error "sstat --parsable failed ($matches != 4)"
526	set exit_code 1
527}
528
529set matches [sstat_job P $job_id3 0]
530if {$matches != 4} {
531	log_error "sstat -P failed ($matches != 4)"
532	set exit_code 1
533}
534
535set matches [sstat_job -parsable2 $job_id3 0]
536if {$matches != 4} {
537	log_error "sstat --parsable2 failed ($matches != 4)"
538	set exit_code 1
539}
540
541set matches [sstat_job v $job_id3 0]
542if {$matches != 4} {
543	log_error "sstat -v failed ($matches != 4)"
544	set exit_code 1
545}
546
547set matches [sstat_job -verbose $job_id3 0]
548if {$matches != 4} {
549	log_error "sstat --verbose failed ($matches != 4)"
550	set exit_code 1
551}
552
553set matches [sstat_vargs o $ac,$ap,$ar,$av,$ji,$mp,$mpn,$mpt,$mr,$mrn,$mrt,$mvs,$mvn,$mvt,$mc,$mn,$mt,$nt $job_id3]
554if {$matches != 6} {
555	log_error "sstat -o failed ($matches != 6)"
556	set exit_code 1
557}
558
559set matches [sstat_vargs -format $ac,$ap,$ar,$av,$ji,$mp,$mpn,$mpt,$mr,$mrn,$mrt,$mvs,$mvn,$mvt,$mc,$mn,$mt,$nt $job_id3]
560if {$matches != 6} {
561	log_error "sstat --format failed ($matches != 6)"
562	set exit_code 1
563}
564
565#
566# Use sacctmgr to delete the test account (shouldn't work, because of running jobs)
567#
568set damatches 0
569set needtodel 0
570set sadel_pid [spawn $sacctmgr -i delete account $test_acct]
571expect {
572	-re "Deleting account" {
573		incr damatches
574		exp_continue
575	}
576	"Job(s) active, cancel job(s) before remove" {
577		log_debug "This error was expected, no worries"
578		set needtodel 1
579	}
580
581	timeout {
582		slow_kill $sadel_pid
583		fail "sacctmgr delete not responding"
584	}
585	eof {
586		wait
587	}
588}
589
590if {$needtodel == 1} {
591	#
592	# Use scancel to quite jobs
593	#
594	spawn $scancel -A $test_acct
595	expect {
596		eof {
597			wait
598		}
599	}
600
601	if {[wait_for_job $job_id3 DONE] != 0} {
602		log_error "Error completing job $job_id3"
603		cancel_job $job_id3
604		set exit_code 1
605	}
606
607	#
608	# Use sacctmgr to delete the test account
609	#
610	set damatches 0
611	set sadel_pid [spawn $sacctmgr -i delete account $test_acct]
612	expect {
613		-re "Deleting account" {
614			incr damatches
615			exp_continue
616		}
617
618		timeout {
619			slow_kill $sadel_pid
620			fail "sacctmgr delete not responding"
621		}
622		eof {
623			wait
624		}
625	}
626}
627
628if {$damatches != 1} {
629	fail "sacctmgr had problems deleting the account"
630}
631
632if {$exit_code != 0} {
633	fail "Test failed due to previous errors (\$exit_code = $exit_code)"
634}
635