1#!/usr/bin/env expect 2############################################################################ 3# Purpose: Test of Slurm sstat functionality 4# sstat a, n, o, p, P and v options. 5############################################################################ 6# Copyright (C) 2009 Lawrence Livermore National Security. 7# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). 8# Written by Joseph Donaghy <donaghy1@llnl.gov> 9# CODE-OCEC-09-009. All rights reserved. 10# 11# This file is part of Slurm, a resource management program. 12# For details, see <https://slurm.schedmd.com/>. 13# Please also read the included file: DISCLAIMER. 14# 15# Slurm is free software; you can redistribute it and/or modify it under 16# the terms of the GNU General Public License as published by the Free 17# Software Foundation; either version 2 of the License, or (at your option) 18# any later version. 19# 20# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY 21# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 22# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 23# details. 24# 25# You should have received a copy of the GNU General Public License along 26# with Slurm; if not, write to the Free Software Foundation, Inc., 27# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 28############################################################################ 29source ./globals 30 31set exit_code 0 32set file_in1 "test$test_id.input1" 33set file_in2 "test$test_id.input2" 34set file_in3 "test$test_id.input3" 35set test_acct "test$test_id\_acct" 36set timeout $max_job_delay 37 38proc cleanup {} { 39 global bin_rm file_in1 file_in2 file_in3 40 41 exec $bin_rm -f $file_in1 $file_in2 $file_in3 42} 43 44set ac AveCPU 45set ap AvePages 46set ar AveRSS 47set av AveVMSize 48set ji JobID 49set mp MaxPages 50set mpn MaxPagesNode 51set mpt MaxPagesTask 52set mr MaxRSS 53set mrn MaxRSSNode 54set mrt MaxRSSTask 55set mvs MaxVMSize 56set mvn MaxVMSizeNode 57set mvt MaxVMSizeTask 58set mc MinCPU 59set mn MinCPUNode 60set mt MinCPUTask 61set nt NTasks 62 63# 64# Check accounting config and bail if not found. 65# 66if {[get_config_param "AccountingStorageType"] ne "accounting_storage/slurmdbd"} { 67 skip "This test can't be run without a usable AccountStorageType" 68} 69 70if {[get_config_param "FrontendName"] ne "MISSING"} { 71 skip "This test can't be run on a front-end system" 72} 73 74if {[string compare [get_admin_level] "Administrator"]} { 75 skip "This test can't be run without being an Accounting administrator.\nUse: sacctmgr mod user \$USER set admin=admin" 76} 77 78# 79# Identify the user and his current default account 80# 81set acct_name "" 82set user_name [get_my_user_name] 83 84set s_pid [spawn $sacctmgr show user $user_name] 85expect { 86 -re "$user_name *($re_word_str)" { 87 set acct_name $expect_out(1,string) 88 exp_continue 89 } 90 timeout { 91 slow_kill $s_pid 92 fail "sacctmgr add not responding" 93 } 94 eof { 95 wait 96 } 97} 98 99# 100# Use sacctmgr to add an account 101# 102set aamatches 0 103set sadd_pid [spawn $sacctmgr -i add account $test_acct] 104expect { 105 -re "Adding Account" { 106 incr aamatches 107 exp_continue 108 } 109 -re "Nothing new added" { 110 log_warn "Vestigial account $test_acct found" 111 incr aamatches 112 exp_continue 113 } 114 timeout { 115 slow_kill $sadd_pid 116 log_error "sacctmgr add not responding" 117 } 118 eof { 119 wait 120 } 121} 122if {$aamatches != 1} { 123 log_error "sacctmgr had a problem adding account" 124} 125 126# 127# Add self to this new account 128# 129set sadd_pid [spawn $sacctmgr -i create user name=$user_name account=$test_acct] 130expect { 131 timeout { 132 slow_kill $sadd_pid 133 fail "sacctmgr add not responding" 134 } 135 eof { 136 wait 137 } 138} 139 140log_user 0 141set real_memory 0 142spawn $scontrol show node 143expect { 144 -re "RealMemory=($number) " { 145 set real_memory $expect_out(1,string) 146# exp_continue 147 } 148 eof { 149 wait 150 } 151} 152log_user 1 153if {$real_memory < 100} { 154 set job_mem_opt "--comment=no_mem_limit" 155 set step_mem_opt "-t1" 156} else { 157 set job_mem_opt "--mem=100" 158 set step_mem_opt "--mem=20" 159} 160 161# 162# Build input script file1 163# 164make_bash_script $file_in1 " 165 $srun $step_mem_opt --gres=craynetwork:0 sleep 60 & 166 $srun $step_mem_opt --gres=craynetwork:0 sleep 60 & 167 $srun $step_mem_opt --gres=craynetwork:0 sleep 60 168" 169# 170# Spawn a job via srun using this account 171# NOTE: --mem option here and in scripts above to permit running more than one 172# step per job if DefMemPerCPU or DefMemPerNode is configured. 173# 174set job_id3 0 175spawn $sbatch -N1 -t1 -v $job_mem_opt --gres=craynetwork:0 --account=$test_acct --output=/dev/null $file_in1 176expect { 177 -re "Submitted batch job ($number)" { 178 set job_id3 $expect_out(1,string) 179 log_debug "Found JobID to be $job_id3" 180 exp_continue 181 } 182 timeout { 183 fail "sbatch not responding" 184 } 185 eof { 186 wait 187 } 188} 189if {$job_id3 == 0} { 190 log_error "Did not get srun job_id3" 191 set exit_code 1 192} else { 193 set matches 0 194 spawn $scontrol show job $job_id3 195 expect { 196 -re "Account=$test_acct" { 197 incr matches 198 exp_continue 199 } 200 timeout { 201 fail "scontrol not responding" 202 } 203 eof { 204 wait 205 } 206 } 207 if {$matches != 1} { 208 log_error "srun failed to use specified account" 209 set exit_code 1 210 } 211} 212 213# 214# Wait for job to start running, then status it 215# 216if {[wait_for_job $job_id3 "RUNNING"] != 0} { 217 fail "Waiting for job to start running" 218} 219sleep 5 220################################################################ 221# 222# Proc: sstat_job 223# 224# Purpose: Pass sstat options and test 225# 226# Returns: Number of matches. 227# 228# Input: Switch options not requiring arguments 229# 230################################################################ 231 232proc sstat_job { soption job_id step_id } { 233 global sstat number 234 set debug 0 235 set matches 0 236 set not_support 0 237 log_debug "sstat -$soption -p -j $job_id" 238 239 if { $soption == "-allsteps" || $soption == "a" } { 240 spawn $sstat -$soption -p -j $job_id 241 expect { 242 -re "Slurm accounting storage is disabled" { 243 set not_support 1 244 exp_continue 245 } 246 -re "$job_id\\.($number)" { 247 if {$debug} {log_debug "Match $expect_out(1,string)"} 248 incr matches 249 exp_continue 250 } 251 -re "$job_id\\.batch" { 252 if {$debug} {log_debug "Match batch"} 253 incr matches 254 exp_continue 255 } 256 timeout { 257 fail "sstat not responding" 258 } 259 eof { 260 wait 261 } 262 } 263 if {$not_support != 0} { 264 skip "Can not test without accounting enabled" 265 } 266 return $matches 267 } 268 269 if { $soption == "-noheader" || $soption == "n" } { 270 spawn $sstat -$soption -p -j $job_id 271 expect { 272 -re "Slurm accounting storage is disabled" { 273 set not_support 1 274 exp_continue 275 } 276 -re "JobID|MaxVMSize|MaxVMSizeNode|MaxVMSizeTask| \ 277 AveVMSize|MaxRSS|MaxRSSNode|MaxRSSTask|AveRSS| \ 278 MaxPages|MaxPagesNode|MaxPagesTask|AvePages| \ 279 MinCPU|MinCPUNode|MinCPUTask|AveCPU|NTasks" { 280 if {$debug} {log_debug "Match4"} 281 incr matches 282 exp_continue 283 } 284 -re "$job_id" { 285 if {$debug} {log_debug "Match5"} 286 incr matches 287 exp_continue 288 } 289 timeout { 290 fail "sstat not responding" 291 } 292 eof { 293 wait 294 } 295 } 296 if {$not_support != 0} { 297 skip "Can not test without accounting enabled" 298 } 299 return $matches 300 } 301 302 if { $soption == "-parsable" || $soption == "p" } { 303 spawn $sstat -$soption -p -j $job_id.$step_id 304 expect { 305 -re "Slurm accounting storage is disabled" { 306 set not_support 1 307 exp_continue 308 } 309 -re "JobID\\|MaxVMSize\\|MaxVMSizeNode\\|MaxVMSizeTask\\|AveVMSize\\|MaxRSS\\|" { 310 if {$debug} {log_debug "Match6"} 311 incr matches 312 exp_continue 313 } 314 -re "MaxRSSNode\\|MaxRSSTask\\|AveRSS\\|MaxPages\\|MaxPagesNode\\|MaxPagesTask\\|" { 315 if {$debug} {log_debug "Match7"} 316 incr matches 317 exp_continue 318 } 319 -re "AvePages\\|MinCPU\\|MinCPUNode\\|MinCPUTask\\|AveCPU\\|NTasks\\|" { 320 if {$debug} {log_debug "Match8"} 321 incr matches 322 exp_continue 323 } 324 -re "$job_id\\.$step_id\\|" { 325 if {$debug} {log_debug "Match9"} 326 incr matches 327 exp_continue 328 } 329 timeout { 330 fail "sstat not responding" 331 } 332 eof { 333 wait 334 } 335 } 336 if {$not_support != 0} { 337 skip "Can not test without accounting enabled" 338 } 339 return $matches 340 } 341 342 if { $soption == "-parsable2" || $soption == "P" } { 343 spawn $sstat -$soption -j $job_id.$step_id 344 expect { 345 -re "Slurm accounting storage is disabled" { 346 set not_support 1 347 exp_continue 348 } 349 -re "JobID\\|MaxVMSize\\|MaxVMSizeNode\\|MaxVMSizeTask\\|AveVMSize\\|MaxRSS\\|" { 350 if {$debug} {log_debug "Match10"} 351 incr matches 352 exp_continue 353 } 354 -re "MaxRSSNode\\|MaxRSSTask\\|AveRSS\\|MaxPages\\|MaxPagesNode\\|MaxPagesTask\\|" { 355 if {$debug} {log_debug "Match11"} 356 incr matches 357 exp_continue 358 } 359 -re "AvePages\\|MinCPU\\|MinCPUNode\\|MinCPUTask\\|AveCPU\\|NTasks" { 360 if {$debug} {log_debug "Match12"} 361 incr matches 362 exp_continue 363 } 364 -re "$job_id\\.$step_id\\|" { 365 if {$debug} {log_debug "Match13"} 366 incr matches 367 exp_continue 368 } 369 timeout { 370 fail "sstat not responding" 371 } 372 eof { 373 wait 374 } 375 } 376 if {$not_support != 0} { 377 skip "Can not test without accounting enabled" 378 } 379 return $matches 380 } 381 382 if { $soption == "-verbose" || $soption == "v" } { 383 spawn $sstat -$soption -p -j $job_id 384 expect { 385 -re "Slurm accounting storage is disabled" { 386 set not_support 1 387 exp_continue 388 } 389 -re "JobID.MaxVMSize.MaxVMSizeNode.MaxVMSizeTask.AveVMSize.MaxRSS" { 390 if {$debug} {log_debug "Match14"} 391 incr matches 392 exp_continue 393 } 394 -re "MaxRSSNode.MaxRSSTask.AveRSS.MaxPages.MaxPagesNode.MaxPagesTask" { 395 if {$debug} {log_debug "Match15"} 396 incr matches 397 exp_continue 398 } 399 -re "AvePages.MinCPU.MinCPUNode.MinCPUTask.AveCPU.NTasks" { 400 if {$debug} {log_debug "Match16"} 401 incr matches 402 exp_continue 403 } 404 -re "$job_id" { 405 if {$debug} {log_debug "Match17"} 406 incr matches 407 exp_continue 408 } 409 timeout { 410 fail "sstat not responding" 411 } 412 eof { 413 wait 414 } 415 } 416 if {$not_support != 0} { 417 skip "Can not test without accounting enabled" 418 } 419 return $matches 420 } 421} 422 423################################################################ 424# 425# Proc: sstat_vargs 426# 427# Purpose: Pass sstat options with arguments and test 428# 429# Returns: Number of matches. 430# 431# Input: Switch options with argument 432# 433################################################################ 434 435proc sstat_vargs { soption vargs job_id} { 436 global sstat 437 set debug 0 438 set matches 0 439 set not_support 0 440 log_debug "sstat -$soption $vargs -p -j $job_id" 441 442 if { $soption == "o" || $soption == "-format" } { 443 spawn $sstat -$soption $vargs -p -j $job_id 444 expect { 445 -re "Slurm accounting storage is disabled" { 446 set not_support 1 447 exp_continue 448 } 449 -re "AveCPU.AvePages.AveRSS.AveVMSize" { 450 if {$debug} {log_debug "Match18"} 451 incr matches 452 exp_continue 453 } 454 -re "JobID.MaxPages.MaxPagesNode.MaxPagesTask" { 455 if {$debug} {log_debug "Match19"} 456 incr matches 457 exp_continue 458 } 459 -re "MaxRSS.MaxRSSNode.MaxRSSTask.MaxVMSize" { 460 if {$debug} {log_debug "Match20"} 461 incr matches 462 exp_continue 463 } 464 -re "MaxVMSizeNode.MaxVMSizeTask.MinCPU.MinCPUNode" { 465 if {$debug} {log_debug "Match21"} 466 incr matches 467 exp_continue 468 } 469 -re "MinCPUTask.NTasks" { 470 if {$debug} {log_debug "Match22"} 471 incr matches 472 exp_continue 473 } 474 -re "$job_id" { 475 if {$debug} {log_debug "Match23"} 476 incr matches 477 exp_continue 478 } 479 timeout { 480 fail "sstat not responding" 481 } 482 eof { 483 wait 484 } 485 } 486 if {$not_support != 0} { 487 skip "Can not test without accounting enabled" 488 } 489 return $matches 490 } 491} 492################################################################ 493set matches [sstat_job a $job_id3 ""] 494if {$matches != 4} { 495 log_error "sstat -a failed ($matches != 3)" 496 set exit_code 1 497} 498 499set matches [sstat_job -allsteps $job_id3 ""] 500if {$matches != 4} { 501 log_error "sstat --allsteps failed ($matches != 3)" 502 set exit_code 1 503} 504 505set matches [sstat_job n $job_id3 ""] 506if {$matches != 1} { 507 log_error "sstat -n failed ($matches != 1)" 508 set exit_code 1 509} 510 511set matches [sstat_job -noheader $job_id3 ""] 512if {$matches != 1} { 513 log_error "sstat --noheader failed ($matches != 1)" 514 set exit_code 1 515} 516 517set matches [sstat_job p $job_id3 0] 518if {$matches != 4} { 519 log_error "sstat -p failed ($matches != 4)" 520 set exit_code 1 521} 522 523set matches [sstat_job -parsable $job_id3 0] 524if {$matches != 4} { 525 log_error "sstat --parsable failed ($matches != 4)" 526 set exit_code 1 527} 528 529set matches [sstat_job P $job_id3 0] 530if {$matches != 4} { 531 log_error "sstat -P failed ($matches != 4)" 532 set exit_code 1 533} 534 535set matches [sstat_job -parsable2 $job_id3 0] 536if {$matches != 4} { 537 log_error "sstat --parsable2 failed ($matches != 4)" 538 set exit_code 1 539} 540 541set matches [sstat_job v $job_id3 0] 542if {$matches != 4} { 543 log_error "sstat -v failed ($matches != 4)" 544 set exit_code 1 545} 546 547set matches [sstat_job -verbose $job_id3 0] 548if {$matches != 4} { 549 log_error "sstat --verbose failed ($matches != 4)" 550 set exit_code 1 551} 552 553set matches [sstat_vargs o $ac,$ap,$ar,$av,$ji,$mp,$mpn,$mpt,$mr,$mrn,$mrt,$mvs,$mvn,$mvt,$mc,$mn,$mt,$nt $job_id3] 554if {$matches != 6} { 555 log_error "sstat -o failed ($matches != 6)" 556 set exit_code 1 557} 558 559set matches [sstat_vargs -format $ac,$ap,$ar,$av,$ji,$mp,$mpn,$mpt,$mr,$mrn,$mrt,$mvs,$mvn,$mvt,$mc,$mn,$mt,$nt $job_id3] 560if {$matches != 6} { 561 log_error "sstat --format failed ($matches != 6)" 562 set exit_code 1 563} 564 565# 566# Use sacctmgr to delete the test account (shouldn't work, because of running jobs) 567# 568set damatches 0 569set needtodel 0 570set sadel_pid [spawn $sacctmgr -i delete account $test_acct] 571expect { 572 -re "Deleting account" { 573 incr damatches 574 exp_continue 575 } 576 "Job(s) active, cancel job(s) before remove" { 577 log_debug "This error was expected, no worries" 578 set needtodel 1 579 } 580 581 timeout { 582 slow_kill $sadel_pid 583 fail "sacctmgr delete not responding" 584 } 585 eof { 586 wait 587 } 588} 589 590if {$needtodel == 1} { 591 # 592 # Use scancel to quite jobs 593 # 594 spawn $scancel -A $test_acct 595 expect { 596 eof { 597 wait 598 } 599 } 600 601 if {[wait_for_job $job_id3 DONE] != 0} { 602 log_error "Error completing job $job_id3" 603 cancel_job $job_id3 604 set exit_code 1 605 } 606 607 # 608 # Use sacctmgr to delete the test account 609 # 610 set damatches 0 611 set sadel_pid [spawn $sacctmgr -i delete account $test_acct] 612 expect { 613 -re "Deleting account" { 614 incr damatches 615 exp_continue 616 } 617 618 timeout { 619 slow_kill $sadel_pid 620 fail "sacctmgr delete not responding" 621 } 622 eof { 623 wait 624 } 625 } 626} 627 628if {$damatches != 1} { 629 fail "sacctmgr had problems deleting the account" 630} 631 632if {$exit_code != 0} { 633 fail "Test failed due to previous errors (\$exit_code = $exit_code)" 634} 635