1#!/usr/bin/env expect 2############################################################################ 3# Purpose: Test of Slurm functionality 4# Test --mem-per-gpu option 5############################################################################ 6# Copyright (C) 2018 SchedMD LLC 7# Written by Morris Jette 8# 9# This file is part of Slurm, a resource management program. 10# For details, see <https://slurm.schedmd.com/>. 11# Please also read the included file: DISCLAIMER. 12# 13# Slurm is free software; you can redistribute it and/or modify it under 14# the terms of the GNU General Public License as published by the Free 15# Software Foundation; either version 2 of the License, or (at your option) 16# any later version. 17# 18# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY 19# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 20# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 21# details. 22# 23# You should have received a copy of the GNU General Public License along 24# with Slurm; if not, write to the Free Software Foundation, Inc., 25# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. 26############################################################################ 27source ./globals 28 29set exit_code 0 30set file_in "test$test_id.input" 31 32proc cleanup {} { 33 global bin_rm file_in 34 35 exec $bin_rm -f $file_in 36} 37 38proc run_gpu_per_job { mem_per_gpu } { 39 global exit_code file_in number srun test_id 40 41 set mem_size 0 42 set srun_pid [spawn $srun --gpus=1 --mem-per-gpu=$mem_per_gpu -J "test$test_id" -t1 ./$file_in] 43 expect { 44 -re "TRES=cpu=($number),mem=($number)M" { 45 set mem_size $expect_out(2,string) 46 exp_continue 47 } 48 timeout { 49 slow_kill $srun_pid 50 fail "srun not responding" 51 } 52 eof { 53 wait 54 } 55 } 56 if {$mem_size != $mem_per_gpu} { 57 log_error "srun --mem-per-gpu failure ($mem_size != $mem_per_gpu)" 58 set exit_code 1 59 } 60} 61 62proc run_gpu_per_node { mem_per_gpu } { 63 global exit_code file_in number srun test_id 64 65 set mem_size 0 66 set srun_pid [spawn $srun --gpus-per-node=1 -N1 --mem-per-gpu=$mem_per_gpu -J "test$test_id" -t1 ./$file_in] 67 expect { 68 -re "TRES=cpu=($number),mem=($number)M" { 69 set mem_size $expect_out(2,string) 70 exp_continue 71 } 72 timeout { 73 slow_kill $srun_pid 74 fail "srun not responding" 75 } 76 eof { 77 wait 78 } 79 } 80 if {$mem_size != $mem_per_gpu} { 81 log_error "srun --mem-per-gpu failure ($mem_size != $mem_per_gpu)" 82 set exit_code 1 83 } 84} 85 86proc run_gpu_per_task { mem_per_gpu gpu_cnt } { 87 global exit_code file_in number srun test_id 88 89 set mem_size 0 90 set srun_pid [spawn $srun --gpus-per-task=$gpu_cnt -n1 --mem-per-gpu=$mem_per_gpu -J "test$test_id" -t1 ./$file_in] 91 expect { 92 -re "TRES=cpu=($number),mem=($number)M" { 93 set mem_size $expect_out(2,string) 94 exp_continue 95 } 96 timeout { 97 slow_kill $srun_pid 98 fail "srun not responding" 99 } 100 eof { 101 wait 102 } 103 } 104 set mem_target [expr $mem_per_gpu * $gpu_cnt] 105 if {$mem_size != $mem_target} { 106 log_error "srun --mem-per-gpu failure ($mem_size != $mem_target)" 107 set exit_code 1 108 } 109} 110 111proc run_gpu_check_mem { srun_opts mem_target node_target } { 112 global exit_code file_in number srun test_name 113 114 set mem_size 0 115 set node_count 0 116 117 set output [run_command_output -fail "$srun $srun_opts -J $test_name -t1 ./$file_in"] 118 regexp "NumNodes=($number)" $output - node_count 119 regexp "TRES=cpu=($number),mem=($number)M" $output - - mem_size 120 121 if {$node_count < $node_target} { 122 log_error "srun --mem-per-gpu failure, bad node count ($node_count < $node_target)" 123 set exit_code 1 124 } 125 if {$mem_size != $mem_target} { 126 log_error "srun $srun_opts failure ($mem_size != $mem_target)" 127 set exit_code 1 128 } 129} 130 131if {![check_config_select "cons_tres"]} { 132 skip "This test is only compatible with select/cons_tres" 133} 134if {![param_contains [get_config_param "SelectTypeParameters"] "*MEMORY"]} { 135 skip "This test requires memory allocation management" 136} 137 138set nb_nodes 2 139set gpu_cnt [get_highest_gres_count $nb_nodes "gpu"] 140if {$gpu_cnt < 2} { 141 skip "This test requires 2 or more GPUs on $nb_nodes nodes of the default partition" 142} 143 144set nodes [get_nodes_by_request "--gres=gpu:$gpu_cnt -t1 -N $nb_nodes"] 145if { [llength $nodes] != $nb_nodes } { 146 skip "This test need to be able to submit jobs with at least --gres=gpu:$gpu_cnt to $nb_nodes nodes" 147} 148 149# Get the node with the maximum number of GPUs 150dict for {node gpus} [get_gres_count "gpu" [join $nodes ,]] { 151 if {$gpus >= $gpu_cnt} { 152 set node_name $node 153 set gpu_cnt $gpus 154 } 155} 156set node_memory [get_node_param $node_name "RealMemory"] 157 158log_debug "GPU count is $gpu_cnt" 159log_debug "Memory Size is $node_memory" 160log_debug "Node count used $nb_nodes" 161 162# 163# Build input script file 164# 165exec $bin_rm -f $file_in 166make_bash_script $file_in "echo HOST:\$SLURMD_NODENAME CUDA_VISIBLE_DEVICES:\$CUDA_VISIBLE_DEVICES 167$scontrol show job \$SLURM_JOB_ID 168exit 0" 169 170# 171# Run test job with global GPU count 172# Increase mem_per_gpu value 10x on each iteration 173# 174for {set inx 12} {$inx <= $node_memory} {set inx [expr $inx * 10]} { 175 run_gpu_per_job $inx 176 if {$exit_code != 0} { 177 break 178 } 179} 180 181# 182# Run test job with gpus-per-node count 183# Increase mem_per_gpu value 10x on each iteration 184# 185for {set inx 12} {$inx <= $node_memory} {set inx [expr $inx * 10]} { 186 run_gpu_per_node $inx 187 if {$exit_code != 0} { 188 break 189 } 190} 191 192# 193# Run test job with gpus-per-task count and one GPU 194# Increase mem_per_gpu value 10x on each iteration 195# 196for {set inx 12} {$inx <= $node_memory} {set inx [expr $inx * 10]} { 197 run_gpu_per_task $inx 1 198 if {$exit_code != 0} { 199 break 200 } 201} 202 203# 204# Run test job with gpus-per-task count and two GPUs 205# Increase mem_per_gpu value 10x on each iteration 206# 207if {$gpu_cnt > 1} { 208 for {set inx 13} {$inx <= [expr $node_memory / 2]} \ 209 {set inx [expr $inx * 10]} { 210 run_gpu_per_task $inx 2 211 if {$exit_code != 0} { 212 break 213 } 214 } 215} 216 217# 218# Test heterogeneous GPU allocation (gpu_cnt GPUs on one node, 1 GPU on another node) 219# 220if {$gpu_cnt > 1 && $nb_nodes > 1} { 221 set gpu_target [expr $gpu_cnt + 1] 222 set mem_spec 13 223 set node_target 2 224 set mem_target [expr $mem_spec * $gpu_target] 225 run_gpu_check_mem "--gpus=$gpu_target --mem-per-gpu=$mem_spec" $mem_target $node_target 226} 227 228# 229# Run test with --gpus=2 and mem_per_gpu value that pushed job to 2 nodes 230# 231if {$gpu_cnt > 1 && $nb_nodes > 1} { 232 set mem_spec [expr $node_memory / $gpu_cnt + 1] 233 set node_target 2 234 set mem_target [expr $mem_spec * $gpu_cnt] 235 run_gpu_check_mem "--gpus=$gpu_cnt --mem-per-gpu=$mem_spec" $mem_target $node_target 236} 237 238log_info "Testing --mem-per-gpu with --exclusive and --gres=gpu:1" 239for {set inx 12} {$inx <= [expr $node_memory / $gpu_cnt]} {set inx [expr $inx * 10]} { 240 run_gpu_check_mem "--gres=gpu:1 --mem-per-gpu=$inx --exclusive -w $node_name" [expr $gpu_cnt * $inx] 1 241 if {$exit_code != 0} { 242 break 243 } 244} 245 246log_info "Testing --mem-per-gpu with --exclusie and --gpus=1" 247for {set inx 12} {$inx <= [expr $node_memory / $gpu_cnt]} {set inx [expr $inx * 10]} { 248 run_gpu_check_mem "--gpus=1 --mem-per-gpu=$inx --exclusive -w $node_name" [expr $gpu_cnt * $inx] 1 249 if {$exit_code != 0} { 250 break 251 } 252} 253 254log_info "Testing --mem-per-gpu with --exclusie and --gpus-per-task=1" 255for {set inx 12} {$inx <= [expr $node_memory / $gpu_cnt]} {set inx [expr $inx * 10]} { 256 run_gpu_check_mem "--gpus-per-task=1 --ntasks-per-node=1 --mem-per-gpu=$inx --exclusive -w $node_name" [expr $gpu_cnt * $inx] 1 257 if {$exit_code != 0} { 258 break 259 } 260} 261 262log_info "Testing --mem-per-gpu with --exclusie and --gpus-per-socket=1" 263for {set inx 12} {$inx <= [expr $node_memory / $gpu_cnt]} {set inx [expr $inx * 10]} { 264 run_gpu_check_mem "--gpus-per-socket=1 --sockets-per-node=1 --mem-per-gpu=$inx --exclusive -w $node_name" [expr $gpu_cnt * $inx] 1 265 if {$exit_code != 0} { 266 break 267 } 268} 269 270if {$exit_code != 0} { 271 fail "Test failed due to previous errors (\$exit_code = $exit_code)" 272} 273