xref: /dports/sysutils/slurm-wlm/slurm-20.02.7/testsuite/expect/test37.1

#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
#          sacctmgr federation functionality
############################################################################
# Copyright (C) 2016 SchedMD LLC.
# Written by Brian Christiansen <brian@schedmd.com>
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################

source ./globals
source ./globals_accounting

set test_id_2       "37_1"
set max_federations 64

# Test Clusters
set tc1		"tc1_$test_id_2"
set tc2		"tc2_$test_id_2"
set tc3		"tc3_$test_id_2"
set tc4		"tc4_$test_id_2"
set tc5		"tc5_$test_id_2"

set fed1 	"fed1_$test_id_2"
set fed2 	"fed2_$test_id_2"
set fed3 	"fed3_$test_id_2"

set timeout	60

#
# Check accounting config and bail if not found.
#
if {[get_config_param "AccountingStorageType"] ne "accounting_storage/slurmdbd"} {
	skip "This test can't be run without a usable AccountStorageType"
}

if {[string compare [get_admin_level] "Administrator"]} {
	skip "This test can't be run without being an Accounting administrator. Use: sacctmgr mod user \$USER set admin=admin"
}

proc delete_object { object names } {
	global sacctmgr
	set matches 0
	set rc 0
	set my_pid [spawn $sacctmgr -i delete $object $names]
	expect {
		-re "privilege to perform this action" {
			log_error "Don't have privileges"
			incr rc
			exp_continue
		}
		-re "(There was a problem|Unknown condition|Bad format on|Bad MaxWall|Unknown option)" {
			log_error "There was a problem with the sacctmgr command"
			incr rc
			exp_continue
		}
		-re "Problem getting" {
			log_error "There was a problem getting information from the database"
			incr rc
			exp_continue
		}
		-re "Problem adding" {
			log_error "There was an unknown problem"
			incr rc
			exp_continue
		}
		-re "No associations" {
			log_error "Your command didn't return anything"
			incr rc
			exp_continue
		}
		-re "Deleting $object" {
			incr matches
			exp_continue
		}
		-re " Nothing deleted" {
			incr matches
			exp_continue
		}
		timeout {
			log_error "sacctmgr delete not responding"
			slow_kill $my_pid
			incr rc
		}
		eof {
			wait
		}
	}

	if {$matches != 1} {
		log_error "sacctmgr had a problem deleting $object got $matches"
		incr rc
	}

	return $rc
}

proc delete_clusters { names } {
	return [delete_object "cluster" $names]
}

proc delete_federations { names } {
	return [delete_object "federation" $names]
}

proc cleanup { } {
	global tc1 tc2 tc3 tc4 tc5 fed1 fed2 fed3 test_id_2 max_federations

	set tmp_clusters ""
	for {set i 0} {$i <= $max_federations} {incr i} {
		append  tmp_clusters "max${i}_$test_id_2,"
	}

	if [delete_clusters "$tc1,$tc2,$tc3,$tc4,$tc5,${tmp_clusters}"] {
		log_error "Unable to delete clusters ($tc1,$tc2,$tc3,$tc4,$tc5,${tmp_clusters})"
	}
	if [delete_federations "$fed1,$fed2,$fed3"] {
		log_error "Unable to delete federations ($fed1,$fed2,$fed3)"
	}
}

#
# Use sacctmgr to delete the test cluster
#
set nothing 0
set matches 0

######MAIN######
#####################################
# TEST: add federation with non-existant cluster(s)
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add federation $fed1 cluster=$tc1]
expect {
	-re "The cluster $tc1 doesn't exist\\. Please add first\\." {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches == 0} {
	fail "Adding invalid federation failed unexpectedly"
}

set matches 0
set my_pid [spawn $sacctmgr -i add federation $fed1 cluster=$tc1,$tc2]
expect {
	-re "The cluster $tc1 doesn't exist\\. Please add first\\." {
		incr matches
		exp_continue
	}
	-re "The cluster $tc2 doesn't exist\\. Please add first\\." {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Adding invalid federation failed unexpectedly"
}

#####################################
# TEST: add cluster to non-existant federation
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add cluster $tc1 federation=$fed1]
expect {
	-re "The federation $fed1 doesn't exist\\." {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 1} {
	fail "Unexpected error"
}

#####################################
# TEST: add new federation
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add federation $fed1]
expect {
	-re "Adding Federation\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "$fed1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Adding federation failed unexpectedly"
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format=federation%20]
expect {
	-re "Federation $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Adding federation failed unexpectedly. Got $matches"
}

#####################################
# TEST: add new federation - already exists
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add federation $fed1]
expect {
	-re "This federation $fed1 already exists\\.  Not adding\\.$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 1} {
	fail "Adding federation failed unexpectedly"
}

#####################################
# TEST: add second federation and make sure that you can select only one federation
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add federation $fed2]
expect {
	-re "Adding Federation\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Adding federation failed unexpectedly"
}

set matches 0
set my_pid [spawn $sacctmgr show federation format=federation%20]
expect {
	-re "Federation $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 3} {
	fail "Adding federation failed unexpectedly"
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format=federation%20]
expect {
	-re "Federation $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Adding federation failed unexpectedly"
}

#####################################
# TEST: add new cluster to existing federation
# TEST: add new cluster with state
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add cluster $tc1 federation=$fed1 fedstate=drain]
expect {
	-re "Adding Cluster\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Setting$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Federation\\s+=\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+FedState\\s+=\\s+DRAIN$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error"
}

#####################################
# TEST: add multiple new clusters to single federation
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add cluster $tc2 $tc3 federation=$fed2]
expect {
	-re "Adding Cluster\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Setting$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Federation\\s+=\\s+$fed2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc1 $tc2 $tc3 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1\\s+$fed1\\s+DRAIN\\s+1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2\\s+$fed2\\s+ACTIVE\\s+1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3\\s+$fed2\\s+ACTIVE\\s+2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error"
}


#Test show cluster WithFed
set matches 0
set my_pid [spawn $sacctmgr show cluster $tc1 $tc2 $tc3 WithFed]
expect {
	-re "Cluster\\s+.*?Federation\\s+ID\\s+Features\\s+FedState $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1\\s+.*?$fed1\\s+1\\s+DRAIN $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2\\s+.*?$fed2\\s+1\\s+ACTIVE $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3\\s+.*?$fed2\\s+2\\s+ACTIVE $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error"
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 $fed2 format="federation%20,cluster%20"]
expect {
	-re "Federation\\s+Cluster $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+$tc1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2\\s+$tc2 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2\\s+$tc3 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error"
}

# TEST tree option - shows one federation line per federation.
set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 $fed2 format="federation%20,cluster%20" tree]
expect {
	-re "Federation\\s+Cluster $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2\\s+$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc2 $eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc3 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 6} {
	fail "Unexpected error"
}


#####################################
# TEST: attempt to set clusters to multiple federations
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 $fed2 set clusters=$tc1]
expect {
	-re "Can't assign clusters to multiple federations\\." {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 1} {
	fail "Unexpected error"
}

#####################################
# TEST: attempt to set clusters with no where clause
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i modify federation set clusters=$tc1]
expect {
	-re "Can't assign clusters to multiple federations\\." {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 1} {
	fail "Unexpected error"
}

set matches 0
set my_pid [spawn $sacctmgr -i modify federation set clusters=$tc1 where cluster=$tc1]
expect {
	-re "Can't assign clusters to multiple federations\\." {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 1} {
	fail "Unexpected error"
}

#####################################
# TEST: modify clusters with fed options
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc1 $tc3 set fedstate=DRAIN]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+FedState\\s+=\\s+DRAIN$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc1$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc3$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error"
}

set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc2 set fedstate=DRAIN+REMOVE]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+FedState\\s+=\\s+DRAIN\\+REMOVE$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error"
}

set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc1 set fedstate=ACTIVE]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+FedState\\s+=\\s+ACTIVE$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc1 $tc2 $tc3 format="cluster%20,federation%20,fedstate%20"]
expect {
	-re "Cluster\\s+Federation\\s+FedState $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1\\s+$fed1\\s+ACTIVE $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2\\s+$fed2\\s+DRAIN\\+REMOVE $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3\\s+$fed2\\s+DRAIN $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error"
}

######################################
## TEST: modify federation with flags
######################################
#set matches 0
#set my_pid [spawn $sacctmgr -i modify federation $fed1 $fed2 set flags+=LLC]
#expect {
#	-re "Setting$eol" {
#		incr matches
#		exp_continue
#	}
#	-re "^\\s+Flags\\s+\\+=\\s+LLC$eol" {
#		incr matches
#		exp_continue
#	}
#	-re "^\\s+Modified federation...$eol" {
#		incr matches
#		exp_continue
#	}
#	-re "^\\s+$fed1$eol" {
#		incr matches
#		exp_continue
#	}
#	-re "^\\s+$fed2$eol" {
#		incr matches
#		exp_continue
#	}
#	timeout {
#		slow_kill $my_pid
#		fail "sacctmgr add not responding"
#	}
#	eof {
#		wait
#	}
#}
#if {$matches != 5} {
#	fail "Unexpected error"
#}
#
#set matches 0
#set my_pid [spawn $sacctmgr show federation $fed1 $fed2 format="federation%20,flags,cluster%20,fedstate%20"]
#expect {
#	-re "Federation\\s+Flags\\s+Cluster\\s+FedState $eol" {
#		incr matches
#		exp_continue
#	}
#	-re "\\s+$fed1\\s+LLC\\s+$tc1\\s+ACTIVE $eol" {
#		incr matches
#		exp_continue
#	}
#	-re "\\s+$fed2\\s+LLC\\s+$tc2\\s+DRAIN\\+REMOVE $eol" {
#		incr matches
#		exp_continue
#	}
#	-re "\\s+$fed2\\s+LLC\\s+$tc3\\s+DRAIN $eol" {
#		incr matches
#		exp_continue
#	}
#	timeout {
#		slow_kill $my_pid
#		fail "sacctmgr add not responding"
#	}
#	eof {
#		wait
#	}
#}
#if {$matches != 4} {
#	fail "Unexpected error"
#}
#
#set matches 0
#set my_pid [spawn $sacctmgr show federation $fed1 $fed2 format="federation%20,flags,cluster%20,fedstate%20" tree]
#expect {
#	-re "Federation\\s+Flags\\s+Cluster\\s+FedState $eol" {
#		incr matches
#		exp_continue
#	}
#	-re "\\s+$fed1\\s+LLC\\s+ $eol" {
#		incr matches
#		exp_continue
#	}
#	-re "\\s+$tc1\\s+ACTIVE $eol" {
#		incr matches
#		exp_continue
#	}
#	-re "\\s+$fed2\\s+LLC\\s+ $eol" {
#		incr matches
#		exp_continue
#	}
#	-re "\\s+$tc2\\s+DRAIN\\+REMOVE $eol" {
#		incr matches
#		exp_continue
#	}
#	-re "\\s+$tc3\\s+DRAIN $eol" {
#		incr matches
#		exp_continue
#	}
#	timeout {
#		slow_kill $my_pid
#		fail "sacctmgr add not responding"
#	}
#	eof {
#		wait
#	}
#}
#if {$matches != 6} {
#	fail "Unexpected error"
#}
#
######################################
## TEST: modify federation clear flags
######################################
#set matches 0
#set my_pid [spawn $sacctmgr -i modify federation $fed1 $fed2 set flags-=LLC]
#expect {
#	-re "Setting$eol" {
#		incr matches
#		exp_continue
#	}
#	-re "^\\s+Flags\\s+-=\\s+LLC$eol" {
#		incr matches
#		exp_continue
#	}
#	-re "^\\s+Modified federation...$eol" {
#		incr matches
#		exp_continue
#	}
#	-re "^\\s+$fed1$eol" {
#		incr matches
#		exp_continue
#	}
#	-re "^\\s+$fed2$eol" {
#		incr matches
#		exp_continue
#	}
#	timeout {
#		slow_kill $my_pid
#		fail "sacctmgr add not responding"
#	}
#	eof {
#		wait
#	}
#}
#if {$matches != 5} {
#	fail "Unexpected error"
#}
#
## Verify that LLC is removed and that only the federations are listed since no
## cluster print fields are requested.
#set matches 0
#set my_pid [spawn $sacctmgr show federation $fed1 $fed2 format="federation%20,flags"]
#expect {
#	-re "Federation\\s+Flags $eol" {
#		incr matches
#		exp_continue
#	}
#	-re "\\s+$fed1\\s+ $eol" {
#		incr matches
#		exp_continue
#	}
#	-re "\\s+$fed2\\s+ $eol" {
#		incr matches
#		exp_continue
#	}
#	timeout {
#		slow_kill $my_pid
#		fail "sacctmgr add not responding"
#	}
#	eof {
#		wait
#	}
#}
#if {$matches != 3} {
#	fail "Unexpected error"
#}

#####################################
# TEST: mod cluster to federation -- check ids
# create hole in fed2 ids
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc2 set federation=$fed1]
expect {
	-re "The cluster $tc2 is assigned to federation $fed2$eol" {
		incr matches
		exp_continue
	}
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Federation\\s+=\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc1 $tc2 $tc3 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1\\s+$fed1\\s+ACTIVE\\s+1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2\\s+$fed1\\s+ACTIVE\\s+2 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3\\s+$fed2\\s+DRAIN\\s+2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error"
}

# move tc1 into whole.
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc1 set federation=$fed2]
expect {
	-re "The cluster $tc1 is assigned to federation $fed1$eol" {
		incr matches
		exp_continue
	}
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Federation\\s+=\\s+$fed2$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc1 $tc2 $tc3 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1\\s+$fed2\\s+ACTIVE\\s+1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2\\s+$fed1\\s+ACTIVE\\s+2 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3\\s+$fed2\\s+DRAIN\\s+2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error"
}

# move tc2 back to fed2 and get new id -- 3.
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc2 set federation=$fed2]
expect {
	-re "The cluster $tc2 is assigned to federation $fed1$eol" {
		incr matches
		exp_continue
	}
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Federation\\s+=\\s+$fed2$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc1 $tc2 $tc3 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1\\s+$fed2\\s+ACTIVE\\s+1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2\\s+$fed2\\s+ACTIVE\\s+3 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3\\s+$fed2\\s+DRAIN\\s+2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error"
}


#####################################
# TEST: add federation with existing clusters
#####################################
set matches 0
set my_pid [spawn $sacctmgr add federation $fed3 clusters=$tc1,$tc2]
expect {
	-re "The cluster $tc1 is assigned to federation $fed2$eol" {
		incr matches
		exp_continue
	}
	-re "The cluster $tc2 is assigned to federation $fed2$eol" {
		incr matches
		exp_continue
	}
	"Are you sure" {
		incr matches
		exp_continue
	}
	"(N/y):" {
		incr matches
		send "y"
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Adding invalid federation failed unexpectedly"
}

set matches 0
expect {
	-re "Adding Federation\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "$fed3$eol" {
		incr matches
		exp_continue
	}
	-re "Settings$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Cluster\\s+=\\s+$tc1$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Cluster\\s+=\\s+$tc2$eol" {
		incr matches
		exp_continue
	}
	"Would you like" {
		incr matches
		exp_continue
	}
	"(N/y):" {
		incr matches
		send "y"
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
expect {
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 7} {
	fail "Adding invalid federation failed unexpectedly"
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 $fed2 $fed3 format="federation%20,cluster%20,fedstate%20,id"]
expect {
	-re "Federation\\s+Cluster\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed2\\s+$tc3\\s+DRAIN\\s+2 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed3\\s+$tc1\\s+ACTIVE\\s+1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed3\\s+$tc2\\s+ACTIVE\\s+2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error"
}

#####################################
# modify cluster, clear federation
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc3 set federation=]
expect {
	-re "Setting" {
		incr matches
		exp_continue
	}
	-re "^\\s+Federation\\s+=\\s+$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc3$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc3 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc3\\s+NA\\s+0 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

#####################################
# TEST: test selecting clusters by federations
#####################################
set matches 0
set my_pid [spawn $sacctmgr show cluster where fed=$fed3 format="cluster%20,federation%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1\\s+$fed3\\s+1 $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2\\s+$fed3\\s+2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 3} {
	fail "Unexpected error. Got $matches"
}



#####################################
# TEST: test adding new cluster and verify state and change states
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add cluster $tc4]
expect {
	-re "Adding Cluster\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc4$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

# New clusters should have fed_id=0, federation="", fed_state=NA
set matches 0
set my_pid [spawn $sacctmgr show cluster $tc4 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc4\\s+NA\\s+0 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

# Test adding cluster to federation sets state to ACTIVE
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc4 set federation=$fed3]
expect {
	-re "Setting" {
		incr matches
		exp_continue
	}
	-re "^\\s+Federation\\s+=\\s+$fed3$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc4$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc4 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc4\\s+$fed3\\s+ACTIVE\\s+3 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}


# Modifying cluster to same federation shouldln't change fed_id or fed_state
set matches 0
# set state to something other than ACTIVE, it should stay the same.
set my_pid [spawn $sacctmgr -i modify cluster $tc4 set fedstate=DRAIN]
expect {
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}

set my_pid [spawn $sacctmgr -i modify cluster $tc4 set federation=$fed3]
expect {
	-re "The cluster $tc4 is already assigned to federation $fed3" {
		incr matches
		exp_continue
	}
	"Nothing to change" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc4 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc4\\s+$fed3\\s+DRAIN\\s+3 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

#same thing for modifying the federaiton.
set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed3 set clusters+=$tc4]
expect {
	-re "The cluster $tc4 is already assigned to federation $fed3" {
		incr matches
		exp_continue
	}
	"Nothing to change" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc4 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc4\\s+$fed3\\s+DRAIN\\s+3 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

# changing from one federation to another should set the state to active
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc4 set federation=$fed2]
expect {
	-re "The cluster $tc4 is assigned to federation $fed3$eol" {
		incr matches
		exp_continue
	}
	-re "Setting" {
		incr matches
		exp_continue
	}
	-re "^\\s+Federation\\s+=\\s+$fed2$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc4$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc4 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc4\\s+$fed2\\s+ACTIVE\\s+1 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}


# same thing for modifying federation - state should go to active
set my_pid [spawn $sacctmgr -i modify cluster $tc4 set fedstate=DRAIN]
expect {
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}

set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed3 set clusters+=$tc4]
expect {
	-re "The cluster $tc4 is assigned to federation $fed2$eol" {
		incr matches
		exp_continue
	}
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ \\+= $tc4$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Modified federation...$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed3$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc4 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc4\\s+$fed3\\s+ACTIVE\\s+3 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

# Test setting state to INACTIVE
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc4 set fedstate=INACTIVE]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+FedState\\s+=\\s+INACTIVE$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc4$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc4 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc4\\s+$fed3\\s+INACTIVE\\s+3 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

# Test invalid state
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc4 set fedstate=abcdefg]
expect {
	-re "Invalid FedState abcdefg" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 1} {
	fail "Unexpected error. Got $matches"
}

#####################################
# TEST: modify federation change clusters - must remove others
#####################################
#sacctmgr modify federation fed set clusters=test3,test4
#sacctmgr modify federation fed set clusters=test1,test2
#sacctmgr modify federation fed set clusters+=test1,test2
#sacctmgr modify federation fed set clusters-=test1,test2
# unset clusters
set my_pid [spawn $sacctmgr -i modify cluster $tc1 $tc2 $tc3 set federation=]
expect {
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}

set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 set clusters=$tc1,$tc2]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ \\+= $tc1$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ \\+= $tc2$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Modified federation...$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20,fedstate%20"]
expect {
	-re "Federation\\s+Cluster\\s+FedState $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+(\\S+)\\s+ACTIVE $eol" {
		set match $expect_out(1,string)
		if {[string equal $match $tc1] ||
		    [string equal $match $tc2]} {
			incr matches
		}
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 3} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 set clusters=$tc1,$tc3]
expect {
	-re "The cluster $tc1 is already assigned to federation $fed1" {
		incr matches
		exp_continue
	}
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ \\+= $tc3$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ -= $tc2$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Modified federation...$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 6} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20,fedstate%20"]
expect {
	-re "Federation\\s+Cluster\\s+FedState $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+(\\S+)\\s+ACTIVE $eol" {
		set match $expect_out(1,string)
		if {[string equal $match $tc1] ||
		    [string equal $match $tc3]} {
			incr matches
		}
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 3} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 set clusters+=$tc2]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ \\+= $tc2$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Modified federation...$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20,fedstate%20"]
expect {
	-re "Federation\\s+Cluster\\s+FedState $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+(\\S+)\\s+ACTIVE $eol" {
		set match $expect_out(1,string)
		if {[string equal $match $tc1] ||
		    [string equal $match $tc2] ||
		    [string equal $match $tc3]} {
			incr matches
		}
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 set clusters-=$tc1,$tc2]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ -= $tc1$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ -= $tc2$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Modified federation...$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20,fedstate%20"]
expect {
	-re "Federation\\s+Cluster\\s+FedState $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+(\\S+)\\s+ACTIVE $eol" {
		set match $expect_out(1,string)
		if {[string equal $match $tc3]} {
			incr matches
		}
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 set clusters-=$tc1,$tc3]
expect {
	-re "The cluster $tc1 isn't assigned to federation $fed1" {
		incr matches
		exp_continue
	}
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ -= $tc3$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Modified federation...$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20,fedstate%20"]
expect {
	-re "Federation\\s+Cluster\\s+FedState $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+ $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

#####################################
# TEST: error checking on using +, - and =
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 set cluster=$tc1,+$tc2]
expect {
	"sacctmgr: error: You can't use '=' and '+' or '-' in the same line" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 1} {
	fail "Failed testing mod with + and ="
}

set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 set cluster=$tc1,-$tc2]
expect {
	"sacctmgr: error: You can't use '=' and '+' or '-' in the same line" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 1} {
	fail "Failed testing mod with - and ="
}


#####################################
# TEST: modify federation, clear clusters
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc1 $tc2 $tc3 set federation=$fed1]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Federation\\s+=\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc1$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc2$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc3$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr mod not responding"
	}
	eof {
		wait
	}
}
if {$matches != 6} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr -i modify federation $fed1 set clusters=]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ -= $tc1$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ -= $tc2$eol" {
		incr matches
		exp_continue
	}
	-re "Cluster\\s+ -= $tc3$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Modified federation...$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 6} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20,fedstate%20"]
expect {
	-re "Federation\\s+Cluster\\s+FedState $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+ $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

# Verify clusters fed_id=0, federation="", fed_state=NA after being removed from
# federation
set matches 0
set my_pid [spawn $sacctmgr show cluster $tc1 $tc2 $tc3 format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+($tc1|$tc2|$tc3)\\s+NA\\s+0 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error. Got $matches"
}


#####################################
# TEST: test deleting cluster with where federation= clause
#####################################
set my_pid [spawn $sacctmgr -i delete federation $fed2 $fed3]
expect {
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}

set matches 0
set my_pid [spawn $sacctmgr -i add federation $fed2 clusters=$tc1,$tc2]
expect {
	-re "Adding Federation\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "$fed2$eol" {
		incr matches
		exp_continue
	}
	-re "Settings$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Cluster\\s+=\\s+$tc1$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Cluster\\s+=\\s+$tc2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Adding federation with clusters failed. Got $matches"
}

# add second cluster to make sure selectin only on federation
set matches 0
set my_pid [spawn $sacctmgr -i add federation $fed3 clusters=$tc3,$tc4]
expect {
	-re "Adding Federation\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "$fed3$eol" {
		incr matches
		exp_continue
	}
	-re "Settings$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Cluster\\s+=\\s+$tc3$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Cluster\\s+=\\s+$tc4$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Adding federation with clusters failed. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr -i delete cluster where federation=$fed2]
expect {
	-re "Deleting clusters...$eol" {
		incr matches
		exp_continue
	}
	-re "$tc1$eol" {
		incr matches
		exp_continue
	}
	-re "$tc2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 3} {
	fail "Deleting clusters with federation=. Got $matches"
}

# add back clusters back to federation to verify both federation clusters are
# selected.
set matches 0
set my_pid [spawn $sacctmgr -i add cluster $tc1,$tc2 federation=$fed2]
expect {
	-re "Adding Cluster\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc1$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc2$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Setting$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Federation\\s+=\\s+$fed2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr -i delete cluster where federation=$fed2,$fed3]
expect {
	-re "Deleting clusters...$eol" {
		incr matches
		exp_continue
	}
	-re "$tc1$eol" {
		incr matches
		exp_continue
	}
	-re "$tc2$eol" {
		incr matches
		exp_continue
	}
	-re "$tc3$eol" {
		incr matches
		exp_continue
	}
	-re "$tc4$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr delete cluster not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Deleting clusters with federation=. Got $matches"
}


#####################################
# TEST: test adding more than 63 clusters to a federation
#####################################
# unset federations
set my_pid [spawn $sacctmgr -i modify federation $fed1 set clusters=]
expect {
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}

set matches 0
set tmp_clusters ""
for {set i 1} {$i < $max_federations} {incr i} {
	if {$i > 1} {
		append tmp_clusters ","
	}
	append tmp_clusters "max${i}_$test_id_2"
}
set timeout 300
set my_pid [spawn $sacctmgr -i add cluster $tmp_clusters federation=$fed1]
expect {
	-re "Adding Cluster\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+max\[1-6\]{0,1}\\d{1}_$test_id_2$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Setting$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Federation\\s+=\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 66} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set tmpc "max${i}_$test_id_2"
set my_pid [spawn $sacctmgr -i add cluster $tmpc federation=$fed1]
expect {
	-re "Adding Cluster\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tmpc$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Setting$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Federation\\s+=\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Problem adding clusters: Too many clusters in federation?" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error. Got $matches"
}

set matches 0
#####################################
# TEST: modify cluster to exceed max clusters in federation
#####################################
#add last cluster without federation
set my_pid [spawn $sacctmgr -i add cluster $tmpc]
expect {
	-re "Adding Cluster\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tmpc$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tmpc set federation=$fed1]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Federation\\s+=\\s+$fed1$eol" {
		incr matches
		exp_continue
	}
	-re "sacctmgr: error: slurmdbd: Too many clusters in federation$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 3} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20"]
expect {
	-re "Federation\\s+Cluster $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$fed1\\s+max\[1-6\]{0,1}\\d{1}_$test_id_2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != $max_federations} {
	fail "Unexpected error. Got $matches"
}

#####################################
# TEST: delete cluster - should delete it from federation.
#####################################
delete_clusters "max1_$test_id_2"

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20"]
expect {
	-re "\\s+$fed1\\s+max1_$test_id_2 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches } {
	fail "Found cluster still on federation when it should have been deleted. Got $matches"
}


#####################################
# TEST: delete federation - should clean clusters from federation.
#####################################
delete_federations $fed1

set matches 0
set my_pid [spawn $sacctmgr show federation $fed1 format="federation%20,cluster%20"]
expect {
	-re "\\s+$fed1.*?$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches } {
	fail "Found federation when it should have been deleted. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster format="cluster%20,federation%20"]
expect {
	-re "$fed1" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches } {
	fail "Found federation on cluster when it was deleted. Got $mattches"
}


# Verify clusters fed_id=0, federation="", fed_state=NA after federation is deleted.

set tmp_clusters ""
for {set i 0} {$i <= $max_federations} {incr i} {
	append  tmp_clusters "max${i}_$test_id_2,"
}
set matches 0
set my_pid [spawn $sacctmgr show cluster $tmp_clusters format="cluster%20,federation%20,fedstate%20,id"]
expect {
	-re "Cluster\\s+Federation\\s+FedState\\s+ID $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+max\[1-6\]{0,1}\\d{1}_$test_id_2\\s+NA\\s+0 $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != $max_federations} {
	fail "Unexpected error. Got $matches"
}


#####################################
# TEST: add/modify cluster features
#####################################
set matches 0
set my_pid [spawn $sacctmgr -i add cluster $tc5 features=a,b]
expect {
	-re "Adding Cluster\\(s\\)$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+$tc5$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Setting$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Feature\\s+=\\s+a$eol" {
		incr matches
		exp_continue
	}
	-re "\\s+Feature\\s+=\\s+b$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc5 format="cluster%20,features%20"]
expect {
	-re "Cluster\\s+Features $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+a,b $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc5 set features=aa,ab]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Feature\\s+=\\s+aa" {
		incr matches
		exp_continue
	}
	-re "^\\s+Feature\\s+=\\s+ab" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc5$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr mod not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc5 format="cluster%20,features%20"]
expect {
	-re "Cluster\\s+Features $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+aa,ab $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc5 set features+=fc]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Feature\\s+\\+=\\s+fc" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc5$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr mod not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc5 format="cluster%20,features%20"]
expect {
	-re "Cluster\\s+Features $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+aa,ab,fc $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc5 set features-=ab]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Feature\\s+-=\\s+ab" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc5$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr mod not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc5 format="cluster%20,features%20"]
expect {
	-re "Cluster\\s+Features $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+aa,fc $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr -i modify cluster $tc5 set features-=aa,fc]
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Feature\\s+-=\\s+aa" {
		incr matches
		exp_continue
	}
	-re "^\\s+Feature\\s+-=\\s+fc" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$tc5$eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr mod not responding"
	}
	eof {
		wait
	}
}
if {$matches != 5} {
	fail "Unexpected error. Got $matches"
}

set matches 0
set my_pid [spawn $sacctmgr show cluster $tc5 format="cluster%20,features%20"]
expect {
	-re "Cluster\\s+Features $eol" {
		incr matches
		exp_continue
	}
	-re "\\s+ $eol" {
		incr matches
		exp_continue
	}
	timeout {
		slow_kill $my_pid
		fail "sacctmgr add not responding"
	}
	eof {
		wait
	}
}
if {$matches != 2} {
	fail "Unexpected error. Got $matches"
}