#!/usr/bin/env expect
############################################################################
# Purpose: Test federated job updates
#
# Reqs:    1. Using slurmdbd accounting storage type and is up
#          2. fed_slurm_base is defined in globals.local - set to directory that
#          has access to each federation configure (fedc1, fedc2, fedc3).
#          Eg.
#          fedr/slurm/ (src)
#          fedr/fed1/bin
#          fedr/fed1/sbin
#          fedr/fed1/etc
#          fedr/fed1/...
#          fedr/fed2/...
#          fedr/fed3/...
#          3. controllers are up and running.
############################################################################
# Copyright (C) 2017 SchedMD LLC.
# Written by Isaac Hartung <ihartung@schedmd.com>
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################

source ./globals
source ./globals_accounting
source ./globals_federation

set fed_name    "feda"
set user_name   ""
set srun_job_cnt 0

#
# Check accounting config and bail if not found.
#
if {[get_config_param "AccountingStorageType"] ne "accounting_storage/slurmdbd"} {
	skip "This test can't be run without a usable AccountStorageType"
}

if {[get_admin_level] ne "Administrator"} {
	skip "This test can't be run without being an Accounting administrator. Use: sacctmgr mod user \$USER set admin=admin"
}

proc modify {job_id constraint reg_ex error_message} {
	global fedc1 fed_slurm_base bin_sleep

	set my_scontrol "${fed_slurm_base}/$fedc1/bin/scontrol"
	set matches 0

	spawn $my_scontrol update jobid=$job_id clusterfeatures=$constraint
	expect {
		-re "$reg_ex" {
			incr matches
		}
		timeout {
			fail "scontrol not responding"
		}
		eof {
			wait
		}
	}
	if {![string match "" $reg_ex] && $matches != 1} {
		fail "Didn't match regex ($reg_ex: $matches)"
	}

	sleep 3
}

proc modify_with_clusters {job_id constraint reg_ex error_message} {
	global fedc1 fed_slurm_base bin_sleep

	set my_scontrol "${fed_slurm_base}/$fedc1/bin/scontrol"
	set matches 0

	spawn $my_scontrol update jobid=$job_id clusters=$constraint
	expect {
		-re "$reg_ex" {
			incr matches
		}
		timeout {
			fail "scontrol not responding"
		}
		eof {
			wait
		}
	}
	if {![string match "" $reg_ex] && $matches != 1} {
		fail "Didn't match regex ($reg_ex: $matches)"
	}

	sleep 3
}

proc sbatch {my_sbatch script constraint args} {

	global number bin_sleep node_count

	set job_id 0
	set command "$my_sbatch -N$node_count --exclusive --output=/dev/null \
		--error=/dev/null -t5 --wrap $script --cluster-constraint=$constraint"

	if {$args ne ""} {
		append command " $args"
	}
	spawn {*}$command
	expect {
		-re "Submitted batch job ($number)" {
			set job_id $expect_out(1,string)
		}
		timeout {
			fail "sbatch not responding"
		}
		eof {
			wait
		}
	}
	if {$job_id == 0} {
		fail "Batch submit failure"
	}

	sleep 3
	return $job_id
}

proc verify {fed job_id pattern error_message} {
	global fed_slurm_base

	set matches 0
	set my_squeue "${fed_slurm_base}/$fed/bin/squeue"
	spawn $my_squeue -a --local -j $job_id --noheader \
		-Ostatecompact:.5,siblingsviable:.20,siblingsactive:.20
	expect {
		-re "\\s+$pattern" {
			incr matches
		}
		eof {
			wait
		}
	}
	if {$matches != 1} {
		fail "$error_message"
	}
}

proc cancel_federation_jobs { } {
	global scancel user_name fedc1 fedc2 fedc3

	spawn $scancel -M$fedc1,$fedc2,$fedc3 --user $user_name
	expect {
		eof {
			wait
		}
	}
	sleep 5
}

proc cleanup { } {
	global fed_name bin_bash bin_rm test_name

	cancel_federation_jobs
	delete_federations $fed_name
	exec $bin_bash -c "$bin_rm -f $test_name*.out"
}

# Start test

if {![check_federation_setup]} {
	skip "This test can't be run without fed_slurm_base, fedc1, fedc2, fedc3 setup in globals.local"
}

if {![check_federation_up]} {
	skip "This test can't be run without all clusters up"
}

set user_name [get_my_user_name]

# Remove existing setup
cleanup

# Add clusters to federation
if [setup_federation $fed_name] {
	fail "Failed to setup federation"
}

# Get number of nodes per cluster
set node_count [llength [get_nodes_by_state]]

log_info "################################################################"
log_info "Setup cluster features"
log_info "################################################################"

set matches 0
spawn $sacctmgr -i modify cluster $fedc1 set features=fa
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Feature\\s+=\\s+fa" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$fedc1$eol" {
		incr matches
		exp_continue
	}
	timeout {
		fail "sacctmgr mod not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error (got $matches)"
}

set matches 0
spawn $sacctmgr -i modify cluster $fedc2 set features=fb
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Feature\\s+=\\s+fb" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$fedc2$eol" {
		incr matches
		exp_continue
	}
	timeout {
		fail "sacctmgr mod not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error (got $matches)"
}

set matches 0
spawn $sacctmgr -i modify cluster $fedc3 set features=fc
expect {
	-re "Setting$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+Feature\\s+=\\s+fc" {
		incr matches
		exp_continue
	}
	-re "Modified cluster...$eol" {
		incr matches
		exp_continue
	}
	-re "^\\s+$fedc3$eol" {
		incr matches
		exp_continue
	}
	timeout {
		fail "sacctmgr mod not responding"
	}
	eof {
		wait
	}
}
if {$matches != 4} {
	fail "Unexpected error (got $matches)"
}

set my_scontrol "${fed_slurm_base}/$fedc1/bin/scontrol"

set job_id 0
set script "\"sleep 300\""
set my_sbatch "${fed_slurm_base}/$fedc1/bin/sbatch"

log_info "################################################################"
log_info "Modify Feature of Running Job"
log_info "################################################################"
cancel_federation_jobs

# First job

set job_id [sbatch $my_sbatch $script fa]

verify $fedc1 $job_id "R\\s+$fedc1\\s+$fedc1" "Job not running on designated cluster"

modify $job_id fb "Job is no longer pending execution for job $job_id" "allowed user to modify features of running job"

log_info "################################################################"
log_info "Test invalid feature"
log_info "################################################################"
# Second job

set job_id [sbatch $my_sbatch $script fa]

verify $fedc1 $job_id "PD\\s+$fedc1\\s+$fedc1" "Job not running on designated cluster"

# Invalid feature

modify $job_id invalid "Invalid cluster feature specification for job" "allowed invalid feature"

log_info "################################################################"
log_info "Move pending job to $fedc2 with feature fb"
log_info "################################################################"
modify $job_id fb "" "did not allow user to modify features of pending job"

set run_cluster [wait_for_fed_job $job_id RUNNING ""]
if {$run_cluster ne $fedc2} {
	fail "Job not running on cluster 2 ($fedc2)"
}
verify $fedc1 $job_id "RV\\s+$fedc2\\s+$fedc2" "Job not revoked"
verify $fedc2 $job_id "R\\s+$fedc2\\s+$fedc2" "Job not running on designated cluster"

log_info "################################################################"
log_info "New job to fa, then move job to $fedc2,$fedc3. Runs on $fedc3"
log_info "################################################################"
# Third job

set job_id [sbatch $my_sbatch $script fa]

modify $job_id "fb,fc" "" "did not allow user to modify features of pending job"

set run_cluster [wait_for_fed_job $job_id RUNNING ""]
if {$run_cluster ne $fedc3} {
	fail "Job not running on cluster 3 ($fedc3)"
}
verify $fedc1 $job_id "RV\\s+$fedc2,$fedc3\\s+$fedc3" "Job not revoked"
verify $fedc3 $job_id "R\\s+$fedc2,$fedc3\\s+$fedc3" "Job not running on designated cluster"

log_info "################################################################"
log_info "New job to fa, then move job to $fedc3"
log_info "################################################################"
# Fourth job

set job_id [sbatch $my_sbatch $script fa]

modify $job_id "fc" "" "did not allow user to modify features of pending job"

verify $fedc1 $job_id "RV\\s+$fedc3\\s+$fedc3" "Job not revoked"
verify $fedc3 $job_id "PD\\s+$fedc3\\s+NA" "Job not running on designated cluster"

log_info "################################################################"
log_info "Move job to $fedc1,$fedc3"
log_info "################################################################"
modify $job_id "fa,fc" "" "did not allow user to modify features of pending job"

verify $fedc1 $job_id "PD\\s+$fedc1,$fedc3\\s+$fedc1,$fedc3" "Job not running on designated cluster"
verify $fedc2 $job_id ".+Invalid.+" "Job found on wrong cluster"
verify $fedc3 $job_id "PD\\s+$fedc1,$fedc3\\s+NA" "Job found on wrong cluster"

log_info "################################################################"
log_info "Move job to $fedc1"
log_info "################################################################"
modify $job_id "fa" "" "did not allow user to modify features of pending job"

verify $fedc1 $job_id "PD\\s+$fedc1\\s+$fedc1" "Job not running on designated cluster"
verify $fedc2 $job_id ".+Invalid.+" "Job found on wrong cluster"
verify $fedc3 $job_id ".+Invalid.+" "Job found on wrong cluster"

log_info "################################################################"
log_info "Clear features. Should go to pending on $fedc1,$fedc2,$fedc3"
log_info "################################################################"
# Clear Features

modify $job_id "" "" "did not allow user to modify features of pending job"

verify $fedc1 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+$fedc1,$fedc2,$fedc3" "Job not running on designated cluster"
verify $fedc2 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"
verify $fedc3 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"

log_info "################################################################"
log_info "Modify Feature of Held Job"
log_info "################################################################"
cancel_federation_jobs

set job_id [sbatch $my_sbatch $script fa -H]

verify $fedc1 $job_id "PD\\s+$fedc1\\s+$fedc1" "Job not running on designated cluster"
verify $fedc2 $job_id ".+Invalid.+" "Job found on wrong cluster"
verify $fedc3 $job_id ".+Invalid.+" "Job found on wrong cluster"

modify $job_id "fb,fc" "" "did not allow user to modify features of pending job"

verify $fedc1 $job_id "RV\\s+$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"
verify $fedc2 $job_id ".+Invalid.+" "Job found on wrong cluster"
verify $fedc3 $job_id ".+Invalid.+" "Job found on wrong cluster"

modify $job_id "" "" "did not allow user to modify features of pending job"

verify $fedc1 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"
verify $fedc2 $job_id ".+Invalid.+" "Job found on wrong cluster"
verify $fedc3 $job_id ".+Invalid.+" "Job found on wrong cluster"

modify $job_id "fc" "" "did not allow user to modify features of pending job"

verify $fedc1 $job_id "PD\\s+$fedc3\\s+NA" "Job not running on designated cluster"
verify $fedc2 $job_id ".+Invalid.+" "Job found on wrong cluster"
verify $fedc3 $job_id ".+Invalid.+" "Job found on wrong cluster"

set my_scontrol "${fed_slurm_base}/$fedc1/bin/scontrol"

log_debug "Releasing job"

exec $my_scontrol release $job_id

sleep 3

set run_cluster [wait_for_fed_job $job_id RUNNING ""]
if {$run_cluster ne $fedc3} {
	fail "Job not running on cluster 3 ($fedc3)"
}
verify $fedc1 $job_id "RV\\s+$fedc3\\s+$fedc3" "Job not running on designated cluster"
verify $fedc2 $job_id ".+Invalid.+" "Job found on wrong cluster"
verify $fedc3 $job_id "R\\s+$fedc3\\s+$fedc3" "Job found on wrong cluster"

log_info "################################################################"
log_info "Test Clearing Features"
log_info "################################################################"

cancel_federation_jobs

# Fill up clusters with jobs

sbatch $my_sbatch $script fa
sbatch $my_sbatch $script fb
sbatch $my_sbatch $script fc

# Add pending job

set job_id [sbatch $my_sbatch $script "" --requeue]

verify $fedc1 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+$fedc1,$fedc2,$fedc3" "Job not running on designated cluster"
verify $fedc2 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"
verify $fedc3 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"

modify $job_id "fa" "" "did not allow user to modify features of pending job"

verify $fedc1 $job_id "PD\\s+$fedc1\\s+$fedc1" "Job not running on designated cluster"
verify $fedc2 $job_id ".+Invalid.+" "Job found on wrong cluster"
verify $fedc3 $job_id ".+Invalid.+" "Job found on wrong cluster"

modify $job_id "" "" "did not allow user to modify features of pending job"

verify $fedc1 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+$fedc1,$fedc2,$fedc3" "Job not running on designated cluster"
verify $fedc2 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"
verify $fedc3 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"

log_info "################################################################"
log_info "Modify Federation Cluster Features"
log_info "################################################################"
cancel_federation_jobs

sbatch $my_sbatch $script fa
sbatch $my_sbatch $script fb
sbatch $my_sbatch $script fc

set job_id [sbatch $my_sbatch $script ""]

verify $fedc1 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+$fedc1,$fedc2,$fedc3" "Job not running on designated cluster"
verify $fedc2 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"
verify $fedc3 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"

modify_with_clusters $job_id "$fedc1" "" "did not allow user to modify features of pending job"

verify $fedc1 $job_id "PD\\s+$fedc1\\s+$fedc1" "Job not running on designated cluster"
verify $fedc2 $job_id ".+Invalid.+" "Job found on wrong cluster"
verify $fedc3 $job_id ".+Invalid.+" "Job found on wrong cluster"

modify_with_clusters $job_id "$fedc1,$fedc2" "" "did not allow user to modify features of pending job"

verify $fedc1 $job_id "PD\\s+$fedc1,$fedc2\\s+$fedc1,$fedc2" "Job not running on designated cluster"
verify $fedc2 $job_id "PD\\s+$fedc1,$fedc2\\s+NA" "Job not running on designated cluster"
verify $fedc3 $job_id ".+Invalid.+" "Job found on wrong cluster"

modify_with_clusters $job_id "$fedc2" "" "did not allow user to modify features of pending job"

verify $fedc1 $job_id "RV\\s+$fedc2\\s+$fedc2" "Job not running on designated cluster"
verify $fedc2 $job_id "PD\\s+$fedc2\\s+NA" "Job not running on designated cluster"
verify $fedc3 $job_id ".+Invalid.+" "Job found on wrong cluster"

modify_with_clusters $job_id "" "" "did not allow user to modify features of pending job"

verify $fedc1 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+$fedc1,$fedc2,$fedc3" "Job not running on designated cluster"
verify $fedc2 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"
verify $fedc3 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"

# This one shows that the $fedc1 is routing the update from $fedc1 to $fedc2

log_info "################################################################"
log_info "$fedc1 routing the update from $fedc1 to $fedc2"
log_info "################################################################"

set job_id [sbatch $my_sbatch $script "" -M$fedc2]

verify $fedc2 $job_id "PD\\s+$fedc2\\s+$fedc2" "Job not running on designated cluster"
verify $fedc1 $job_id ".+Invalid.+" "Job found on wrong cluster"
verify $fedc3 $job_id ".+Invalid.+" "Job found on wrong cluster"

modify_with_clusters $job_id "$fedc1" "" "did not allow user to modify features of pending job"

verify $fedc2 $job_id "RV\\s+$fedc1\\s+$fedc1" "Job not running on designated cluster"
verify $fedc1 $job_id "PD\\s+$fedc1\\s+NA" "Job not running on designated cluster"
verify $fedc3 $job_id ".+Invalid.+" "Job found on wrong cluster"

modify_with_clusters $job_id "" "" "did not allow user to modify features of pending job"

verify $fedc2 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+$fedc1,$fedc2,$fedc3" "Job not running on designated cluster"
verify $fedc1 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"
verify $fedc3 $job_id "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"

# Federations and clusterfeatures ($fedc1 and fb - so the job is not eligible
#				  anywhere beacause it can't have both)

log_info "################################################################"
log_info "Federations and ClusterFeatures"
log_info "################################################################"

modify_with_clusters $job_id "$fedc1" "" "did not allow user to modify features of pending job"

verify $fedc2 $job_id "RV\\s+$fedc1\\s+$fedc1" "Job not running on designated cluster"
verify $fedc1 $job_id "PD\\s+$fedc1\\s+NA" "Job not running on designated cluster"
verify $fedc3 $job_id ".+Invalid.+" "Job found on wrong cluster"

modify $job_id "fb" "" "did not allow user to modify features of pending job"

verify $fedc2 $job_id "RV\\s+NA\\s+NA" "Job not running on designated cluster"
verify $fedc1 $job_id ".+Invalid.+" "Job found on wrong cluster"
verify $fedc3 $job_id ".+Invalid.+" "Job found on wrong cluster"

modify $job_id "fa" "" "did not allow user to modify features of pending job"

verify $fedc2 $job_id "RV\\s+$fedc1\\s+$fedc1" "Job not running on designated cluster"
verify $fedc1 $job_id "PD\\s+$fedc1\\s+NA" "Job not running on designated cluster"
verify $fedc3 $job_id ".+Invalid.+" "Job found on wrong cluster"

log_info "################################################################"
log_info "Federations and Negative ClusterFeatures"
log_info "################################################################"

cancel_federation_jobs

set jid(0) [sbatch $my_sbatch $script !fc,fb]
set jid(1) [sbatch $my_sbatch $script !fa,fc]
set jid(2) [sbatch $my_sbatch $script !fb,fa]

verify $fedc1 $jid(0) "R\\s+$fedc1\\s+$fedc1" "Job not running on designated cluster"
verify $fedc2 $jid(1) "R\\s+$fedc2\\s+$fedc2" "Job not running on designated cluster"
verify $fedc3 $jid(2) "R\\s+$fedc3\\s+$fedc3" "Job not running on designated cluster"

# Submit pending job on all clusters then move around using negative constraints

set jid(4) [sbatch $my_sbatch $script ""]

verify $fedc1 $jid(4) "PD\\s+$fedc1,$fedc2,$fedc3\\s+$fedc1,$fedc2,$fedc3" "Job not running on designated cluster"
verify $fedc2 $jid(4) "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"
verify $fedc3 $jid(4) "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"

modify $jid(4) "!fb" "" "did not allow user to modify features of pending job"

verify $fedc2 $jid(4) ".+Invalid.+" "Job found on wrong cluster"
verify $fedc1 $jid(4) "PD\\s+$fedc1,$fedc3\\s+" "Job not running on designated cluster"
verify $fedc3 $jid(4) "PD\\s+$fedc1,$fedc3\\s+" "Job not running on designated cluster"

modify $jid(4) "!fc" "" "did not allow user to modify features of pending job"

verify $fedc2 $jid(4) "PD\\s+$fedc1,$fedc2\\s+" "Job not running on designated cluster"
verify $fedc1 $jid(4) "PD\\s+$fedc1,$fedc2\\s+" "Job not running on designated cluster"
verify $fedc3 $jid(4) ".+Invalid.+" "Job found on wrong cluster"

modify $jid(4) "!fa" "" "did not allow user to modify features of pending job"

verify $fedc1 $jid(4) "RV\\s+$fedc2,$fedc3\\s+" "Job found on wrong cluster"
verify $fedc2 $jid(4) "PD\\s+$fedc2,$fedc3\\s+" "Job not running on designated cluster"
verify $fedc3 $jid(4) "PD\\s+$fedc2,$fedc3\\s+" "Job not running on designated cluster"

modify $jid(4) "!fb,fc" "" "did not allow user to modify features of pending job"

verify $fedc2 $jid(4) ".+Invalid.+" "Job found on wrong cluster"
verify $fedc1 $jid(4) "PD\\s+$fedc1\\s+" "Job not running on designated cluster"
verify $fedc3 $jid(4) ".+Invalid.+" "Job found on wrong cluster"

modify $jid(4) "!fa,fc" "" "did not allow user to modify features of pending job"

verify $fedc1 $jid(4) "RV\\s+$fedc2\\s+$fedc2" "Job found on wrong cluster"
verify $fedc2 $jid(4) "PD\\s+$fedc2\\s+" "Job not running on designated cluster"
verify $fedc3 $jid(4) ".+Invalid.+" "Job found on wrong cluster"

modify $jid(4) "!fb,fa" "" "did not allow user to modify features of pending job"

verify $fedc2 $jid(4) ".+Invalid.+" "Job found on wrong cluster"
verify $fedc3 $jid(4) "PD\\s+$fedc3\\s+" "Job not running on designated cluster"
verify $fedc1 $jid(4) "RV\\s+$fedc3\\s+$fedc3" "Job found on wrong cluster"

# Clear features again

modify $jid(4) "" "" "did not allow user to modify features of pending job"

verify $fedc1 $jid(4) "PD\\s+$fedc1,$fedc2,$fedc3\\s+$fedc1,$fedc2,$fedc3" "Job not running on designated cluster"
verify $fedc2 $jid(4) "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"
verify $fedc3 $jid(4) "PD\\s+$fedc1,$fedc2,$fedc3\\s+NA" "Job not running on designated cluster"

# Submit pending jobs to single cluster using negative constraints

set jid(0) [sbatch $my_sbatch $script !fc,fb]

wait_for_fed_job $jid(0) PENDING $fedc1

verify $fedc2 $jid(0) ".+Invalid.+" "Job found on wrong cluster"
verify $fedc1 $jid(0) "PD\\s+$fedc1\\s+$fedc1" "Job not running on designated cluster"
verify $fedc3 $jid(0) ".+Invalid.+" "Job found on wrong cluster"

set jid(0) [sbatch $my_sbatch $script !fa,fb]

sleep 5

verify $fedc2 $jid(0) ".+Invalid.+" "Job found on wrong cluster"
verify $fedc3 $jid(0) "PD\\s+$fedc3\\s+NA" "Job not running on designated cluster"
verify $fedc1 $jid(0) "RV\\s+$fedc3\\s+$fedc3" "Job found on wrong cluster"

set jid(0) [sbatch $my_sbatch $script !fc,fa]

sleep 5

verify $fedc1 $jid(0) "RV\\s+$fedc2\\s+$fedc2" "Job found on wrong cluster"
verify $fedc2 $jid(0) "PD\\s+$fedc2\\s+NA" "Job not running on designated cluster"
verify $fedc3 $jid(0) ".+Invalid.+" "Job found on wrong cluster"

# Error test--negate all clusters

set matches 0

set command "$my_sbatch -N$node_count --exclusive --output=/dev/null \
	--error=/dev/null -t5 --wrap $script --cluster-constraint=!fa,fb,fc"
spawn {*}$command
expect {
	-re ".+Invalid cluster feature specification" {
		incr matches
	}
	timeout {
		fail "sbatch not responding"
	}
	eof {
		wait
	}
}

if {$matches != 1} {
	fail "Batch submit failure"
}