1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
|
#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
# Validate smd -d (--drain-node), and -R (--reason) set.
############################################################################
# Copyright (C) 2011-2014 SchedMD LLC
# Written by Nathan Yee <nyee32@schedmd.com>
# All rights reserved
############################################################################
source ./globals
set job_id 0
set node_name ""
set file_in "$test_dir/script"
set reason_str "test"
if {![param_contains [get_config_param "SlurmctldPlugstack"] "nonstop"]} {
skip "This test is only compatible when SlurmctldPlugstack includes nonstop"
}
if {![is_super_user]} {
skip "This test is only suitable for a super user (to restore down nodes)"
}
proc cleanup {} {
global job_id node_name
cancel_job $job_id
if {$node_name ne ""} {
reset_state $node_name
}
}
proc get_node {job_id} {
global scontrol squeue re_word_str re_word_str
set node_list ""
spawn $squeue -j $job_id --noheader -o NodeList=%N
expect {
-re "NodeList=($re_word_str)" {
set node_list $expect_out(1,string)
exp_continue
}
timeout {
fail "squeue is not responding"
}
eof {
wait
}
}
set node_name ""
spawn $scontrol show hostnames $node_list
expect {
-re "($re_word_str)" {
set node_name $expect_out(1,string)
exp_continue
}
timeout {
fail "scontrol is not responding"
}
eof {
wait
}
}
return $node_name
}
proc reset_state {node} {
global scontrol
spawn $scontrol update NodeName=$node State=RESUME
expect {
timeout {
fail "scontrol is not responding"
}
eof {
wait
}
}
}
make_bash_script $file_in "$bin_sleep 100"
#
# Submit batch script to test
#
set job_id 0
spawn $sbatch -N2 --no-kill --output=/dev/null $file_in
expect {
-re "Submitted batch job ($number)" {
set job_id $expect_out(1,string)
exp_continue
}
timeout {
fail "sbatch is not responding"
}
eof {
wait
}
}
if {$job_id == 0} {
fail "sbatch did not submit job"
}
#
# Wait for job to start
#
wait_for_job -fail $job_id "RUNNING"
set node_name [get_node $job_id]
#
# Validate smd -d option
#
set sub_match 0
spawn $smd -d $node_name $job_id -R $reason_str
expect {
-re "Job $job_id node $node_name is being drained" {
set sub_match 1
exp_continue
}
timeout {
fail "smd is not reponding"
}
eof {
wait
}
}
if {$sub_match != 1} {
fail "smd did not drain node"
}
#
# Check that smd reason is set and node was drained
#
set match 0
spawn $scontrol show node $node_name
expect {
-re "State=.*FAIL" {
incr match
exp_continue
}
-re "Reason=$reason_str" {
incr match
exp_continue
}
timeout {
fail "scontrol is not responding"
}
eof {
wait
}
}
if {$match != 2} {
fail "smd did not set correct state or reason"
}
|
