1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
|
#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
# validate smd -r (--replace-node) option.
############################################################################
# Copyright (C) 2011-2014 SchedMD LLC
# Written by Nathan Yee <nyee32@schedmd.com>
# All rights reserved
############################################################################
source ./globals
set file_in "$test_dir/script"
set node_name ""
set job_id 0
# this test is for the nonstop --replace-node test
if {![param_contains [get_config_param "SlurmctldPlugstack"] "nonstop"]} {
skip "This test is only compatible when SlurmctldPlugstack includes nonstop"
}
if {![is_super_user]} {
skip "This test is only suitable for a super user (to restore down nodes)"
}
proc cleanup {} {
global job_id node_name
cancel_job $job_id
if {$node_name ne ""} {
reset_state $node_name
}
}
proc get_node {job_id} {
global scontrol squeue re_word_str re_word_str
set node_list ""
spawn $squeue -j $job_id --noheader -o NodeList=%N
expect {
-re "NodeList=($re_word_str)" {
set node_list $expect_out(1,string)
exp_continue
}
timeout {
fail "squeue is not responding"
}
eof {
wait
}
}
set node_name ""
spawn $scontrol show hostnames $node_list
expect {
-re "($re_word_str)" {
set node_name $expect_out(1,string)
exp_continue
}
timeout {
fail "scontrol is not responding"
}
eof {
wait
}
}
return $node_name
}
proc reset_state {node} {
global scontrol
spawn $scontrol update NodeName=$node State=RESUME
expect {
timeout {
fail "scontrol is not responding"
}
eof {
wait
}
}
}
make_bash_script $file_in "$bin_sleep 100"
#
# Submit batch script to test
#
set job_id 0
spawn $sbatch -N2 --no-kill --output=/dev/null $file_in
expect {
-re "Submitted batch job ($number)" {
set job_id $expect_out(1,string)
exp_continue
}
timeout {
fail "sbatch is not responding"
}
eof {
wait
}
}
if {$job_id == 0} {
fail "sbatch did not submit job"
}
#
# Wait for job to start
#
wait_for_job -fail $job_id "RUNNING"
set node_name [get_node $job_id]
#
# replace a node that has not failed
#
set sub_match 0
spawn $smd -r $node_name $job_id
expect {
-re "The specified node has not failed" {
log_debug "This error is expected"
set sub_match 1
exp_continue
}
timeout {
fail "smd is not reponding"
}
eof {
wait
}
}
if {$sub_match != 1} {
fail "There are nodes that are drained when there should not be"
}
#
# Drain the node so we can replace it
#
set sub_match 0
spawn $smd -d $node_name $job_id -R test
expect {
-re "Job $job_id node $node_name is being drained" {
set sub_match 1
exp_continue
}
timeout {
fail "smd is not reponding"
}
eof {
wait
}
}
if {$sub_match != 1} {
fail "smd did not drain node"
}
#
# replace the drained node
#
set sub_match 0
spawn $smd -r $node_name $job_id
expect {
-re "Job $job_id got node $node_name replaced" {
set sub_match 1
exp_continue
}
timeout {
fail "smd is not reponding"
}
eof {
wait
}
}
if {$sub_match != 1} {
fail "smd did not replace the node"
}
|
