1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
|
#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
# Validates that the smd -j (--job-info) display.
############################################################################
# Copyright (C) 2011-2014 SchedMD LLC
# Written by Nathan Yee <nyee32@schedmd.com>
# All rights reserved
############################################################################
source ./globals
set node_name ""
set file_in "$test_dir/script"
set job_id 0
if {![param_contains [get_config_param "SlurmctldPlugstack"] "nonstop"]} {
skip "This test is only compatible when SlurmctldPlugstack includes nonstop"
}
if {![is_super_user]} {
skip "This test is only suitable for a super user (to restore down nodes)"
}
proc cleanup {} {
global job_id node_name
cancel_job $job_id
if {$node_name ne ""} {
reset_state $node_name
}
}
make_bash_script $file_in "$bin_sleep 100"
proc get_node {job_id} {
global scontrol squeue re_word_str re_word_str
set node_list ""
spawn $squeue -j $job_id --noheader -o NodeList=%N
expect {
-re "NodeList=($re_word_str)" {
set node_list $expect_out(1,string)
exp_continue
}
timeout {
fail "squeue is not responding"
}
eof {
wait
}
}
set node_name ""
spawn $scontrol show hostnames $node_list
expect {
-re "($re_word_str)" {
set node_name $expect_out(1,string)
exp_continue
}
timeout {
fail "scontrol is not responding"
}
eof {
wait
}
}
return $node_name
}
proc reset_state {node} {
global scontrol
spawn $scontrol update NodeName=$node State=RESUME
expect {
timeout {
fail "scontrol is not responding"
}
eof {
wait
}
}
}
#
# Submit batch script to test
#
set job_id 0
spawn $sbatch -N2 --no-kill -o /dev/null $file_in
expect {
-re "Submitted batch job ($number)" {
set job_id $expect_out(1,string)
exp_continue
}
timeout {
fail "sbatch is not responding"
}
eof {
wait
}
}
if {$job_id == 0} {
fail "sbatch did not submit job"
}
#
# Wait for job to start
#
wait_for_job -fail $job_id "RUNNING"
set node_name [get_node $job_id]
#
# Drain the node
#
set sub_match 0
spawn $smd -d $node_name $job_id -R test
expect {
-re "Job $job_id node $node_name is being drained" {
set sub_match 1
exp_continue
}
timeout {
fail "smd is not responding"
}
eof {
wait
}
}
if {$sub_match != 1} {
fail "smd did not drain node"
}
#
# Now drop the node, which may extend our time limit
#
set sub_match 0
spawn $smd -D $node_name $job_id
expect {
-re "Job $job_id node $node_name dropped" {
set sub_match 1
exp_continue
}
timeout {
fail "smd is not responding"
}
eof {
wait
}
}
if {$sub_match != 1} {
fail "smd did not drain node"
}
#
# Test the -j option
#
set sub_match 0
spawn $smd -j $job_id
expect {
-re "Job $job_id .*" {
set sub_match 1
exp_continue
}
timeout {
fail "smd is not responding"
}
eof {
wait
}
}
if {$sub_match != 1} {
fail "smd did not get information about the job"
}
|
