1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
|
/*
* Copyright 2009- ECMWF.
*
* This software is licensed under the terms of the Apache Licence version 2.0
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
* In applying this licence, ECMWF does not waive the privileges and immunities
* granted to it by virtue of its status as an intergovernmental organisation
* nor does it submit to any jurisdiction.
*/
#include <cstdlib>
#include <fstream>
#include <iostream>
#include <boost/test/unit_test.hpp>
#include "ServerTestHarness.hpp"
#include "TestFixture.hpp"
#include "ecflow/attribute/VerifyAttr.hpp"
#include "ecflow/core/AssertTimer.hpp"
#include "ecflow/core/Chrono.hpp"
#include "ecflow/core/Converter.hpp"
#include "ecflow/core/Timer.hpp"
#include "ecflow/node/Defs.hpp"
#include "ecflow/node/Family.hpp"
#include "ecflow/node/Suite.hpp"
#include "ecflow/node/Task.hpp"
#include "ecflow/test/scaffold/Naming.hpp"
using namespace ecf;
#if defined(_AIX)
static int timeout = 30;
#else
static int timeout = 20;
#endif
BOOST_AUTO_TEST_SUITE(S_Test)
BOOST_AUTO_TEST_SUITE(T_Suspend)
static void waitForTimeDependenciesToBeFree(int max_time_to_wait) {
// wait for a period of time, while time dependencies fire.
TestFixture::client().set_throw_on_error(false);
AssertTimer assertTimer(max_time_to_wait, false); // Bomb out after n seconds, fall back if test fail
while (1) {
BOOST_REQUIRE_MESSAGE(TestFixture::client().sync_local() == 0,
"sync_local failed should return 0\n"
<< TestFixture::client().errorMsg());
defs_ptr defs = TestFixture::client().defs();
std::vector<Task*> tasks;
defs->getAllTasks(tasks);
size_t taskTimeDepIsFree = 0;
for (Task* task : tasks) {
size_t attSetFree = 0;
for (const ecf::TimeAttr& timeAttr : task->timeVec()) {
if (timeAttr.isFree(task->suite()->calendar())) {
attSetFree++;
}
}
if (attSetFree == task->timeVec().size()) {
taskTimeDepIsFree++;
}
}
if (taskTimeDepIsFree == tasks.size()) {
break;
}
BOOST_REQUIRE_MESSAGE(assertTimer.duration() < assertTimer.timeConstraint(),
"waitForTimeDependenciesToBeFree Test wait "
<< assertTimer.duration() << " taking longer than time constraint of "
<< assertTimer.timeConstraint() << " aborting");
sleep(1);
}
}
// test:: suspend/shutdown. During suspend the time dependencies should still
// be handled. When the server/node is restarted/resumed the task should
// be submitted straight away.(i.e providing no trigger/complete dependencies)
BOOST_AUTO_TEST_CASE(test_shutdown) {
ECF_NAME_THIS_TEST();
DurationTimer timer;
TestClean clean_at_start_and_end;
// Create the defs file corresponding to the text below
// ECF_HOME variable is automatically added by the test harness.
// ECF_INCLUDE variable is automatically added by the test harness.
// SLEEPTIME variable is automatically added by the test harness.
// ECF_CLIENT_EXE_PATH variable is automatically added by the test harness.
// This is substituted in sms includes
// Allows test to run without requiring installation
Defs theDefs;
{
// Initialise clock with today's date and time, then create a time attribute
// with today's time + minute Avoid adding directly to TimeSlot
// i.e. if local time is 9:59, and we create a TimeSlot like
// task->addTime( ecf::TimeAttr( ecf::TimeSlot(theTm.tm_hour,theTm.tm_min+3) ) );
// The minute will be 62, which is illegal and will not parse
auto theLocalTime = Calendar::second_clock_time();
// For each 2 seconds of poll in the server update calendar by 1 minute
// Note: if we use 1 seconds poll to update calendar by 1 minute, then
// we will find that state change happens at job submission interval,
// and hence skews time series. Which can leave state in a queued state,
// and hence test never completes
suite_ptr suite = theDefs.add_suite("test_shutdown");
ClockAttr clockAttr(theLocalTime);
suite->addClock(clockAttr);
family_ptr fam = suite->add_family("family");
int taskSize = 2; // on linux 1024 tasks take ~4 seconds for job submission
for (int i = 0; i < taskSize; i++) {
task_ptr task = fam->add_task("t" + ecf::convert_to<std::string>(i));
auto time1 = theLocalTime + boost::posix_time::minutes(1 + i);
task->addTime(ecf::TimeAttr(ecf::TimeSlot(time1.time_of_day())));
task->addVerify(VerifyAttr(NState::COMPLETE, 1)); // task should complete 1 times
}
}
// The test harness will create corresponding directory structure and populate with standard sms files.
ServerTestHarness serverTestHarness;
serverTestHarness.run(theDefs,
ServerTestHarness::testDataDefsLocation("test_shutdown.def"),
1,
false /* don't wait for test to finish */);
// Shutdown the server. The time dependencies *should* still be handled
// *** If this test fails, in that we fail to restart() server it will mess up
// *** any following test.
TestFixture::client().set_throw_on_error(false);
BOOST_REQUIRE_MESSAGE(TestFixture::client().shutdownServer() == 0,
CtsApi::shutdownServer() << " failed should return 0.\n"
<< TestFixture::client().errorMsg());
// wait for a period of time, while time dependencies fire.
(void)waitForTimeDependenciesToBeFree(timeout);
// restart server, all jobs should be launched straight away
BOOST_REQUIRE_MESSAGE(TestFixture::client().restartServer() == 0,
CtsApi::restartServer() << " failed should return 0.\n"
<< TestFixture::client().errorMsg());
// Wait for submitted jobs in restart server to complete
bool verifyAttrInServer = true;
defs_ptr serverDefs = serverTestHarness.testWaiter(theDefs, timeout, verifyAttrInServer);
BOOST_REQUIRE_MESSAGE(serverDefs.get(), " Failed to return server after restartServer");
// cout << "Printing Defs \n";
// std::cout << *serverDefs.get();
std::cout << timer.duration() << " update-calendar-count(" << serverDefs->updateCalendarCount() << ")\n";
}
// test:: suspend/shutdown. During suspend the time dependencies should still
// be handled. When the server/node is restarted/resumed the task should
// be submitted straight away.(i.e providing no trigger/complete dependencies)
BOOST_AUTO_TEST_CASE(test_suspend_node) {
ECF_NAME_THIS_TEST();
DurationTimer timer;
TestClean clean_at_start_and_end;
// Create the defs file corresponding to the text below
// ECF_HOME variable is automatically added by the test harness.
// ECF_INCLUDE variable is automatically added by the test harness.
// SLEEPTIME variable is automatically added by the test harness.
// ECF_CLIENT_EXE_PATH variable is automatically added by the test harness.
// This is substituted in sms includes
// Allows test to run without requiring installation
Defs theDefs;
{
// Initialise clock with today's date and time, then create a time attribute
// with today's time + minute Avoid adding directly to TimeSlot
// i.e. if local time is 9:59, and we create a TimeSlot like
// task->addTime( ecf::TimeAttr( ecf::TimeSlot(theTm.tm_hour,theTm.tm_min+3) ) );
// The minute will be 62, which is illegal and will not parse
auto theLocalTime = Calendar::second_clock_time();
suite_ptr suite = theDefs.add_suite("test_suspend_node");
ClockAttr clockAttr(theLocalTime);
suite->addClock(clockAttr);
task_ptr t1 = suite->add_task("t1");
auto time1 = theLocalTime + boost::posix_time::minutes(1);
t1->addTime(ecf::TimeAttr(ecf::TimeSlot(time1.time_of_day())));
task_ptr t2 = suite->add_task("t2");
auto time2 = theLocalTime + boost::posix_time::minutes(2);
t2->addTime(ecf::TimeAttr(ecf::TimeSlot(time2.time_of_day())));
family_ptr fam = suite->add_family("family");
for (int i = 0; i < 2; i++) {
task_ptr task = fam->add_task("t" + ecf::convert_to<std::string>(i));
task->addVerify(VerifyAttr(NState::COMPLETE, 1)); // task should complete 1 times
auto time3 = theLocalTime + boost::posix_time::minutes(1 + i);
task->addTime(ecf::TimeAttr(ecf::TimeSlot(time3.time_of_day())));
}
}
// The test harness will create corresponding directory structure and populate with standard sms files.
ServerTestHarness serverTestHarness;
serverTestHarness.run(theDefs,
ServerTestHarness::testDataDefsLocation("test_suspend_node.def"),
1,
false /* don't wait for test to finish */);
// SUSPEND the family. The time dependencies *should* still be handled
TestFixture::client().set_throw_on_error(false);
BOOST_REQUIRE_MESSAGE(TestFixture::client().suspend("/test_suspend_node/family") == 0,
CtsApi::to_string(CtsApi::suspend("/test_suspend_node/family"))
<< " failed should return 0.\n"
<< TestFixture::client().errorMsg());
// wait for a period of time, while time dependencies fire.
waitForTimeDependenciesToBeFree(timeout);
// RESUME the family, all jobs should be launched straight away, since time dependencies should be free
BOOST_REQUIRE_MESSAGE(TestFixture::client().resume("/test_suspend_node/family") == 0,
CtsApi::to_string(CtsApi::resume("/test_suspend_node/family"))
<< " failed should return 0.\n"
<< TestFixture::client().errorMsg());
// Wait for submitted jobs to complete
bool verifyAttrInServer = true;
defs_ptr serverDefs = serverTestHarness.testWaiter(theDefs, timeout, verifyAttrInServer);
BOOST_REQUIRE_MESSAGE(serverDefs.get(), " Failed to return server after restartServer");
// cout << "Printing Defs \n";
// std::cout << *serverDefs.get();
std::cout << timer.duration() << " update-calendar-count(" << serverDefs->updateCalendarCount() << ")\n";
}
BOOST_AUTO_TEST_SUITE_END()
BOOST_AUTO_TEST_SUITE_END()
|
