1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
|
/*
* Copyright 2009- ECMWF.
*
* This software is licensed under the terms of the Apache Licence version 2.0
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
* In applying this licence, ECMWF does not waive the privileges and immunities
* granted to it by virtue of its status as an intergovernmental organisation
* nor does it submit to any jurisdiction.
*/
#include <iostream>
#include <boost/test/unit_test.hpp>
#include "TestFixture.hpp"
#include "ZombieUtil.hpp"
#include "ecflow/attribute/Zombie.hpp"
#include "ecflow/core/AssertTimer.hpp"
#include "ecflow/core/Environment.hpp"
using namespace std;
using namespace ecf;
void ZombieUtil::test_clean_up(int timeout) {
// try to tidy up. Avoid leaving zombie process that mess up tests
TestFixture::client().zombieGet();
std::vector<Zombie> zombies = TestFixture::client().server_reply().zombies();
if (!zombies.empty()) {
cout << "\nZombieUtil::test_clean_up: Found Zombies:\n";
cout << "Client Environment:\n" << TestFixture::client().to_string() << "\n";
cout << Zombie::pretty_print(zombies, 9) << "\n, attempting to *fob* then *remove* ...\n";
int no_fobed = do_zombie_user_action(
ZombieCtrlAction::FOB, zombies.size(), timeout, false /* don't fail if it takes to long */);
// In order to FOB, we must wait, till a child command, talks to the server.
if (no_fobed) {
int wait = 5;
#if defined(HPUX) || defined(_AIX)
wait += 5; // On these platforms wait longer,
#endif
cout << " Fobed " << no_fobed << " left over zombies. sleeping for " << wait
<< "s before attempting to remove\n";
sleep(wait);
}
(void)do_zombie_user_action(
ZombieCtrlAction::REMOVE, no_fobed, timeout, false /* don't fail if it takes to long */);
}
}
int ZombieUtil::do_zombie_user_action(ZombieCtrlAction uc,
int expected_action_cnt,
int max_time_to_wait,
bool fail_if_to_long) {
/// return the number of zombies set to user action;
bool ecf_debug_zombies = false;
if (ecf::environment::has("ECF_DEBUG_ZOMBIES")) {
ecf_debug_zombies = true;
cout << "\n do_zombie_user_action " << ecf::to_string(uc) << " expected_action_cnt " << expected_action_cnt
<< "\n";
}
int action_set = 0;
std::vector<Zombie> action_set_zombies;
AssertTimer assertTimer(max_time_to_wait, false); // Bomb out after n seconds, fall back if test fail
while (true) {
BOOST_REQUIRE_MESSAGE(TestFixture::client().zombieGet() == 0,
"zombieGet failed should return 0\n"
<< TestFixture::client().errorMsg());
std::vector<Zombie> zombies = TestFixture::client().server_reply().zombies();
bool continue_looping = false;
for (const Zombie& z : zombies) {
switch (uc) {
case ZombieCtrlAction::FOB: {
if (!z.fob()) {
TestFixture::client().zombieFob(z); // UNBLOCK, child commands, allow zombie to complete, will
// clear server_reply().zombies()
continue_looping = true;
action_set++;
action_set_zombies.push_back(z);
}
break;
}
case ZombieCtrlAction::FAIL: {
if (!z.fail()) {
TestFixture::client().zombieFail(z); // UNBLOCK, child commands, allow zombie to complete, will
// clear server_reply().zombies()
continue_looping = true;
action_set++;
action_set_zombies.push_back(z);
}
break;
}
case ZombieCtrlAction::ADOPT: {
if (!z.adopt()) {
TestFixture::client().zombieAdopt(z); // UNBLOCK, child commands, allow zombie to complete, will
// clear server_reply().zombies()
continue_looping = true;
action_set++;
action_set_zombies.push_back(z);
}
break;
}
case ZombieCtrlAction::REMOVE: {
if (!z.remove()) { // should always return false
TestFixture::client().zombieRemove(z); // This should be immediate, and is not remembered
continue_looping = true;
action_set++;
action_set_zombies.push_back(z);
}
break;
}
case ZombieCtrlAction::BLOCK: {
if (!z.block()) {
TestFixture::client().zombieBlock(z);
continue_looping = true;
action_set++;
action_set_zombies.push_back(z);
}
break;
}
case ZombieCtrlAction::KILL: {
if (!z.kill()) {
TestFixture::client().zombieKill(z);
continue_looping = true;
action_set++;
action_set_zombies.push_back(z);
}
break;
}
}
}
if (expected_action_cnt == action_set) {
break; // return
}
if (!continue_looping && action_set > 0) {
if (expected_action_cnt == 0) {
break; // return, some clients set this as 0
}
}
// make sure test does not take too long.
if (assertTimer.duration() >= assertTimer.timeConstraint()) {
if (expected_action_cnt > 0 && action_set > 0) {
if (ecf_debug_zombies) {
cout << " timeing out after action_set = " << action_set
<< " expected_action_cnt = " << expected_action_cnt << "\n";
}
break;
}
std::stringstream ss;
ss << "do_zombie_user_action:\nExpected " << expected_action_cnt << " zombies with user action "
<< ecf::to_string(uc) << " but found " << action_set << "\naction set zombies\n"
<< Zombie::pretty_print(action_set_zombies, 6) << ", Test taking longer than time constraint of "
<< assertTimer.timeConstraint();
if (fail_if_to_long) {
BOOST_REQUIRE_MESSAGE(false, ss.str() << " aborting\n" << Zombie::pretty_print(zombies, 6));
}
else {
cout << ss.str() << " breaking out\n" << Zombie::pretty_print(zombies, 6) << "\n";
break;
}
}
sleep(1);
}
if (ecf_debug_zombies) {
cout << " " << action_set << " zombies set to user action " << ecf::to_string(uc) << " returning\n";
cout << Zombie::pretty_print(action_set_zombies, 6);
}
return action_set;
}
|
