1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
|
"""reaper self-test"""
import logging
import time
import warnings
from unittest import mock
import pytest
from tests.integration_tests import reaper
from tests.integration_tests.instances import IntegrationInstance
LOG = logging.Logger(__name__)
class MockInstance(IntegrationInstance):
# because of instance id printing
instance = mock.Mock()
def __init__(self, times_refused):
self.times_refused = times_refused
self.call_count = 0
# assert that destruction succeeded
self.stopped = False
def destroy(self):
"""destroy() only succeeds after failing N=times_refused times"""
if self.call_count == self.times_refused:
self.stopped = True
return
self.call_count += 1
raise RuntimeError("I object!")
@pytest.mark.ci
class TestReaper:
def test_start_stop(self):
"""basic setup teardown"""
instance = MockInstance(0)
r = reaper.Reaper()
# start / stop
r.start()
r.stop()
# start / reap / stop
r.start()
r.reap(instance)
r.stop()
# start / stop
r.start()
r.stop()
assert instance.stopped
def test_basic_reap(self):
"""basic setup teardown"""
i_1 = MockInstance(0)
r = reaper.Reaper()
r.start()
r.reap(i_1)
r.stop()
assert i_1.stopped
def test_unreaped_instance(self):
"""a single warning should print for any number of leaked instances"""
i_1 = MockInstance(64)
i_2 = MockInstance(64)
r = reaper.Reaper()
r.start()
r.reap(i_1)
r.reap(i_2)
with warnings.catch_warnings(record=True) as w:
r.stop()
assert len(w) == 1
def test_stubborn_reap(self):
"""verify that stubborn instances are cleaned"""
sleep_time = 0.000_001
sleep_total = 0.0
instances = [
MockInstance(0),
MockInstance(3),
MockInstance(6),
MockInstance(9),
MockInstance(12),
MockInstance(9),
MockInstance(6),
MockInstance(3),
MockInstance(0),
]
# forcibly disallow sleeping, to avoid wasted time during tests
r = reaper.Reaper(timeout=0.0)
r.start()
for i in instances:
r.reap(i)
# this should really take no time at all, waiting 1s should be plenty
# of time for the reaper to reap it when not sleeping
while sleep_total < 1.0:
# are any still undead?
any_undead = False
for i in instances:
if not i.stopped:
any_undead = True
break
if not any_undead:
# test passed
# Advance to GO, collect $400
break
# sleep then recheck, incremental backoff
sleep_total += sleep_time
sleep_time *= 2
time.sleep(sleep_time)
r.stop()
for i in instances:
assert i.stopped, (
f"Reaper didn't reap stubborn instance {i} in {sleep_total}s. "
"Something appears to be broken in the reaper logic or test."
)
def test_start_stop_multiple(self):
"""reap lots of instances
obedient ones
"""
num = 64
instances = []
r = reaper.Reaper()
r.start()
for _ in range(num):
i = MockInstance(0)
instances.append(i)
r.reap(i)
r.stop()
for i in instances:
assert i.stopped
|
