1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
|
# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import cylc
from cylc.flow.task_remote_mgr import (
REMOTE_FILE_INSTALL_DONE,
REMOTE_FILE_INSTALL_FAILED
)
async def test_remote_tidy(
flow,
scheduler,
start,
mock_glbl_cfg,
one_conf,
monkeypatch
):
"""Remote tidy gets platforms for install targets.
In particular, referencing https://github.com/cylc/cylc-flow/issues/5429,
ensure that install targets defined implicitly by platform name are found.
Mock remote init map:
- Include an install target (quiz) with
message != REMOTE_FILE_INSTALL_DONE to ensure that
this is picked out.
- Install targets where we can get a platform
- foo - Install target is implicitly the platfrom name.
- bar9 - The install target is implicitly the plaform name,
and the platform name matches a platform regex.
- baz - Install target is set explicitly.
- An install target (qux) where we cannot get a platform: Ensure
that we get the desired error.
Test that platforms with no good hosts (no host not in bad hosts).
"""
# Monkeypatch away subprocess.Popen calls - prevent any interaction with
# remotes actually happening:
class MockProc:
def __init__(self, *args, **kwargs):
self.poll = lambda: True
if (
'baum' in args[0]
or 'bay' in args[0]
):
self.returncode = 255
else:
self.returncode = 0
self.communicate = lambda: ('out', 'err')
monkeypatch.setattr(
cylc.flow.task_remote_mgr,
'Popen',
lambda *args, **kwargs: MockProc(*args, **kwargs)
)
# Monkeypath function to add a sort order which we don't need in the
# real code but rely to prevent the test becoming flaky:
def mock_get_install_target_platforms_map(*args, **kwargs):
"""Add sort to original function to ensure test consistency"""
from cylc.flow.platforms import get_install_target_to_platforms_map
result = get_install_target_to_platforms_map(*args, **kwargs)
sorted_result = {}
for key in sorted(result):
sorted_result[key] = sorted(
result[key], key=lambda x: x['name'], reverse=True)
return sorted_result
monkeypatch.setattr(
cylc.flow.task_remote_mgr,
'get_install_target_to_platforms_map',
mock_get_install_target_platforms_map
)
# Set up global config
mock_glbl_cfg(
'cylc.flow.platforms.glbl_cfg',
'''
[platforms]
[[foo]]
# install target = foo (implicit)
# hosts = foo (implicit)
[[bar.]]
# install target = bar1 to bar9 (implicit)
# hosts = bar1 to bar9 (implicit)
[[baz]]
install target = baz
# baum and bay should be uncontactable:
hosts = baum, bay, baz
[[[selection]]]
method = definition order
[[notthisone]]
install target = baz
hosts = baum, bay
[[bay]]
''',
)
# Get a scheduler:
id_ = flow(one_conf)
schd = scheduler(id_)
async with start(schd) as log:
# Write database with 6 tasks using 3 platforms:
platforms = ['baz', 'bar9', 'foo', 'notthisone', 'bay']
line = r"('', '', {}, 0, 1, '', '', 0,'', '', '', 0, '', '{}', 4, '')"
stmt = r"INSERT INTO task_jobs VALUES" + r','.join([
line.format(i, platform) for i, platform in enumerate(platforms)
])
schd.workflow_db_mgr.pri_dao.connect().execute(stmt)
schd.workflow_db_mgr.pri_dao.connect().commit()
# Mock a remote init map.
schd.task_job_mgr.task_remote_mgr.remote_init_map = {
'baz': REMOTE_FILE_INSTALL_DONE, # Should match platform baz
'bar9': REMOTE_FILE_INSTALL_DONE, # Should match platform bar.
'foo': REMOTE_FILE_INSTALL_DONE, # Should match plaform foo
'qux': REMOTE_FILE_INSTALL_DONE, # Should not match a plaform
'quiz': REMOTE_FILE_INSTALL_FAILED, # Should not be considered
'bay': REMOTE_FILE_INSTALL_DONE, # Should return NoPlatforms
}
# Clear the log, run the test:
log.clear()
schd.task_job_mgr.task_remote_mgr.bad_hosts.update(['baum', 'bay'])
schd.task_job_mgr.task_remote_mgr.remote_tidy()
pass
records = [str(r.msg) for r in log.records]
# We can't get qux, no defined platform has a matching install target:
qux_msg = 'No platforms available to remote tidy install targets:\n * qux'
assert qux_msg in records
# We can get foo bar baz, and we try to remote tidy them.
# (This will ultimately fail, but past the point we are testing).
for target in ['foo', 'bar9', 'baz']:
msg = f'platform: {target} - remote tidy (on {target})'
assert msg in records
# We haven't done anything with Quiz because we're only looking
# at cases where platform == REMOTE_FILE_INSTALL_DONE
assert not [r for r in records if 'quiz' in r]
notthisone_msg = (
'platform: notthisone - clean up did not complete'
'\nUnable to find valid host for notthisone'
)
assert notthisone_msg in records
bay_msg = (
'Unable to find a platform from install target'
' bay during remote tidy.')
assert bay_msg in records
|
