1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
|
#!/usr/bin/env python3
"""Search an outcome file for configurations with given settings.
Read an outcome file and report the configurations in which test_suite_config
runs with the required settings (compilation option enabled or disabled).
"""
import argparse
import os
import re
import subprocess
from typing import Dict, FrozenSet, Iterator, List, Set
import tempfile
import unittest
from mbedtls_framework import build_tree
def make_regexp_for_settings(settings: List[str]) -> str:
"""Construct a regexp matching the interesting outcome lines.
Interesting outcome lines are from test_suite_config where the given
setting is passing.
We assume that the elements of settings don't contain regexp special
characters.
"""
return (r';test_suite_config[^;]*;Config: (' +
'|'.join(settings) +
r');PASS;')
def run_grep(regexp: str, outcome_file: str) -> List[str]:
"""Run grep on the outcome file and return the matching lines."""
env = os.environ.copy()
env['LC_ALL'] = 'C' # Speeds up some versions of GNU grep
try:
return subprocess.check_output(['grep', '-E', regexp, outcome_file],
encoding='ascii',
env=env).splitlines()
except subprocess.CalledProcessError as exn:
if exn.returncode == 1:
return [] # No results. We don't consider this an error.
raise
OUTCOME_LINE_RE = re.compile(r'[^;]*;'
r'([^;]*);'
r'test_suite_config\.(?:[^;]*);'
r'Config: ([^;]*);'
r'PASS;')
def extract_configuration_data(outcome_lines: List[str]) -> Dict[str, FrozenSet[str]]:
"""Extract the configuration data from outcome lines.
The result maps a configuration name to the list of passing settings
in that configuration.
"""
config_data = {} #type: Dict[str, Set[str]]
for line in outcome_lines:
m = OUTCOME_LINE_RE.match(line)
# Assuming a well-formed outcome file, make_regexp_for_settings()
# arranges to only return lines that should match OUTCOME_LINE_RE.
# So this assertion can't fail unless there is an unexpected
# divergence between OUTCOME_LINE_RE, make_regexp_for_settings()
# and the format of the given outcome file
assert m is not None
config_name, setting = m.groups()
if config_name not in config_data:
config_data[config_name] = set()
config_data[config_name].add(setting)
return dict((name, frozenset(settings))
for name, settings in config_data.items())
def matching_configurations(config_data: Dict[str, FrozenSet[str]],
required: List[str]) -> Iterator[str]:
"""Search configurations with the given passing settings.
config_data maps a configuration name to the list of passing settings
in that configuration.
Each setting should be an Mbed TLS compile setting (MBEDTLS_xxx or
PSA_xxx), optionally prefixed with "!".
"""
required_set = frozenset(required)
for config, observed in config_data.items():
if required_set.issubset(observed):
yield config
def search_config_outcomes(outcome_file: str, settings: List[str]) -> List[str]:
"""Search the given outcome file for reports of the given settings.
Each setting should be an Mbed TLS compile setting (MBEDTLS_xxx or
PSA_xxx), optionally prefixed with "!".
"""
# The outcome file is large enough (hundreds of MB) that parsing it
# in Python is slow. Use grep to speed this up considerably.
regexp = make_regexp_for_settings(settings)
outcome_lines = run_grep(regexp, outcome_file)
config_data = extract_configuration_data(outcome_lines)
return sorted(matching_configurations(config_data, settings))
class TestSearch(unittest.TestCase):
"""Tests of search functionality."""
OUTCOME_FILE_CONTENT = """\
whatever;foobar;test_suite_config.part;Config: MBEDTLS_FOO;PASS;
whatever;foobar;test_suite_config.part;Config: !MBEDTLS_FOO;SKIP;
whatever;foobar;test_suite_config.part;Config: MBEDTLS_BAR;PASS;
whatever;foobar;test_suite_config.part;Config: !MBEDTLS_BAR;SKIP;
whatever;foobar;test_suite_config.part;Config: MBEDTLS_QUX;SKIP;
whatever;foobar;test_suite_config.part;Config: !MBEDTLS_QUX;PASS;
whatever;fooqux;test_suite_config.part;Config: MBEDTLS_FOO;PASS;
whatever;fooqux;test_suite_config.part;Config: !MBEDTLS_FOO;SKIP;
whatever;fooqux;test_suite_config.part;Config: MBEDTLS_BAR;SKIP;
whatever;fooqux;test_suite_config.part;Config: !MBEDTLS_BAR;PASS;
whatever;fooqux;test_suite_config.part;Config: MBEDTLS_QUX;PASS;
whatever;fooqux;test_suite_config.part;Config: !MBEDTLS_QUX;SKIP;
whatever;fooqux;test_suite_something.else;Config: MBEDTLS_BAR;PASS;
whatever;boring;test_suite_config.part;Config: BORING;PASS;
whatever;parasite;not_test_suite_config.not;Config: MBEDTLS_FOO;PASS;
whatever;parasite;test_suite_config.but;Config: MBEDTLS_QUX with bells on;PASS;
whatever;parasite;test_suite_config.but;Not Config: MBEDTLS_QUX;PASS;
"""
def search(self, settings: List[str], expected: List[str]) -> None:
"""Test the search functionality.
* settings: settings to search.
* expected: expected search results.
"""
with tempfile.NamedTemporaryFile() as tmp:
tmp.write(self.OUTCOME_FILE_CONTENT.encode())
tmp.flush()
actual = search_config_outcomes(tmp.name, settings)
self.assertEqual(actual, expected)
def test_foo(self) -> None:
self.search(['MBEDTLS_FOO'], ['foobar', 'fooqux'])
def test_bar(self) -> None:
self.search(['MBEDTLS_BAR'], ['foobar'])
def test_foo_bar(self) -> None:
self.search(['MBEDTLS_FOO', 'MBEDTLS_BAR'], ['foobar'])
def test_foo_notbar(self) -> None:
self.search(['MBEDTLS_FOO', '!MBEDTLS_BAR'], ['fooqux'])
class TestOutcome(unittest.TestCase):
"""Tests of outcome file format expectations.
This class builds and runs the config tests in the current configuration.
The configuration must have at least one feature enabled and at least
one feature disabled in each category: MBEDTLS_xxx and PSA_WANT_xxx.
It needs a C compiler.
"""
outcome_content = '' # Let mypy know this field can be used in test case methods
@classmethod
def setUpClass(cls) -> None:
"""Generate, build and run the config tests."""
root_dir = build_tree.guess_project_root()
tests_dir = os.path.join(root_dir, 'tests')
suites = ['test_suite_config.mbedtls_boolean',
'test_suite_config.psa_boolean']
_output = subprocess.check_output(['make'] + suites,
cwd=tests_dir,
stderr=subprocess.STDOUT)
with tempfile.NamedTemporaryFile(dir=tests_dir) as outcome_file:
env = os.environ.copy()
env['MBEDTLS_TEST_PLATFORM'] = 'some_platform'
env['MBEDTLS_TEST_CONFIGURATION'] = 'some_configuration'
env['MBEDTLS_TEST_OUTCOME_FILE'] = outcome_file.name
for suite in suites:
_output = subprocess.check_output([os.path.join(os.path.curdir, suite)],
cwd=tests_dir,
env=env,
stderr=subprocess.STDOUT)
cls.outcome_content = outcome_file.read().decode('ascii')
def test_outcome_format(self) -> None:
"""Check that there are outcome lines matching the expected general format."""
def regex(prefix: str, result: str) -> str:
return (r'(?:\A|\n)some_platform;some_configuration;'
r'test_suite_config\.\w+;Config: {}_\w+;{};'
.format(prefix, result))
self.assertRegex(self.outcome_content, regex('MBEDTLS', 'PASS'))
self.assertRegex(self.outcome_content, regex('MBEDTLS', 'SKIP'))
self.assertRegex(self.outcome_content, regex('!MBEDTLS', 'PASS'))
self.assertRegex(self.outcome_content, regex('!MBEDTLS', 'SKIP'))
self.assertRegex(self.outcome_content, regex('PSA_WANT', 'PASS'))
self.assertRegex(self.outcome_content, regex('PSA_WANT', 'SKIP'))
self.assertRegex(self.outcome_content, regex('!PSA_WANT', 'PASS'))
self.assertRegex(self.outcome_content, regex('!PSA_WANT', 'SKIP'))
def test_outcome_lines(self) -> None:
"""Look for some sample outcome lines."""
def regex(setting: str) -> str:
return (r'(?:\A|\n)some_platform;some_configuration;'
r'test_suite_config\.\w+;Config: {};(PASS|SKIP);'
.format(setting))
self.assertRegex(self.outcome_content, regex('MBEDTLS_AES_C'))
self.assertRegex(self.outcome_content, regex('MBEDTLS_AES_ROM_TABLES'))
self.assertRegex(self.outcome_content, regex('MBEDTLS_SSL_CLI_C'))
self.assertRegex(self.outcome_content, regex('MBEDTLS_X509_CRT_PARSE_C'))
self.assertRegex(self.outcome_content, regex('PSA_WANT_ALG_HMAC'))
self.assertRegex(self.outcome_content, regex('PSA_WANT_KEY_TYPE_AES'))
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--outcome-file', '-f', metavar='FILE',
default='outcomes.csv',
help='Outcome file to read (default: outcomes.csv)')
parser.add_argument('settings', metavar='SETTING', nargs='+',
help='Required setting (e.g. "MBEDTLS_RSA_C" or "!PSA_WANT_ALG_SHA256")')
options = parser.parse_args()
found = search_config_outcomes(options.outcome_file, options.settings)
for name in found:
print(name)
if __name__ == '__main__':
main()
|
