1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
|
#!/usr/bin/env python3
"""Sanity checks for test data.
This program contains a class for traversing test cases that can be used
independently of the checks.
"""
# Copyright The Mbed TLS Contributors
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import glob
import os
import re
import sys
class Results:
"""Store file and line information about errors or warnings in test suites."""
def __init__(self, options):
self.errors = 0
self.warnings = 0
self.ignore_warnings = options.quiet
def error(self, file_name, line_number, fmt, *args):
sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n').
format(file_name, line_number, *args))
self.errors += 1
def warning(self, file_name, line_number, fmt, *args):
if not self.ignore_warnings:
sys.stderr.write(('{}:{}:Warning:' + fmt + '\n')
.format(file_name, line_number, *args))
self.warnings += 1
class TestDescriptionExplorer:
"""An iterator over test cases with descriptions.
The test cases that have descriptions are:
* Individual unit tests (entries in a .data file) in test suites.
* Individual test cases in ssl-opt.sh.
This is an abstract class. To use it, derive a class that implements
the process_test_case method, and call walk_all().
"""
def process_test_case(self, per_file_state,
file_name, line_number, description):
"""Process a test case.
per_file_state: an object created by new_per_file_state() at the beginning
of each file.
file_name: a relative path to the file containing the test case.
line_number: the line number in the given file.
description: the test case description as a byte string.
"""
raise NotImplementedError
def new_per_file_state(self):
"""Return a new per-file state object.
The default per-file state object is None. Child classes that require per-file
state may override this method.
"""
#pylint: disable=no-self-use
return None
def walk_test_suite(self, data_file_name):
"""Iterate over the test cases in the given unit test data file."""
in_paragraph = False
descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
with open(data_file_name, 'rb') as data_file:
for line_number, line in enumerate(data_file, 1):
line = line.rstrip(b'\r\n')
if not line:
in_paragraph = False
continue
if line.startswith(b'#'):
continue
if not in_paragraph:
# This is a test case description line.
self.process_test_case(descriptions,
data_file_name, line_number, line)
in_paragraph = True
def walk_ssl_opt_sh(self, file_name):
"""Iterate over the test cases in ssl-opt.sh or a file with a similar format."""
descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
with open(file_name, 'rb') as file_contents:
for line_number, line in enumerate(file_contents, 1):
# Assume that all run_test calls have the same simple form
# with the test description entirely on the same line as the
# function name.
m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line)
if not m:
continue
description = m.group(1)
self.process_test_case(descriptions,
file_name, line_number, description)
@staticmethod
def collect_test_directories():
"""Get the relative path for the TLS and Crypto test directories."""
if os.path.isdir('tests'):
tests_dir = 'tests'
elif os.path.isdir('suites'):
tests_dir = '.'
elif os.path.isdir('../suites'):
tests_dir = '..'
directories = [tests_dir]
return directories
def walk_all(self):
"""Iterate over all named test cases."""
test_directories = self.collect_test_directories()
for directory in test_directories:
for data_file_name in glob.glob(os.path.join(directory, 'suites',
'*.data')):
self.walk_test_suite(data_file_name)
ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh')
if os.path.exists(ssl_opt_sh):
self.walk_ssl_opt_sh(ssl_opt_sh)
for ssl_opt_file_name in glob.glob(os.path.join(directory, 'opt-testcases',
'*.sh')):
self.walk_ssl_opt_sh(ssl_opt_file_name)
class TestDescriptions(TestDescriptionExplorer):
"""Collect the available test cases."""
def __init__(self):
super().__init__()
self.descriptions = set()
def process_test_case(self, _per_file_state,
file_name, _line_number, description):
"""Record an available test case."""
base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name))
key = ';'.join([base_name, description.decode('utf-8')])
self.descriptions.add(key)
def collect_available_test_cases():
"""Collect the available test cases."""
explorer = TestDescriptions()
explorer.walk_all()
return sorted(explorer.descriptions)
class DescriptionChecker(TestDescriptionExplorer):
"""Check all test case descriptions.
* Check that each description is valid (length, allowed character set, etc.).
* Check that there is no duplicated description inside of one test suite.
"""
def __init__(self, results):
self.results = results
def new_per_file_state(self):
"""Dictionary mapping descriptions to their line number."""
return {}
def process_test_case(self, per_file_state,
file_name, line_number, description):
"""Check test case descriptions for errors."""
results = self.results
seen = per_file_state
if description in seen:
results.error(file_name, line_number,
'Duplicate description (also line {})',
seen[description])
return
if re.search(br'[\t;]', description):
results.error(file_name, line_number,
'Forbidden character \'{}\' in description',
re.search(br'[\t;]', description).group(0).decode('ascii'))
if re.search(br'[^ -~]', description):
results.error(file_name, line_number,
'Non-ASCII character in description')
if len(description) > 66:
results.warning(file_name, line_number,
'Test description too long ({} > 66)',
len(description))
seen[description] = line_number
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--list-all',
action='store_true',
help='List all test cases, without doing checks')
parser.add_argument('--quiet', '-q',
action='store_true',
help='Hide warnings')
parser.add_argument('--verbose', '-v',
action='store_false', dest='quiet',
help='Show warnings (default: on; undoes --quiet)')
options = parser.parse_args()
if options.list_all:
descriptions = collect_available_test_cases()
sys.stdout.write('\n'.join(descriptions + ['']))
return
results = Results(options)
checker = DescriptionChecker(results)
checker.walk_all()
if (results.warnings or results.errors) and not options.quiet:
sys.stderr.write('{}: {} errors, {} warnings\n'
.format(sys.argv[0], results.errors, results.warnings))
sys.exit(1 if results.errors else 0)
if __name__ == '__main__':
main()
|
