1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
#!/usr/bin/env python3
# Copyright Catch2 Authors
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE.txt or copy at
# https://www.boost.org/LICENSE_1_0.txt)
# SPDX-License-Identifier: BSL-1.0
import os
import subprocess
import sys
import re
import json
cmake_version_regex = re.compile('cmake version (\d+)\.(\d+)\.(\d+)')
def get_cmake_version():
result = subprocess.run(['cmake', '--version'],
capture_output = True,
check = True,
text = True)
version_match = cmake_version_regex.match(result.stdout)
if not version_match:
print('Could not find cmake version in output')
print(f"output: '{result.stdout}'")
exit(4)
return (int(version_match.group(1)),
int(version_match.group(2)),
int(version_match.group(3)))
def build_project(sources_dir, output_base_path, catch2_path):
build_dir = os.path.join(output_base_path, 'ctest-registration-test')
config_cmd = ['cmake',
'-B', build_dir,
'-S', sources_dir,
f'-DCATCH2_PATH={catch2_path}',
'-DCMAKE_BUILD_TYPE=Debug']
build_cmd = ['cmake',
'--build', build_dir,
'--config', 'Debug']
try:
subprocess.run(config_cmd,
capture_output = True,
check = True,
text = True)
subprocess.run(build_cmd,
capture_output = True,
check = True,
text = True)
except subprocess.CalledProcessError as err:
print('Error when building the test project')
print(f'cmd: {err.cmd}')
print(f'stderr: {err.stderr}')
print(f'stdout: {err.stdout}')
exit(3)
return build_dir
def get_test_names(build_path):
# For now we assume that Windows builds are done using MSBuild under
# Debug configuration. This means that we need to add "Debug" folder
# to the path when constructing it. On Linux, we don't add anything.
config_path = "Debug" if os.name == 'nt' else ""
full_path = os.path.join(build_path, config_path, 'tests')
cmd = [full_path, '--reporter', 'xml', '--list-tests']
result = subprocess.run(cmd,
capture_output = True,
check = True,
text = True)
import xml.etree.ElementTree as ET
root = ET.fromstring(result.stdout)
return [tc.text for tc in root.findall('TestCase/Name')]
def get_ctest_listing(build_path):
old_path = os.getcwd()
os.chdir(build_path)
cmd = ['ctest', '-C', 'debug', '--show-only=json-v1']
result = subprocess.run(cmd,
capture_output = True,
check = True,
text = True)
os.chdir(old_path)
return result.stdout
def extract_tests_from_ctest(ctest_output):
ctest_response = json.loads(ctest_output)
tests = ctest_response['tests']
test_names = []
for test in tests:
test_command = test['command']
# First part of the command is the binary, second is the filter.
# If there are less, registration has failed. If there are more,
# registration has changed and the script needs updating.
assert len(test_command) == 2
test_names.append(test_command[1])
test_name = test_command[1]
return test_names
def check_DL_PATHS(ctest_output):
ctest_response = json.loads(ctest_output)
tests = ctest_response['tests']
for test in tests:
properties = test['properties']
for property in properties:
if property['name'] == 'ENVIRONMENT_MODIFICATION':
assert len(property['value']) == 2, f"The test provides 2 arguments to DL_PATHS, but instead found {len(property['value'])}"
def escape_catch2_test_name(name):
for char in ('\\', ',', '[', ']'):
name = name.replace(char, f"\\{char}")
return name
if __name__ == '__main__':
if len(sys.argv) != 3:
print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path')
exit(2)
catch2_path = sys.argv[1]
output_base_path = sys.argv[2]
sources_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
build_path = build_project(sources_dir, output_base_path, catch2_path)
catch_test_names = [escape_catch2_test_name(name) for name in get_test_names(build_path)]
ctest_output = get_ctest_listing(build_path)
ctest_test_names = extract_tests_from_ctest(ctest_output)
mismatched = 0
for catch_test in catch_test_names:
if catch_test not in ctest_test_names:
print(f"Catch2 test '{catch_test}' not found in CTest")
mismatched += 1
for ctest_test in ctest_test_names:
if ctest_test not in catch_test_names:
print(f"CTest test '{ctest_test}' not found in Catch2")
mismatched += 1
if mismatched:
print(f"Found {mismatched} mismatched tests catch test names and ctest test commands!")
exit(1)
cmake_version = get_cmake_version()
if cmake_version >= (3, 27):
check_DL_PATHS(ctest_output)
|
