1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
|
#!/usr/bin/env python3
# Copyright Catch2 Authors
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE.txt or copy at
# https://www.boost.org/LICENSE_1_0.txt)
# SPDX-License-Identifier: BSL-1.0
import os
import subprocess
import sys
def build_project(sources_dir, output_base_path, catch2_path):
build_dir = os.path.join(output_base_path, 'ctest-registration-test')
config_cmd = ['cmake',
'-B', build_dir,
'-S', sources_dir,
f'-DCATCH2_PATH={catch2_path}',
'-DCMAKE_BUILD_TYPE=Debug']
build_cmd = ['cmake',
'--build', build_dir,
'--config', 'Debug']
try:
subprocess.run(config_cmd,
capture_output = True,
check = True,
text = True)
subprocess.run(build_cmd,
capture_output = True,
check = True,
text = True)
except subprocess.CalledProcessError as err:
print('Error when building the test project')
print(f'cmd: {err.cmd}')
print(f'stderr: {err.stderr}')
print(f'stdout: {err.stdout}')
exit(3)
return build_dir
def get_test_names(build_path):
# For now we assume that Windows builds are done using MSBuild under
# Debug configuration. This means that we need to add "Debug" folder
# to the path when constructing it. On Linux, we don't add anything.
config_path = "Debug" if os.name == 'nt' else ""
full_path = os.path.join(build_path, config_path, 'tests')
cmd = [full_path, '--reporter', 'xml', '--list-tests']
result = subprocess.run(cmd,
capture_output = True,
check = True,
text = True)
import xml.etree.ElementTree as ET
root = ET.fromstring(result.stdout)
return [tc.text for tc in root.findall('TestCase/Name')]
def list_ctest_tests(build_path):
old_path = os.getcwd()
os.chdir(build_path)
cmd = ['ctest', '-C', 'debug', '--show-only=json-v1']
result = subprocess.run(cmd,
capture_output = True,
check = True,
text = True)
os.chdir(old_path)
import json
ctest_response = json.loads(result.stdout)
tests = ctest_response['tests']
test_names = []
for test in tests:
test_command = test['command']
# First part of the command is the binary, second is the filter.
# If there are less, registration has failed. If there are more,
# registration has changed and the script needs updating.
assert len(test_command) == 2
test_names.append(test_command[1])
test_name = test_command[1]
return test_names
def escape_catch2_test_name(name):
for char in ('\\', ',', '[', ']'):
name = name.replace(char, f"\\{char}")
return name
if __name__ == '__main__':
if len(sys.argv) != 3:
print(f'Usage: {sys.argv[0]} path-to-catch2-cml output-path')
exit(2)
catch2_path = sys.argv[1]
output_base_path = sys.argv[2]
sources_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
build_path = build_project(sources_dir, output_base_path, catch2_path)
catch_test_names = [escape_catch2_test_name(name) for name in get_test_names(build_path)]
ctest_test_names = list_ctest_tests(build_path)
mismatched = 0
for catch_test in catch_test_names:
if catch_test not in ctest_test_names:
print(f"Catch2 test '{catch_test}' not found in CTest")
mismatched += 1
for ctest_test in ctest_test_names:
if ctest_test not in catch_test_names:
print(f"CTest test '{ctest_test}' not found in Catch2")
mismatched += 1
if mismatched:
print(f"Found {mismatched} mismatched tests catch test names and ctest test commands!")
exit(1)
|
