1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
|
// Copyright Catch2 Authors
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.txt or copy at
// https://www.boost.org/LICENSE_1_0.txt)
// SPDX-License-Identifier: BSL-1.0
#include <catch2/catch_tag_alias_autoregistrar.hpp>
#include <catch2/reporters/catch_reporter_event_listener.hpp>
#include <catch2/internal/catch_enforce.hpp>
#include <catch2/catch_test_case_info.hpp>
#include <catch2/reporters/catch_reporter_registrars.hpp>
// Some example tag aliases
CATCH_REGISTER_TAG_ALIAS("[@nhf]", "[failing]~[.]")
CATCH_REGISTER_TAG_ALIAS("[@tricky]", "[tricky]~[.]")
#ifdef __clang__
# pragma clang diagnostic ignored "-Wpadded"
# pragma clang diagnostic ignored "-Wweak-vtables"
# pragma clang diagnostic ignored "-Wc++98-compat"
#endif
/**
* Event listener that internally counts and validates received events.
*
* Currently only performs validation by counting received events, rather
* than performing full matching. This means that it won't fail if the *Ended
* events are provided in wrong order, as long as they come in the right amount
* and with the right nesting.
*/
class ValidatingTestListener : public Catch::EventListenerBase {
struct EventCounter {
int starting = 0;
int ended = 0;
bool hasActiveEvent() const {
return starting > ended;
}
bool hasSingleActiveEvent() const {
return starting - 1 == ended;
}
bool allEventsEnded() const {
return starting == ended;
}
};
public:
static std::string getDescription() {
return "Validates ordering of Catch2's listener events";
}
ValidatingTestListener(Catch::IConfig const* config) :
EventListenerBase(config) {
m_preferences.shouldReportAllAssertions = true;
}
void testRunStarting( Catch::TestRunInfo const& ) override {
CATCH_ENFORCE( m_testRunCounter.starting == 0,
"Test run can only start once" );
++m_testRunCounter.starting;
}
void testCaseStarting(Catch::TestCaseInfo const&) override {
CATCH_ENFORCE( m_testRunCounter.hasActiveEvent(),
"Test case can only be started if the test run has already started" );
CATCH_ENFORCE( m_testCaseCounter.allEventsEnded(),
"Test case cannot start if there is an unfinished one" );
++m_testCaseCounter.starting;
// Reset the part tracking for partial test case events
m_lastSeenPartNumber = uint64_t(-1);
}
void testCasePartialStarting(Catch::TestCaseInfo const&,
uint64_t partNumber) override {
CATCH_ENFORCE( m_testCaseCounter.hasSingleActiveEvent(),
"Test case can only be partially started if the test case has fully started already" );
CATCH_ENFORCE( m_lastSeenPartNumber + 1 == partNumber,
"Partial test case started out of order" );
++m_testCasePartialCounter.starting;
m_lastSeenPartNumber = partNumber;
}
void sectionStarting(Catch::SectionInfo const&) override {
CATCH_ENFORCE( m_testCaseCounter.hasSingleActiveEvent(),
"Section can only start in a test case" );
CATCH_ENFORCE( m_testCasePartialCounter.hasSingleActiveEvent(),
"Section can only start in a test case" );
++m_sectionCounter.starting;
}
void assertionStarting(Catch::AssertionInfo const&) override {
CATCH_ENFORCE( m_testCaseCounter.hasSingleActiveEvent(),
"Assertion can only start if test case is started" );
++m_assertionCounter.starting;
}
void assertionEnded(Catch::AssertionStats const&) override {
// todo:
// * Check that assertions are balanced
// * Check that assertions has started
++m_assertionCounter.ended;
}
void sectionEnded(Catch::SectionStats const&) override {
CATCH_ENFORCE( m_sectionCounter.hasActiveEvent(),
"Section ended without corresponding start" );
// TODO: Check that all assertions ended
++m_sectionCounter.ended;
}
void testCasePartialEnded(Catch::TestCaseStats const&,
uint64_t partNumber) override {
CATCH_ENFORCE( m_lastSeenPartNumber == partNumber,
"Partial test case ended out of order" );
CATCH_ENFORCE( m_testCasePartialCounter.hasSingleActiveEvent(),
"Partial test case ended without corresponding start" );
CATCH_ENFORCE( m_sectionCounter.allEventsEnded(),
"Partial test case ended with unbalanced sections" );
// TODO: Check that all assertions ended
++m_testCasePartialCounter.ended;
}
void testCaseEnded(Catch::TestCaseStats const&) override {
CATCH_ENFORCE( m_testCaseCounter.hasSingleActiveEvent(),
"Test case end is not matched with test case start" );
CATCH_ENFORCE( m_testCasePartialCounter.allEventsEnded(),
"A partial test case has not ended" );
CATCH_ENFORCE( m_sectionCounter.allEventsEnded(),
"Test case ended with unbalanced sections" );
// TODO: Check that all assertions ended
++m_testCaseCounter.ended;
}
void testRunEnded( Catch::TestRunStats const& ) override {
CATCH_ENFORCE( m_testRunCounter.hasSingleActiveEvent(),
"Test run end is not matched with test run start" );
CATCH_ENFORCE( m_testRunCounter.ended == 0,
"Test run can only end once" );
++m_testRunCounter.ended;
}
~ValidatingTestListener() override;
private:
EventCounter m_testRunCounter;
EventCounter m_testCaseCounter;
EventCounter m_testCasePartialCounter;
uint64_t m_lastSeenPartNumber = 0;
EventCounter m_sectionCounter;
EventCounter m_assertionCounter;
};
ValidatingTestListener::~ValidatingTestListener() {
// Throwing from noexcept destructor terminates, but we don't mind
// because this is test-only check and we don't need to try and recover
// from assumption violation here.
CATCH_ENFORCE( m_testRunCounter.ended < 2,
"Test run should be started at most once" );
CATCH_ENFORCE( m_testRunCounter.allEventsEnded(),
"The test run has not finished" );
CATCH_ENFORCE( m_testCaseCounter.allEventsEnded(),
"A test case did not finish" );
// TODO: other counters being balanced?
}
CATCH_REGISTER_LISTENER( ValidatingTestListener )
|
