1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
|
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2016 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See http://swift.org/LICENSE.txt for license information
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//
// XCTestCase+Performance.swift
// Methods on XCTestCase for testing the performance of code blocks.
//
public struct XCTPerformanceMetric : RawRepresentable, Equatable, Hashable {
public let rawValue: String
public init(_ rawValue: String) {
self.rawValue = rawValue
}
public init(rawValue: String) {
self.rawValue = rawValue
}
}
public extension XCTPerformanceMetric {
/// Records wall clock time in seconds between `startMeasuring`/`stopMeasuring`.
static let wallClockTime = XCTPerformanceMetric(rawValue: WallClockTimeMetric.name)
}
/// The following methods are called from within a test method to carry out
/// performance testing on blocks of code.
public extension XCTestCase {
/// The names of the performance metrics to measure when invoking `measure(block:)`.
/// Returns `XCTPerformanceMetric_WallClockTime` by default. Subclasses can
/// override this to change the behavior of `measure(block:)`
class var defaultPerformanceMetrics: [XCTPerformanceMetric] {
return [.wallClockTime]
}
/// Call from a test method to measure resources (`defaultPerformanceMetrics`)
/// used by the block in the current process.
///
/// func testPerformanceOfMyFunction() {
/// measure {
/// // Do that thing you want to measure.
/// MyFunction();
/// }
/// }
///
/// - Parameter block: A block whose performance to measure.
/// - Bug: The `block` param should have no external label, but there seems
/// to be a swiftc bug that causes issues when such a parameter comes
/// after a defaulted arg. See https://bugs.swift.org/browse/SR-1483 This
/// API incompatibility with Apple XCTest can be worked around in practice
/// by using trailing closure syntax when calling this method.
/// - Note: Whereas Apple XCTest determines the file and line number of
/// measurements by using symbolication, this implementation opts to take
/// `file` and `line` as parameters instead. As a result, the interface to
/// these methods are not exactly identical between these environments. To
/// ensure compatibility of tests between swift-corelibs-xctest and Apple
/// XCTest, it is not recommended to pass explicit values for `file` and `line`.
func measure(file: StaticString = #file, line: Int = #line, block: () -> Void) {
measureMetrics(type(of: self).defaultPerformanceMetrics,
automaticallyStartMeasuring: true,
file: file,
line: line,
for: block)
}
/// Call from a test method to measure resources (XCTPerformanceMetrics) used
/// by the block in the current process. Each metric will be measured across
/// calls to the block. The number of times the block will be called is undefined
/// and may change in the future. For one example of why, as long as the requested
/// performance metrics do not interfere with each other the API will measure
/// all metrics across the same calls to the block. If the performance metrics
/// may interfere the API will measure them separately.
///
/// func testMyFunction2_WallClockTime() {
/// measureMetrics(type(of: self).defaultPerformanceMetrics, automaticallyStartMeasuring: false) {
///
/// // Do setup work that needs to be done for every iteration but
/// // you don't want to measure before the call to `startMeasuring()`
/// SetupSomething();
/// self.startMeasuring()
///
/// // Do that thing you want to measure.
/// MyFunction()
/// self.stopMeasuring()
///
/// // Do teardown work that needs to be done for every iteration
/// // but you don't want to measure after the call to `stopMeasuring()`
/// TeardownSomething()
/// }
/// }
///
/// Caveats:
/// * If `true` was passed for `automaticallyStartMeasuring` and `startMeasuring()`
/// is called anyway, the test will fail.
/// * If `false` was passed for `automaticallyStartMeasuring` then `startMeasuring()`
/// must be called once and only once before the end of the block or the test will fail.
/// * If `stopMeasuring()` is called multiple times during the block the test will fail.
///
/// - Parameter metrics: An array of Strings (XCTPerformanceMetrics) to measure.
/// Providing an unrecognized string is a test failure.
/// - Parameter automaticallyStartMeasuring: If `false`, `XCTestCase` will
/// not take any measurements until -startMeasuring is called.
/// - Parameter block: A block whose performance to measure.
/// - Note: Whereas Apple XCTest determines the file and line number of
/// measurements by using symbolication, this implementation opts to take
/// `file` and `line` as parameters instead. As a result, the interface to
/// these methods are not exactly identical between these environments. To
/// ensure compatibility of tests between swift-corelibs-xctest and Apple
/// XCTest, it is not recommended to pass explicit values for `file` and `line`.
func measureMetrics(_ metrics: [XCTPerformanceMetric], automaticallyStartMeasuring: Bool, file: StaticString = #file, line: Int = #line, for block: () -> Void) {
guard _performanceMeter == nil else {
return recordAPIViolation(description: "Can only record one set of metrics per test method.", file: file, line: line)
}
PerformanceMeter.measureMetrics(metrics.map({ $0.rawValue }), delegate: self, file: file, line: line) { meter in
self._performanceMeter = meter
if automaticallyStartMeasuring {
meter.startMeasuring(file: file, line: line)
}
block()
}
}
/// Call this from within a measure block to set the beginning of the critical
/// section. Measurement of metrics will start at this point.
/// - Note: Whereas Apple XCTest determines the file and line number of
/// measurements by using symbolication, this implementation opts to take
/// `file` and `line` as parameters instead. As a result, the interface to
/// these methods are not exactly identical between these environments. To
/// ensure compatibility of tests between swift-corelibs-xctest and Apple
/// XCTest, it is not recommended to pass explicit values for `file` and `line`.
func startMeasuring(file: StaticString = #file, line: Int = #line) {
guard let performanceMeter = _performanceMeter, !performanceMeter.didFinishMeasuring else {
return recordAPIViolation(description: "Cannot start measuring. startMeasuring() is only supported from a block passed to measureMetrics(...).", file: file, line: line)
}
performanceMeter.startMeasuring(file: file, line: line)
}
/// Call this from within a measure block to set the ending of the critical
/// section. Measurement of metrics will stop at this point.
/// - Note: Whereas Apple XCTest determines the file and line number of
/// measurements by using symbolication, this implementation opts to take
/// `file` and `line` as parameters instead. As a result, the interface to
/// these methods are not exactly identical between these environments. To
/// ensure compatibility of tests between swift-corelibs-xctest and Apple
/// XCTest, it is not recommended to pass explicit values for `file` and `line`.
func stopMeasuring(file: StaticString = #file, line: Int = #line) {
guard let performanceMeter = _performanceMeter, !performanceMeter.didFinishMeasuring else {
return recordAPIViolation(description: "Cannot stop measuring. stopMeasuring() is only supported from a block passed to measureMetrics(...).", file: file, line: line)
}
performanceMeter.stopMeasuring(file: file, line: line)
}
}
extension XCTestCase: PerformanceMeterDelegate {
internal func recordAPIViolation(description: String, file: StaticString, line: Int) {
recordFailure(withDescription: "API violation - \(description)",
inFile: String(describing: file),
atLine: line,
expected: false)
}
internal func recordMeasurements(results: String, file: StaticString, line: Int) {
XCTestObservationCenter.shared.testCase(self, didMeasurePerformanceResults: results, file: file, line: line)
}
internal func recordFailure(description: String, file: StaticString, line: Int) {
recordFailure(withDescription: "failed: " + description, inFile: String(describing: file), atLine: line, expected: true)
}
}
|
