1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
|
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2016 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See http://swift.org/LICENSE.txt for license information
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//
// PerformanceMeter.swift
// Measures the performance of a block of code and reports the results.
//
/// Describes a type that is capable of measuring some aspect of code performance
/// over time.
internal protocol PerformanceMetric {
/// Called once per iteration immediately before the tested code is executed.
/// The metric should do whatever work is required to begin a new measurement.
func startMeasuring()
/// Called once per iteration immediately after the tested code is executed.
/// The metric should do whatever work is required to finalize measurement.
func stopMeasuring()
/// Called once, after all measurements have been taken, to provide feedback
/// about the collected measurements.
/// - Returns: Measurement results to present to the user.
func calculateResults() -> String
/// Called once, after all measurements have been taken, to determine whether
/// the measurements should be treated as a test failure or not.
/// - Returns: A diagnostic message if the results indicate failure, else nil.
func failureMessage() -> String?
}
/// Protocol used by `PerformanceMeter` to report measurement results
internal protocol PerformanceMeterDelegate {
/// Reports a string representation of the gathered performance metrics
/// - Parameter results: The raw measured values, and some derived data such
/// as average, and standard deviation
/// - Parameter file: The source file name where the measurement was invoked
/// - Parameter line: The source line number where the measurement was invoked
func recordMeasurements(results: String, file: StaticString, line: Int)
/// Reports a test failure from the analysis of performance measurements.
/// This can currently be caused by an unexpectedly large standard deviation
/// calculated over the data.
/// - Parameter description: An explanation of the failure
/// - Parameter file: The source file name where the measurement was invoked
/// - Parameter line: The source line number where the measurement was invoked
func recordFailure(description: String, file: StaticString, line: Int)
/// Reports a misuse of the `PerformanceMeter` API, such as calling `
/// startMeasuring` multiple times.
/// - Parameter description: An explanation of the misuse
/// - Parameter file: The source file name where the misuse occurred
/// - Parameter line: The source line number where the misuse occurred
func recordAPIViolation(description: String, file: StaticString, line: Int)
}
/// - Bug: This class is intended to be `internal` but is public to work around
/// a toolchain bug on Linux. See `XCTestCase._performanceMeter` for more info.
public final class PerformanceMeter {
enum Error: Swift.Error, CustomStringConvertible {
case noMetrics
case unknownMetric(metricName: String)
case startMeasuringAlreadyCalled
case stopMeasuringAlreadyCalled
case startMeasuringNotCalled
case stopBeforeStarting
var description: String {
switch self {
case .noMetrics: return "At least one metric must be provided to measure."
case .unknownMetric(let name): return "Unknown metric: \(name)"
case .startMeasuringAlreadyCalled: return "Already called startMeasuring() once this iteration."
case .stopMeasuringAlreadyCalled: return "Already called stopMeasuring() once this iteration."
case .startMeasuringNotCalled: return "startMeasuring() must be called during the block."
case .stopBeforeStarting: return "Cannot stop measuring before starting measuring."
}
}
}
internal var didFinishMeasuring: Bool {
return state == .measurementFinished || state == .measurementAborted
}
private enum State {
case iterationUnstarted
case iterationStarted
case iterationFinished
case measurementFinished
case measurementAborted
}
private var state: State = .iterationUnstarted
private let metrics: [PerformanceMetric]
private let delegate: PerformanceMeterDelegate
private let invocationFile: StaticString
private let invocationLine: Int
private init(metrics: [PerformanceMetric], delegate: PerformanceMeterDelegate, file: StaticString, line: Int) {
self.metrics = metrics
self.delegate = delegate
self.invocationFile = file
self.invocationLine = line
}
static func measureMetrics(_ metricNames: [String], delegate: PerformanceMeterDelegate, file: StaticString = #file, line: Int = #line, for block: (PerformanceMeter) -> Void) {
do {
let metrics = try self.metrics(forNames: metricNames)
let meter = PerformanceMeter(metrics: metrics, delegate: delegate, file: file, line: line)
meter.measure(block)
} catch let e {
delegate.recordAPIViolation(description: String(describing: e), file: file, line: line)
}
}
func startMeasuring(file: StaticString = #file, line: Int = #line) {
guard state == .iterationUnstarted else {
return recordAPIViolation(.startMeasuringAlreadyCalled, file: file, line: line)
}
state = .iterationStarted
metrics.forEach { $0.startMeasuring() }
}
func stopMeasuring(file: StaticString = #file, line: Int = #line) {
guard state != .iterationUnstarted else {
return recordAPIViolation(.stopBeforeStarting, file: file, line: line)
}
guard state != .iterationFinished else {
return recordAPIViolation(.stopMeasuringAlreadyCalled, file: file, line: line)
}
state = .iterationFinished
metrics.forEach { $0.stopMeasuring() }
}
func abortMeasuring() {
state = .measurementAborted
}
private static func metrics(forNames names: [String]) throws -> [PerformanceMetric] {
guard !names.isEmpty else { throw Error.noMetrics }
let metricsMapping = [WallClockTimeMetric.name : WallClockTimeMetric.self]
return try names.map({
guard let metricType = metricsMapping[$0] else { throw Error.unknownMetric(metricName: $0) }
return metricType.init()
})
}
private var numberOfIterations: Int {
return 10
}
private func measure(_ block: (PerformanceMeter) -> Void) {
for _ in (0..<numberOfIterations) {
state = .iterationUnstarted
block(self)
stopMeasuringIfNeeded()
if state == .measurementAborted { return }
if state == .iterationUnstarted {
recordAPIViolation(.startMeasuringNotCalled, file: invocationFile, line: invocationLine)
return
}
}
state = .measurementFinished
recordResults()
recordFailures()
}
private func stopMeasuringIfNeeded() {
if state == .iterationStarted {
stopMeasuring(file: invocationFile, line: invocationLine)
}
}
private func recordResults() {
for metric in metrics {
delegate.recordMeasurements(results: metric.calculateResults(), file: invocationFile, line: invocationLine)
}
}
private func recordFailures() {
metrics.compactMap({ $0.failureMessage() }).forEach { message in
delegate.recordFailure(description: message, file: invocationFile, line: invocationLine)
}
}
private func recordAPIViolation(_ error: Error, file: StaticString, line: Int) {
state = .measurementAborted
delegate.recordAPIViolation(description: String(describing: error), file: file, line: line)
}
}
|
