1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
|
// RUN: %{swiftc} %s -o %T/Performance
// RUN: %T/Performance > %t || true
// RUN: %{xctest_checker} %t %s
#if os(macOS)
import SwiftXCTest
#else
import XCTest
#endif
// CHECK: Test Suite 'All tests' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: Test Suite '.*\.xctest' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: Test Suite 'PerformanceTestCase' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
class PerformanceTestCase: XCTestCase {
// CHECK: Test Case 'PerformanceTestCase.test_measureBlockIteratesTenTimes' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+4]]: Test Case 'PerformanceTestCase.test_measureBlockIteratesTenTimes' measured \[Time, seconds\] .*
// CHECK: Test Case 'PerformanceTestCase.test_measureBlockIteratesTenTimes' passed \(\d+\.\d+ seconds\)
func test_measureBlockIteratesTenTimes() {
var iterationCount = 0
measure(block: {
iterationCount += 1
})
XCTAssertEqual(iterationCount, 10)
}
// CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithAutomaticStartAndStop' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+4]]: Test Case 'PerformanceTestCase.test_measuresMetricsWithAutomaticStartAndStop' measured \[Time, seconds\] .*
// CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithAutomaticStartAndStop' passed \(\d+\.\d+ seconds\)
func test_measuresMetricsWithAutomaticStartAndStop() {
var iterationCount = 0
measureMetrics([.wallClockTime], automaticallyStartMeasuring: true, for: {
iterationCount += 1
})
XCTAssertEqual(iterationCount, 10)
}
// CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithManualStartAndStop' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+3]]: Test Case 'PerformanceTestCase.test_measuresMetricsWithManualStartAndStop' measured \[Time, seconds\] .*
// CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithManualStartAndStop' passed \(\d+\.\d+ seconds\)
func test_measuresMetricsWithManualStartAndStop() {
measureMetrics([.wallClockTime], automaticallyStartMeasuring: false) {
self.startMeasuring()
self.stopMeasuring()
}
}
// CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithoutExplicitStop' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+3]]: Test Case 'PerformanceTestCase.test_measuresMetricsWithoutExplicitStop' measured \[Time, seconds\] .*
// CHECK: Test Case 'PerformanceTestCase.test_measuresMetricsWithoutExplicitStop' passed \(\d+\.\d+ seconds\)
func test_measuresMetricsWithoutExplicitStop() {
measureMetrics([.wallClockTime], automaticallyStartMeasuring: false) {
self.startMeasuring()
}
}
// CHECK: Test Case 'PerformanceTestCase.test_hasWallClockAsDefaultPerformanceMetric' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: Test Case 'PerformanceTestCase.test_hasWallClockAsDefaultPerformanceMetric' passed \(\d+\.\d+ seconds\)
func test_hasWallClockAsDefaultPerformanceMetric() {
XCTAssertEqual(PerformanceTestCase.defaultPerformanceMetrics, [.wallClockTime])
}
// CHECK: Test Case 'PerformanceTestCase.test_printsValuesAfterMeasuring' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+3]]: Test Case 'PerformanceTestCase.test_printsValuesAfterMeasuring' measured \[Time, seconds\] average: \d+.\d{3}, relative standard deviation: \d+.\d{3}%, values: \[\d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}, \d+.\d{6}\], performanceMetricID:org.swift.XCTPerformanceMetric_WallClockTime, maxPercentRelativeStandardDeviation: \d+.\d{3}%, maxStandardDeviation: \d.\d{3}
// CHECK: Test Case 'PerformanceTestCase.test_printsValuesAfterMeasuring' passed \(\d+\.\d+ seconds\)
func test_printsValuesAfterMeasuring() {
measure {
Thread.sleep(forTimeInterval: 1)
}
}
// CHECK: Test Case 'PerformanceTestCase.test_abortsMeasurementsAfterTestFailure' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
func test_abortsMeasurementsAfterTestFailure() {
var iterationCount = 0
measure {
iterationCount += 1
// CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+1]]: error: PerformanceTestCase.test_abortsMeasurementsAfterTestFailure : XCTAssertLessThan failed: \("3"\) is not less than \("3"\) -
XCTAssertLessThan(iterationCount, 3)
}
XCTAssertEqual(iterationCount, 3)
}
// CHECK: Test Case 'PerformanceTestCase.test_abortsMeasurementsAfterTestFailure' failed \(\d+\.\d+ seconds\)
// CHECK: Test Case 'PerformanceTestCase.test_measuresWallClockTimeInBlock' started at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
func test_measuresWallClockTimeInBlock() {
var hasWaited = false
// CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+2]]: Test Case 'PerformanceTestCase.test_measuresWallClockTimeInBlock' measured \[Time, seconds\] average: \d+.\d{3}, relative standard deviation: \d+.\d{3}%, values: \[1.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}, 0.\d{6}\], performanceMetricID:org.swift.XCTPerformanceMetric_WallClockTime, maxPercentRelativeStandardDeviation: \d+.\d{3}%, maxStandardDeviation: \d.\d{3}
// CHECK: .*[/\\]Performance[/\\]main.swift:[[@LINE+1]]: error: PerformanceTestCase.test_measuresWallClockTimeInBlock : failed: The relative standard deviation of the measurements is \d+.\d{3}% which is higher than the max allowed of \d+.\d{3}%.
measure {
if !hasWaited {
Thread.sleep(forTimeInterval: 1)
hasWaited = true
}
}
}
// CHECK: Test Case 'PerformanceTestCase.test_measuresWallClockTimeInBlock' failed \(\d+\.\d+ seconds\)
static var allTests = {
return [
("test_measureBlockIteratesTenTimes", test_measureBlockIteratesTenTimes),
("test_measuresMetricsWithAutomaticStartAndStop", test_measuresMetricsWithAutomaticStartAndStop),
("test_measuresMetricsWithManualStartAndStop", test_measuresMetricsWithManualStartAndStop),
("test_measuresMetricsWithoutExplicitStop", test_measuresMetricsWithoutExplicitStop),
("test_hasWallClockAsDefaultPerformanceMetric", test_hasWallClockAsDefaultPerformanceMetric),
("test_printsValuesAfterMeasuring", test_printsValuesAfterMeasuring),
("test_abortsMeasurementsAfterTestFailure", test_abortsMeasurementsAfterTestFailure),
("test_measuresWallClockTimeInBlock", test_measuresWallClockTimeInBlock),
]
}()
}
// CHECK: Test Suite 'PerformanceTestCase' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: \t Executed \d+ tests, with 2 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
XCTMain([testCase(PerformanceTestCase.allTests)])
// CHECK: Test Suite '.*\.xctest' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: \t Executed \d+ tests, with \d failures? \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
// CHECK: Test Suite 'All tests' failed at \d+-\d+-\d+ \d+:\d+:\d+\.\d+
// CHECK: \t Executed \d+ tests, with \d failures? \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds
|
