1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
|
//===----------------------------------------------------------------------===//
//
// This source file is part of the SwiftNIO open source project
//
// Copyright (c) 2020 Apple Inc. and the SwiftNIO project authors
// Licensed under Apache License v2.0
//
// See LICENSE.txt for license information
// See CONTRIBUTORS.txt for the list of SwiftNIO project authors
//
// SPDX-License-Identifier: Apache-2.0
//
//===----------------------------------------------------------------------===//
import NIO
import class Foundation.Process
import struct Foundation.URL
import class Foundation.FileHandle
struct CrashTest {
let crashRegex: String
let runTest: () -> Void
init(regex: String, _ runTest: @escaping () -> Void) {
self.crashRegex = regex
self.runTest = runTest
}
}
// Compatible with Swift on all macOS versions as well as Linux
extension Process {
var binaryPath: String? {
get {
if #available(macOS 10.13, /* Linux */ *) {
return self.executableURL?.path
} else {
return self.launchPath
}
}
set {
if #available(macOS 10.13, /* Linux */ *) {
self.executableURL = newValue.map { URL(fileURLWithPath: $0) }
} else {
self.launchPath = newValue
}
}
}
func runProcess() throws {
if #available(macOS 10.13, *) {
try self.run()
} else {
self.launch()
}
}
}
func main() throws {
enum RunResult {
case signal(Int)
case exit(Int)
}
enum InterpretedRunResult {
case crashedAsExpected
case regexDidNotMatch(regex: String, output: String)
case unexpectedRunResult(RunResult)
case outputError(String)
}
struct CrashTestNotFound: Error {
let suite: String
let test: String
}
func allTestsForSuite(_ testSuite: String) -> [(String, CrashTest)] {
return crashTestSuites[testSuite].map { testSuiteObject in
Mirror(reflecting: testSuiteObject)
.children
.filter { $0.label?.starts(with: "test") ?? false }
.compactMap { crashTestDescriptor in
crashTestDescriptor.label.flatMap { label in
(crashTestDescriptor.value as? CrashTest).map { crashTest in
return (label, crashTest)
}
}
}
} ?? []
}
func findCrashTest(_ testName: String, suite: String) -> CrashTest? {
return allTestsForSuite(suite)
.first(where: { $0.0 == testName })?
.1
}
func interpretOutput(_ result: Result<ProgramOutput, Error>,
regex: String,
runResult: RunResult) throws -> InterpretedRunResult {
struct NoOutputFound: Error {}
guard case .signal(Int(SIGILL)) = runResult else {
return .unexpectedRunResult(runResult)
}
let output = try result.get()
if output.range(of: regex, options: .regularExpression) != nil {
return .crashedAsExpected
} else {
return .regexDidNotMatch(regex: regex, output: output)
}
}
func usage() {
print("\(CommandLine.arguments.first ?? "NIOCrashTester") COMMAND [OPTIONS]")
print()
print("COMMAND is:")
print(" run-all to run all crash tests")
print(" run SUITE TEST-NAME to run the crash test SUITE.TEST-NAME")
print("")
print("For debugging purposes, you can also directly run the crash test binary that will crash using")
print(" \(CommandLine.arguments.first ?? "NIOCrashTester") _exec SUITE TEST-NAME")
}
let group = MultiThreadedEventLoopGroup(numberOfThreads: 1)
defer {
try! group.syncShutdownGracefully()
}
signal(SIGPIPE, SIG_IGN)
func runCrashTest(_ name: String, suite: String, binary: String) throws -> InterpretedRunResult {
guard let crashTest = findCrashTest(name, suite: suite) else {
throw CrashTestNotFound(suite: suite, test: name)
}
let grepper = OutputGrepper.make(group: group)
let devNull = try FileHandle(forUpdating: URL(fileURLWithPath: "/dev/null"))
defer {
devNull.closeFile()
}
let processOutputPipe = FileHandle(fileDescriptor: try! grepper.processOutputPipe.takeDescriptorOwnership())
let process = Process()
process.binaryPath = binary
process.standardInput = devNull
process.standardOutput = devNull
process.standardError = processOutputPipe
process.arguments = ["_exec", suite, name]
try process.runProcess()
process.waitUntilExit()
processOutputPipe.closeFile()
let result: Result<ProgramOutput, Error> = Result {
try grepper.result.wait()
}
return try interpretOutput(result,
regex: crashTest.crashRegex,
runResult: process.terminationReason == .exit ?
.exit(Int(process.terminationStatus)) :
.signal(Int(process.terminationStatus)))
}
var failedTests = 0
func runAndEval(_ test: String, suite: String) throws {
print("running crash test \(suite).\(test)", terminator: " ")
switch try runCrashTest(test, suite: suite, binary: CommandLine.arguments.first!) {
case .regexDidNotMatch(regex: let regex, output: let output):
print("FAILED: regex did not match output", "regex: \(regex)", "output: \(output)",
separator: "\n", terminator: "")
failedTests += 1
case .unexpectedRunResult(let runResult):
print("FAILED: unexpected run result: \(runResult)")
failedTests += 1
case .outputError(let description):
print("FAILED: \(description)")
failedTests += 1
case .crashedAsExpected:
print("OK")
}
}
switch CommandLine.arguments.dropFirst().first {
case .some("run-all"):
for testSuite in crashTestSuites {
for test in allTestsForSuite(testSuite.key) {
try runAndEval(test.0, suite: testSuite.key)
}
}
case .some("run"):
if let suite = CommandLine.arguments.dropFirst(2).first {
for test in CommandLine.arguments.dropFirst(3) {
try runAndEval(test, suite: suite)
}
} else {
usage()
exit(EXIT_FAILURE)
}
case .some("_exec"):
if let testSuiteName = CommandLine.arguments.dropFirst(2).first,
let testName = CommandLine.arguments.dropFirst(3).first,
let crashTest = findCrashTest(testName, suite: testSuiteName) {
crashTest.runTest()
} else {
fatalError("can't find/create test for \(Array(CommandLine.arguments.dropFirst(2)))")
}
default:
usage()
exit(EXIT_FAILURE)
}
exit(CInt(failedTests == 0 ? EXIT_SUCCESS : EXIT_FAILURE))
}
try main()
|
