1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
|
import ArgumentParser
import Foundation
import SystemPackage
import WasmKit
@main
struct Spectest: AsyncParsableCommand {
@Argument
var path: String
@Option
var include: String?
@Option
var exclude: String?
@Flag
var verbose = false
@Flag(inversion: .prefixedNo)
var parallel: Bool = true
func run() async throws {
let printVerbose = self.verbose
@Sendable func log(_ message: String, verbose: Bool = false) {
if !verbose || printVerbose {
fputs(message + "\n", stderr)
}
}
let include = self.include.flatMap { $0.split(separator: ",").map(String.init) } ?? []
let exclude = self.exclude.flatMap { $0.split(separator: ",").map(String.init) } ?? []
let testCases: [TestCase]
do {
testCases = try TestCase.load(include: include, exclude: exclude, in: path, log: { log($0) })
} catch {
fatalError("failed to load test: \(error)")
}
let rootPath: String
let filePath = FilePath(path)
if (try? FileDescriptor.open(filePath, FileDescriptor.AccessMode.readOnly, options: .directory)) != nil {
rootPath = path
} else {
rootPath = URL(fileURLWithPath: path).deletingLastPathComponent().path
}
// https://github.com/WebAssembly/spec/tree/8a352708cffeb71206ca49a0f743bdc57269fb1a/interpreter#spectest-host-module
let hostModulePath = FilePath(rootPath).appending("host.wasm")
let hostModule = try parseWasm(filePath: hostModulePath)
@Sendable func runTestCase(testCase: TestCase) throws -> [Result] {
var testCaseResults = [Result]()
try testCase.run(spectestModule: hostModule) { testCase, command, result in
switch result {
case let .failed(reason):
log("\(testCase.content.sourceFilename):\(command.line): \(result.banner) \(reason)")
case let .skipped(reason):
log("\(testCase.content.sourceFilename):\(command.line): \(result.banner) \(reason)", verbose: true)
case .passed:
log("\(testCase.content.sourceFilename):\(command.line): \(result.banner)", verbose: true)
default:
log("\(testCase.content.sourceFilename):\(command.line): \(result.banner)")
}
testCaseResults.append(result)
}
return testCaseResults
}
let results: [Result]
if parallel {
results = try await withThrowingTaskGroup(of: [Result].self) { group in
for testCase in testCases {
group.addTask {
try await Task { try runTestCase(testCase: testCase) }.value
}
}
var results = [Result]()
for try await testCaseResults in group {
results.append(contentsOf: testCaseResults)
}
return results
}
} else {
results = try testCases.flatMap { try runTestCase(testCase: $0) }
}
let passingCount = results.filter { if case .passed = $0 { return true } else { return false } }.count
let skippedCount = results.filter { if case .skipped = $0 { return true } else { return false } }.count
let failedCount = results.filter { if case .failed = $0 { return true } else { return false } }.count
print(
"""
\(passingCount)/\(results.count) (\(
percentage(passingCount, results.count)
) passing, \(
percentage(skippedCount, results.count)
) skipped, \(
percentage(failedCount, results.count)
) failed)
"""
)
// Exit with non-zero status when there is any failure
if failedCount > 0 {
throw ArgumentParser.ExitCode(1)
}
}
private func percentage(_ numerator: Int, _ denominator: Int) -> String {
"\(Int(Double(numerator) / Double(denominator) * 100))%"
}
}
|
