1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
|
//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift open source project
//
// Copyright (c) 2025 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See http://swift.org/LICENSE.txt for license information
// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
package import SWBCore
import SWBProtocol
import SWBUtil
import Foundation
#if canImport(os)
import os
#endif
/// Manages uploading compilation caching outputs in the background, when a remote cache is enabled.
/// The network tasks are managed using Swift concurrency, without blocking execution lanes
/// or being constrained by them.
///
/// FIXME: Ideally the uploading tasks would not block build completion, for example when doing
/// `xcodebuild test` uploading tasks could continue in the background while the tests start running.
/// This would require designing a way for the uploading tasks to be reported and entered in the result bundle
/// without being tied to a particular build.
///
/// FIXME: Implement cancellation functionality.
package final class CompilationCachingUploader {
private let group: SWBDispatchGroup = .init()
private let lock: Lock = .init()
private var uploadedKeys: Set<String> = []
private var pendingUploads: Int = 0
deinit {
precondition(pendingUploads == 0)
}
private func startedUpload() {
group.enter()
lock.withLock { pendingUploads += 1 }
}
private func finishedUpload() {
lock.withLock { pendingUploads -= 1 }
group.leave()
}
/// Schedule a cached compilation for uploading. The call will return immediately and the
/// upload action will be performed in the background.
/// If the `cacheKey` was already scheduled for uploading before the operation is a no-op.
/// This is thread-safe.
package func upload(
clangCompilation: ClangCASCachedCompilation,
cacheKey: String,
enableDiagnosticRemarks: Bool,
enableStrictCASErrors: Bool,
activityReporter: any ActivityReporter
) {
let inserted = lock.withLock { uploadedKeys.insert(cacheKey).inserted }
guard inserted else {
return // already uploaded
}
startedUpload()
let signatureCtx = InsecureHashContext()
signatureCtx.add(string: "ClangCachingUpload")
signatureCtx.add(string: cacheKey)
let signature = signatureCtx.signature
let activityID = activityReporter.beginActivity(
ruleInfo: "ClangCachingUpload \(cacheKey)",
executionDescription: "Clang caching upload key \(cacheKey)",
signature: signature,
target: nil,
parentActivity: nil
)
if enableDiagnosticRemarks {
for output in clangCompilation.getOutputs() {
activityReporter.emit(
diagnostic: Diagnostic(behavior: .note, location: .unknown, data: DiagnosticData("uploaded CAS output \(output.name): \(output.casID)")),
for: activityID,
signature: signature
)
}
}
// Avoiding the swift concurrency variant because it may lead to starvation when `waitForCompletion()`
// blocks on such tasks. Before using a swift concurrency task here make sure there's no deadlock
// when setting `LIBDISPATCH_COOPERATIVE_POOL_STRICT`.
clangCompilation.makeGlobalAsync { error in
let status: BuildOperationTaskEnded.Status
if let error {
activityReporter.emit(
diagnostic: Diagnostic(behavior: enableStrictCASErrors ? .error : .warning, location: .unknown, data: DiagnosticData(error.localizedDescription)),
for: activityID,
signature: signature
)
status = enableStrictCASErrors ? .failed : .succeeded
} else {
status = .succeeded
}
activityReporter.endActivity(
id: activityID,
signature: signature,
status: status
)
self.finishedUpload()
}
}
/// Schedule upload for swift compilation output
package func upload(
swiftCompilation: SwiftCachedCompilation,
cacheKey: String,
enableDiagnosticRemarks: Bool,
enableStrictCASErrors: Bool,
activityReporter: any ActivityReporter
) {
let inserted = lock.withLock { uploadedKeys.insert(cacheKey).inserted }
guard inserted else {
return // already uploaded
}
startedUpload()
let signatureCtx = InsecureHashContext()
signatureCtx.add(string: "SwiftCachingUpload")
signatureCtx.add(string: cacheKey)
let signature = signatureCtx.signature
let activityID = activityReporter.beginActivity(
ruleInfo: "SwiftCachingUpload \(cacheKey)",
executionDescription: "Swift caching upload key \(cacheKey)",
signature: signature,
target: nil,
parentActivity: nil
)
do {
if enableDiagnosticRemarks {
for output in try swiftCompilation.getOutputs() {
activityReporter.emit(
diagnostic: Diagnostic(behavior: .note,
location: .unknown,
data: DiagnosticData("uploaded CAS output \(output.kindName): \(output.casID)")),
for: activityID,
signature: signature
)
}
}
} catch {
// failed to print remarks. warn about the returned error
activityReporter.emit(
diagnostic: Diagnostic(behavior: enableStrictCASErrors ? .error : .warning, location: .unknown, data: DiagnosticData(error.localizedDescription)),
for: activityID,
signature: signature
)
}
// Avoiding the swift concurrency variant because it may lead to starvation when `waitForCompletion()`
// blocks on such tasks. Before using a swift concurrency task here make sure there's no deadlock
// when setting `LIBDISPATCH_COOPERATIVE_POOL_STRICT`.
swiftCompilation.makeGlobal { error in
let status: BuildOperationTaskEnded.Status
if let error {
activityReporter.emit(
diagnostic: Diagnostic(behavior: enableStrictCASErrors ? .error : .warning, location: .unknown, data: DiagnosticData(error.localizedDescription)),
for: activityID,
signature: signature
)
status = enableStrictCASErrors ? .failed : .succeeded
} else {
status = .succeeded
}
activityReporter.endActivity(
id: activityID,
signature: signature,
status: status
)
self.finishedUpload()
}
}
package func waitForCompletion() async {
await group.wait(queue: .global())
}
}
|
