File: IncrementalDependencyAndInputSetup.swift

package info (click to toggle)
swiftlang 6.0.3-2
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid, trixie
  • size: 2,519,992 kB
  • sloc: cpp: 9,107,863; ansic: 2,040,022; asm: 1,135,751; python: 296,500; objc: 82,456; f90: 60,502; lisp: 34,951; pascal: 19,946; sh: 18,133; perl: 7,482; ml: 4,937; javascript: 4,117; makefile: 3,840; awk: 3,535; xml: 914; fortran: 619; cs: 573; ruby: 573
file content (395 lines) | stat: -rw-r--r-- 17,440 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
//===----- IncrementalDependencyAndInputSetup.swift - Incremental --------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

import SwiftOptions
import class Dispatch.DispatchQueue

import class TSCBasic.DiagnosticsEngine
import protocol TSCBasic.FileSystem

// Initial incremental state computation
extension IncrementalCompilationState {
  static func computeIncrementalStateForPlanning(driver: inout Driver)
    throws -> IncrementalCompilationState.InitialStateForPlanning?
  {
    guard driver.shouldAttemptIncrementalCompilation else { return nil }

    let options = computeIncrementalOptions(driver: &driver)

    guard let outputFileMap = driver.outputFileMap else {
      driver.diagnosticEngine.emit(.warning_incremental_requires_output_file_map)
      return nil
    }

    let reporter: IncrementalCompilationState.Reporter?
    if options.contains(.showIncremental) {
      reporter = IncrementalCompilationState.Reporter(
        diagnosticEngine: driver.diagnosticEngine,
        outputFileMap: outputFileMap)
    } else {
      reporter = nil
    }

    guard let buildRecordInfo = driver.buildRecordInfo else {
      reporter?.reportDisablingIncrementalBuild("no build record path")
      return nil
    }

    // FIXME: This should work without an output file map. We should have
    // another way to specify a build record and where to put intermediates.
    guard
      let initialState =
        try IncrementalCompilationState
        .IncrementalDependencyAndInputSetup(
          options, outputFileMap,
          buildRecordInfo,
          reporter, driver.inputFiles,
          driver.fileSystem,
          driver.diagnosticEngine
        ).computeInitialStateForPlanning(driver: &driver)
    else {
      Self.removeDependencyGraphFile(driver)
      if options.contains(.explicitModuleBuild) {
        Self.removeInterModuleDependencyGraphFile(driver)
      }
      return nil
    }

    return initialState
  }

  // Extract options relevant to incremental builds
  static func computeIncrementalOptions(driver: inout Driver) -> IncrementalCompilationState.Options {
    var options: IncrementalCompilationState.Options = []
    if driver.parsedOptions.contains(.driverAlwaysRebuildDependents) {
      options.formUnion(.alwaysRebuildDependents)
    }
    if driver.parsedOptions.contains(.driverShowIncremental) || driver.showJobLifecycle {
      options.formUnion(.showIncremental)
    }
    let emitOpt = Option.driverEmitFineGrainedDependencyDotFileAfterEveryImport
    if driver.parsedOptions.contains(emitOpt) {
      options.formUnion(.emitDependencyDotFileAfterEveryImport)
    }
    let veriOpt = Option.driverVerifyFineGrainedDependencyGraphAfterEveryImport
    if driver.parsedOptions.contains(veriOpt) {
      options.formUnion(.verifyDependencyGraphAfterEveryImport)
    }
    if driver.parsedOptions.hasFlag(positive: .enableIncrementalImports,
                                  negative: .disableIncrementalImports,
                                  default: true) {
      options.formUnion(.enableCrossModuleIncrementalBuild)
      options.formUnion(.readPriorsFromModuleDependencyGraph)
    }
    if driver.parsedOptions.contains(.driverExplicitModuleBuild) {
      options.formUnion(.explicitModuleBuild)
    }
    return options
  }
}

/// Validate if a prior inter-module dependency graph is still valid
extension IncrementalCompilationState.IncrementalDependencyAndInputSetup {
  static func readAndValidatePriorInterModuleDependencyGraph(
    driver: inout Driver,
    buildRecordInfo: BuildRecordInfo,
    reporter: IncrementalCompilationState.Reporter?
  ) throws -> InterModuleDependencyGraph? {
    // Attempt to read a serialized inter-module dependency graph from a prior build
    guard let priorInterModuleDependencyGraph =
        buildRecordInfo.readPriorInterModuleDependencyGraph(reporter: reporter),
          let priorImports = priorInterModuleDependencyGraph.mainModule.directDependencies?.map({ $0.moduleName }) else {
      reporter?.reportExplicitBuildMustReScan("Could not read inter-module dependency graph at \(buildRecordInfo.interModuleDependencyGraphPath)")
      return nil
    }

    // Verify that import sets match
    let currentImports = try driver.performImportPrescan().imports
    guard Set(priorImports) == Set(currentImports) else {
      reporter?.reportExplicitBuildMustReScan("Target import set has changed.")
      return nil
    }

    // Verify that each dependnecy is up-to-date with respect to its inputs
    guard try priorInterModuleDependencyGraph.computeInvalidatedModuleDependencies(fileSystem: buildRecordInfo.fileSystem,
                                                                                   forRebuild: false,
                                                                                   reporter: reporter).isEmpty else {
      reporter?.reportExplicitBuildMustReScan("Not all dependencies are up-to-date.")
      return nil
    }

    reporter?.report("Confirmed prior inter-module dependency graph is up-to-date at: \(buildRecordInfo.interModuleDependencyGraphPath)")
    return priorInterModuleDependencyGraph
  }
}

/// Builds the `InitialState`
/// Also bundles up an bunch of configuration info
extension IncrementalCompilationState {

  /// A collection of immutable state that is handy to access.
  public struct IncrementalDependencyAndInputSetup: IncrementalCompilationSynchronizer {
    @_spi(Testing) public let outputFileMap: OutputFileMap
    @_spi(Testing) public let buildRecordInfo: BuildRecordInfo
    @_spi(Testing) public let reporter: IncrementalCompilationState.Reporter?
    @_spi(Testing) public let options: IncrementalCompilationState.Options
    @_spi(Testing) public let inputFiles: [TypedVirtualPath]
    @_spi(Testing) public let fileSystem: FileSystem

    /// The state managing incremental compilation gets mutated every time a compilation job completes.
    /// This queue ensures that the access and mutation of that state is thread-safe.
    @_spi(Testing) public let incrementalCompilationQueue: DispatchQueue

    @_spi(Testing) public let diagnosticEngine: DiagnosticsEngine

    /// Options, someday
    @_spi(Testing) public let dependencyDotFilesIncludeExternals: Bool = true
    @_spi(Testing) public let dependencyDotFilesIncludeAPINotes: Bool = false

    @_spi(Testing) public var readPriorsFromModuleDependencyGraph: Bool {
      options.contains(.readPriorsFromModuleDependencyGraph)
    }
    @_spi(Testing) public var explicitModuleBuild: Bool {
      options.contains(.explicitModuleBuild)
    }
    @_spi(Testing) public var alwaysRebuildDependents: Bool {
      options.contains(.alwaysRebuildDependents)
    }
    @_spi(Testing) public var isCrossModuleIncrementalBuildEnabled: Bool {
      options.contains(.enableCrossModuleIncrementalBuild)
    }
    @_spi(Testing) public var verifyDependencyGraphAfterEveryImport: Bool {
      options.contains(.verifyDependencyGraphAfterEveryImport)
    }
    @_spi(Testing) public var emitDependencyDotFileAfterEveryImport: Bool {
      options.contains(.emitDependencyDotFileAfterEveryImport)
    }

    @_spi(Testing) public init(
      _ options: Options,
      _ outputFileMap: OutputFileMap,
      _ buildRecordInfo: BuildRecordInfo,
      _ reporter: IncrementalCompilationState.Reporter?,
      _ inputFiles: [TypedVirtualPath],
      _ fileSystem: FileSystem,
      _ diagnosticEngine: DiagnosticsEngine
    ) {
      self.outputFileMap = outputFileMap
      self.buildRecordInfo = buildRecordInfo
      self.reporter = reporter
      self.options = options
      self.inputFiles = inputFiles
      self.fileSystem = fileSystem
      assert(outputFileMap.onlySourceFilesHaveSwiftDeps())
      self.diagnosticEngine = diagnosticEngine
      self.incrementalCompilationQueue = DispatchQueue(
        label: "com.apple.swift-driver.incremental-compilation-state",
        qos: .userInteractive,
        attributes: .concurrent)
    }

    func computeInitialStateForPlanning(driver: inout Driver) throws -> InitialStateForPlanning? {
      guard let priors = computeGraphAndInputsInvalidatedByExternals() else {
        return nil
      }

      // If a valid build record could not be produced, do not bother here
      let priorInterModuleDependencyGraph: InterModuleDependencyGraph?
      if options.contains(.explicitModuleBuild) {
        if priors.graph.buildRecord.inputInfos.isEmpty {
          reporter?.report("Incremental compilation did not attempt to read inter-module dependency graph.")
          priorInterModuleDependencyGraph = nil
        } else {
          priorInterModuleDependencyGraph = try Self.readAndValidatePriorInterModuleDependencyGraph(
            driver: &driver, buildRecordInfo: buildRecordInfo, reporter: reporter)
        }
      } else {
        priorInterModuleDependencyGraph = nil
      }

      return InitialStateForPlanning(
        graph: priors.graph, buildRecordInfo: buildRecordInfo,
        upToDatePriorInterModuleDependencyGraph: priorInterModuleDependencyGraph,
        inputsInvalidatedByExternals: priors.fileSet,
        incrementalOptions: options)
    }

    /// Is this source file part of this build?
    ///
    /// - Parameter sourceFile: the Swift source-code file in question
    /// - Returns: true iff this file was in the command-line invocation of the driver
    func isPartOfBuild(_ sourceFile: SwiftSourceFile) -> Bool {
      return self.inputFiles.contains(sourceFile.typedFile)
    }
  }
}


// MARK: - building/reading the ModuleDependencyGraph & scheduling externals for 1st wave
extension IncrementalCompilationState.IncrementalDependencyAndInputSetup {
  struct PriorState {
    var graph: ModuleDependencyGraph
    var fileSet: TransitivelyInvalidatedSwiftSourceFileSet
  }

  /// Builds or reads the graph
  /// Returns nil if some input (i.e. .swift file) has no corresponding swiftdeps file.
  /// Does not cope with disappeared inputs -- would be left in graph
  /// For inputs with swiftDeps in OFM, but no readable file, puts input in graph map, but no nodes in graph:
  ///   caller must ensure scheduling of those
  private func computeGraphAndInputsInvalidatedByExternals() -> PriorState? {
    return blockingConcurrentAccessOrMutation {
      if readPriorsFromModuleDependencyGraph {
        return readPriorGraphAndCollectInputsInvalidatedByChangedOrAddedExternals()
      }
      // Every external is added, but don't want to compile an unchanged input that has an import
      // so just changed, not changedOrAdded.
      return buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals()
    }
  }

  private func readPriorGraphAndCollectInputsInvalidatedByChangedOrAddedExternals() -> PriorState? {
    let dependencyGraphPath = buildRecordInfo.dependencyGraphPath
    let graphIfPresent: ModuleDependencyGraph?
    do {
      graphIfPresent = try ModuleDependencyGraph.read(from: dependencyGraphPath, info: self)
    }
    catch let ModuleDependencyGraph.ReadError.mismatchedSerializedGraphVersion(expected, read) {
      reporter?.report("Will not do cross-module incremental builds, wrong version of priors; expected \(expected) but read \(read) at '\(dependencyGraphPath)'")
      graphIfPresent = nil
    }
    catch {
      diagnosticEngine.emit(.warning("Could not read priors, will not do cross-module incremental builds: \(error.localizedDescription), at \(dependencyGraphPath)"),
                            location: nil)
      graphIfPresent = nil
    }
    guard let graph = graphIfPresent, self.validateBuildRecord(graph.buildRecord) != nil else {
      // Do not fall back to `buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals`
      // because it would be unsound to read a `swiftmodule` file with only a partial set of integrated `swiftdeps`.
      // A fingerprint change in such a `swiftmodule` would not be able to propagate and invalidate a use
      // in a as-yet-unread swiftdeps file.
      //
      // Instead, just compile everything. It's OK to be unsound then because every file will be compiled anyway.
      return buildEmptyGraphAndCompileEverything()
    }

    let sourceFiles = SourceFiles(
      inputFiles: inputFiles,
      buildRecord: graph.buildRecord)

    if !sourceFiles.disappeared.isEmpty {
      // Would have to cleanse nodes of disappeared inputs from graph
      // and would have to schedule files depending on defs from disappeared nodes
      if let reporter = reporter {
        reporter.report(
          "Incremental compilation has been disabled, "
          + "because the following inputs were used in the previous compilation but not in this one: "
          + sourceFiles.disappeared.map { $0.typedFile.file.basename }.joined(separator: ", "))
      }
      return buildEmptyGraphAndCompileEverything()
    }

    graph.dotFileWriter?.write(graph)

    // Any externals not already in graph must be additions which should trigger
    // recompilation. Thus, `ChangedOrAdded`.
    let nodesDirectlyInvalidatedByExternals =
      graph.collectNodesInvalidatedByChangedOrAddedExternals()
    // Wait till the last minute to do the transitive closure as an optimization.
    guard let inputsInvalidatedByExternals = graph.collectInputsInBuildUsingInvalidated(
      nodes: nodesDirectlyInvalidatedByExternals)
    else {
      return nil
    }
    return PriorState(graph: graph, fileSet: inputsInvalidatedByExternals)
  }

  /// Builds a graph
  /// Returns nil if some input (i.e. .swift file) has no corresponding swiftdeps file.
  /// Does not cope with disappeared inputs
  /// For inputs with swiftDeps in OFM, but no readable file, puts input in graph map, but no nodes in graph:
  ///   caller must ensure scheduling of those
  /// For externalDependencies, puts then in graph.fingerprintedExternalDependencies, but otherwise
  /// does nothing special.
  private func buildInitialGraphFromSwiftDepsAndCollectInputsInvalidatedByChangedExternals() -> PriorState? {
    guard
      let contents = try? fileSystem.readFileContents(self.buildRecordInfo.buildRecordPath).cString
    else {
      reporter?.report("Incremental compilation could not read build record at ", self.buildRecordInfo.buildRecordPath)
      reporter?.reportDisablingIncrementalBuild("could not read build record")
      return nil
    }

    func failedToReadOutOfDateMap(_ reason: String) {
      let why = "malformed build record file\(reason.isEmpty ? "" : (" " + reason))"
      reporter?.report(
        "Incremental compilation has been disabled due to \(why)", self.buildRecordInfo.buildRecordPath)
      reporter?.reportDisablingIncrementalBuild(why)
    }

    do {
      guard let buildRecord = try self.validateBuildRecord(BuildRecord(contents: contents)) else {
        return nil
      }

      let graph = ModuleDependencyGraph.createForBuildingFromSwiftDeps(buildRecord, self)
      var inputsInvalidatedByChangedExternals = TransitivelyInvalidatedSwiftSourceFileSet()
      for input in self.inputFiles {
        guard let invalidatedInputs =
                graph.collectInputsRequiringCompilationFromExternalsFoundByCompiling(input: SwiftSourceFile(input.fileHandle))
        else {
          return nil
        }
        inputsInvalidatedByChangedExternals.formUnion(invalidatedInputs)
      }
      reporter?.report("Created dependency graph from swiftdeps files")
      return PriorState(graph: graph, fileSet: inputsInvalidatedByChangedExternals)
    } catch let error as BuildRecord.Error {
      failedToReadOutOfDateMap(error.reason)
      return nil
    } catch {
      return nil
    }
  }

  private func buildEmptyGraphAndCompileEverything() -> PriorState {
    let buildRecord = BuildRecord(
      argsHash: self.buildRecordInfo.currentArgsHash,
      swiftVersion: self.buildRecordInfo.actualSwiftVersion,
      buildStartTime: .distantPast,
      buildEndTime: .distantFuture,
      inputInfos: [:])
    let graph = ModuleDependencyGraph.createForBuildingAfterEachCompilation(buildRecord, self)
    return PriorState(graph: graph, fileSet: TransitivelyInvalidatedSwiftSourceFileSet())
  }

  private func validateBuildRecord(
    _ outOfDateBuildRecord: BuildRecord
  ) -> BuildRecord? {
    let actualSwiftVersion = self.buildRecordInfo.actualSwiftVersion
    guard actualSwiftVersion == outOfDateBuildRecord.swiftVersion else {
      let why = "compiler version mismatch. Compiling with: \(actualSwiftVersion). Previously compiled with: \(outOfDateBuildRecord.swiftVersion)"
      // mimic legacy
      reporter?.reportIncrementalCompilationHasBeenDisabled("due to a " + why)
      reporter?.reportDisablingIncrementalBuild(why)
      return nil
    }
    guard outOfDateBuildRecord.argsHash == self.buildRecordInfo.currentArgsHash else {
      let why = "different arguments were passed to the compiler"
      // mimic legacy
      reporter?.reportIncrementalCompilationHasBeenDisabled("because " + why)
      reporter?.reportDisablingIncrementalBuild(why)
      return nil
    }
    return outOfDateBuildRecord
  }
}