File: Event.HumanReadableOutputRecorder.swift

package info (click to toggle)
swiftlang 6.2.3-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 2,856,264 kB
  • sloc: cpp: 9,995,718; ansic: 2,234,019; asm: 1,092,167; python: 313,940; objc: 82,726; f90: 80,126; lisp: 38,373; pascal: 25,580; sh: 20,378; ml: 5,058; perl: 4,751; makefile: 4,725; awk: 3,535; javascript: 3,018; xml: 918; fortran: 664; cs: 573; ruby: 396
file content (622 lines) | stat: -rw-r--r-- 21,959 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2023 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
//

extension Event {
  /// A type which handles ``Event`` instances and outputs representations of
  /// them as human-readable messages.
  ///
  /// This type can be used compositionally to produce output in other
  /// human-readable formats such as rich text or HTML.
  ///
  /// The format of the output is not meant to be machine-readable and is
  /// subject to change. For machine-readable output, use ``JUnitXMLRecorder``.
  @_spi(ForToolsIntegrationOnly)
  public struct HumanReadableOutputRecorder: Sendable/*, ~Copyable*/ {
    /// A type describing a human-readable message produced by an instance of
    /// ``Event/HumanReadableOutputRecorder``.
    public struct Message: Sendable {
      /// The symbol associated with this message, if any.
      var symbol: Symbol?

      /// The human-readable message.
      var stringValue: String

      /// A concise version of ``stringValue``, if available.
      ///
      /// Not all messages include a concise string.
      var conciseStringValue: String?
    }

    /// A type that contains mutable context for
    /// ``Event/ConsoleOutputRecorder``.
    fileprivate struct Context {
      /// The instant at which the run started.
      var runStartInstant: Test.Clock.Instant?

      /// The instant at which the current iteration started.
      var iterationStartInstant: Test.Clock.Instant?

      /// The number of tests started or skipped during the run.
      ///
      /// This value does not include test suites.
      var testCount = 0

      /// The number of test suites started or skipped during the run.
      var suiteCount = 0

      /// An enumeration describing the various keys which can be used in a test
      /// data graph for an output recorder.
      enum TestDataKey: Hashable {
        /// A string key, typically containing one key from the key path
        /// representation of a ``Test/ID`` instance.
        case string(String)

        /// A test case ID.
        case testCaseID(Test.Case.ID)
      }

      /// A type describing data tracked on a per-test basis.
      struct TestData {
        /// The instant at which the test started.
        var startInstant: Test.Clock.Instant

        /// The number of issues recorded for the test, grouped by their
        /// level of severity.
        var issueCount: [Issue.Severity: Int] = [:]

        /// The number of known issues recorded for the test.
        var knownIssueCount = 0
      }

      /// Data tracked on a per-test basis.
      var testData = Graph<TestDataKey, TestData?>()
    }

    /// This event recorder's mutable context about events it has received,
    /// which may be used to inform how subsequent events are written.
    private var _context = Locked(rawValue: Context())

    /// Initialize a new human-readable event recorder.
    ///
    /// Output from the testing library is converted to "messages" using the
    /// ``Event/HumanReadableOutputRecorder/record(_:in:verbosity:)`` function.
    /// The format of those messages is, as the type's name suggests, not meant
    /// to be machine-readable and is subject to change.
    public init() {}
  }
}

// MARK: -

extension Event.HumanReadableOutputRecorder {
  /// Get a string representing an array of comments, formatted for output.
  ///
  /// - Parameters:
  ///   - comments: The comments that should be formatted.
  ///
  /// - Returns: An array of formatted messages representing `comments`, or an
  ///   empty array if there are none.
  private func _formattedComments(_ comments: [Comment]) -> [Message] {
    comments.map(_formattedComment)
  }

  /// Get a string representing a single comment, formatted for output.
  ///
  /// - Parameters:
  ///   - comment: The comment that should be formatted.
  ///
  /// - Returns: A formatted message representing `comment`.
  private func _formattedComment(_ comment: Comment) -> Message {
    Message(symbol: .details, stringValue: comment.rawValue)
  }

  /// Get a string representing the comments attached to a test, formatted for
  /// output.
  ///
  /// - Parameters:
  ///   - test: The test whose comments should be formatted.
  ///
  /// - Returns: A formatted string representing the comments attached to `test`,
  ///   or `nil` if there are none.
  private func _formattedComments(for test: Test) -> [Message] {
    _formattedComments(test.comments(from: Comment.self))
  }

  /// Get the total number of issues recorded in a graph of test data
  /// structures.
  ///
  /// - Parameters:
  ///   - graph: The graph to walk while counting issues.
  ///
  /// - Returns: A tuple containing the number of issues recorded in `graph`.
  private func _issueCounts(
    in graph: Graph<Event.HumanReadableOutputRecorder.Context.TestDataKey, Event.HumanReadableOutputRecorder.Context.TestData?>?
  ) -> (errorIssueCount: Int, warningIssueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) {
    guard let graph else {
      return (0, 0, 0, 0, "")
    }
    let errorIssueCount = graph.compactMap { $0.value?.issueCount[.error] }.reduce(into: 0, +=)
    let warningIssueCount = graph.compactMap { $0.value?.issueCount[.warning] }.reduce(into: 0, +=)
    let knownIssueCount = graph.compactMap(\.value?.knownIssueCount).reduce(into: 0, +=)
    let totalIssueCount = errorIssueCount + warningIssueCount + knownIssueCount

    // Construct a string describing the issue counts.
    let description = switch (errorIssueCount > 0, warningIssueCount > 0, knownIssueCount > 0) {
    case (true, true, true):
      " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue")))"
    case (true, false, true):
      " with \(totalIssueCount.counting("issue")) (including \(knownIssueCount.counting("known issue")))"
    case (false, true, true):
      " with \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue"))"
    case (false, false, true):
      " with \(knownIssueCount.counting("known issue"))"
    case (true, true, false):
      " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")))"
    case (true, false, false):
      " with \(totalIssueCount.counting("issue"))"
    case(false, true, false):
      " with \(warningIssueCount.counting("warning"))"
    case(false, false, false):
      ""
    }

    return (errorIssueCount, warningIssueCount, knownIssueCount, totalIssueCount,  description)
  }
}

/// Generate a title for the specified test (either "Test" or "Suite"),
/// capitalized and suitable for use as the leading word of a human-readable
/// message string.
///
/// - Parameters:
///   - test: The test to generate a description for, if any.
///
/// - Returns: A human-readable title for the specified test. Defaults to "Test"
///   if `test` is `nil`.
private func _capitalizedTitle(for test: Test?) -> String {
  test?.isSuite == true ? "Suite" : "Test"
}

extension Test.Case {
  /// The arguments of this test case, formatted for presentation, prefixed by
  /// their corresponding parameter label when available.
  ///
  /// - Parameters:
  ///   - includeTypeNames: Whether the qualified type name of each argument's
  ///     runtime type should be included. Defaults to `false`.
  ///
  /// - Returns: A string containing the arguments of this test case formatted
  ///   for presentation, or an empty string if this test cases is
  ///   non-parameterized.
  fileprivate func labeledArguments(includingQualifiedTypeNames includeTypeNames: Bool = false) -> String {
    guard let arguments else { return "" }

    return arguments.lazy
      .map { argument in
        let valueDescription = String(describingForTest: argument.value)

        let label = argument.parameter.secondName ?? argument.parameter.firstName
        let labeledArgument = if label == "_" {
          valueDescription
        } else {
          "\(label) → \(valueDescription)"
        }

        if includeTypeNames {
          let typeInfo = TypeInfo(describingTypeOf: argument.value)
          return "\(labeledArgument) (\(typeInfo.fullyQualifiedName))"
        } else {
          return labeledArgument
        }
      }
      .joined(separator: ", ")
  }
}

// MARK: -

extension Event.HumanReadableOutputRecorder {
  /// Record the specified event by generating zero or more messages that
  /// describe it.
  ///
  /// - Parameters:
  ///   - event: The event to record.
  ///   - eventContext: The context associated with the event.
  ///   - verbosity: How verbose output should be. When the value of this
  ///     argument is greater than `0`, additional output is provided. When the
  ///     value of this argument is less than `0`, some output is suppressed.
  ///     If the value of this argument is `nil`, the value set for the current
  ///     configuration is used instead. The exact effects of this argument are
  ///     implementation-defined and subject to change.
  ///
  /// - Returns: An array of zero or more messages that can be displayed to the
  ///   user.
  @discardableResult public func record(
    _ event: borrowing Event,
    in eventContext: borrowing Event.Context,
    verbosity: Int? = nil
  ) -> [Message] {
    let verbosity: Int = if let verbosity {
      verbosity
    } else if let verbosity = eventContext.configuration?.verbosity {
      verbosity
    } else {
      0
    }
    let test = eventContext.test
    let keyPath = eventContext.keyPath
    let testName = if let test {
      if let displayName = test.displayName {
        if verbosity > 0 {
          "\"\(displayName)\" (aka '\(test.name)')"
        } else {
          "\"\(displayName)\""
        }
      } else {
        test.name
      }
    } else {
      "«unknown»"
    }
    let instant = event.instant
    let iterationCount = eventContext.configuration?.repetitionPolicy.maximumIterationCount

    // First, make any updates to the context/state associated with this
    // recorder.
    let context = _context.withLock { context in
      switch event.kind {
      case .runStarted:
        context.runStartInstant = instant

      case .iterationStarted:
        if let iterationCount, iterationCount > 1 {
          context.iterationStartInstant = instant
        }

      case .testStarted:
        let test = test!
        context.testData[keyPath] = .init(startInstant: instant)
        if test.isSuite {
          context.suiteCount += 1
        } else {
          context.testCount += 1
        }

      case .testSkipped:
        let test = test!
        if test.isSuite {
          context.suiteCount += 1
        } else {
          context.testCount += 1
        }

      case let .issueRecorded(issue):
        var testData = context.testData[keyPath] ?? .init(startInstant: instant)
        if issue.isKnown {
          testData.knownIssueCount += 1
        } else {
          let issueCount = testData.issueCount[issue.severity] ?? 0
          testData.issueCount[issue.severity] = issueCount + 1
        }
        context.testData[keyPath] = testData
      
      case .testCaseStarted:
        context.testData[keyPath] = .init(startInstant: instant)

      default:
        // These events do not manipulate the context structure.
        break
      }

      return context
    }

    // If in quiet mode, only produce messages for a subset of events we'd
    // otherwise log.
    if verbosity == .min {
      // Quietest mode: no messages at all.
      return []
    } else if verbosity < 0 {
      switch event.kind {
      case .runStarted, .issueRecorded, .runEnded:
        break
      default:
        return []
      }
    }

    // Finally, produce any messages for the event.
    switch event.kind {
    case .testDiscovered:
      // Suppress events of this kind from output as they are not generally
      // interesting in human-readable output.
      break

    case .runStarted:
      var comments = [Comment]()
      if verbosity > 0 {
        comments.append("Swift Version: \(swiftStandardLibraryVersion)")
      }
      comments.append("Testing Library Version: \(testingLibraryVersion)")
      if let targetTriple {
        comments.append("Target Platform: \(targetTriple)")
      }
      if verbosity > 0 {
#if targetEnvironment(simulator)
        comments.append("OS Version (Simulator): \(simulatorVersion)")
        comments.append("OS Version (Host): \(operatingSystemVersion)")
#else
        comments.append("OS Version: \(operatingSystemVersion)")
#endif
      }
      return CollectionOfOne(
        Message(
          symbol: .default,
          stringValue: "Test run started."
        )
      ) + _formattedComments(comments)

    case let .iterationStarted(index):
      if let iterationCount, iterationCount > 1 {
        return [
          Message(
            symbol: .default,
            stringValue: "Iteration \(index + 1) started."
          )
        ]
      }

    case .planStepStarted, .planStepEnded:
      // Suppress events of these kinds from output as they are not generally
      // interesting in human-readable output.
      break

    case .testStarted:
      let test = test!
      return [
        Message(
          symbol: .default,
          stringValue: "\(_capitalizedTitle(for: test)) \(testName) started."
        )
      ]

    case .testEnded:
      let test = test!
      let testDataGraph = context.testData.subgraph(at: keyPath)
      let testData = testDataGraph?.value ?? .init(startInstant: instant)
      let issues = _issueCounts(in: testDataGraph)
      let duration = testData.startInstant.descriptionOfDuration(to: instant)
      let testCasesCount = if test.isParameterized, let testDataGraph {
        " with \(testDataGraph.children.count.counting("test case"))"
      } else {
        ""
      }
      return if issues.errorIssueCount > 0 {
        CollectionOfOne(
          Message(
            symbol: .fail,
            stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) failed after \(duration)\(issues.description)."
          )
        ) + _formattedComments(for: test)
      } else {
        [
          Message(
            symbol: .pass(knownIssueCount: issues.knownIssueCount),
            stringValue: "\(_capitalizedTitle(for: test)) \(testName)\(testCasesCount) passed after \(duration)\(issues.description)."
          )
        ]
      }

    case let .testSkipped(skipInfo):
      let test = test!
      return if let comment = skipInfo.comment {
        [
          Message(symbol: .skip, stringValue: "\(_capitalizedTitle(for: test)) \(testName) skipped: \"\(comment.rawValue)\"")
        ]
      } else {
        [
          Message(symbol: .skip, stringValue: "\(_capitalizedTitle(for: test)) \(testName) skipped.")
        ]
      }

    case .expectationChecked:
      // Suppress events of this kind from output as they are not generally
      // interesting in human-readable output.
      break

    case let .issueRecorded(issue):
      let parameterCount = if let parameters = test?.parameters {
        parameters.count
      } else {
        0
      }
      let labeledArguments = if let testCase = eventContext.testCase {
        testCase.labeledArguments()
      } else {
        ""
      }
      let symbol: Event.Symbol
      let subject: String
      if issue.isKnown {
        symbol = .pass(knownIssueCount: 1)
        subject = "a known issue"
      } else {
        switch issue.severity {
        case .warning:
          symbol = .passWithWarnings
          subject = "a warning"
        case .error:
          symbol = .fail
          subject = "an issue"
        }
      }

      var additionalMessages = [Message]()
      if case let .expectationFailed(expectation) = issue.kind, let differenceDescription = expectation.differenceDescription {
        additionalMessages.append(Message(symbol: .difference, stringValue: differenceDescription))
      }
      additionalMessages += _formattedComments(issue.comments)
      if let knownIssueComment = issue.knownIssueContext?.comment {
        additionalMessages.append(_formattedComment(knownIssueComment))
      }

      if verbosity > 0, case let .expectationFailed(expectation) = issue.kind {
        let expression = expectation.evaluatedExpression
        func addMessage(about expression: __Expression) {
          let description = expression.expandedDebugDescription()
          additionalMessages.append(Message(symbol: .details, stringValue: description))
        }
        let subexpressions = expression.subexpressions
        if subexpressions.isEmpty {
          addMessage(about: expression)
        } else {
          for subexpression in subexpressions {
            addMessage(about: subexpression)
          }
        }
      }

      let atSourceLocation = issue.sourceLocation.map { " at \($0)" } ?? ""
      let primaryMessage: Message = if parameterCount == 0 {
        Message(
          symbol: symbol,
          stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(subject)\(atSourceLocation): \(issue.kind)",
          conciseStringValue: String(describing: issue.kind)
        )
      } else {
        Message(
          symbol: symbol,
          stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(subject) with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)",
          conciseStringValue: String(describing: issue.kind)
        )
      }
      return CollectionOfOne(primaryMessage) + additionalMessages

    case let .valueAttached(attachment):
      var result = [
        Message(
          symbol: .attachment,
          stringValue: "Attached '\(attachment.preferredName)' to \(testName)."
        )
      ]
      if let path = attachment.fileSystemPath {
        result.append(
          Message(
            symbol: .details,
            stringValue: "Written to '\(path)'."
          )
        )
      }
      return result

    case .testCaseStarted:
      guard let testCase = eventContext.testCase, testCase.isParameterized, let arguments = testCase.arguments else {
        break
      }

      return [
        Message(
          symbol: .default,
          stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) started."
        )
      ]

    case .testCaseEnded:
      guard verbosity > 0, let testCase = eventContext.testCase, testCase.isParameterized, let arguments = testCase.arguments else {
        break
      }

      let testDataGraph = context.testData.subgraph(at: keyPath)
      let testData = testDataGraph?.value ?? .init(startInstant: instant)
      let issues = _issueCounts(in: testDataGraph)
      let duration = testData.startInstant.descriptionOfDuration(to: instant)

      let message = if issues.errorIssueCount > 0 {
        Message(
          symbol: .fail,
          stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) failed after \(duration)\(issues.description)."
        )
      } else {
        Message(
          symbol: .pass(knownIssueCount: issues.knownIssueCount),
          stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) passed after \(duration)\(issues.description)."
        )
      }
      return [message]

    case let .iterationEnded(index):
      guard let iterationStartInstant = context.iterationStartInstant else {
        break
      }
      let duration = iterationStartInstant.descriptionOfDuration(to: instant)

      return [
        Message(
          symbol: .default,
          stringValue: "Iteration \(index + 1) ended after \(duration)."
        )
      ]

    case .runEnded:
      let testCount = context.testCount
      let suiteCount = context.suiteCount
      let issues = _issueCounts(in: context.testData)
      let runStartInstant = context.runStartInstant ?? instant
      let duration = runStartInstant.descriptionOfDuration(to: instant)

      return if issues.errorIssueCount > 0 {
        [
          Message(
            symbol: .fail,
            stringValue: "Test run with \(testCount.counting("test")) in \(suiteCount.counting("suite")) failed after \(duration)\(issues.description)."
          )
        ]
      } else {
        [
          Message(
            symbol: .pass(knownIssueCount: issues.knownIssueCount),
            stringValue: "Test run with \(testCount.counting("test")) in \(suiteCount.counting("suite")) passed after \(duration)\(issues.description)."
          )
        ]
      }
    }

    return []
  }
}

extension Test.ID {
  /// The key path in a test data graph representing this test ID.
  fileprivate var keyPath: some Collection<Event.HumanReadableOutputRecorder.Context.TestDataKey> {
    keyPathRepresentation.map { .string($0) }
  }
}

extension Event.Context {
  /// The key path in a test data graph representing this event this context is
  /// associated with, including its test and/or test case IDs.
  fileprivate var keyPath: some Collection<Event.HumanReadableOutputRecorder.Context.TestDataKey> {
    var keyPath = [Event.HumanReadableOutputRecorder.Context.TestDataKey]()

    if let test {
      keyPath.append(contentsOf: test.id.keyPath)

      if let testCase {
        keyPath.append(.testCaseID(testCase.id))
      }
    }

    return keyPath
  }
}

// MARK: - Codable

extension Event.HumanReadableOutputRecorder.Message: Codable {}