File: TokenSpec.swift

package info (click to toggle)
swiftlang 6.0.3-2
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid, trixie
  • size: 2,519,992 kB
  • sloc: cpp: 9,107,863; ansic: 2,040,022; asm: 1,135,751; python: 296,500; objc: 82,456; f90: 60,502; lisp: 34,951; pascal: 19,946; sh: 18,133; perl: 7,482; ml: 4,937; javascript: 4,117; makefile: 3,840; awk: 3,535; xml: 914; fortran: 619; cs: 573; ruby: 573
file content (223 lines) | stat: -rw-r--r-- 7,937 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

#if swift(>=6)
@_spi(RawSyntax) public import SwiftSyntax
#else
@_spi(RawSyntax) import SwiftSyntax
#endif

/// Pre-computes the keyword a lexeme might represent. This makes matching
/// a lexeme that has been converted into `PrepareForKeyword` match cheaper to
/// match against multiple ``TokenSpec`` that assume a keyword.
struct PrepareForKeywordMatch {
  /// The kind of the lexeme.
  fileprivate let rawTokenKind: RawTokenKind

  /// If the lexeme has the same text as a keyword, that keyword, otherwise `nil`.
  fileprivate let keyword: Keyword?

  /// Whether to lexeme occurred at the start of a line.
  fileprivate let isAtStartOfLine: Bool

  @inline(__always)
  init(_ lexeme: Lexer.Lexeme) {
    self.rawTokenKind = lexeme.rawTokenKind
    switch lexeme.rawTokenKind {
    case .keyword, .identifier:
      keyword = Keyword(lexeme.tokenText)
    default:
      keyword = nil
    }
    self.isAtStartOfLine = lexeme.isAtStartOfLine
  }
}

/// Describes a token that should be consumed by the parser.
///
/// All the methods in here and all functions that take a ``TokenSpec`` need to be
/// marked `@inline(__always)` so the compiler inlines the ``RawTokenKind`` we are
/// matching against and is thus able to rule out one of the branches in
/// `matches(rawTokenKind:text:)` based on the matched kind.
@_spi(AlternateTokenIntrospection)
public struct TokenSpec: Sendable {
  /// The kind we expect the token that we want to consume to have.
  /// This can be a keyword, in which case the ``TokenSpec`` will also match an
  /// identifier with the same text as the keyword and remap it to that keyword
  /// when consumed.
  ///
  /// `fileprivate` because only functions in this file should access it since
  /// they know how to handle the identifier -> keyword remapping.
  fileprivate let rawTokenKind: RawTokenKind

  /// If `rawTokenKind` is `keyword`, the keyword we are expecting. For all other
  /// values of `rawTokenKind`, this is `nil`.
  fileprivate let keyword: Keyword?

  /// If not nil, the token will be remapped to the provided kind when consumed.
  ///
  /// `fileprivate` because only functions in this file should access it since
  /// they know how to handle the identifier -> keyword remapping.
  fileprivate let remapping: RawTokenKind?

  /// The recovery precedence that should be used when consuming this token. By
  /// default this is the token precedence of `rawTokenKind` but it can be
  /// overridden.
  let recoveryPrecedence: TokenPrecedence

  /// Whether the token is allowed to be at the start of a line. Defaults to
  /// `true` but can be set to `false` to consume a token for recovery purposes
  /// that is not allowed to start a new line.
  let allowAtStartOfLine: Bool

  @inline(__always)
  init(
    _ rawTokenKind: RawTokenKind,
    remapping: RawTokenKind? = nil,
    recoveryPrecedence: TokenPrecedence? = nil,
    allowAtStartOfLine: Bool = true
  ) {
    precondition(
      rawTokenKind != .keyword,
      "To create a TokenSpec for a keyword use the initializer that takes a keyword"
    )
    self.rawTokenKind = rawTokenKind
    self.keyword = nil
    self.remapping = remapping
    self.recoveryPrecedence = recoveryPrecedence ?? TokenPrecedence(nonKeyword: rawTokenKind)
    self.allowAtStartOfLine = allowAtStartOfLine
  }

  @inline(__always)
  init(
    _ keyword: Keyword,
    remapping: RawTokenKind? = nil,
    recoveryPrecedence: TokenPrecedence? = nil,
    allowAtStartOfLine: Bool = true
  ) {
    self.rawTokenKind = .keyword
    self.keyword = keyword
    self.remapping = remapping
    self.recoveryPrecedence = recoveryPrecedence ?? TokenPrecedence(keyword)
    self.allowAtStartOfLine = allowAtStartOfLine
  }

  @inline(__always)
  func matches(
    rawTokenKind: RawTokenKind,
    keyword: @autoclosure () -> Keyword?,
    atStartOfLine: @autoclosure () -> Bool
  ) -> Bool {
    if !allowAtStartOfLine && atStartOfLine() {
      return false
    }
    if self.rawTokenKind == .keyword {
      precondition(self.keyword != nil)
      switch rawTokenKind {
      case .keyword, .identifier:
        return keyword() == self.keyword
      default:
        return false
      }
    } else {
      return rawTokenKind == self.rawTokenKind
    }
  }

  @inline(__always)
  static func ~= (kind: TokenSpec, lexeme: Lexer.Lexeme) -> Bool {
    return kind.matches(
      rawTokenKind: lexeme.rawTokenKind,
      keyword: Keyword(lexeme.tokenText),
      atStartOfLine: lexeme.isAtStartOfLine
    )
  }

  @inline(__always)
  static func ~= (kind: TokenSpec, token: TokenSyntax) -> Bool {
    return kind.matches(
      rawTokenKind: token.tokenView.rawKind,
      keyword: Keyword(token.tokenView.rawText),
      atStartOfLine: token.leadingTrivia.contains(where: { $0.isNewline })
    )
  }

  @inline(__always)
  static func ~= (kind: TokenSpec, token: RawTokenSyntax) -> Bool {
    return kind.matches(
      rawTokenKind: token.tokenKind,
      keyword: Keyword(token.tokenView.rawText),
      atStartOfLine: token.leadingTriviaPieces.contains(where: \.isNewline)
    )
  }

  @inline(__always)
  static func ~= (kind: TokenSpec, lexeme: PrepareForKeywordMatch) -> Bool {
    return kind.matches(
      rawTokenKind: lexeme.rawTokenKind,
      keyword: lexeme.keyword,
      atStartOfLine: lexeme.isAtStartOfLine
    )
  }

  /// Returns a ``TokenKind`` that will most likely be parsed as a token that
  /// matches this ``TokenSpec``.
  ///
  /// IMPORTANT: Should only be used when generating tokens during the
  /// modification of test cases. This should never be used in the parser itself.
  @_spi(AlternateTokenIntrospection)
  public var synthesizedTokenKind: TokenKind {
    switch rawTokenKind {
    case .binaryOperator: return .binaryOperator("+")
    case .dollarIdentifier: return .dollarIdentifier("$0")
    case .floatLiteral: return .floatLiteral("1.0")
    case .identifier: return .identifier("myIdent")
    case .integerLiteral: return .integerLiteral("1")
    case .keyword: return .keyword(keyword!)
    case .postfixOperator: return .postfixOperator("++")
    case .prefixOperator: return .prefixOperator("!")
    case .rawStringPoundDelimiter: return .rawStringPoundDelimiter("#")
    case .regexLiteralPattern: return .regexLiteralPattern(".*")
    case .regexPoundDelimiter: return .regexPoundDelimiter("#")
    case .stringSegment: return .stringSegment("abc")
    default: return TokenKind.fromRaw(kind: rawTokenKind, text: "")
    }
  }
}

extension TokenConsumer {
  /// Generates a missing token that has the expected kind of `spec`.
  @inline(__always)
  mutating func missingToken(_ spec: TokenSpec) -> Token {
    return missingToken(
      spec.remapping ?? spec.rawTokenKind,
      text: spec.keyword?.defaultText ?? spec.rawTokenKind.defaultText
    )
  }

  /// Asserts that the current token matches `spec` and consumes it, performing
  /// any necessary token kind remapping.
  ///
  /// This should only be called from parsing primitives like `consume(if:)` and
  /// `eat`. Introduce new users of this very sparingly.
  @inline(__always)
  mutating func eat(_ spec: TokenSpec) -> Token {
    precondition(spec ~= self.currentToken)
    if let remapping = spec.remapping {
      return self.consumeAnyToken(remapping: remapping)
    } else if spec.rawTokenKind == .keyword {
      return self.consumeAnyToken(remapping: .keyword)
    } else {
      return self.consumeAnyToken()
    }
  }
}