1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
|
//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
#if swift(>=6)
@_spi(RawSyntax) public import SwiftSyntax
#else
@_spi(RawSyntax) import SwiftSyntax
#endif
extension Lexer {
/// A trivia-delimited region of source text.
///
/// A lexeme is the fundamental output unit of lexical analysis. Each lexeme
/// represents a fully identified, meaningful part of the input text that
/// will can be consumed by a ``Parser``.
@_spi(Testing)
public struct Lexeme: CustomDebugStringConvertible {
@_spi(Testing)
public struct Flags: OptionSet, CustomDebugStringConvertible, Sendable {
@_spi(Testing)
public var rawValue: UInt8
@_spi(Testing)
public init(rawValue: UInt8) {
self.rawValue = rawValue
}
@_spi(Testing)
public static let isAtStartOfLine = Flags(rawValue: 1 << 0)
@_spi(Testing)
public var debugDescription: String {
var descriptionComponents: [String] = []
if self.contains(.isAtStartOfLine) {
descriptionComponents.append("isAtStartOfLine")
}
return "[\(descriptionComponents.joined(separator: ", "))]"
}
}
@_spi(Testing)
public var rawTokenKind: RawTokenKind
@_spi(Testing)
public var flags: Lexeme.Flags
@_spi(Testing)
public var diagnostic: TokenDiagnostic?
var start: UnsafePointer<UInt8>
var leadingTriviaByteLength: Int
var textByteLength: Int
var trailingTriviaByteLength: Int
/// The cursor that produces this lexeme by calling `nextToken` on it.
/// Used if the token needs to be re-lexed in a different lexer state.
var cursor: Lexer.Cursor
var isAtStartOfLine: Bool {
return self.flags.contains(.isAtStartOfLine)
}
var isEditorPlaceholder: Bool {
return self.rawTokenKind == .identifier && self.tokenText.isEditorPlaceholder
}
init(
tokenKind: RawTokenKind,
flags: Flags,
diagnostic: TokenDiagnostic?,
start: UnsafePointer<UInt8>,
leadingTriviaLength: Int,
textLength: Int,
trailingTriviaLength: Int,
cursor: Lexer.Cursor
) {
self.rawTokenKind = tokenKind
self.flags = flags
self.diagnostic = diagnostic
self.start = start
self.leadingTriviaByteLength = leadingTriviaLength
self.textByteLength = textLength
self.trailingTriviaByteLength = trailingTriviaLength
self.cursor = cursor
}
@_spi(Testing)
public var byteLength: Int {
leadingTriviaByteLength + textByteLength + trailingTriviaByteLength
}
@_spi(Testing)
public var wholeText: SyntaxText {
SyntaxText(baseAddress: start, count: byteLength)
}
@_spi(Testing)
public var textRange: Range<SyntaxText.Index> {
leadingTriviaByteLength..<leadingTriviaByteLength + textByteLength
}
@_spi(Testing)
public var tokenText: SyntaxText {
SyntaxText(
baseAddress: start.advanced(by: leadingTriviaByteLength),
count: textByteLength
)
}
@_spi(Testing)
public var leadingTriviaText: SyntaxText {
SyntaxText(
baseAddress: start,
count: leadingTriviaByteLength
)
}
@_spi(Testing)
public var trailingTriviaText: SyntaxText {
SyntaxText(
baseAddress: start.advanced(by: leadingTriviaByteLength + textByteLength),
count: trailingTriviaByteLength
)
}
@_spi(Testing)
public var debugDescription: String {
return String(syntaxText: SyntaxText(baseAddress: start, count: byteLength))
}
}
}
|
