1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
|
//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2024 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
import LSPLogging
import LanguageServerProtocol
import SourceKitD
/// A wrapper around an array of syntax highlighting tokens.
public struct SyntaxHighlightingTokens: Sendable {
public var tokens: [SyntaxHighlightingToken]
public init(tokens: [SyntaxHighlightingToken]) {
self.tokens = tokens
}
/// The LSP representation of syntax highlighting tokens. Note that this
/// requires the tokens in this array to be sorted.
public var lspEncoded: [UInt32] {
var previous = Position(line: 0, utf16index: 0)
var rawTokens: [UInt32] = []
rawTokens.reserveCapacity(tokens.count * 5)
for token in self.tokens {
let lineDelta = token.start.line - previous.line
let charDelta =
token.start.utf16index - (
// The character delta is relative to the previous token's start
// only if the token is on the previous token's line.
previous.line == token.start.line ? previous.utf16index : 0)
// We assert that the tokens are actually sorted
assert(lineDelta >= 0)
assert(charDelta >= 0)
previous = token.start
rawTokens += [
UInt32(lineDelta),
UInt32(charDelta),
UInt32(token.utf16length),
token.kind.tokenType,
token.modifiers.rawValue,
]
}
return rawTokens
}
/// Merges the tokens in this array into a new token array,
/// preferring the given array's tokens if duplicate ranges are
/// found.
public func mergingTokens(with other: SyntaxHighlightingTokens) -> SyntaxHighlightingTokens {
let otherRanges = Set(other.tokens.map(\.range))
return SyntaxHighlightingTokens(tokens: tokens.filter { !otherRanges.contains($0.range) } + other.tokens)
}
public func mergingTokens(with other: [SyntaxHighlightingToken]) -> SyntaxHighlightingTokens {
let otherRanges = Set(other.map(\.range))
return SyntaxHighlightingTokens(tokens: tokens.filter { !otherRanges.contains($0.range) } + other)
}
/// Sorts the tokens in this array by their start position.
public func sorted(_ areInIncreasingOrder: (SyntaxHighlightingToken, SyntaxHighlightingToken) -> Bool)
-> SyntaxHighlightingTokens
{
SyntaxHighlightingTokens(tokens: tokens.sorted(by: areInIncreasingOrder))
}
}
extension SyntaxHighlightingTokens {
/// Decodes the LSP representation of syntax highlighting tokens
public init(lspEncodedTokens rawTokens: [UInt32]) {
self.init(tokens: [])
assert(rawTokens.count.isMultiple(of: 5))
self.tokens.reserveCapacity(rawTokens.count / 5)
var current = Position(line: 0, utf16index: 0)
for i in stride(from: 0, to: rawTokens.count, by: 5) {
let lineDelta = Int(rawTokens[i])
let charDelta = Int(rawTokens[i + 1])
let length = Int(rawTokens[i + 2])
let rawKind = rawTokens[i + 3]
let rawModifiers = rawTokens[i + 4]
current.line += lineDelta
if lineDelta == 0 {
current.utf16index += charDelta
} else {
current.utf16index = charDelta
}
let kind = SemanticTokenTypes.all[Int(rawKind)]
let modifiers = SemanticTokenModifiers(rawValue: rawModifiers)
self.tokens.append(
SyntaxHighlightingToken(
start: current,
utf16length: length,
kind: kind,
modifiers: modifiers
)
)
}
}
}
|
