1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
|
//===--- PPCaching.cpp - Handle caching lexed tokens ----------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements pieces of the Preprocessor interface that manage the
// caching of lexed tokens.
//
//===----------------------------------------------------------------------===//
#include "clang/Lex/Preprocessor.h"
using namespace clang;
std::pair<Preprocessor::CachedTokensTy::size_type, bool>
Preprocessor::LastBacktrackPos() {
assert(isBacktrackEnabled());
auto BacktrackPos = BacktrackPositions.back();
bool Unannotated =
static_cast<CachedTokensTy::difference_type>(BacktrackPos) < 0;
return {Unannotated ? ~BacktrackPos : BacktrackPos, Unannotated};
}
// EnableBacktrackAtThisPos - From the point that this method is called, and
// until CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
// keeps track of the lexed tokens so that a subsequent Backtrack() call will
// make the Preprocessor re-lex the same tokens.
//
// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
// be combined with the EnableBacktrackAtThisPos calls in reverse order.
void Preprocessor::EnableBacktrackAtThisPos(bool Unannotated) {
assert(LexLevel == 0 && "cannot use lookahead while lexing");
BacktrackPositions.push_back(Unannotated ? ~CachedLexPos : CachedLexPos);
if (Unannotated)
UnannotatedBacktrackTokens.emplace_back(CachedTokens, CachedTokens.size());
EnterCachingLexMode();
}
Preprocessor::CachedTokensTy Preprocessor::PopUnannotatedBacktrackTokens() {
assert(isUnannotatedBacktrackEnabled() && "missing unannotated tokens?");
auto [UnannotatedTokens, NumCachedToks] =
std::move(UnannotatedBacktrackTokens.back());
UnannotatedBacktrackTokens.pop_back();
// If another unannotated backtrack is active, propagate any tokens that were
// lexed (not cached) since EnableBacktrackAtThisPos was last called.
if (isUnannotatedBacktrackEnabled())
UnannotatedBacktrackTokens.back().first.append(
UnannotatedTokens.begin() + NumCachedToks, UnannotatedTokens.end());
return std::move(UnannotatedTokens);
}
// Disable the last EnableBacktrackAtThisPos call.
void Preprocessor::CommitBacktrackedTokens() {
assert(isBacktrackEnabled() && "EnableBacktrackAtThisPos was not called!");
auto [BacktrackPos, Unannotated] = LastBacktrackPos();
BacktrackPositions.pop_back();
if (Unannotated)
PopUnannotatedBacktrackTokens();
}
// Make Preprocessor re-lex the tokens that were lexed since
// EnableBacktrackAtThisPos() was previously called.
void Preprocessor::Backtrack() {
assert(isBacktrackEnabled() && "EnableBacktrackAtThisPos was not called!");
auto [BacktrackPos, Unannotated] = LastBacktrackPos();
BacktrackPositions.pop_back();
CachedLexPos = BacktrackPos;
if (Unannotated)
CachedTokens = PopUnannotatedBacktrackTokens();
recomputeCurLexerKind();
}
void Preprocessor::CachingLex(Token &Result) {
if (!InCachingLexMode())
return;
// The assert in EnterCachingLexMode should prevent this from happening.
assert(LexLevel == 1 &&
"should not use token caching within the preprocessor");
if (CachedLexPos < CachedTokens.size()) {
Result = CachedTokens[CachedLexPos++];
Result.setFlag(Token::IsReinjected);
return;
}
ExitCachingLexMode();
Lex(Result);
if (isBacktrackEnabled()) {
// Cache the lexed token.
EnterCachingLexModeUnchecked();
CachedTokens.push_back(Result);
++CachedLexPos;
if (isUnannotatedBacktrackEnabled())
UnannotatedBacktrackTokens.back().first.push_back(Result);
return;
}
if (CachedLexPos < CachedTokens.size()) {
EnterCachingLexModeUnchecked();
} else {
// All cached tokens were consumed.
CachedTokens.clear();
CachedLexPos = 0;
}
}
void Preprocessor::EnterCachingLexMode() {
// The caching layer sits on top of all the other lexers, so it's incorrect
// to cache tokens while inside a nested lex action. The cached tokens would
// be retained after returning to the enclosing lex action and, at best,
// would appear at the wrong position in the token stream.
assert(LexLevel == 0 &&
"entered caching lex mode while lexing something else");
if (InCachingLexMode()) {
assert(CurLexerCallback == CLK_CachingLexer && "Unexpected lexer kind");
return;
}
EnterCachingLexModeUnchecked();
}
void Preprocessor::EnterCachingLexModeUnchecked() {
assert(CurLexerCallback != CLK_CachingLexer && "already in caching lex mode");
PushIncludeMacroStack();
CurLexerCallback = CLK_CachingLexer;
}
const Token &Preprocessor::PeekAhead(unsigned N) {
assert(CachedLexPos + N > CachedTokens.size() && "Confused caching.");
ExitCachingLexMode();
for (size_t C = CachedLexPos + N - CachedTokens.size(); C > 0; --C) {
CachedTokens.push_back(Token());
Lex(CachedTokens.back());
if (isUnannotatedBacktrackEnabled())
UnannotatedBacktrackTokens.back().first.push_back(CachedTokens.back());
}
EnterCachingLexMode();
return CachedTokens.back();
}
void Preprocessor::AnnotatePreviousCachedTokens(const Token &Tok) {
assert(Tok.isAnnotation() && "Expected annotation token");
assert(CachedLexPos != 0 && "Expected to have some cached tokens");
assert(CachedTokens[CachedLexPos-1].getLastLoc() == Tok.getAnnotationEndLoc()
&& "The annotation should be until the most recent cached token");
// Start from the end of the cached tokens list and look for the token
// that is the beginning of the annotation token.
for (CachedTokensTy::size_type i = CachedLexPos; i != 0; --i) {
CachedTokensTy::iterator AnnotBegin = CachedTokens.begin() + i-1;
if (AnnotBegin->getLocation() == Tok.getLocation()) {
assert((!isBacktrackEnabled() || LastBacktrackPos().first <= i) &&
"The backtrack pos points inside the annotated tokens!");
// Replace the cached tokens with the single annotation token.
if (i < CachedLexPos)
CachedTokens.erase(AnnotBegin + 1, CachedTokens.begin() + CachedLexPos);
*AnnotBegin = Tok;
CachedLexPos = i;
return;
}
}
}
bool Preprocessor::IsPreviousCachedToken(const Token &Tok) const {
// There's currently no cached token...
if (!CachedLexPos)
return false;
const Token LastCachedTok = CachedTokens[CachedLexPos - 1];
if (LastCachedTok.getKind() != Tok.getKind())
return false;
SourceLocation::IntTy RelOffset = 0;
if ((!getSourceManager().isInSameSLocAddrSpace(
Tok.getLocation(), getLastCachedTokenLocation(), &RelOffset)) ||
RelOffset)
return false;
return true;
}
void Preprocessor::ReplacePreviousCachedToken(ArrayRef<Token> NewToks) {
assert(CachedLexPos != 0 && "Expected to have some cached tokens");
CachedTokens.insert(CachedTokens.begin() + CachedLexPos - 1, NewToks.begin(),
NewToks.end());
CachedTokens.erase(CachedTokens.begin() + CachedLexPos - 1 + NewToks.size());
CachedLexPos += NewToks.size() - 1;
}
|
