1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
|
//===--- PPCaching.cpp - Handle caching lexed tokens ----------------------===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// This file implements pieces of the Preprocessor interface that manage the
// caching of lexed tokens.
//
//===----------------------------------------------------------------------===//
#include "clang/Lex/Preprocessor.h"
using namespace clang;
// EnableBacktrackAtThisPos - From the point that this method is called, and
// until CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
// keeps track of the lexed tokens so that a subsequent Backtrack() call will
// make the Preprocessor re-lex the same tokens.
//
// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
// be combined with the EnableBacktrackAtThisPos calls in reverse order.
void Preprocessor::EnableBacktrackAtThisPos() {
BacktrackPositions.push_back(CachedLexPos);
EnterCachingLexMode();
}
// Disable the last EnableBacktrackAtThisPos call.
void Preprocessor::CommitBacktrackedTokens() {
assert(!BacktrackPositions.empty()
&& "EnableBacktrackAtThisPos was not called!");
BacktrackPositions.pop_back();
}
Preprocessor::CachedTokensRange Preprocessor::LastCachedTokenRange() {
assert(isBacktrackEnabled());
auto PrevCachedLexPos = BacktrackPositions.back();
return CachedTokensRange{PrevCachedLexPos, CachedLexPos};
}
void Preprocessor::EraseCachedTokens(CachedTokensRange TokenRange) {
assert(TokenRange.Begin <= TokenRange.End);
if (CachedLexPos == TokenRange.Begin && TokenRange.Begin != TokenRange.End) {
// We have backtracked to the start of the token range as we want to consume
// them again. Erase the tokens only after consuming then.
assert(!CachedTokenRangeToErase);
CachedTokenRangeToErase = TokenRange;
return;
}
// The cached tokens were committed, so they should be erased now.
assert(TokenRange.End == CachedLexPos);
CachedTokens.erase(CachedTokens.begin() + TokenRange.Begin,
CachedTokens.begin() + TokenRange.End);
CachedLexPos = TokenRange.Begin;
ExitCachingLexMode();
}
// Make Preprocessor re-lex the tokens that were lexed since
// EnableBacktrackAtThisPos() was previously called.
void Preprocessor::Backtrack() {
assert(!BacktrackPositions.empty()
&& "EnableBacktrackAtThisPos was not called!");
CachedLexPos = BacktrackPositions.back();
BacktrackPositions.pop_back();
recomputeCurLexerKind();
}
void Preprocessor::CachingLex(Token &Result) {
if (!InCachingLexMode())
return;
if (CachedLexPos < CachedTokens.size()) {
Result = CachedTokens[CachedLexPos++];
// Erase the some of the cached tokens after they are consumed when
// asked to do so.
if (CachedTokenRangeToErase &&
CachedTokenRangeToErase->End == CachedLexPos) {
EraseCachedTokens(*CachedTokenRangeToErase);
CachedTokenRangeToErase = None;
}
return;
}
ExitCachingLexMode();
Lex(Result);
if (isBacktrackEnabled()) {
// Cache the lexed token.
EnterCachingLexMode();
CachedTokens.push_back(Result);
++CachedLexPos;
return;
}
if (CachedLexPos < CachedTokens.size()) {
EnterCachingLexMode();
} else {
// All cached tokens were consumed.
CachedTokens.clear();
CachedLexPos = 0;
}
}
void Preprocessor::EnterCachingLexMode() {
if (InCachingLexMode()) {
assert(CurLexerKind == CLK_CachingLexer && "Unexpected lexer kind");
return;
}
PushIncludeMacroStack();
CurLexerKind = CLK_CachingLexer;
}
const Token &Preprocessor::PeekAhead(unsigned N) {
assert(CachedLexPos + N > CachedTokens.size() && "Confused caching.");
ExitCachingLexMode();
for (size_t C = CachedLexPos + N - CachedTokens.size(); C > 0; --C) {
CachedTokens.push_back(Token());
Lex(CachedTokens.back());
}
EnterCachingLexMode();
return CachedTokens.back();
}
void Preprocessor::AnnotatePreviousCachedTokens(const Token &Tok) {
assert(Tok.isAnnotation() && "Expected annotation token");
assert(CachedLexPos != 0 && "Expected to have some cached tokens");
assert(CachedTokens[CachedLexPos-1].getLastLoc() == Tok.getAnnotationEndLoc()
&& "The annotation should be until the most recent cached token");
// Start from the end of the cached tokens list and look for the token
// that is the beginning of the annotation token.
for (CachedTokensTy::size_type i = CachedLexPos; i != 0; --i) {
CachedTokensTy::iterator AnnotBegin = CachedTokens.begin() + i-1;
if (AnnotBegin->getLocation() == Tok.getLocation()) {
assert((BacktrackPositions.empty() || BacktrackPositions.back() <= i) &&
"The backtrack pos points inside the annotated tokens!");
// Replace the cached tokens with the single annotation token.
if (i < CachedLexPos)
CachedTokens.erase(AnnotBegin + 1, CachedTokens.begin() + CachedLexPos);
*AnnotBegin = Tok;
CachedLexPos = i;
return;
}
}
}
bool Preprocessor::IsPreviousCachedToken(const Token &Tok) const {
// There's currently no cached token...
if (!CachedLexPos)
return false;
const Token LastCachedTok = CachedTokens[CachedLexPos - 1];
if (LastCachedTok.getKind() != Tok.getKind())
return false;
int RelOffset = 0;
if ((!getSourceManager().isInSameSLocAddrSpace(
Tok.getLocation(), getLastCachedTokenLocation(), &RelOffset)) ||
RelOffset)
return false;
return true;
}
void Preprocessor::ReplacePreviousCachedToken(ArrayRef<Token> NewToks) {
assert(CachedLexPos != 0 && "Expected to have some cached tokens");
CachedTokens.insert(CachedTokens.begin() + CachedLexPos - 1, NewToks.begin(),
NewToks.end());
CachedTokens.erase(CachedTokens.begin() + CachedLexPos - 1 + NewToks.size());
CachedLexPos += NewToks.size() - 1;
}
|
