1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
|
//===--- TokenAnalyzer.cpp - Analyze Token Streams --------------*- C++ -*-===//
//
// The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
///
/// \file
/// \brief This file implements an abstract TokenAnalyzer and associated helper
/// classes. TokenAnalyzer can be extended to generate replacements based on
/// an annotated and pre-processed token stream.
///
//===----------------------------------------------------------------------===//
#include "TokenAnalyzer.h"
#include "AffectedRangeManager.h"
#include "Encoding.h"
#include "FormatToken.h"
#include "FormatTokenLexer.h"
#include "TokenAnnotator.h"
#include "UnwrappedLineParser.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/DiagnosticOptions.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Format/Format.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/Debug.h"
#define DEBUG_TYPE "format-formatter"
namespace clang {
namespace format {
// This sets up an virtual file system with file \p FileName containing \p
// Code.
std::unique_ptr<Environment>
Environment::CreateVirtualEnvironment(StringRef Code, StringRef FileName,
ArrayRef<tooling::Range> Ranges,
unsigned FirstStartColumn,
unsigned NextStartColumn,
unsigned LastStartColumn) {
// This is referenced by `FileMgr` and will be released by `FileMgr` when it
// is deleted.
IntrusiveRefCntPtr<vfs::InMemoryFileSystem> InMemoryFileSystem(
new vfs::InMemoryFileSystem);
// This is passed to `SM` as reference, so the pointer has to be referenced
// in `Environment` so that `FileMgr` can out-live this function scope.
std::unique_ptr<FileManager> FileMgr(
new FileManager(FileSystemOptions(), InMemoryFileSystem));
// This is passed to `SM` as reference, so the pointer has to be referenced
// by `Environment` due to the same reason above.
std::unique_ptr<DiagnosticsEngine> Diagnostics(new DiagnosticsEngine(
IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs),
new DiagnosticOptions));
// This will be stored as reference, so the pointer has to be stored in
// due to the same reason above.
std::unique_ptr<SourceManager> VirtualSM(
new SourceManager(*Diagnostics, *FileMgr));
InMemoryFileSystem->addFile(
FileName, 0,
llvm::MemoryBuffer::getMemBuffer(Code, FileName,
/*RequiresNullTerminator=*/false));
FileID ID = VirtualSM->createFileID(FileMgr->getFile(FileName),
SourceLocation(), clang::SrcMgr::C_User);
assert(ID.isValid());
SourceLocation StartOfFile = VirtualSM->getLocForStartOfFile(ID);
std::vector<CharSourceRange> CharRanges;
for (const tooling::Range &Range : Ranges) {
SourceLocation Start = StartOfFile.getLocWithOffset(Range.getOffset());
SourceLocation End = Start.getLocWithOffset(Range.getLength());
CharRanges.push_back(CharSourceRange::getCharRange(Start, End));
}
return llvm::make_unique<Environment>(
ID, std::move(FileMgr), std::move(VirtualSM), std::move(Diagnostics),
CharRanges, FirstStartColumn, NextStartColumn, LastStartColumn);
}
TokenAnalyzer::TokenAnalyzer(const Environment &Env, const FormatStyle &Style)
: Style(Style), Env(Env),
AffectedRangeMgr(Env.getSourceManager(), Env.getCharRanges()),
UnwrappedLines(1),
Encoding(encoding::detectEncoding(
Env.getSourceManager().getBufferData(Env.getFileID()))) {
DEBUG(
llvm::dbgs() << "File encoding: "
<< (Encoding == encoding::Encoding_UTF8 ? "UTF8" : "unknown")
<< "\n");
DEBUG(llvm::dbgs() << "Language: " << getLanguageName(Style.Language)
<< "\n");
}
std::pair<tooling::Replacements, unsigned> TokenAnalyzer::process() {
tooling::Replacements Result;
FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(),
Env.getFirstStartColumn(), Style, Encoding);
UnwrappedLineParser Parser(Style, Tokens.getKeywords(),
Env.getFirstStartColumn(), Tokens.lex(), *this);
Parser.parse();
assert(UnwrappedLines.rbegin()->empty());
unsigned Penalty = 0;
for (unsigned Run = 0, RunE = UnwrappedLines.size(); Run + 1 != RunE; ++Run) {
DEBUG(llvm::dbgs() << "Run " << Run << "...\n");
SmallVector<AnnotatedLine *, 16> AnnotatedLines;
TokenAnnotator Annotator(Style, Tokens.getKeywords());
for (unsigned i = 0, e = UnwrappedLines[Run].size(); i != e; ++i) {
AnnotatedLines.push_back(new AnnotatedLine(UnwrappedLines[Run][i]));
Annotator.annotate(*AnnotatedLines.back());
}
std::pair<tooling::Replacements, unsigned> RunResult =
analyze(Annotator, AnnotatedLines, Tokens);
DEBUG({
llvm::dbgs() << "Replacements for run " << Run << ":\n";
for (tooling::Replacements::const_iterator I = RunResult.first.begin(),
E = RunResult.first.end();
I != E; ++I) {
llvm::dbgs() << I->toString() << "\n";
}
});
for (unsigned i = 0, e = AnnotatedLines.size(); i != e; ++i) {
delete AnnotatedLines[i];
}
Penalty += RunResult.second;
for (const auto &R : RunResult.first) {
auto Err = Result.add(R);
// FIXME: better error handling here. For now, simply return an empty
// Replacements to indicate failure.
if (Err) {
llvm::errs() << llvm::toString(std::move(Err)) << "\n";
return {tooling::Replacements(), 0};
}
}
}
return {Result, Penalty};
}
void TokenAnalyzer::consumeUnwrappedLine(const UnwrappedLine &TheLine) {
assert(!UnwrappedLines.empty());
UnwrappedLines.back().push_back(TheLine);
}
void TokenAnalyzer::finishRun() {
UnwrappedLines.push_back(SmallVector<UnwrappedLine, 16>());
}
} // end namespace format
} // end namespace clang
|
