1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
//===-- lib/Parser/token-sequence.h -----------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef FORTRAN_PARSER_TOKEN_SEQUENCE_H_
#define FORTRAN_PARSER_TOKEN_SEQUENCE_H_
// A buffer class capable of holding a contiguous sequence of characters
// and a partitioning thereof into preprocessing tokens, along with their
// associated provenances.
#include "flang/Parser/char-block.h"
#include "flang/Parser/provenance.h"
#include <cstddef>
#include <cstring>
#include <string>
#include <utility>
#include <vector>
namespace llvm {
class raw_ostream;
}
namespace Fortran::parser {
// Buffers a contiguous sequence of characters that has been partitioned into
// a sequence of preprocessing tokens with provenances.
class TokenSequence {
public:
TokenSequence() {}
TokenSequence(const TokenSequence &that) { Put(that); }
TokenSequence(
const TokenSequence &that, std::size_t at, std::size_t count = 1) {
Put(that, at, count);
}
TokenSequence(TokenSequence &&that)
: start_{std::move(that.start_)}, nextStart_{that.nextStart_},
char_{std::move(that.char_)}, provenances_{
std::move(that.provenances_)} {}
TokenSequence(const std::string &s, Provenance p) { Put(s, p); }
TokenSequence &operator=(const TokenSequence &that) {
clear();
Put(that);
return *this;
}
TokenSequence &operator=(TokenSequence &&that);
bool empty() const { return start_.empty(); }
void clear();
void pop_back();
void shrink_to_fit();
void swap(TokenSequence &);
std::size_t SizeInTokens() const { return start_.size(); }
std::size_t SizeInChars() const { return char_.size(); }
CharBlock ToCharBlock() const { return {&char_[0], char_.size()}; }
std::string ToString() const { return ToCharBlock().ToString(); }
CharBlock TokenAt(std::size_t token) const {
return {&char_[start_.at(token)], TokenBytes(token)};
}
char CharAt(std::size_t j) const { return char_.at(j); }
CharBlock CurrentOpenToken() const {
return {&char_[nextStart_], char_.size() - nextStart_};
}
std::size_t SkipBlanks(std::size_t) const;
void PutNextTokenChar(char ch, Provenance provenance) {
char_.emplace_back(ch);
provenances_.Put({provenance, 1});
}
void CloseToken() {
start_.emplace_back(nextStart_);
nextStart_ = char_.size();
}
void ReopenLastToken() {
nextStart_ = start_.back();
start_.pop_back();
}
void RemoveLastToken();
void Put(const TokenSequence &);
void Put(const TokenSequence &, ProvenanceRange);
void Put(const TokenSequence &, std::size_t at, std::size_t tokens = 1);
void Put(const char *, std::size_t, Provenance);
void Put(const CharBlock &, Provenance);
void Put(const std::string &, Provenance);
void Put(llvm::raw_string_ostream &, Provenance);
Provenance GetTokenProvenance(
std::size_t token, std::size_t offset = 0) const;
ProvenanceRange GetTokenProvenanceRange(
std::size_t token, std::size_t offset = 0) const;
ProvenanceRange GetIntervalProvenanceRange(
std::size_t token, std::size_t tokens = 1) const;
ProvenanceRange GetProvenanceRange() const;
char *GetMutableCharData() { return &char_[0]; }
TokenSequence &ToLowerCase();
bool HasBlanks(std::size_t firstChar = 0) const;
bool HasRedundantBlanks(std::size_t firstChar = 0) const;
TokenSequence &RemoveBlanks(std::size_t firstChar = 0);
TokenSequence &RemoveRedundantBlanks(std::size_t firstChar = 0);
TokenSequence &ClipComment(bool skipFirst = false);
void Emit(CookedSource &) const;
void Dump(llvm::raw_ostream &) const;
private:
std::size_t TokenBytes(std::size_t token) const {
return (token + 1 >= start_.size() ? char_.size() : start_[token + 1]) -
start_[token];
}
std::vector<std::size_t> start_;
std::size_t nextStart_{0};
std::vector<char> char_;
OffsetToProvenanceMappings provenances_;
};
} // namespace Fortran::parser
#endif // FORTRAN_PARSER_TOKEN_SEQUENCE_H_
|
