1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
//===- Token.h - MLIR Token Interface ---------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#ifndef MLIR_LIB_PARSER_TOKEN_H
#define MLIR_LIB_PARSER_TOKEN_H
#include "mlir/Support/LLVM.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/SMLoc.h"
#include <optional>
namespace mlir {
/// This represents a token in the MLIR syntax.
class Token {
public:
enum Kind {
#define TOK_MARKER(NAME) NAME,
#define TOK_IDENTIFIER(NAME) NAME,
#define TOK_LITERAL(NAME) NAME,
#define TOK_PUNCTUATION(NAME, SPELLING) NAME,
#define TOK_KEYWORD(SPELLING) kw_##SPELLING,
#include "TokenKinds.def"
};
Token(Kind kind, StringRef spelling) : kind(kind), spelling(spelling) {}
// Return the bytes that make up this token.
StringRef getSpelling() const { return spelling; }
// Token classification.
Kind getKind() const { return kind; }
bool is(Kind k) const { return kind == k; }
bool isAny(Kind k1, Kind k2) const { return is(k1) || is(k2); }
/// Return true if this token is one of the specified kinds.
template <typename... T>
bool isAny(Kind k1, Kind k2, Kind k3, T... others) const {
if (is(k1))
return true;
return isAny(k2, k3, others...);
}
bool isNot(Kind k) const { return kind != k; }
/// Return true if this token isn't one of the specified kinds.
template <typename... T>
bool isNot(Kind k1, Kind k2, T... others) const {
return !isAny(k1, k2, others...);
}
/// Return true if this is one of the keyword token kinds (e.g. kw_if).
bool isKeyword() const;
/// Returns true if the current token represents a code completion.
bool isCodeCompletion() const { return is(code_complete); }
/// Returns true if the current token represents a code completion for the
/// "normal" token type.
bool isCodeCompletionFor(Kind kind) const;
/// Returns true if the current token is the given type, or represents a code
/// completion for that type.
bool isOrIsCodeCompletionFor(Kind kind) const {
return is(kind) || isCodeCompletionFor(kind);
}
// Helpers to decode specific sorts of tokens.
/// For an integer token, return its value as an unsigned. If it doesn't fit,
/// return std::nullopt.
std::optional<unsigned> getUnsignedIntegerValue() const;
/// For an integer token, return its value as an uint64_t. If it doesn't fit,
/// return std::nullopt.
static std::optional<uint64_t> getUInt64IntegerValue(StringRef spelling);
std::optional<uint64_t> getUInt64IntegerValue() const {
return getUInt64IntegerValue(getSpelling());
}
/// For a floatliteral token, return its value as a double. Returns
/// std::nullopt in the case of underflow or overflow.
std::optional<double> getFloatingPointValue() const;
/// For an inttype token, return its bitwidth.
std::optional<unsigned> getIntTypeBitwidth() const;
/// For an inttype token, return its signedness semantics: std::nullopt means
/// no signedness semantics; true means signed integer type; false means
/// unsigned integer type.
std::optional<bool> getIntTypeSignedness() const;
/// Given a hash_identifier token like #123, try to parse the number out of
/// the identifier, returning std::nullopt if it is a named identifier like #x
/// or if the integer doesn't fit.
std::optional<unsigned> getHashIdentifierNumber() const;
/// Given a token containing a string literal, return its value, including
/// removing the quote characters and unescaping the contents of the string.
std::string getStringValue() const;
/// Given a token containing a hex string literal, return its value or
/// std::nullopt if the token does not contain a valid hex string. A hex
/// string literal is a string starting with `0x` and only containing hex
/// digits.
std::optional<std::string> getHexStringValue() const;
/// Given a token containing a symbol reference, return the unescaped string
/// value.
std::string getSymbolReference() const;
// Location processing.
SMLoc getLoc() const;
SMLoc getEndLoc() const;
SMRange getLocRange() const;
/// Given a punctuation or keyword token kind, return the spelling of the
/// token as a string. Warning: This will abort on markers, identifiers and
/// literal tokens since they have no fixed spelling.
static StringRef getTokenSpelling(Kind kind);
private:
/// Discriminator that indicates the sort of token this is.
Kind kind;
/// A reference to the entire token contents; this is always a pointer into
/// a memory buffer owned by the source manager.
StringRef spelling;
};
} // namespace mlir
#endif // MLIR_LIB_PARSER_TOKEN_H
|
