1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
|
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
class TokenType:
name: str
contains_syntax: bool
STRING: TokenType = ...
NAME: TokenType = ...
NUMBER: TokenType = ...
OP: TokenType = ...
NEWLINE: TokenType = ...
INDENT: TokenType = ...
DEDENT: TokenType = ...
ASYNC: TokenType = ...
AWAIT: TokenType = ...
FSTRING_START: TokenType = ...
FSTRING_STRING: TokenType = ...
FSTRING_END: TokenType = ...
ENDMARKER: TokenType = ...
# unused dummy tokens for backwards compat with the parso tokenizer
ERRORTOKEN: TokenType = ...
ERROR_DEDENT: TokenType = ...
|
