1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
|
import sys
from _typeshed import FileDescriptorOrPath
from collections.abc import Callable, Generator, Iterable, Sequence
from re import Pattern
from token import *
from token import EXACT_TOKEN_TYPES as EXACT_TOKEN_TYPES
from typing import Any, NamedTuple, TextIO, type_check_only
from typing_extensions import TypeAlias
__all__ = [
"AMPER",
"AMPEREQUAL",
"AT",
"ATEQUAL",
"CIRCUMFLEX",
"CIRCUMFLEXEQUAL",
"COLON",
"COLONEQUAL",
"COMMA",
"COMMENT",
"DEDENT",
"DOT",
"DOUBLESLASH",
"DOUBLESLASHEQUAL",
"DOUBLESTAR",
"DOUBLESTAREQUAL",
"ELLIPSIS",
"ENCODING",
"ENDMARKER",
"EQEQUAL",
"EQUAL",
"ERRORTOKEN",
"GREATER",
"GREATEREQUAL",
"INDENT",
"ISEOF",
"ISNONTERMINAL",
"ISTERMINAL",
"LBRACE",
"LEFTSHIFT",
"LEFTSHIFTEQUAL",
"LESS",
"LESSEQUAL",
"LPAR",
"LSQB",
"MINEQUAL",
"MINUS",
"NAME",
"NEWLINE",
"NL",
"NOTEQUAL",
"NT_OFFSET",
"NUMBER",
"N_TOKENS",
"OP",
"PERCENT",
"PERCENTEQUAL",
"PLUS",
"PLUSEQUAL",
"RARROW",
"RBRACE",
"RIGHTSHIFT",
"RIGHTSHIFTEQUAL",
"RPAR",
"RSQB",
"SEMI",
"SLASH",
"SLASHEQUAL",
"STAR",
"STAREQUAL",
"STRING",
"TILDE",
"TYPE_COMMENT",
"TYPE_IGNORE",
"TokenInfo",
"VBAR",
"VBAREQUAL",
"detect_encoding",
"generate_tokens",
"tok_name",
"tokenize",
"untokenize",
]
if sys.version_info < (3, 13):
__all__ += ["ASYNC", "AWAIT"]
if sys.version_info >= (3, 10):
__all__ += ["SOFT_KEYWORD"]
if sys.version_info >= (3, 12):
__all__ += ["EXCLAMATION", "FSTRING_END", "FSTRING_MIDDLE", "FSTRING_START", "EXACT_TOKEN_TYPES"]
if sys.version_info >= (3, 13):
__all__ += ["TokenError", "open"]
cookie_re: Pattern[str]
blank_re: Pattern[bytes]
_Position: TypeAlias = tuple[int, int]
# This class is not exposed. It calls itself tokenize.TokenInfo.
@type_check_only
class _TokenInfo(NamedTuple):
type: int
string: str
start: _Position
end: _Position
line: str
class TokenInfo(_TokenInfo):
@property
def exact_type(self) -> int: ...
# Backwards compatible tokens can be sequences of a shorter length too
_Token: TypeAlias = TokenInfo | Sequence[int | str | _Position]
class TokenError(Exception): ...
if sys.version_info < (3, 13):
class StopTokenizing(Exception): ... # undocumented
class Untokenizer:
tokens: list[str]
prev_row: int
prev_col: int
encoding: str | None
def add_whitespace(self, start: _Position) -> None: ...
def untokenize(self, iterable: Iterable[_Token]) -> str: ...
def compat(self, token: Sequence[int | str], iterable: Iterable[_Token]) -> None: ...
if sys.version_info >= (3, 12):
def escape_brackets(self, token: str) -> str: ...
# the docstring says "returns bytes" but is incorrect --
# if the ENCODING token is missing, it skips the encode
def untokenize(iterable: Iterable[_Token]) -> Any: ...
def detect_encoding(readline: Callable[[], bytes | bytearray]) -> tuple[str, Sequence[bytes]]: ...
def tokenize(readline: Callable[[], bytes | bytearray]) -> Generator[TokenInfo, None, None]: ...
def generate_tokens(readline: Callable[[], str]) -> Generator[TokenInfo, None, None]: ...
def open(filename: FileDescriptorOrPath) -> TextIO: ...
def group(*choices: str) -> str: ... # undocumented
def any(*choices: str) -> str: ... # undocumented
def maybe(*choices: str) -> str: ... # undocumented
Whitespace: str # undocumented
Comment: str # undocumented
Ignore: str # undocumented
Name: str # undocumented
Hexnumber: str # undocumented
Binnumber: str # undocumented
Octnumber: str # undocumented
Decnumber: str # undocumented
Intnumber: str # undocumented
Exponent: str # undocumented
Pointfloat: str # undocumented
Expfloat: str # undocumented
Floatnumber: str # undocumented
Imagnumber: str # undocumented
Number: str # undocumented
def _all_string_prefixes() -> set[str]: ... # undocumented
StringPrefix: str # undocumented
Single: str # undocumented
Double: str # undocumented
Single3: str # undocumented
Double3: str # undocumented
Triple: str # undocumented
String: str # undocumented
Special: str # undocumented
Funny: str # undocumented
PlainToken: str # undocumented
Token: str # undocumented
ContStr: str # undocumented
PseudoExtras: str # undocumented
PseudoToken: str # undocumented
endpats: dict[str, str] # undocumented
single_quoted: set[str] # undocumented
triple_quoted: set[str] # undocumented
tabsize: int # undocumented
|
