1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
|
class Tokenizer:
def __init__(self, tokengen, vis=None):
"""Call with tokenize.generate_tokens(...)."""
self.tokengen = tokengen
self.vis = vis
self.tokens = []
self.pos = 0
def mark(self):
return self.pos
def reset(self, pos):
if pos == self.pos:
return
self.pos = pos
self.report()
def get_token(self):
token = self.peek_token()
self.pos += 1
self.report()
return token
def peek_token(self):
if self.pos == len(self.tokens):
self.tokens.append(next(self.tokengen))
self.report()
return self.tokens[self.pos]
def report(self):
if self.vis is not None:
self.vis.vis_tokens(self.tokens, self.pos)
|
