File: tokenizer.py

package info (click to toggle)
python-pegen 0.3.0-2
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 11,148 kB
  • sloc: python: 15,081; makefile: 89
file content (23 lines) | stat: -rw-r--r-- 541 bytes parent folder | download | duplicates (2) 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
 
class Tokenizer:

    def __init__(self, tokengen):
        """Call with tokenize.generate_tokens(...)."""
        self.tokengen = tokengen
        self.tokens = []
        self.pos = 0

    def mark(self):
        return self.pos

    def reset(self, pos):
        self.pos = pos

    def get_token(self):
        token = self.peek_token()
        self.pos += 1
        return token

    def peek_token(self):
        if self.pos == len(self.tokens):
            self.tokens.append(next(self.tokengen))
        return self.tokens[self.pos]