1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
|
package com.metaweb.lessen.tokenizers;
import java.util.LinkedList;
import java.util.List;
import com.metaweb.lessen.tokens.Token;
import com.metaweb.lessen.tokens.Token.Type;
public class BufferedTokenizer implements Tokenizer {
final protected Tokenizer _tokenizer;
final protected List<Token> _tokens = new LinkedList<Token>();
public BufferedTokenizer(Tokenizer tokenizer) {
_tokenizer = tokenizer;
Token t = _tokenizer.getToken();
if (t != null) {
_tokens.add(t);
}
}
public Token getToken(int lookahead) {
while (lookahead >= _tokens.size() && getMore());
return lookahead < _tokens.size() ? _tokens.get(lookahead) : null;
}
@Override
public Token getToken() {
return getToken(0);
}
public void next(int count) {
while (count > 0) {
if (_tokens.size() > 0 || getMore()) {
_tokens.remove(0);
count--;
} else {
break;
}
}
}
@Override
public void next() {
next(1);
}
public int lookOverWhitespaceAndComment(int lookahead) {
Token t;
while ((t = getToken(lookahead)) != null &&
(t.type == Type.Whitespace || t.type == Type.Comment)) {
lookahead++;
}
return lookahead;
}
protected boolean getMore() {
_tokenizer.next();
Token t = _tokenizer.getToken();
if (t != null) {
_tokens.add(t);
return true;
} else {
return false;
}
}
}
|
