1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
|
package com.metaweb.lessen.tokenizers;
import com.metaweb.lessen.tokens.Token;
import com.metaweb.lessen.tokens.Token.Type;
public class CondensingTokenizer implements Tokenizer {
final protected BufferedTokenizer _tokenizer;
final protected boolean _removeComments;
protected Token _token;
public CondensingTokenizer(Tokenizer tokenizer) {
this(tokenizer, true);
}
public CondensingTokenizer(Tokenizer tokenizer, boolean removeComments) {
_tokenizer = new BufferedTokenizer(tokenizer);
_removeComments = removeComments;
_token = _tokenizer.getToken();
resolve();
}
@Override
public Token getToken() {
return _token;
}
@Override
public void next() {
_tokenizer.next();
_token = _tokenizer.getToken();
resolve();
}
protected void resolve() {
while (_removeComments && _token != null && _token.type == Type.Comment) {
_tokenizer.next();
_token = _tokenizer.getToken();
}
if (_token != null && _token.type == Type.Whitespace) {
StringBuffer sb = new StringBuffer();
sb.append(_token.text);
int lookahead = 1;
Token t;
while ((t = _tokenizer.getToken(lookahead)) != null &&
(t.type == Type.Whitespace ||
(_removeComments && t.type == Type.Comment))) {
if (t.type == Type.Whitespace) {
sb.append(t.text);
}
lookahead++;
}
_tokenizer.next(lookahead - 1);
_token = new Token(
_token.type,
_token.start,
_token.end,
sb.toString().indexOf('\n') < 0 ? " " : "\n"
);
}
}
}
|
