1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
|
package com.metaweb.lessen.tokenizers;
import java.io.IOException;
import java.io.Reader;
public class ReaderTokenizer extends TokenizerBase {
final static protected int s_bufferSize = 32;
final protected Reader _reader;
final protected StringBuffer _sb = new StringBuffer(s_bufferSize * 2);
final protected char[] _chars = new char[s_bufferSize];
protected int _sbStartIndex = 0;
protected int _next = 0;
protected boolean _eof = false;
public ReaderTokenizer(Reader reader) {
_reader = reader;
next();
}
@Override
protected boolean hasMoreChar() {
return hasMoreChar(0);
}
@Override
protected void advance() {
advance(1, false);
}
@Override
protected void advance(int by, boolean flush) {
_next += by;
if (flush) {
flush(_next);
}
}
@Override
protected void flush(int upToIndex) {
int chop = upToIndex - _sbStartIndex;
if (chop > 0.75 * s_bufferSize) {
_sb.delete(0, chop);
_sbStartIndex += chop;
}
}
@Override
protected char getCharRelative(int offset) {
tryLoadAhead(offset + 1);
return _sb.charAt(_next - _sbStartIndex + offset);
}
@Override
protected char getCurrentChar() {
return getCharRelative(0);
}
@Override
protected int getCurrentOffset() {
return _next;
}
@Override
protected String getText(int from, int to) {
return _sb.substring(from - _sbStartIndex, to - _sbStartIndex);
}
@Override
protected boolean hasMoreChar(int offset) {
tryLoadAhead(offset + 1);
return _next - _sbStartIndex + offset < _sb.length();
}
protected void tryLoadAhead(int distance) {
if (_next - _sbStartIndex + distance > _sb.length() && !_eof) {
try {
int c = _reader.read(_chars);
if (c < 0) {
_eof = true;
_reader.close();
} else {
_sb.append(_chars, 0, c);
}
} catch (IOException e) {
_eof = true;
}
}
}
}
|
