1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
|
package com.wcohen.ss.api;
import java.util.Iterator;
/**
* Split a string into tokens.
*/
public interface Tokenizer
{
/** Return tokenized version of a string */
public Token[] tokenize(String input);
/** Convert a given string into a token. The intern function
* should have these properties: (1) If s1.equals(s2), then
* intern(s1)==intern(s2). (2) If no string equal to s1 has ever
* been interned before, then intern(s1).getIndex() will be larger
* than every previously-assigned index--i.e, token 'indexes' are
* assigned in increasing order.
*/
public Token intern(String s);
/** Return an iterator over interned tokens */
public Iterator tokenIterator();
/** Return the higest index of any interned token */
public int maxTokenIndex();
}
|
