File: SourcedTokenizer.java

package info (click to toggle)
libsecondstring-java 0.1~dfsg-2
  • links: PTS, VCS
  • area: main
  • in suites: bookworm, forky, sid, trixie
  • size: 764 kB
  • sloc: java: 9,592; xml: 114; makefile: 6
file content (17 lines) | stat: -rw-r--r-- 347 bytes parent folder | download 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
 
package com.wcohen.ss.api;

import java.util.Iterator;

/**
 * Split a string into tokens, retaining provinance.
 */

public interface SourcedTokenizer extends Tokenizer
{
    /**  Return tokenized version of a string, as an array of
     *  SourcedToken objects.
     */
    public SourcedToken[] sourcedTokenize(String input,String source);
}