1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
|
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <iostream>
#include "asserttestexit.h"
#include "srchilite/wordtokenizer.h"
#include "srchilite/tostringcollection.h"
using namespace std;
using namespace srchilite;
static ostream &operator <<(ostream &os, const WordTokenizer::WordTokenizerResults::value_type &);
ostream &operator <<(ostream &os, const WordTokenizer::WordTokenizerResults::value_type &token) {
if (token.first.size()) {
os << "space: \"" << token.first << "\"" << endl;
} else {
os << "word : \"" << token.second << "\"" << endl;
}
return os;
}
int main() {
WordTokenizer::WordTokenizerResults tokens;
WordTokenizer::tokenize(" Here are\t \t some_words!", tokens);
cout << "tokens: " << collectionToString(&tokens) << endl;
WordTokenizer::WordTokenizerResults::const_iterator tok = tokens.begin();
assertTrue(tok != tokens.end());
assertEquals(" ", (tok++)->first);
assertEquals("Here", (tok++)->second);
assertEquals(" ", (tok++)->first);
assertEquals("are", (tok++)->second);
assertEquals("\t \t ", (tok++)->first);
assertEquals("some_words!", (tok++)->second);
cout << "test_wordtokenizer: SUCCESS" << endl;
return 0;
}
|
