1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
|
// This file is part of The New Aspell
// Copyright (C) 2002 by Kevin Atkinson under the GNU LGPL license
// version 2.0 or 2.1. You should have received a copy of the LGPL
// license along with this library if you did not you can find
// it at http://www.gnu.org/.
#include "tokenizer.hpp"
#include "convert.hpp"
namespace acommon
{
Tokenizer::Tokenizer()
: word_begin(0), word_end(0), end(0),
begin_pos(0), end_pos(0),
conv_(0)
{}
Tokenizer::~Tokenizer()
{}
void Tokenizer::reset (FilterChar * start, FilterChar * stop)
{
bool can_encode = conv_->encode(start, stop, buf_);
assert(can_encode);
end_pos = 0;
word_end = start;
end = stop;
}
}
|
