File: tokenizer.cpp

package info (click to toggle)
aspell 0.60.2%2B20050121-2
  • links: PTS
  • area: main
  • in suites: sarge
  • size: 7,856 kB
  • ctags: 4,781
  • sloc: cpp: 22,068; sh: 9,197; perl: 1,541; ansic: 1,530; makefile: 672; sed: 16
file content (31 lines) | stat: -rw-r--r-- 701 bytes parent folder | download | duplicates (8) 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
 
// This file is part of The New Aspell
// Copyright (C) 2002 by Kevin Atkinson under the GNU LGPL license
// version 2.0 or 2.1.  You should have received a copy of the LGPL
// license along with this library if you did not you can find
// it at http://www.gnu.org/.

#include "tokenizer.hpp"
#include "convert.hpp"

namespace acommon
{

  Tokenizer::Tokenizer() 
    : word_begin(0), word_end(0), end(0), 
      begin_pos(0), end_pos(0),
      conv_(0) 
  {}

  Tokenizer::~Tokenizer()
  {}

  void Tokenizer::reset (FilterChar * begin, FilterChar * end) 
  {
    bool can_encode = conv_->encode(begin, end, buf_);
    assert(can_encode);
    end_pos = 0;
    word_end = begin;
    end = end;
  }

}