File: tokenizer.cpp

package info (click to toggle)
aspell 0.60.6-1
  • links: PTS
  • area: main
  • in suites: lenny
  • size: 10,000 kB
  • ctags: 4,862
  • sloc: sh: 48,145; cpp: 22,153; perl: 1,546; ansic: 1,535; makefile: 684; sed: 16
file content (31 lines) | stat: -rw-r--r-- 701 bytes parent folder | download | duplicates (8) 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
 
// This file is part of The New Aspell
// Copyright (C) 2002 by Kevin Atkinson under the GNU LGPL license
// version 2.0 or 2.1.  You should have received a copy of the LGPL
// license along with this library if you did not you can find
// it at http://www.gnu.org/.

#include "tokenizer.hpp"
#include "convert.hpp"

namespace acommon
{

  Tokenizer::Tokenizer() 
    : word_begin(0), word_end(0), end(0), 
      begin_pos(0), end_pos(0),
      conv_(0) 
  {}

  Tokenizer::~Tokenizer()
  {}

  void Tokenizer::reset (FilterChar * begin, FilterChar * end) 
  {
    bool can_encode = conv_->encode(begin, end, buf_);
    assert(can_encode);
    end_pos = 0;
    word_end = begin;
    end = end;
  }

}