File: tokenizer.cc

package info (click to toggle)
chromium 138.0.7204.183-1
  • links: PTS, VCS
  • area: main
  • in suites: trixie
  • size: 6,071,908 kB
  • sloc: cpp: 34,937,088; ansic: 7,176,967; javascript: 4,110,704; python: 1,419,953; asm: 946,768; xml: 739,971; pascal: 187,324; sh: 89,623; perl: 88,663; objc: 79,944; sql: 50,304; cs: 41,786; fortran: 24,137; makefile: 21,806; php: 13,980; tcl: 13,166; yacc: 8,925; ruby: 7,485; awk: 3,720; lisp: 3,096; lex: 1,327; ada: 727; jsp: 228; sed: 36
file content (39 lines) | stat: -rw-r--r-- 1,171 bytes parent folder | download | duplicates (6) 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
// Copyright 2024 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "chrome/renderer/accessibility/phrase_segmentation/tokenizer.h"

#include <cstdint>
#include <string>
#include <string_view>
#include <utility>
#include <vector>

#include "base/i18n/break_iterator.h"
#include "base/numerics/safe_conversions.h"

namespace {
bool is_all_spaces(std::u16string_view str) {
  return str.find_first_not_of(u' ') == std::string::npos;
}
}  // namespace

std::vector<std::pair<int, int>> Tokenizer::Tokenize(
    const std::u16string& text) {
  std::vector<std::pair<int, int>> ret;
  base::i18n::BreakIterator iter(text, base::i18n::BreakIterator::BREAK_WORD);
  if (!iter.Init()) {
    return ret;
  }

  // iter.Advance() returns false if we've run past end of the text.
  while (iter.Advance()) {
    // Tokenize words as well as punctuations.
    if (iter.IsWord() || !is_all_spaces(iter.GetString())) {
      ret.emplace_back(base::checked_cast<int>(iter.prev()) /* start index */,
                       base::checked_cast<int>(iter.pos()) /* end index */);
    }
  }
  return ret;
}