File: tokenized_string.h

package info (click to toggle)
chromium 139.0.7258.127-1
  • links: PTS, VCS
  • area: main
  • in suites:
  • size: 6,122,068 kB
  • sloc: cpp: 35,100,771; ansic: 7,163,530; javascript: 4,103,002; python: 1,436,920; asm: 946,517; xml: 746,709; pascal: 187,653; perl: 88,691; sh: 88,436; objc: 79,953; sql: 51,488; cs: 44,583; fortran: 24,137; makefile: 22,147; tcl: 15,277; php: 13,980; yacc: 8,984; ruby: 7,485; awk: 3,720; lisp: 3,096; lex: 1,327; ada: 727; jsp: 228; sed: 36
file content (54 lines) | stat: -rw-r--r-- 1,492 bytes parent folder | download | duplicates (6)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
// Copyright 2019 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef CHROMEOS_ASH_COMPONENTS_STRING_MATCHING_TOKENIZED_STRING_H_
#define CHROMEOS_ASH_COMPONENTS_STRING_MATCHING_TOKENIZED_STRING_H_

#include <string_view>
#include <vector>

#include "ui/gfx/range/range.h"

namespace ash::string_matching {

// TokenizedString takes a string and breaks it down into token words.
class TokenizedString {
 public:
  enum class Mode {
    // Break words into tokens at camel case, alpha/num, and white space
    // boundaries.
    kCamelCase,
    // Break words into tokens at white space.
    kWords,
  };

  typedef std::vector<std::u16string> Tokens;
  typedef std::vector<gfx::Range> Mappings;

  explicit TokenizedString(std::u16string text, Mode mode = Mode::kCamelCase);

  TokenizedString(const TokenizedString&) = delete;
  TokenizedString& operator=(const TokenizedString&) = delete;

  ~TokenizedString();

  const std::u16string& text() const { return text_; }
  const Tokens& tokens() const { return tokens_; }
  const Mappings& mappings() const { return mappings_; }

 private:
  void Tokenize();
  void TokenizeWords();

  // Input text.
  const std::u16string text_;

  // Broken down tokens and the index mapping of tokens in original string.
  Tokens tokens_;
  Mappings mappings_;
};

}  // namespace ash::string_matching

#endif  // CHROMEOS_ASH_COMPONENTS_STRING_MATCHING_TOKENIZED_STRING_H_