1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
|
From fe993dc755f0d067280f438fd097b9db53edc237 Mon Sep 17 00:00:00 2001
From: Robert Ogden <robertogden@chromium.org>
Date: Wed, 30 Nov 2022 10:24:02 -0800
Subject: [PATCH 01/11] sentencepiece tokenization not supported
---
.../cc/text/tokenizers/tokenizer_utils.cc | 14 ++++----------
1 file changed, 4 insertions(+), 10 deletions(-)
diff --git a/third_party/tflite_support/src/tensorflow_lite_support/cc/text/tokenizers/tokenizer_utils.cc b/third_party/tflite_support/src/tensorflow_lite_support/cc/text/tokenizers/tokenizer_utils.cc
index 0ba0f370cc305..bd4f18fca7f08 100644
--- a/third_party/tflite_support/src/tensorflow_lite_support/cc/text/tokenizers/tokenizer_utils.cc
+++ b/third_party/tflite_support/src/tensorflow_lite_support/cc/text/tokenizers/tokenizer_utils.cc
@@ -23,7 +23,6 @@ limitations under the License.
#include "tensorflow_lite_support/cc/port/status_macros.h"
#include "tensorflow_lite_support/cc/text/tokenizers/bert_tokenizer.h"
#include "tensorflow_lite_support/cc/text/tokenizers/regex_tokenizer.h"
-#include "tensorflow_lite_support/cc/text/tokenizers/sentencepiece_tokenizer.h"
#include "tensorflow_lite_support/metadata/metadata_schema_generated.h"
namespace tflite {
@@ -32,7 +31,6 @@ namespace text {
namespace tokenizer {
using ::tflite::ProcessUnit;
-using ::tflite::SentencePieceTokenizerOptions;
using ::tflite::support::CreateStatusWithPayload;
using ::tflite::support::StatusOr;
using ::tflite::support::TfLiteSupportStatus;
@@ -77,14 +75,10 @@ StatusOr<std::unique_ptr<Tokenizer>> CreateTokenizerFromProcessUnit(
vocab_buffer.size());
}
case ProcessUnitOptions_SentencePieceTokenizerOptions: {
- const tflite::SentencePieceTokenizerOptions* options =
- tokenizer_process_unit->options_as<SentencePieceTokenizerOptions>();
- ASSIGN_OR_RETURN(absl::string_view model_buffer,
- CheckAndLoadFirstAssociatedFile(
- options->sentencePiece_model(), metadata_extractor));
- // TODO(b/160647204): Extract sentence piece model vocabulary
- return absl::make_unique<SentencePieceTokenizer>(model_buffer.data(),
- model_buffer.size());
+ return CreateStatusWithPayload(
+ absl::StatusCode::kInvalidArgument,
+ "Chromium does not support sentencepiece tokenization",
+ TfLiteSupportStatus::kMetadataInvalidTokenizerError);
}
case ProcessUnitOptions_RegexTokenizerOptions: {
const tflite::RegexTokenizerOptions* options =
--
2.49.0.777.g153de2bbd5-goog
|
