Searched defs:subwords (Results 1 – 7 of 7) sorted by relevance
| /aosp_15_r20/external/libtextclassifier/native/utils/ |
| H A D | wordpiece_tokenizer.cc | 94 std::vector<std::string>* subwords, in NoTokenFound()
114 std::vector<std::string>* subwords, std::vector<int>* begin_offset, in AddWord()
135 std::vector<std::string>* subwords, in AddUnknownCharacter()
164 std::vector<std::string>* subwords, std::vector<int>* begin_offset, in TokenizeL2RGreedy()
214 std::vector<std::string>* subwords, std::vector<int>* begin_offset, in WordpieceTokenize()
238 std::vector<std::string>* subwords, std::vector<int>* begin_offset, in WordpieceTokenize()
|
| H A D | bert_tokenizer.cc | 98 std::vector<std::string>& subwords = result.subwords; in TokenizeIntoWordpieces() local
|
| /aosp_15_r20/external/tflite-support/tensorflow_lite_support/cc/test/text/ |
| H A D | regex_tokenizer_test.cc | 69 std::vector<std::string> subwords = {"good", "morning", "i'm", "your", in TEST() local
85 std::vector<std::string> subwords = {"good", "morning", "i'm", "your", in TEST() local
|
| /aosp_15_r20/external/tflite-support/tensorflow_lite_support/cc/text/tokenizers/ |
| H A D | tokenizer.h | 31 std::vector<std::string> subwords; member
|
| H A D | sentencepiece_tokenizer.h | 48 std::vector<std::string>& subwords = result.subwords; in Tokenize() local
|
| H A D | tokenizer_jni_lib.cc | 39 std::vector<std::string> subwords = tokenize_result.subwords; in nativeTokenize() local
|
| H A D | bert_tokenizer.cc | 65 std::vector<std::string>& subwords = result.subwords; in TokenizeWordpiece() local
|