Searched refs:codepoint_index (Results 1 – 1 of 1) sorted by relevance
| /aosp_15_r20/external/libtextclassifier/native/utils/ |
| H A D | tokenizer.cc | 142 int codepoint_index = 0; in InternalTokenize() local
146 ++it, ++codepoint_index) { in InternalTokenize()
157 new_token = Token("", codepoint_index, codepoint_index); in InternalTokenize()
167 new_token = Token("", codepoint_index + 1, codepoint_index + 1); in InternalTokenize()
286 int codepoint_index = 0; in NumberTokenize() local
296 int codepoint_index, NumberTokenType token_type, in NumberTokenize()
300 new_token = Token("", codepoint_index, codepoint_index, in NumberTokenize()
310 int codepoint_index, UnicodeText::const_iterator it) { in NumberTokenize() argument
313 result->emplace_back("", codepoint_index, codepoint_index + 1); in NumberTokenize()
316 new_token = Token("", codepoint_index + 1, codepoint_index + 1); in NumberTokenize()
[all …]
|