Searched refs:split_on_script_change (Results 1 – 3 of 3) sorted by relevance
| /aosp_15_r20/external/libtextclassifier/native/utils/ |
| H A D | tokenizer.h | 58 const bool split_on_script_change,
67 const bool split_on_script_change, in Tokenizer() argument
70 internal_tokenizer_codepoint_ranges, split_on_script_change, in Tokenizer()
76 const bool split_on_script_change) in Tokenizer() argument
79 split_on_script_change,
|
| H A D | tokenizer.cc | 34 const bool split_on_script_change, in Tokenizer() argument
39 split_on_script_change_(split_on_script_change), in Tokenizer()
55 if (type_ == TokenizationType_MIXED && split_on_script_change) { in Tokenizer()
|
| H A D | tokenizer_test.cc | 36 const bool split_on_script_change, in TestingTokenizer() argument
40 internal_tokenizer_codepoint_ranges, split_on_script_change, in TestingTokenizer()
52 const bool split_on_script_change, in TestingTokenizerProxy() argument
80 type, &unilib_, configs_fb, internal_configs_fb, split_on_script_change, in TestingTokenizerProxy()
|