Searched refs:tokenization_codepoint_config (Results 1 – 9 of 9) sorted by relevance
33 if (options->tokenization_codepoint_config() != nullptr) { in BuildTokenizer()35 options->tokenization_codepoint_config()->begin(), in BuildTokenizer()36 options->tokenization_codepoint_config()->end()); in BuildTokenizer()47 options->tokenization_codepoint_config() != nullptr && in BuildTokenizer()
48 if (options->tokenization_codepoint_config() != nullptr) { in CreateTokenizer()50 options->tokenization_codepoint_config()->begin(), in CreateTokenizer()51 options->tokenization_codepoint_config()->end()); in CreateTokenizer()61 options->tokenization_codepoint_config() != nullptr && in CreateTokenizer()
146 tokenization_codepoint_config:[TokenizationCodepointRange];
372 options.tokenization_codepoint_config.emplace_back( in TEST_F()374 auto& config = options.tokenization_codepoint_config.back(); in TEST_F()446 options.tokenization_codepoint_config.emplace_back( in TEST_F()448 auto& config = options.tokenization_codepoint_config.back(); in TEST_F()596 options.tokenization_codepoint_config.emplace_back( in TEST_F()598 auto& config = options.tokenization_codepoint_config.back(); in TEST_F()
34 if (options->tokenization_codepoint_config() != nullptr) { in BuildTokenizer()36 options->tokenization_codepoint_config()->begin(), in BuildTokenizer()37 options->tokenization_codepoint_config()->end()); in BuildTokenizer()47 options->tokenization_codepoint_config() != nullptr && in BuildTokenizer()
365 tokenization_codepoint_config:[TokenizationCodepointRange];743 tokenization_codepoint_config:[TokenizationCodepointRange];
1925 …<std::unique_ptr<libtextclassifier3::TokenizationCodepointRangeT>> tokenization_codepoint_config{};1950 …ers::Offset<libtextclassifier3::TokenizationCodepointRange>> *tokenization_codepoint_config() cons… in tokenization_codepoint_config() function1964 verifier.VerifyVector(tokenization_codepoint_config()) && in Verify()1965 verifier.VerifyVectorOfTables(tokenization_codepoint_config()) && in Verify()1987 …tbuffers::Offset<libtextclassifier3::TokenizationCodepointRange>>> tokenization_codepoint_config) { in add_tokenization_codepoint_config()1988 …ddOffset(GrammarTokenizerOptions::VT_TOKENIZATION_CODEPOINT_CONFIG, tokenization_codepoint_config); in add_tokenization_codepoint_config()2011 …uffers::Offset<libtextclassifier3::TokenizationCodepointRange>>> tokenization_codepoint_config = 0,2016 builder_.add_tokenization_codepoint_config(tokenization_codepoint_config);2027 …ers::Offset<libtextclassifier3::TokenizationCodepointRange>> *tokenization_codepoint_config = null…2030 …= tokenization_codepoint_config ? _fbb.CreateVector<flatbuffers::Offset<libtextclassifier3::Tokeni…[all …]
686 …<std::unique_ptr<libtextclassifier3::TokenizationCodepointRangeT>> tokenization_codepoint_config{};711 …ers::Offset<libtextclassifier3::TokenizationCodepointRange>> *tokenization_codepoint_config() cons… in tokenization_codepoint_config() function725 verifier.VerifyVector(tokenization_codepoint_config()) && in Verify()726 verifier.VerifyVectorOfTables(tokenization_codepoint_config()) && in Verify()748 …tbuffers::Offset<libtextclassifier3::TokenizationCodepointRange>>> tokenization_codepoint_config) { in add_tokenization_codepoint_config()749 …ddOffset(ActionsTokenizerOptions::VT_TOKENIZATION_CODEPOINT_CONFIG, tokenization_codepoint_config); in add_tokenization_codepoint_config()772 …uffers::Offset<libtextclassifier3::TokenizationCodepointRange>>> tokenization_codepoint_config = 0,777 builder_.add_tokenization_codepoint_config(tokenization_codepoint_config);788 …ers::Offset<libtextclassifier3::TokenizationCodepointRange>> *tokenization_codepoint_config = null…791 …= tokenization_codepoint_config ? _fbb.CreateVector<flatbuffers::Offset<libtextclassifier3::Tokeni…[all …]
106 options.tokenization_codepoint_config.emplace_back( in BuildFeatureProcessor()108 auto& config = options.tokenization_codepoint_config.back(); in BuildFeatureProcessor()