| /aosp_15_r20/external/icing/icing/tokenization/ |
| H A D | trigram-tokenizer_test.cc | 39 TrigramTokenizer tokenizer; in TEST() local
52 TrigramTokenizer tokenizer; in TEST() local
66 TrigramTokenizer tokenizer; in TEST() local
81 TrigramTokenizer tokenizer; in TEST() local
94 TrigramTokenizer tokenizer; in TEST() local
137 TrigramTokenizer tokenizer; in TEST() local
144 TrigramTokenizer tokenizer; in TEST() local
154 TrigramTokenizer tokenizer; in TEST() local
165 TrigramTokenizer tokenizer; in TEST() local
175 TrigramTokenizer tokenizer; in TEST() local
[all …]
|
| /aosp_15_r20/external/libtextclassifier/native/utils/ |
| H A D | tokenizer_test.cc | 126 TestingTokenizerProxy tokenizer(TokenizationType_INTERNAL_TOKENIZER, configs, in TEST() local
173 TestingTokenizerProxy tokenizer(TokenizationType_INTERNAL_TOKENIZER, configs, in TEST() local
208 TestingTokenizerProxy tokenizer(TokenizationType_INTERNAL_TOKENIZER, configs, in TEST() local
344 TestingTokenizerProxy tokenizer(TokenizationType_INTERNAL_TOKENIZER, configs, in TEST() local
375 TestingTokenizerProxy tokenizer(TokenizationType_ICU, {}, {}, in TEST() local
395 TestingTokenizerProxy tokenizer(TokenizationType_ICU, {}, {}, in TEST() local
423 TestingTokenizerProxy tokenizer(TokenizationType_ICU, {}, {}, in TEST() local
441 TestingTokenizerProxy tokenizer(TokenizationType_ICU, {}, {}, in TEST() local
489 TestingTokenizerProxy tokenizer(TokenizationType_MIXED, configs, in TEST() local
520 TestingTokenizerProxy tokenizer(TokenizationType_INTERNAL_TOKENIZER, in TEST() local
[all …]
|
| H A D | bert_tokenizer_test.cc | 32 void AssertTokenizerResults(std::unique_ptr<BertTokenizer> tokenizer) { in AssertTokenizerResults()
41 auto tokenizer = in TEST() local
48 auto tokenizer = in TEST() local
60 auto tokenizer = std::make_unique<BertTokenizer>(vocab); in TEST() local
66 auto tokenizer = in TEST() local
76 auto tokenizer = in TEST() local
89 auto tokenizer = in TEST() local
110 auto tokenizer = std::make_unique<BertTokenizer>(vocab); in TEST() local
124 auto tokenizer = std::make_unique<BertTokenizer>(vocab); in TEST() local
145 auto tokenizer = std::make_unique<BertTokenizer>(vocab); in TEST() local
[all …]
|
| /aosp_15_r20/system/core/init/ |
| H A D | tokenizer_test.cpp | 60 TEST(tokenizer, null) { in TEST() argument
64 TEST(tokenizer, simple_oneline) { in TEST() argument
68 TEST(tokenizer, simple_multiline) { in TEST() argument
72 TEST(tokenizer, preceding_space) { in TEST() argument
78 TEST(tokenizer, comments) { in TEST() argument
89 TEST(tokenizer, control_chars) { in TEST() argument
118 TEST(tokenizer, cr_lf) { in TEST() argument
139 TEST(tokenizer, quoted) { in TEST() argument
|
| /aosp_15_r20/external/ktfmt/core/src/test/java/com/facebook/ktfmt/format/ |
| H A D | TokenizerTest.kt | 37 val tokenizer = Tokenizer(code, file) in PsiWhiteSpace are split to newlines and maximal-length whitespaces() constant
63 val tokenizer = Tokenizer(code, file) in Strings are returns as a single token() constant
106 val tokenizer = Tokenizer(code, file) in Token index is advanced after a string token() constant
133 val tokenizer = Tokenizer(code, file) in Context receivers are parsed correctly() constant
|
| /aosp_15_r20/external/parameter-framework/upstream/test/tokenizer/ |
| H A D | Test.cpp | 49 Tokenizer tokenizer("a bcd ef"); variable
58 Tokenizer tokenizer(""); variable
67 Tokenizer tokenizer(" a \n\t bc "); variable
77 Tokenizer tokenizer("/a/bcd/ef g/h/", "/"); variable
88 Tokenizer tokenizer("", Tokenizer::defaultDelimiters, false); variable
97 Tokenizer tokenizer(",", ",", false); variable
106 Tokenizer tokenizer(" a b \nc d ", Tokenizer::defaultDelimiters, false); variable
|
| /aosp_15_r20/external/protobuf/csharp/src/Google.Protobuf/ |
| H A D | JsonParser.cs | 97 …private static void MergeWrapperField(JsonParser parser, IMessage message, JsonTokenizer tokenizer) in MergeWrapperField()
151 private void Merge(IMessage message, JsonTokenizer tokenizer) in Merge()
220 private void MergeField(IMessage message, FieldDescriptor field, JsonTokenizer tokenizer) in MergeField()
253 … private void MergeRepeatedField(IMessage message, FieldDescriptor field, JsonTokenizer tokenizer) in MergeRepeatedField()
279 private void MergeMapField(IMessage message, FieldDescriptor field, JsonTokenizer tokenizer) in MergeMapField()
326 private object ParseSingleValue(FieldDescriptor field, JsonTokenizer tokenizer) in ParseSingleValue()
445 private void MergeStructValue(IMessage message, JsonTokenizer tokenizer) in MergeStructValue()
487 private void MergeStruct(IMessage message, JsonTokenizer tokenizer) in MergeStruct()
500 private void MergeAny(IMessage message, JsonTokenizer tokenizer) in MergeAny()
565 private void MergeWellKnownTypeAnyBody(IMessage body, JsonTokenizer tokenizer) in MergeWellKnownTypeAnyBody()
|
| /aosp_15_r20/external/cronet/third_party/protobuf/csharp/src/Google.Protobuf/ |
| H A D | JsonParser.cs | 97 …private static void MergeWrapperField(JsonParser parser, IMessage message, JsonTokenizer tokenizer) in MergeWrapperField()
151 private void Merge(IMessage message, JsonTokenizer tokenizer) in Merge()
220 private void MergeField(IMessage message, FieldDescriptor field, JsonTokenizer tokenizer) in MergeField()
253 … private void MergeRepeatedField(IMessage message, FieldDescriptor field, JsonTokenizer tokenizer) in MergeRepeatedField()
279 private void MergeMapField(IMessage message, FieldDescriptor field, JsonTokenizer tokenizer) in MergeMapField()
326 private object ParseSingleValue(FieldDescriptor field, JsonTokenizer tokenizer) in ParseSingleValue()
445 private void MergeStructValue(IMessage message, JsonTokenizer tokenizer) in MergeStructValue()
487 private void MergeStruct(IMessage message, JsonTokenizer tokenizer) in MergeStruct()
500 private void MergeAny(IMessage message, JsonTokenizer tokenizer) in MergeAny()
565 private void MergeWellKnownTypeAnyBody(IMessage body, JsonTokenizer tokenizer) in MergeWellKnownTypeAnyBody()
|
| /aosp_15_r20/external/doclava/src/com/google/doclava/apicheck/ |
| H A D | ApiFile.java | 70 final Tokenizer tokenizer = new Tokenizer(filename, (new String(buf, 0, size)).toCharArray()); in parseApi() local
91 private static void parsePackage(ApiInfo api, Tokenizer tokenizer) in parsePackage()
116 private static void parseClass(ApiInfo api, PackageInfo pkg, Tokenizer tokenizer, String token) in parseClass()
232 private static void parseConstructor(Tokenizer tokenizer, ClassInfo cl, String token) in parseConstructor()
287 private static void parseMethod(Tokenizer tokenizer, ClassInfo cl, String token) in parseMethod()
378 private static void parseField(Tokenizer tokenizer, ClassInfo cl, String token, boolean isEnum) in parseField()
507 private static void parseTypeParameterList(Tokenizer tokenizer, in parseTypeParameterList()
546 private static void parseParameterList(Tokenizer tokenizer, AbstractMethodInfo method, in parseParameterList()
583 private static String parseThrows(Tokenizer tokenizer, AbstractMethodInfo method) in parseThrows()
615 public static void assertIdent(Tokenizer tokenizer, String token) throws ApiParseException { in assertIdent()
|
| /aosp_15_r20/external/cronet/third_party/boringssl/src/pki/ |
| H A D | pem_unittest.cc | 20 PEMTokenizer tokenizer(string_piece, accepted_types); in TEST() local
38 PEMTokenizer tokenizer(string_piece, accepted_types); in TEST() local
56 PEMTokenizer tokenizer(string_piece, accepted_types); in TEST() local
73 PEMTokenizer tokenizer(string_piece, accepted_types); in TEST() local
98 PEMTokenizer tokenizer(string_piece, accepted_types); in TEST() local
122 PEMTokenizer tokenizer(string_piece, accepted_types); in TEST() local
139 PEMTokenizer tokenizer(string_piece, accepted_types); in TEST() local
160 PEMTokenizer tokenizer(string_piece, accepted_types); in TEST() local
|
| /aosp_15_r20/external/protobuf/python/google/protobuf/ |
| H A D | text_format.py | 866 def _MergeField(self, tokenizer, message): argument
1009 def _ConsumeAnyTypeUrl(self, tokenizer): argument
1024 def _MergeMessageField(self, tokenizer, message, field): argument
1088 def _MergeScalarField(self, tokenizer, message, field): argument
1164 def _SkipFieldContents(tokenizer): argument
1186 def _SkipField(tokenizer): argument
1209 def _SkipFieldMessage(tokenizer): argument
1228 def _SkipFieldValue(tokenizer): argument
1250 def _SkipRepeatedFieldValue(tokenizer): argument
1608 def _ConsumeInt32(tokenizer): argument
[all …]
|
| /aosp_15_r20/external/cronet/third_party/protobuf/python/google/protobuf/ |
| H A D | text_format.py | 866 def _MergeField(self, tokenizer, message): argument
1009 def _ConsumeAnyTypeUrl(self, tokenizer): argument
1024 def _MergeMessageField(self, tokenizer, message, field): argument
1088 def _MergeScalarField(self, tokenizer, message, field): argument
1164 def _SkipFieldContents(tokenizer): argument
1186 def _SkipField(tokenizer): argument
1209 def _SkipFieldMessage(tokenizer): argument
1228 def _SkipFieldValue(tokenizer): argument
1250 def _SkipRepeatedFieldValue(tokenizer): argument
1608 def _ConsumeInt32(tokenizer): argument
[all …]
|
| /aosp_15_r20/external/tflite-support/tensorflow_lite_support/cc/test/text/ |
| H A D | regex_tokenizer_test.cc | 26 namespace tokenizer { namespace
46 auto tokenizer = in TEST() local
57 auto tokenizer = in TEST() local
67 auto tokenizer = in TEST() local
82 auto tokenizer = in TEST() local
101 auto tokenizer = in TEST() local
120 auto tokenizer = in TEST() local
|
| /aosp_15_r20/external/cronet/third_party/protobuf/src/google/protobuf/io/ |
| H A D | tokenizer_unittest.cc | 254 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
298 Tokenizer tokenizer(&input, &error_collector); in TEST_1D() local
339 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
348 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
473 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
546 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
613 Tokenizer tokenizer(&input, &error_collector); in TEST_1D() local
773 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
1136 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
1162 Tokenizer tokenizer(&input, &error_collector); in TEST_1D() local
|
| /aosp_15_r20/external/protobuf/src/google/protobuf/io/ |
| H A D | tokenizer_unittest.cc | 254 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
298 Tokenizer tokenizer(&input, &error_collector); in TEST_1D() local
339 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
348 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
473 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
546 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
613 Tokenizer tokenizer(&input, &error_collector); in TEST_1D() local
773 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
1136 Tokenizer tokenizer(&input, &error_collector); in TEST_2D() local
1162 Tokenizer tokenizer(&input, &error_collector); in TEST_1D() local
|
| /aosp_15_r20/external/perfetto/src/protozero/filtering/ |
| H A D | message_tokenizer_unittest.cc | 70 MessageTokenizer tokenizer; in TEST() local
123 MessageTokenizer tokenizer; in TEST() local
147 MessageTokenizer tokenizer; in TEST() local
157 MessageTokenizer tokenizer; in TEST() local
169 MessageTokenizer tokenizer; in TEST() local
|
| /aosp_15_r20/external/apache-commons-lang/src/test/java/org/apache/commons/lang3/text/ |
| H A D | StrTokenizerTest.java | 46 private void checkClone(final StrTokenizer tokenizer) { in checkClone()
47 assertNotSame(StrTokenizer.getCSVInstance(), tokenizer); in checkClone() local
48 assertNotSame(StrTokenizer.getTSVInstance(), tokenizer); in checkClone() local
534 void testEmpty(final StrTokenizer tokenizer) { in testEmpty()
591 final StrTokenizer tokenizer = new StrTokenizer((char[]) null); in testCloneNull() local
606 final StrTokenizer tokenizer = new StrTokenizer(input); in testCloneReset() local
761 void testXSVAbc(final StrTokenizer tokenizer) { in testXSVAbc()
|
| /aosp_15_r20/external/executorch/examples/mediatek/executor_runner/ |
| H A D | mtk_llama_executor_runner.cpp | 179 const std::unique_ptr<Tokenizer>& tokenizer, in digest_prompt()
230 const std::unique_ptr<Tokenizer>& tokenizer, in gen_response()
303 const std::unique_ptr<Tokenizer>& tokenizer, in inference()
324 std::unique_ptr<Tokenizer> tokenizer; in load_tokenizer() local
370 const auto tokenizer = load_tokenizer(); in main() local
|
| H A D | mtk_llama_runner.cpp | 188 const std::unique_ptr<Tokenizer>& tokenizer, in digest_prompt()
239 const std::unique_ptr<Tokenizer>& tokenizer, in gen_response()
312 const std::unique_ptr<Tokenizer>& tokenizer, in inference()
333 std::unique_ptr<Tokenizer> tokenizer; in load_tokenizer() local
|
| /aosp_15_r20/external/sdv/vsomeip/third_party/boost/tokenizer/include/boost/ |
| D | tokenizer.hpp | 32 class tokenizer { class
53 tokenizer(Iterator first, Iterator last, in tokenizer() function in boost::tokenizer
58 tokenizer(const Container& c) in tokenizer() function in boost::tokenizer
62 tokenizer(const Container& c,const TokenizerFunc& f) in tokenizer() function in boost::tokenizer
|
| /aosp_15_r20/frameworks/base/tools/aapt2/util/ |
| H A D | Util_test.cpp | 56 auto tokenizer = util::Tokenize(StringPiece("this| is|the|end"), '|'); in TEST() local
70 auto tokenizer = util::Tokenize(StringPiece(""), '|'); in TEST() local
79 auto tokenizer = util::Tokenize(StringPiece("one."), '.'); in TEST() local
88 auto tokenizer = util::Tokenize(StringPiece("none"), '.'); in TEST() local
|
| /aosp_15_r20/external/perfetto/src/trace_processor/importers/proto/ |
| H A D | proto_trace_tokenizer_unittest.cc | 41 ProtoTraceTokenizer tokenizer; in TEST() local
63 ProtoTraceTokenizer tokenizer; in TEST() local
93 ProtoTraceTokenizer tokenizer; in TEST() local
125 ProtoTraceTokenizer tokenizer; in TEST() local
|
| /aosp_15_r20/external/aws-sdk-java-v2/codegen/src/test/java/software/amazon/awssdk/codegen/poet/rules2/ |
| H A D | TokenizerTest.java | 27 Tokenizer tokenizer = new Tokenizer("{foobar}"); in recognizesReferenceExpression() local
35 Tokenizer tokenizer = new Tokenizer("foobar[123]"); in recognizesIndexedExpression() local
46 Tokenizer tokenizer = new Tokenizer("{url#authority}"); in recognizesNamedAccessExpression() local
|
| /aosp_15_r20/external/deqp-deps/amber/src/vkscript/ |
| D | parser.cc | 43 std::string Parser::make_error(const Tokenizer& tokenizer, in make_error()
152 Tokenizer tokenizer(section.contents); in ProcessRequireBlock() local
278 Tokenizer tokenizer(section.contents); in ProcessIndicesBlock() local
321 Tokenizer tokenizer(section.contents); in ProcessVertexDataBlock() local
|
| /aosp_15_r20/external/protobuf/java/core/src/main/java/com/google/protobuf/ |
| H A D | TextFormat.java | 1554 private void detectSilentMarker(Tokenizer tokenizer) { in detectSilentMarker()
1787 final Tokenizer tokenizer = new Tokenizer(input); in merge() local
1801 final Tokenizer tokenizer, in mergeField()
1816 final Tokenizer tokenizer, in mergeField()
1974 final Tokenizer tokenizer, in consumeFieldValues()
2016 final Tokenizer tokenizer, in consumeFieldValue()
2186 final Tokenizer tokenizer, in mergeAnyFieldValue()
2246 private void skipField(Tokenizer tokenizer) throws ParseException { in skipField()
2279 private void skipFieldMessage(Tokenizer tokenizer) throws ParseException { in skipFieldMessage()
2294 private void skipFieldValue(Tokenizer tokenizer) throws ParseException { in skipFieldValue()
|