Searched defs:token_id (Results 1 – 8 of 8) sorted by relevance
226 std::size_t token_id, char_type const* targetstate) in add_token()240 std::size_t token_id, char_type const* targetstate) in add_token()
227 #define EXECUTORCH_END_PROF(token_id) \ argument259 #define EXECUTORCH_END_PROF(token_id) \ argument
74 void end_profiling(uint32_t token_id) { in end_profiling()
32 enum token_id enum
51 test(Lexer& lex, Char const* input, std::size_t token_id = 0, in test()
254 int token_id = 0; in Preprocess() local
375 TokenId token_id = token_info.id << 1; in Parse() local
472 bool ActionsSuggestions::EmbedTokenId(const int32 token_id, in EmbedTokenId()