Searched defs:TokenSpan (Results 1 – 1 of 1) sorted by relevance
145 struct TokenSpan { struct146 static const TokenSpan kInvalid; argument148 TokenSpan() : first(kInvalidIndex), second(kInvalidIndex) {} in TokenSpan() argument150 TokenSpan(TokenIndex start, TokenIndex end) : first(start), second(end) {} in TokenSpan() argument153 explicit TokenSpan(int token_index) in TokenSpan() function156 TokenSpan& operator=(const TokenSpan& other) = default; argument180 TokenSpan Expand(int num_tokens_left, int num_tokens_right) const { in Expand() argument188 // Pretty-printing function for TokenSpan. argument