1 use crate::lexer::json_number_lit::JsonNumberLit; 2 use crate::lexer::lexer_impl::LexerError; 3 use crate::lexer::lexer_impl::LexerResult; 4 use crate::lexer::loc::Loc; 5 use crate::lexer::num_lit::NumLit; 6 use crate::lexer::str_lit::StrLit; 7 8 #[derive(Clone, Debug, PartialEq)] 9 pub enum Token { 10 Ident(String), 11 Symbol(char), 12 // Protobuf tokenizer has separate tokens for int and float. 13 // Tokens do not include sign. 14 IntLit(u64), 15 FloatLit(f64), 16 JsonNumber(JsonNumberLit), 17 // including quotes 18 StrLit(StrLit), 19 } 20 21 impl Token { 22 /// Back to original format(&self) -> String23 pub fn format(&self) -> String { 24 match self { 25 &Token::Ident(ref s) => s.clone(), 26 &Token::Symbol(c) => c.to_string(), 27 &Token::IntLit(ref i) => i.to_string(), 28 &Token::StrLit(ref s) => s.quoted(), 29 &Token::FloatLit(ref f) => f.to_string(), 30 &Token::JsonNumber(ref f) => f.to_string(), 31 } 32 } 33 to_num_lit(&self) -> LexerResult<NumLit>34 pub fn to_num_lit(&self) -> LexerResult<NumLit> { 35 match self { 36 &Token::IntLit(i) => Ok(NumLit::U64(i)), 37 &Token::FloatLit(f) => Ok(NumLit::F64(f)), 38 _ => Err(LexerError::IncorrectInput), 39 } 40 } 41 } 42 43 #[derive(Clone)] 44 pub struct TokenWithLocation { 45 pub token: Token, 46 pub loc: Loc, 47 } 48