1 use crate::lexer::json_number_lit::JsonNumberLit;
2 use crate::lexer::lexer_impl::LexerError;
3 use crate::lexer::lexer_impl::LexerResult;
4 use crate::lexer::loc::Loc;
5 use crate::lexer::num_lit::NumLit;
6 use crate::lexer::str_lit::StrLit;
7 
8 #[derive(Clone, Debug, PartialEq)]
9 pub enum Token {
10     Ident(String),
11     Symbol(char),
12     // Protobuf tokenizer has separate tokens for int and float.
13     // Tokens do not include sign.
14     IntLit(u64),
15     FloatLit(f64),
16     JsonNumber(JsonNumberLit),
17     // including quotes
18     StrLit(StrLit),
19 }
20 
21 impl Token {
22     /// Back to original
format(&self) -> String23     pub fn format(&self) -> String {
24         match self {
25             &Token::Ident(ref s) => s.clone(),
26             &Token::Symbol(c) => c.to_string(),
27             &Token::IntLit(ref i) => i.to_string(),
28             &Token::StrLit(ref s) => s.quoted(),
29             &Token::FloatLit(ref f) => f.to_string(),
30             &Token::JsonNumber(ref f) => f.to_string(),
31         }
32     }
33 
to_num_lit(&self) -> LexerResult<NumLit>34     pub fn to_num_lit(&self) -> LexerResult<NumLit> {
35         match self {
36             &Token::IntLit(i) => Ok(NumLit::U64(i)),
37             &Token::FloatLit(f) => Ok(NumLit::F64(f)),
38             _ => Err(LexerError::IncorrectInput),
39         }
40     }
41 }
42 
43 #[derive(Clone)]
44 pub struct TokenWithLocation {
45     pub token: Token,
46     pub loc: Loc,
47 }
48