use crate::lexer::Span; use std::error::Error; use std::fmt::{format, Debug, Display, Formatter}; #[derive(PartialEq, Clone, Debug)] pub enum ScriptTokenType { // Structural Tokens LeftParen, RightParen, LeftBrace, RightBrace, Comma, Dot, Semicolon, // Unary Operators Bang, Minus, // Binary Operators Asterisk, Slash, Plus, BangEqual, Equal, EqualEqual, Greater, GreaterEqual, Less, LessEqual, DoublePipe, DoubleAmpersand, Modulo, Caret, // Literals Identifier(String), String(String), OwnedString(String), Integer(i64), Float(f64), Boolean(bool), // Keywords Class, Else, Function, For, If, Null, Print, Return, Super, This, Let, While, Export, Import, Alias, From, Typeof, Finally, // Misc Eof, } impl ScriptTokenType { pub fn len(&self) -> usize { match self { ScriptTokenType::LeftParen => 1, ScriptTokenType::RightParen => 1, ScriptTokenType::LeftBrace => 2, ScriptTokenType::RightBrace => 2, ScriptTokenType::Comma => 1, ScriptTokenType::Dot => 1, ScriptTokenType::Minus => 1, ScriptTokenType::Plus => 1, ScriptTokenType::Semicolon => 1, ScriptTokenType::Slash => 1, ScriptTokenType::Asterisk => 1, ScriptTokenType::Bang => 1, ScriptTokenType::BangEqual => 2, ScriptTokenType::Equal => 1, ScriptTokenType::EqualEqual => 2, ScriptTokenType::Greater => 1, ScriptTokenType::GreaterEqual => 2, ScriptTokenType::Less => 1, ScriptTokenType::LessEqual => 2, ScriptTokenType::DoublePipe => 2, ScriptTokenType::DoubleAmpersand => 2, ScriptTokenType::Modulo => 1, ScriptTokenType::Caret => 1, ScriptTokenType::Identifier(value) => value.len(), ScriptTokenType::String(value) => value.len() + 2, ScriptTokenType::OwnedString(value) => value.len() + 2, ScriptTokenType::Integer(value) => format!("{}", value).len(), ScriptTokenType::Float(value) => format!("{}", value).len(), ScriptTokenType::Boolean(value) => { if *value { 4 } else { 5 } } ScriptTokenType::Class => 6, ScriptTokenType::Else => 4, ScriptTokenType::Function => 2, ScriptTokenType::For => 3, ScriptTokenType::If => 2, ScriptTokenType::Null => 4, ScriptTokenType::Print => 5, ScriptTokenType::Return => 6, ScriptTokenType::Super => 5, ScriptTokenType::This => 4, ScriptTokenType::Let => 3, ScriptTokenType::While => 5, ScriptTokenType::Export => 6, ScriptTokenType::Import => 6, ScriptTokenType::Alias => 2, ScriptTokenType::From => 4, ScriptTokenType::Eof => 0, ScriptTokenType::Typeof => 6, ScriptTokenType::Finally => 7, } } } impl Display for ScriptTokenType { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { ScriptTokenType::LeftParen => write!(f, "("), ScriptTokenType::RightParen => write!(f, ")"), ScriptTokenType::LeftBrace => write!(f, "{{"), ScriptTokenType::RightBrace => write!(f, "}}"), ScriptTokenType::Comma => write!(f, ","), ScriptTokenType::Dot => write!(f, "."), ScriptTokenType::Minus => write!(f, "-"), ScriptTokenType::Plus => write!(f, "+"), ScriptTokenType::Semicolon => write!(f, ";"), ScriptTokenType::Slash => write!(f, "/"), ScriptTokenType::Asterisk => write!(f, "*"), ScriptTokenType::Bang => write!(f, "!"), ScriptTokenType::BangEqual => write!(f, "!="), ScriptTokenType::Equal => write!(f, "="), ScriptTokenType::EqualEqual => write!(f, "=="), ScriptTokenType::Greater => write!(f, ">"), ScriptTokenType::GreaterEqual => write!(f, ">="), ScriptTokenType::Less => write!(f, "<"), ScriptTokenType::LessEqual => write!(f, "<="), ScriptTokenType::DoublePipe => write!(f, "||"), ScriptTokenType::DoubleAmpersand => write!(f, "&&"), ScriptTokenType::Modulo => write!(f, "%"), ScriptTokenType::Caret => write!(f, "^"), ScriptTokenType::Identifier(value) => write!(f, "{}", value), ScriptTokenType::String(value) => write!(f, "{}", value), ScriptTokenType::OwnedString(value) => write!(f, "{}", value), ScriptTokenType::Integer(value) => write!(f, "{}", value), ScriptTokenType::Float(value) => write!(f, "{}", value), ScriptTokenType::Boolean(value) => write!(f, "{}", value), ScriptTokenType::Class => write!(f, "struct"), ScriptTokenType::Else => write!(f, "else"), ScriptTokenType::Function => write!(f, "fn"), ScriptTokenType::For => write!(f, "for"), ScriptTokenType::If => write!(f, "if"), ScriptTokenType::Null => write!(f, "null"), ScriptTokenType::Print => write!(f, "print"), ScriptTokenType::Return => write!(f, "return"), ScriptTokenType::Super => write!(f, "super"), ScriptTokenType::This => write!(f, "this"), ScriptTokenType::Let => write!(f, "let"), ScriptTokenType::While => write!(f, "while"), ScriptTokenType::Export => write!(f, "export"), ScriptTokenType::Import => write!(f, "import"), ScriptTokenType::Alias => write!(f, "as"), ScriptTokenType::From => write!(f, "from"), ScriptTokenType::Eof => write!(f, ""), ScriptTokenType::Typeof => write!(f, "typeof"), ScriptTokenType::Finally => write!(f, "finally"), } } } #[derive(Copy, Clone, Debug, PartialEq)] pub struct TokenFromStringError<'a> { source: &'a str, } impl<'a> Display for TokenFromStringError<'a> { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "Failed to parse into token; value {}", self.source) } } impl<'a> Error for TokenFromStringError<'a> {} impl<'a> TryFrom<&'a str> for ScriptTokenType { type Error = TokenFromStringError<'a>; fn try_from(value: &'a str) -> Result<Self, Self::Error> { match value { "(" => Ok(ScriptTokenType::LeftParen), ")" => Ok(ScriptTokenType::RightParen), "{" => Ok(ScriptTokenType::LeftBrace), "}" => Ok(ScriptTokenType::RightBrace), "," => Ok(ScriptTokenType::Comma), "." => Ok(ScriptTokenType::Dot), "-" => Ok(ScriptTokenType::Minus), "+" => Ok(ScriptTokenType::Plus), ";" => Ok(ScriptTokenType::Semicolon), "/" => Ok(ScriptTokenType::Slash), "*" => Ok(ScriptTokenType::Asterisk), "!" => Ok(ScriptTokenType::Bang), "!=" => Ok(ScriptTokenType::BangEqual), "=" => Ok(ScriptTokenType::Equal), "==" => Ok(ScriptTokenType::EqualEqual), ">" => Ok(ScriptTokenType::Greater), ">=" => Ok(ScriptTokenType::GreaterEqual), "<" => Ok(ScriptTokenType::Less), "<=" => Ok(ScriptTokenType::LessEqual), "||" => Ok(ScriptTokenType::DoublePipe), "&&" => Ok(ScriptTokenType::DoubleAmpersand), "%" => Ok(ScriptTokenType::Modulo), "^" => Ok(ScriptTokenType::Caret), "struct" => Ok(ScriptTokenType::Class), "else" => Ok(ScriptTokenType::Else), "fn" => Ok(ScriptTokenType::Function), "for" => Ok(ScriptTokenType::For), "if" => Ok(ScriptTokenType::If), "null" => Ok(ScriptTokenType::Null), "print" => Ok(ScriptTokenType::Print), "return" => Ok(ScriptTokenType::Return), "super" => Ok(ScriptTokenType::Super), "this" => Ok(ScriptTokenType::This), "let" => Ok(ScriptTokenType::Let), "while" => Ok(ScriptTokenType::While), "export" => Ok(ScriptTokenType::Export), "import" => Ok(ScriptTokenType::Import), "as" => Ok(ScriptTokenType::Alias), "from" => Ok(ScriptTokenType::From), "typeof" => Ok(ScriptTokenType::Typeof), "finally" => Ok(ScriptTokenType::Finally), "false" => Ok(ScriptTokenType::Boolean(false)), "true" => Ok(ScriptTokenType::Boolean(true)), _ => Err(TokenFromStringError { source: value }), } } } #[derive(Clone, PartialEq)] pub struct ScriptToken<'a> { pub position: Span<'a>, pub token_type: ScriptTokenType, } impl<'a> Display for ScriptToken<'a> { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.token_type) } } impl<'a> Debug for ScriptToken<'a> { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, "[{}:{}] {:?}", self.position.location_line(), self.position.get_column(), self.token_type ) } } #[cfg(test)] mod token_tests { use super::{ScriptTokenType, TokenFromStringError}; use test_case::test_case; #[test_case("(" => Ok(ScriptTokenType::LeftParen); r#"Parse LeftParen"#)] #[test_case(")" => Ok(ScriptTokenType::RightParen); r#"Parse RightParen"#)] #[test_case("{" => Ok(ScriptTokenType::LeftBrace); r#"Parse LeftBrace"#)] #[test_case("}" => Ok(ScriptTokenType::RightBrace); r#"Parse RightBrace"#)] #[test_case("," => Ok(ScriptTokenType::Comma); r#"Parse Comma"#)] #[test_case("." => Ok(ScriptTokenType::Dot); r#"Parse Dot"#)] #[test_case("-" => Ok(ScriptTokenType::Minus); r#"Parse Minus"#)] #[test_case("+" => Ok(ScriptTokenType::Plus); r#"Parse Plus"#)] #[test_case(";" => Ok(ScriptTokenType::Semicolon); r#"Parse Semicolon"#)] #[test_case("/" => Ok(ScriptTokenType::Slash); r#"Parse Slash"#)] #[test_case("*" => Ok(ScriptTokenType::Asterisk); r#"Parse Asterisk"#)] #[test_case("!" => Ok(ScriptTokenType::Bang); r#"Parse Bang"#)] #[test_case("!=" => Ok(ScriptTokenType::BangEqual); r#"Parse BangEqual"#)] #[test_case("=" => Ok(ScriptTokenType::Equal); r#"Parse Equal"#)] #[test_case("==" => Ok(ScriptTokenType::EqualEqual); r#"Parse EqualEqual"#)] #[test_case(">" => Ok(ScriptTokenType::Greater); r#"Parse Greater"#)] #[test_case(">=" => Ok(ScriptTokenType::GreaterEqual); r#"Parse GreaterEqual"#)] #[test_case("<" => Ok(ScriptTokenType::Less); r#"Parse Less"#)] #[test_case("<=" => Ok(ScriptTokenType::LessEqual); r#"Parse LessEqual"#)] #[test_case("||" => Ok(ScriptTokenType::DoublePipe); r#"Parse DoublePipe"#)] #[test_case("&&" => Ok(ScriptTokenType::DoubleAmpersand); r#"Parse DoubleAmpersand"#)] #[test_case("%" => Ok(ScriptTokenType::Modulo); r#"Parse Modulo"#)] #[test_case("^" => Ok(ScriptTokenType::Caret); r#"Parse Caret"#)] #[test_case("struct" => Ok(ScriptTokenType::Class); r#"Parse Class"#)] #[test_case("else" => Ok(ScriptTokenType::Else); r#"Parse Else"#)] #[test_case("fn" => Ok(ScriptTokenType::Function); r#"Parse Function"#)] #[test_case("for" => Ok(ScriptTokenType::For); r#"Parse For"#)] #[test_case("if" => Ok(ScriptTokenType::If); r#"Parse If"#)] #[test_case("null" => Ok(ScriptTokenType::Null); r#"Parse Null"#)] #[test_case("print" => Ok(ScriptTokenType::Print); r#"Parse Print"#)] #[test_case("return" => Ok(ScriptTokenType::Return); r#"Parse Return"#)] #[test_case("super" => Ok(ScriptTokenType::Super); r#"Parse Super"#)] #[test_case("this" => Ok(ScriptTokenType::This); r#"Parse This"#)] #[test_case("let" => Ok(ScriptTokenType::Let); r#"Parse Let"#)] #[test_case("while" => Ok(ScriptTokenType::While); r#"Parse While"#)] #[test_case("export" => Ok(ScriptTokenType::Export); r#"Parse Export"#)] #[test_case("import" => Ok(ScriptTokenType::Import); r#"Parse Import"#)] #[test_case("as" => Ok(ScriptTokenType::Alias); r#"Parse Alias"#)] #[test_case("from" => Ok(ScriptTokenType::From); r#"Parse From"#)] #[test_case("typeof" => Ok(ScriptTokenType::Typeof); r#"Parse Typeof"#)] #[test_case("true" => Ok(ScriptTokenType::Boolean(true)); r#"Parse true"#)] #[test_case("false" => Ok(ScriptTokenType::Boolean(false)); r#"Parse false"#)] #[test_case("finally" => Ok(ScriptTokenType::Finally); r#"Parse finally"#)] fn parse_token_type(value: &'static str) -> Result<ScriptTokenType, TokenFromStringError> { ScriptTokenType::try_from(value) } }