diff --git a/examples/math.fx b/examples/math.fx index 5c4af57..447fbf2 100644 --- a/examples/math.fx +++ b/examples/math.fx @@ -1,12 +1,12 @@ fn main() { - let num = 10 - print(fib(num)) + let num = 10; + print(fib(num)); } fn fib(n int) int { if (n <= 1) { - return n + return n; } - return fib(n-1) + fib(n-2) + return fib(n-1) + fib(n-2); } \ No newline at end of file diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs index c56952a..5415898 100644 --- a/src/lexer/mod.rs +++ b/src/lexer/mod.rs @@ -20,17 +20,14 @@ impl Token { } /// Enum representing common lexeme types. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum TokenKind { /// Any whitespace characters sequence. Whitespace, - Literal { - kind: LiteralKind, - }, + Identifier(String), + Literal(Value), /// Keywords such as 'if' or 'else' - Identifier { - kind: IdentifierKind, - }, + Keyword(Keyword), /// // Lorem Ipsum Comment, /// "+" @@ -43,6 +40,8 @@ pub enum TokenKind { Slash, /// ":" Colon, + /// ";" + SemiColon, /// "=" Equals, /// "==" @@ -72,13 +71,13 @@ pub enum TokenKind { } #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub enum LiteralKind { +pub enum Value { Int, Str, } #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub enum IdentifierKind { +pub enum Keyword { Let, If, Else, @@ -132,19 +131,13 @@ impl Cursor<'_> { fn advance_token(&mut self) -> Token { // Original chars used to identify the token later on let original_chars = self.chars(); + // FIXME: Identical value, since it will be used twice and is not clonable later + let original_chars2 = self.chars(); let first_char = self.bump().unwrap(); let token_kind = match first_char { c if is_whitespace(c) => self.whitespace(), - '0'..='9' => { - let kind = self.number(); - - TokenKind::Literal { kind } - } - '"' | '\'' => { - let kind = self.string(); - - TokenKind::Literal { kind } - } + '0'..='9' => self.number(), + '"' | '\'' => self.string(), '+' => Plus, '-' => Minus, '*' => Star, @@ -157,6 +150,7 @@ impl Cursor<'_> { _ => Equals, }, ':' => Colon, + ';' => SemiColon, '<' => SmallerThen, '>' => LargerThen, '(' => BraceOpen, @@ -167,12 +161,13 @@ impl Cursor<'_> { '}' => CurlyBracesClose, c if is_id_start(c) => { let kind = self.identifier(c); - if kind == IdentifierKind::Unknown { - Literal { - kind: LiteralKind::Str, - } + println!("Identifier Type: {:?}", kind); + if kind == Keyword::Unknown { + let mut ch: String = original_chars.collect(); + ch.truncate(self.len_consumed()); + TokenKind::Identifier(ch) } else { - Identifier { kind } + TokenKind::Keyword(kind) } } '\n' => CarriageReturn, @@ -181,7 +176,7 @@ impl Cursor<'_> { }; let len = self.len_consumed(); - let mut raw = original_chars.collect::(); + let mut raw = original_chars2.collect::(); // Cut the original tokens to the length of the token raw.truncate(len); Token::new(token_kind, len, raw) @@ -208,18 +203,18 @@ impl Cursor<'_> { Whitespace } - fn number(&mut self) -> LiteralKind { + fn number(&mut self) -> TokenKind { self.eat_digits(); - LiteralKind::Int + TokenKind::Literal(Value::Int) } - fn string(&mut self) -> LiteralKind { + fn string(&mut self) -> TokenKind { self.eat_string(); - LiteralKind::Str + TokenKind::Literal(Value::Str) } - fn identifier(&mut self, first_char: char) -> IdentifierKind { + fn identifier(&mut self, first_char: char) -> Keyword { let mut original: String = self.chars().collect::(); let len = self.eat_while(is_id_continue); @@ -229,12 +224,12 @@ impl Cursor<'_> { original = format!("{}{}", first_char, original); match original { - c if c == "if" => IdentifierKind::If, - c if c == "else" => IdentifierKind::Else, - c if c == "fn" => IdentifierKind::Function, - c if c == "true" || c == "false" => IdentifierKind::Boolean, - c if c == "let" => IdentifierKind::Let, - _ => IdentifierKind::Unknown, + c if c == "if" => Keyword::If, + c if c == "else" => Keyword::Else, + c if c == "fn" => Keyword::Function, + c if c == "true" || c == "false" => Keyword::Boolean, + c if c == "let" => Keyword::Let, + _ => Keyword::Unknown, } } diff --git a/src/lexer/tests.rs b/src/lexer/tests.rs index 46016e1..7df5e93 100644 --- a/src/lexer/tests.rs +++ b/src/lexer/tests.rs @@ -101,7 +101,7 @@ mod tests { Token { len: 4, kind: TokenKind::Identifier { - kind: IdentifierKind::Boolean + kind: Keyword::Boolean }, raw: "true".to_owned() } @@ -112,7 +112,7 @@ mod tests { Token { len: 5, kind: TokenKind::Identifier { - kind: IdentifierKind::Boolean + kind: Keyword::Boolean }, raw: "false".to_owned() } @@ -128,7 +128,7 @@ mod tests { Token { len: 2, kind: TokenKind::Identifier { - kind: IdentifierKind::Function + kind: Keyword::Function }, raw: "fn".to_owned() } @@ -164,7 +164,7 @@ mod tests { Token { len: 2, kind: TokenKind::Identifier { - kind: IdentifierKind::Function + kind: Keyword::Function }, raw: "fn".to_owned(), } diff --git a/src/main.rs b/src/main.rs index b714e0b..17aa7c9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,3 @@ -use lexer::TokenKind; use std::fs::File; use std::io::Read; diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 430b66f..07371c2 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1,5 +1,5 @@ -use crate::lexer::IdentifierKind; -use crate::lexer::{LiteralKind, Token, TokenKind}; +use crate::lexer::Keyword; +use crate::lexer::{Token, TokenKind}; use crate::parser::node_type::*; use std::iter::Peekable; use std::vec::IntoIter; @@ -85,44 +85,22 @@ impl Parser { } } - fn match_identifier_kind(&mut self, identifier_kind: IdentifierKind) -> Result<(), String> { + fn match_keyword(&mut self, keyword: Keyword) -> Result<(), String> { let token = self.next_token(); println!( "match_identifier_kind: Token: {:?}, identifier_kind: {:?}", - token, identifier_kind + token, keyword ); match token.kind { - TokenKind::Identifier { - kind: identifier_kind, - } => Ok(()), + TokenKind::Keyword(_) => Ok(()), other => Err(format!("Expected SemiColon, found {:?}", other)), } } fn match_identifier(&mut self) -> Result { - let token = self.next_token(); - - // TODO: Match any IdentifierKind. This can definetely be prettier, but I couldn't figure it out in a hurry - match &token.kind { - TokenKind::Identifier { - kind: IdentifierKind::Boolean, - } - | TokenKind::Identifier { - kind: IdentifierKind::Else, - } - | TokenKind::Identifier { - kind: IdentifierKind::Function, - } - | TokenKind::Identifier { - kind: IdentifierKind::If, - } - | TokenKind::Identifier { - kind: IdentifierKind::Let, - } - | TokenKind::Identifier { - kind: IdentifierKind::Unknown, - } => Ok(token.raw), + match self.next_token().kind { + TokenKind::Identifier(n) => Ok(n), other => Err(format!("Expected Identifier, found {:?}", other)), } } @@ -144,12 +122,8 @@ impl Parser { } fn parse_function(&mut self) -> Result { - self.match_identifier_kind(IdentifierKind::Function)?; - let name = self - .match_token(TokenKind::Literal { - kind: LiteralKind::Str, - })? - .raw; + self.match_keyword(Keyword::Function)?; + let name = self.match_identifier()?; self.match_token(TokenKind::BraceOpen)?; self.match_token(TokenKind::BraceClose)?;