Browse Source

Refactor TokenType

github-actions
Garrit Franke 3 years ago
parent
commit
de7163c40e
  1. 8
      examples/math.fx
  2. 67
      src/lexer/mod.rs
  3. 8
      src/lexer/tests.rs
  4. 1
      src/main.rs
  5. 44
      src/parser/mod.rs

8
examples/math.fx

@ -1,12 +1,12 @@
fn main() {
let num = 10
print(fib(num))
let num = 10;
print(fib(num));
}
fn fib(n int) int {
if (n <= 1) {
return n
return n;
}
return fib(n-1) + fib(n-2)
return fib(n-1) + fib(n-2);
}

67
src/lexer/mod.rs

@ -20,17 +20,14 @@ impl Token {
}
/// Enum representing common lexeme types.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum TokenKind {
/// Any whitespace characters sequence.
Whitespace,
Literal {
kind: LiteralKind,
},
Identifier(String),
Literal(Value),
/// Keywords such as 'if' or 'else'
Identifier {
kind: IdentifierKind,
},
Keyword(Keyword),
/// // Lorem Ipsum
Comment,
/// "+"
@ -43,6 +40,8 @@ pub enum TokenKind {
Slash,
/// ":"
Colon,
/// ";"
SemiColon,
/// "="
Equals,
/// "=="
@ -72,13 +71,13 @@ pub enum TokenKind {
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum LiteralKind {
pub enum Value {
Int,
Str,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum IdentifierKind {
pub enum Keyword {
Let,
If,
Else,
@ -132,19 +131,13 @@ impl Cursor<'_> {
fn advance_token(&mut self) -> Token {
// Original chars used to identify the token later on
let original_chars = self.chars();
// FIXME: Identical value, since it will be used twice and is not clonable later
let original_chars2 = self.chars();
let first_char = self.bump().unwrap();
let token_kind = match first_char {
c if is_whitespace(c) => self.whitespace(),
'0'..='9' => {
let kind = self.number();
TokenKind::Literal { kind }
}
'"' | '\'' => {
let kind = self.string();
TokenKind::Literal { kind }
}
'0'..='9' => self.number(),
'"' | '\'' => self.string(),
'+' => Plus,
'-' => Minus,
'*' => Star,
@ -157,6 +150,7 @@ impl Cursor<'_> {
_ => Equals,
},
':' => Colon,
';' => SemiColon,
'<' => SmallerThen,
'>' => LargerThen,
'(' => BraceOpen,
@ -167,12 +161,13 @@ impl Cursor<'_> {
'}' => CurlyBracesClose,
c if is_id_start(c) => {
let kind = self.identifier(c);
if kind == IdentifierKind::Unknown {
Literal {
kind: LiteralKind::Str,
}
println!("Identifier Type: {:?}", kind);
if kind == Keyword::Unknown {
let mut ch: String = original_chars.collect();
ch.truncate(self.len_consumed());
TokenKind::Identifier(ch)
} else {
Identifier { kind }
TokenKind::Keyword(kind)
}
}
'\n' => CarriageReturn,
@ -181,7 +176,7 @@ impl Cursor<'_> {
};
let len = self.len_consumed();
let mut raw = original_chars.collect::<String>();
let mut raw = original_chars2.collect::<String>();
// Cut the original tokens to the length of the token
raw.truncate(len);
Token::new(token_kind, len, raw)
@ -208,18 +203,18 @@ impl Cursor<'_> {
Whitespace
}
fn number(&mut self) -> LiteralKind {
fn number(&mut self) -> TokenKind {
self.eat_digits();
LiteralKind::Int
TokenKind::Literal(Value::Int)
}
fn string(&mut self) -> LiteralKind {
fn string(&mut self) -> TokenKind {
self.eat_string();
LiteralKind::Str
TokenKind::Literal(Value::Str)
}
fn identifier(&mut self, first_char: char) -> IdentifierKind {
fn identifier(&mut self, first_char: char) -> Keyword {
let mut original: String = self.chars().collect::<String>();
let len = self.eat_while(is_id_continue);
@ -229,12 +224,12 @@ impl Cursor<'_> {
original = format!("{}{}", first_char, original);
match original {
c if c == "if" => IdentifierKind::If,
c if c == "else" => IdentifierKind::Else,
c if c == "fn" => IdentifierKind::Function,
c if c == "true" || c == "false" => IdentifierKind::Boolean,
c if c == "let" => IdentifierKind::Let,
_ => IdentifierKind::Unknown,
c if c == "if" => Keyword::If,
c if c == "else" => Keyword::Else,
c if c == "fn" => Keyword::Function,
c if c == "true" || c == "false" => Keyword::Boolean,
c if c == "let" => Keyword::Let,
_ => Keyword::Unknown,
}
}

8
src/lexer/tests.rs

@ -101,7 +101,7 @@ mod tests {
Token {
len: 4,
kind: TokenKind::Identifier {
kind: IdentifierKind::Boolean
kind: Keyword::Boolean
},
raw: "true".to_owned()
}
@ -112,7 +112,7 @@ mod tests {
Token {
len: 5,
kind: TokenKind::Identifier {
kind: IdentifierKind::Boolean
kind: Keyword::Boolean
},
raw: "false".to_owned()
}
@ -128,7 +128,7 @@ mod tests {
Token {
len: 2,
kind: TokenKind::Identifier {
kind: IdentifierKind::Function
kind: Keyword::Function
},
raw: "fn".to_owned()
}
@ -164,7 +164,7 @@ mod tests {
Token {
len: 2,
kind: TokenKind::Identifier {
kind: IdentifierKind::Function
kind: Keyword::Function
},
raw: "fn".to_owned(),
}

1
src/main.rs

@ -1,4 +1,3 @@
use lexer::TokenKind;
use std::fs::File;
use std::io::Read;

44
src/parser/mod.rs

@ -1,5 +1,5 @@
use crate::lexer::IdentifierKind;
use crate::lexer::{LiteralKind, Token, TokenKind};
use crate::lexer::Keyword;
use crate::lexer::{Token, TokenKind};
use crate::parser::node_type::*;
use std::iter::Peekable;
use std::vec::IntoIter;
@ -85,44 +85,22 @@ impl Parser {
}
}
fn match_identifier_kind(&mut self, identifier_kind: IdentifierKind) -> Result<(), String> {
fn match_keyword(&mut self, keyword: Keyword) -> Result<(), String> {
let token = self.next_token();
println!(
"match_identifier_kind: Token: {:?}, identifier_kind: {:?}",
token, identifier_kind
token, keyword
);
match token.kind {
TokenKind::Identifier {
kind: identifier_kind,
} => Ok(()),
TokenKind::Keyword(_) => Ok(()),
other => Err(format!("Expected SemiColon, found {:?}", other)),
}
}
fn match_identifier(&mut self) -> Result<String, String> {
let token = self.next_token();
// TODO: Match any IdentifierKind. This can definetely be prettier, but I couldn't figure it out in a hurry
match &token.kind {
TokenKind::Identifier {
kind: IdentifierKind::Boolean,
}
| TokenKind::Identifier {
kind: IdentifierKind::Else,
}
| TokenKind::Identifier {
kind: IdentifierKind::Function,
}
| TokenKind::Identifier {
kind: IdentifierKind::If,
}
| TokenKind::Identifier {
kind: IdentifierKind::Let,
}
| TokenKind::Identifier {
kind: IdentifierKind::Unknown,
} => Ok(token.raw),
match self.next_token().kind {
TokenKind::Identifier(n) => Ok(n),
other => Err(format!("Expected Identifier, found {:?}", other)),
}
}
@ -144,12 +122,8 @@ impl Parser {
}
fn parse_function(&mut self) -> Result<Function, String> {
self.match_identifier_kind(IdentifierKind::Function)?;
let name = self
.match_token(TokenKind::Literal {
kind: LiteralKind::Str,
})?
.raw;
self.match_keyword(Keyword::Function)?;
let name = self.match_identifier()?;
self.match_token(TokenKind::BraceOpen)?;
self.match_token(TokenKind::BraceClose)?;

Loading…
Cancel
Save