Browse Source

Parse functions

github-actions
Garrit Franke 3 years ago
parent
commit
86ca1cd2dd
  1. 4
      examples/hello_world.fx
  2. 9
      src/lexer/mod.rs
  3. 8
      src/main.rs
  4. 173
      src/parser/mod.rs
  5. 39
      src/parser/node_type.rs

4
examples/hello_world.fx

@ -1,4 +1,4 @@
fn main() {
let message = "Hello World!"
print(message)
}

9
src/lexer/mod.rs

@ -6,7 +6,7 @@ use cursor::Cursor;
#[cfg(test)]
mod tests;
#[derive(Debug, PartialEq, Eq)]
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct Token {
pub kind: TokenKind,
pub len: usize,
@ -167,9 +167,14 @@ impl Cursor<'_> {
'}' => CurlyBracesClose,
c if is_id_start(c) => {
let kind = self.identifier(c);
if kind == IdentifierKind::Unknown {
Literal {
kind: LiteralKind::Str,
}
} else {
Identifier { kind }
}
}
'\n' => CarriageReturn,
'\t' => Tab,
_ => Unknown,

8
src/main.rs

@ -13,10 +13,8 @@ fn main() -> std::io::Result<()> {
let tokens = lexer::tokenize(&contents);
// let ast = parser::parse(tokens.into_iter());
for token in tokens {
if token.kind != TokenKind::Whitespace && token.kind != TokenKind::CarriageReturn {
println!("{:?}", token);
}
}
let program = parser::parse(tokens);
println!("{:?}", program);
Ok(())
}

173
src/parser/mod.rs

@ -1,36 +1,173 @@
use crate::lexer::Token;
use crate::lexer::IdentifierKind;
use crate::lexer::{LiteralKind, Token, TokenKind};
use crate::parser::node_type::*;
use std::iter::Peekable;
use std::vec::IntoIter;
mod node_type;
pub struct Parser {
tokens: Box<dyn Iterator<Item = Token>>,
current: Option<Token>,
indentation_level: usize,
tokens: Peekable<IntoIter<Token>>,
peeked: Vec<Token>,
}
impl Parser {
pub(crate) fn new(tokens: impl Iterator<Item = Token> + 'static) -> Self {
pub fn new(tokens: Vec<Token>) -> Parser {
Parser {
tokens: Box::new(tokens),
current: None,
indentation_level: 0,
tokens: tokens.into_iter().peekable(),
peeked: vec![],
}
}
pub fn parse(&mut self) -> Result<Program, String> {
self.parse_program()
}
fn next(&mut self) {
self.current = self.tokens.next();
fn next(&mut self) -> Option<Token> {
if self.peeked.is_empty() {
self.tokens.next()
} else {
self.peeked.pop()
}
}
#[derive(Debug)]
pub struct AST;
fn peek(&mut self) -> Option<Token> {
if let Some(token) = self.next() {
self.push(Some(token.to_owned()));
Some(token)
} else {
None
}
}
pub fn parse(tokens: impl Iterator<Item = Token> + 'static) -> AST {
let mut parser = Parser::new(tokens);
let ast = AST {};
fn drop(&mut self, count: usize) {
for _ in 0..count {
self.next();
}
}
fn push(&mut self, token: Option<Token>) {
if let Some(t) = token {
self.peeked.push(t);
}
}
fn has_more(&mut self) -> bool {
!self.peeked.is_empty() || self.tokens.peek().is_some()
}
fn next_token(&mut self) -> Token {
self.next().expect("failed to parse")
}
fn match_token(&mut self, token_kind: TokenKind) -> Result<Token, String> {
loop {
parser.next();
break;
match self.peek().expect("Failed to peek token").kind {
TokenKind::Whitespace | TokenKind::Tab | TokenKind::CarriageReturn => {
self.next_token();
}
_ => break,
}
}
match self.next() {
Some(token) if token.kind == token_kind => Ok(token),
other => Err(format!(
"Token {:?} not found, found {:?}",
token_kind, other
)),
}
}
fn peek_token(&mut self, token_kind: TokenKind) -> Result<Token, String> {
match self.peek() {
Some(token) if token.kind == token_kind => Ok(token),
other => Err(format!(
"Token {:?} not found, found {:?}",
token_kind, other
)),
}
}
fn match_identifier_kind(&mut self, identifier_kind: IdentifierKind) -> Result<(), String> {
let token = self.next_token();
match token.kind {
TokenKind::Identifier {
kind: identifier_kind,
} => Ok(()),
other => Err(format!("Expected SemiColon, found {:?}", other)),
}
}
fn match_identifier(&mut self) -> Result<String, String> {
let token = self.next_token();
// TODO: Match any IdentifierKind. This can definetely be prettier, but I couldn't figure it out in a hurry
match &token.kind {
TokenKind::Identifier {
kind: IdentifierKind::Boolean,
}
| TokenKind::Identifier {
kind: IdentifierKind::Else,
}
| TokenKind::Identifier {
kind: IdentifierKind::Function,
}
| TokenKind::Identifier {
kind: IdentifierKind::If,
}
| TokenKind::Identifier {
kind: IdentifierKind::Let,
}
| TokenKind::Identifier {
kind: IdentifierKind::Unknown,
} => Ok(token.raw),
other => Err(format!("Expected Identifier, found {:?}", other)),
}
}
}
impl Parser {
fn parse_program(&mut self) -> Result<Program, String> {
let mut functions = Vec::new();
let globals = Vec::new();
while self.has_more() {
match self.next_token() {
t if t.kind == TokenKind::Whitespace => continue,
_ => functions.push(self.parse_function().expect("Failed to parse function")),
}
}
Ok(Program {
func: functions,
globals: globals,
})
}
fn parse_function(&mut self) -> Result<Function, String> {
self.match_identifier_kind(IdentifierKind::Function);
let name = self
.match_token(TokenKind::Literal {
kind: LiteralKind::Str,
})?
.raw;
self.match_token(TokenKind::BraceOpen);
self.match_token(TokenKind::BraceClose);
self.match_token(TokenKind::CurlyBracesOpen);
self.match_token(TokenKind::CurlyBracesClose);
Ok(Function {
name: name,
arguments: Vec::new(),
statements: Vec::new(),
})
}
}
pub fn parse(tokens: Vec<Token>) -> Result<node_type::Program, String> {
let mut parser = Parser::new(tokens);
ast
parser.parse()
}

39
src/parser/node_type.rs

@ -0,0 +1,39 @@
#[derive(Debug)]
pub struct Program {
pub func: Vec<Function>,
pub globals: Vec<String>,
}
#[derive(Debug)]
pub struct Function {
pub name: String,
pub arguments: Vec<Variable>,
pub statements: Vec<Statement>,
}
#[derive(Debug, Eq, PartialEq)]
pub struct Variable {
pub name: String,
}
#[derive(Debug, Eq, PartialEq)]
pub enum Statement {
Declare(Variable, Option<Expression>),
Return(Expression),
If(Expression, Box<Statement>, Option<Box<Statement>>),
While(Expression, Box<Statement>),
Exp(Expression),
Compound(Vec<Statement>),
}
#[derive(Debug, Eq, PartialEq)]
pub enum Expression {
Int(u32),
Char(u8),
FunctionCall(String, Vec<Expression>),
Variable(String),
VariableRef(String),
Assign(String, Box<Expression>),
AssignPostfix(String, Box<Expression>),
Ternary(Box<Expression>, Box<Expression>, Box<Expression>),
}
Loading…
Cancel
Save