schala/src/maaru_lang/parser.rs
greg 737dad6438 Added some tests
And commented out old tests for Maaru that don't compile
2017-09-19 22:11:02 -07:00

756 lines
23 KiB
Rust

use maaru_lang::tokenizer::{Token, Kw, OpTok};
use maaru_lang::tokenizer::Token::*;
use std::fmt;
use std::collections::VecDeque;
use std::rc::Rc;
use std::convert::From;
// Grammar
// program := (statement delimiter ?)*
// delimiter := Newline | Semicolon
// statement := declaration | expression
// declaration := FN prototype LCurlyBrace (statement)* RCurlyBrace
// prototype := identifier LParen identlist RParen
// identlist := Ident (Comma Ident)* | ε
// exprlist := Expression (Comma Expression)* | ε
// itemlist := Ident COLON Expression (Comma Ident COLON Expression)* | ε
//
// expression := postop_expression (op postop_expression)*
// postop_expression := primary_expression postop
// primary_expression := number_expr | String | identifier_expr | paren_expr | conditional_expr | while_expr | lambda_expr | list_expr | struct_expr
// number_expr := (PLUS | MINUS ) number_expr | Number
// identifier_expr := call_expression | Variable
// list_expr := LSquareBracket exprlist RSquareBracket
// struct_expr := LCurlyBrace itemlist RCurlyBrace
// call_expression := Identifier LParen exprlist RParen
// while_expr := WHILE primary_expression LCurlyBrace (expression delimiter)* RCurlyBrace
// paren_expr := LParen expression RParen
// conditional_expr := IF expression LCurlyBrace (expression delimiter)* RCurlyBrace (LCurlyBrace (expresion delimiter)* RCurlyBrace)?
// lambda_expr := FN LParen identlist RParen LCurlyBrace (expression delimiter)* RCurlyBrace
// lambda_call := | LParen exprlist RParen
// postop := ε | LParen exprlist RParen | LBracket expression RBracket
// op := '+', '-', etc.
//
pub type AST = Vec<Statement>;
#[derive(Debug, Clone)]
pub enum Statement {
ExprNode(Expression),
FuncDefNode(Function),
}
impl fmt::Display for Statement {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
use self::Statement::*;
match *self {
ExprNode(ref expr) => write!(f, "{}", expr),
FuncDefNode(_) => write!(f, "UNIMPLEMENTED"),
}
}
}
#[derive(Debug, Clone)]
pub struct Function {
pub prototype: Prototype,
pub body: Vec<Statement>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct Prototype {
pub name: Rc<String>,
pub parameters: Vec<Rc<String>>,
}
#[derive(Debug, Clone)]
pub enum Expression {
Null,
StringLiteral(Rc<String>),
Number(f64),
Variable(Rc<String>),
BinExp(BinOp, Box<Expression>, Box<Expression>),
Call(Callable, Vec<Expression>),
Conditional(Box<Expression>, Box<Expression>, Option<Box<Expression>>),
Lambda(Function),
Block(VecDeque<Expression>),
While(Box<Expression>, Vec<Expression>),
Index(Box<Expression>, Box<Expression>),
ListLiteral(VecDeque<Expression>),
StructLiteral(VecDeque<(Rc<String>, Expression)>),
}
#[derive(Clone, Debug)]
pub enum Callable {
NamedFunction(Rc<String>),
Lambda(Function),
}
//TODO this ought to be ReducedExpression
impl fmt::Display for Expression {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
use self::Expression::*;
match *self {
Null => write!(f, "null"),
StringLiteral(ref s) => write!(f, "\"{}\"", s),
Number(n) => write!(f, "{}", n),
Lambda(Function { prototype: Prototype { ref name, ref parameters, .. }, .. }) => {
write!(f, "«function: {}, {} arg(s)»", name, parameters.len())
}
ListLiteral(ref items) => {
write!(f, "[ ")?;
let mut iter = items.iter().peekable();
while let Some(item) = iter.next() {
write!(f, "{}", item)?;
if let Some(_) = iter.peek() {
write!(f, ", ")?;
}
}
write!(f, " ]")
}
StructLiteral(ref items) => {
write!(f, "{} ", "{")?;
let mut iter = items.iter().peekable();
while let Some(pair) = iter.next() {
write!(f, "{}: {}", pair.0, pair.1)?;
if let Some(_) = iter.peek() {
write!(f, ", ")?;
}
}
write!(f, "{} ", "}")
}
_ => write!(f, "UNIMPLEMENTED"),
}
}
}
#[derive(Debug, Clone)]
pub enum BinOp {
Add,
AddAssign,
Sub,
SubAssign,
Mul,
MulAssign,
Div,
DivAssign,
Mod,
Less,
LessEq,
Greater,
GreaterEq,
Equal,
Assign,
Custom(String),
}
impl From<OpTok> for BinOp {
fn from(token: OpTok) -> BinOp {
use self::BinOp::*;
match &token.0[..] {
"+" => Add,
"+=" => AddAssign,
"-" => Sub,
"-=" => SubAssign,
"*" => Mul,
"*=" => MulAssign,
"/" => Div,
"/=" => DivAssign,
"%" => Mod,
"<" => Less,
"<=" => LessEq,
">" => Greater,
">=" => GreaterEq,
"==" => Equal,
"=" => Assign,
op => Custom(op.to_string()),
}
}
}
type Precedence = u8;
// TODO make this support incomplete parses
pub type ParseResult<T> = Result<T, ParseError>;
#[derive(Debug)]
pub struct ParseError {
pub msg: String,
pub remaining_tokens: Vec<Token>,
}
impl ParseError {
fn result_from_str<T>(msg: &str) -> ParseResult<T> {
Err(ParseError {
msg: msg.to_string(),
remaining_tokens: vec![],
})
}
}
struct Parser {
tokens: Vec<Token>,
}
impl Parser {
fn initialize(tokens: &[Token]) -> Parser {
let mut tokens = tokens.to_vec();
tokens.reverse();
Parser { tokens: tokens }
}
fn peek(&self) -> Option<Token> {
self.tokens.last().map(|x| x.clone())
}
fn next(&mut self) -> Option<Token> {
self.tokens.pop()
}
fn get_precedence(&self, op: &OpTok) -> Precedence {
match &op.0[..] {
"+" => 10,
"-" => 10,
"*" => 20,
"/" => 20,
"%" => 20,
"==" => 40,
"=" | "+=" | "-=" | "*=" | "/=" => 1,
">" | ">=" | "<" | "<=" => 30,
_ => 255,
}
}
}
macro_rules! expect {
($self_:expr, $token:pat) => {
match $self_.peek() {
Some($token) => {$self_.next();},
Some(x) => {
let err = format!("Expected `{:?}` but got `{:?}`", stringify!($token), x);
return ParseError::result_from_str(&err)
},
None => {
let err = format!("Expected `{:?}` but got end of input", stringify!($token));
return ParseError::result_from_str(&err) //TODO make this not require 2 stringifications
}
}
}
}
macro_rules! expect_identifier {
($self_:expr) => {
match $self_.peek() {
Some(Identifier(s)) => {$self_.next(); s},
Some(x) => return ParseError::result_from_str(&format!("Expected identifier, but got {:?}", x)),
None => return ParseError::result_from_str("Expected identifier, but got end of input"),
}
}
}
macro_rules! skip_whitespace {
($_self: expr) => {
loop {
match $_self.peek() {
Some(ref t) if is_delimiter(t) => {
$_self.next();
continue;
}
_ => break,
}
}
}
}
macro_rules! delimiter_block {
($_self: expr, $try_parse: ident, $($break_pattern: pat)|+) => {
{
let mut acc = Vec::new();
loop {
match $_self.peek() {
None => break,
Some(ref t) if is_delimiter(t) => { $_self.next(); continue; },
$($break_pattern)|+ => break,
_ => {
let a = try!($_self.$try_parse());
acc.push(a);
}
}
}
acc
}
}
}
fn is_delimiter(token: &Token) -> bool {
match *token {
Newline | Semicolon => true,
_ => false,
}
}
impl Parser {
fn program(&mut self) -> ParseResult<AST> {
let mut ast = Vec::new(); //TODO have this come from previously-parsed tree
loop {
let result: ParseResult<Statement> = match self.peek() {
Some(ref t) if is_delimiter(t) => {
self.next();
continue;
}
Some(_) => self.statement(),
None => break,
};
match result {
Ok(node) => ast.push(node),
Err(mut err) => {
err.remaining_tokens = self.tokens.clone();
err.remaining_tokens.reverse();
return Err(err);
}
}
}
Ok(ast)
}
fn statement(&mut self) -> ParseResult<Statement> {
let node: Statement = match self.peek() {
Some(Keyword(Kw::Fn)) => self.declaration()?,
Some(_) => Statement::ExprNode(self.expression()?),
None => panic!("Unexpected end of tokens"),
};
Ok(node)
}
fn declaration(&mut self) -> ParseResult<Statement> {
expect!(self, Keyword(Kw::Fn));
let prototype = self.prototype()?;
expect!(self, LCurlyBrace);
let body = self.body()?;
expect!(self, RCurlyBrace);
Ok(Statement::FuncDefNode(Function {
prototype: prototype,
body: body,
}))
}
fn prototype(&mut self) -> ParseResult<Prototype> {
let name = expect_identifier!(self);
expect!(self, LParen);
let parameters = self.identlist()?;
expect!(self, RParen);
Ok(Prototype {
name: name,
parameters: parameters,
})
}
fn identlist(&mut self) -> ParseResult<Vec<Rc<String>>> {
let mut args = Vec::new();
while let Some(Identifier(name)) = self.peek() {
args.push(name.clone());
self.next();
match self.peek() {
Some(Comma) => {self.next();},
_ => break,
}
}
Ok(args)
}
fn exprlist(&mut self) -> ParseResult<Vec<Expression>> {
let mut exprs = Vec::new();
loop {
if let Some(RParen) = self.peek() {
break;
}
let exp = self.expression()?;
exprs.push(exp);
match self.peek() {
Some(Comma) => {self.next();},
_ => break,
}
}
Ok(exprs)
}
fn itemlist(&mut self) -> ParseResult<VecDeque<(Rc<String>, Expression)>> {
let mut items = VecDeque::new();
loop {
if let Some(RCurlyBrace) = self.peek() {
break;
}
let name = expect_identifier!(self);
expect!(self, Colon);
let expr = self.expression()?;
items.push_back((name, expr));
match self.peek() {
Some(Comma) => {self.next();},
_ => break,
};
}
Ok(items)
}
fn body(&mut self) -> ParseResult<Vec<Statement>> {
let statements = delimiter_block!(
self,
statement,
Some(RCurlyBrace)
);
Ok(statements)
}
fn expression(&mut self) -> ParseResult<Expression> {
let lhs: Expression = self.postop_expression()?;
self.precedence_expr(lhs, 0)
}
fn precedence_expr(&mut self,
mut lhs: Expression,
min_precedence: u8)
-> ParseResult<Expression> {
while let Some(Operator(op)) = self.peek() {
let precedence = self.get_precedence(&op);
if precedence < min_precedence {
break;
}
self.next();
let mut rhs = self.postop_expression()?;
while let Some(Operator(ref op)) = self.peek() {
if self.get_precedence(op) > precedence {
let new_prec = self.get_precedence(op);
rhs = self.precedence_expr(rhs, new_prec)?;
} else {
break;
}
}
lhs = Expression::BinExp(op.into(), Box::new(lhs), Box::new(rhs));
}
Ok(lhs)
}
fn postop_expression(&mut self) -> ParseResult<Expression> {
use self::Expression::*;
let expr = self.primary_expression()?;
let ret = match self.peek() {
Some(LParen) => {
let args = self.call_expression()?;
match expr {
Lambda(f) => Call(Callable::Lambda(f), args),
e => {
let err = format!("Expected lambda expression before a call, got {:?}", e);
return ParseError::result_from_str(&err);
},
}
},
Some(LSquareBracket) => {
expect!(self, LSquareBracket);
let index_expr = self.expression()?;
expect!(self, RSquareBracket);
Index(Box::new(expr), Box::new(index_expr))
},
_ => {
expr
}
};
Ok(ret)
}
fn primary_expression(&mut self) -> ParseResult<Expression> {
Ok(match self.peek() {
Some(Keyword(Kw::Null)) => {
self.next();
Expression::Null
}
Some(NumLiteral(_)) => self.number_expression()?,
Some(Operator(OpTok(ref a))) if **a == "+" || **a == "-" => self.number_expression()?,
Some(StrLiteral(s)) => {
self.next();
Expression::StringLiteral(s)
}
Some(Keyword(Kw::If)) => self.conditional_expr()?,
Some(Keyword(Kw::While)) => self.while_expr()?,
Some(Identifier(_)) => self.identifier_expr()?,
Some(Token::LParen) => self.paren_expr()?,
Some(Keyword(Kw::Fn)) => self.lambda_expr()?,
Some(Token::LSquareBracket) => self.list_expr()?,
Some(Token::LCurlyBrace) => self.struct_expr()?,
Some(e) => {
return ParseError::result_from_str(&format!("Expected primary expression, got \
{:?}",
e));
}
None => return ParseError::result_from_str("Expected primary expression received EoI"),
})
}
fn list_expr(&mut self) -> ParseResult<Expression> {
expect!(self, LSquareBracket);
let exprlist: Vec<Expression> = self.exprlist()?;
expect!(self, RSquareBracket);
Ok(Expression::ListLiteral(VecDeque::from(exprlist)))
}
fn struct_expr(&mut self) -> ParseResult<Expression> {
expect!(self, LCurlyBrace);
let struct_items = self.itemlist()?;
expect!(self, RCurlyBrace);
Ok(Expression::StructLiteral(struct_items))
}
fn number_expression(&mut self) -> ParseResult<Expression> {
let mut multiplier = 1;
loop {
match self.peek() {
Some(NumLiteral(n)) => {
self.next();
return Ok(Expression::Number(n * multiplier as f64));
}
Some(Operator(OpTok(ref a))) if **a == "+" => {
self.next();
}
Some(Operator(OpTok(ref a))) if **a == "-" => {
multiplier *= -1;
self.next();
}
Some(e) => {
return ParseError::result_from_str(
&format!("Expected +, - or number, got {:?}", e));
}
None => {
return ParseError::result_from_str(
&format!("Expected +, - or number, got EoI"));
}
}
}
}
fn lambda_expr(&mut self) -> ParseResult<Expression> {
use self::Expression::*;
expect!(self, Keyword(Kw::Fn));
skip_whitespace!(self);
expect!(self, LParen);
let parameters = self.identlist()?;
expect!(self, RParen);
skip_whitespace!(self);
expect!(self, LCurlyBrace);
let body = self.body()?;
expect!(self, RCurlyBrace);
let prototype = Prototype {
name: Rc::new("a lambda yo!".to_string()),
parameters: parameters,
};
let function = Function {
prototype: prototype,
body: body,
};
Ok(Lambda(function))
}
fn while_expr(&mut self) -> ParseResult<Expression> {
use self::Expression::*;
expect!(self, Keyword(Kw::While));
let test = self.expression()?;
expect!(self, LCurlyBrace);
let body = delimiter_block!(
self,
expression,
Some(RCurlyBrace)
);
expect!(self, RCurlyBrace);
Ok(While(Box::new(test), body))
}
fn conditional_expr(&mut self) -> ParseResult<Expression> {
use self::Expression::*;
expect!(self, Keyword(Kw::If));
let test = self.expression()?;
skip_whitespace!(self);
expect!(self, LCurlyBrace);
skip_whitespace!(self);
let then_block = delimiter_block!(
self,
expression,
Some(RCurlyBrace)
);
expect!(self, RCurlyBrace);
skip_whitespace!(self);
let else_block = if let Some(Keyword(Kw::Else)) = self.peek() {
self.next();
skip_whitespace!(self);
expect!(self, LCurlyBrace);
let else_exprs = delimiter_block!(
self,
expression,
Some(RCurlyBrace)
);
Some(else_exprs)
} else {
None
};
expect!(self, RCurlyBrace);
Ok(Conditional(Box::new(test),
Box::new(Block(VecDeque::from(then_block))),
else_block.map(|list| Box::new(Block(VecDeque::from(list))))))
}
fn identifier_expr(&mut self) -> ParseResult<Expression> {
let name = expect_identifier!(self);
let expr = match self.peek() {
Some(LParen) => {
let args = self.call_expression()?;
Expression::Call(Callable::NamedFunction(name), args)
}
__ => Expression::Variable(name),
};
Ok(expr)
}
fn call_expression(&mut self) -> ParseResult<Vec<Expression>> {
expect!(self, LParen);
let args: Vec<Expression> = self.exprlist()?;
expect!(self, RParen);
Ok(args)
}
fn paren_expr(&mut self) -> ParseResult<Expression> {
expect!(self, Token::LParen);
let expr = self.expression()?;
expect!(self, Token::RParen);
Ok(expr)
}
}
pub fn parse(tokens: &[Token], _parsed_tree: &[Statement]) -> ParseResult<AST> {
let mut parser = Parser::initialize(tokens);
parser.program()
}
/*
#[cfg(test)]
mod tests {
use schala_lang::tokenizer;
use super::*;
use super::Statement::*;
use super::Expression::*;
macro_rules! parsetest {
($input:expr, $output:pat, $ifexpr:expr) => {
{
let tokens = tokenizer::tokenize($input).unwrap();
let ast = parse(&tokens, &[]).unwrap();
match &ast[..] {
$output if $ifexpr => (),
x => panic!("Error in parse test, got {:?} instead", x)
}
}
}
}
#[test]
fn function_parse_test() {
use super::Function;
parsetest!(
"fn a() { 1 + 2 }",
&[FuncDefNode(Function {prototype: Prototype { ref name, ref parameters }, ref body})],
match &body[..] { &[ExprNode(BinExp(_, box Number(1.0), box Number(2.0)))] => true, _ => false }
&& **name == "a" && match &parameters[..] { &[] => true, _ => false }
);
parsetest!(
"fn a(x,y){ 1 + 2 }",
&[FuncDefNode(Function {prototype: Prototype { ref name, ref parameters }, ref body})],
match &body[..] { &[ExprNode(BinExp(_, box Number(1.0), box Number(2.0)))] => true, _ => false }
&& **name == "a" && *parameters[0] == "x" && *parameters[1] == "y" && parameters.len() == 2
);
let t3 = "fn (x) { x + 2 }";
let tokens3 = tokenizer::tokenize(t3).unwrap();
assert!(parse(&tokens3, &[]).is_err());
}
#[test]
fn expression_parse_test() {
parsetest!("a", &[ExprNode(Variable(ref s))], **s == "a");
parsetest!("a + b",
&[ExprNode(BinExp(BinOp::Add, box Variable(ref a), box Variable(ref b)))],
**a == "a" && **b == "b");
parsetest!("a + b * c",
&[ExprNode(BinExp(BinOp::Add, box Variable(ref a), box BinExp(BinOp::Mul, box Variable(ref b), box Variable(ref c))))],
**a == "a" && **b == "b" && **c == "c");
parsetest!("a * b + c",
&[ExprNode(BinExp(BinOp::Add, box BinExp(BinOp::Mul, box Variable(ref a), box Variable(ref b)), box Variable(ref c)))],
**a == "a" && **b == "b" && **c == "c");
parsetest!("(a + b) * c",
&[ExprNode(BinExp(BinOp::Mul, box BinExp(BinOp::Add, box Variable(ref a), box Variable(ref b)), box Variable(ref c)))],
**a == "a" && **b == "b" && **c == "c");
}
#[test]
fn lambda_parse_test() {
use schala_lang::tokenizer;
let t1 = "(fn(x) { x + 2 })";
let tokens1 = tokenizer::tokenize(t1).unwrap();
match parse(&tokens1, &[]).unwrap()[..] {
_ => (),
}
let t2 = "fn(x) { x + 2 }";
let tokens2 = tokenizer::tokenize(t2).unwrap();
assert!(parse(&tokens2, &[]).is_err());
let t3 = "(fn(x) { x + 10 })(20)";
let tokens3 = tokenizer::tokenize(t3).unwrap();
match parse(&tokens3, &[]).unwrap() {
_ => (),
};
}
#[test]
fn conditional_parse_test() {
use schala_lang::tokenizer;
let t1 = "if null { 20 } else { 40 }";
let tokens = tokenizer::tokenize(t1).unwrap();
match parse(&tokens, &[]).unwrap()[..] {
[ExprNode(Conditional(box Null, box Block(_), Some(box Block(_))))] => (),
_ => panic!(),
}
let t2 = r"
if null {
20
} else {
40
}
";
let tokens2 = tokenizer::tokenize(t2).unwrap();
match parse(&tokens2, &[]).unwrap()[..] {
[ExprNode(Conditional(box Null, box Block(_), Some(box Block(_))))] => (),
_ => panic!(),
}
let t2 = r"
if null {
20 } else
{
40
}
";
let tokens3 = tokenizer::tokenize(t2).unwrap();
match parse(&tokens3, &[]).unwrap()[..] {
[ExprNode(Conditional(box Null, box Block(_), Some(box Block(_))))] => (),
_ => panic!(),
}
}
}
*/