greg/schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor parsing structure

Browse Source 
TokenHandler should contain all the methods for actually manipulating
tokens, Parser should only contain the recursive descent methods
This commit is contained in:
greg 2019-01-05 18:38:18 -08:00
parent 2b338fd3c9
commit 5491169d55
2 changed files with 53 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
schala-lang/codegen/src/lib.rs
View File

@ -19,7 +19,7 @@ impl Fold for RecursiveDescentFn {
let new_block: syn::Block = parse_quote! {
{
let next_token_before_parse = self.token_handler.peek_token();
let next_token_before_parse = self.token_handler.peek();
let record = ParseRecord {
production_name: stringify!(#ident).to_string(),
next_token: format!("{}", next_token_before_parse.to_string_with_metadata()),
@ -34,7 +34,7 @@ impl Fold for RecursiveDescentFn {
}
match result {
Err(ParseError { token: None, msg }) => {
let next_token_after_parse = self.token_handler.peek_token();
let next_token_after_parse = self.token_handler.peek();
println!("HERE? {:?}", next_token_after_parse);
Err(ParseError { token: Some(next_token_after_parse), msg })
},

106
schala-lang/language/src/parsing.rs
View File

@ -67,10 +67,10 @@ impl TokenHandler {
TokenHandler { tokens, end_of_file }
}
fn peek(&mut self) -> TokenKind {
fn peek_kind(&mut self) -> TokenKind {
self.tokens.peek().map(|ref t| { t.kind.clone() }).unwrap_or(TokenKind::EOF)
}
fn peek_token(&mut self) -> Token {
fn peek(&mut self) -> Token {
self.tokens.peek().map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, offset: self.end_of_file})
}
fn next(&mut self) -> Token {
@ -88,10 +88,6 @@ impl Parser {
}
}
fn peek(&mut self) -> TokenKind {
self.token_handler.peek()
}
pub fn parse(&mut self) -> ParseResult<AST> {
self.program()
}
@ -121,7 +117,7 @@ macro_rules! expect {
($self:expr, $token_kind:pat) => { expect!($self, $token_kind if true) };
($self:expr, $expected_kind:pat if $cond:expr) => {
{
let tok = $self.token_handler.peek_token();
let tok = $self.token_handler.peek();
match tok.get_kind() {
$expected_kind if $cond => $self.token_handler.next(),
actual_kind => {
@ -145,19 +141,19 @@ macro_rules! delimited {
expect!($self, $start);
let mut acc = vec![];
loop {
let peek = $self.peek();
match peek {
let peek = $self.token_handler.peek();
match peek.get_kind() {
$end | EOF => break,
_ => (),
}
if !$strictness {
match peek {
match peek.get_kind() {
$( $delim )|* => { $self.token_handler.next(); continue },
_ => ()
}
}
acc.push($self.$parse_fn()?);
match $self.peek() {
match $self.token_handler.peek().get_kind() {
$( $delim )|* => { $self.token_handler.next(); continue },
_ if $strictness => break,
_ => continue,
@ -296,7 +292,7 @@ impl Parser {
fn program(&mut self) -> ParseResult<AST> {
let mut statements = Vec::new();
loop {
match self.peek() {
match self.token_handler.peek().get_kind() {
EOF => break,
Newline | Semicolon => {
self.token_handler.next();
@ -313,7 +309,7 @@ impl Parser {
#[recursive_descent_method]
fn statement(&mut self) -> ParseResult<Statement> {
//TODO handle error recovery here
match self.peek() {
match self.token_handler.peek().get_kind() {
Keyword(Type) => self.type_declaration().map(|decl| { Statement::Declaration(decl) }),
Keyword(Func)=> self.func_declaration().map(|func| { Statement::Declaration(func) }),
Keyword(Let) => self.binding_declaration().map(|decl| Statement::Declaration(decl)),
@ -331,10 +327,10 @@ impl Parser {
#[recursive_descent_method]
fn type_declaration_body(&mut self) -> ParseResult<Declaration> {
if let Keyword(Alias) = self.peek() {
if let Keyword(Alias) = self.token_handler.peek_kind() {
self.type_alias()
} else {
let mutable = if let Keyword(Mut) = self.peek() {
let mutable = if let Keyword(Mut) = self.token_handler.peek_kind() {
self.token_handler.next();
true
} else {
@ -361,7 +357,7 @@ impl Parser {
let mut variants = Vec::new();
variants.push(self.variant_specifier()?);
loop {
if let Pipe = self.peek() {
if let Pipe = self.token_handler.peek_kind() {
self.token_handler.next();
variants.push(self.variant_specifier()?);
} else {
@ -376,7 +372,7 @@ impl Parser {
use self::Variant::*;
let name = self.identifier()?;
match self.peek() {
match self.token_handler.peek_kind() {
LParen => {
let tuple_members = delimited!(self, LParen, type_name, Comma, RParen);
Ok(TupleStruct(name, tuple_members))
@ -400,7 +396,7 @@ impl Parser {
#[recursive_descent_method]
fn func_declaration(&mut self) -> ParseResult<Declaration> {
let signature = self.func_signature()?;
if let LCurlyBrace = self.peek() {
if let LCurlyBrace = self.token_handler.peek_kind() {
let statements = self.nonempty_func_body()?;
Ok(Declaration::FuncDecl(signature, statements))
} else {
@ -411,7 +407,7 @@ impl Parser {
#[recursive_descent_method]
fn func_signature(&mut self) -> ParseResult<Signature> {
expect!(self, Keyword(Func));
let (name, operator) = match self.peek() {
let (name, operator) = match self.token_handler.peek_kind() {
Operator(s) => {
let name = s.clone();
self.token_handler.next();
@ -420,7 +416,7 @@ impl Parser {
_ => (self.identifier()?, false)
};
let params = self.formal_param_list()?;
let type_anno = match self.peek() {
let type_anno = match self.token_handler.peek_kind() {
Colon => Some(self.type_anno()?),
_ => None,
};
@ -441,7 +437,7 @@ impl Parser {
#[recursive_descent_method]
fn formal_param(&mut self) -> ParseResult<FormalParam> {
let name = self.identifier()?;
let ty = match self.peek() {
let ty = match self.token_handler.peek_kind() {
Colon => Some(self.type_anno()?),
_ => None
};
@ -451,7 +447,7 @@ impl Parser {
#[recursive_descent_method]
fn binding_declaration(&mut self) -> ParseResult<Declaration> {
expect!(self, Keyword(Kw::Let));
let constant = match self.peek() {
let constant = match self.token_handler.peek_kind() {
Keyword(Kw::Mut) => {
self.token_handler.next();
false
@ -482,7 +478,7 @@ impl Parser {
fn impl_declaration(&mut self) -> ParseResult<Declaration> {
expect!(self, Keyword(Impl));
let first = self.type_name()?;
let second = if let Keyword(For) = self.peek() {
let second = if let Keyword(For) = self.token_handler.peek_kind() {
self.token_handler.next();
Some(self.type_name()?)
} else {
@ -512,7 +508,7 @@ impl Parser {
#[recursive_descent_method]
fn expression(&mut self) -> ParseResult<Expression> {
let mut expr_body = self.precedence_expr(BinOp::min_precedence())?;
let type_anno = match self.peek() {
let type_anno = match self.token_handler.peek_kind() {
Colon => Some(self.type_anno()?),
_ => None
};
@ -532,7 +528,7 @@ impl Parser {
#[recursive_descent_method]
fn type_name(&mut self) -> ParseResult<TypeIdentifier> {
use self::TypeIdentifier::*;
Ok(match self.peek() {
Ok(match self.token_handler.peek_kind() {
LParen => Tuple(delimited!(self, LParen, type_name, Comma, RParen)),
_ => Singleton(self.type_singleton_name()?),
})
@ -542,7 +538,7 @@ impl Parser {
fn type_singleton_name(&mut self) -> ParseResult<TypeSingletonName> {
Ok(TypeSingletonName {
name: self.identifier()?,
params: match self.peek() {
params: match self.token_handler.peek_kind() {
LAngleBracket => delimited!(self, LAngleBracket, type_name, Comma, RAngleBracket),
_ => vec![],
}
@ -553,7 +549,7 @@ impl Parser {
fn precedence_expr(&mut self, precedence: i32) -> ParseResult<Expression> {
let record = ParseRecord {
production_name: "precedence_expr".to_string(),
next_token: format!("{}", self.token_handler.peek_token().to_string_with_metadata()),
next_token: format!("{}", self.token_handler.peek().to_string_with_metadata()),
level: self.parse_level,
};
self.parse_level += 1;
@ -561,7 +557,7 @@ impl Parser {
let mut lhs = self.prefix_expr()?;
loop {
let new_precedence = match BinOp::get_precedence_from_token(&self.peek()) {
let new_precedence = match BinOp::get_precedence_from_token(&self.token_handler.peek_kind()) {
Some(p) => p,
None => break,
};
@ -583,7 +579,7 @@ impl Parser {
#[recursive_descent_method]
fn prefix_expr(&mut self) -> ParseResult<Expression> {
match self.peek() {
match self.token_handler.peek_kind() {
Operator(ref op) if PrefixOp::is_prefix(&*op) => {
let sigil = match self.token_handler.next().kind {
Operator(op) => op,
@ -602,7 +598,7 @@ impl Parser {
#[recursive_descent_method]
fn call_expr(&mut self) -> ParseResult<Expression> {
let mut expr = self.index_expr()?;
while let LParen = self.peek() {
while let LParen = self.token_handler.peek_kind() {
let arguments = delimited!(self, LParen, expression, Comma, RParen);
let arguments = arguments.into_iter().map(|s| Node::new(s)).collect();
expr = Expression(ExpressionType::Call { f: bx!(expr), arguments }, None); //TODO none is incorrect
@ -614,7 +610,7 @@ impl Parser {
#[recursive_descent_method]
fn index_expr(&mut self) -> ParseResult<Expression> {
let primary = self.primary()?;
Ok(if let LSquareBracket = self.peek() {
Ok(if let LSquareBracket = self.token_handler.peek_kind() {
let indexers = delimited!(self, LSquareBracket, expression, Comma, RSquareBracket);
Expression(ExpressionType::Index {
indexee: bx!(Expression(primary.0, None)),
@ -627,7 +623,7 @@ impl Parser {
#[recursive_descent_method]
fn primary(&mut self) -> ParseResult<Expression> {
match self.peek() {
match self.token_handler.peek_kind() {
LCurlyBrace => self.curly_brace_expr(),
Backslash => self.lambda_expr(),
LParen => self.paren_expr(),
@ -655,7 +651,7 @@ impl Parser {
fn lambda_expr(&mut self) -> ParseResult<Expression> {
expect!(self, Backslash);
let params = self.lambda_param_list()?;
let type_anno = match self.peek() {
let type_anno = match self.token_handler.peek_kind() {
Colon => Some(self.type_anno()?),
_ => None,
};
@ -665,7 +661,7 @@ impl Parser {
#[recursive_descent_method]
fn lambda_param_list(&mut self) -> ParseResult<Vec<FormalParam>> {
if let LParen = self.peek() {
if let LParen = self.token_handler.peek_kind() {
self.formal_param_list()
} else {
let single_param = self.formal_param()?;
@ -697,7 +693,7 @@ impl Parser {
fn identifier_expr(&mut self) -> ParseResult<Expression> {
use self::ExpressionType::*;
let identifier = self.identifier()?;
Ok(match self.peek() {
Ok(match self.token_handler.peek_kind() {
LCurlyBrace if !self.restrictions.no_struct_literal => {
let fields = self.record_block()?;
Expression(NamedStruct { name: identifier, fields }, None)
@ -729,7 +725,7 @@ impl Parser {
x?
});
let body = Box::new(match self.peek() {
let body = Box::new(match self.token_handler.peek_kind() {
Keyword(Kw::Then) => self.conditional()?,
Keyword(Kw::Is) => self.simple_pattern_match()? ,
_ => self.guard_block()?
@ -741,7 +737,7 @@ impl Parser {
#[recursive_descent_method]
fn discriminator(&mut self) -> ParseResult<Discriminator> {
let lhs = self.prefix_expr()?;
let ref next = self.peek();
let ref next = self.token_handler.peek_kind();
Ok(if let Some(op) = BinOp::from_sigil_token(next) {
Discriminator::BinOp(lhs, op)
} else {
@ -769,7 +765,7 @@ impl Parser {
#[recursive_descent_method]
fn else_clause(&mut self) -> ParseResult<Option<Block>> {
Ok(if let Keyword(Kw::Else) = self.peek() {
Ok(if let Keyword(Kw::Else) = self.token_handler.peek_kind() {
self.token_handler.next();
Some(self.expr_or_block()?)
} else {
@ -785,19 +781,19 @@ impl Parser {
let mut guards = vec![];
loop {
match self.peek() {
match self.token_handler.peek_kind() {
RCurlyBrace | EOF => break,
Semicolon | Newline => { self.token_handler.next(); continue},
_ => {
let guard_arm = self.guard_arm()?;
guards.push(guard_arm);
loop {
match self.peek() {
match self.token_handler.peek_kind() {
Semicolon | Newline => { self.token_handler.next(); continue; },
_ => break,
}
}
if let RCurlyBrace = self.peek() {
if let RCurlyBrace = self.token_handler.peek_kind() {
break;
}
expect!(self, Comma);
@ -818,7 +814,7 @@ impl Parser {
#[recursive_descent_method]
fn guard(&mut self) -> ParseResult<Guard> {
Ok(match self.peek() {
Ok(match self.token_handler.peek_kind() {
Keyword(Kw::Is) => {
self.token_handler.next();
let pat = self.pattern()?;
@ -840,7 +836,7 @@ impl Parser {
#[recursive_descent_method]
fn pattern(&mut self) -> ParseResult<Pattern> {
if let LParen = self.peek() {
if let LParen = self.token_handler.peek_kind() {
let tuple_pattern_variants = delimited!(self, LParen, pattern, Comma, RParen);
Ok(Pattern::TuplePattern(tuple_pattern_variants))
} else {
@ -851,11 +847,11 @@ impl Parser {
#[recursive_descent_method]
fn simple_pattern(&mut self) -> ParseResult<Pattern> {
Ok({
let tok = self.token_handler.peek_token();
let tok = self.token_handler.peek();
match tok.get_kind() {
Identifier(_) => {
let id = self.identifier()?;
match self.peek() {
match self.token_handler.peek_kind() {
LCurlyBrace => {
let members = delimited!(self, LCurlyBrace, record_pattern_entry, Comma, RCurlyBrace);
Pattern::Record(id, members)
@ -892,7 +888,7 @@ impl Parser {
#[recursive_descent_method]
fn signed_number_literal(&mut self) -> ParseResult<Pattern> {
let neg = match self.peek() {
let neg = match self.token_handler.peek_kind() {
Operator(ref op) if **op == "-" => {
self.token_handler.next();
true
@ -906,7 +902,7 @@ impl Parser {
#[recursive_descent_method]
fn record_pattern_entry(&mut self) -> ParseResult<(Rc<String>, Pattern)> {
let name = self.identifier()?;
Ok(match self.peek() {
Ok(match self.token_handler.peek_kind() {
Colon => {
expect!(self, Colon);
let pat = self.pattern()?;
@ -924,7 +920,7 @@ impl Parser {
#[recursive_descent_method]
fn expr_or_block(&mut self) -> ParseResult<Block> {
match self.peek() {
match self.token_handler.peek_kind() {
LCurlyBrace => self.block(),
_ => {
let expr = self.expression()?;
@ -949,7 +945,7 @@ impl Parser {
#[recursive_descent_method]
fn while_cond(&mut self) -> ParseResult<Option<Expression>> {
Ok(match self.peek() {
Ok(match self.token_handler.peek_kind() {
LCurlyBrace => None,
_ => Some(self.expression()?),
})
@ -958,7 +954,7 @@ impl Parser {
#[recursive_descent_method]
fn for_expr(&mut self) -> ParseResult<Expression> {
expect!(self, Keyword(Kw::For));
let enumerators = if let LCurlyBrace = self.peek() {
let enumerators = if let LCurlyBrace = self.token_handler.peek_kind() {
delimited!(self, LCurlyBrace, enumerator, Comma | Newline, RCurlyBrace)
} else {
let single_enum = {
@ -984,7 +980,7 @@ impl Parser {
#[recursive_descent_method]
fn for_expr_body(&mut self) -> ParseResult<ForBody> {
use self::ForBody::*;
let tok = self.token_handler.peek_token();
let tok = self.token_handler.peek();
Ok(match tok.get_kind() {
LCurlyBrace => {
let statements = delimited!(self, LCurlyBrace, statement, Newline | Semicolon, RCurlyBrace, nonstrict);
@ -1011,7 +1007,7 @@ impl Parser {
fn literal(&mut self) -> ParseResult<Expression> {
use self::ExpressionType::*;
let tok = self.token_handler.peek_token();
let tok = self.token_handler.peek();
match tok.get_kind() {
DigitGroup(_) | HexLiteral(_) | BinNumberSigil | Period => self.number_literal(),
Keyword(Kw::True) => {
@ -1032,7 +1028,7 @@ impl Parser {
#[recursive_descent_method]
fn number_literal(&mut self) -> ParseResult<Expression> {
match self.peek() {
match self.token_handler.peek_kind() {
HexLiteral(_) | BinNumberSigil => self.int_literal(),
_ => self.float_literal(),
}
@ -1061,7 +1057,7 @@ impl Parser {
fn float_literal(&mut self) -> ParseResult<Expression> {
use self::ExpressionType::*;
let mut digits = self.digits()?;
if let Period = self.peek() {
if let Period = self.token_handler.peek_kind() {
self.token_handler.next();
digits.push_str(".");
digits.push_str(&self.digits()?);
@ -1082,7 +1078,7 @@ impl Parser {
fn digits(&mut self) -> ParseResult<String> {
let mut ds = String::new();
loop {
match self.peek() {
match self.token_handler.peek_kind() {
Underscore => { self.token_handler.next(); continue; },
DigitGroup(ref s) => { self.token_handler.next(); ds.push_str(s)},
_ => break,