From 5491169d559e87f2c6f380d2c08b32b853a38ddc Mon Sep 17 00:00:00 2001 From: greg Date: Sat, 5 Jan 2019 18:38:18 -0800 Subject: [PATCH] Refactor parsing structure TokenHandler should contain all the methods for actually manipulating tokens, Parser should only contain the recursive descent methods --- schala-lang/codegen/src/lib.rs | 4 +- schala-lang/language/src/parsing.rs | 106 +++++++++++++--------------- 2 files changed, 53 insertions(+), 57 deletions(-) diff --git a/schala-lang/codegen/src/lib.rs b/schala-lang/codegen/src/lib.rs index b28ddbc..b3678a6 100644 --- a/schala-lang/codegen/src/lib.rs +++ b/schala-lang/codegen/src/lib.rs @@ -19,7 +19,7 @@ impl Fold for RecursiveDescentFn { let new_block: syn::Block = parse_quote! { { - let next_token_before_parse = self.token_handler.peek_token(); + let next_token_before_parse = self.token_handler.peek(); let record = ParseRecord { production_name: stringify!(#ident).to_string(), next_token: format!("{}", next_token_before_parse.to_string_with_metadata()), @@ -34,7 +34,7 @@ impl Fold for RecursiveDescentFn { } match result { Err(ParseError { token: None, msg }) => { - let next_token_after_parse = self.token_handler.peek_token(); + let next_token_after_parse = self.token_handler.peek(); println!("HERE? {:?}", next_token_after_parse); Err(ParseError { token: Some(next_token_after_parse), msg }) }, diff --git a/schala-lang/language/src/parsing.rs b/schala-lang/language/src/parsing.rs index 06f4129..54546d7 100644 --- a/schala-lang/language/src/parsing.rs +++ b/schala-lang/language/src/parsing.rs @@ -67,10 +67,10 @@ impl TokenHandler { TokenHandler { tokens, end_of_file } } - fn peek(&mut self) -> TokenKind { + fn peek_kind(&mut self) -> TokenKind { self.tokens.peek().map(|ref t| { t.kind.clone() }).unwrap_or(TokenKind::EOF) } - fn peek_token(&mut self) -> Token { + fn peek(&mut self) -> Token { self.tokens.peek().map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, offset: self.end_of_file}) } fn next(&mut self) -> Token { @@ -88,10 +88,6 @@ impl Parser { } } - fn peek(&mut self) -> TokenKind { - self.token_handler.peek() - } - pub fn parse(&mut self) -> ParseResult { self.program() } @@ -121,7 +117,7 @@ macro_rules! expect { ($self:expr, $token_kind:pat) => { expect!($self, $token_kind if true) }; ($self:expr, $expected_kind:pat if $cond:expr) => { { - let tok = $self.token_handler.peek_token(); + let tok = $self.token_handler.peek(); match tok.get_kind() { $expected_kind if $cond => $self.token_handler.next(), actual_kind => { @@ -145,19 +141,19 @@ macro_rules! delimited { expect!($self, $start); let mut acc = vec![]; loop { - let peek = $self.peek(); - match peek { + let peek = $self.token_handler.peek(); + match peek.get_kind() { $end | EOF => break, _ => (), } if !$strictness { - match peek { + match peek.get_kind() { $( $delim )|* => { $self.token_handler.next(); continue }, _ => () } } acc.push($self.$parse_fn()?); - match $self.peek() { + match $self.token_handler.peek().get_kind() { $( $delim )|* => { $self.token_handler.next(); continue }, _ if $strictness => break, _ => continue, @@ -296,7 +292,7 @@ impl Parser { fn program(&mut self) -> ParseResult { let mut statements = Vec::new(); loop { - match self.peek() { + match self.token_handler.peek().get_kind() { EOF => break, Newline | Semicolon => { self.token_handler.next(); @@ -313,7 +309,7 @@ impl Parser { #[recursive_descent_method] fn statement(&mut self) -> ParseResult { //TODO handle error recovery here - match self.peek() { + match self.token_handler.peek().get_kind() { Keyword(Type) => self.type_declaration().map(|decl| { Statement::Declaration(decl) }), Keyword(Func)=> self.func_declaration().map(|func| { Statement::Declaration(func) }), Keyword(Let) => self.binding_declaration().map(|decl| Statement::Declaration(decl)), @@ -331,10 +327,10 @@ impl Parser { #[recursive_descent_method] fn type_declaration_body(&mut self) -> ParseResult { - if let Keyword(Alias) = self.peek() { + if let Keyword(Alias) = self.token_handler.peek_kind() { self.type_alias() } else { - let mutable = if let Keyword(Mut) = self.peek() { + let mutable = if let Keyword(Mut) = self.token_handler.peek_kind() { self.token_handler.next(); true } else { @@ -361,7 +357,7 @@ impl Parser { let mut variants = Vec::new(); variants.push(self.variant_specifier()?); loop { - if let Pipe = self.peek() { + if let Pipe = self.token_handler.peek_kind() { self.token_handler.next(); variants.push(self.variant_specifier()?); } else { @@ -376,7 +372,7 @@ impl Parser { use self::Variant::*; let name = self.identifier()?; - match self.peek() { + match self.token_handler.peek_kind() { LParen => { let tuple_members = delimited!(self, LParen, type_name, Comma, RParen); Ok(TupleStruct(name, tuple_members)) @@ -400,7 +396,7 @@ impl Parser { #[recursive_descent_method] fn func_declaration(&mut self) -> ParseResult { let signature = self.func_signature()?; - if let LCurlyBrace = self.peek() { + if let LCurlyBrace = self.token_handler.peek_kind() { let statements = self.nonempty_func_body()?; Ok(Declaration::FuncDecl(signature, statements)) } else { @@ -411,7 +407,7 @@ impl Parser { #[recursive_descent_method] fn func_signature(&mut self) -> ParseResult { expect!(self, Keyword(Func)); - let (name, operator) = match self.peek() { + let (name, operator) = match self.token_handler.peek_kind() { Operator(s) => { let name = s.clone(); self.token_handler.next(); @@ -420,7 +416,7 @@ impl Parser { _ => (self.identifier()?, false) }; let params = self.formal_param_list()?; - let type_anno = match self.peek() { + let type_anno = match self.token_handler.peek_kind() { Colon => Some(self.type_anno()?), _ => None, }; @@ -441,7 +437,7 @@ impl Parser { #[recursive_descent_method] fn formal_param(&mut self) -> ParseResult { let name = self.identifier()?; - let ty = match self.peek() { + let ty = match self.token_handler.peek_kind() { Colon => Some(self.type_anno()?), _ => None }; @@ -451,7 +447,7 @@ impl Parser { #[recursive_descent_method] fn binding_declaration(&mut self) -> ParseResult { expect!(self, Keyword(Kw::Let)); - let constant = match self.peek() { + let constant = match self.token_handler.peek_kind() { Keyword(Kw::Mut) => { self.token_handler.next(); false @@ -482,7 +478,7 @@ impl Parser { fn impl_declaration(&mut self) -> ParseResult { expect!(self, Keyword(Impl)); let first = self.type_name()?; - let second = if let Keyword(For) = self.peek() { + let second = if let Keyword(For) = self.token_handler.peek_kind() { self.token_handler.next(); Some(self.type_name()?) } else { @@ -512,7 +508,7 @@ impl Parser { #[recursive_descent_method] fn expression(&mut self) -> ParseResult { let mut expr_body = self.precedence_expr(BinOp::min_precedence())?; - let type_anno = match self.peek() { + let type_anno = match self.token_handler.peek_kind() { Colon => Some(self.type_anno()?), _ => None }; @@ -532,7 +528,7 @@ impl Parser { #[recursive_descent_method] fn type_name(&mut self) -> ParseResult { use self::TypeIdentifier::*; - Ok(match self.peek() { + Ok(match self.token_handler.peek_kind() { LParen => Tuple(delimited!(self, LParen, type_name, Comma, RParen)), _ => Singleton(self.type_singleton_name()?), }) @@ -542,7 +538,7 @@ impl Parser { fn type_singleton_name(&mut self) -> ParseResult { Ok(TypeSingletonName { name: self.identifier()?, - params: match self.peek() { + params: match self.token_handler.peek_kind() { LAngleBracket => delimited!(self, LAngleBracket, type_name, Comma, RAngleBracket), _ => vec![], } @@ -553,7 +549,7 @@ impl Parser { fn precedence_expr(&mut self, precedence: i32) -> ParseResult { let record = ParseRecord { production_name: "precedence_expr".to_string(), - next_token: format!("{}", self.token_handler.peek_token().to_string_with_metadata()), + next_token: format!("{}", self.token_handler.peek().to_string_with_metadata()), level: self.parse_level, }; self.parse_level += 1; @@ -561,7 +557,7 @@ impl Parser { let mut lhs = self.prefix_expr()?; loop { - let new_precedence = match BinOp::get_precedence_from_token(&self.peek()) { + let new_precedence = match BinOp::get_precedence_from_token(&self.token_handler.peek_kind()) { Some(p) => p, None => break, }; @@ -583,7 +579,7 @@ impl Parser { #[recursive_descent_method] fn prefix_expr(&mut self) -> ParseResult { - match self.peek() { + match self.token_handler.peek_kind() { Operator(ref op) if PrefixOp::is_prefix(&*op) => { let sigil = match self.token_handler.next().kind { Operator(op) => op, @@ -602,7 +598,7 @@ impl Parser { #[recursive_descent_method] fn call_expr(&mut self) -> ParseResult { let mut expr = self.index_expr()?; - while let LParen = self.peek() { + while let LParen = self.token_handler.peek_kind() { let arguments = delimited!(self, LParen, expression, Comma, RParen); let arguments = arguments.into_iter().map(|s| Node::new(s)).collect(); expr = Expression(ExpressionType::Call { f: bx!(expr), arguments }, None); //TODO none is incorrect @@ -614,7 +610,7 @@ impl Parser { #[recursive_descent_method] fn index_expr(&mut self) -> ParseResult { let primary = self.primary()?; - Ok(if let LSquareBracket = self.peek() { + Ok(if let LSquareBracket = self.token_handler.peek_kind() { let indexers = delimited!(self, LSquareBracket, expression, Comma, RSquareBracket); Expression(ExpressionType::Index { indexee: bx!(Expression(primary.0, None)), @@ -627,7 +623,7 @@ impl Parser { #[recursive_descent_method] fn primary(&mut self) -> ParseResult { - match self.peek() { + match self.token_handler.peek_kind() { LCurlyBrace => self.curly_brace_expr(), Backslash => self.lambda_expr(), LParen => self.paren_expr(), @@ -655,7 +651,7 @@ impl Parser { fn lambda_expr(&mut self) -> ParseResult { expect!(self, Backslash); let params = self.lambda_param_list()?; - let type_anno = match self.peek() { + let type_anno = match self.token_handler.peek_kind() { Colon => Some(self.type_anno()?), _ => None, }; @@ -665,7 +661,7 @@ impl Parser { #[recursive_descent_method] fn lambda_param_list(&mut self) -> ParseResult> { - if let LParen = self.peek() { + if let LParen = self.token_handler.peek_kind() { self.formal_param_list() } else { let single_param = self.formal_param()?; @@ -697,7 +693,7 @@ impl Parser { fn identifier_expr(&mut self) -> ParseResult { use self::ExpressionType::*; let identifier = self.identifier()?; - Ok(match self.peek() { + Ok(match self.token_handler.peek_kind() { LCurlyBrace if !self.restrictions.no_struct_literal => { let fields = self.record_block()?; Expression(NamedStruct { name: identifier, fields }, None) @@ -729,7 +725,7 @@ impl Parser { x? }); - let body = Box::new(match self.peek() { + let body = Box::new(match self.token_handler.peek_kind() { Keyword(Kw::Then) => self.conditional()?, Keyword(Kw::Is) => self.simple_pattern_match()? , _ => self.guard_block()? @@ -741,7 +737,7 @@ impl Parser { #[recursive_descent_method] fn discriminator(&mut self) -> ParseResult { let lhs = self.prefix_expr()?; - let ref next = self.peek(); + let ref next = self.token_handler.peek_kind(); Ok(if let Some(op) = BinOp::from_sigil_token(next) { Discriminator::BinOp(lhs, op) } else { @@ -769,7 +765,7 @@ impl Parser { #[recursive_descent_method] fn else_clause(&mut self) -> ParseResult> { - Ok(if let Keyword(Kw::Else) = self.peek() { + Ok(if let Keyword(Kw::Else) = self.token_handler.peek_kind() { self.token_handler.next(); Some(self.expr_or_block()?) } else { @@ -785,19 +781,19 @@ impl Parser { let mut guards = vec![]; loop { - match self.peek() { + match self.token_handler.peek_kind() { RCurlyBrace | EOF => break, Semicolon | Newline => { self.token_handler.next(); continue}, _ => { let guard_arm = self.guard_arm()?; guards.push(guard_arm); loop { - match self.peek() { + match self.token_handler.peek_kind() { Semicolon | Newline => { self.token_handler.next(); continue; }, _ => break, } } - if let RCurlyBrace = self.peek() { + if let RCurlyBrace = self.token_handler.peek_kind() { break; } expect!(self, Comma); @@ -818,7 +814,7 @@ impl Parser { #[recursive_descent_method] fn guard(&mut self) -> ParseResult { - Ok(match self.peek() { + Ok(match self.token_handler.peek_kind() { Keyword(Kw::Is) => { self.token_handler.next(); let pat = self.pattern()?; @@ -840,7 +836,7 @@ impl Parser { #[recursive_descent_method] fn pattern(&mut self) -> ParseResult { - if let LParen = self.peek() { + if let LParen = self.token_handler.peek_kind() { let tuple_pattern_variants = delimited!(self, LParen, pattern, Comma, RParen); Ok(Pattern::TuplePattern(tuple_pattern_variants)) } else { @@ -851,11 +847,11 @@ impl Parser { #[recursive_descent_method] fn simple_pattern(&mut self) -> ParseResult { Ok({ - let tok = self.token_handler.peek_token(); + let tok = self.token_handler.peek(); match tok.get_kind() { Identifier(_) => { let id = self.identifier()?; - match self.peek() { + match self.token_handler.peek_kind() { LCurlyBrace => { let members = delimited!(self, LCurlyBrace, record_pattern_entry, Comma, RCurlyBrace); Pattern::Record(id, members) @@ -892,7 +888,7 @@ impl Parser { #[recursive_descent_method] fn signed_number_literal(&mut self) -> ParseResult { - let neg = match self.peek() { + let neg = match self.token_handler.peek_kind() { Operator(ref op) if **op == "-" => { self.token_handler.next(); true @@ -906,7 +902,7 @@ impl Parser { #[recursive_descent_method] fn record_pattern_entry(&mut self) -> ParseResult<(Rc, Pattern)> { let name = self.identifier()?; - Ok(match self.peek() { + Ok(match self.token_handler.peek_kind() { Colon => { expect!(self, Colon); let pat = self.pattern()?; @@ -924,7 +920,7 @@ impl Parser { #[recursive_descent_method] fn expr_or_block(&mut self) -> ParseResult { - match self.peek() { + match self.token_handler.peek_kind() { LCurlyBrace => self.block(), _ => { let expr = self.expression()?; @@ -949,7 +945,7 @@ impl Parser { #[recursive_descent_method] fn while_cond(&mut self) -> ParseResult> { - Ok(match self.peek() { + Ok(match self.token_handler.peek_kind() { LCurlyBrace => None, _ => Some(self.expression()?), }) @@ -958,7 +954,7 @@ impl Parser { #[recursive_descent_method] fn for_expr(&mut self) -> ParseResult { expect!(self, Keyword(Kw::For)); - let enumerators = if let LCurlyBrace = self.peek() { + let enumerators = if let LCurlyBrace = self.token_handler.peek_kind() { delimited!(self, LCurlyBrace, enumerator, Comma | Newline, RCurlyBrace) } else { let single_enum = { @@ -984,7 +980,7 @@ impl Parser { #[recursive_descent_method] fn for_expr_body(&mut self) -> ParseResult { use self::ForBody::*; - let tok = self.token_handler.peek_token(); + let tok = self.token_handler.peek(); Ok(match tok.get_kind() { LCurlyBrace => { let statements = delimited!(self, LCurlyBrace, statement, Newline | Semicolon, RCurlyBrace, nonstrict); @@ -1011,7 +1007,7 @@ impl Parser { fn literal(&mut self) -> ParseResult { use self::ExpressionType::*; - let tok = self.token_handler.peek_token(); + let tok = self.token_handler.peek(); match tok.get_kind() { DigitGroup(_) | HexLiteral(_) | BinNumberSigil | Period => self.number_literal(), Keyword(Kw::True) => { @@ -1032,7 +1028,7 @@ impl Parser { #[recursive_descent_method] fn number_literal(&mut self) -> ParseResult { - match self.peek() { + match self.token_handler.peek_kind() { HexLiteral(_) | BinNumberSigil => self.int_literal(), _ => self.float_literal(), } @@ -1061,7 +1057,7 @@ impl Parser { fn float_literal(&mut self) -> ParseResult { use self::ExpressionType::*; let mut digits = self.digits()?; - if let Period = self.peek() { + if let Period = self.token_handler.peek_kind() { self.token_handler.next(); digits.push_str("."); digits.push_str(&self.digits()?); @@ -1082,7 +1078,7 @@ impl Parser { fn digits(&mut self) -> ParseResult { let mut ds = String::new(); loop { - match self.peek() { + match self.token_handler.peek_kind() { Underscore => { self.token_handler.next(); continue; }, DigitGroup(ref s) => { self.token_handler.next(); ds.push_str(s)}, _ => break,