greg/schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor parsing structure

Browse Source 
TokenHandler should contain all the methods for actually manipulating
tokens, Parser should only contain the recursive descent methods
This commit is contained in:
greg 2019-01-05 18:38:18 -08:00
parent 2b338fd3c9
commit 5491169d55
2 changed files with 53 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
schala-lang/codegen/src/lib.rs
View File

@ -19,7 +19,7 @@ impl Fold for RecursiveDescentFn {
let new_block: syn::Block = parse_quote! { let new_block: syn::Block = parse_quote! {
{ {
let next_token_before_parse = self.token_handler.peek_token(); let next_token_before_parse = self.token_handler.peek();
let record = ParseRecord { let record = ParseRecord {
production_name: stringify!(#ident).to_string(), production_name: stringify!(#ident).to_string(),
next_token: format!("{}", next_token_before_parse.to_string_with_metadata()), next_token: format!("{}", next_token_before_parse.to_string_with_metadata()),
@ -34,7 +34,7 @@ impl Fold for RecursiveDescentFn {
} }
match result { match result {
Err(ParseError { token: None, msg }) => { Err(ParseError { token: None, msg }) => {
let next_token_after_parse = self.token_handler.peek_token(); let next_token_after_parse = self.token_handler.peek();
println!("HERE? {:?}", next_token_after_parse); println!("HERE? {:?}", next_token_after_parse);
Err(ParseError { token: Some(next_token_after_parse), msg }) Err(ParseError { token: Some(next_token_after_parse), msg })
}, },

106
schala-lang/language/src/parsing.rs
View File

@ -67,10 +67,10 @@ impl TokenHandler {
TokenHandler { tokens, end_of_file } TokenHandler { tokens, end_of_file }
} }
fn peek(&mut self) -> TokenKind { fn peek_kind(&mut self) -> TokenKind {
self.tokens.peek().map(|ref t| { t.kind.clone() }).unwrap_or(TokenKind::EOF) self.tokens.peek().map(|ref t| { t.kind.clone() }).unwrap_or(TokenKind::EOF)
} }
fn peek_token(&mut self) -> Token { fn peek(&mut self) -> Token {
self.tokens.peek().map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, offset: self.end_of_file}) self.tokens.peek().map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, offset: self.end_of_file})
} }
fn next(&mut self) -> Token { fn next(&mut self) -> Token {
@ -88,10 +88,6 @@ impl Parser {
} }
} }
fn peek(&mut self) -> TokenKind {
self.token_handler.peek()
}
pub fn parse(&mut self) -> ParseResult<AST> { pub fn parse(&mut self) -> ParseResult<AST> {
self.program() self.program()
} }
@ -121,7 +117,7 @@ macro_rules! expect {
($self:expr, $token_kind:pat) => { expect!($self, $token_kind if true) }; ($self:expr, $token_kind:pat) => { expect!($self, $token_kind if true) };
($self:expr, $expected_kind:pat if $cond:expr) => { ($self:expr, $expected_kind:pat if $cond:expr) => {
{ {
let tok = $self.token_handler.peek_token(); let tok = $self.token_handler.peek();
match tok.get_kind() { match tok.get_kind() {
$expected_kind if $cond => $self.token_handler.next(), $expected_kind if $cond => $self.token_handler.next(),
actual_kind => { actual_kind => {
@ -145,19 +141,19 @@ macro_rules! delimited {
expect!($self, $start); expect!($self, $start);
let mut acc = vec![]; let mut acc = vec![];
loop { loop {
let peek = $self.peek(); let peek = $self.token_handler.peek();
match peek { match peek.get_kind() {
$end | EOF => break, $end | EOF => break,
_ => (), _ => (),
} }
if !$strictness { if !$strictness {
match peek { match peek.get_kind() {
$( $delim )|* => { $self.token_handler.next(); continue }, $( $delim )|* => { $self.token_handler.next(); continue },
_ => () _ => ()
} }
} }
acc.push($self.$parse_fn()?); acc.push($self.$parse_fn()?);
match $self.peek() { match $self.token_handler.peek().get_kind() {
$( $delim )|* => { $self.token_handler.next(); continue }, $( $delim )|* => { $self.token_handler.next(); continue },
_ if $strictness => break, _ if $strictness => break,
_ => continue, _ => continue,
@ -296,7 +292,7 @@ impl Parser {
fn program(&mut self) -> ParseResult<AST> { fn program(&mut self) -> ParseResult<AST> {
let mut statements = Vec::new(); let mut statements = Vec::new();
loop { loop {
match self.peek() { match self.token_handler.peek().get_kind() {
EOF => break, EOF => break,
Newline | Semicolon => { Newline | Semicolon => {
self.token_handler.next(); self.token_handler.next();
@ -313,7 +309,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn statement(&mut self) -> ParseResult<Statement> { fn statement(&mut self) -> ParseResult<Statement> {
//TODO handle error recovery here //TODO handle error recovery here
match self.peek() { match self.token_handler.peek().get_kind() {
Keyword(Type) => self.type_declaration().map(|decl| { Statement::Declaration(decl) }), Keyword(Type) => self.type_declaration().map(|decl| { Statement::Declaration(decl) }),
Keyword(Func)=> self.func_declaration().map(|func| { Statement::Declaration(func) }), Keyword(Func)=> self.func_declaration().map(|func| { Statement::Declaration(func) }),
Keyword(Let) => self.binding_declaration().map(|decl| Statement::Declaration(decl)), Keyword(Let) => self.binding_declaration().map(|decl| Statement::Declaration(decl)),
@ -331,10 +327,10 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn type_declaration_body(&mut self) -> ParseResult<Declaration> { fn type_declaration_body(&mut self) -> ParseResult<Declaration> {
if let Keyword(Alias) = self.peek() { if let Keyword(Alias) = self.token_handler.peek_kind() {
self.type_alias() self.type_alias()
} else { } else {
let mutable = if let Keyword(Mut) = self.peek() { let mutable = if let Keyword(Mut) = self.token_handler.peek_kind() {
self.token_handler.next(); self.token_handler.next();
true true
} else { } else {
@ -361,7 +357,7 @@ impl Parser {
let mut variants = Vec::new(); let mut variants = Vec::new();
variants.push(self.variant_specifier()?); variants.push(self.variant_specifier()?);
loop { loop {
if let Pipe = self.peek() { if let Pipe = self.token_handler.peek_kind() {
self.token_handler.next(); self.token_handler.next();
variants.push(self.variant_specifier()?); variants.push(self.variant_specifier()?);
} else { } else {
@ -376,7 +372,7 @@ impl Parser {
use self::Variant::*; use self::Variant::*;
let name = self.identifier()?; let name = self.identifier()?;
match self.peek() { match self.token_handler.peek_kind() {
LParen => { LParen => {
let tuple_members = delimited!(self, LParen, type_name, Comma, RParen); let tuple_members = delimited!(self, LParen, type_name, Comma, RParen);
Ok(TupleStruct(name, tuple_members)) Ok(TupleStruct(name, tuple_members))
@ -400,7 +396,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn func_declaration(&mut self) -> ParseResult<Declaration> { fn func_declaration(&mut self) -> ParseResult<Declaration> {
let signature = self.func_signature()?; let signature = self.func_signature()?;
if let LCurlyBrace = self.peek() { if let LCurlyBrace = self.token_handler.peek_kind() {
let statements = self.nonempty_func_body()?; let statements = self.nonempty_func_body()?;
Ok(Declaration::FuncDecl(signature, statements)) Ok(Declaration::FuncDecl(signature, statements))
} else { } else {
@ -411,7 +407,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn func_signature(&mut self) -> ParseResult<Signature> { fn func_signature(&mut self) -> ParseResult<Signature> {
expect!(self, Keyword(Func)); expect!(self, Keyword(Func));
let (name, operator) = match self.peek() { let (name, operator) = match self.token_handler.peek_kind() {
Operator(s) => { Operator(s) => {
let name = s.clone(); let name = s.clone();
self.token_handler.next(); self.token_handler.next();
@ -420,7 +416,7 @@ impl Parser {
_ => (self.identifier()?, false) _ => (self.identifier()?, false)
}; };
let params = self.formal_param_list()?; let params = self.formal_param_list()?;
let type_anno = match self.peek() { let type_anno = match self.token_handler.peek_kind() {
Colon => Some(self.type_anno()?), Colon => Some(self.type_anno()?),
_ => None, _ => None,
}; };
@ -441,7 +437,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn formal_param(&mut self) -> ParseResult<FormalParam> { fn formal_param(&mut self) -> ParseResult<FormalParam> {
let name = self.identifier()?; let name = self.identifier()?;
let ty = match self.peek() { let ty = match self.token_handler.peek_kind() {
Colon => Some(self.type_anno()?), Colon => Some(self.type_anno()?),
_ => None _ => None
}; };
@ -451,7 +447,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn binding_declaration(&mut self) -> ParseResult<Declaration> { fn binding_declaration(&mut self) -> ParseResult<Declaration> {
expect!(self, Keyword(Kw::Let)); expect!(self, Keyword(Kw::Let));
let constant = match self.peek() { let constant = match self.token_handler.peek_kind() {
Keyword(Kw::Mut) => { Keyword(Kw::Mut) => {
self.token_handler.next(); self.token_handler.next();
false false
@ -482,7 +478,7 @@ impl Parser {
fn impl_declaration(&mut self) -> ParseResult<Declaration> { fn impl_declaration(&mut self) -> ParseResult<Declaration> {
expect!(self, Keyword(Impl)); expect!(self, Keyword(Impl));
let first = self.type_name()?; let first = self.type_name()?;
let second = if let Keyword(For) = self.peek() { let second = if let Keyword(For) = self.token_handler.peek_kind() {
self.token_handler.next(); self.token_handler.next();
Some(self.type_name()?) Some(self.type_name()?)
} else { } else {
@ -512,7 +508,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn expression(&mut self) -> ParseResult<Expression> { fn expression(&mut self) -> ParseResult<Expression> {
let mut expr_body = self.precedence_expr(BinOp::min_precedence())?; let mut expr_body = self.precedence_expr(BinOp::min_precedence())?;
let type_anno = match self.peek() { let type_anno = match self.token_handler.peek_kind() {
Colon => Some(self.type_anno()?), Colon => Some(self.type_anno()?),
_ => None _ => None
}; };
@ -532,7 +528,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn type_name(&mut self) -> ParseResult<TypeIdentifier> { fn type_name(&mut self) -> ParseResult<TypeIdentifier> {
use self::TypeIdentifier::*; use self::TypeIdentifier::*;
Ok(match self.peek() { Ok(match self.token_handler.peek_kind() {
LParen => Tuple(delimited!(self, LParen, type_name, Comma, RParen)), LParen => Tuple(delimited!(self, LParen, type_name, Comma, RParen)),
_ => Singleton(self.type_singleton_name()?), _ => Singleton(self.type_singleton_name()?),
}) })
@ -542,7 +538,7 @@ impl Parser {
fn type_singleton_name(&mut self) -> ParseResult<TypeSingletonName> { fn type_singleton_name(&mut self) -> ParseResult<TypeSingletonName> {
Ok(TypeSingletonName { Ok(TypeSingletonName {
name: self.identifier()?, name: self.identifier()?,
params: match self.peek() { params: match self.token_handler.peek_kind() {
LAngleBracket => delimited!(self, LAngleBracket, type_name, Comma, RAngleBracket), LAngleBracket => delimited!(self, LAngleBracket, type_name, Comma, RAngleBracket),
_ => vec![], _ => vec![],
} }
@ -553,7 +549,7 @@ impl Parser {
fn precedence_expr(&mut self, precedence: i32) -> ParseResult<Expression> { fn precedence_expr(&mut self, precedence: i32) -> ParseResult<Expression> {
let record = ParseRecord { let record = ParseRecord {
production_name: "precedence_expr".to_string(), production_name: "precedence_expr".to_string(),
next_token: format!("{}", self.token_handler.peek_token().to_string_with_metadata()), next_token: format!("{}", self.token_handler.peek().to_string_with_metadata()),
level: self.parse_level, level: self.parse_level,
}; };
self.parse_level += 1; self.parse_level += 1;
@ -561,7 +557,7 @@ impl Parser {
let mut lhs = self.prefix_expr()?; let mut lhs = self.prefix_expr()?;
loop { loop {
let new_precedence = match BinOp::get_precedence_from_token(&self.peek()) { let new_precedence = match BinOp::get_precedence_from_token(&self.token_handler.peek_kind()) {
Some(p) => p, Some(p) => p,
None => break, None => break,
}; };
@ -583,7 +579,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn prefix_expr(&mut self) -> ParseResult<Expression> { fn prefix_expr(&mut self) -> ParseResult<Expression> {
match self.peek() { match self.token_handler.peek_kind() {
Operator(ref op) if PrefixOp::is_prefix(&*op) => { Operator(ref op) if PrefixOp::is_prefix(&*op) => {
let sigil = match self.token_handler.next().kind { let sigil = match self.token_handler.next().kind {
Operator(op) => op, Operator(op) => op,
@ -602,7 +598,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn call_expr(&mut self) -> ParseResult<Expression> { fn call_expr(&mut self) -> ParseResult<Expression> {
let mut expr = self.index_expr()?; let mut expr = self.index_expr()?;
while let LParen = self.peek() { while let LParen = self.token_handler.peek_kind() {
let arguments = delimited!(self, LParen, expression, Comma, RParen); let arguments = delimited!(self, LParen, expression, Comma, RParen);
let arguments = arguments.into_iter().map(|s| Node::new(s)).collect(); let arguments = arguments.into_iter().map(|s| Node::new(s)).collect();
expr = Expression(ExpressionType::Call { f: bx!(expr), arguments }, None); //TODO none is incorrect expr = Expression(ExpressionType::Call { f: bx!(expr), arguments }, None); //TODO none is incorrect
@ -614,7 +610,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn index_expr(&mut self) -> ParseResult<Expression> { fn index_expr(&mut self) -> ParseResult<Expression> {
let primary = self.primary()?; let primary = self.primary()?;
Ok(if let LSquareBracket = self.peek() { Ok(if let LSquareBracket = self.token_handler.peek_kind() {
let indexers = delimited!(self, LSquareBracket, expression, Comma, RSquareBracket); let indexers = delimited!(self, LSquareBracket, expression, Comma, RSquareBracket);
Expression(ExpressionType::Index { Expression(ExpressionType::Index {
indexee: bx!(Expression(primary.0, None)), indexee: bx!(Expression(primary.0, None)),
@ -627,7 +623,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn primary(&mut self) -> ParseResult<Expression> { fn primary(&mut self) -> ParseResult<Expression> {
match self.peek() { match self.token_handler.peek_kind() {
LCurlyBrace => self.curly_brace_expr(), LCurlyBrace => self.curly_brace_expr(),
Backslash => self.lambda_expr(), Backslash => self.lambda_expr(),
LParen => self.paren_expr(), LParen => self.paren_expr(),
@ -655,7 +651,7 @@ impl Parser {
fn lambda_expr(&mut self) -> ParseResult<Expression> { fn lambda_expr(&mut self) -> ParseResult<Expression> {
expect!(self, Backslash); expect!(self, Backslash);
let params = self.lambda_param_list()?; let params = self.lambda_param_list()?;
let type_anno = match self.peek() { let type_anno = match self.token_handler.peek_kind() {
Colon => Some(self.type_anno()?), Colon => Some(self.type_anno()?),
_ => None, _ => None,
}; };
@ -665,7 +661,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn lambda_param_list(&mut self) -> ParseResult<Vec<FormalParam>> { fn lambda_param_list(&mut self) -> ParseResult<Vec<FormalParam>> {
if let LParen = self.peek() { if let LParen = self.token_handler.peek_kind() {
self.formal_param_list() self.formal_param_list()
} else { } else {
let single_param = self.formal_param()?; let single_param = self.formal_param()?;
@ -697,7 +693,7 @@ impl Parser {
fn identifier_expr(&mut self) -> ParseResult<Expression> { fn identifier_expr(&mut self) -> ParseResult<Expression> {
use self::ExpressionType::*; use self::ExpressionType::*;
let identifier = self.identifier()?; let identifier = self.identifier()?;
Ok(match self.peek() { Ok(match self.token_handler.peek_kind() {
LCurlyBrace if !self.restrictions.no_struct_literal => { LCurlyBrace if !self.restrictions.no_struct_literal => {
let fields = self.record_block()?; let fields = self.record_block()?;
Expression(NamedStruct { name: identifier, fields }, None) Expression(NamedStruct { name: identifier, fields }, None)
@ -729,7 +725,7 @@ impl Parser {
x? x?
}); });
let body = Box::new(match self.peek() { let body = Box::new(match self.token_handler.peek_kind() {
Keyword(Kw::Then) => self.conditional()?, Keyword(Kw::Then) => self.conditional()?,
Keyword(Kw::Is) => self.simple_pattern_match()? , Keyword(Kw::Is) => self.simple_pattern_match()? ,
_ => self.guard_block()? _ => self.guard_block()?
@ -741,7 +737,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn discriminator(&mut self) -> ParseResult<Discriminator> { fn discriminator(&mut self) -> ParseResult<Discriminator> {
let lhs = self.prefix_expr()?; let lhs = self.prefix_expr()?;
let ref next = self.peek(); let ref next = self.token_handler.peek_kind();
Ok(if let Some(op) = BinOp::from_sigil_token(next) { Ok(if let Some(op) = BinOp::from_sigil_token(next) {
Discriminator::BinOp(lhs, op) Discriminator::BinOp(lhs, op)
} else { } else {
@ -769,7 +765,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn else_clause(&mut self) -> ParseResult<Option<Block>> { fn else_clause(&mut self) -> ParseResult<Option<Block>> {
Ok(if let Keyword(Kw::Else) = self.peek() { Ok(if let Keyword(Kw::Else) = self.token_handler.peek_kind() {
self.token_handler.next(); self.token_handler.next();
Some(self.expr_or_block()?) Some(self.expr_or_block()?)
} else { } else {
@ -785,19 +781,19 @@ impl Parser {
let mut guards = vec![]; let mut guards = vec![];
loop { loop {
match self.peek() { match self.token_handler.peek_kind() {
RCurlyBrace | EOF => break, RCurlyBrace | EOF => break,
Semicolon | Newline => { self.token_handler.next(); continue}, Semicolon | Newline => { self.token_handler.next(); continue},
_ => { _ => {
let guard_arm = self.guard_arm()?; let guard_arm = self.guard_arm()?;
guards.push(guard_arm); guards.push(guard_arm);
loop { loop {
match self.peek() { match self.token_handler.peek_kind() {
Semicolon | Newline => { self.token_handler.next(); continue; }, Semicolon | Newline => { self.token_handler.next(); continue; },
_ => break, _ => break,
} }
} }
if let RCurlyBrace = self.peek() { if let RCurlyBrace = self.token_handler.peek_kind() {
break; break;
} }
expect!(self, Comma); expect!(self, Comma);
@ -818,7 +814,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn guard(&mut self) -> ParseResult<Guard> { fn guard(&mut self) -> ParseResult<Guard> {
Ok(match self.peek() { Ok(match self.token_handler.peek_kind() {
Keyword(Kw::Is) => { Keyword(Kw::Is) => {
self.token_handler.next(); self.token_handler.next();
let pat = self.pattern()?; let pat = self.pattern()?;
@ -840,7 +836,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn pattern(&mut self) -> ParseResult<Pattern> { fn pattern(&mut self) -> ParseResult<Pattern> {
if let LParen = self.peek() { if let LParen = self.token_handler.peek_kind() {
let tuple_pattern_variants = delimited!(self, LParen, pattern, Comma, RParen); let tuple_pattern_variants = delimited!(self, LParen, pattern, Comma, RParen);
Ok(Pattern::TuplePattern(tuple_pattern_variants)) Ok(Pattern::TuplePattern(tuple_pattern_variants))
} else { } else {
@ -851,11 +847,11 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn simple_pattern(&mut self) -> ParseResult<Pattern> { fn simple_pattern(&mut self) -> ParseResult<Pattern> {
Ok({ Ok({
let tok = self.token_handler.peek_token(); let tok = self.token_handler.peek();
match tok.get_kind() { match tok.get_kind() {
Identifier(_) => { Identifier(_) => {
let id = self.identifier()?; let id = self.identifier()?;
match self.peek() { match self.token_handler.peek_kind() {
LCurlyBrace => { LCurlyBrace => {
let members = delimited!(self, LCurlyBrace, record_pattern_entry, Comma, RCurlyBrace); let members = delimited!(self, LCurlyBrace, record_pattern_entry, Comma, RCurlyBrace);
Pattern::Record(id, members) Pattern::Record(id, members)
@ -892,7 +888,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn signed_number_literal(&mut self) -> ParseResult<Pattern> { fn signed_number_literal(&mut self) -> ParseResult<Pattern> {
let neg = match self.peek() { let neg = match self.token_handler.peek_kind() {
Operator(ref op) if **op == "-" => { Operator(ref op) if **op == "-" => {
self.token_handler.next(); self.token_handler.next();
true true
@ -906,7 +902,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn record_pattern_entry(&mut self) -> ParseResult<(Rc<String>, Pattern)> { fn record_pattern_entry(&mut self) -> ParseResult<(Rc<String>, Pattern)> {
let name = self.identifier()?; let name = self.identifier()?;
Ok(match self.peek() { Ok(match self.token_handler.peek_kind() {
Colon => { Colon => {
expect!(self, Colon); expect!(self, Colon);
let pat = self.pattern()?; let pat = self.pattern()?;
@ -924,7 +920,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn expr_or_block(&mut self) -> ParseResult<Block> { fn expr_or_block(&mut self) -> ParseResult<Block> {
match self.peek() { match self.token_handler.peek_kind() {
LCurlyBrace => self.block(), LCurlyBrace => self.block(),
_ => { _ => {
let expr = self.expression()?; let expr = self.expression()?;
@ -949,7 +945,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn while_cond(&mut self) -> ParseResult<Option<Expression>> { fn while_cond(&mut self) -> ParseResult<Option<Expression>> {
Ok(match self.peek() { Ok(match self.token_handler.peek_kind() {
LCurlyBrace => None, LCurlyBrace => None,
_ => Some(self.expression()?), _ => Some(self.expression()?),
}) })
@ -958,7 +954,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn for_expr(&mut self) -> ParseResult<Expression> { fn for_expr(&mut self) -> ParseResult<Expression> {
expect!(self, Keyword(Kw::For)); expect!(self, Keyword(Kw::For));
let enumerators = if let LCurlyBrace = self.peek() { let enumerators = if let LCurlyBrace = self.token_handler.peek_kind() {
delimited!(self, LCurlyBrace, enumerator, Comma | Newline, RCurlyBrace) delimited!(self, LCurlyBrace, enumerator, Comma | Newline, RCurlyBrace)
} else { } else {
let single_enum = { let single_enum = {
@ -984,7 +980,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn for_expr_body(&mut self) -> ParseResult<ForBody> { fn for_expr_body(&mut self) -> ParseResult<ForBody> {
use self::ForBody::*; use self::ForBody::*;
let tok = self.token_handler.peek_token(); let tok = self.token_handler.peek();
Ok(match tok.get_kind() { Ok(match tok.get_kind() {
LCurlyBrace => { LCurlyBrace => {
let statements = delimited!(self, LCurlyBrace, statement, Newline | Semicolon, RCurlyBrace, nonstrict); let statements = delimited!(self, LCurlyBrace, statement, Newline | Semicolon, RCurlyBrace, nonstrict);
@ -1011,7 +1007,7 @@ impl Parser {
fn literal(&mut self) -> ParseResult<Expression> { fn literal(&mut self) -> ParseResult<Expression> {
use self::ExpressionType::*; use self::ExpressionType::*;
let tok = self.token_handler.peek_token(); let tok = self.token_handler.peek();
match tok.get_kind() { match tok.get_kind() {
DigitGroup(_) | HexLiteral(_) | BinNumberSigil | Period => self.number_literal(), DigitGroup(_) | HexLiteral(_) | BinNumberSigil | Period => self.number_literal(),
Keyword(Kw::True) => { Keyword(Kw::True) => {
@ -1032,7 +1028,7 @@ impl Parser {
#[recursive_descent_method] #[recursive_descent_method]
fn number_literal(&mut self) -> ParseResult<Expression> { fn number_literal(&mut self) -> ParseResult<Expression> {
match self.peek() { match self.token_handler.peek_kind() {
HexLiteral(_) | BinNumberSigil => self.int_literal(), HexLiteral(_) | BinNumberSigil => self.int_literal(),
_ => self.float_literal(), _ => self.float_literal(),
} }
@ -1061,7 +1057,7 @@ impl Parser {
fn float_literal(&mut self) -> ParseResult<Expression> { fn float_literal(&mut self) -> ParseResult<Expression> {
use self::ExpressionType::*; use self::ExpressionType::*;
let mut digits = self.digits()?; let mut digits = self.digits()?;
if let Period = self.peek() { if let Period = self.token_handler.peek_kind() {
self.token_handler.next(); self.token_handler.next();
digits.push_str("."); digits.push_str(".");
digits.push_str(&self.digits()?); digits.push_str(&self.digits()?);
@ -1082,7 +1078,7 @@ impl Parser {
fn digits(&mut self) -> ParseResult<String> { fn digits(&mut self) -> ParseResult<String> {
let mut ds = String::new(); let mut ds = String::new();
loop { loop {
match self.peek() { match self.token_handler.peek_kind() {
Underscore => { self.token_handler.next(); continue; }, Underscore => { self.token_handler.next(); continue; },
DigitGroup(ref s) => { self.token_handler.next(); ds.push_str(s)}, DigitGroup(ref s) => { self.token_handler.next(); ds.push_str(s)},
_ => break, _ => break,