greg/schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Get rid of Separator token

Browse Source 
Have separate newline and semicolon tokens
This commit is contained in:
greg 2015-12-31 22:20:59 -08:00
parent 51745fd800
commit dd2b4893a4
2 changed files with 9 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/parser.rs
View File

@ -99,7 +99,7 @@ impl Parser {
fn parse(&mut self) -> ParseResult<AST> {
let r = self.expr();
try!(self.expect(Token::Separator));
try!(self.expect(Token::Newline));
try!(self.expect(Token::EOF));
r
}

17
src/tokenizer.rs
View File

@ -1,7 +1,8 @@
#[derive(Debug, Clone, PartialEq)]
pub enum Token {
EOF,
Separator,
Newline,
Semicolon,
LParen,
RParen,
Comma,
@ -62,12 +63,10 @@ pub fn tokenize(input: &str) -> Vec<Token> {
break;
}
}
} else if c == ';' || c == '\n' {
if let Some(&Token::Separator) = tokens.last() {
//skip past multiple separators
} else {
tokens.push(Token::Separator);
}
} else if c == ';' {
tokens.push(Token::Semicolon);
} else if c == '\n' {
tokens.push(Token::Newline);
} else if c == '(' {
tokens.push(Token::LParen);
} else if c == ')' {
@ -123,12 +122,12 @@ mod tests {
fn tokeniziation_tests() {
let t1 = "let a = 3\n";
assert_eq!(format!("{:?}", tokenize(t1)),
"[Keyword(Let), Identifier(\"a\"), Keyword(Assign), NumLiteral(3), Separator, EOF]");
"[Keyword(Let), Identifier(\"a\"), Keyword(Assign), NumLiteral(3), Newline, EOF]");
// this is intentional
let t2 = "a + b*c\n";
assert_eq!(format!("{:?}", tokenize(t2)),
"[Identifier(\"a\"), Identifier(\"+\"), Identifier(\"b*c\"), Separator, EOF]");
"[Identifier(\"a\"), Identifier(\"+\"), Identifier(\"b*c\"), Newline, EOF]");
}
}