greg/schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Get rid of Separator token

Browse Source 
Have separate newline and semicolon tokens
This commit is contained in:
greg 2015-12-31 22:20:59 -08:00
parent 51745fd800
commit dd2b4893a4
2 changed files with 9 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/parser.rs
View File

@ -99,7 +99,7 @@ impl Parser {
fn parse(&mut self) -> ParseResult<AST> { fn parse(&mut self) -> ParseResult<AST> {
let r = self.expr(); let r = self.expr();
try!(self.expect(Token::Separator)); try!(self.expect(Token::Newline));
try!(self.expect(Token::EOF)); try!(self.expect(Token::EOF));
r r
} }

17
src/tokenizer.rs
View File

@ -1,7 +1,8 @@
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub enum Token { pub enum Token {
EOF, EOF,
Separator, Newline,
Semicolon,
LParen, LParen,
RParen, RParen,
Comma, Comma,
@ -62,12 +63,10 @@ pub fn tokenize(input: &str) -> Vec<Token> {
break; break;
} }
} }
} else if c == ';' || c == '\n' { } else if c == ';' {
if let Some(&Token::Separator) = tokens.last() { tokens.push(Token::Semicolon);
//skip past multiple separators } else if c == '\n' {
} else { tokens.push(Token::Newline);
tokens.push(Token::Separator);
}
} else if c == '(' { } else if c == '(' {
tokens.push(Token::LParen); tokens.push(Token::LParen);
} else if c == ')' { } else if c == ')' {
@ -123,12 +122,12 @@ mod tests {
fn tokeniziation_tests() { fn tokeniziation_tests() {
let t1 = "let a = 3\n"; let t1 = "let a = 3\n";
assert_eq!(format!("{:?}", tokenize(t1)), assert_eq!(format!("{:?}", tokenize(t1)),
"[Keyword(Let), Identifier(\"a\"), Keyword(Assign), NumLiteral(3), Separator, EOF]"); "[Keyword(Let), Identifier(\"a\"), Keyword(Assign), NumLiteral(3), Newline, EOF]");
// this is intentional // this is intentional
let t2 = "a + b*c\n"; let t2 = "a + b*c\n";
assert_eq!(format!("{:?}", tokenize(t2)), assert_eq!(format!("{:?}", tokenize(t2)),
"[Identifier(\"a\"), Identifier(\"+\"), Identifier(\"b*c\"), Separator, EOF]"); "[Identifier(\"a\"), Identifier(\"+\"), Identifier(\"b*c\"), Newline, EOF]");
} }
} }