Comma as separate token

This commit is contained in:
greg 2015-07-19 14:24:42 -07:00
parent c6059ada7d
commit 8e3774ffca

View File

@ -14,6 +14,7 @@ enum Token {
Separator,
LParen,
RParen,
Comma,
NumLiteral(i32),
StrLiteral(String),
Identifier(String)
@ -85,8 +86,9 @@ fn tokenize(input: &str) -> Vec<Token> {
tokens.push(Token::LParen);
} else if c == ')' {
tokens.push(Token::RParen);
} else if c == ',' {
tokens.push(Token::Comma);
} else {
let mut buffer = String::with_capacity(20);
buffer.push(c);