Collapse Separator tokens

only ever gonna be one in a row
This commit is contained in:
greg 2015-07-24 15:46:53 -07:00
parent 46999beabf
commit 169e662049

View File

@ -60,7 +60,10 @@ pub fn tokenize(input: &str) -> Vec<Token> {
}
}
} else if c == ';' || c == '\n' {
tokens.push(Token::Separator);
if let Some(&Token::Separator) = tokens.last() {
} else {
tokens.push(Token::Separator);
}
} else if c == '(' {
tokens.push(Token::LParen);
} else if c == ')' {