Add comma tokenization

This commit is contained in:
greg 2016-01-09 23:22:46 -08:00
parent 570650cbfa
commit 3e04cbfa29

View File

@ -39,6 +39,7 @@ fn ends_identifier(c: &char) -> bool {
c == ')' || c == ')' ||
c == ',' || c == ',' ||
c == '.' || c == '.' ||
c == ',' ||
c == ':' c == ':'
} }
@ -67,6 +68,8 @@ pub fn tokenize(input: &str) -> Option<Vec<Token>> {
RParen RParen
} else if c == ':' { } else if c == ':' {
Colon Colon
else if c == ',' {
Comma
} else if c == '"' { } else if c == '"' {
let mut buffer = String::with_capacity(20); let mut buffer = String::with_capacity(20);
loop { loop {