From 3e04cbfa29777a73f970419d4962d740d3aae587 Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Sat, 9 Jan 2016 23:22:46 -0800
Subject: [PATCH] Add comma tokenization

---
 src/tokenizer.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 3eeb7ee..7a31af3 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -39,6 +39,7 @@ fn ends_identifier(c: &char) -> bool {
     c == ')' ||
     c == ',' ||
     c == '.' ||
+    c == ',' ||
     c == ':'
 }
 
@@ -67,6 +68,8 @@ pub fn tokenize(input: &str) -> Option<Vec<Token>> {
             RParen
         } else if c == ':' {
             Colon
+        else if  c == ',' {
+            Comma
         } else if c == '"' {
             let mut buffer = String::with_capacity(20);
             loop {