From c6059ada7d778b2a046f3a6e2d6f70b21efce403 Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Sun, 19 Jul 2015 13:55:34 -0700
Subject: [PATCH] Separators and parens

Separator = ; or \n, they are equivalent
---
 src/main.rs | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/main.rs b/src/main.rs
index 121aa13..85852b7 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -11,6 +11,9 @@ fn main() {
 #[derive(Debug)]
 enum Token {
     EOF,
+    Separator,
+    LParen,
+    RParen,
     NumLiteral(i32),
     StrLiteral(String),
     Identifier(String)
@@ -76,19 +79,26 @@ fn tokenize(input: &str) -> Vec<Token> {
                     break;
                 }
             }
+        } else if c == ';' || c == '\n' {
+            tokens.push(Token::Separator);
+        } else if c == '(' {
+            tokens.push(Token::LParen);
+        } else if c == ')' {
+            tokens.push(Token::RParen);
         } else {
+
             let mut buffer = String::with_capacity(20);
             buffer.push(c);
-            while let Some(x) = iterator.next() {
-                if char::is_whitespace(x) {
+
+            while let Some(x) = iterator.peek().cloned() {
+                if !char::is_alphanumeric(x) {
                     break;
                 }
-                buffer.push(x);
+                buffer.push(iterator.next().unwrap());
             }
             tokens.push(Token::Identifier(buffer));
         }
     }
-
     tokens.push(Token::EOF);
     tokens
 }