From e1aa7ecb17cb431ac6a97f95bc35619dfad39e3f Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Fri, 15 Jan 2016 00:31:00 -0800
Subject: [PATCH] Finish tokenizing Op separately

---
 src/tokenizer.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index d00f48b..e7d0469 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -98,11 +98,11 @@ pub fn tokenize(input: &str) -> Option<Vec<Token>> {
                 Ok(f) => NumLiteral(f),
                 Err(_) => return None
             }
-        } else if !char::is_alphanumeric(c) { //TODO see if this what I want
+        } else if !char::is_alphanumeric(c) {
             let mut buffer = String::with_capacity(20);
             buffer.push(c);
             loop {
-                if iter.peek().map_or(false, |x| is_digit(x) || *x == '.') {
+                if iter.peek().map_or(false, |x| !char::is_alphanumeric(*x)) {
                     let n = iter.next().unwrap();
                     buffer.push(n);
                 } else {