From 8662a3ba0e7e55290ccec78adfccd1a8dad86a3a Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Wed, 6 Jan 2016 23:48:53 -0800
Subject: [PATCH] Make tokenize error-able

---
 src/main.rs      |  8 ++++++--
 src/tokenizer.rs | 10 ++++++----
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/main.rs b/src/main.rs
index 9aff043..aeb31cb 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -55,15 +55,19 @@ impl ReplState for InterpreterState {
 }
 
 fn repl_handler(input: &str, state: &mut InterpreterState) -> String {
+    let tokens = match tokenize(input) {
+        None => return format!("Failure to tokenize"),
+        Some(t) => t
+    };
+
     if state.show_tokens {
-        println!("Tokens: {:?}", tokenize(input));
+        println!("Tokens: {:?}", tokens);
     }
 
     if state.show_parse {
         println!("not implemented")
     }
 
-    let tokens = tokenize(input);
     let ast = parse(&tokens);
     format!("{:?}", tokens)
 }
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 77e8df3..59418f5 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -31,7 +31,7 @@ fn is_digit(c: &char) -> bool {
     c.is_digit(10)
 }
 
-pub fn tokenize(input: &str) -> Vec<Token> {
+pub fn tokenize(input: &str) -> Option<Vec<Token>> {
     use self::Token::*;
     let mut tokens = Vec::new();
     let mut iter = input.chars().peekable();
@@ -63,18 +63,20 @@ pub fn tokenize(input: &str) -> Vec<Token> {
                 match iter.next() {
                     Some(x) if x == '"' => break,
                     Some(x) => buffer.push(x),
-                    None => return tokens,
+                    None => return None,
                 }
             }
             StrLiteral(buffer)
+        } else if is_digit(&c) {
+            NumLiteral(45.0)
         } else {
-            StrLiteral("DUMMY".to_string())
+            Identifier("DUMMY".to_string())
         };
 
         tokens.push(cur_tok);
     }
 
-    tokens
+    Some(tokens)
 }
 
 #[cfg(test)]