From 71aef379d38e200af1e68ca2843ed6959ff8b177 Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Thu, 7 Jan 2016 01:09:18 -0800
Subject: [PATCH] Tokenize number literals

TODO: expand this bit of code to handle 0x12, etc. syntax
---
 src/tokenizer.rs | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 59418f5..5bb3aab 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -68,7 +68,20 @@ pub fn tokenize(input: &str) -> Option<Vec<Token>> {
             }
             StrLiteral(buffer)
         } else if is_digit(&c) {
-            NumLiteral(45.0)
+            let mut buffer = String::with_capacity(20);
+            buffer.push(c);
+            loop {
+                if iter.peek().map_or(false, |x| is_digit(x) || *x == '.') {
+                    let n = iter.next().unwrap();
+                    buffer.push(n);
+                } else {
+                    break;
+                }
+            }
+            match buffer.parse::<f64>() {
+                Ok(f) => NumLiteral(f),
+                Err(_) => return None
+            }
         } else {
             Identifier("DUMMY".to_string())
         };