From 2aaa600d530d62e1ba7440d1c63c08cc5e56b393 Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Sun, 19 Jul 2015 00:12:27 -0700
Subject: [PATCH] More tokenizer stuff

---
 src/main.rs | 38 ++++++++++++++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 4 deletions(-)

diff --git a/src/main.rs b/src/main.rs
index 21c5c4d..121aa13 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,6 +1,7 @@
 use std::io;
 use std::io::Write;
 use std::io::BufRead;
+use std::char;
 
 fn main() {
     println!("Unnamed language 0.01");
@@ -34,6 +35,7 @@ fn repl() {
         match line {
             Ok(n) => {
                 let tokens = tokenize(&buf);
+                buf.clear();
                 println!("Tokens: {:?}", tokens);
 
                 let ast = parse(tokens);
@@ -52,11 +54,39 @@ fn repl() {
 
 fn tokenize(input: &str) -> Vec<Token> {
     let mut tokens = Vec::new();
-    let mut buffer = String::with_capacity(20);
-    let mut iterator = input.chars();
-    loop {
+    let mut iterator = input.chars().peekable();
 
-        break;
+    while let Some(c) = iterator.next() {
+        if char::is_whitespace(c) {
+            continue;
+        } else if c == '"' {
+
+            let mut buffer = String::with_capacity(20);
+            while let Some(x) = iterator.next() {
+                if x == '"' {
+                    break;
+                }
+                buffer.push(x);
+            }
+            tokens.push(Token::StrLiteral(buffer));
+
+        } else if c == '#' {
+            while let Some(x) = iterator.next() {
+                if x == '\n' {
+                    break;
+                }
+            }
+        } else {
+            let mut buffer = String::with_capacity(20);
+            buffer.push(c);
+            while let Some(x) = iterator.next() {
+                if char::is_whitespace(x) {
+                    break;
+                }
+                buffer.push(x);
+            }
+            tokens.push(Token::Identifier(buffer));
+        }
     }
 
     tokens.push(Token::EOF);