From 815f2b8242ec561d6712c4cbfad94efc703b4062 Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Fri, 3 Feb 2017 11:34:26 -0800
Subject: [PATCH] Starting work on Maaru tokens

---
 src/maaru_lang/mod.rs | 60 ++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 57 insertions(+), 3 deletions(-)

diff --git a/src/maaru_lang/mod.rs b/src/maaru_lang/mod.rs
index e016e1a..84a8506 100644
--- a/src/maaru_lang/mod.rs
+++ b/src/maaru_lang/mod.rs
@@ -1,3 +1,6 @@
+extern crate itertools;
+use self::itertools::Itertools;
+
 use language::{ProgrammingLanguage, EvaluationMachine, ParseError, TokenError, LLVMCodeString};
 
 pub struct Maaru {
@@ -14,7 +17,25 @@ pub struct MaaruEvaluator {
 }
 
 #[derive(Debug)]
-pub struct Token { }
+pub enum Token {
+    StrLiteral(String),
+    Newline,
+    LParen,
+    RParen,
+    LBracket,
+    RBracket,
+    LBrace,
+    RBrace,
+    Comma,
+    Identifier(String),
+    Operator(String),
+    NumLiteral(Number),
+}
+
+#[derive(Debug)]
+pub enum Number {
+    Integer(u64),
+}
 
 #[derive(Debug)]
 pub struct AST { }
@@ -28,8 +49,41 @@ impl ProgrammingLanguage for Maaru {
         "Maaru".to_string()
     }
 
-    fn tokenize(_input: &str) -> Result<Vec<Self::Token>, TokenError> {
-        Ok(vec![Token { }])
+    fn tokenize(input: &str) -> Result<Vec<Self::Token>, TokenError> {
+        use self::Token::*;
+        let mut tokens = Vec::new();
+        let mut iter = input.chars().peekable();
+        while let Some(c) = iter.next() {
+            if c == ';' {
+                while let Some(c) = iter.next() {
+                    if c == '\n' {
+                        break;
+                    }
+                }
+                continue;
+            }
+            let cur_tok = match c {
+                c if char::is_whitespace(c) && c != '\n' => continue,
+                '\n' => Newline,
+                '(' => LParen,
+                ')' => RParen,
+                '[' => LBracket,
+                ']' => RBracket,
+                '{' => LBrace,
+                '}' => RBrace,
+                ',' => Comma,
+                c if char::is_alphanumeric(c) => {
+                    let mut buffer = String::new();
+                    buffer.push(c);
+                    buffer.extend(iter.peeking_take_while(|x| char::is_whitespace(*x)));
+                    Identifier(buffer)
+                },
+                _ => unimplemented!(),
+            };
+            tokens.push(cur_tok);
+        }
+
+        Ok(tokens)
     }
 
     fn parse(_input: Vec<Self::Token>) -> Result<Self::AST, ParseError> {