From c0a5418c2732b879229cf635cb8c98e2bf36b3d6 Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Thu, 30 Nov 2017 22:37:49 -0800
Subject: [PATCH] Tokens

---
 src/rukka_lang/mod.rs | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/rukka_lang/mod.rs b/src/rukka_lang/mod.rs
index 998023e..e92ca9d 100644
--- a/src/rukka_lang/mod.rs
+++ b/src/rukka_lang/mod.rs
@@ -41,6 +41,28 @@ fn parse(input: &str) -> Result<Sexp, String> {
   }
 }
 
+#[derive(Debug)]
+enum Token {
+  LParen,
+  RParen,
+  Symbol(String)
+}
+
+fn tokenize(input: &mut Peekable<Chars>) -> Vec<Token> {
+  let mut tokens = Vec::new();
+  loop {
+    match input.peek().map(|x| *x) {
+      None => break,
+      Some('(') => tokens.push(LParen),
+      Some(')') => tokens.push(RParen),
+      Some(c) if c.is_whitespace() => { tokens.next(); continue },
+      Some(c) => {
+
+      }
+    }
+  }
+}
+
 fn read_sexp(input: &mut Peekable<Chars>) -> Result<Sexp, String> {
   if input.next() != Some('(') {
     return Err(format!("Expected '('"));