From 429ace73bd6394a67d54ad6913e38f59f73dde71 Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Wed, 22 Jul 2015 03:02:55 -0700
Subject: [PATCH] Move tokenizing into separate module

---
 src/main.rs      | 17 ++++-------------
 src/tokenizer.rs | 13 +++++++++++++
 2 files changed, 17 insertions(+), 13 deletions(-)
 create mode 100644 src/tokenizer.rs

diff --git a/src/main.rs b/src/main.rs
index 3eab353..3e05f1e 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -4,7 +4,10 @@ use std::io::BufRead;
 use std::char;
 use std::slice::Iter;
 
-use Token::*;
+use tokenizer::Token;
+use tokenizer::Token::*;
+
+mod tokenizer;
 
 
 fn main() {
@@ -12,18 +15,6 @@ fn main() {
     repl();
 }
 
-#[derive(Debug, Clone)]
-enum Token {
-    EOF,
-    Separator,
-    LParen,
-    RParen,
-    Comma,
-    NumLiteral(f64),
-    StrLiteral(String),
-    Identifier(String)
-    /* Keyword(Keyword) */ //implement in future
-}
 
 #[derive(Debug)]
 enum AST {
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
new file mode 100644
index 0000000..994f5be
--- /dev/null
+++ b/src/tokenizer.rs
@@ -0,0 +1,13 @@
+
+#[derive(Debug, Clone)]
+pub enum Token {
+    EOF,
+    Separator,
+    LParen,
+    RParen,
+    Comma,
+    NumLiteral(f64),
+    StrLiteral(String),
+    Identifier(String)
+    /* Keyword(Keyword) */ //implement in future
+}