From 841b38d5b122415e3a0b66c2d4c9c5e86a100ab1 Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Sun, 20 Dec 2015 17:03:03 -0800
Subject: [PATCH] Add tokenization test

---
 src/tokenizer.rs | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index f5b19ab..b3f96ea 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -115,3 +115,15 @@ fn handle_identifier(identifier: String) -> Token {
 
     return Token::Keyword(keyword);
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn tokeniziation_tests() {
+        let t1 = "let a = 3\n";
+        assert_eq!(format!("{:?}", tokenize(t1)),
+            "[Keyword(Let), Identifier(\"a\"), Keyword(Assign), NumLiteral(3), Separator, EOF]");
+    }
+}