From e436533638dbeabce98727227a1833af4d216b89 Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Fri, 8 Jan 2016 23:59:15 -0800
Subject: [PATCH] Passing test

---
 src/tokenizer.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index fae8a73..0aea013 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -117,6 +117,8 @@ pub fn tokenize(input: &str) -> Option<Vec<Token>> {
         tokens.push(cur_tok);
     }
 
+    tokens.push(EOF);
+
     Some(tokens)
 }
 
@@ -126,6 +128,9 @@ mod tests {
 
     #[test]
     fn tokeniziation_tests() {
-        let t1 = "let a = 3\n";
+        let input1 = "let a = 3\n";
+        let token1 = tokenize(input1).unwrap();
+        assert_eq!(format!("{:?}", token1),
+            "[Identifier(\"let\"), Identifier(\"a\"), Identifier(\"=\"), NumLiteral(3), Newline, EOF]");
     }
 }