From ecebbb2eaea4f4f434119a859bdb1d39a382f2b1 Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Sat, 24 Mar 2018 18:38:28 -0700
Subject: [PATCH] Fix interspersing of newlines in tokenizer infra

---
 schala-lang/src/lib.rs           | 1 -
 schala-lang/src/tokenizing.rs    | 4 +++-
 source_files/schala/first.schala | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/schala-lang/src/lib.rs b/schala-lang/src/lib.rs
index ca006a5..ca20e71 100644
--- a/schala-lang/src/lib.rs
+++ b/schala-lang/src/lib.rs
@@ -82,7 +82,6 @@ impl ProgrammingLanguageInterface for Schala {
       }
     };
 
-
     //symbol table
     match self.type_context.add_top_level_types(&ast) {
       Ok(()) => (),
diff --git a/schala-lang/src/tokenizing.rs b/schala-lang/src/tokenizing.rs
index e2a6c08..2692aeb 100644
--- a/schala-lang/src/tokenizing.rs
+++ b/schala-lang/src/tokenizing.rs
@@ -111,9 +111,11 @@ pub fn tokenize(input: &str) -> Vec<Token> {
   let mut tokens: Vec<Token> = Vec::new();
 
   let mut input  = input.lines().enumerate()
+      .intersperse((0, "\n"))
       .flat_map(|(line_idx, ref line)| {
           line.chars().enumerate().map(move |(ch_idx, ch)| (line_idx, ch_idx, ch))
-      }).peekable();
+      })
+      .peekable();
 
   while let Some((line_idx, ch_idx, c)) = input.next() {
     let cur_tok_type = match c {
diff --git a/source_files/schala/first.schala b/source_files/schala/first.schala
index a8cca94..25e958d 100644
--- a/source_files/schala/first.schala
+++ b/source_files/schala/first.schala
@@ -4,7 +4,7 @@ fn main() {
   a + b
 }
 
-foo
+//foo
 
 print(main())