Fix interspersing of newlines in tokenizer infra

schala-lang/src/lib.rs

@@ -82,7 +82,6 @@ impl ProgrammingLanguageInterface for Schala {
       }
     };
 
-
     //symbol table
     match self.type_context.add_top_level_types(&ast) {
       Ok(()) => (),

schala-lang/src/tokenizing.rs

@@ -111,9 +111,11 @@ pub fn tokenize(input: &str) -> Vec<Token> {
   let mut tokens: Vec<Token> = Vec::new();
 
   let mut input  = input.lines().enumerate()
+      .intersperse((0, "\n"))
       .flat_map(|(line_idx, ref line)| {
           line.chars().enumerate().map(move |(ch_idx, ch)| (line_idx, ch_idx, ch))
-      }).peekable();
+      })
+      .peekable();
 
   while let Some((line_idx, ch_idx, c)) = input.next() {
     let cur_tok_type = match c {

source_files/schala/first.schala

@@ -4,7 +4,7 @@ fn main() {
   a + b
 }
 
-foo
+//foo
 
 print(main())