Make token debug nicer

src/main.rs

@@ -1,6 +1,7 @@
 #![feature(advanced_slice_patterns, slice_patterns, box_patterns)]
 extern crate getopts;
 extern crate linefeed;
+extern crate itertools;
 #[macro_use]
 extern crate lazy_static;
 #[macro_use]

src/robo_lang/mod.rs

@@ -1,5 +1,4 @@
-extern crate itertools;
+use itertools::Itertools;
-use self::itertools::Itertools;
 
 use language::{ProgrammingLanguageInterface, EvalOptions, ReplOutput, TokenError};
 

src/schala_lang/mod.rs

@@ -1,3 +1,4 @@
+use itertools::Itertools;
 use language::{ProgrammingLanguageInterface, EvalOptions, TraceArtifact, ReplOutput};
 
 mod parsing;
@@ -20,7 +21,9 @@ impl ProgrammingLanguageInterface for Schala {
     let mut output = ReplOutput::default();
     let tokens = parsing::tokenize(input);
     if options.debug_tokens {
-      output.add_artifact(TraceArtifact::new("tokens", format!("{:?}", tokens)));
+      let token_string = tokens.iter().map(|t| format!("{:?}<{}>", t.token_type, t.offset)).join(", ");
+      output.add_artifact(TraceArtifact::new("tokens", format!("{:?}", token_string)));
+
     }
 
     {

src/schala_lang/parsing.rs

@@ -1,9 +1,7 @@
-extern crate itertools;
+use itertools::Itertools;
-
 use std::collections::HashMap;
 use std::rc::Rc;
 use std::iter::{Enumerate, Peekable};
-use self::itertools::Itertools;
 use std::vec::IntoIter;
 use std::str::Chars;
 
@@ -71,8 +69,8 @@ lazy_static! {
 
 #[derive(Debug)]
 pub struct Token {
-  token_type: TokenType,
+  pub token_type: TokenType,
-  offset: usize,
+  pub offset: usize,
 }
 
 impl Token {