Autoparser - token output

src/schala_lang/autoparser.rs

@@ -1,4 +1,5 @@
 use schala_lib::{ProgrammingLanguageInterface, EvalOptions, TraceArtifact, LanguageOutput};
+use itertools::Itertools;
 
 use schala_lang::{tokenizing, parsing};
 
@@ -20,6 +21,14 @@ impl ProgrammingLanguageInterface for Schala {
 
   fn evaluate_in_repl(&mut self, input: &str, options: &EvalOptions) -> LanguageOutput {
     let mut output = LanguageOutput::default();
+
+    let tokens = tokenizing::tokenize(input);
+    if options.debug_tokens {
+      let token_string = tokens.iter().map(|t| format!("{:?}<L:{},C:{}>", t.token_type, t.offset.0, t.offset.1)).join(", ");
+      output.add_artifact(TraceArtifact::new("tokens", format!("{:?}", token_string)));
+    }
+
+    output.add_output(format!("{:?}", tokens));
     output
   }
 }