diff --git a/src/schala_lang/autoparser.rs b/src/schala_lang/autoparser.rs index 9a235f8..8816178 100644 --- a/src/schala_lang/autoparser.rs +++ b/src/schala_lang/autoparser.rs @@ -1,4 +1,5 @@ use schala_lib::{ProgrammingLanguageInterface, EvalOptions, TraceArtifact, LanguageOutput}; +use itertools::Itertools; use schala_lang::{tokenizing, parsing}; @@ -20,6 +21,14 @@ impl ProgrammingLanguageInterface for Schala { fn evaluate_in_repl(&mut self, input: &str, options: &EvalOptions) -> LanguageOutput { let mut output = LanguageOutput::default(); + + let tokens = tokenizing::tokenize(input); + if options.debug_tokens { + let token_string = tokens.iter().map(|t| format!("{:?}", t.token_type, t.offset.0, t.offset.1)).join(", "); + output.add_artifact(TraceArtifact::new("tokens", format!("{:?}", token_string))); + } + + output.add_output(format!("{:?}", tokens)); output } }