Replace regex-based lexer with character-at-a-time lexer (#406)

Cargo.lock

@@ -206,7 +206,6 @@ dependencies = [
  "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.51 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "regex 1.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "target 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",

Cargo.toml

@@ -21,7 +21,6 @@ itertools      = "0.8.0"
 lazy_static    = "1.0.0"
 libc           = "0.2.21"
 log            = "0.4.4"
-regex          = "1.0.0"
 target         = "1.0.0"
 tempdir        = "0.3.5"
 unicode-width  = "0.1.3"

README.adoc

@@ -710,40 +710,6 @@ When a script with a shebang is executed, the system supplies the path to the sc
 
 With the above shebang, `just` will change its working directory to the location of the script. If you'd rather leave the working directory unchanged, use `#!/usr/bin/env just --working-directory . --justfile`.
 
-== Frequently Asked Questions
-
-=== What are the idiosyncrasies of make that just avoids?
-
-Make has some behaviors which are either confusing, complicated, or make it unsuitable for use as a general command runner.
-
-One example is that sometimes make won't run the commands in a recipe. For example, if you have a file called `test` and the following makefile that runs it:
-
-```make
-test:
-  ./test
-```
-
-Make will actually refuse to run it:
-
-```sh
-$ make test
-make: `test' is up to date.
-```
-
-Make sees the recipe `test` and assumes that it produces a file called `test`. It then sees that this file exists and thus assumes that the recipe doesn't need to be run.
-
-To be fair, this behavior is desirable when using make as a build system, but not when using it as a command runner.
-
-Some other examples include having to understand the difference between `=` and `:=` assignment, the confusing error messages that can be produced if you mess up your makefile, having to use `$$` to write recipes that use environment variables, and incompatibilites between different flavors of make.
-
-=== What's the relationship between just and cargo build scripts?
-
-http://doc.crates.io/build-script.html[Cargo build scripts] have a pretty specific use, which is to control how cargo builds your rust project. This might include adding flags to `rustc` invocations, building an external dependency, or running some kind of codegen step.
-
-`just`, on the other hand, is for all the other miscellaneous commands you might run as part of development. Things like running tests in different configurations, linting your code, pushing build artifacts to a server, removing temporary files, and the like.
-
-Also, although `just` is written in rust, it can be used regardless of the language or build system your project uses.
-
 == Miscellanea
 
 === Companion Tools
@@ -813,7 +779,7 @@ Before `just` was a fancy rust program it was a tiny shell script that called `m
 
 === Non-Project Specific Justfile
 
-If you want some commands to be available everwhere, put them in `~/.justfile` and add the following to your shell's initialization file:
+If you want some commands to be available everywhere, put them in `~/.justfile` and add the following to your shell's initialization file:
 
 ```sh
 alias .j='just --justfile ~/.justfile --working-directory ~'
@@ -829,6 +795,50 @@ I'm pretty sure that nobody actually uses this feature, but it's there.
 
 ¯\\_(ツ)_/¯
 
+== Contributing
+
+`just` welcomes your contributions! `just` is released under the maximally permissive [CC0](https://creativecommons.org/publicdomain/zero/1.0/legalcode.txt) public domain dedication and fallback license, so your changes must also released under this license.
+
+=== Janus
+
+[Janus](https://github.com/casey/janus) is a tool that collects and analyzes justfiles, and can determine if a new version of `just` breaks or changes the interpretation of existing justfiles.
+
+Before merging a particularly large or gruesome change, Janus should be run to make sure that nothing breaks. Don't worry about running Janus yourself, Casey will happily run it for you on changes that need it.
+
+== Frequently Asked Questions
+
+=== What are the idiosyncrasies of make that just avoids?
+
+Make has some behaviors which are either confusing, complicated, or make it unsuitable for use as a general command runner.
+
+One example is that sometimes make won't run the commands in a recipe. For example, if you have a file called `test` and the following makefile that runs it:
+
+```make
+test:
+  ./test
+```
+
+Make will actually refuse to run it:
+
+```sh
+$ make test
+make: `test' is up to date.
+```
+
+Make sees the recipe `test` and assumes that it produces a file called `test`. It then sees that this file exists and thus assumes that the recipe doesn't need to be run.
+
+To be fair, this behavior is desirable when using make as a build system, but not when using it as a command runner.
+
+Some other examples include having to understand the difference between `=` and `:=` assignment, the confusing error messages that can be produced if you mess up your makefile, having to use `$$` to write recipes that use environment variables, and incompatibilites between different flavors of make.
+
+=== What's the relationship between just and cargo build scripts?
+
+http://doc.crates.io/build-script.html[Cargo build scripts] have a pretty specific use, which is to control how cargo builds your rust project. This might include adding flags to `rustc` invocations, building an external dependency, or running some kind of codegen step.
+
+`just`, on the other hand, is for all the other miscellaneous commands you might run as part of development. Things like running tests in different configurations, linting your code, pushing build artifacts to a server, removing temporary files, and the like.
+
+Also, although `just` is written in rust, it can be used regardless of the language or build system your project uses.
+
 == Further Ramblings
 
 I personally find it very useful to write a `justfile` for almost every project, big or small.

functions.rs

@@ -0,0 +1,30 @@
+use crate::common::*;
+
+pub struct Functions<'a> {
+  stack: Vec<&'a Expression<'a>>,
+}
+
+impl<'a> Iterator for Functions<'a> {
+  type Item = (&'a Token<'a>, usize);
+
+  fn next(&mut self) -> Option<Self::Item> {
+    match self.stack.pop() {
+      None
+      | Some(Expression::String { .. })
+      | Some(Expression::Backtick { .. })
+      | Some(Expression::Variable { .. }) => None,
+      Some(Expression::Call {
+        token, arguments, ..
+      }) => Some((token, arguments.len())),
+      Some(Expression::Concatination { lhs, rhs }) => {
+        self.stack.push(lhs);
+        self.stack.push(rhs);
+        self.next()
+      }
+      Some(Expression::Group { expression }) => {
+        self.stack.push(expression);
+        self.next()
+      }
+    }
+  }
+}

justfile

@@ -83,7 +83,7 @@ sloc:
 	! grep --color -En '.{101}' src/*.rs
 
 replace FROM TO:
-	find src -name '*.rs' | xargs sed -i '' -E 's/{{FROM}}/{{TO}}/g'
+	sd -i '{{FROM}}' '{{TO}}' src/*.rs
 
 test-quine:
 	cargo run -- quine

src/assignment_evaluator.rs

@@ -1,7 +1,5 @@
 use crate::common::*;
 
-use brev;
-
 pub struct AssignmentEvaluator<'a: 'b, 'b> {
   pub assignments: &'b BTreeMap<&'a str, Expression<'a>>,
   pub invocation_directory: &'b Result<PathBuf, String>,
@@ -53,7 +51,7 @@ impl<'a, 'b> AssignmentEvaluator<'a, 'b> {
     let mut evaluated = String::new();
     for fragment in line {
       match *fragment {
-        Fragment::Text { ref text } => evaluated += text.lexeme,
+        Fragment::Text { ref text } => evaluated += text.lexeme(),
         Fragment::Expression { ref expression } => {
           evaluated += &self.evaluate_expression(expression, arguments)?;
         }
@@ -183,7 +181,7 @@ mod test {
         output_error: OutputError::Code(code),
       } => {
         assert_eq!(code, 100);
-        assert_eq!(token.lexeme, "`f() { return 100; }; f`");
+        assert_eq!(token.lexeme(), "`f() { return 100; }; f`");
       }
       other => panic!("expected a code run error, but got: {}", other),
     }
@@ -211,7 +209,7 @@ recipe:
         token,
         output_error: OutputError::Code(_),
       } => {
-        assert_eq!(token.lexeme, "`echo $exported_variable`");
+        assert_eq!(token.lexeme(), "`echo $exported_variable`");
       }
       other => panic!("expected a backtick code errror, but got: {}", other),
     }

src/assignment_resolver.rs

@@ -45,10 +45,10 @@ impl<'a: 'b, 'b> AssignmentResolver<'a, 'b> {
       let message = format!("attempted to resolve unknown assignment `{}`", name);
       return Err(CompilationError {
         text: "",
-        index: 0,
+        offset: 0,
         line: 0,
         column: 0,
-        width: None,
+        width: 0,
         kind: Internal { message },
       });
     }
@@ -96,40 +96,40 @@ mod test {
   compilation_error_test! {
     name:   circular_variable_dependency,
     input:   "a = b\nb = a",
-    index:  0,
+    offset:  0,
     line:   0,
     column: 0,
-    width:  Some(1),
+    width:  1,
     kind:   CircularVariableDependency{variable: "a", circle: vec!["a", "b", "a"]},
   }
 
   compilation_error_test! {
     name:   self_variable_dependency,
     input:  "a = a",
-    index:  0,
+    offset:  0,
     line:   0,
     column: 0,
-    width:  Some(1),
+    width:  1,
     kind:   CircularVariableDependency{variable: "a", circle: vec!["a", "a"]},
   }
 
   compilation_error_test! {
     name:   unknown_expression_variable,
     input:  "x = yy",
-    index:  4,
+    offset:  4,
     line:   0,
     column: 4,
-    width:  Some(2),
+    width:  2,
     kind:   UndefinedVariable{variable: "yy"},
   }
 
   compilation_error_test! {
     name:   unknown_function,
     input:  "a = foo()",
-    index:  4,
+    offset:  4,
     line:   0,
     column: 4,
-    width:  Some(3),
+    width:  3,
     kind:   UnknownFunction{function: "foo"},
   }
 

src/color.rs

@@ -2,16 +2,8 @@ use crate::common::*;
 
 use ansi_term::Color::*;
 use ansi_term::{ANSIGenericString, Prefix, Style, Suffix};
-use atty::is as is_atty;
 use atty::Stream;
 
-#[derive(Copy, Clone)]
-pub enum UseColor {
-  Auto,
-  Always,
-  Never,
-}
-
 #[derive(Copy, Clone)]
 pub struct Color {
   use_color: UseColor,
@@ -19,16 +11,6 @@ pub struct Color {
   style: Style,
 }
 
-impl Default for Color {
-  fn default() -> Color {
-    Color {
-      use_color: UseColor::Never,
-      atty: false,
-      style: Style::new(),
-    }
-  }
-}
-
 impl Color {
   fn restyle(self, style: Style) -> Color {
     Color { style, ..self }
@@ -36,7 +18,7 @@ impl Color {
 
   fn redirect(self, stream: Stream) -> Color {
     Color {
-      atty: is_atty(stream),
+      atty: atty::is(stream),
       ..self
     }
   }
@@ -138,3 +120,13 @@ impl Color {
     self.effective_style().suffix()
   }
 }
+
+impl Default for Color {
+  fn default() -> Color {
+    Color {
+      use_color: UseColor::Never,
+      atty: false,
+      style: Style::new(),
+    }
+  }
+}

src/common.rs

@@ -9,6 +9,7 @@ pub(crate) use std::{
   path::{Path, PathBuf},
   process,
   process::Command,
+  str::Chars,
   sync::{Mutex, MutexGuard},
   usize, vec,
 };
@@ -16,7 +17,6 @@ pub(crate) use std::{
 pub(crate) use edit_distance::edit_distance;
 pub(crate) use libc::{EXIT_FAILURE, EXIT_SUCCESS};
 pub(crate) use log::warn;
-pub(crate) use regex::Regex;
 pub(crate) use tempdir::TempDir;
 pub(crate) use unicode_width::UnicodeWidthChar;
 
@@ -28,10 +28,12 @@ pub(crate) use crate::{
   color::Color,
   compilation_error::{CompilationError, CompilationErrorKind, CompilationResult},
   configuration::Configuration,
-  cooked_string::CookedString,
   expression::Expression,
   fragment::Fragment,
-  function::{evaluate_function, resolve_function, FunctionContext},
+  function::{evaluate_function, resolve_function},
+  function_context::FunctionContext,
+  functions::Functions,
+  interrupt_guard::InterruptGuard,
   interrupt_handler::InterruptHandler,
   justfile::Justfile,
   lexer::Lexer,
@@ -39,11 +41,18 @@ pub(crate) use crate::{
   misc::{default, empty},
   parameter::Parameter,
   parser::Parser,
-  recipe::{Recipe, RecipeContext},
+  position::Position,
+  recipe::Recipe,
+  recipe_context::RecipeContext,
   recipe_resolver::RecipeResolver,
   runtime_error::{RunResult, RuntimeError},
   shebang::Shebang,
-  token::{Token, TokenKind},
+  state::State,
+  string_literal::StringLiteral,
+  token::Token,
+  token_kind::TokenKind,
+  use_color::UseColor,
+  variables::Variables,
   verbosity::Verbosity,
 };
 

src/compilation_error.rs

@@ -7,10 +7,10 @@ pub type CompilationResult<'a, T> = Result<T, CompilationError<'a>>;
 #[derive(Debug, PartialEq)]
 pub struct CompilationError<'a> {
   pub text: &'a str,
-  pub index: usize,
+  pub offset: usize,
   pub line: usize,
   pub column: usize,
-  pub width: Option<usize>,
+  pub width: usize,
   pub kind: CompilationErrorKind<'a>,
 }
 
@@ -98,8 +98,10 @@ pub enum CompilationErrorKind<'a> {
     function: &'a str,
   },
   UnknownStartOfToken,
+  UnpairedCarriageReturn,
   UnterminatedInterpolation,
   UnterminatedString,
+  UnterminatedBacktick,
 }
 
 impl<'a> Display for CompilationError<'a> {
@@ -277,12 +279,18 @@ impl<'a> Display for CompilationError<'a> {
       UnknownStartOfToken => {
         writeln!(f, "Unknown start of token:")?;
       }
+      UnpairedCarriageReturn => {
+        writeln!(f, "Unpaired carriage return")?;
+      }
       UnterminatedInterpolation => {
         writeln!(f, "Unterminated interpolation")?;
       }
       UnterminatedString => {
         writeln!(f, "Unterminated string")?;
       }
+      UnterminatedBacktick => {
+        writeln!(f, "Unterminated backtick")?;
+      }
       Internal { ref message } => {
         writeln!(
           f,
@@ -295,6 +303,13 @@ impl<'a> Display for CompilationError<'a> {
 
     write!(f, "{}", message.suffix())?;
 
-    write_error_context(f, self.text, self.index, self.line, self.column, self.width)
+    write_error_context(
+      f,
+      self.text,
+      self.offset,
+      self.line,
+      self.column,
+      self.width,
+    )
   }
 }

src/expression.rs

@@ -16,7 +16,7 @@ pub enum Expression<'a> {
     rhs: Box<Expression<'a>>,
   },
   String {
-    cooked_string: CookedString<'a>,
+    cooked_string: StringLiteral<'a>,
   },
   Variable {
     name: &'a str,
@@ -29,11 +29,11 @@ pub enum Expression<'a> {
 
 impl<'a> Expression<'a> {
   pub fn variables(&'a self) -> Variables<'a> {
-    Variables { stack: vec![self] }
+    Variables::new(self)
   }
 
   pub fn functions(&'a self) -> Functions<'a> {
-    Functions { stack: vec![self] }
+    Functions::new(self)
   }
 }
 
@@ -64,59 +64,3 @@ impl<'a> Display for Expression<'a> {
     Ok(())
   }
 }
-
-pub struct Variables<'a> {
-  stack: Vec<&'a Expression<'a>>,
-}
-
-impl<'a> Iterator for Variables<'a> {
-  type Item = &'a Token<'a>;
-
-  fn next(&mut self) -> Option<&'a Token<'a>> {
-    match self.stack.pop() {
-      None
-      | Some(Expression::String { .. })
-      | Some(Expression::Backtick { .. })
-      | Some(Expression::Call { .. }) => None,
-      Some(Expression::Variable { token, .. }) => Some(token),
-      Some(Expression::Concatination { lhs, rhs }) => {
-        self.stack.push(lhs);
-        self.stack.push(rhs);
-        self.next()
-      }
-      Some(Expression::Group { expression }) => {
-        self.stack.push(expression);
-        self.next()
-      }
-    }
-  }
-}
-
-pub struct Functions<'a> {
-  stack: Vec<&'a Expression<'a>>,
-}
-
-impl<'a> Iterator for Functions<'a> {
-  type Item = (&'a Token<'a>, usize);
-
-  fn next(&mut self) -> Option<Self::Item> {
-    match self.stack.pop() {
-      None
-      | Some(Expression::String { .. })
-      | Some(Expression::Backtick { .. })
-      | Some(Expression::Variable { .. }) => None,
-      Some(Expression::Call {
-        token, arguments, ..
-      }) => Some((token, arguments.len())),
-      Some(Expression::Concatination { lhs, rhs }) => {
-        self.stack.push(lhs);
-        self.stack.push(rhs);
-        self.next()
-      }
-      Some(Expression::Group { expression }) => {
-        self.stack.push(expression);
-        self.next()
-      }
-    }
-  }
-}

src/fragment.rs

@@ -9,7 +9,7 @@ pub enum Fragment<'a> {
 impl<'a> Fragment<'a> {
   pub fn continuation(&self) -> bool {
     match *self {
-      Fragment::Text { ref text } => text.lexeme.ends_with('\\'),
+      Fragment::Text { ref text } => text.lexeme().ends_with('\\'),
       _ => false,
     }
   }

src/function.rs

@@ -37,13 +37,8 @@ impl Function {
   }
 }
 
-pub struct FunctionContext<'a> {
-  pub invocation_directory: &'a Result<PathBuf, String>,
-  pub dotenv: &'a BTreeMap<String, String>,
-}
-
 pub fn resolve_function<'a>(token: &Token<'a>, argc: usize) -> CompilationResult<'a, ()> {
-  let name = token.lexeme;
+  let name = token.lexeme();
   if let Some(function) = FUNCTIONS.get(&name) {
     use self::Function::*;
     match (function, argc) {
@@ -58,7 +53,7 @@ pub fn resolve_function<'a>(token: &Token<'a>, argc: usize) -> CompilationResult
     }
   } else {
     Err(token.error(CompilationErrorKind::UnknownFunction {
-      function: token.lexeme,
+      function: token.lexeme(),
     }))
   }
 }

src/function_context.rs

@@ -0,0 +1,6 @@
+use crate::common::*;
+
+pub struct FunctionContext<'a> {
+  pub invocation_directory: &'a Result<PathBuf, String>,
+  pub dotenv: &'a BTreeMap<String, String>,
+}

src/functions.rs

@@ -0,0 +1,36 @@
+use crate::common::*;
+
+pub struct Functions<'a> {
+  stack: Vec<&'a Expression<'a>>,
+}
+
+impl<'a> Functions<'a> {
+  pub fn new(root: &'a Expression<'a>) -> Functions<'a> {
+    Functions { stack: vec![root] }
+  }
+}
+
+impl<'a> Iterator for Functions<'a> {
+  type Item = (&'a Token<'a>, usize);
+
+  fn next(&mut self) -> Option<Self::Item> {
+    match self.stack.pop() {
+      None
+      | Some(Expression::String { .. })
+      | Some(Expression::Backtick { .. })
+      | Some(Expression::Variable { .. }) => None,
+      Some(Expression::Call {
+        token, arguments, ..
+      }) => Some((token, arguments.len())),
+      Some(Expression::Concatination { lhs, rhs }) => {
+        self.stack.push(lhs);
+        self.stack.push(rhs);
+        self.next()
+      }
+      Some(Expression::Group { expression }) => {
+        self.stack.push(expression);
+        self.next()
+      }
+    }
+  }
+}

src/interrupt_guard.rs

@@ -0,0 +1,16 @@
+use crate::common::*;
+
+pub struct InterruptGuard;
+
+impl InterruptGuard {
+  pub fn new() -> InterruptGuard {
+    InterruptHandler::instance().block();
+    InterruptGuard
+  }
+}
+
+impl Drop for InterruptGuard {
+  fn drop(&mut self) {
+    InterruptHandler::instance().unblock();
+  }
+}

src/interrupt_handler.rs

@@ -1,7 +1,5 @@
 use crate::common::*;
 
-use ctrlc;
-
 pub struct InterruptHandler {
   blocks: u32,
   interrupted: bool,
@@ -12,7 +10,7 @@ impl InterruptHandler {
     ctrlc::set_handler(|| InterruptHandler::instance().interrupt())
   }
 
-  fn instance() -> MutexGuard<'static, InterruptHandler> {
+  pub fn instance() -> MutexGuard<'static, InterruptHandler> {
     lazy_static! {
       static ref INSTANCE: Mutex<InterruptHandler> = Mutex::new(InterruptHandler::new());
     }
@@ -49,11 +47,11 @@ impl InterruptHandler {
     process::exit(130);
   }
 
-  fn block(&mut self) {
+  pub fn block(&mut self) {
     self.blocks += 1;
   }
 
-  fn unblock(&mut self) {
+  pub fn unblock(&mut self) {
     if self.blocks == 0 {
       die!(
         "{}",
@@ -76,18 +74,3 @@ impl InterruptHandler {
     function()
   }
 }
-
-pub struct InterruptGuard;
-
-impl InterruptGuard {
-  fn new() -> InterruptGuard {
-    InterruptHandler::instance().block();
-    InterruptGuard
-  }
-}
-
-impl Drop for InterruptGuard {
-  fn drop(&mut self) {
-    InterruptHandler::instance().unblock();
-  }
-}

src/lib.rs

@@ -20,10 +20,12 @@ mod command_ext;
 mod common;
 mod compilation_error;
 mod configuration;
-mod cooked_string;
 mod expression;
 mod fragment;
 mod function;
+mod function_context;
+mod functions;
+mod interrupt_guard;
 mod interrupt_handler;
 mod justfile;
 mod lexer;
@@ -32,13 +34,20 @@ mod misc;
 mod parameter;
 mod parser;
 mod platform;
+mod position;
 mod range_ext;
 mod recipe;
+mod recipe_context;
 mod recipe_resolver;
 mod run;
 mod runtime_error;
 mod shebang;
+mod state;
+mod string_literal;
 mod token;
+mod token_kind;
+mod use_color;
+mod variables;
 mod verbosity;
 
 pub use crate::run::run;

src/misc.rs

@@ -58,11 +58,13 @@ pub fn conjoin<T: Display>(
 pub fn write_error_context(
   f: &mut Formatter,
   text: &str,
-  index: usize,
+  offset: usize,
   line: usize,
   column: usize,
-  width: Option<usize>,
+  width: usize,
 ) -> Result<(), fmt::Error> {
+  let width = if width == 0 { 1 } else { width };
+
   let line_number = line + 1;
   let red = Color::fmt(f).error();
   match text.lines().nth(line) {
@@ -77,14 +79,14 @@ pub fn write_error_context(
           if i < column {
             space_column += 4;
           }
-          if i >= column && i < column + width.unwrap_or(1) {
+          if i >= column && i < column + width {
             space_width += 4;
           }
         } else {
           if i < column {
             space_column += UnicodeWidthChar::width(c).unwrap_or(0);
           }
-          if i >= column && i < column + width.unwrap_or(1) {
+          if i >= column && i < column + width {
             space_width += UnicodeWidthChar::width(c).unwrap_or(0);
           }
           space_line.push(c);
@@ -95,16 +97,6 @@ pub fn write_error_context(
       writeln!(f, "{0:1$} |", "", line_number_width)?;
       writeln!(f, "{} | {}", line_number, space_line)?;
       write!(f, "{0:1$} |", "", line_number_width)?;
-      if width == None {
-        write!(
-          f,
-          " {0:1$}{2}^{3}",
-          "",
-          space_column,
-          red.prefix(),
-          red.suffix()
-        )?;
-      } else {
       write!(
         f,
         " {0:1$}{2}{3:^<4$}{5}",
@@ -116,9 +108,8 @@ pub fn write_error_context(
         red.suffix()
       )?;
     }
-    }
     None => {
-      if index != text.len() {
+      if offset != text.len() {
         write!(
           f,
           "internal error: Error has invalid line number: {}",

src/parser.rs

@@ -16,7 +16,8 @@ pub struct Parser<'a> {
 
 impl<'a> Parser<'a> {
   pub fn parse(text: &'a str) -> CompilationResult<'a, Justfile> {
-    let tokens = Lexer::lex(text)?;
+    let mut tokens = Lexer::lex(text)?;
+    tokens.retain(|token| token.kind != Whitespace);
     let parser = Parser::new(text, tokens);
     parser.justfile()
   }
@@ -87,7 +88,7 @@ impl<'a> Parser<'a> {
     doc: Option<Token<'a>>,
     quiet: bool,
   ) -> CompilationResult<'a, ()> {
-    if let Some(recipe) = self.recipes.get(name.lexeme) {
+    if let Some(recipe) = self.recipes.get(name.lexeme()) {
       return Err(name.error(DuplicateRecipe {
         recipe: recipe.name,
         first: recipe.line_number,
@@ -115,14 +116,14 @@ impl<'a> Parser<'a> {
 
       if parsed_variadic_parameter {
         return Err(parameter.error(ParameterFollowsVariadicParameter {
-          parameter: parameter.lexeme,
+          parameter: parameter.lexeme(),
         }));
       }
 
-      if parameters.iter().any(|p| p.name == parameter.lexeme) {
+      if parameters.iter().any(|p| p.name == parameter.lexeme()) {
         return Err(parameter.error(DuplicateParameter {
-          recipe: name.lexeme,
+          recipe: name.lexeme(),
-          parameter: parameter.lexeme,
+          parameter: parameter.lexeme(),
         }));
       }
 
@@ -135,7 +136,7 @@ impl<'a> Parser<'a> {
 
       if parsed_parameter_with_default && default.is_none() {
         return Err(parameter.error(RequiredParameterFollowsDefaultParameter {
-          parameter: parameter.lexeme,
+          parameter: parameter.lexeme(),
         }));
       }
 
@@ -143,7 +144,7 @@ impl<'a> Parser<'a> {
       parsed_variadic_parameter = variadic;
 
       parameters.push(Parameter {
-        name: parameter.lexeme,
+        name: parameter.lexeme(),
         token: parameter,
         default,
         variadic,
@@ -163,13 +164,13 @@ impl<'a> Parser<'a> {
     let mut dependencies = vec![];
     let mut dependency_tokens = vec![];
     while let Some(dependency) = self.accept(Name) {
-      if dependencies.contains(&dependency.lexeme) {
+      if dependencies.contains(&dependency.lexeme()) {
         return Err(dependency.error(DuplicateDependency {
-          recipe: name.lexeme,
+          recipe: name.lexeme(),
-          dependency: dependency.lexeme,
+          dependency: dependency.lexeme(),
         }));
       }
-      dependencies.push(dependency.lexeme);
+      dependencies.push(dependency.lexeme());
       dependency_tokens.push(dependency);
     }
 
@@ -197,7 +198,7 @@ impl<'a> Parser<'a> {
           if let Some(token) = self.accept(Text) {
             if fragments.is_empty() {
               if lines.is_empty() {
-                if token.lexeme.starts_with("#!") {
+                if token.lexeme().starts_with("#!") {
                   shebang = true;
                 }
               } else if !shebang
@@ -206,7 +207,7 @@ impl<'a> Parser<'a> {
                   .and_then(|line| line.last())
                   .map(Fragment::continuation)
                   .unwrap_or(false)
-                && (token.lexeme.starts_with(' ') || token.lexeme.starts_with('\t'))
+                && (token.lexeme().starts_with(' ') || token.lexeme().starts_with('\t'))
               {
                 return Err(token.error(ExtraLeadingWhitespace));
               }
@@ -234,12 +235,12 @@ impl<'a> Parser<'a> {
     }
 
     self.recipes.insert(
-      name.lexeme,
+      name.lexeme(),
       Recipe {
         line_number: name.line,
-        name: name.lexeme,
+        name: name.lexeme(),
-        doc: doc.map(|t| t.lexeme[1..].trim()),
+        doc: doc.map(|t| t.lexeme()[1..].trim()),
-        private: &name.lexeme[0..1] == "_",
+        private: &name.lexeme()[0..1] == "_",
         dependencies,
         dependency_tokens,
         lines,
@@ -263,26 +264,26 @@ impl<'a> Parser<'a> {
           }
           let arguments = self.arguments()?;
           if let Some(token) = self.expect(ParenR) {
-            return Err(self.unexpected_token(&token, &[Name, StringToken, ParenR]));
+            return Err(self.unexpected_token(&token, &[Name, StringCooked, ParenR]));
           }
           Ok(Expression::Call {
-            name: first.lexeme,
+            name: first.lexeme(),
             token: first,
             arguments,
           })
         } else {
           Ok(Expression::Variable {
-            name: first.lexeme,
+            name: first.lexeme(),
             token: first,
           })
         }
       }
       Backtick => Ok(Expression::Backtick {
-        raw: &first.lexeme[1..first.lexeme.len() - 1],
+        raw: &first.lexeme()[1..first.lexeme().len() - 1],
         token: first,
       }),
-      RawString | StringToken => Ok(Expression::String {
+      StringRaw | StringCooked => Ok(Expression::String {
-        cooked_string: CookedString::new(&first)?,
+        cooked_string: StringLiteral::new(&first)?,
       }),
       ParenL => {
         let expression = self.expression()?;
@@ -295,7 +296,7 @@ impl<'a> Parser<'a> {
           expression: Box::new(expression),
         })
       }
-      _ => Err(self.unexpected_token(&first, &[Name, StringToken])),
+      _ => Err(self.unexpected_token(&first, &[Name, StringCooked])),
     }
   }
 
@@ -333,13 +334,13 @@ impl<'a> Parser<'a> {
   }
 
   fn assignment(&mut self, name: Token<'a>, export: bool) -> CompilationResult<'a, ()> {
-    if self.assignments.contains_key(name.lexeme) {
+    if self.assignments.contains_key(name.lexeme()) {
       return Err(name.error(DuplicateVariable {
-        variable: name.lexeme,
+        variable: name.lexeme(),
       }));
     }
     if export {
-      self.exports.insert(name.lexeme);
+      self.exports.insert(name.lexeme());
     }
 
     let expression = self.expression()?;
@@ -347,14 +348,14 @@ impl<'a> Parser<'a> {
       return Err(self.unexpected_token(&token, &[Plus, Eol]));
     }
 
-    self.assignments.insert(name.lexeme, expression);
+    self.assignments.insert(name.lexeme(), expression);
-    self.assignment_tokens.insert(name.lexeme, name);
+    self.assignment_tokens.insert(name.lexeme(), name);
     Ok(())
   }
 
   fn alias(&mut self, name: Token<'a>) -> CompilationResult<'a, ()> {
     // Make sure alias doesn't already exist
-    if let Some(alias) = self.aliases.get(name.lexeme) {
+    if let Some(alias) = self.aliases.get(name.lexeme()) {
       return Err(name.error(DuplicateAlias {
         alias: alias.name,
         first: alias.line_number,
@@ -363,7 +364,7 @@ impl<'a> Parser<'a> {
 
     // Make sure the next token is of kind Name and keep it
     let target = if let Some(next) = self.accept(Name) {
-      next.lexeme
+      next.lexeme()
     } else {
       let unexpected = self.tokens.next().unwrap();
       return Err(self.unexpected_token(&unexpected, &[Name]));
@@ -375,15 +376,15 @@ impl<'a> Parser<'a> {
     }
 
     self.aliases.insert(
-      name.lexeme,
+      name.lexeme(),
       Alias {
-        name: name.lexeme,
+        name: name.lexeme(),
         line_number: name.line,
-        private: name.lexeme.starts_with('_'),
+        private: name.lexeme().starts_with('_'),
         target,
       },
     );
-    self.alias_tokens.insert(name.lexeme, name);
+    self.alias_tokens.insert(name.lexeme(), name);
 
     Ok(())
   }
@@ -416,7 +417,7 @@ impl<'a> Parser<'a> {
             }
           }
           Name => {
-            if token.lexeme == "export" {
+            if token.lexeme() == "export" {
               let next = self.tokens.next().unwrap();
               if next.kind == Name && self.accepted(Equals) {
                 self.assignment(next, true)?;
@@ -426,7 +427,7 @@ impl<'a> Parser<'a> {
                 self.recipe(&token, doc, false)?;
                 doc = None;
               }
-            } else if token.lexeme == "alias" {
+            } else if token.lexeme() == "alias" {
               let next = self.tokens.next().unwrap();
               if next.kind == Name && self.accepted(Equals) {
                 self.alias(next)?;
@@ -449,10 +450,10 @@ impl<'a> Parser<'a> {
         None => {
           return Err(CompilationError {
             text: self.text,
-            index: 0,
+            offset: 0,
             line: 0,
             column: 0,
-            width: None,
+            width: 0,
             kind: Internal {
               message: "unexpected end of token stream".to_string(),
             },
@@ -476,18 +477,18 @@ impl<'a> Parser<'a> {
 
     for recipe in self.recipes.values() {
       for parameter in &recipe.parameters {
-        if self.assignments.contains_key(parameter.token.lexeme) {
+        if self.assignments.contains_key(parameter.token.lexeme()) {
           return Err(parameter.token.error(ParameterShadowsVariable {
-            parameter: parameter.token.lexeme,
+            parameter: parameter.token.lexeme(),
           }));
         }
       }
 
       for dependency in &recipe.dependency_tokens {
-        if !self.recipes[dependency.lexeme].parameters.is_empty() {
+        if !self.recipes[dependency.lexeme()].parameters.is_empty() {
           return Err(dependency.error(DependencyHasParameters {
             recipe: recipe.name,
-            dependency: dependency.lexeme,
+            dependency: dependency.lexeme(),
           }));
         }
       }
@@ -863,253 +864,261 @@ f y=(`echo hello` + x) +z=("foo" + "bar"):"#,
     "x = ('0')",
   }
 
+  #[rustfmt::skip]
+  summary_test! {
+    escaped_dos_newlines,
+    "@spam:\r
+\t{ \\\r
+\t\tfiglet test; \\\r
+\t\tcargo build --color always 2>&1; \\\r
+\t\tcargo test  --color always -- --color always 2>&1; \\\r
+\t} | less\r
+",
+"@spam:
+    { \\
+    \tfiglet test; \\
+    \tcargo build --color always 2>&1; \\
+    \tcargo test  --color always -- --color always 2>&1; \\
+    } | less",
+  }
+
   compilation_error_test! {
     name: duplicate_alias,
     input: "alias foo = bar\nalias foo = baz",
-    index: 22,
+    offset: 22,
     line: 1,
     column: 6,
-    width: Some(3),
+    width: 3,
     kind: DuplicateAlias { alias: "foo", first: 0 },
   }
 
   compilation_error_test! {
     name: alias_syntax_multiple_rhs,
     input: "alias foo = bar baz",
-    index: 16,
+    offset: 16,
     line: 0,
     column: 16,
-    width: Some(3),
+    width: 3,
     kind: UnexpectedToken { expected: vec![Eol, Eof], found: Name },
   }
 
   compilation_error_test! {
     name: alias_syntax_no_rhs,
     input: "alias foo = \n",
-    index: 12,
+    offset: 12,
     line: 0,
     column: 12,
-    width: Some(1),
+    width: 1,
     kind: UnexpectedToken {expected: vec![Name], found:Eol},
   }
 
   compilation_error_test! {
     name: unknown_alias_target,
     input: "alias foo = bar\n",
-    index: 6,
+    offset: 6,
     line: 0,
     column: 6,
-    width: Some(3),
+    width: 3,
     kind: UnknownAliasTarget {alias: "foo", target: "bar"},
   }
 
   compilation_error_test! {
     name: alias_shadows_recipe_before,
     input: "bar: \n  echo bar\nalias foo = bar\nfoo:\n  echo foo",
-    index: 23,
+    offset: 23,
     line: 2,
     column: 6,
-    width: Some(3),
+    width: 3,
     kind: AliasShadowsRecipe {alias: "foo", recipe_line: 3},
   }
 
   compilation_error_test! {
     name: alias_shadows_recipe_after,
     input: "foo:\n  echo foo\nalias foo = bar\nbar:\n  echo bar",
-    index: 22,
+    offset: 22,
     line: 2,
     column: 6,
-    width: Some(3),
+    width: 3,
     kind: AliasShadowsRecipe { alias: "foo", recipe_line: 0 },
   }
 
   compilation_error_test! {
     name:   missing_colon,
     input:  "a b c\nd e f",
-    index:  5,
+    offset:  5,
     line:   0,
     column: 5,
-    width:  Some(1),
+    width:  1,
     kind:   UnexpectedToken{expected: vec![Name, Plus, Colon], found: Eol},
   }
 
   compilation_error_test! {
     name:   missing_default_eol,
     input:  "hello arg=\n",
-    index:  10,
+    offset:  10,
     line:   0,
     column: 10,
-    width:  Some(1),
+    width:  1,
-    kind:   UnexpectedToken{expected: vec![Name, StringToken], found: Eol},
+    kind:   UnexpectedToken{expected: vec![Name, StringCooked], found: Eol},
   }
 
   compilation_error_test! {
     name:   missing_default_eof,
     input:  "hello arg=",
-    index:  10,
+    offset:  10,
     line:   0,
     column: 10,
-    width:  Some(0),
+    width:  0,
-    kind:   UnexpectedToken{expected: vec![Name, StringToken], found: Eof},
+    kind:   UnexpectedToken{expected: vec![Name, StringCooked], found: Eof},
   }
 
   compilation_error_test! {
     name:   parameter_after_variadic,
     input:  "foo +a bbb:",
-    index:  7,
+    offset:  7,
     line:   0,
     column: 7,
-    width:  Some(3),
+    width:  3,
     kind:   ParameterFollowsVariadicParameter{parameter: "bbb"},
   }
 
   compilation_error_test! {
     name:   required_after_default,
     input:  "hello arg='foo' bar:",
-    index:  16,
+    offset:  16,
     line:   0,
     column: 16,
-    width:  Some(3),
+    width:  3,
     kind:   RequiredParameterFollowsDefaultParameter{parameter: "bar"},
   }
 
   compilation_error_test! {
     name:   missing_eol,
     input:  "a b c: z =",
-    index:  9,
+    offset:  9,
     line:   0,
     column: 9,
-    width:  Some(1),
+    width:  1,
     kind:   UnexpectedToken{expected: vec![Name, Eol, Eof], found: Equals},
   }
 
   compilation_error_test! {
     name:   duplicate_parameter,
     input:  "a b b:",
-    index:  4,
+    offset:  4,
     line:   0,
     column: 4,
-    width:  Some(1),
+    width:  1,
     kind:   DuplicateParameter{recipe: "a", parameter: "b"},
   }
 
   compilation_error_test! {
     name:   parameter_shadows_varible,
     input:  "foo = \"h\"\na foo:",
-    index:  12,
+    offset:  12,
     line:   1,
     column: 2,
-    width:  Some(3),
+    width:  3,
     kind:   ParameterShadowsVariable{parameter: "foo"},
   }
 
   compilation_error_test! {
     name:   dependency_has_parameters,
     input:  "foo arg:\nb: foo",
-    index:  12,
+    offset:  12,
     line:   1,
     column: 3,
-    width:  Some(3),
+    width:  3,
     kind:   DependencyHasParameters{recipe: "b", dependency: "foo"},
   }
 
   compilation_error_test! {
     name:   duplicate_dependency,
     input:  "a b c: b c z z",
-    index:  13,
+    offset:  13,
     line:   0,
     column: 13,
-    width:  Some(1),
+    width:  1,
     kind:   DuplicateDependency{recipe: "a", dependency: "z"},
   }
 
   compilation_error_test! {
     name:   duplicate_recipe,
     input:  "a:\nb:\na:",
-    index:  6,
+    offset:  6,
     line:   2,
     column: 0,
-    width:  Some(1),
+    width:  1,
     kind:   DuplicateRecipe{recipe: "a", first: 0},
   }
 
   compilation_error_test! {
     name:   duplicate_variable,
     input:  "a = \"0\"\na = \"0\"",
-    index:  8,
+    offset:  8,
     line:   1,
     column: 0,
-    width:  Some(1),
+    width:  1,
     kind:   DuplicateVariable{variable: "a"},
   }
 
   compilation_error_test! {
     name:   extra_whitespace,
     input:  "a:\n blah\n  blarg",
-    index:  10,
+    offset:  10,
     line:   2,
     column: 1,
-    width:  Some(6),
+    width:  6,
     kind:   ExtraLeadingWhitespace,
   }
 
   compilation_error_test! {
     name:   interpolation_outside_of_recipe,
     input:  "{{",
-    index:  0,
+    offset:  0,
     line:   0,
     column: 0,
-    width:  Some(2),
+    width:  2,
     kind:   UnexpectedToken{expected: vec![Name, At], found: InterpolationStart},
   }
 
-  compilation_error_test! {
-    name:   unclosed_interpolation_delimiter,
-    input:  "a:\n echo {{ foo",
-    index:  15,
-    line:   1,
-    column: 12,
-    width:  Some(0),
-    kind:   UnexpectedToken{expected: vec![Plus, InterpolationEnd], found: Dedent},
-  }
-
   compilation_error_test! {
     name:   unclosed_parenthesis_in_expression,
     input:  "x = foo(",
-    index:  8,
+    offset:  8,
     line:   0,
     column: 8,
-    width:  Some(0),
+    width:  0,
-    kind:   UnexpectedToken{expected: vec![Name, StringToken, ParenR], found: Eof},
+    kind:   UnexpectedToken{expected: vec![Name, StringCooked, ParenR], found: Eof},
   }
 
   compilation_error_test! {
     name:   unclosed_parenthesis_in_interpolation,
     input:  "a:\n echo {{foo(}}",
-    index:  15,
+    offset:  15,
     line:   1,
     column: 12,
-    width:  Some(2),
+    width:  2,
-    kind:   UnexpectedToken{expected: vec![Name, StringToken, ParenR], found: InterpolationEnd},
+    kind:   UnexpectedToken{expected: vec![Name, StringCooked, ParenR], found: InterpolationEnd},
   }
 
   compilation_error_test! {
     name:   plus_following_parameter,
     input:  "a b c+:",
-    index:  5,
+    offset:  5,
     line:   0,
     column: 5,
-    width:  Some(1),
+    width:  1,
     kind:   UnexpectedToken{expected: vec![Name], found: Plus},
   }
 
   compilation_error_test! {
     name:   bad_export,
     input:  "export a",
-    index:  8,
+    offset:  8,
     line:   0,
     column: 8,
-    width:  Some(0),
+    width:  0,
     kind:   UnexpectedToken{expected: vec![Name, Plus, Colon], found: Eof},
   }
 

src/position.rs

@@ -0,0 +1,7 @@
+/// Source position
+#[derive(Copy, Clone, PartialEq, Debug)]
+pub struct Position {
+  pub offset: usize,
+  pub column: usize,
+  pub line: usize,
+}

src/recipe.rs

@@ -38,12 +38,6 @@ pub struct Recipe<'a> {
   pub shebang: bool,
 }
 
-pub struct RecipeContext<'a> {
-  pub invocation_directory: &'a Result<PathBuf, String>,
-  pub configuration: &'a Configuration<'a>,
-  pub scope: BTreeMap<&'a str, String>,
-}
-
 impl<'a> Recipe<'a> {
   pub fn argument_range(&self) -> RangeInclusive<usize> {
     self.min_arguments()..=self.max_arguments()
@@ -319,7 +313,13 @@ impl<'a> Display for Recipe<'a> {
     if let Some(doc) = self.doc {
       writeln!(f, "# {}", doc)?;
     }
+
+    if self.quiet {
+      write!(f, "@{}", self.name)?;
+    } else {
       write!(f, "{}", self.name)?;
+    }
+
     for parameter in &self.parameters {
       write!(f, " {}", parameter)?;
     }
@@ -337,7 +337,7 @@ impl<'a> Display for Recipe<'a> {
           write!(f, "    ")?;
         }
         match *piece {
-          Fragment::Text { ref text } => write!(f, "{}", text.lexeme)?,
+          Fragment::Text { ref text } => write!(f, "{}", text.lexeme())?,
           Fragment::Expression { ref expression, .. } => write!(f, "{{{{{}}}}}", expression)?,
         }
       }

src/recipe_context.rs

@@ -0,0 +1,7 @@
+use crate::common::*;
+
+pub struct RecipeContext<'a> {
+  pub invocation_directory: &'a Result<PathBuf, String>,
+  pub configuration: &'a Configuration<'a>,
+  pub scope: BTreeMap<&'a str, String>,
+}

src/recipe_resolver.rs

@@ -72,12 +72,12 @@ impl<'a, 'b> RecipeResolver<'a, 'b> {
 
   fn resolve_function(&self, function: &Token, argc: usize) -> CompilationResult<'a, ()> {
     resolve_function(function, argc).map_err(|error| CompilationError {
-      index: error.index,
+      offset: error.offset,
       line: error.line,
       column: error.column,
       width: error.width,
       kind: UnknownFunction {
-        function: &self.text[error.index..error.index + error.width.unwrap()],
+        function: &self.text[error.offset..error.offset + error.width],
       },
       text: self.text,
     })
@@ -88,18 +88,18 @@ impl<'a, 'b> RecipeResolver<'a, 'b> {
     variable: &Token,
     parameters: &[Parameter],
   ) -> CompilationResult<'a, ()> {
-    let name = variable.lexeme;
+    let name = variable.lexeme();
     let undefined =
       !self.assignments.contains_key(name) && !parameters.iter().any(|p| p.name == name);
     if undefined {
       let error = variable.error(UndefinedVariable { variable: name });
       return Err(CompilationError {
-        index: error.index,
+        offset: error.offset,
         line: error.line,
         column: error.column,
         width: error.width,
         kind: UndefinedVariable {
-          variable: &self.text[error.index..error.index + error.width.unwrap()],
+          variable: &self.text[error.offset..error.offset + error.width],
         },
         text: self.text,
       });
@@ -115,7 +115,7 @@ impl<'a, 'b> RecipeResolver<'a, 'b> {
     self.stack.push(recipe.name);
     self.seen.insert(recipe.name);
     for dependency_token in &recipe.dependency_tokens {
-      match self.recipes.get(dependency_token.lexeme) {
+      match self.recipes.get(dependency_token.lexeme()) {
         Some(dependency) => {
           if !self.resolved.contains(dependency.name) {
             if self.seen.contains(dependency.name) {
@@ -139,7 +139,7 @@ impl<'a, 'b> RecipeResolver<'a, 'b> {
         None => {
           return Err(dependency_token.error(UnknownDependency {
             recipe: recipe.name,
-            unknown: dependency_token.lexeme,
+            unknown: dependency_token.lexeme(),
           }));
         }
       }
@@ -157,80 +157,80 @@ mod test {
   compilation_error_test! {
     name:   circular_recipe_dependency,
     input:  "a: b\nb: a",
-    index:  8,
+    offset: 8,
     line:   1,
     column: 3,
-    width:  Some(1),
+    width:  1,
     kind:   CircularRecipeDependency{recipe: "b", circle: vec!["a", "b", "a"]},
   }
 
   compilation_error_test! {
     name:   self_recipe_dependency,
     input:  "a: a",
-    index:  3,
+    offset: 3,
     line:   0,
     column: 3,
-    width:  Some(1),
+    width:  1,
     kind:   CircularRecipeDependency{recipe: "a", circle: vec!["a", "a"]},
   }
 
   compilation_error_test! {
     name:   unknown_dependency,
     input:  "a: b",
-    index:  3,
+    offset: 3,
     line:   0,
     column: 3,
-    width:  Some(1),
+    width:  1,
     kind:   UnknownDependency{recipe: "a", unknown: "b"},
   }
 
   compilation_error_test! {
     name:   unknown_interpolation_variable,
     input:  "x:\n {{   hello}}",
-    index:  9,
+    offset: 9,
     line:   1,
     column: 6,
-    width:  Some(5),
+    width:  5,
     kind:   UndefinedVariable{variable: "hello"},
   }
 
   compilation_error_test! {
     name:   unknown_second_interpolation_variable,
     input:  "wtf=\"x\"\nx:\n echo\n foo {{wtf}} {{ lol }}",
-    index:  33,
+    offset: 33,
     line:   3,
     column: 16,
-    width:  Some(3),
+    width:  3,
     kind:   UndefinedVariable{variable: "lol"},
   }
 
   compilation_error_test! {
     name:   unknown_function_in_interpolation,
     input:  "a:\n echo {{bar()}}",
-    index:  11,
+    offset: 11,
     line:   1,
     column: 8,
-    width:  Some(3),
+    width:  3,
     kind:   UnknownFunction{function: "bar"},
   }
 
   compilation_error_test! {
     name:   unknown_function_in_default,
     input:  "a f=baz():",
-    index:  4,
+    offset: 4,
     line:   0,
     column: 4,
-    width:  Some(3),
+    width:  3,
     kind:   UnknownFunction{function: "baz"},
   }
 
   compilation_error_test! {
     name:   unknown_variable_in_default,
     input:  "a f=foo:",
-    index:  4,
+    offset: 4,
     line:   0,
     column: 4,
-    width:  Some(3),
+    width:  3,
     kind:   UndefinedVariable{variable: "foo"},
   }
 }

src/run.rs

@@ -194,27 +194,33 @@ pub fn run() {
     }
   }
 
-  let override_re = Regex::new("^([^=]+)=(.*)$").unwrap();
+  fn is_override(arg: &&str) -> bool {
+    arg.chars().skip(1).any(|c| c == '=')
+  }
 
-  let raw_arguments: Vec<_> = matches
+  let raw_arguments: Vec<&str> = matches
     .values_of("ARGUMENTS")
     .map(Iterator::collect)
     .unwrap_or_default();
 
-  for argument in raw_arguments
+  for argument in raw_arguments.iter().cloned().take_while(is_override) {
-    .iter()
+    let i = argument
-    .take_while(|arg| override_re.is_match(arg))
+      .char_indices()
-  {
+      .skip(1)
-    let captures = override_re.captures(argument).unwrap();
+      .filter(|&(_, c)| c == '=')
-    overrides.insert(
+      .next()
-      captures.get(1).unwrap().as_str(),
+      .unwrap()
-      captures.get(2).unwrap().as_str(),
+      .0;
-    );
+
+    let name = &argument[..i];
+    let value = &argument[i + 1..];
+
+    overrides.insert(name, value);
   }
 
   let rest = raw_arguments
-    .iter()
+    .into_iter()
-    .skip_while(|arg| override_re.is_match(arg))
+    .skip_while(is_override)
     .enumerate()
     .flat_map(|(i, argument)| {
       if i == 0 {
@@ -237,7 +243,7 @@ pub fn run() {
         }
       }
 
-      Some(*argument)
+      Some(argument)
     })
     .collect::<Vec<&str>>();
 

src/runtime_error.rs

@@ -10,10 +10,10 @@ fn write_token_error_context(f: &mut Formatter, token: &Token) -> Result<(), fmt
   write_error_context(
     f,
     token.text,
-    token.index,
+    token.offset,
     token.line,
-    token.column + token.prefix.len(),
+    token.column,
-    Some(token.lexeme.len()),
+    token.lexeme().len(),
   )
 }
 
@@ -255,7 +255,12 @@ impl<'a> Display for RuntimeError<'a> {
         ref token,
         ref message,
       } => {
-        writeln!(f, "Call to function `{}` failed: {}", token.lexeme, message)?;
+        writeln!(
+          f,
+          "Call to function `{}` failed: {}",
+          token.lexeme(),
+          message
+        )?;
         error_token = Some(token);
       }
       Shebang {

src/state.rs

@@ -0,0 +1,9 @@
+use crate::common::*;
+
+#[derive(Copy, Clone, PartialEq, Debug)]
+pub enum State<'a> {
+  Normal,
+  Indented { indentation: &'a str },
+  Text,
+  Interpolation { interpolation_start: Position },
+}

src/string_literal.rs

@@ -1,21 +1,21 @@
 use crate::common::*;
 
 #[derive(PartialEq, Debug)]
-pub struct CookedString<'a> {
+pub struct StringLiteral<'a> {
   pub raw: &'a str,
   pub cooked: Cow<'a, str>,
 }
 
-impl<'a> CookedString<'a> {
+impl<'a> StringLiteral<'a> {
-  pub fn new(token: &Token<'a>) -> CompilationResult<'a, CookedString<'a>> {
+  pub fn new(token: &Token<'a>) -> CompilationResult<'a, StringLiteral<'a>> {
-    let raw = &token.lexeme[1..token.lexeme.len() - 1];
+    let raw = &token.lexeme()[1..token.lexeme().len() - 1];
 
-    if let TokenKind::RawString = token.kind {
+    if let TokenKind::StringRaw = token.kind {
-      Ok(CookedString {
+      Ok(StringLiteral {
         cooked: Cow::Borrowed(raw),
         raw,
       })
-    } else if let TokenKind::StringToken = token.kind {
+    } else if let TokenKind::StringCooked = token.kind {
       let mut cooked = String::new();
       let mut escape = false;
       for c in raw.chars() {
@@ -41,7 +41,7 @@ impl<'a> CookedString<'a> {
         }
         cooked.push(c);
       }
-      Ok(CookedString {
+      Ok(StringLiteral {
         raw,
         cooked: Cow::Owned(cooked),
       })
@@ -53,7 +53,7 @@ impl<'a> CookedString<'a> {
   }
 }
 
-impl<'a> Display for CookedString<'a> {
+impl<'a> Display for StringLiteral<'a> {
   fn fmt(&self, f: &mut Formatter) -> fmt::Result {
     match self.cooked {
       Cow::Borrowed(raw) => write!(f, "'{}'", raw),

src/summary.rs

@@ -138,7 +138,7 @@ impl Fragment {
   fn new(fragment: fragment::Fragment) -> Fragment {
     match fragment {
       fragment::Fragment::Text { text } => Fragment::Text {
-        text: text.lexeme.to_owned(),
+        text: text.lexeme().to_owned(),
       },
       fragment::Fragment::Expression { expression } => Fragment::Expression {
         expression: Expression::new(expression),

src/testing.rs

@@ -7,11 +7,43 @@ pub fn parse_success(text: &str) -> Justfile {
   }
 }
 
+pub fn token_summary(tokens: &[Token]) -> String {
+  use TokenKind::*;
+
+  tokens
+    .iter()
+    .map(|t| match t.kind {
+      At => "@",
+      Backtick => "`",
+      Colon => ":",
+      Comma => ",",
+      Comment => "#",
+      Dedent => "<",
+      Eof => ".",
+      Eol => "$",
+      Equals => "=",
+      Indent => ">",
+      InterpolationEnd => "}",
+      InterpolationStart => "{",
+      Line => "^",
+      Name => "N",
+      ParenL => "(",
+      ParenR => ")",
+      Plus => "+",
+      StringRaw => "'",
+      StringCooked => "\"",
+      Text => "_",
+      Whitespace => " ",
+    })
+    .collect::<Vec<&str>>()
+    .join("")
+}
+
 macro_rules! compilation_error_test {
   (
     name:   $name:ident,
     input:  $input:expr,
-    index:    $index:expr,
+    offset: $offset:expr,
     line:   $line:expr,
     column: $column:expr,
     width:  $width:expr,
@@ -23,19 +55,22 @@ macro_rules! compilation_error_test {
 
       let expected = crate::compilation_error::CompilationError {
         text: input,
-        index: $index,
+        offset: $offset,
         line: $line,
         column: $column,
         width: $width,
         kind: $kind,
       };
 
-      let tokens = crate::lexer::Lexer::lex(input).unwrap();
+      let mut tokens = Lexer::lex(input).unwrap();
+
+      tokens.retain(|token| token.kind != TokenKind::Whitespace);
+
       let parser = crate::parser::Parser::new(input, tokens);
 
       if let Err(error) = parser.justfile() {
         assert_eq!(error.text, expected.text);
-        assert_eq!(error.index, expected.index);
+        assert_eq!(error.offset, expected.offset);
         assert_eq!(error.line, expected.line);
         assert_eq!(error.column, expected.column);
         assert_eq!(error.width, expected.width);

src/token.rs

@@ -2,80 +2,27 @@ use crate::common::*;
 
 #[derive(Debug, PartialEq, Clone)]
 pub struct Token<'a> {
-  pub index: usize,
+  pub offset: usize,
+  pub length: usize,
   pub line: usize,
   pub column: usize,
   pub text: &'a str,
-  pub prefix: &'a str,
-  pub lexeme: &'a str,
   pub kind: TokenKind,
 }
 
 impl<'a> Token<'a> {
+  pub fn lexeme(&self) -> &'a str {
+    &self.text[self.offset..self.offset + self.length]
+  }
+
   pub fn error(&self, kind: CompilationErrorKind<'a>) -> CompilationError<'a> {
     CompilationError {
-      column: self.column + self.prefix.len(),
+      column: self.column,
-      index: self.index + self.prefix.len(),
+      offset: self.offset,
       line: self.line,
       text: self.text,
-      width: Some(self.lexeme.len()),
+      width: self.length,
       kind,
     }
   }
 }
-
-#[derive(Debug, PartialEq, Clone, Copy)]
-pub enum TokenKind {
-  At,
-  Backtick,
-  Colon,
-  Comma,
-  Comment,
-  Dedent,
-  Eof,
-  Eol,
-  Equals,
-  Indent,
-  InterpolationEnd,
-  InterpolationStart,
-  Line,
-  Name,
-  ParenL,
-  ParenR,
-  Plus,
-  RawString,
-  StringToken,
-  Text,
-}
-
-impl Display for TokenKind {
-  fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
-    use TokenKind::*;
-    write!(
-      f,
-      "{}",
-      match *self {
-        Backtick => "backtick",
-        Colon => "':'",
-        Comma => "','",
-        Comment => "comment",
-        Dedent => "dedent",
-        Eof => "end of file",
-        Eol => "end of line",
-        Equals => "'='",
-        Indent => "indent",
-        InterpolationEnd => "'}}'",
-        InterpolationStart => "'{{'",
-        Line => "command",
-        Name => "name",
-        Plus => "'+'",
-        At => "'@'",
-        ParenL => "'('",
-        ParenR => "')'",
-        StringToken => "string",
-        RawString => "raw string",
-        Text => "command text",
-      }
-    )
-  }
-}

src/token_kind.rs

@@ -0,0 +1,59 @@
+use crate::common::*;
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub enum TokenKind {
+  At,
+  Backtick,
+  Colon,
+  Comma,
+  Comment,
+  Dedent,
+  Eof,
+  Eol,
+  Equals,
+  Indent,
+  InterpolationEnd,
+  InterpolationStart,
+  Line,
+  Name,
+  ParenL,
+  ParenR,
+  Plus,
+  StringRaw,
+  StringCooked,
+  Text,
+  Whitespace,
+}
+
+impl Display for TokenKind {
+  fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
+    use TokenKind::*;
+    write!(
+      f,
+      "{}",
+      match *self {
+        At => "'@'",
+        Backtick => "backtick",
+        Colon => "':'",
+        Comma => "','",
+        Comment => "comment",
+        Dedent => "dedent",
+        Eof => "end of file",
+        Eol => "end of line",
+        Equals => "'='",
+        Indent => "indent",
+        InterpolationEnd => "'}}'",
+        InterpolationStart => "'{{'",
+        Line => "command",
+        Name => "name",
+        ParenL => "'('",
+        ParenR => "')'",
+        Plus => "'+'",
+        StringRaw => "raw string",
+        StringCooked => "cooked string",
+        Text => "command text",
+        Whitespace => "whitespace",
+      }
+    )
+  }
+}

src/use_color.rs

@@ -0,0 +1,6 @@
+#[derive(Copy, Clone)]
+pub enum UseColor {
+  Auto,
+  Always,
+  Never,
+}

src/variables.rs

@@ -0,0 +1,34 @@
+use crate::common::*;
+
+pub struct Variables<'a> {
+  stack: Vec<&'a Expression<'a>>,
+}
+
+impl<'a> Variables<'a> {
+  pub fn new(root: &'a Expression<'a>) -> Variables<'a> {
+    Variables { stack: vec![root] }
+  }
+}
+
+impl<'a> Iterator for Variables<'a> {
+  type Item = &'a Token<'a>;
+
+  fn next(&mut self) -> Option<&'a Token<'a>> {
+    match self.stack.pop() {
+      None
+      | Some(Expression::String { .. })
+      | Some(Expression::Backtick { .. })
+      | Some(Expression::Call { .. }) => None,
+      Some(Expression::Variable { token, .. }) => Some(token),
+      Some(Expression::Concatination { lhs, rhs }) => {
+        self.stack.push(lhs);
+        self.stack.push(rhs);
+        self.next()
+      }
+      Some(Expression::Group { expression }) => {
+        self.stack.push(expression);
+        self.next()
+      }
+    }
+  }
+}

tests/integration.rs

@@ -431,6 +431,20 @@ integration_test! {
   status:   200,
 }
 
+// 😬鎌
+integration_test! {
+  name:     backtick_code_interpolation_mod,
+  justfile: "f:\n 無{{`exit 200`}}",
+  args:     (),
+  stdout:   "",
+  stderr:   "error: Backtick failed with exit code 200
+  |
+2 |  無{{`exit 200`}}
+  |      ^^^^^^^^^^
+",
+  status:   200,
+}
+
 integration_test! {
   name:     backtick_code_interpolation_tab,
   justfile: "
@@ -1029,7 +1043,7 @@ integration_test! {
 Leading whitespace may consist of tabs or spaces, but not both
   |
 2 |      echo hello
-  | ^
+  | ^^^^^
 ",
   status:   EXIT_FAILURE,
 }
@@ -1056,7 +1070,7 @@ integration_test! {
             Recipe started with `␉␉` but found line with `␉␠`
   |
 3 |      echo goodbye
-  | ^
+  | ^^^^^
 ",
   status:   EXIT_FAILURE,
 }
@@ -1725,13 +1739,13 @@ a:
 integration_test! {
   name:     unterminated_raw_string,
   justfile: "
-a b=':
+a b= ':
 ",
   args:     ("a"),
   stdout:   "",
   stderr:   "error: Unterminated string
   |
-2 | a b=':
+2 | a b= ':
   |      ^
 ",
   status:   EXIT_FAILURE,
@@ -1740,13 +1754,13 @@ a b=':
 integration_test! {
   name:     unterminated_string,
   justfile: r#"
-a b=":
+a b= ":
 "#,
   args:     ("a"),
   stdout:   "",
   stderr:   r#"error: Unterminated string
   |
-2 | a b=":
+2 | a b= ":
   |      ^
 "#,
   status:   EXIT_FAILURE,
@@ -2051,3 +2065,64 @@ foo a=arch() o=os() f=os_family():
   stderr:   format!("echo {} {} {}\n", target::arch(), target::os(), target::os_family()).as_str(),
   status:   EXIT_SUCCESS,
 }
+
+integration_test! {
+   name:     unterminated_interpolation_eol,
+   justfile: "
+foo:
+  echo {{
+",
+   args:     (),
+   stdout:   "",
+   stderr:   r#"error: Unterminated interpolation
+  |
+3 |   echo {{
+  |        ^^
+"#,
+   status:   EXIT_FAILURE,
+}
+
+integration_test! {
+   name:     unterminated_interpolation_eof,
+   justfile: "
+foo:
+  echo {{",
+   args:     (),
+   stdout:   "",
+   stderr:   r#"error: Unterminated interpolation
+  |
+3 |   echo {{
+  |        ^^
+"#,
+   status:   EXIT_FAILURE,
+}
+
+integration_test! {
+   name:     unterminated_backtick,
+   justfile: "
+foo a=\t`echo blaaaaaah:
+  echo {{a}}",
+   args:     (),
+   stdout:   "",
+   stderr:   r#"error: Unterminated backtick
+  |
+2 | foo a=    `echo blaaaaaah:
+  |           ^
+"#,
+   status:   EXIT_FAILURE,
+}
+
+integration_test! {
+   name:     unknown_start_of_token,
+   justfile: "
+assembly_source_files = $(wildcard src/arch/$(arch)/*.s)
+",
+   args:     (),
+   stdout:   "",
+   stderr:   r#"error: Unknown start of token:
+  |
+2 | assembly_source_files = $(wildcard src/arch/$(arch)/*.s)
+  |                         ^
+"#,
+   status:   EXIT_FAILURE,
+}

tests/interrupts.rs

@@ -52,6 +52,7 @@ mod unix {
   }
 
   #[test]
+  #[ignore]
   fn interrupt_shebang() {
     interrupt_test(
       "
@@ -63,6 +64,7 @@ default:
   }
 
   #[test]
+  #[ignore]
   fn interrupt_line() {
     interrupt_test(
       "

variables.rs

@@ -0,0 +1,28 @@
+use crate::common::*;
+
+pub struct Variables<'a> {
+  stack: Vec<&'a Expression<'a>>,
+}
+
+impl<'a> Iterator for Variables<'a> {
+  type Item = &'a Token<'a>;
+
+  fn next(&mut self) -> Option<&'a Token<'a>> {
+    match self.stack.pop() {
+      None
+      | Some(Expression::String { .. })
+      | Some(Expression::Backtick { .. })
+      | Some(Expression::Call { .. }) => None,
+      Some(Expression::Variable { token, .. }) => Some(token),
+      Some(Expression::Concatination { lhs, rhs }) => {
+        self.stack.push(lhs);
+        self.stack.push(rhs);
+        self.next()
+      }
+      Some(Expression::Group { expression }) => {
+        self.stack.push(expression);
+        self.next()
+      }
+    }
+  }
+}