Parsing and tokenizing tests are now mostly passsing, not running

recipes though.

notes

@@ -37,6 +37,8 @@ notes
     or should non-slash recipes still run in this directory?
     will need to change things a great deal
 - indentation is line continuation
+- should i disallow a shebang recipe where the shebang isn't on the first line?
+- add insane borrow checker issue to issue tracker
 - add context to unexpected_token error
   "while parsing a recipe"
   "while parsing an expression"

src/lib.rs

@@ -13,7 +13,7 @@ extern crate tempdir;
 use std::io::prelude::*;
 
 use std::{fs, fmt, process, io};
-use std::collections::{BTreeMap, BTreeSet, HashSet};
+use std::collections::{BTreeMap, HashSet};
 use std::fmt::Display;
 use regex::Regex;
 
@@ -54,10 +54,11 @@ fn re(pattern: &str) -> Regex {
 struct Recipe<'a> {
   line_number:        usize,
   name:               &'a str,
-  lines:              Vec<&'a str>,
+  lines:              Vec<String>,
-  fragments:          Vec<Vec<Fragment<'a>>>,
+  // fragments:          Vec<Vec<Fragment<'a>>>,
-  variables:          BTreeSet<&'a str>,
+  // variables:          BTreeSet<&'a str>,
-  variable_tokens:    Vec<Token<'a>>,
+  // variable_tokens:    Vec<Token<'a>>,
+  new_lines:          Vec<Vec<Fragmant<'a>>>,
   dependencies:       Vec<&'a str>,
   dependency_tokens:  Vec<Token<'a>>,
   arguments:          Vec<&'a str>,
@@ -66,17 +67,47 @@ struct Recipe<'a> {
 }
 
 #[derive(PartialEq, Debug)]
-enum Fragment<'a> {
+enum Fragmant<'a> {
-  Text{text: &'a str},
+  Text{text: Token<'a>},
-  Variable{name: &'a str},
+  Expression{expression: Expression<'a>},
 }
 
+#[derive(PartialEq, Debug)]
 enum Expression<'a> {
   Variable{name: &'a str, token: Token<'a>},
   String{contents: &'a str},
   Concatination{lhs: Box<Expression<'a>>, rhs: Box<Expression<'a>>},
 }
 
+impl<'a> Expression<'a> {
+  fn variables(&'a self) -> Variables<'a> {
+    Variables {
+      stack: vec![self],
+    }
+  }
+}
+
+struct Variables<'a> {
+  stack: Vec<&'a Expression<'a>>,
+}
+
+impl<'a> Iterator for Variables<'a> {
+  type Item = &'a Token<'a>;
+
+  fn next(&mut self) -> Option<&'a Token<'a>> {
+    match self.stack.pop() {
+      None                                               => None,
+      Some(&Expression::Variable{ref token,..})          => Some(token),
+      Some(&Expression::String{..})                      => None,
+      Some(&Expression::Concatination{ref lhs, ref rhs}) => {
+        self.stack.push(lhs);
+        self.stack.push(rhs);
+        self.next()
+      }
+    }
+  }
+}
+
 impl<'a> Display for Expression<'a> {
   fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
     match *self {
@@ -118,7 +149,7 @@ impl<'a> Recipe<'a> {
         );
         let mut text = String::new();
         // add the shebang
-        text += self.lines[0];
+        text += &self.lines[0];
         text += "\n";
         // add blank lines so that lines in the generated script
         // have the same line number as the corresponding lines
@@ -127,7 +158,7 @@ impl<'a> Recipe<'a> {
           text += "\n"
         }
         for line in &self.lines[1..] {
-          text += line;
+          text += &line;
           text += "\n";
         }
         try!(
@@ -163,7 +194,7 @@ impl<'a> Recipe<'a> {
       });
     } else {
       for command in &self.lines {
-        let mut command = *command;
+        let mut command = &command[0..];
         if !command.starts_with('@') {
           warn!("{}", command);
         } else {
@@ -202,21 +233,20 @@ impl<'a> Display for Recipe<'a> {
       try!(write!(f, " {}", dependency))
     }
 
-
+    for (i, pieces) in self.new_lines.iter().enumerate() {
-    for (i, fragments) in self.fragments.iter().enumerate() {
       if i == 0 {
         try!(writeln!(f, ""));
       }
-      for (j, fragment) in fragments.iter().enumerate() {
+      for (j, piece) in pieces.iter().enumerate() {
         if j == 0 {
           try!(write!(f, "    "));
         }
-        match *fragment {
+        match piece {
-          Fragment::Text{text} => try!(write!(f, "{}", text)),
+          &Fragmant::Text{ref text} => try!(write!(f, "{}", text.lexeme)),
-          Fragment::Variable{name} => try!(write!(f, "{}{}{}", "{{", name, "}}")),
+          &Fragmant::Expression{ref expression} => try!(write!(f, "{}{}{}", "{{", expression, "}}")),
         }
       }
-      if i + 1 < self.fragments.len() {
+      if i + 1 < self.new_lines.len() {
         try!(write!(f, "\n"));
       }
     }
@@ -378,8 +408,6 @@ enum ErrorKind<'a> {
   DuplicateVariable{variable: &'a str},
   ArgumentShadowsVariable{argument: &'a str},
   MixedLeadingWhitespace{whitespace: &'a str},
-  UnclosedInterpolationDelimiter,
-  BadInterpolationVariableName{recipe: &'a str, text: &'a str},
   ExtraLeadingWhitespace,
   InconsistentLeadingWhitespace{expected: &'a str, found: &'a str},
   OuterShebang,
@@ -478,12 +506,6 @@ impl<'a> Display for Error<'a> {
       ErrorKind::OuterShebang => {
         try!(writeln!(f, "a shebang \"#!\" is reserved syntax outside of recipes"))
       }
-      ErrorKind::UnclosedInterpolationDelimiter => {
-        try!(writeln!(f, "unmatched {}", "{{"))
-      }
-      ErrorKind::BadInterpolationVariableName{recipe, text} => {
-        try!(writeln!(f, "recipe {} contains a bad variable interpolation: {}", recipe, text))
-      }
       ErrorKind::UnknownDependency{recipe, unknown} => {
         try!(writeln!(f, "recipe {} has unknown dependency {}", recipe, unknown));
       }
@@ -660,22 +682,6 @@ impl<'a> Token<'a> {
       kind:   kind,
     }
   }
-
-  /*
-  fn split(
-    self,
-    leading_prefix_len:  usize,
-    lexeme_len:          usize,
-    trailing_prefix_len: usize,
-  ) -> (Token<'a>, Token<'a>) {
-    let len = self.prefix.len() + self.lexeme.len();
-
-    // let length = self.prefix.len() + self.lexeme.len();
-    // if lexeme_start > lexeme_end || lexeme_end > length {
-    // }
-    // panic!("Tried to split toke
-  }
-  */
 }
 
 #[derive(Debug, PartialEq, Clone, Copy)]
@@ -1071,29 +1077,37 @@ impl<'a> Parser<'a> {
       return Err(self.unexpected_token(&token, &[Name, Eol, Eof]));
     }
 
-    enum Piece<'a> {
-      Text{text: Token<'a>},
-      Expression{expression: Expression<'a>},
-    }
-
     let mut new_lines = vec![];
+    let mut shebang = false;
 
     if self.accepted(Indent) {
       while !self.accepted(Dedent) {
+        if self.accepted(Eol) {
+          continue;
+        }
         if let Some(token) = self.expect(Line) {
           return Err(token.error(ErrorKind::InternalError{
-            message: format!("Expected a dedent but got {}", token.class)
+            message: format!("Expected a line but got {}", token.class)
           }))
         }
         let mut pieces = vec![];
 
-        while !self.accepted(Eol) {
+        while !(self.accepted(Eol) || self.peek(Dedent)) {
           if let Some(token) = self.accept(Text) {
-            pieces.push(Piece::Text{text: token});
+            if pieces.is_empty() {
+              if new_lines.is_empty() {
+                if token.lexeme.starts_with("#!") {
+                  shebang = true;
+                }
+              } else if !shebang && token.lexeme.starts_with(" ") || token.lexeme.starts_with("\t") {
+                return Err(token.error(ErrorKind::ExtraLeadingWhitespace));
+              }
+            }
+            pieces.push(Fragmant::Text{text: token});
           } else if let Some(token) = self.expect(InterpolationStart) {
             return Err(self.unexpected_token(&token, &[Text, InterpolationStart, Eol]));
           } else {
-            pieces.push(Piece::Expression{expression: try!(self.expression(true))});
+            pieces.push(Fragmant::Expression{expression: try!(self.expression(true))});
             if let Some(token) = self.expect(InterpolationEnd) {
               return Err(self.unexpected_token(&token, &[InterpolationEnd]));
             }
@@ -1104,8 +1118,7 @@ impl<'a> Parser<'a> {
       }
     }
 
-    panic!("done!");
+    /*
-
     let mut lines = vec![];
     let mut line_tokens = vec![];
     let mut shebang = false;
@@ -1195,6 +1208,7 @@ impl<'a> Parser<'a> {
       }
       fragments.push(line_fragments);
     }
+    */
 
     Ok(Recipe {
       line_number:       line_number,
@@ -1203,10 +1217,12 @@ impl<'a> Parser<'a> {
       dependency_tokens: dependency_tokens,
       arguments:         arguments,
       argument_tokens:   argument_tokens,
-      fragments:         fragments,
+      // fragments:         fragments,
-      variables:         variables,
+      // variables:         variables,
-      variable_tokens:   variable_tokens,
+      // variable_tokens:   variable_tokens,
-      lines:             lines,
+      lines:             vec![],
+      new_lines:         new_lines,
+      // lines:             lines,
       shebang:           shebang,
     })
   }
@@ -1226,7 +1242,7 @@ impl<'a> Parser<'a> {
       Ok(lhs)
     } else if let Some(token) = self.expect_eol() {
       if interpolation {
-        Err(self.unexpected_token(&token, &[Plus, Eol, InterpolationEnd]))
+        return Err(self.unexpected_token(&token, &[Plus, Eol, InterpolationEnd]))
       } else {
         Err(self.unexpected_token(&token, &[Plus, Eol]))
       }
@@ -1299,10 +1315,35 @@ impl<'a> Parser<'a> {
         }
       }
 
-      for variable in &recipe.variable_tokens {
+      for line in &recipe.new_lines {
+        for piece in line {
+          if let &Fragmant::Expression{ref expression} = piece {
+            for variable in expression.variables() {
               let name = variable.lexeme;
               if !(assignments.contains_key(&name) || recipe.arguments.contains(&name)) {
-          return Err(variable.error(ErrorKind::UnknownVariable{variable: name}));
+                // There's a borrow issue here that seems to difficult to solve.
+                // The error derived from the variable token has too short a lifetime,
+                // so we create a new error from its contents, which do live long
+                // enough.
+                //
+                // I suspect the solution here is to give recipes, pieces, and expressions
+                // two lifetime parameters instead of one, with one being the lifetime
+                // of the struct, and the second being the lifetime of the tokens
+                // that it contains
+                let error = variable.error(ErrorKind::UnknownVariable{variable: name});
+                return Err(Error {
+                  text:   self.text,
+                  index:  error.index,
+                  line:   error.line,
+                  column: error.column,
+                  width:  error.width,
+                  kind:   ErrorKind::UnknownVariable {
+                    variable: &self.text[error.index..error.index + error.width.unwrap()],
+                  }
+                });
+              }
+            }
+          }
         }
       }
     }

src/tests.rs

@@ -53,7 +53,6 @@ fn token_summary(tokens: &[Token]) -> String {
   }).collect::<Vec<_>>().join("")
 }
 
-/*
 fn parse_success(text: &str) -> Justfile {
   match super::parse(text) {
     Ok(justfile) => justfile,
@@ -84,7 +83,6 @@ fn parse_error(text: &str, expected: Error) {
     panic!("Expected {:?} but parse succeeded", expected.kind);
   }
 }
-*/
 
 #[test]
 fn tokenize_recipe_interpolation_eol() {
@@ -196,7 +194,7 @@ fn tokenize_tabs_then_tab_space() {
 }
 
 #[test]
-fn outer_shebang() {
+fn tokenize_outer_shebang() {
   let text = "#!/usr/bin/env bash";
   tokenize_error(text, Error {
     text:   text,
@@ -209,7 +207,7 @@ fn outer_shebang() {
 }
 
 #[test]
-fn unknown_start_of_token() {
+fn tokenize_unknown() {
   let text = "~";
   tokenize_error(text, Error {
     text:   text,
@@ -221,7 +219,6 @@ fn unknown_start_of_token() {
   });
 }
 
-/*
 #[test]
 fn parse_empty() {
   parse_summary("
@@ -239,20 +236,22 @@ x:
 y:
 z:
 foo = \"x\"
+bar = foo
 goodbye = \"y\"
 hello a b    c   : x y    z #hello
   #! blah
   #blarg
-  {{ foo }}abc{{ goodbye\t   }}xyz
+  {{ foo + bar}}abc{{ goodbye\t  + \"x\" }}xyz
   1
   2
   3
-", "foo = \"x\" # \"x\"
+", "bar = foo # \"x\"
+foo = \"x\" # \"x\"
 goodbye = \"y\" # \"y\"
 hello a b c: x y z
     #! blah
     #blarg
-    {{foo}}abc{{goodbye}}xyz
+    {{foo + bar}}abc{{goodbye + \"x\"}}xyz
     1
     2
     3
@@ -456,54 +455,6 @@ fn write_or() {
   assert_eq!("1, 2, 3, or 4", super::Or(&[1,2,3,4]).to_string());
 }
 
-#[test]
-fn run_shebang() {
-  // this test exists to make sure that shebang recipes
-  // run correctly. although this script is still
-  // executed by sh its behavior depends on the value of a
-  // variable and continuing even though a command fails,
-  // whereas in plain recipes variables are not available
-  // in subsequent lines and execution stops when a line
-  // fails
-  let text = "
-a:
- #!/usr/bin/env sh
- code=200
- function x { return $code; }
- x
- x
-";
-
-  match parse_success(text).run(&["a"]).unwrap_err() {
-    super::RunError::Code{recipe, code} => {
-      assert_eq!(recipe, "a");
-      assert_eq!(code, 200);
-    },
-    other => panic!("expected an code run error, but got: {}", other),
-  }
-}
-
-#[test]
-fn run_order() {
-  let tmp = tempdir::TempDir::new("run_order").unwrap_or_else(|err| panic!("tmpdir: failed to create temporary directory: {}", err));
-  let path = tmp.path().to_str().unwrap_or_else(|| panic!("tmpdir: path was not valid UTF-8")).to_owned();
-  let text = r"
-b: a
-  @mv a b
-
-a:
-  @touch a
-
-d: c
-  @rm c
-
-c: b
-  @mv b c
-";
-  super::std::env::set_current_dir(path).expect("failed to set current directory");
-  parse_success(text).run(&["a", "d"]).unwrap();
-}
-
 #[test]
 fn unknown_recipes() {
   match parse_success("a:\nb:\nc:").run(&["a", "x", "y", "z"]).unwrap_err() {
@@ -512,17 +463,6 @@ fn unknown_recipes() {
   }
 }
 
-#[test]
-fn code_error() {
-  match parse_success("fail:\n @function x { return 100; }; x").run(&["fail"]).unwrap_err() {
-    super::RunError::Code{recipe, code} => {
-      assert_eq!(recipe, "fail");
-      assert_eq!(code, 100);
-    },
-    other @ _ => panic!("expected a code run error, but got: {}", other),
-  }
-}
-
 #[test]
 fn extra_whitespace() {
   // we might want to make extra leading whitespace a line continuation in the future,
@@ -576,24 +516,24 @@ fn bad_interpolation_variable_name() {
   let text = "a:\n echo {{hello--hello}}";
   parse_error(text, Error {
     text:   text,
-    index:  4,
+    index:  11,
     line:   1,
-    column: 1,
+    column: 8,
-    width:  Some(21),
+    width:  Some(12),
-    kind:   ErrorKind::BadInterpolationVariableName{recipe: "a", text: "hello--hello"}
+    kind:   ErrorKind::BadName{name: "hello--hello"}
   });
 }
 
 #[test]
 fn unclosed_interpolation_delimiter() {
-  let text = "a:\n echo {{";
+  let text = "a:\n echo {{ foo";
   parse_error(text, Error {
     text:   text,
-    index:  4,
+    index:  15,
     line:   1,
-    column: 1,
+    column: 12,
-    width:  Some(7),
+    width:  Some(0),
-    kind:   ErrorKind::UnclosedInterpolationDelimiter,
+    kind:   ErrorKind::UnexpectedToken{expected: vec![Plus, Eol, InterpolationEnd], found: Dedent},
   });
 }
 
@@ -621,15 +561,82 @@ fn unknown_interpolation_variable() {
     width:  Some(5),
     kind:   ErrorKind::UnknownVariable{variable: "hello"},
   });
+}
 
-  // let text = "x:\n echo\n {{ lol }}";
+#[test]
-  // parse_error(text, Error {
+fn unknown_second_interpolation_variable() {
-  //   text:   text,
+  let text = "wtf=\"x\"\nx:\n echo\n foo {{wtf}} {{ lol }}";
-  //   index:  11,
+  parse_error(text, Error {
-  //   line:   2,
+    text:   text,
-  //   column: 2,
+    index:  33,
-  //   width:  Some(3),
+    line:   3,
-  //   kind:   ErrorKind::UnknownVariable{variable: "lol"},
+    column: 16,
-  // });
+    width:  Some(3),
+    kind:   ErrorKind::UnknownVariable{variable: "lol"},
+  });
+}
+
+#[test]
+fn run_order() {
+  let tmp = tempdir::TempDir::new("run_order").unwrap_or_else(|err| panic!("tmpdir: failed to create temporary directory: {}", err));
+  let path = tmp.path().to_str().unwrap_or_else(|| panic!("tmpdir: path was not valid UTF-8")).to_owned();
+  let text = r"
+b: a
+  @mv a b
+
+a:
+  @touch F
+  @touch a
+
+d: c
+  @rm c
+
+c: b
+  @mv b c";
+  tokenize_success(text, "$N:N$>^_$$<N:$>^_$^_$$<N:N$>^_$$<N:N$>^_<.");
+  super::std::env::set_current_dir(path).expect("failed to set current directory");
+  parse_success(text).run(&["a", "d"]).unwrap();
+  if let Err(_) = super::std::fs::metadata("F") {
+    panic!("recipes did not run");
+  }
+}
+
+/*
+#[test]
+fn run_shebang() {
+  // this test exists to make sure that shebang recipes
+  // run correctly. although this script is still
+  // executed by sh its behavior depends on the value of a
+  // variable and continuing even though a command fails,
+  // whereas in plain recipes variables are not available
+  // in subsequent lines and execution stops when a line
+  // fails
+  let text = "
+a:
+ #!/usr/bin/env sh
+ code=200
+ function x { return $code; }
+ x
+ x
+";
+
+  match parse_success(text).run(&["a"]).unwrap_err() {
+    super::RunError::Code{recipe, code} => {
+      assert_eq!(recipe, "a");
+      assert_eq!(code, 200);
+    },
+    other => panic!("expected an code run error, but got: {}", other),
+  }
+}
+
+#[test]
+fn code_error() {
+  match parse_success("fail:\n @function x { return 100; }; x").run(&["fail"]).unwrap_err() {
+    super::RunError::Code{recipe, code} => {
+      assert_eq!(recipe, "fail");
+      assert_eq!(code, 100);
+    },
+    other @ _ => panic!("expected a code run error, but got: {}", other),
+  }
 }
 */