diff --git a/notes b/notes index d51cc88..db4c522 100644 --- a/notes +++ b/notes @@ -37,6 +37,8 @@ notes or should non-slash recipes still run in this directory? will need to change things a great deal - indentation is line continuation +- should i disallow a shebang recipe where the shebang isn't on the first line? +- add insane borrow checker issue to issue tracker - add context to unexpected_token error "while parsing a recipe" "while parsing an expression" diff --git a/src/lib.rs b/src/lib.rs index 5e89370..06cb848 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,7 +13,7 @@ extern crate tempdir; use std::io::prelude::*; use std::{fs, fmt, process, io}; -use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::collections::{BTreeMap, HashSet}; use std::fmt::Display; use regex::Regex; @@ -54,10 +54,11 @@ fn re(pattern: &str) -> Regex { struct Recipe<'a> { line_number: usize, name: &'a str, - lines: Vec<&'a str>, - fragments: Vec>>, - variables: BTreeSet<&'a str>, - variable_tokens: Vec>, + lines: Vec, + // fragments: Vec>>, + // variables: BTreeSet<&'a str>, + // variable_tokens: Vec>, + new_lines: Vec>>, dependencies: Vec<&'a str>, dependency_tokens: Vec>, arguments: Vec<&'a str>, @@ -66,17 +67,47 @@ struct Recipe<'a> { } #[derive(PartialEq, Debug)] -enum Fragment<'a> { - Text{text: &'a str}, - Variable{name: &'a str}, +enum Fragmant<'a> { + Text{text: Token<'a>}, + Expression{expression: Expression<'a>}, } +#[derive(PartialEq, Debug)] enum Expression<'a> { Variable{name: &'a str, token: Token<'a>}, String{contents: &'a str}, Concatination{lhs: Box>, rhs: Box>}, } +impl<'a> Expression<'a> { + fn variables(&'a self) -> Variables<'a> { + Variables { + stack: vec![self], + } + } +} + +struct Variables<'a> { + stack: Vec<&'a Expression<'a>>, +} + +impl<'a> Iterator for Variables<'a> { + type Item = &'a Token<'a>; + + fn next(&mut self) -> Option<&'a Token<'a>> { + match self.stack.pop() { + None => None, + Some(&Expression::Variable{ref token,..}) => Some(token), + Some(&Expression::String{..}) => None, + Some(&Expression::Concatination{ref lhs, ref rhs}) => { + self.stack.push(lhs); + self.stack.push(rhs); + self.next() + } + } + } +} + impl<'a> Display for Expression<'a> { fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { match *self { @@ -118,7 +149,7 @@ impl<'a> Recipe<'a> { ); let mut text = String::new(); // add the shebang - text += self.lines[0]; + text += &self.lines[0]; text += "\n"; // add blank lines so that lines in the generated script // have the same line number as the corresponding lines @@ -127,7 +158,7 @@ impl<'a> Recipe<'a> { text += "\n" } for line in &self.lines[1..] { - text += line; + text += &line; text += "\n"; } try!( @@ -163,7 +194,7 @@ impl<'a> Recipe<'a> { }); } else { for command in &self.lines { - let mut command = *command; + let mut command = &command[0..]; if !command.starts_with('@') { warn!("{}", command); } else { @@ -202,21 +233,20 @@ impl<'a> Display for Recipe<'a> { try!(write!(f, " {}", dependency)) } - - for (i, fragments) in self.fragments.iter().enumerate() { + for (i, pieces) in self.new_lines.iter().enumerate() { if i == 0 { try!(writeln!(f, "")); } - for (j, fragment) in fragments.iter().enumerate() { + for (j, piece) in pieces.iter().enumerate() { if j == 0 { try!(write!(f, " ")); } - match *fragment { - Fragment::Text{text} => try!(write!(f, "{}", text)), - Fragment::Variable{name} => try!(write!(f, "{}{}{}", "{{", name, "}}")), + match piece { + &Fragmant::Text{ref text} => try!(write!(f, "{}", text.lexeme)), + &Fragmant::Expression{ref expression} => try!(write!(f, "{}{}{}", "{{", expression, "}}")), } } - if i + 1 < self.fragments.len() { + if i + 1 < self.new_lines.len() { try!(write!(f, "\n")); } } @@ -378,8 +408,6 @@ enum ErrorKind<'a> { DuplicateVariable{variable: &'a str}, ArgumentShadowsVariable{argument: &'a str}, MixedLeadingWhitespace{whitespace: &'a str}, - UnclosedInterpolationDelimiter, - BadInterpolationVariableName{recipe: &'a str, text: &'a str}, ExtraLeadingWhitespace, InconsistentLeadingWhitespace{expected: &'a str, found: &'a str}, OuterShebang, @@ -478,12 +506,6 @@ impl<'a> Display for Error<'a> { ErrorKind::OuterShebang => { try!(writeln!(f, "a shebang \"#!\" is reserved syntax outside of recipes")) } - ErrorKind::UnclosedInterpolationDelimiter => { - try!(writeln!(f, "unmatched {}", "{{")) - } - ErrorKind::BadInterpolationVariableName{recipe, text} => { - try!(writeln!(f, "recipe {} contains a bad variable interpolation: {}", recipe, text)) - } ErrorKind::UnknownDependency{recipe, unknown} => { try!(writeln!(f, "recipe {} has unknown dependency {}", recipe, unknown)); } @@ -660,22 +682,6 @@ impl<'a> Token<'a> { kind: kind, } } - - /* - fn split( - self, - leading_prefix_len: usize, - lexeme_len: usize, - trailing_prefix_len: usize, - ) -> (Token<'a>, Token<'a>) { - let len = self.prefix.len() + self.lexeme.len(); - - // let length = self.prefix.len() + self.lexeme.len(); - // if lexeme_start > lexeme_end || lexeme_end > length { - // } - // panic!("Tried to split toke - } - */ } #[derive(Debug, PartialEq, Clone, Copy)] @@ -1071,29 +1077,37 @@ impl<'a> Parser<'a> { return Err(self.unexpected_token(&token, &[Name, Eol, Eof])); } - enum Piece<'a> { - Text{text: Token<'a>}, - Expression{expression: Expression<'a>}, - } - let mut new_lines = vec![]; + let mut shebang = false; if self.accepted(Indent) { while !self.accepted(Dedent) { + if self.accepted(Eol) { + continue; + } if let Some(token) = self.expect(Line) { return Err(token.error(ErrorKind::InternalError{ - message: format!("Expected a dedent but got {}", token.class) + message: format!("Expected a line but got {}", token.class) })) } let mut pieces = vec![]; - while !self.accepted(Eol) { + while !(self.accepted(Eol) || self.peek(Dedent)) { if let Some(token) = self.accept(Text) { - pieces.push(Piece::Text{text: token}); + if pieces.is_empty() { + if new_lines.is_empty() { + if token.lexeme.starts_with("#!") { + shebang = true; + } + } else if !shebang && token.lexeme.starts_with(" ") || token.lexeme.starts_with("\t") { + return Err(token.error(ErrorKind::ExtraLeadingWhitespace)); + } + } + pieces.push(Fragmant::Text{text: token}); } else if let Some(token) = self.expect(InterpolationStart) { return Err(self.unexpected_token(&token, &[Text, InterpolationStart, Eol])); } else { - pieces.push(Piece::Expression{expression: try!(self.expression(true))}); + pieces.push(Fragmant::Expression{expression: try!(self.expression(true))}); if let Some(token) = self.expect(InterpolationEnd) { return Err(self.unexpected_token(&token, &[InterpolationEnd])); } @@ -1104,8 +1118,7 @@ impl<'a> Parser<'a> { } } - panic!("done!"); - + /* let mut lines = vec![]; let mut line_tokens = vec![]; let mut shebang = false; @@ -1195,6 +1208,7 @@ impl<'a> Parser<'a> { } fragments.push(line_fragments); } + */ Ok(Recipe { line_number: line_number, @@ -1203,10 +1217,12 @@ impl<'a> Parser<'a> { dependency_tokens: dependency_tokens, arguments: arguments, argument_tokens: argument_tokens, - fragments: fragments, - variables: variables, - variable_tokens: variable_tokens, - lines: lines, + // fragments: fragments, + // variables: variables, + // variable_tokens: variable_tokens, + lines: vec![], + new_lines: new_lines, + // lines: lines, shebang: shebang, }) } @@ -1226,7 +1242,7 @@ impl<'a> Parser<'a> { Ok(lhs) } else if let Some(token) = self.expect_eol() { if interpolation { - Err(self.unexpected_token(&token, &[Plus, Eol, InterpolationEnd])) + return Err(self.unexpected_token(&token, &[Plus, Eol, InterpolationEnd])) } else { Err(self.unexpected_token(&token, &[Plus, Eol])) } @@ -1298,11 +1314,36 @@ impl<'a> Parser<'a> { })); } } - - for variable in &recipe.variable_tokens { - let name = variable.lexeme; - if !(assignments.contains_key(&name) || recipe.arguments.contains(&name)) { - return Err(variable.error(ErrorKind::UnknownVariable{variable: name})); + + for line in &recipe.new_lines { + for piece in line { + if let &Fragmant::Expression{ref expression} = piece { + for variable in expression.variables() { + let name = variable.lexeme; + if !(assignments.contains_key(&name) || recipe.arguments.contains(&name)) { + // There's a borrow issue here that seems to difficult to solve. + // The error derived from the variable token has too short a lifetime, + // so we create a new error from its contents, which do live long + // enough. + // + // I suspect the solution here is to give recipes, pieces, and expressions + // two lifetime parameters instead of one, with one being the lifetime + // of the struct, and the second being the lifetime of the tokens + // that it contains + let error = variable.error(ErrorKind::UnknownVariable{variable: name}); + return Err(Error { + text: self.text, + index: error.index, + line: error.line, + column: error.column, + width: error.width, + kind: ErrorKind::UnknownVariable { + variable: &self.text[error.index..error.index + error.width.unwrap()], + } + }); + } + } + } } } } diff --git a/src/tests.rs b/src/tests.rs index 12fdf4e..e271a77 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -53,7 +53,6 @@ fn token_summary(tokens: &[Token]) -> String { }).collect::>().join("") } -/* fn parse_success(text: &str) -> Justfile { match super::parse(text) { Ok(justfile) => justfile, @@ -84,7 +83,6 @@ fn parse_error(text: &str, expected: Error) { panic!("Expected {:?} but parse succeeded", expected.kind); } } -*/ #[test] fn tokenize_recipe_interpolation_eol() { @@ -196,7 +194,7 @@ fn tokenize_tabs_then_tab_space() { } #[test] -fn outer_shebang() { +fn tokenize_outer_shebang() { let text = "#!/usr/bin/env bash"; tokenize_error(text, Error { text: text, @@ -209,7 +207,7 @@ fn outer_shebang() { } #[test] -fn unknown_start_of_token() { +fn tokenize_unknown() { let text = "~"; tokenize_error(text, Error { text: text, @@ -221,7 +219,6 @@ fn unknown_start_of_token() { }); } -/* #[test] fn parse_empty() { parse_summary(" @@ -239,20 +236,22 @@ x: y: z: foo = \"x\" +bar = foo goodbye = \"y\" hello a b c : x y z #hello #! blah #blarg - {{ foo }}abc{{ goodbye\t }}xyz + {{ foo + bar}}abc{{ goodbye\t + \"x\" }}xyz 1 2 3 -", "foo = \"x\" # \"x\" +", "bar = foo # \"x\" +foo = \"x\" # \"x\" goodbye = \"y\" # \"y\" hello a b c: x y z #! blah #blarg - {{foo}}abc{{goodbye}}xyz + {{foo + bar}}abc{{goodbye + \"x\"}}xyz 1 2 3 @@ -456,54 +455,6 @@ fn write_or() { assert_eq!("1, 2, 3, or 4", super::Or(&[1,2,3,4]).to_string()); } -#[test] -fn run_shebang() { - // this test exists to make sure that shebang recipes - // run correctly. although this script is still - // executed by sh its behavior depends on the value of a - // variable and continuing even though a command fails, - // whereas in plain recipes variables are not available - // in subsequent lines and execution stops when a line - // fails - let text = " -a: - #!/usr/bin/env sh - code=200 - function x { return $code; } - x - x -"; - - match parse_success(text).run(&["a"]).unwrap_err() { - super::RunError::Code{recipe, code} => { - assert_eq!(recipe, "a"); - assert_eq!(code, 200); - }, - other => panic!("expected an code run error, but got: {}", other), - } -} - -#[test] -fn run_order() { - let tmp = tempdir::TempDir::new("run_order").unwrap_or_else(|err| panic!("tmpdir: failed to create temporary directory: {}", err)); - let path = tmp.path().to_str().unwrap_or_else(|| panic!("tmpdir: path was not valid UTF-8")).to_owned(); - let text = r" -b: a - @mv a b - -a: - @touch a - -d: c - @rm c - -c: b - @mv b c -"; - super::std::env::set_current_dir(path).expect("failed to set current directory"); - parse_success(text).run(&["a", "d"]).unwrap(); -} - #[test] fn unknown_recipes() { match parse_success("a:\nb:\nc:").run(&["a", "x", "y", "z"]).unwrap_err() { @@ -512,17 +463,6 @@ fn unknown_recipes() { } } -#[test] -fn code_error() { - match parse_success("fail:\n @function x { return 100; }; x").run(&["fail"]).unwrap_err() { - super::RunError::Code{recipe, code} => { - assert_eq!(recipe, "fail"); - assert_eq!(code, 100); - }, - other @ _ => panic!("expected a code run error, but got: {}", other), - } -} - #[test] fn extra_whitespace() { // we might want to make extra leading whitespace a line continuation in the future, @@ -576,24 +516,24 @@ fn bad_interpolation_variable_name() { let text = "a:\n echo {{hello--hello}}"; parse_error(text, Error { text: text, - index: 4, + index: 11, line: 1, - column: 1, - width: Some(21), - kind: ErrorKind::BadInterpolationVariableName{recipe: "a", text: "hello--hello"} + column: 8, + width: Some(12), + kind: ErrorKind::BadName{name: "hello--hello"} }); } #[test] fn unclosed_interpolation_delimiter() { - let text = "a:\n echo {{"; + let text = "a:\n echo {{ foo"; parse_error(text, Error { text: text, - index: 4, + index: 15, line: 1, - column: 1, - width: Some(7), - kind: ErrorKind::UnclosedInterpolationDelimiter, + column: 12, + width: Some(0), + kind: ErrorKind::UnexpectedToken{expected: vec![Plus, Eol, InterpolationEnd], found: Dedent}, }); } @@ -621,15 +561,82 @@ fn unknown_interpolation_variable() { width: Some(5), kind: ErrorKind::UnknownVariable{variable: "hello"}, }); +} - // let text = "x:\n echo\n {{ lol }}"; - // parse_error(text, Error { - // text: text, - // index: 11, - // line: 2, - // column: 2, - // width: Some(3), - // kind: ErrorKind::UnknownVariable{variable: "lol"}, - // }); +#[test] +fn unknown_second_interpolation_variable() { + let text = "wtf=\"x\"\nx:\n echo\n foo {{wtf}} {{ lol }}"; + parse_error(text, Error { + text: text, + index: 33, + line: 3, + column: 16, + width: Some(3), + kind: ErrorKind::UnknownVariable{variable: "lol"}, + }); +} + +#[test] +fn run_order() { + let tmp = tempdir::TempDir::new("run_order").unwrap_or_else(|err| panic!("tmpdir: failed to create temporary directory: {}", err)); + let path = tmp.path().to_str().unwrap_or_else(|| panic!("tmpdir: path was not valid UTF-8")).to_owned(); + let text = r" +b: a + @mv a b + +a: + @touch F + @touch a + +d: c + @rm c + +c: b + @mv b c"; + tokenize_success(text, "$N:N$>^_$$^_$^_$$^_$$^_<."); + super::std::env::set_current_dir(path).expect("failed to set current directory"); + parse_success(text).run(&["a", "d"]).unwrap(); + if let Err(_) = super::std::fs::metadata("F") { + panic!("recipes did not run"); + } +} + +/* +#[test] +fn run_shebang() { + // this test exists to make sure that shebang recipes + // run correctly. although this script is still + // executed by sh its behavior depends on the value of a + // variable and continuing even though a command fails, + // whereas in plain recipes variables are not available + // in subsequent lines and execution stops when a line + // fails + let text = " +a: + #!/usr/bin/env sh + code=200 + function x { return $code; } + x + x +"; + + match parse_success(text).run(&["a"]).unwrap_err() { + super::RunError::Code{recipe, code} => { + assert_eq!(recipe, "a"); + assert_eq!(code, 200); + }, + other => panic!("expected an code run error, but got: {}", other), + } +} + +#[test] +fn code_error() { + match parse_success("fail:\n @function x { return 100; }; x").run(&["fail"]).unwrap_err() { + super::RunError::Code{recipe, code} => { + assert_eq!(recipe, "fail"); + assert_eq!(code, 100); + }, + other @ _ => panic!("expected a code run error, but got: {}", other), + } } */