Parsing and tokenizing tests are now mostly passsing, not running
recipes though.
This commit is contained in:
Casey Rodarmor
parent
aae665a4e9
commit
d5f81dc0b4
2
notes
2
notes
@ -37,6 +37,8 @@ notes
|
||||
or should non-slash recipes still run in this directory?
|
||||
will need to change things a great deal
|
||||
- indentation is line continuation
|
||||
- should i disallow a shebang recipe where the shebang isn't on the first line?
|
||||
- add insane borrow checker issue to issue tracker
|
||||
- add context to unexpected_token error
|
||||
"while parsing a recipe"
|
||||
"while parsing an expression"
|
||||
|
||||
161
src/lib.rs
161
src/lib.rs
@ -13,7 +13,7 @@ extern crate tempdir;
|
||||
use std::io::prelude::*;
|
||||
|
||||
use std::{fs, fmt, process, io};
|
||||
use std::collections::{BTreeMap, BTreeSet, HashSet};
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::fmt::Display;
|
||||
use regex::Regex;
|
||||
|
||||
@ -54,10 +54,11 @@ fn re(pattern: &str) -> Regex {
|
||||
struct Recipe<'a> {
|
||||
line_number: usize,
|
||||
name: &'a str,
|
||||
lines: Vec<&'a str>,
|
||||
fragments: Vec<Vec<Fragment<'a>>>,
|
||||
variables: BTreeSet<&'a str>,
|
||||
variable_tokens: Vec<Token<'a>>,
|
||||
lines: Vec<String>,
|
||||
// fragments: Vec<Vec<Fragment<'a>>>,
|
||||
// variables: BTreeSet<&'a str>,
|
||||
// variable_tokens: Vec<Token<'a>>,
|
||||
new_lines: Vec<Vec<Fragmant<'a>>>,
|
||||
dependencies: Vec<&'a str>,
|
||||
dependency_tokens: Vec<Token<'a>>,
|
||||
arguments: Vec<&'a str>,
|
||||
@ -66,17 +67,47 @@ struct Recipe<'a> {
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Debug)]
|
||||
enum Fragment<'a> {
|
||||
Text{text: &'a str},
|
||||
Variable{name: &'a str},
|
||||
enum Fragmant<'a> {
|
||||
Text{text: Token<'a>},
|
||||
Expression{expression: Expression<'a>},
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Debug)]
|
||||
enum Expression<'a> {
|
||||
Variable{name: &'a str, token: Token<'a>},
|
||||
String{contents: &'a str},
|
||||
Concatination{lhs: Box<Expression<'a>>, rhs: Box<Expression<'a>>},
|
||||
}
|
||||
|
||||
impl<'a> Expression<'a> {
|
||||
fn variables(&'a self) -> Variables<'a> {
|
||||
Variables {
|
||||
stack: vec![self],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Variables<'a> {
|
||||
stack: Vec<&'a Expression<'a>>,
|
||||
}
|
||||
|
||||
impl<'a> Iterator for Variables<'a> {
|
||||
type Item = &'a Token<'a>;
|
||||
|
||||
fn next(&mut self) -> Option<&'a Token<'a>> {
|
||||
match self.stack.pop() {
|
||||
None => None,
|
||||
Some(&Expression::Variable{ref token,..}) => Some(token),
|
||||
Some(&Expression::String{..}) => None,
|
||||
Some(&Expression::Concatination{ref lhs, ref rhs}) => {
|
||||
self.stack.push(lhs);
|
||||
self.stack.push(rhs);
|
||||
self.next()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Display for Expression<'a> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
|
||||
match *self {
|
||||
@ -118,7 +149,7 @@ impl<'a> Recipe<'a> {
|
||||
);
|
||||
let mut text = String::new();
|
||||
// add the shebang
|
||||
text += self.lines[0];
|
||||
text += &self.lines[0];
|
||||
text += "\n";
|
||||
// add blank lines so that lines in the generated script
|
||||
// have the same line number as the corresponding lines
|
||||
@ -127,7 +158,7 @@ impl<'a> Recipe<'a> {
|
||||
text += "\n"
|
||||
}
|
||||
for line in &self.lines[1..] {
|
||||
text += line;
|
||||
text += &line;
|
||||
text += "\n";
|
||||
}
|
||||
try!(
|
||||
@ -163,7 +194,7 @@ impl<'a> Recipe<'a> {
|
||||
});
|
||||
} else {
|
||||
for command in &self.lines {
|
||||
let mut command = *command;
|
||||
let mut command = &command[0..];
|
||||
if !command.starts_with('@') {
|
||||
warn!("{}", command);
|
||||
} else {
|
||||
@ -202,21 +233,20 @@ impl<'a> Display for Recipe<'a> {
|
||||
try!(write!(f, " {}", dependency))
|
||||
}
|
||||
|
||||
|
||||
for (i, fragments) in self.fragments.iter().enumerate() {
|
||||
for (i, pieces) in self.new_lines.iter().enumerate() {
|
||||
if i == 0 {
|
||||
try!(writeln!(f, ""));
|
||||
}
|
||||
for (j, fragment) in fragments.iter().enumerate() {
|
||||
for (j, piece) in pieces.iter().enumerate() {
|
||||
if j == 0 {
|
||||
try!(write!(f, " "));
|
||||
}
|
||||
match *fragment {
|
||||
Fragment::Text{text} => try!(write!(f, "{}", text)),
|
||||
Fragment::Variable{name} => try!(write!(f, "{}{}{}", "{{", name, "}}")),
|
||||
match piece {
|
||||
&Fragmant::Text{ref text} => try!(write!(f, "{}", text.lexeme)),
|
||||
&Fragmant::Expression{ref expression} => try!(write!(f, "{}{}{}", "{{", expression, "}}")),
|
||||
}
|
||||
}
|
||||
if i + 1 < self.fragments.len() {
|
||||
if i + 1 < self.new_lines.len() {
|
||||
try!(write!(f, "\n"));
|
||||
}
|
||||
}
|
||||
@ -378,8 +408,6 @@ enum ErrorKind<'a> {
|
||||
DuplicateVariable{variable: &'a str},
|
||||
ArgumentShadowsVariable{argument: &'a str},
|
||||
MixedLeadingWhitespace{whitespace: &'a str},
|
||||
UnclosedInterpolationDelimiter,
|
||||
BadInterpolationVariableName{recipe: &'a str, text: &'a str},
|
||||
ExtraLeadingWhitespace,
|
||||
InconsistentLeadingWhitespace{expected: &'a str, found: &'a str},
|
||||
OuterShebang,
|
||||
@ -478,12 +506,6 @@ impl<'a> Display for Error<'a> {
|
||||
ErrorKind::OuterShebang => {
|
||||
try!(writeln!(f, "a shebang \"#!\" is reserved syntax outside of recipes"))
|
||||
}
|
||||
ErrorKind::UnclosedInterpolationDelimiter => {
|
||||
try!(writeln!(f, "unmatched {}", "{{"))
|
||||
}
|
||||
ErrorKind::BadInterpolationVariableName{recipe, text} => {
|
||||
try!(writeln!(f, "recipe {} contains a bad variable interpolation: {}", recipe, text))
|
||||
}
|
||||
ErrorKind::UnknownDependency{recipe, unknown} => {
|
||||
try!(writeln!(f, "recipe {} has unknown dependency {}", recipe, unknown));
|
||||
}
|
||||
@ -660,22 +682,6 @@ impl<'a> Token<'a> {
|
||||
kind: kind,
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
fn split(
|
||||
self,
|
||||
leading_prefix_len: usize,
|
||||
lexeme_len: usize,
|
||||
trailing_prefix_len: usize,
|
||||
) -> (Token<'a>, Token<'a>) {
|
||||
let len = self.prefix.len() + self.lexeme.len();
|
||||
|
||||
// let length = self.prefix.len() + self.lexeme.len();
|
||||
// if lexeme_start > lexeme_end || lexeme_end > length {
|
||||
// }
|
||||
// panic!("Tried to split toke
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Clone, Copy)]
|
||||
@ -1071,29 +1077,37 @@ impl<'a> Parser<'a> {
|
||||
return Err(self.unexpected_token(&token, &[Name, Eol, Eof]));
|
||||
}
|
||||
|
||||
enum Piece<'a> {
|
||||
Text{text: Token<'a>},
|
||||
Expression{expression: Expression<'a>},
|
||||
}
|
||||
|
||||
let mut new_lines = vec![];
|
||||
let mut shebang = false;
|
||||
|
||||
if self.accepted(Indent) {
|
||||
while !self.accepted(Dedent) {
|
||||
if self.accepted(Eol) {
|
||||
continue;
|
||||
}
|
||||
if let Some(token) = self.expect(Line) {
|
||||
return Err(token.error(ErrorKind::InternalError{
|
||||
message: format!("Expected a dedent but got {}", token.class)
|
||||
message: format!("Expected a line but got {}", token.class)
|
||||
}))
|
||||
}
|
||||
let mut pieces = vec![];
|
||||
|
||||
while !self.accepted(Eol) {
|
||||
while !(self.accepted(Eol) || self.peek(Dedent)) {
|
||||
if let Some(token) = self.accept(Text) {
|
||||
pieces.push(Piece::Text{text: token});
|
||||
if pieces.is_empty() {
|
||||
if new_lines.is_empty() {
|
||||
if token.lexeme.starts_with("#!") {
|
||||
shebang = true;
|
||||
}
|
||||
} else if !shebang && token.lexeme.starts_with(" ") || token.lexeme.starts_with("\t") {
|
||||
return Err(token.error(ErrorKind::ExtraLeadingWhitespace));
|
||||
}
|
||||
}
|
||||
pieces.push(Fragmant::Text{text: token});
|
||||
} else if let Some(token) = self.expect(InterpolationStart) {
|
||||
return Err(self.unexpected_token(&token, &[Text, InterpolationStart, Eol]));
|
||||
} else {
|
||||
pieces.push(Piece::Expression{expression: try!(self.expression(true))});
|
||||
pieces.push(Fragmant::Expression{expression: try!(self.expression(true))});
|
||||
if let Some(token) = self.expect(InterpolationEnd) {
|
||||
return Err(self.unexpected_token(&token, &[InterpolationEnd]));
|
||||
}
|
||||
@ -1104,8 +1118,7 @@ impl<'a> Parser<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
panic!("done!");
|
||||
|
||||
/*
|
||||
let mut lines = vec![];
|
||||
let mut line_tokens = vec![];
|
||||
let mut shebang = false;
|
||||
@ -1195,6 +1208,7 @@ impl<'a> Parser<'a> {
|
||||
}
|
||||
fragments.push(line_fragments);
|
||||
}
|
||||
*/
|
||||
|
||||
Ok(Recipe {
|
||||
line_number: line_number,
|
||||
@ -1203,10 +1217,12 @@ impl<'a> Parser<'a> {
|
||||
dependency_tokens: dependency_tokens,
|
||||
arguments: arguments,
|
||||
argument_tokens: argument_tokens,
|
||||
fragments: fragments,
|
||||
variables: variables,
|
||||
variable_tokens: variable_tokens,
|
||||
lines: lines,
|
||||
// fragments: fragments,
|
||||
// variables: variables,
|
||||
// variable_tokens: variable_tokens,
|
||||
lines: vec![],
|
||||
new_lines: new_lines,
|
||||
// lines: lines,
|
||||
shebang: shebang,
|
||||
})
|
||||
}
|
||||
@ -1226,7 +1242,7 @@ impl<'a> Parser<'a> {
|
||||
Ok(lhs)
|
||||
} else if let Some(token) = self.expect_eol() {
|
||||
if interpolation {
|
||||
Err(self.unexpected_token(&token, &[Plus, Eol, InterpolationEnd]))
|
||||
return Err(self.unexpected_token(&token, &[Plus, Eol, InterpolationEnd]))
|
||||
} else {
|
||||
Err(self.unexpected_token(&token, &[Plus, Eol]))
|
||||
}
|
||||
@ -1299,10 +1315,35 @@ impl<'a> Parser<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
for variable in &recipe.variable_tokens {
|
||||
for line in &recipe.new_lines {
|
||||
for piece in line {
|
||||
if let &Fragmant::Expression{ref expression} = piece {
|
||||
for variable in expression.variables() {
|
||||
let name = variable.lexeme;
|
||||
if !(assignments.contains_key(&name) || recipe.arguments.contains(&name)) {
|
||||
return Err(variable.error(ErrorKind::UnknownVariable{variable: name}));
|
||||
// There's a borrow issue here that seems to difficult to solve.
|
||||
// The error derived from the variable token has too short a lifetime,
|
||||
// so we create a new error from its contents, which do live long
|
||||
// enough.
|
||||
//
|
||||
// I suspect the solution here is to give recipes, pieces, and expressions
|
||||
// two lifetime parameters instead of one, with one being the lifetime
|
||||
// of the struct, and the second being the lifetime of the tokens
|
||||
// that it contains
|
||||
let error = variable.error(ErrorKind::UnknownVariable{variable: name});
|
||||
return Err(Error {
|
||||
text: self.text,
|
||||
index: error.index,
|
||||
line: error.line,
|
||||
column: error.column,
|
||||
width: error.width,
|
||||
kind: ErrorKind::UnknownVariable {
|
||||
variable: &self.text[error.index..error.index + error.width.unwrap()],
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
177
src/tests.rs
177
src/tests.rs
@ -53,7 +53,6 @@ fn token_summary(tokens: &[Token]) -> String {
|
||||
}).collect::<Vec<_>>().join("")
|
||||
}
|
||||
|
||||
/*
|
||||
fn parse_success(text: &str) -> Justfile {
|
||||
match super::parse(text) {
|
||||
Ok(justfile) => justfile,
|
||||
@ -84,7 +83,6 @@ fn parse_error(text: &str, expected: Error) {
|
||||
panic!("Expected {:?} but parse succeeded", expected.kind);
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
#[test]
|
||||
fn tokenize_recipe_interpolation_eol() {
|
||||
@ -196,7 +194,7 @@ fn tokenize_tabs_then_tab_space() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outer_shebang() {
|
||||
fn tokenize_outer_shebang() {
|
||||
let text = "#!/usr/bin/env bash";
|
||||
tokenize_error(text, Error {
|
||||
text: text,
|
||||
@ -209,7 +207,7 @@ fn outer_shebang() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_start_of_token() {
|
||||
fn tokenize_unknown() {
|
||||
let text = "~";
|
||||
tokenize_error(text, Error {
|
||||
text: text,
|
||||
@ -221,7 +219,6 @@ fn unknown_start_of_token() {
|
||||
});
|
||||
}
|
||||
|
||||
/*
|
||||
#[test]
|
||||
fn parse_empty() {
|
||||
parse_summary("
|
||||
@ -239,20 +236,22 @@ x:
|
||||
y:
|
||||
z:
|
||||
foo = \"x\"
|
||||
bar = foo
|
||||
goodbye = \"y\"
|
||||
hello a b c : x y z #hello
|
||||
#! blah
|
||||
#blarg
|
||||
{{ foo }}abc{{ goodbye\t }}xyz
|
||||
{{ foo + bar}}abc{{ goodbye\t + \"x\" }}xyz
|
||||
1
|
||||
2
|
||||
3
|
||||
", "foo = \"x\" # \"x\"
|
||||
", "bar = foo # \"x\"
|
||||
foo = \"x\" # \"x\"
|
||||
goodbye = \"y\" # \"y\"
|
||||
hello a b c: x y z
|
||||
#! blah
|
||||
#blarg
|
||||
{{foo}}abc{{goodbye}}xyz
|
||||
{{foo + bar}}abc{{goodbye + \"x\"}}xyz
|
||||
1
|
||||
2
|
||||
3
|
||||
@ -456,54 +455,6 @@ fn write_or() {
|
||||
assert_eq!("1, 2, 3, or 4", super::Or(&[1,2,3,4]).to_string());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_shebang() {
|
||||
// this test exists to make sure that shebang recipes
|
||||
// run correctly. although this script is still
|
||||
// executed by sh its behavior depends on the value of a
|
||||
// variable and continuing even though a command fails,
|
||||
// whereas in plain recipes variables are not available
|
||||
// in subsequent lines and execution stops when a line
|
||||
// fails
|
||||
let text = "
|
||||
a:
|
||||
#!/usr/bin/env sh
|
||||
code=200
|
||||
function x { return $code; }
|
||||
x
|
||||
x
|
||||
";
|
||||
|
||||
match parse_success(text).run(&["a"]).unwrap_err() {
|
||||
super::RunError::Code{recipe, code} => {
|
||||
assert_eq!(recipe, "a");
|
||||
assert_eq!(code, 200);
|
||||
},
|
||||
other => panic!("expected an code run error, but got: {}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_order() {
|
||||
let tmp = tempdir::TempDir::new("run_order").unwrap_or_else(|err| panic!("tmpdir: failed to create temporary directory: {}", err));
|
||||
let path = tmp.path().to_str().unwrap_or_else(|| panic!("tmpdir: path was not valid UTF-8")).to_owned();
|
||||
let text = r"
|
||||
b: a
|
||||
@mv a b
|
||||
|
||||
a:
|
||||
@touch a
|
||||
|
||||
d: c
|
||||
@rm c
|
||||
|
||||
c: b
|
||||
@mv b c
|
||||
";
|
||||
super::std::env::set_current_dir(path).expect("failed to set current directory");
|
||||
parse_success(text).run(&["a", "d"]).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_recipes() {
|
||||
match parse_success("a:\nb:\nc:").run(&["a", "x", "y", "z"]).unwrap_err() {
|
||||
@ -512,17 +463,6 @@ fn unknown_recipes() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_error() {
|
||||
match parse_success("fail:\n @function x { return 100; }; x").run(&["fail"]).unwrap_err() {
|
||||
super::RunError::Code{recipe, code} => {
|
||||
assert_eq!(recipe, "fail");
|
||||
assert_eq!(code, 100);
|
||||
},
|
||||
other @ _ => panic!("expected a code run error, but got: {}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extra_whitespace() {
|
||||
// we might want to make extra leading whitespace a line continuation in the future,
|
||||
@ -576,24 +516,24 @@ fn bad_interpolation_variable_name() {
|
||||
let text = "a:\n echo {{hello--hello}}";
|
||||
parse_error(text, Error {
|
||||
text: text,
|
||||
index: 4,
|
||||
index: 11,
|
||||
line: 1,
|
||||
column: 1,
|
||||
width: Some(21),
|
||||
kind: ErrorKind::BadInterpolationVariableName{recipe: "a", text: "hello--hello"}
|
||||
column: 8,
|
||||
width: Some(12),
|
||||
kind: ErrorKind::BadName{name: "hello--hello"}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unclosed_interpolation_delimiter() {
|
||||
let text = "a:\n echo {{";
|
||||
let text = "a:\n echo {{ foo";
|
||||
parse_error(text, Error {
|
||||
text: text,
|
||||
index: 4,
|
||||
index: 15,
|
||||
line: 1,
|
||||
column: 1,
|
||||
width: Some(7),
|
||||
kind: ErrorKind::UnclosedInterpolationDelimiter,
|
||||
column: 12,
|
||||
width: Some(0),
|
||||
kind: ErrorKind::UnexpectedToken{expected: vec![Plus, Eol, InterpolationEnd], found: Dedent},
|
||||
});
|
||||
}
|
||||
|
||||
@ -621,15 +561,82 @@ fn unknown_interpolation_variable() {
|
||||
width: Some(5),
|
||||
kind: ErrorKind::UnknownVariable{variable: "hello"},
|
||||
});
|
||||
}
|
||||
|
||||
// let text = "x:\n echo\n {{ lol }}";
|
||||
// parse_error(text, Error {
|
||||
// text: text,
|
||||
// index: 11,
|
||||
// line: 2,
|
||||
// column: 2,
|
||||
// width: Some(3),
|
||||
// kind: ErrorKind::UnknownVariable{variable: "lol"},
|
||||
// });
|
||||
#[test]
|
||||
fn unknown_second_interpolation_variable() {
|
||||
let text = "wtf=\"x\"\nx:\n echo\n foo {{wtf}} {{ lol }}";
|
||||
parse_error(text, Error {
|
||||
text: text,
|
||||
index: 33,
|
||||
line: 3,
|
||||
column: 16,
|
||||
width: Some(3),
|
||||
kind: ErrorKind::UnknownVariable{variable: "lol"},
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_order() {
|
||||
let tmp = tempdir::TempDir::new("run_order").unwrap_or_else(|err| panic!("tmpdir: failed to create temporary directory: {}", err));
|
||||
let path = tmp.path().to_str().unwrap_or_else(|| panic!("tmpdir: path was not valid UTF-8")).to_owned();
|
||||
let text = r"
|
||||
b: a
|
||||
@mv a b
|
||||
|
||||
a:
|
||||
@touch F
|
||||
@touch a
|
||||
|
||||
d: c
|
||||
@rm c
|
||||
|
||||
c: b
|
||||
@mv b c";
|
||||
tokenize_success(text, "$N:N$>^_$$<N:$>^_$^_$$<N:N$>^_$$<N:N$>^_<.");
|
||||
super::std::env::set_current_dir(path).expect("failed to set current directory");
|
||||
parse_success(text).run(&["a", "d"]).unwrap();
|
||||
if let Err(_) = super::std::fs::metadata("F") {
|
||||
panic!("recipes did not run");
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
#[test]
|
||||
fn run_shebang() {
|
||||
// this test exists to make sure that shebang recipes
|
||||
// run correctly. although this script is still
|
||||
// executed by sh its behavior depends on the value of a
|
||||
// variable and continuing even though a command fails,
|
||||
// whereas in plain recipes variables are not available
|
||||
// in subsequent lines and execution stops when a line
|
||||
// fails
|
||||
let text = "
|
||||
a:
|
||||
#!/usr/bin/env sh
|
||||
code=200
|
||||
function x { return $code; }
|
||||
x
|
||||
x
|
||||
";
|
||||
|
||||
match parse_success(text).run(&["a"]).unwrap_err() {
|
||||
super::RunError::Code{recipe, code} => {
|
||||
assert_eq!(recipe, "a");
|
||||
assert_eq!(code, 200);
|
||||
},
|
||||
other => panic!("expected an code run error, but got: {}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn code_error() {
|
||||
match parse_success("fail:\n @function x { return 100; }; x").run(&["fail"]).unwrap_err() {
|
||||
super::RunError::Code{recipe, code} => {
|
||||
assert_eq!(recipe, "fail");
|
||||
assert_eq!(code, 100);
|
||||
},
|
||||
other @ _ => panic!("expected a code run error, but got: {}", other),
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
Loading…
Reference in New Issue
Block a user