Allow ignore line endings inside delimiters (#717)

Modify the lexer to keep track of opening `({[` and closing `]})` delimiters.
When the lexer would emit an eol or indent outside of a recipe when there
is at least one open delimiter, emit a whitespace token instead.

This allows expressions to be split on multiple lines, like so:

    x := if 'a' == 'b' {
      'x'
    } else {
      'y'
    }

This does not work inside of recipe body interpolations, although this
restriction might relaxed in the future.
This commit is contained in:
Casey Rodarmor 2020-10-27 23:51:17 -07:00 committed by GitHub
parent 70768eb24c
commit aa506fa5bd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 282 additions and 43 deletions

View File

@ -45,16 +45,16 @@ pub(crate) use crate::{
assignment_resolver::AssignmentResolver, binding::Binding, color::Color, assignment_resolver::AssignmentResolver, binding::Binding, color::Color,
compilation_error::CompilationError, compilation_error_kind::CompilationErrorKind, compilation_error::CompilationError, compilation_error_kind::CompilationErrorKind,
compiler::Compiler, config::Config, config_error::ConfigError, count::Count, compiler::Compiler, config::Config, config_error::ConfigError, count::Count,
dependency::Dependency, enclosure::Enclosure, evaluator::Evaluator, expression::Expression, delimiter::Delimiter, dependency::Dependency, enclosure::Enclosure, evaluator::Evaluator,
fragment::Fragment, function::Function, function_context::FunctionContext, expression::Expression, fragment::Fragment, function::Function,
interrupt_guard::InterruptGuard, interrupt_handler::InterruptHandler, item::Item, function_context::FunctionContext, interrupt_guard::InterruptGuard,
justfile::Justfile, keyword::Keyword, lexer::Lexer, line::Line, list::List, interrupt_handler::InterruptHandler, item::Item, justfile::Justfile, keyword::Keyword,
load_error::LoadError, module::Module, name::Name, output_error::OutputError, lexer::Lexer, line::Line, list::List, load_error::LoadError, module::Module, name::Name,
parameter::Parameter, parameter_kind::ParameterKind, parser::Parser, platform::Platform, output_error::OutputError, parameter::Parameter, parameter_kind::ParameterKind, parser::Parser,
position::Position, positional::Positional, recipe::Recipe, recipe_context::RecipeContext, platform::Platform, position::Position, positional::Positional, recipe::Recipe,
recipe_resolver::RecipeResolver, runtime_error::RuntimeError, scope::Scope, search::Search, recipe_context::RecipeContext, recipe_resolver::RecipeResolver, runtime_error::RuntimeError,
search_config::SearchConfig, search_error::SearchError, set::Set, setting::Setting, scope::Scope, search::Search, search_config::SearchConfig, search_error::SearchError, set::Set,
settings::Settings, shebang::Shebang, show_whitespace::ShowWhitespace, setting::Setting, settings::Settings, shebang::Shebang, show_whitespace::ShowWhitespace,
string_literal::StringLiteral, subcommand::Subcommand, suggestion::Suggestion, table::Table, string_literal::StringLiteral, subcommand::Subcommand, suggestion::Suggestion, table::Table,
thunk::Thunk, token::Token, token_kind::TokenKind, unresolved_dependency::UnresolvedDependency, thunk::Thunk, token::Token, token_kind::TokenKind, unresolved_dependency::UnresolvedDependency,
unresolved_recipe::UnresolvedRecipe, use_color::UseColor, variables::Variables, unresolved_recipe::UnresolvedRecipe, use_color::UseColor, variables::Variables,

View File

@ -209,6 +209,22 @@ impl Display for CompilationError<'_> {
UnknownStartOfToken => { UnknownStartOfToken => {
writeln!(f, "Unknown start of token:")?; writeln!(f, "Unknown start of token:")?;
}, },
MismatchedClosingDelimiter {
open,
open_line,
close,
} => {
writeln!(
f,
"Mismatched closing delimiter `{}`. (Did you mean to close the `{}` on line {}?)",
close.close(),
open.open(),
open_line.ordinal(),
)?;
},
UnexpectedClosingDelimiter { close } => {
writeln!(f, "Unexpected closing delimiter `{}`", close.close())?;
},
UnpairedCarriageReturn => { UnpairedCarriageReturn => {
writeln!(f, "Unpaired carriage return")?; writeln!(f, "Unpaired carriage return")?;
}, },

View File

@ -97,6 +97,14 @@ pub(crate) enum CompilationErrorKind<'src> {
setting: &'src str, setting: &'src str,
}, },
UnpairedCarriageReturn, UnpairedCarriageReturn,
UnexpectedClosingDelimiter {
close: Delimiter,
},
MismatchedClosingDelimiter {
close: Delimiter,
open: Delimiter,
open_line: usize,
},
UnterminatedInterpolation, UnterminatedInterpolation,
UnterminatedString, UnterminatedString,
UnterminatedBacktick, UnterminatedBacktick,

24
src/delimiter.rs Normal file
View File

@ -0,0 +1,24 @@
#[derive(PartialEq, Eq, Debug, Copy, Clone)]
pub(crate) enum Delimiter {
Brace,
Bracket,
Paren,
}
impl Delimiter {
pub(crate) fn open(self) -> char {
match self {
Self::Brace => '{',
Self::Bracket => '[',
Self::Paren => '(',
}
}
pub(crate) fn close(self) -> char {
match self {
Self::Brace => '}',
Self::Bracket => ']',
Self::Paren => ')',
}
}
}

View File

@ -32,6 +32,8 @@ pub(crate) struct Lexer<'src> {
indentation: Vec<&'src str>, indentation: Vec<&'src str>,
/// Current interpolation start token /// Current interpolation start token
interpolation_start: Option<Token<'src>>, interpolation_start: Option<Token<'src>>,
/// Current open delimiters
open_delimiters: Vec<(Delimiter, usize)>,
} }
impl<'src> Lexer<'src> { impl<'src> Lexer<'src> {
@ -59,6 +61,7 @@ impl<'src> Lexer<'src> {
recipe_body_pending: false, recipe_body_pending: false,
recipe_body: false, recipe_body: false,
interpolation_start: None, interpolation_start: None,
open_delimiters: Vec::new(),
chars, chars,
next, next,
src, src,
@ -431,14 +434,15 @@ impl<'src> Lexer<'src> {
self.advance()?; self.advance()?;
} }
let indentation = self.lexeme(); if self.open_delimiters() {
self.token(Whitespace);
self.indentation.push(indentation); } else {
let indentation = self.lexeme();
self.token(Indent); self.indentation.push(indentation);
self.token(Indent);
if self.recipe_body_pending { if self.recipe_body_pending {
self.recipe_body = true; self.recipe_body = true;
}
} }
Ok(()) Ok(())
@ -452,23 +456,24 @@ impl<'src> Lexer<'src> {
'!' => self.lex_bang(), '!' => self.lex_bang(),
'*' => self.lex_single(Asterisk), '*' => self.lex_single(Asterisk),
'@' => self.lex_single(At), '@' => self.lex_single(At),
'[' => self.lex_single(BracketL), '[' => self.lex_delimiter(BracketL),
']' => self.lex_single(BracketR), ']' => self.lex_delimiter(BracketR),
'=' => self.lex_choice('=', EqualsEquals, Equals), '=' => self.lex_choice('=', EqualsEquals, Equals),
',' => self.lex_single(Comma), ',' => self.lex_single(Comma),
':' => self.lex_colon(), ':' => self.lex_colon(),
'(' => self.lex_single(ParenL), '(' => self.lex_delimiter(ParenL),
')' => self.lex_single(ParenR), ')' => self.lex_delimiter(ParenR),
'{' => self.lex_single(BraceL), '{' => self.lex_delimiter(BraceL),
'}' => self.lex_single(BraceR), '}' => self.lex_delimiter(BraceR),
'+' => self.lex_single(Plus), '+' => self.lex_single(Plus),
'\n' => self.lex_single(Eol),
'\r' => self.lex_cr_lf(),
'#' => self.lex_comment(), '#' => self.lex_comment(),
'`' => self.lex_backtick(), '`' => self.lex_backtick(),
' ' | '\t' => self.lex_whitespace(), ' ' => self.lex_whitespace(),
'\'' => self.lex_raw_string(),
'"' => self.lex_cooked_string(), '"' => self.lex_cooked_string(),
'\'' => self.lex_raw_string(),
'\n' => self.lex_eol(),
'\r' => self.lex_eol(),
'\t' => self.lex_whitespace(),
_ if Self::is_identifier_start(start) => self.lex_identifier(), _ if Self::is_identifier_start(start) => self.lex_identifier(),
_ => { _ => {
self.advance()?; self.advance()?;
@ -589,6 +594,53 @@ impl<'src> Lexer<'src> {
Ok(()) Ok(())
} }
/// Lex an opening or closing delimiter
fn lex_delimiter(&mut self, kind: TokenKind) -> CompilationResult<'src, ()> {
use Delimiter::*;
match kind {
BraceL => self.open_delimiter(Brace),
BraceR => self.close_delimiter(Brace)?,
BracketL => self.open_delimiter(Bracket),
BracketR => self.close_delimiter(Bracket)?,
ParenL => self.open_delimiter(Paren),
ParenR => self.close_delimiter(Paren)?,
_ =>
return Err(self.internal_error(format!(
"Lexer::lex_delimiter called with non-delimiter token: `{}`",
kind,
))),
}
// Emit the delimiter token
self.lex_single(kind)
}
/// Push a delimiter onto the open delimiter stack
fn open_delimiter(&mut self, delimiter: Delimiter) {
self
.open_delimiters
.push((delimiter, self.token_start.line));
}
/// Pop a delimiter from the open delimiter stack and error if incorrect type
fn close_delimiter(&mut self, close: Delimiter) -> CompilationResult<'src, ()> {
match self.open_delimiters.pop() {
Some((open, _)) if open == close => Ok(()),
Some((open, open_line)) => Err(self.error(MismatchedClosingDelimiter {
open,
close,
open_line,
})),
None => Err(self.error(UnexpectedClosingDelimiter { close })),
}
}
/// Return true if there are any unclosed delimiters
fn open_delimiters(&self) -> bool {
!self.open_delimiters.is_empty()
}
/// Lex a token starting with '!' /// Lex a token starting with '!'
fn lex_bang(&mut self) -> CompilationResult<'src, ()> { fn lex_bang(&mut self) -> CompilationResult<'src, ()> {
self.presume('!')?; self.presume('!')?;
@ -621,14 +673,22 @@ impl<'src> Lexer<'src> {
} }
/// Lex a carriage return and line feed /// Lex a carriage return and line feed
fn lex_cr_lf(&mut self) -> CompilationResult<'src, ()> { fn lex_eol(&mut self) -> CompilationResult<'src, ()> {
self.presume('\r')?; if self.accepted('\r')? {
if !self.accepted('\n')? {
if !self.accepted('\n')? { return Err(self.error(UnpairedCarriageReturn));
return Err(self.error(UnpairedCarriageReturn)); }
} else {
self.presume('\n')?;
} }
self.token(Eol); // Emit an eol if there are no open delimiters, otherwise emit a whitespace
// token.
if self.open_delimiters() {
self.token(Whitespace);
} else {
self.token(Eol);
}
Ok(()) Ok(())
} }
@ -958,8 +1018,8 @@ mod tests {
test! { test! {
name: brace_r, name: brace_r,
text: "}", text: "{}",
tokens: (BraceR), tokens: (BraceL, BraceR),
} }
test! { test! {
@ -970,8 +1030,8 @@ mod tests {
test! { test! {
name: brace_rrr, name: brace_rrr,
text: "}}}", text: "{{{}}}",
tokens: (BraceR, BraceR, BraceR), tokens: (BraceL, BraceL, BraceL, BraceR, BraceR, BraceR),
} }
test! { test! {
@ -1801,7 +1861,7 @@ mod tests {
test! { test! {
name: tokenize_parens, name: tokenize_parens,
text: "((())) )abc(+", text: "((())) ()abc(+",
tokens: ( tokens: (
ParenL, ParenL,
ParenL, ParenL,
@ -1810,6 +1870,7 @@ mod tests {
ParenR, ParenR,
ParenR, ParenR,
Whitespace, Whitespace,
ParenL,
ParenR, ParenR,
Identifier:"abc", Identifier:"abc",
ParenL, ParenL,
@ -1846,8 +1907,18 @@ mod tests {
test! { test! {
name: brackets, name: brackets,
text: "][", text: "[][]",
tokens: (BracketR, BracketL), tokens: (BracketL, BracketR, BracketL, BracketR),
}
test! {
name: open_delimiter_eol,
text: "[\n](\n){\n}",
tokens: (
BracketL, Whitespace:"\n", BracketR,
ParenL, Whitespace:"\n", ParenR,
BraceL, Whitespace:"\n", BraceR
),
} }
error! { error! {
@ -2049,6 +2120,20 @@ mod tests {
kind: UnexpectedCharacter { expected: '=' }, kind: UnexpectedCharacter { expected: '=' },
} }
error! {
name: mismatched_closing_brace,
input: "(]",
offset: 1,
line: 0,
column: 1,
width: 0,
kind: MismatchedClosingDelimiter {
open: Delimiter::Paren,
close: Delimiter::Bracket,
open_line: 0,
},
}
#[test] #[test]
fn presume_error() { fn presume_error() {
assert_matches!( assert_matches!(

View File

@ -65,6 +65,7 @@ mod config;
mod config_error; mod config_error;
mod count; mod count;
mod default; mod default;
mod delimiter;
mod dependency; mod dependency;
mod empty; mod empty;
mod enclosure; mod enclosure;

View File

@ -244,8 +244,8 @@ impl Expression {
} => Expression::Conditional { } => Expression::Conditional {
lhs: Box::new(Expression::new(lhs)), lhs: Box::new(Expression::new(lhs)),
rhs: Box::new(Expression::new(rhs)), rhs: Box::new(Expression::new(rhs)),
then: Box::new(Expression::new(lhs)), then: Box::new(Expression::new(then)),
otherwise: Box::new(Expression::new(rhs)), otherwise: Box::new(Expression::new(otherwise)),
inverted: *inverted, inverted: *inverted,
}, },
StringLiteral { string_literal } => Expression::String { StringLiteral { string_literal } => Expression::String {

104
tests/delimiters.rs Normal file
View File

@ -0,0 +1,104 @@
use crate::common::*;
test! {
name: mismatched_delimiter,
justfile: "(]",
stderr: "
error: Mismatched closing delimiter `]`. (Did you mean to close the `(` on line 1?)
|
1 | (]
| ^
",
status: EXIT_FAILURE,
}
test! {
name: unexpected_delimiter,
justfile: "]",
stderr: "
error: Unexpected closing delimiter `]`
|
1 | ]
| ^
",
status: EXIT_FAILURE,
}
test! {
name: paren_continuation,
justfile: "
x := (
'a'
+
'b'
)
foo:
echo {{x}}
",
stdout: "ab\n",
stderr: "echo ab\n",
}
test! {
name: brace_continuation,
justfile: "
x := if '' == '' {
'a'
} else {
'b'
}
foo:
echo {{x}}
",
stdout: "a\n",
stderr: "echo a\n",
}
test! {
name: bracket_continuation,
justfile: "
set shell := [
'sh',
'-cu',
]
foo:
echo foo
",
stdout: "foo\n",
stderr: "echo foo\n",
}
test! {
name: dependency_continuation,
justfile: "
foo: (
bar 'bar'
)
echo foo
bar x:
echo {{x}}
",
stdout: "bar\nfoo\n",
stderr: "echo bar\necho foo\n",
}
test! {
name: no_interpolation_continuation,
justfile: "
foo:
echo {{ (
'a' + 'b')}}
",
stdout: "",
stderr: "
error: Unterminated interpolation
|
2 | echo {{ (
| ^^
",
status: EXIT_FAILURE,
}

View File

@ -6,6 +6,7 @@ mod common;
mod choose; mod choose;
mod completions; mod completions;
mod conditional; mod conditional;
mod delimiters;
mod dotenv; mod dotenv;
mod edit; mod edit;
mod error_messages; mod error_messages;