just/src/lexer.rs
2022-01-30 20:16:10 +00:00

2303 lines
45 KiB
Rust

use crate::common::*;
use CompileErrorKind::*;
use TokenKind::*;
/// Just language lexer
///
/// The lexer proceeds character-by-character, as opposed to using regular
/// expressions to lex tokens or semi-tokens at a time. As a result, it is
/// verbose and straightforward. Just used to have a regex-based lexer, which
/// was slower and generally godawful. However, this should not be taken as a
/// slight against regular expressions, the lexer was just idiosyncratically
/// bad.
pub(crate) struct Lexer<'src> {
/// Source text
src: &'src str,
/// Char iterator
chars: Chars<'src>,
/// Tokens
tokens: Vec<Token<'src>>,
/// Current token start
token_start: Position,
/// Current token end
token_end: Position,
/// Next character to be lexed
next: Option<char>,
/// Next indent will start a recipe body
recipe_body_pending: bool,
/// Inside recipe body
recipe_body: bool,
/// Indentation stack
indentation: Vec<&'src str>,
/// Interpolation token start stack
interpolation_stack: Vec<Token<'src>>,
/// Current open delimiters
open_delimiters: Vec<(Delimiter, usize)>,
}
impl<'src> Lexer<'src> {
/// Lex `text`
pub(crate) fn lex(src: &'src str) -> CompileResult<Vec<Token<'src>>> {
Lexer::new(src).tokenize()
}
/// Create a new Lexer to lex `text`
fn new(src: &'src str) -> Lexer<'src> {
let mut chars = src.chars();
let next = chars.next();
let start = Position {
offset: 0,
column: 0,
line: 0,
};
Lexer {
indentation: vec![""],
tokens: Vec::new(),
token_start: start,
token_end: start,
recipe_body_pending: false,
recipe_body: false,
interpolation_stack: Vec::new(),
open_delimiters: Vec::new(),
chars,
next,
src,
}
}
/// Advance over the character in `self.next`, updating `self.token_end`
/// accordingly.
fn advance(&mut self) -> CompileResult<'src, ()> {
match self.next {
Some(c) => {
let len_utf8 = c.len_utf8();
self.token_end.offset += len_utf8;
self.token_end.column += len_utf8;
if c == '\n' {
self.token_end.column = 0;
self.token_end.line += 1;
}
self.next = self.chars.next();
Ok(())
}
None => Err(self.internal_error("Lexer advanced past end of text")),
}
}
/// Advance over N characters.
fn skip(&mut self, n: usize) -> CompileResult<'src, ()> {
for _ in 0..n {
self.advance()?;
}
Ok(())
}
/// Lexeme of in-progress token
fn lexeme(&self) -> &'src str {
&self.src[self.token_start.offset..self.token_end.offset]
}
/// Length of current token
fn current_token_length(&self) -> usize {
self.token_end.offset - self.token_start.offset
}
fn accepted(&mut self, c: char) -> CompileResult<'src, bool> {
if self.next_is(c) {
self.advance()?;
Ok(true)
} else {
Ok(false)
}
}
fn presume(&mut self, c: char) -> CompileResult<'src, ()> {
if !self.next_is(c) {
return Err(self.internal_error(format!("Lexer presumed character `{}`", c)));
}
self.advance()?;
Ok(())
}
fn presume_str(&mut self, s: &str) -> CompileResult<'src, ()> {
for c in s.chars() {
self.presume(c)?;
}
Ok(())
}
/// Is next character c?
fn next_is(&self, c: char) -> bool {
self.next == Some(c)
}
/// Is next character ' ' or '\t'?
fn next_is_whitespace(&self) -> bool {
self.next_is(' ') || self.next_is('\t')
}
/// Un-lexed text
fn rest(&self) -> &'src str {
&self.src[self.token_end.offset..]
}
/// Check if unlexed text begins with prefix
fn rest_starts_with(&self, prefix: &str) -> bool {
self.rest().starts_with(prefix)
}
/// Does rest start with "\n" or "\r\n"?
fn at_eol(&self) -> bool {
self.next_is('\n') || self.rest_starts_with("\r\n")
}
/// Are we at end-of-file?
fn at_eof(&self) -> bool {
self.rest().is_empty()
}
/// Are we at end-of-line or end-of-file?
fn at_eol_or_eof(&self) -> bool {
self.at_eol() || self.at_eof()
}
/// Get current indentation
fn indentation(&self) -> &'src str {
self.indentation.last().unwrap()
}
/// Are we currently indented
fn indented(&self) -> bool {
!self.indentation().is_empty()
}
/// Create a new token with `kind` whose lexeme is between `self.token_start`
/// and `self.token_end`
fn token(&mut self, kind: TokenKind) {
self.tokens.push(Token {
offset: self.token_start.offset,
column: self.token_start.column,
line: self.token_start.line,
src: self.src,
length: self.token_end.offset - self.token_start.offset,
kind,
});
// Set `token_start` to point after the lexed token
self.token_start = self.token_end;
}
/// Create an internal error with `message`
fn internal_error(&self, message: impl Into<String>) -> CompileError<'src> {
// Use `self.token_end` as the location of the error
let token = Token {
src: self.src,
offset: self.token_end.offset,
line: self.token_end.line,
column: self.token_end.column,
length: 0,
kind: Unspecified,
};
CompileError {
kind: CompileErrorKind::Internal {
message: message.into(),
},
token,
}
}
/// Create a compilation error with `kind`
fn error(&self, kind: CompileErrorKind<'src>) -> CompileError<'src> {
// Use the in-progress token span as the location of the error.
// The width of the error site to highlight depends on the kind of error:
let length = match kind {
UnterminatedString | UnterminatedBacktick => {
let kind = match StringKind::from_token_start(self.lexeme()) {
Some(kind) => kind,
None => {
return self.internal_error("Lexer::error: expected string or backtick token start")
}
};
kind.delimiter().len()
}
// highlight the full token
_ => self.lexeme().len(),
};
let token = Token {
kind: Unspecified,
src: self.src,
offset: self.token_start.offset,
line: self.token_start.line,
column: self.token_start.column,
length,
};
CompileError { token, kind }
}
fn unterminated_interpolation_error(interpolation_start: Token<'src>) -> CompileError<'src> {
CompileError {
token: interpolation_start,
kind: UnterminatedInterpolation,
}
}
/// True if `text` could be an identifier
pub(crate) fn is_identifier(text: &str) -> bool {
if !text.chars().next().map_or(false, Self::is_identifier_start) {
return false;
}
for c in text.chars().skip(1) {
if !Self::is_identifier_continue(c) {
return false;
}
}
true
}
/// True if `c` can be the first character of an identifier
fn is_identifier_start(c: char) -> bool {
matches!(c, 'a'..='z' | 'A'..='Z' | '_')
}
/// True if `c` can be a continuation character of an idenitifier
fn is_identifier_continue(c: char) -> bool {
if Self::is_identifier_start(c) {
return true;
}
matches!(c, '0'..='9' | '-')
}
/// Consume the text and produce a series of tokens
fn tokenize(mut self) -> CompileResult<'src, Vec<Token<'src>>> {
loop {
if self.token_start.column == 0 {
self.lex_line_start()?;
}
match self.next {
Some(first) => {
if let Some(&interpolation_start) = self.interpolation_stack.last() {
self.lex_interpolation(interpolation_start, first)?;
} else if self.recipe_body {
self.lex_body()?;
} else {
self.lex_normal(first)?;
};
}
None => break,
}
}
if let Some(&interpolation_start) = self.interpolation_stack.last() {
return Err(Self::unterminated_interpolation_error(interpolation_start));
}
while self.indented() {
self.lex_dedent();
}
self.token(Eof);
assert_eq!(self.token_start.offset, self.token_end.offset);
assert_eq!(self.token_start.offset, self.src.len());
assert_eq!(self.indentation.len(), 1);
Ok(self.tokens)
}
/// Handle blank lines and indentation
fn lex_line_start(&mut self) -> CompileResult<'src, ()> {
enum Indentation<'src> {
// Line only contains whitespace
Blank,
// Indentation continues
Continue,
// Indentation decreases
Decrease,
// Indentation isn't consistent
Inconsistent,
// Indentation increases
Increase,
// Indentation mixes spaces and tabs
Mixed { whitespace: &'src str },
}
use Indentation::*;
let nonblank_index = self
.rest()
.char_indices()
.skip_while(|&(_, c)| c == ' ' || c == '\t')
.map(|(i, _)| i)
.next()
.unwrap_or_else(|| self.rest().len());
let rest = &self.rest()[nonblank_index..];
let whitespace = &self.rest()[..nonblank_index];
let body_whitespace = &whitespace[..whitespace
.char_indices()
.take(self.indentation().chars().count())
.map(|(i, _c)| i)
.next()
.unwrap_or(0)];
let spaces = whitespace.chars().any(|c| c == ' ');
let tabs = whitespace.chars().any(|c| c == '\t');
let body_spaces = body_whitespace.chars().any(|c| c == ' ');
let body_tabs = body_whitespace.chars().any(|c| c == '\t');
#[allow(clippy::if_same_then_else)]
let indentation = if rest.starts_with('\n') || rest.starts_with("\r\n") || rest.is_empty() {
Blank
} else if whitespace == self.indentation() {
Continue
} else if self.indentation.contains(&whitespace) {
Decrease
} else if self.recipe_body && whitespace.starts_with(self.indentation()) {
Continue
} else if self.recipe_body && body_spaces && body_tabs {
Mixed {
whitespace: body_whitespace,
}
} else if !self.recipe_body && spaces && tabs {
Mixed { whitespace }
} else if whitespace.len() < self.indentation().len() {
Inconsistent
} else if self.recipe_body
&& body_whitespace.len() >= self.indentation().len()
&& !body_whitespace.starts_with(self.indentation())
{
Inconsistent
} else if whitespace.len() >= self.indentation().len()
&& !whitespace.starts_with(self.indentation())
{
Inconsistent
} else {
Increase
};
match indentation {
Blank => {
if !whitespace.is_empty() {
while self.next_is_whitespace() {
self.advance()?;
}
self.token(Whitespace);
};
Ok(())
}
Continue => {
if !self.indentation().is_empty() {
for _ in self.indentation().chars() {
self.advance()?;
}
self.token(Whitespace);
}
Ok(())
}
Decrease => {
while self.indentation() != whitespace {
self.lex_dedent();
}
if !whitespace.is_empty() {
while self.next_is_whitespace() {
self.advance()?;
}
self.token(Whitespace);
}
Ok(())
}
Mixed { whitespace } => {
for _ in whitespace.chars() {
self.advance()?;
}
Err(self.error(MixedLeadingWhitespace { whitespace }))
}
Inconsistent => {
for _ in whitespace.chars() {
self.advance()?;
}
Err(self.error(InconsistentLeadingWhitespace {
expected: self.indentation(),
found: whitespace,
}))
}
Increase => {
while self.next_is_whitespace() {
self.advance()?;
}
if self.open_delimiters() {
self.token(Whitespace);
} else {
let indentation = self.lexeme();
self.indentation.push(indentation);
self.token(Indent);
if self.recipe_body_pending {
self.recipe_body = true;
}
}
Ok(())
}
}
}
/// Lex token beginning with `start` outside of a recipe body
fn lex_normal(&mut self, start: char) -> CompileResult<'src, ()> {
match start {
' ' | '\t' => self.lex_whitespace(),
'!' => self.lex_digraph('!', '=', BangEquals),
'#' => self.lex_comment(),
'$' => self.lex_single(Dollar),
'&' => self.lex_digraph('&', '&', AmpersandAmpersand),
'(' => self.lex_delimiter(ParenL),
')' => self.lex_delimiter(ParenR),
'*' => self.lex_single(Asterisk),
'+' => self.lex_single(Plus),
',' => self.lex_single(Comma),
':' => self.lex_colon(),
'=' => self.lex_choices('=', &[('=', EqualsEquals), ('~', EqualsTilde)], Equals),
'@' => self.lex_single(At),
'[' => self.lex_delimiter(BracketL),
'\n' | '\r' => self.lex_eol(),
'\u{feff}' => self.lex_single(ByteOrderMark),
']' => self.lex_delimiter(BracketR),
'`' | '"' | '\'' => self.lex_string(),
'{' => self.lex_delimiter(BraceL),
'}' => self.lex_delimiter(BraceR),
_ if Self::is_identifier_start(start) => self.lex_identifier(),
_ => {
self.advance()?;
Err(self.error(UnknownStartOfToken))
}
}
}
/// Lex token beginning with `start` inside an interpolation
fn lex_interpolation(
&mut self,
interpolation_start: Token<'src>,
start: char,
) -> CompileResult<'src, ()> {
if self.rest_starts_with("}}") {
// end current interpolation
if self.interpolation_stack.pop().is_none() {
self.advance()?;
self.advance()?;
return Err(self.internal_error(
"Lexer::lex_interpolation found `}}` but was called with empty interpolation stack.",
));
}
// Emit interpolation end token
self.lex_double(InterpolationEnd)
} else if self.at_eol_or_eof() {
// Return unterminated interpolation error that highlights the opening
// {{
Err(Self::unterminated_interpolation_error(interpolation_start))
} else {
// Otherwise lex as per normal
self.lex_normal(start)
}
}
/// Lex token while in recipe body
fn lex_body(&mut self) -> CompileResult<'src, ()> {
enum Terminator {
Newline,
NewlineCarriageReturn,
Interpolation,
EndOfFile,
}
use Terminator::*;
let terminator = loop {
if self.rest_starts_with("{{{{") {
self.skip(4)?;
continue;
}
if self.rest_starts_with("\n") {
break Newline;
}
if self.rest_starts_with("\r\n") {
break NewlineCarriageReturn;
}
if self.rest_starts_with("{{") {
break Interpolation;
}
if self.at_eof() {
break EndOfFile;
}
self.advance()?;
};
// emit text token containing text so far
if self.current_token_length() > 0 {
self.token(Text);
}
match terminator {
Newline => self.lex_single(Eol),
NewlineCarriageReturn => self.lex_double(Eol),
Interpolation => {
self.lex_double(InterpolationStart)?;
self
.interpolation_stack
.push(self.tokens[self.tokens.len() - 1]);
Ok(())
}
EndOfFile => Ok(()),
}
}
fn lex_dedent(&mut self) {
assert_eq!(self.current_token_length(), 0);
self.token(Dedent);
self.indentation.pop();
self.recipe_body_pending = false;
self.recipe_body = false;
}
/// Lex a single-character token
fn lex_single(&mut self, kind: TokenKind) -> CompileResult<'src, ()> {
self.advance()?;
self.token(kind);
Ok(())
}
/// Lex a double-character token
fn lex_double(&mut self, kind: TokenKind) -> CompileResult<'src, ()> {
self.advance()?;
self.advance()?;
self.token(kind);
Ok(())
}
/// Lex a double-character token of kind `then` if the second character of
/// that token would be `second`, otherwise lex a single-character token of
/// kind `otherwise`
fn lex_choices(
&mut self,
first: char,
choices: &[(char, TokenKind)],
otherwise: TokenKind,
) -> CompileResult<'src, ()> {
self.presume(first)?;
for (second, then) in choices {
if self.accepted(*second)? {
self.token(*then);
return Ok(());
}
}
self.token(otherwise);
Ok(())
}
/// Lex an opening or closing delimiter
fn lex_delimiter(&mut self, kind: TokenKind) -> CompileResult<'src, ()> {
use Delimiter::*;
match kind {
BraceL => self.open_delimiter(Brace),
BraceR => self.close_delimiter(Brace)?,
BracketL => self.open_delimiter(Bracket),
BracketR => self.close_delimiter(Bracket)?,
ParenL => self.open_delimiter(Paren),
ParenR => self.close_delimiter(Paren)?,
_ => {
return Err(self.internal_error(format!(
"Lexer::lex_delimiter called with non-delimiter token: `{}`",
kind,
)))
}
}
// Emit the delimiter token
self.lex_single(kind)
}
/// Push a delimiter onto the open delimiter stack
fn open_delimiter(&mut self, delimiter: Delimiter) {
self
.open_delimiters
.push((delimiter, self.token_start.line));
}
/// Pop a delimiter from the open delimiter stack and error if incorrect type
fn close_delimiter(&mut self, close: Delimiter) -> CompileResult<'src, ()> {
match self.open_delimiters.pop() {
Some((open, _)) if open == close => Ok(()),
Some((open, open_line)) => Err(self.error(MismatchedClosingDelimiter {
open,
close,
open_line,
})),
None => Err(self.error(UnexpectedClosingDelimiter { close })),
}
}
/// Return true if there are any unclosed delimiters
fn open_delimiters(&self) -> bool {
!self.open_delimiters.is_empty()
}
/// Lex a two-character digraph
fn lex_digraph(&mut self, left: char, right: char, token: TokenKind) -> CompileResult<'src, ()> {
self.presume(left)?;
if self.accepted(right)? {
self.token(token);
Ok(())
} else {
// Emit an unspecified token to consume the current character,
self.token(Unspecified);
if self.at_eof() {
return Err(self.error(UnexpectedEndOfToken { expected: right }));
}
// …and advance past another character,
self.advance()?;
// …so that the error we produce highlights the unexpected character.
Err(self.error(UnexpectedCharacter { expected: right }))
}
}
/// Lex a token starting with ':'
fn lex_colon(&mut self) -> CompileResult<'src, ()> {
self.presume(':')?;
if self.accepted('=')? {
self.token(ColonEquals);
} else {
self.token(Colon);
self.recipe_body_pending = true;
}
Ok(())
}
/// Lex a carriage return and line feed
fn lex_eol(&mut self) -> CompileResult<'src, ()> {
if self.accepted('\r')? {
if !self.accepted('\n')? {
return Err(self.error(UnpairedCarriageReturn));
}
} else {
self.presume('\n')?;
}
// Emit an eol if there are no open delimiters, otherwise emit a whitespace
// token.
if self.open_delimiters() {
self.token(Whitespace);
} else {
self.token(Eol);
}
Ok(())
}
/// Lex name: [a-zA-Z_][a-zA-Z0-9_]*
fn lex_identifier(&mut self) -> CompileResult<'src, ()> {
self.advance()?;
while let Some(c) = self.next {
if !Self::is_identifier_continue(c) {
break;
}
self.advance()?;
}
self.token(Identifier);
Ok(())
}
/// Lex comment: #[^\r\n]
fn lex_comment(&mut self) -> CompileResult<'src, ()> {
self.presume('#')?;
while !self.at_eol_or_eof() {
self.advance()?;
}
self.token(Comment);
Ok(())
}
/// Lex whitespace: [ \t]+
fn lex_whitespace(&mut self) -> CompileResult<'src, ()> {
while self.next_is_whitespace() {
self.advance()?;
}
self.token(Whitespace);
Ok(())
}
/// Lex a backtick, cooked string, or raw string.
///
/// Backtick: `[^`]*`
/// Cooked string: "[^"]*" # also processes escape sequences
/// Raw string: '[^']*'
fn lex_string(&mut self) -> CompileResult<'src, ()> {
let kind = if let Some(kind) = StringKind::from_token_start(self.rest()) {
kind
} else {
self.advance()?;
return Err(self.internal_error("Lexer::lex_string: invalid string start"));
};
self.presume_str(kind.delimiter())?;
let mut escape = false;
loop {
if self.next == None {
return Err(self.error(kind.unterminated_error_kind()));
} else if kind.processes_escape_sequences() && self.next_is('\\') && !escape {
escape = true;
} else if self.rest_starts_with(kind.delimiter()) && !escape {
break;
} else {
escape = false;
}
self.advance()?;
}
self.presume_str(kind.delimiter())?;
self.token(kind.token_kind());
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use pretty_assertions::assert_eq;
macro_rules! test {
{
name: $name:ident,
text: $text:expr,
tokens: ($($kind:ident $(: $lexeme:literal)?),* $(,)?)$(,)?
} => {
#[test]
fn $name() {
let kinds: &[TokenKind] = &[$($kind,)* Eof];
let lexemes: &[&str] = &[$(lexeme!($kind $(, $lexeme)?),)* ""];
test($text, true, kinds, lexemes);
}
};
{
name: $name:ident,
text: $text:expr,
tokens: ($($kind:ident $(: $lexeme:literal)?),* $(,)?)$(,)?
unindent: $unindent:expr,
} => {
#[test]
fn $name() {
let kinds: &[TokenKind] = &[$($kind,)* Eof];
let lexemes: &[&str] = &[$(lexeme!($kind $(, $lexeme)?),)* ""];
test($text, $unindent, kinds, lexemes);
}
}
}
macro_rules! lexeme {
{
$kind:ident, $lexeme:literal
} => {
$lexeme
};
{
$kind:ident
} => {
default_lexeme($kind)
}
}
fn test(text: &str, unindent_text: bool, want_kinds: &[TokenKind], want_lexemes: &[&str]) {
let text = if unindent_text {
unindent(text)
} else {
text.to_owned()
};
let have = Lexer::lex(&text).unwrap();
let have_kinds = have
.iter()
.map(|token| token.kind)
.collect::<Vec<TokenKind>>();
let have_lexemes = have.iter().map(Token::lexeme).collect::<Vec<&str>>();
assert_eq!(have_kinds, want_kinds, "Token kind mismatch");
assert_eq!(have_lexemes, want_lexemes, "Token lexeme mismatch");
let mut roundtrip = String::new();
for lexeme in have_lexemes {
roundtrip.push_str(lexeme);
}
assert_eq!(roundtrip, text, "Roundtrip mismatch");
let mut offset = 0;
let mut line = 0;
let mut column = 0;
for token in have {
assert_eq!(token.offset, offset);
assert_eq!(token.line, line);
assert_eq!(token.lexeme().len(), token.length);
assert_eq!(token.column, column);
for c in token.lexeme().chars() {
if c == '\n' {
line += 1;
column = 0;
} else {
column += c.len_utf8();
}
}
offset += token.length;
}
}
fn default_lexeme(kind: TokenKind) -> &'static str {
match kind {
// Fixed lexemes
AmpersandAmpersand => "&&",
Asterisk => "*",
At => "@",
BangEquals => "!=",
BraceL => "{",
BraceR => "}",
BracketL => "[",
BracketR => "]",
ByteOrderMark => "\u{feff}",
Colon => ":",
ColonEquals => ":=",
Comma => ",",
Dollar => "$",
Eol => "\n",
Equals => "=",
EqualsEquals => "==",
EqualsTilde => "=~",
Indent => " ",
InterpolationEnd => "}}",
InterpolationStart => "{{",
ParenL => "(",
ParenR => ")",
Plus => "+",
Whitespace => " ",
// Empty lexemes
Dedent | Eof => "",
// Variable lexemes
Text | StringToken | Backtick | Identifier | Comment | Unspecified => {
panic!("Token {:?} has no default lexeme", kind)
}
}
}
macro_rules! error {
(
name: $name:ident,
input: $input:expr,
offset: $offset:expr,
line: $line:expr,
column: $column:expr,
width: $width:expr,
kind: $kind:expr,
) => {
#[test]
fn $name() {
error($input, $offset, $line, $column, $width, $kind);
}
};
}
fn error(
src: &str,
offset: usize,
line: usize,
column: usize,
length: usize,
kind: CompileErrorKind,
) {
match Lexer::lex(src) {
Ok(_) => panic!("Lexing succeeded but expected"),
Err(have) => {
let want = CompileError {
token: Token {
kind: have.token.kind,
src,
offset,
line,
column,
length,
},
kind,
};
assert_eq!(have, want);
}
}
}
test! {
name: name_new,
text: "foo",
tokens: (Identifier:"foo"),
}
test! {
name: comment,
text: "# hello",
tokens: (Comment:"# hello"),
}
test! {
name: backtick,
text: "`echo`",
tokens: (Backtick:"`echo`"),
}
test! {
name: backtick_multi_line,
text: "`echo\necho`",
tokens: (Backtick:"`echo\necho`"),
}
test! {
name: raw_string,
text: "'hello'",
tokens: (StringToken:"'hello'"),
}
test! {
name: raw_string_multi_line,
text: "'hello\ngoodbye'",
tokens: (StringToken:"'hello\ngoodbye'"),
}
test! {
name: cooked_string,
text: "\"hello\"",
tokens: (StringToken:"\"hello\""),
}
test! {
name: cooked_string_multi_line,
text: "\"hello\ngoodbye\"",
tokens: (StringToken:"\"hello\ngoodbye\""),
}
test! {
name: cooked_multiline_string,
text: "\"\"\"hello\ngoodbye\"\"\"",
tokens: (StringToken:"\"\"\"hello\ngoodbye\"\"\""),
}
test! {
name: ampersand_ampersand,
text: "&&",
tokens: (AmpersandAmpersand),
}
test! {
name: equals,
text: "=",
tokens: (Equals),
}
test! {
name: equals_equals,
text: "==",
tokens: (EqualsEquals),
}
test! {
name: bang_equals,
text: "!=",
tokens: (BangEquals),
}
test! {
name: brace_l,
text: "{",
tokens: (BraceL),
}
test! {
name: brace_r,
text: "{}",
tokens: (BraceL, BraceR),
}
test! {
name: brace_lll,
text: "{{{",
tokens: (BraceL, BraceL, BraceL),
}
test! {
name: brace_rrr,
text: "{{{}}}",
tokens: (BraceL, BraceL, BraceL, BraceR, BraceR, BraceR),
}
test! {
name: dollar,
text: "$",
tokens: (Dollar),
}
test! {
name: export_concatination,
text: "export foo = 'foo' + 'bar'",
tokens: (
Identifier:"export",
Whitespace,
Identifier:"foo",
Whitespace,
Equals,
Whitespace,
StringToken:"'foo'",
Whitespace,
Plus,
Whitespace,
StringToken:"'bar'",
)
}
test! {
name: export_complex,
text: "export foo = ('foo' + 'bar') + `baz`",
tokens: (
Identifier:"export",
Whitespace,
Identifier:"foo",
Whitespace,
Equals,
Whitespace,
ParenL,
StringToken:"'foo'",
Whitespace,
Plus,
Whitespace,
StringToken:"'bar'",
ParenR,
Whitespace,
Plus,
Whitespace,
Backtick:"`baz`",
),
}
test! {
name: eol_linefeed,
text: "\n",
tokens: (Eol),
unindent: false,
}
test! {
name: eol_carriage_return_linefeed,
text: "\r\n",
tokens: (Eol:"\r\n"),
unindent: false,
}
test! {
name: indented_line,
text: "foo:\n a",
tokens: (Identifier:"foo", Colon, Eol, Indent:" ", Text:"a", Dedent),
}
test! {
name: indented_normal,
text: "
a
b
c
",
tokens: (
Identifier:"a",
Eol,
Indent:" ",
Identifier:"b",
Eol,
Whitespace:" ",
Identifier:"c",
Eol,
Dedent,
),
}
test! {
name: indented_normal_nonempty_blank,
text: "a\n b\n\t\t\n c\n",
tokens: (
Identifier:"a",
Eol,
Indent:" ",
Identifier:"b",
Eol,
Whitespace:"\t\t",
Eol,
Whitespace:" ",
Identifier:"c",
Eol,
Dedent,
),
unindent: false,
}
test! {
name: indented_normal_multiple,
text: "
a
b
c
",
tokens: (
Identifier:"a",
Eol,
Indent:" ",
Identifier:"b",
Eol,
Indent:" ",
Identifier:"c",
Eol,
Dedent,
Dedent,
),
}
test! {
name: indent_indent_dedent_indent,
text: "
a
b
c
d
e
",
tokens: (
Identifier:"a",
Eol,
Indent:" ",
Identifier:"b",
Eol,
Indent:" ",
Identifier:"c",
Eol,
Dedent,
Whitespace:" ",
Identifier:"d",
Eol,
Indent:" ",
Identifier:"e",
Eol,
Dedent,
Dedent,
),
}
test! {
name: indent_recipe_dedent_indent,
text: "
a
b:
c
d
e
",
tokens: (
Identifier:"a",
Eol,
Indent:" ",
Identifier:"b",
Colon,
Eol,
Indent:" ",
Text:"c",
Eol,
Dedent,
Whitespace:" ",
Identifier:"d",
Eol,
Indent:" ",
Identifier:"e",
Eol,
Dedent,
Dedent,
),
}
test! {
name: indented_block,
text: "
foo:
a
b
c
",
tokens: (
Identifier:"foo",
Colon,
Eol,
Indent,
Text:"a",
Eol,
Whitespace:" ",
Text:"b",
Eol,
Whitespace:" ",
Text:"c",
Eol,
Dedent,
)
}
test! {
name: brace_escape,
text: "
foo:
{{{{
",
tokens: (
Identifier:"foo",
Colon,
Eol,
Indent,
Text:"{{{{",
Eol,
Dedent,
)
}
test! {
name: indented_block_followed_by_item,
text: "
foo:
a
b:
",
tokens: (
Identifier:"foo",
Colon,
Eol,
Indent,
Text:"a",
Eol,
Dedent,
Identifier:"b",
Colon,
Eol,
)
}
test! {
name: indented_block_followed_by_blank,
text: "
foo:
a
b:
",
tokens: (
Identifier:"foo",
Colon,
Eol,
Indent:" ",
Text:"a",
Eol,
Eol,
Dedent,
Identifier:"b",
Colon,
Eol,
),
}
test! {
name: indented_line_containing_unpaired_carriage_return,
text: "foo:\n \r \n",
tokens: (
Identifier:"foo",
Colon,
Eol,
Indent:" ",
Text:"\r ",
Eol,
Dedent,
),
unindent: false,
}
test! {
name: indented_blocks,
text: "
b: a
@mv a b
a:
@touch F
@touch a
d: c
@rm c
c: b
@mv b c
",
tokens: (
Identifier:"b",
Colon,
Whitespace,
Identifier:"a",
Eol,
Indent,
Text:"@mv a b",
Eol,
Eol,
Dedent,
Identifier:"a",
Colon,
Eol,
Indent,
Text:"@touch F",
Eol,
Whitespace:" ",
Text:"@touch a",
Eol,
Eol,
Dedent,
Identifier:"d",
Colon,
Whitespace,
Identifier:"c",
Eol,
Indent,
Text:"@rm c",
Eol,
Eol,
Dedent,
Identifier:"c",
Colon,
Whitespace,
Identifier:"b",
Eol,
Indent,
Text:"@mv b c",
Eol,
Dedent
),
}
test! {
name: interpolation_empty,
text: "hello:\n echo {{}}",
tokens: (
Identifier:"hello",
Colon,
Eol,
Indent:" ",
Text:"echo ",
InterpolationStart,
InterpolationEnd,
Dedent,
),
}
test! {
name: interpolation_expression,
text: "hello:\n echo {{`echo hello` + `echo goodbye`}}",
tokens: (
Identifier:"hello",
Colon,
Eol,
Indent:" ",
Text:"echo ",
InterpolationStart,
Backtick:"`echo hello`",
Whitespace,
Plus,
Whitespace,
Backtick:"`echo goodbye`",
InterpolationEnd,
Dedent,
),
}
test! {
name: interpolation_raw_multiline_string,
text: "hello:\n echo {{'\n'}}",
tokens: (
Identifier:"hello",
Colon,
Eol,
Indent:" ",
Text:"echo ",
InterpolationStart,
StringToken:"'\n'",
InterpolationEnd,
Dedent,
),
}
test! {
name: tokenize_names,
text: "
foo
bar-bob
b-bob_asdfAAAA
test123
",
tokens: (
Identifier:"foo",
Eol,
Identifier:"bar-bob",
Eol,
Identifier:"b-bob_asdfAAAA",
Eol,
Identifier:"test123",
Eol,
),
}
test! {
name: tokenize_indented_line,
text: "foo:\n a",
tokens: (
Identifier:"foo",
Colon,
Eol,
Indent:" ",
Text:"a",
Dedent,
),
}
test! {
name: tokenize_indented_block,
text: "
foo:
a
b
c
",
tokens: (
Identifier:"foo",
Colon,
Eol,
Indent,
Text:"a",
Eol,
Whitespace:" ",
Text:"b",
Eol,
Whitespace:" ",
Text:"c",
Eol,
Dedent,
),
}
test! {
name: tokenize_strings,
text: r#"a = "'a'" + '"b"' + "'c'" + '"d"'#echo hello"#,
tokens: (
Identifier:"a",
Whitespace,
Equals,
Whitespace,
StringToken:"\"'a'\"",
Whitespace,
Plus,
Whitespace,
StringToken:"'\"b\"'",
Whitespace,
Plus,
Whitespace,
StringToken:"\"'c'\"",
Whitespace,
Plus,
Whitespace,
StringToken:"'\"d\"'",
Comment:"#echo hello",
)
}
test! {
name: tokenize_recipe_interpolation_eol,
text: "
foo: # some comment
{{hello}}
",
tokens: (
Identifier:"foo",
Colon,
Whitespace,
Comment:"# some comment",
Eol,
Indent:" ",
InterpolationStart,
Identifier:"hello",
InterpolationEnd,
Eol,
Dedent
),
}
test! {
name: tokenize_recipe_interpolation_eof,
text: "foo: # more comments
{{hello}}
# another comment
",
tokens: (
Identifier:"foo",
Colon,
Whitespace,
Comment:"# more comments",
Eol,
Indent:" ",
InterpolationStart,
Identifier:"hello",
InterpolationEnd,
Eol,
Dedent,
Comment:"# another comment",
Eol,
),
}
test! {
name: tokenize_recipe_complex_interpolation_expression,
text: "foo: #lol\n {{a + b + \"z\" + blarg}}",
tokens: (
Identifier:"foo",
Colon,
Whitespace:" ",
Comment:"#lol",
Eol,
Indent:" ",
InterpolationStart,
Identifier:"a",
Whitespace,
Plus,
Whitespace,
Identifier:"b",
Whitespace,
Plus,
Whitespace,
StringToken:"\"z\"",
Whitespace,
Plus,
Whitespace,
Identifier:"blarg",
InterpolationEnd,
Dedent,
),
}
test! {
name: tokenize_recipe_multiple_interpolations,
text: "foo:,#ok\n {{a}}0{{b}}1{{c}}",
tokens: (
Identifier:"foo",
Colon,
Comma,
Comment:"#ok",
Eol,
Indent:" ",
InterpolationStart,
Identifier:"a",
InterpolationEnd,
Text:"0",
InterpolationStart,
Identifier:"b",
InterpolationEnd,
Text:"1",
InterpolationStart,
Identifier:"c",
InterpolationEnd,
Dedent,
),
}
test! {
name: tokenize_junk,
text: "
bob
hello blah blah blah : a b c #whatever
",
tokens: (
Identifier:"bob",
Eol,
Eol,
Identifier:"hello",
Whitespace,
Identifier:"blah",
Whitespace,
Identifier:"blah",
Whitespace,
Identifier:"blah",
Whitespace,
Colon,
Whitespace,
Identifier:"a",
Whitespace,
Identifier:"b",
Whitespace,
Identifier:"c",
Whitespace,
Comment:"#whatever",
Eol,
)
}
test! {
name: tokenize_empty_lines,
text: "
# this does something
hello:
asdf
bsdf
csdf
dsdf # whatever
# yolo
",
tokens: (
Eol,
Comment:"# this does something",
Eol,
Identifier:"hello",
Colon,
Eol,
Indent,
Text:"asdf",
Eol,
Whitespace:" ",
Text:"bsdf",
Eol,
Eol,
Whitespace:" ",
Text:"csdf",
Eol,
Eol,
Whitespace:" ",
Text:"dsdf # whatever",
Eol,
Eol,
Dedent,
Comment:"# yolo",
Eol,
),
}
test! {
name: tokenize_comment_before_variable,
text: "
#
A='1'
echo:
echo {{A}}
",
tokens: (
Comment:"#",
Eol,
Identifier:"A",
Equals,
StringToken:"'1'",
Eol,
Identifier:"echo",
Colon,
Eol,
Indent,
Text:"echo ",
InterpolationStart,
Identifier:"A",
InterpolationEnd,
Eol,
Dedent,
),
}
test! {
name: tokenize_interpolation_backticks,
text: "hello:\n echo {{`echo hello` + `echo goodbye`}}",
tokens: (
Identifier:"hello",
Colon,
Eol,
Indent:" ",
Text:"echo ",
InterpolationStart,
Backtick:"`echo hello`",
Whitespace,
Plus,
Whitespace,
Backtick:"`echo goodbye`",
InterpolationEnd,
Dedent
),
}
test! {
name: tokenize_empty_interpolation,
text: "hello:\n echo {{}}",
tokens: (
Identifier:"hello",
Colon,
Eol,
Indent:" ",
Text:"echo ",
InterpolationStart,
InterpolationEnd,
Dedent,
),
}
test! {
name: tokenize_assignment_backticks,
text: "a = `echo hello` + `echo goodbye`",
tokens: (
Identifier:"a",
Whitespace,
Equals,
Whitespace,
Backtick:"`echo hello`",
Whitespace,
Plus,
Whitespace,
Backtick:"`echo goodbye`",
),
}
test! {
name: tokenize_multiple,
text: "
hello:
a
b
c
d
# hello
bob:
frank
\t
",
tokens: (
Eol,
Identifier:"hello",
Colon,
Eol,
Indent,
Text:"a",
Eol,
Whitespace:" ",
Text:"b",
Eol,
Eol,
Whitespace:" ",
Text:"c",
Eol,
Eol,
Whitespace:" ",
Text:"d",
Eol,
Eol,
Dedent,
Comment:"# hello",
Eol,
Identifier:"bob",
Colon,
Eol,
Indent:" ",
Text:"frank",
Eol,
Eol,
Dedent,
),
}
test! {
name: tokenize_comment,
text: "a:=#",
tokens: (
Identifier:"a",
ColonEquals,
Comment:"#",
),
}
test! {
name: tokenize_comment_with_bang,
text: "a:=#foo!",
tokens: (
Identifier:"a",
ColonEquals,
Comment:"#foo!",
),
}
test! {
name: tokenize_order,
text: "
b: a
@mv a b
a:
@touch F
@touch a
d: c
@rm c
c: b
@mv b c
",
tokens: (
Identifier:"b",
Colon,
Whitespace,
Identifier:"a",
Eol,
Indent,
Text:"@mv a b",
Eol,
Eol,
Dedent,
Identifier:"a",
Colon,
Eol,
Indent,
Text:"@touch F",
Eol,
Whitespace:" ",
Text:"@touch a",
Eol,
Eol,
Dedent,
Identifier:"d",
Colon,
Whitespace,
Identifier:"c",
Eol,
Indent,
Text:"@rm c",
Eol,
Eol,
Dedent,
Identifier:"c",
Colon,
Whitespace,
Identifier:"b",
Eol,
Indent,
Text:"@mv b c",
Eol,
Dedent,
),
}
test! {
name: tokenize_parens,
text: "((())) ()abc(+",
tokens: (
ParenL,
ParenL,
ParenL,
ParenR,
ParenR,
ParenR,
Whitespace,
ParenL,
ParenR,
Identifier:"abc",
ParenL,
Plus,
),
}
test! {
name: crlf_newline,
text: "#\r\n#asdf\r\n",
tokens: (
Comment:"#",
Eol:"\r\n",
Comment:"#asdf",
Eol:"\r\n",
),
}
test! {
name: multiple_recipes,
text: "a:\n foo\nb:",
tokens: (
Identifier:"a",
Colon,
Eol,
Indent:" ",
Text:"foo",
Eol,
Dedent,
Identifier:"b",
Colon,
),
}
test! {
name: brackets,
text: "[][]",
tokens: (BracketL, BracketR, BracketL, BracketR),
}
test! {
name: open_delimiter_eol,
text: "[\n](\n){\n}",
tokens: (
BracketL, Whitespace:"\n", BracketR,
ParenL, Whitespace:"\n", ParenR,
BraceL, Whitespace:"\n", BraceR
),
}
error! {
name: tokenize_space_then_tab,
input: "a:
0
1
\t2
",
offset: 9,
line: 3,
column: 0,
width: 1,
kind: InconsistentLeadingWhitespace{expected: " ", found: "\t"},
}
error! {
name: tokenize_tabs_then_tab_space,
input: "a:
\t\t0
\t\t 1
\t 2
",
offset: 12,
line: 3,
column: 0,
width: 3,
kind: InconsistentLeadingWhitespace{expected: "\t\t", found: "\t "},
}
error! {
name: tokenize_unknown,
input: "%",
offset: 0,
line: 0,
column: 0,
width: 1,
kind: UnknownStartOfToken,
}
error! {
name: unterminated_string_with_escapes,
input: r#"a = "\n\t\r\"\\"#,
offset: 4,
line: 0,
column: 4,
width: 1,
kind: UnterminatedString,
}
error! {
name: unterminated_raw_string,
input: "r a='asdf",
offset: 4,
line: 0,
column: 4,
width: 1,
kind: UnterminatedString,
}
error! {
name: unterminated_interpolation,
input: "foo:\n echo {{
",
offset: 11,
line: 1,
column: 6,
width: 2,
kind: UnterminatedInterpolation,
}
error! {
name: unterminated_backtick,
input: "`echo",
offset: 0,
line: 0,
column: 0,
width: 1,
kind: UnterminatedBacktick,
}
error! {
name: unpaired_carriage_return,
input: "foo\rbar",
offset: 3,
line: 0,
column: 3,
width: 1,
kind: UnpairedCarriageReturn,
}
error! {
name: invalid_name_start_dash,
input: "-foo",
offset: 0,
line: 0,
column: 0,
width: 1,
kind: UnknownStartOfToken,
}
error! {
name: invalid_name_start_digit,
input: "0foo",
offset: 0,
line: 0,
column: 0,
width: 1,
kind: UnknownStartOfToken,
}
error! {
name: unterminated_string,
input: r#"a = ""#,
offset: 4,
line: 0,
column: 4,
width: 1,
kind: UnterminatedString,
}
error! {
name: mixed_leading_whitespace_recipe,
input: "a:\n\t echo hello",
offset: 3,
line: 1,
column: 0,
width: 2,
kind: MixedLeadingWhitespace{whitespace: "\t "},
}
error! {
name: mixed_leading_whitespace_normal,
input: "a\n\t echo hello",
offset: 2,
line: 1,
column: 0,
width: 2,
kind: MixedLeadingWhitespace{whitespace: "\t "},
}
error! {
name: mixed_leading_whitespace_indent,
input: "a\n foo\n \tbar",
offset: 7,
line: 2,
column: 0,
width: 2,
kind: MixedLeadingWhitespace{whitespace: " \t"},
}
error! {
name: bad_dedent,
input: "a\n foo\n bar\n baz",
offset: 14,
line: 3,
column: 0,
width: 2,
kind: InconsistentLeadingWhitespace{expected: " ", found: " "},
}
error! {
name: unclosed_interpolation_delimiter,
input: "a:\n echo {{ foo",
offset: 9,
line: 1,
column: 6,
width: 2,
kind: UnterminatedInterpolation,
}
error! {
name: unexpected_character_after_bang,
input: "!{",
offset: 1,
line: 0,
column: 1,
width: 1,
kind: UnexpectedCharacter { expected: '=' },
}
error! {
name: mismatched_closing_brace,
input: "(]",
offset: 1,
line: 0,
column: 1,
width: 0,
kind: MismatchedClosingDelimiter {
open: Delimiter::Paren,
close: Delimiter::Bracket,
open_line: 0,
},
}
error! {
name: bang_eof,
input: "!",
offset: 1,
line: 0,
column: 1,
width: 0,
kind: UnexpectedEndOfToken {
expected: '=',
},
}
error! {
name: bang_unexpected,
input: "!%",
offset: 1,
line: 0,
column: 1,
width: 1,
kind: UnexpectedCharacter {
expected: '=',
},
}
error! {
name: ampersand_eof,
input: "&",
offset: 1,
line: 0,
column: 1,
width: 0,
kind: UnexpectedEndOfToken {
expected: '&',
},
}
error! {
name: ampersand_unexpected,
input: "&%",
offset: 1,
line: 0,
column: 1,
width: 1,
kind: UnexpectedCharacter {
expected: '&',
},
}
#[test]
fn presume_error() {
let compile_error = Lexer::new("!").presume('-').unwrap_err();
assert_matches!(
compile_error,
CompileError {
token: Token {
offset: 0,
line: 0,
column: 0,
length: 0,
src: "!",
kind: Unspecified,
},
kind: Internal {
ref message,
},
} if message == "Lexer presumed character `-`"
);
assert_eq!(
Error::Compile { compile_error }
.color_display(Color::never())
.to_string(),
"error: Internal error, this may indicate a bug in just: \
Lexer presumed character `-`\nconsider filing an issue: \
https://github.com/casey/just/issues/new\n |\n1 | !\n | ^"
);
}
}