Add tokenization for string literal prefixes
This commit is contained in:
parent
efc8497235
commit
d20acf7166
@ -991,7 +991,7 @@ impl Parser {
|
||||
self.token_handler.next();
|
||||
Pattern::Literal(PatternLiteral::BoolPattern(false))
|
||||
},
|
||||
StrLiteral(s) => {
|
||||
StrLiteral { s, .. } => {
|
||||
self.token_handler.next();
|
||||
Pattern::Literal(PatternLiteral::StringPattern(s))
|
||||
},
|
||||
@ -1140,7 +1140,7 @@ impl Parser {
|
||||
let id = self.id_store.fresh();
|
||||
Ok(Expression::new(id, BoolLiteral(false)))
|
||||
},
|
||||
StrLiteral(s) => {
|
||||
StrLiteral {s, ..} => {
|
||||
self.token_handler.next();
|
||||
let id = self.id_store.fresh();
|
||||
Ok(Expression::new(id, StringLiteral(s.clone())))
|
||||
|
@ -21,7 +21,10 @@ pub enum TokenKind {
|
||||
|
||||
Operator(Rc<String>),
|
||||
DigitGroup(Rc<String>), HexLiteral(Rc<String>), BinNumberSigil,
|
||||
StrLiteral(Rc<String>),
|
||||
StrLiteral {
|
||||
s: Rc<String>,
|
||||
prefix: Option<Rc<String>>
|
||||
},
|
||||
Identifier(Rc<String>),
|
||||
Keyword(Kw),
|
||||
|
||||
@ -37,7 +40,7 @@ impl fmt::Display for TokenKind {
|
||||
&Operator(ref s) => write!(f, "Operator({})", **s),
|
||||
&DigitGroup(ref s) => write!(f, "DigitGroup({})", s),
|
||||
&HexLiteral(ref s) => write!(f, "HexLiteral({})", s),
|
||||
&StrLiteral(ref s) => write!(f, "StrLiteral({})", s),
|
||||
&StrLiteral {ref s, .. } => write!(f, "StrLiteral({})", s),
|
||||
&Identifier(ref s) => write!(f, "Identifier({})", s),
|
||||
&Error(ref s) => write!(f, "Error({})", s),
|
||||
other => write!(f, "{:?}", other),
|
||||
@ -163,7 +166,7 @@ pub fn tokenize(input: &str) -> Vec<Token> {
|
||||
'(' => LParen, ')' => RParen,
|
||||
'{' => LCurlyBrace, '}' => RCurlyBrace,
|
||||
'[' => LSquareBracket, ']' => RSquareBracket,
|
||||
'"' => handle_quote(&mut input),
|
||||
'"' => handle_quote(&mut input, None),
|
||||
'\\' => Backslash,
|
||||
c if c.is_digit(10) => handle_digit(c, &mut input),
|
||||
c if c.is_alphabetic() || c == '_' => handle_alphabetic(c, &mut input),
|
||||
@ -191,7 +194,7 @@ fn handle_digit(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>) ->
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_quote(input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind {
|
||||
fn handle_quote(input: &mut Peekable<impl Iterator<Item=CharData>>, quote_prefix: Option<&str>) -> TokenKind {
|
||||
let mut buf = String::new();
|
||||
loop {
|
||||
match input.next().map(|(_, _, c)| { c }) {
|
||||
@ -213,7 +216,7 @@ fn handle_quote(input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind
|
||||
None => return TokenKind::Error(format!("Unclosed string")),
|
||||
}
|
||||
}
|
||||
TokenKind::StrLiteral(Rc::new(buf))
|
||||
TokenKind::StrLiteral { s: Rc::new(buf), prefix: quote_prefix.map(|s| Rc::new(s.to_string())) }
|
||||
}
|
||||
|
||||
fn handle_alphabetic(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind {
|
||||
@ -225,6 +228,10 @@ fn handle_alphabetic(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>
|
||||
|
||||
loop {
|
||||
match input.peek().map(|&(_, _, c)| { c }) {
|
||||
Some(c) if c == '"' => {
|
||||
input.next();
|
||||
return handle_quote(input, Some(&buf));
|
||||
},
|
||||
Some(c) if c.is_alphanumeric() || c == '_' => {
|
||||
input.next();
|
||||
buf.push(c);
|
||||
@ -325,4 +332,13 @@ mod schala_tokenizer_tests {
|
||||
let token_kinds: Vec<TokenKind> = tokenize("1 `plus` 2").into_iter().map(move |t| t.kind).collect();
|
||||
assert_eq!(token_kinds, vec![digit!("1"), op!("plus"), digit!("2")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn string_literals() {
|
||||
let token_kinds: Vec<TokenKind> = tokenize(r#""some string""#).into_iter().map(move |t| t.kind).collect();
|
||||
assert_eq!(token_kinds, vec![StrLiteral { s: Rc::new("some string".to_string()), prefix: None }]);
|
||||
|
||||
let token_kinds: Vec<TokenKind> = tokenize(r#"b"some bytestring""#).into_iter().map(move |t| t.kind).collect();
|
||||
assert_eq!(token_kinds, vec![StrLiteral { s: Rc::new("some bytestring".to_string()), prefix: Some(Rc::new("b".to_string())) }]);
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user