greg/schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Token tests

Browse Source
This commit is contained in:
Greg Shuflin 2021-10-14 05:23:24 -07:00
parent 90ede076cc
commit 63360e5617
1 changed files with 26 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
schala-lang/language/src/tokenizing.rs
View File

@ -302,47 +302,54 @@ mod schala_tokenizer_tests {
macro_rules! ident { ($ident:expr) => { Identifier(Rc::new($ident.to_string())) } } macro_rules! ident { ($ident:expr) => { Identifier(Rc::new($ident.to_string())) } }
macro_rules! op { ($ident:expr) => { Operator(Rc::new($ident.to_string())) } } macro_rules! op { ($ident:expr) => { Operator(Rc::new($ident.to_string())) } }
fn token_kinds(input: &str) -> Vec<TokenKind> {
tokenize(input).into_iter().map(move |tok| tok.kind).collect()
}
#[test] #[test]
fn tokens() { fn tokens() {
let a = tokenize("let a: A<B> = c ++ d"); let output = token_kinds("let a: A<B> = c ++ d");
let token_kinds: Vec<TokenKind> = a.into_iter().map(move |t| t.kind).collect(); assert_eq!(output, vec![Keyword(Let), ident!("a"), Colon, ident!("A"),
assert_eq!(token_kinds, vec![Keyword(Let), ident!("a"), Colon, ident!("A"),
LAngleBracket, ident!("B"), RAngleBracket, Equals, ident!("c"), op!("++"), ident!("d")]); LAngleBracket, ident!("B"), RAngleBracket, Equals, ident!("c"), op!("++"), ident!("d")]);
} }
#[test] #[test]
fn underscores() { fn underscores() {
let token_kinds: Vec<TokenKind> = tokenize("4_8").into_iter().map(move |t| t.kind).collect(); let output = token_kinds("4_8");
assert_eq!(token_kinds, vec![digit!("4"), Underscore, digit!("8")]); assert_eq!(output, vec![digit!("4"), Underscore, digit!("8")]);
let token_kinds2: Vec<TokenKind> = tokenize("aba_yo").into_iter().map(move |t| t.kind).collect(); let output = token_kinds("aba_yo");
assert_eq!(token_kinds2, vec![ident!("aba_yo")]); assert_eq!(output, vec![ident!("aba_yo")]);
} }
#[test] #[test]
fn comments() { fn comments() {
let token_kinds: Vec<TokenKind> = tokenize("1 + /* hella /* bro */ */ 2").into_iter().map(move |t| t.kind).collect(); let output = token_kinds("1 + /* hella /* bro */ */ 2");
assert_eq!(token_kinds, vec![digit!("1"), op!("+"), digit!("2")]); assert_eq!(output, vec![digit!("1"), op!("+"), digit!("2")]);
let token_kinds: Vec<TokenKind> = tokenize("1 + /* hella /* bro */ 2").into_iter().map(move |t| t.kind).collect(); let output = token_kinds("1 + /* hella /* bro */ 2");
assert_eq!(token_kinds, vec![digit!("1"), op!("+"), Error("Unclosed comment".to_string())]); assert_eq!(output, vec![digit!("1"), op!("+"), Error("Unclosed comment".to_string())]);
//TODO not sure if I want this behavior
let output = token_kinds("1 + /* hella */ bro */ 2");
assert_eq!(output, vec![digit!("1"), op!("+"), Identifier(Rc::new("bro".to_string())), Operator(Rc::new("*".to_string())), Slash, DigitGroup(Rc::new("2".to_string()))]);
} }
#[test] #[test]
fn backtick_operators() { fn backtick_operators() {
let token_kinds: Vec<TokenKind> = tokenize("1 `plus` 2").into_iter().map(move |t| t.kind).collect(); let output = token_kinds("1 `plus` 2");
assert_eq!(token_kinds, vec![digit!("1"), op!("plus"), digit!("2")]); assert_eq!(output, vec![digit!("1"), op!("plus"), digit!("2")]);
} }
#[test] #[test]
fn string_literals() { fn string_literals() {
let token_kinds: Vec<TokenKind> = tokenize(r#""some string""#).into_iter().map(move |t| t.kind).collect(); let output = token_kinds(r#""some string""#);
assert_eq!(token_kinds, vec![StrLiteral { s: Rc::new("some string".to_string()), prefix: None }]); assert_eq!(output, vec![StrLiteral { s: Rc::new("some string".to_string()), prefix: None }]);
let token_kinds: Vec<TokenKind> = tokenize(r#"b"some bytestring""#).into_iter().map(move |t| t.kind).collect(); let output = token_kinds(r#"b"some bytestring""#);
assert_eq!(token_kinds, vec![StrLiteral { s: Rc::new("some bytestring".to_string()), prefix: Some(Rc::new("b".to_string())) }]); assert_eq!(output, vec![StrLiteral { s: Rc::new("some bytestring".to_string()), prefix: Some(Rc::new("b".to_string())) }]);
let token_kinds: Vec<TokenKind> = tokenize(r#""Do \n \" escapes work\t""#).into_iter().map(move |t| t.kind).collect(); let output = token_kinds(r#""Do \n \" escapes work\t""#);
assert_eq!(token_kinds, vec![StrLiteral { s: Rc::new("Do \n \" escapes work\t".to_string()), prefix: None }]); assert_eq!(output, vec![StrLiteral { s: Rc::new("Do \n \" escapes work\t".to_string()), prefix: None }]);
} }
} }