diff --git a/schala-lang/language/src/tokenizing.rs b/schala-lang/language/src/tokenizing.rs index 3410e9b..782abfe 100644 --- a/schala-lang/language/src/tokenizing.rs +++ b/schala-lang/language/src/tokenizing.rs @@ -302,47 +302,54 @@ mod schala_tokenizer_tests { macro_rules! ident { ($ident:expr) => { Identifier(Rc::new($ident.to_string())) } } macro_rules! op { ($ident:expr) => { Operator(Rc::new($ident.to_string())) } } + fn token_kinds(input: &str) -> Vec { + tokenize(input).into_iter().map(move |tok| tok.kind).collect() + } + #[test] fn tokens() { - let a = tokenize("let a: A = c ++ d"); - let token_kinds: Vec = a.into_iter().map(move |t| t.kind).collect(); - assert_eq!(token_kinds, vec![Keyword(Let), ident!("a"), Colon, ident!("A"), + let output = token_kinds("let a: A = c ++ d"); + assert_eq!(output, vec![Keyword(Let), ident!("a"), Colon, ident!("A"), LAngleBracket, ident!("B"), RAngleBracket, Equals, ident!("c"), op!("++"), ident!("d")]); } #[test] fn underscores() { - let token_kinds: Vec = tokenize("4_8").into_iter().map(move |t| t.kind).collect(); - assert_eq!(token_kinds, vec![digit!("4"), Underscore, digit!("8")]); + let output = token_kinds("4_8"); + assert_eq!(output, vec![digit!("4"), Underscore, digit!("8")]); - let token_kinds2: Vec = tokenize("aba_yo").into_iter().map(move |t| t.kind).collect(); - assert_eq!(token_kinds2, vec![ident!("aba_yo")]); + let output = token_kinds("aba_yo"); + assert_eq!(output, vec![ident!("aba_yo")]); } #[test] fn comments() { - let token_kinds: Vec = tokenize("1 + /* hella /* bro */ */ 2").into_iter().map(move |t| t.kind).collect(); - assert_eq!(token_kinds, vec![digit!("1"), op!("+"), digit!("2")]); + let output = token_kinds("1 + /* hella /* bro */ */ 2"); + assert_eq!(output, vec![digit!("1"), op!("+"), digit!("2")]); - let token_kinds: Vec = tokenize("1 + /* hella /* bro */ 2").into_iter().map(move |t| t.kind).collect(); - assert_eq!(token_kinds, vec![digit!("1"), op!("+"), Error("Unclosed comment".to_string())]); + let output = token_kinds("1 + /* hella /* bro */ 2"); + assert_eq!(output, vec![digit!("1"), op!("+"), Error("Unclosed comment".to_string())]); + + //TODO not sure if I want this behavior + let output = token_kinds("1 + /* hella */ bro */ 2"); + assert_eq!(output, vec![digit!("1"), op!("+"), Identifier(Rc::new("bro".to_string())), Operator(Rc::new("*".to_string())), Slash, DigitGroup(Rc::new("2".to_string()))]); } #[test] fn backtick_operators() { - let token_kinds: Vec = tokenize("1 `plus` 2").into_iter().map(move |t| t.kind).collect(); - assert_eq!(token_kinds, vec![digit!("1"), op!("plus"), digit!("2")]); + let output = token_kinds("1 `plus` 2"); + assert_eq!(output, vec![digit!("1"), op!("plus"), digit!("2")]); } #[test] fn string_literals() { - let token_kinds: Vec = tokenize(r#""some string""#).into_iter().map(move |t| t.kind).collect(); - assert_eq!(token_kinds, vec![StrLiteral { s: Rc::new("some string".to_string()), prefix: None }]); + let output = token_kinds(r#""some string""#); + assert_eq!(output, vec![StrLiteral { s: Rc::new("some string".to_string()), prefix: None }]); - let token_kinds: Vec = tokenize(r#"b"some bytestring""#).into_iter().map(move |t| t.kind).collect(); - assert_eq!(token_kinds, vec![StrLiteral { s: Rc::new("some bytestring".to_string()), prefix: Some(Rc::new("b".to_string())) }]); + let output = token_kinds(r#"b"some bytestring""#); + assert_eq!(output, vec![StrLiteral { s: Rc::new("some bytestring".to_string()), prefix: Some(Rc::new("b".to_string())) }]); - let token_kinds: Vec = tokenize(r#""Do \n \" escapes work\t""#).into_iter().map(move |t| t.kind).collect(); - assert_eq!(token_kinds, vec![StrLiteral { s: Rc::new("Do \n \" escapes work\t".to_string()), prefix: None }]); + let output = token_kinds(r#""Do \n \" escapes work\t""#); + assert_eq!(output, vec![StrLiteral { s: Rc::new("Do \n \" escapes work\t".to_string()), prefix: None }]); } }