Token tests

schala-lang/language/src/error.rs

@@ -1,7 +1,7 @@
+use crate::parsing::ParseError;
 use crate::schala::{SourceReference, Stage};
 use crate::source_map::Location;
 use crate::tokenizing::{Token, TokenKind};
-use crate::parsing::ParseError;
 use crate::typechecking::TypeError;
 
 pub struct SchalaError {
@@ -11,7 +11,6 @@ pub struct SchalaError {
 }
 
 impl SchalaError {
-
     pub(crate) fn display(&self) -> String {
         if let Some(ref err) = self.formatted_parse_error {
             err.clone()
@@ -23,22 +22,29 @@ impl SchalaError {
     pub(crate) fn from_type_error(err: TypeError) -> Self {
         Self {
             formatted_parse_error: None,
-      errors: vec![
-        Error { location: None, text: Some(err.msg), stage: Stage::Typechecking }
-      ]
+            errors: vec![Error {
+                location: None,
+                text: Some(err.msg),
+                stage: Stage::Typechecking,
+            }],
         }
     }
 
     pub(crate) fn from_string(text: String, stage: Stage) -> Self {
         Self {
             formatted_parse_error: None,
-      errors: vec![
-        Error { location: None, text: Some(text), stage }
-      ]
+            errors: vec![Error {
+                location: None,
+                text: Some(text),
+                stage,
+            }],
         }
     }
 
-  pub(crate) fn from_parse_error(parse_error: ParseError, source_reference: &SourceReference) -> Self {
+    pub(crate) fn from_parse_error(
+        parse_error: ParseError,
+        source_reference: &SourceReference,
+    ) -> Self {
         Self {
             formatted_parse_error: Some(format_parse_error(parse_error, source_reference)),
             errors: vec![],
@@ -46,15 +52,17 @@ impl SchalaError {
     }
 
     pub(crate) fn from_tokens(tokens: &[Token]) -> Option<SchalaError> {
-    let token_errors: Vec<Error> = tokens.iter()
+        let token_errors: Vec<Error> = tokens
+            .iter()
             .filter_map(|tok| match tok.kind {
                 TokenKind::Error(ref err) => Some(Error {
                     location: Some(tok.location),
                     text: Some(err.clone()),
                     stage: Stage::Tokenizing,
                 }),
-        _ => None
-      }).collect();
+                _ => None,
+            })
+            .collect();
 
         if token_errors.is_empty() {
             None
@@ -74,7 +82,6 @@ struct Error {
     stage: Stage,
 }
 
-
 fn format_parse_error(error: ParseError, source_reference: &SourceReference) -> String {
     let line_num = error.token.location.line_num;
     let ch = error.token.location.char_num;
@@ -86,14 +93,21 @@ fn format_parse_error(error: ParseError, source_reference: &SourceReference) ->
 
     let production = match error.production_name {
         Some(n) => format!("\n(from production \"{}\")", n),
-    None => "".to_string()
+        None => "".to_string(),
     };
 
-  format!(r#"
+    format!(
+        r#"
 {error_msg}{production}
 {space_padding} |
 {line_num} | {}
 {space_padding} | {}
-"#, line_from_program, location_pointer, error_msg=error.msg, space_padding=space_padding, line_num=line_num, production=production
+"#,
+        line_from_program,
+        location_pointer,
+        error_msg = error.msg,
+        space_padding = space_padding,
+        line_num = line_num,
+        production = production
     )
 }

schala-lang/language/src/tokenizing.rs

@@ -302,47 +302,54 @@ mod schala_tokenizer_tests {
   macro_rules! ident { ($ident:expr) => { Identifier(Rc::new($ident.to_string())) } }
   macro_rules! op { ($ident:expr) => { Operator(Rc::new($ident.to_string())) } }
 
+  fn token_kinds(input: &str) -> Vec<TokenKind> {
+    tokenize(input).into_iter().map(move |tok| tok.kind).collect()
+  }
+
   #[test]
   fn tokens() {
-    let a = tokenize("let a: A<B> = c ++ d");
-    let token_kinds: Vec<TokenKind> = a.into_iter().map(move |t| t.kind).collect();
-    assert_eq!(token_kinds, vec![Keyword(Let), ident!("a"), Colon, ident!("A"),
+    let output = token_kinds("let a: A<B> = c ++ d");
+    assert_eq!(output, vec![Keyword(Let), ident!("a"), Colon, ident!("A"),
       LAngleBracket, ident!("B"), RAngleBracket, Equals, ident!("c"), op!("++"), ident!("d")]);
   }
 
   #[test]
   fn underscores() {
-    let token_kinds: Vec<TokenKind> = tokenize("4_8").into_iter().map(move |t| t.kind).collect();
-    assert_eq!(token_kinds, vec![digit!("4"), Underscore, digit!("8")]);
+    let output = token_kinds("4_8");
+    assert_eq!(output, vec![digit!("4"), Underscore, digit!("8")]);
 
-    let token_kinds2: Vec<TokenKind> = tokenize("aba_yo").into_iter().map(move |t| t.kind).collect();
-    assert_eq!(token_kinds2, vec![ident!("aba_yo")]);
+    let output = token_kinds("aba_yo");
+    assert_eq!(output, vec![ident!("aba_yo")]);
   }
 
   #[test]
   fn comments() {
-    let token_kinds: Vec<TokenKind> = tokenize("1  + /* hella /* bro */ */ 2").into_iter().map(move |t| t.kind).collect();
-    assert_eq!(token_kinds, vec![digit!("1"), op!("+"), digit!("2")]);
+    let output = token_kinds("1  + /* hella /* bro */ */ 2");
+    assert_eq!(output, vec![digit!("1"), op!("+"), digit!("2")]);
 
-    let token_kinds: Vec<TokenKind> = tokenize("1  + /* hella /* bro */ 2").into_iter().map(move |t| t.kind).collect();
-    assert_eq!(token_kinds, vec![digit!("1"), op!("+"), Error("Unclosed comment".to_string())]);
+    let output = token_kinds("1  + /* hella /* bro */ 2");
+    assert_eq!(output, vec![digit!("1"), op!("+"), Error("Unclosed comment".to_string())]);
+
+    //TODO not sure if I want this behavior
+    let output = token_kinds("1  + /* hella */ bro */ 2");
+    assert_eq!(output, vec![digit!("1"), op!("+"), Identifier(Rc::new("bro".to_string())), Operator(Rc::new("*".to_string())), Slash, DigitGroup(Rc::new("2".to_string()))]);
   }
 
   #[test]
   fn backtick_operators() {
-    let token_kinds: Vec<TokenKind> = tokenize("1 `plus` 2").into_iter().map(move |t| t.kind).collect();
-    assert_eq!(token_kinds, vec![digit!("1"), op!("plus"), digit!("2")]);
+    let output = token_kinds("1 `plus` 2");
+    assert_eq!(output, vec![digit!("1"), op!("plus"), digit!("2")]);
   }
 
   #[test]
   fn string_literals() {
-    let token_kinds: Vec<TokenKind> = tokenize(r#""some string""#).into_iter().map(move |t| t.kind).collect();
-    assert_eq!(token_kinds, vec![StrLiteral { s: Rc::new("some string".to_string()), prefix: None }]);
+    let output = token_kinds(r#""some string""#);
+    assert_eq!(output, vec![StrLiteral { s: Rc::new("some string".to_string()), prefix: None }]);
 
-    let token_kinds: Vec<TokenKind> = tokenize(r#"b"some bytestring""#).into_iter().map(move |t| t.kind).collect();
-    assert_eq!(token_kinds, vec![StrLiteral { s: Rc::new("some bytestring".to_string()), prefix: Some(Rc::new("b".to_string())) }]);
+    let output = token_kinds(r#"b"some bytestring""#);
+    assert_eq!(output, vec![StrLiteral { s: Rc::new("some bytestring".to_string()), prefix: Some(Rc::new("b".to_string())) }]);
 
-    let token_kinds: Vec<TokenKind> = tokenize(r#""Do \n \" escapes work\t""#).into_iter().map(move |t| t.kind).collect();
-    assert_eq!(token_kinds, vec![StrLiteral { s: Rc::new("Do \n \" escapes work\t".to_string()), prefix: None }]);
+    let output = token_kinds(r#""Do \n \" escapes work\t""#);
+    assert_eq!(output, vec![StrLiteral { s: Rc::new("Do \n \" escapes work\t".to_string()), prefix: None }]);
   }
 }