From 2ec7bf3b9a044185370c77c353cd19510775c261 Mon Sep 17 00:00:00 2001
From: greg <greg.shuflin@protonmail.com>
Date: Fri, 16 Nov 2018 03:51:03 -0800
Subject: [PATCH] Some initial work on passing token metadata to AST

---
 schala-lang/language/src/parsing.rs    | 10 +++++++++-
 schala-lang/language/src/tokenizing.rs |  9 +++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/schala-lang/language/src/parsing.rs b/schala-lang/language/src/parsing.rs
index 7cdd1ee..51d849e 100644
--- a/schala-lang/language/src/parsing.rs
+++ b/schala-lang/language/src/parsing.rs
@@ -64,6 +64,10 @@ impl TokenHandler {
   fn next(&mut self) -> TokenType {
     self.tokens.next().map(|ref t| { t.token_type.clone() }).unwrap_or(TokenType::EOF)
   }
+
+  fn next_with_metadata(&mut self) -> Token {
+    self.tokens.next().unwrap_or(Token { token_type: TokenType::EOF, offset: (0, 0) })
+  }
 }
 
 impl Parser {
@@ -86,6 +90,10 @@ impl Parser {
     self.token_handler.next()
   }
 
+  fn next_with_metadata(&mut self) -> Token {
+    self.token_handler.next_with_metadata()
+  }
+
   pub fn parse(&mut self) -> ParseResult<AST> {
     self.program()
   }
@@ -1015,7 +1023,7 @@ impl Parser {
   #[recursive_descent_method]
   fn int_literal(&mut self) -> ParseResult<Expression> {
     use self::ExpressionType::*;
-    match self.next() {
+    match self.next_with_metadata().get_token_type() {
       BinNumberSigil => {
         let digits = self.digits()?;
         let n = parse_binary(digits)?;
diff --git a/schala-lang/language/src/tokenizing.rs b/schala-lang/language/src/tokenizing.rs
index a5bd19d..12b20c0 100644
--- a/schala-lang/language/src/tokenizing.rs
+++ b/schala-lang/language/src/tokenizing.rs
@@ -91,6 +91,11 @@ pub struct Token {
   pub offset: (usize, usize),
 }
 
+#[derive(Debug, Clone)]
+pub struct TokenMetadata {
+  pub offset: (usize, usize)
+}
+
 impl Token {
   pub fn get_error(&self) -> Option<String> {
     match self.token_type {
@@ -101,6 +106,10 @@ impl Token {
   pub fn to_string_with_metadata(&self) -> String {
     format!("{}(L:{},c:{})", self.token_type, self.offset.0, self.offset.1)
   }
+
+  pub fn get_token_type(&self) -> TokenType {
+    self.token_type.clone()
+  }
 }
 
 const OPERATOR_CHARS: [char; 18] = ['!', '$', '%', '&', '*', '+', '-', '.', ':', '<', '>', '=', '?', '@', '^', '|', '~', '`'];