schala-lang/language/src/ast.rs

@@ -9,10 +9,9 @@ mod operators;
 pub use operators::*;
 pub use visitor::ASTVisitor;
 pub use walker::walk_ast;
-use crate::tokenizing::Location;
 
 /// An abstract identifier for an AST node
-#[derive(Debug, PartialEq, Eq, Hash, Clone, Default)]
+#[derive(Debug, PartialEq, Eq, Hash, Clone)]
 pub struct ItemId {
   idx: u32,
 }
@@ -58,8 +57,6 @@ pub struct AST {
 pub struct Statement {
   #[derivative(PartialEq="ignore")]
   pub id: ItemId,
-  #[derivative(PartialEq="ignore")]
-  pub location: Location,
   pub kind: StatementKind,
 }
 

schala-lang/language/src/ast/visitor_test.rs

@@ -22,7 +22,7 @@ impl ASTVisitor for Tester {
 #[test]
 fn foo() {
   let mut tester = Tester { count: 0, float_count: 0 };
-  let ast = quick_ast(r#"
+  let (ast, _) = quick_ast(r#"
 import gragh
 
 let a = 20 + 84

schala-lang/language/src/error.rs

@@ -1,6 +1,7 @@
 use crate::parsing::ParseError;
 use crate::schala::{SourceReference, Stage};
-use crate::tokenizing::{Token, TokenKind, Location};
+use crate::source_map::Location;
+use crate::tokenizing::{Token, TokenKind};
 use crate::typechecking::TypeError;
 
 pub struct SchalaError {
@@ -84,7 +85,7 @@ struct Error {
 fn format_parse_error(error: ParseError, source_reference: &SourceReference) -> String {
     let line_num = error.token.location.line_num;
     let ch = error.token.location.char_num;
-    let line_from_program = source_reference.get_line(line_num as usize);
+    let line_from_program = source_reference.get_line(line_num);
     let location_pointer = format!("{}^", " ".repeat(ch));
 
     let line_num_digits = format!("{}", line_num).chars().count();

schala-lang/language/src/eval/test.rs

@@ -9,8 +9,9 @@ use crate::reduced_ast::reduce;
 use crate::eval::State;
 
 fn evaluate_all_outputs(input: &str) -> Vec<Result<String, String>> {
-  let mut ast = crate::util::quick_ast(input);
+  let (mut ast, source_map) = crate::util::quick_ast(input);
-  let symbol_table = Rc::new(RefCell::new(SymbolTable::new()));
+  let source_map = Rc::new(RefCell::new(source_map));
+  let symbol_table = Rc::new(RefCell::new(SymbolTable::new(source_map)));
   symbol_table.borrow_mut().add_top_level_symbols(&ast).unwrap();
   {
   let mut scope_resolver = ScopeResolver::new(symbol_table.clone());

schala-lang/language/src/lib.rs

@@ -30,6 +30,7 @@ mod scope_resolution;
 mod builtin;
 mod reduced_ast;
 mod eval;
+mod source_map;
 mod error;
 
 mod schala;

schala-lang/language/src/parsing.rs

@@ -166,9 +166,10 @@ use std::str::FromStr;
 use crate::tokenizing::*;
 use crate::tokenizing::Kw::*;
 use crate::tokenizing::TokenKind::*;
-use crate::tokenizing::Location;
 
+use crate::source_map::Location;
 use crate::ast::*;
+use crate::schala::SourceMapHandle;
 
 /// Represents a parsing error
 #[derive(Debug)]
@@ -201,6 +202,7 @@ pub struct Parser {
   parse_level: u32,
   restrictions: ParserRestrictions,
   id_store: ItemIdStore,
+  source_map: SourceMapHandle
 }
 
 
@@ -245,13 +247,14 @@ impl TokenHandler {
 
 impl Parser {
   /// Create a new parser initialized with some tokens.
-  pub fn new() -> Parser {
+  pub fn new(source_map: SourceMapHandle) -> Parser {
     Parser {
       token_handler: TokenHandler::new(vec![]),
       parse_record: vec![],
       parse_level: 0,
       restrictions: ParserRestrictions { no_struct_literal: false },
       id_store: ItemIdStore::new(),
+      source_map,
     }
   }
 
@@ -380,7 +383,8 @@ impl Parser {
       _ => self.expression().map(|expr| { StatementKind::Expression(expr) } ),
     }?;
     let id = self.id_store.fresh();
-    Ok(Statement { kind, id, location: tok.location })
+    self.source_map.borrow_mut().add_location(&id, tok.location); 
+    Ok(Statement { kind, id })
   }
 
   #[recursive_descent_method]
@@ -1036,12 +1040,11 @@ impl Parser {
 
   #[recursive_descent_method]
   fn expr_or_block(&mut self) -> ParseResult<Block> {
-    let tok = self.token_handler.peek();
+    match self.token_handler.peek_kind() {
-    match tok.get_kind() {
       LCurlyBrace => self.block(),
       _ => {
         let expr = self.expression()?;
-        let s = Statement { id: self.id_store.fresh(), location: tok.location, kind: StatementKind::Expression(expr) };
+        let s = Statement { id: self.id_store.fresh(), kind: StatementKind::Expression(expr) };
         Ok(vec![s])
       }
     }

schala-lang/language/src/parsing/test.rs

@@ -1,8 +1,8 @@
 #![cfg(test)]
+use std::cell::RefCell;
 use std::rc::Rc;
 use std::str::FromStr;
 
-use crate::tokenizing::Location;
 use super::{Parser, ParseResult, tokenize};
 use crate::ast::*;
 use super::Declaration::*;
@@ -14,8 +14,10 @@ use super::Variant::*;
 use super::ForBody::*;
 
 fn make_parser(input: &str) -> Parser {
+  let source_map = crate::source_map::SourceMap::new();
+  let source_map_handle = Rc::new(RefCell::new(source_map));
   let tokens: Vec<crate::tokenizing::Token> = tokenize(input);
-  let mut parser = super::Parser::new();
+  let mut parser = super::Parser::new(source_map_handle);
   parser.add_new_tokens(tokens);
   parser
 }
@@ -25,15 +27,6 @@ fn parse(input: &str) -> ParseResult<AST> {
   parser.parse()
 }
 
-//TODO maybe can be const?
-fn make_statement(kind: StatementKind) -> Statement {
-  Statement {
-    location: Location::default(),
-    id: ItemId::default(),
-    kind,
-  }
-}
-
 macro_rules! parse_test {
   ($string:expr, $correct:expr) => {
     assert_eq!(parse($string).unwrap(), $correct)
@@ -68,19 +61,19 @@ macro_rules! tys {
 
 macro_rules! decl {
   ($expr_type:expr) => {
-    make_statement(StatementKind::Declaration($expr_type))
+    Statement { id: ItemIdStore::new_id(), kind: StatementKind::Declaration($expr_type) }
   };
 }
 
 macro_rules! import {
   ($import_spec:expr) => {
-    make_statement(StatementKind::Import($import_spec))
+    Statement { id: ItemIdStore::new_id(), kind: StatementKind::Import($import_spec) }
   }
 }
 
 macro_rules! module {
   ($module_spec:expr) => {
-    make_statement(StatementKind::Module($module_spec))
+    Statement { id: ItemIdStore::new_id(), kind: StatementKind::Module($module_spec) }
   }
 }
 
@@ -106,9 +99,10 @@ macro_rules! prefexp {
   ($op:expr, $lhs:expr) => { PrefixExp(PrefixOp::from_str($op).unwrap(), bx!(Expression::new(ItemIdStore::new_id(), $lhs).into())) }
 }
 macro_rules! exst {
-  ($expr_type:expr) => { make_statement(StatementKind::Expression(Expression::new(ItemIdStore::new_id(), $expr_type).into())) };
+  ($expr_type:expr) => { Statement { id: ItemIdStore::new_id(), kind: StatementKind::Expression(Expression::new(ItemIdStore::new_id(), $expr_type).into())} };
-  ($expr_type:expr, $type_anno:expr) => { make_statement(StatementKind::Expression(Expression::with_anno(ItemIdStore::new_id(), $expr_type, $type_anno).into())) };
+  ($expr_type:expr, $type_anno:expr) => { Statement { id: ItemIdStore::new_id(), kind: StatementKind::Expression(Expression::with_anno(ItemIdStore::new_id(), $expr_type, $type_anno).into())} };
-  ($op:expr, $lhs:expr, $rhs:expr) => { make_statement(StatementKind::Expression(ex!(binexp!($op, $lhs, $rhs)))) };
+  ($op:expr, $lhs:expr, $rhs:expr) => { Statement { id: ItemIdStore::new_id(), ,kind: StatementKind::Expression(ex!(binexp!($op, $lhs, $rhs)))}
+  };
   (s $statement_text:expr) => {
     {
       let mut parser = make_parser($statement_text);

schala-lang/language/src/schala.rs

@@ -6,10 +6,11 @@ use std::rc::Rc;
 use schala_repl::{ProgrammingLanguageInterface,
 ComputationRequest, ComputationResponse,
 LangMetaRequest, LangMetaResponse, GlobalOutputStats};
-use crate::{reduced_ast, tokenizing, parsing, eval, typechecking, symbol_table};
+use crate::{reduced_ast, tokenizing, parsing, eval, typechecking, symbol_table, source_map};
 use crate::error::SchalaError;
 
 pub type SymbolTableHandle = Rc<RefCell<symbol_table::SymbolTable>>;
+pub type SourceMapHandle = Rc<RefCell<source_map::SourceMap>>;
 
 /// All the state necessary to parse and execute a Schala program are stored in this struct.
 /// `state` represents the execution state for the AST-walking interpreter, the other fields
@@ -17,6 +18,7 @@ pub type SymbolTableHandle = Rc<RefCell<symbol_table::SymbolTable>>;
 #[allow(dead_code)]
 pub struct Schala {
   source_reference: SourceReference,
+  source_map: SourceMapHandle,
   state: eval::State<'static>,
   symbol_table: SymbolTableHandle,
   resolver: crate::scope_resolution::ScopeResolver<'static>,
@@ -38,15 +40,17 @@ impl Schala {
 impl Schala {
   /// Creates a new Schala environment *without* any prelude.
   fn new_blank_env() -> Schala {
-    let symbols = Rc::new(RefCell::new(symbol_table::SymbolTable::new()));
+    let source_map = Rc::new(RefCell::new(source_map::SourceMap::new()));
+    let symbols = Rc::new(RefCell::new(symbol_table::SymbolTable::new(source_map.clone())));
     Schala {
       //TODO maybe these can be the same structure
       source_reference: SourceReference::new(),
       symbol_table: symbols.clone(),
+      source_map: source_map.clone(),
       resolver: crate::scope_resolution::ScopeResolver::new(symbols.clone()),
       state: eval::State::new(),
       type_context: typechecking::TypeContext::new(),
-      active_parser: parsing::Parser::new()
+      active_parser: parsing::Parser::new(source_map)
     }
   }
 

schala-lang/language/src/source_map.rs

@@ -0,0 +1,39 @@
+use std::collections::HashMap;
+use std::fmt;
+
+use crate::ast::ItemId;
+
+pub type LineNumber = usize;
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct Location {
+  pub line_num: LineNumber,
+  pub char_num: usize,
+}
+
+impl fmt::Display for Location {
+  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+    write!(f, "{}:{}", self.line_num, self.char_num)
+  }
+}
+
+pub struct SourceMap {
+  map: HashMap<ItemId, Location>
+}
+
+impl SourceMap {
+  pub fn new() -> SourceMap {
+    SourceMap { map: HashMap::new() }
+  }
+
+  pub fn add_location(&mut self, id: &ItemId, loc: Location) {
+    self.map.insert(id.clone(), loc);
+  }
+
+  pub fn lookup(&self, id: &ItemId) -> Option<Location> {
+    match self.map.get(id) {
+      Some(loc) => Some(loc.clone()),
+      None => None
+    }
+  }
+}

schala-lang/language/src/symbol_table.rs

@@ -4,7 +4,8 @@ use std::rc::Rc;
 use std::fmt;
 use std::fmt::Write;
 
-use crate::tokenizing::LineNumber;
+use crate::schala::SourceMapHandle;
+use crate::source_map::{SourceMap, LineNumber};
 use crate::ast;
 use crate::ast::{ItemId, TypeBody, TypeSingletonName, Signature, Statement, StatementKind, ModuleSpecifier};
 use crate::typechecking::TypeName;
@@ -23,8 +24,6 @@ macro_rules! fqsn {
   };
 }
 
-mod source_map;
-use source_map::SourceMap;
 mod symbol_trie;
 use symbol_trie::SymbolTrie;
 mod test;
@@ -94,16 +93,16 @@ impl ScopeSegment {
 
 //cf. p. 150 or so of Language Implementation Patterns
 pub struct SymbolTable {
-  source_map: SourceMap,
+  source_map_handle: SourceMapHandle,
   symbol_path_to_symbol: HashMap<FullyQualifiedSymbolName, Symbol>,
   id_to_fqsn: HashMap<ItemId, FullyQualifiedSymbolName>,
   symbol_trie: SymbolTrie,
 }
 
 impl SymbolTable {
-  pub fn new() -> SymbolTable {
+  pub fn new(source_map_handle: SourceMapHandle) -> SymbolTable {
     SymbolTable {
-      source_map: SourceMap::new(),
+      source_map_handle,
       symbol_path_to_symbol: HashMap::new(),
       id_to_fqsn: HashMap::new(),
       symbol_trie: SymbolTrie::new()
@@ -199,17 +198,15 @@ impl SymbolTable {
 
     for statement in statements.iter() {
       match statement {
-        Statement { kind: StatementKind::Declaration(decl), id, location, } => {
+        Statement { kind: StatementKind::Declaration(decl), id } => {
-          self.source_map.add_location(id, *location);
-
           match decl {
             FuncSig(ref signature) => {
-              seen_identifiers.try_register(&signature.name, &id, &self.source_map)
+              seen_identifiers.try_register(&signature.name, &id, &self.source_map_handle.borrow())
                 .map_err(|line| format!("Duplicate function definition: {}. It's already defined at {}", signature.name, line))?;
               self.add_function_signature(signature, scope_name_stack)?
             }
             FuncDecl(ref signature, ref body) => {
-              seen_identifiers.try_register(&signature.name, &id, &self.source_map)
+              seen_identifiers.try_register(&signature.name, &id, &self.source_map_handle.borrow())
                 .map_err(|line| format!("Duplicate function definition: {}. It's already defined at {}", signature.name, line))?;
               self.add_function_signature(signature, scope_name_stack)?;
               scope_name_stack.push(ScopeSegment{
@@ -220,21 +217,20 @@ impl SymbolTable {
               output?
             },
             TypeDecl { name, body, mutable } => {
-              seen_identifiers.try_register(&name.name, &id, &self.source_map)
+              seen_identifiers.try_register(&name.name, &id, &self.source_map_handle.borrow())
                 .map_err(|line| format!("Duplicate type definition: {}. It's already defined at {}", name.name, line))?;
               self.add_type_decl(name, body, mutable, scope_name_stack)?
             },
             Binding { name, .. } => {
-              seen_identifiers.try_register(&name, &id, &self.source_map)
+              seen_identifiers.try_register(&name, &id, &self.source_map_handle.borrow())
                 .map_err(|line| format!("Duplicate variable definition: {}. It's already defined at {}", name, line))?;
               self.add_new_symbol(name, scope_name_stack, SymbolSpec::Binding);
             }
             _ => ()
           }
         },
-        Statement { kind: StatementKind::Module(ModuleSpecifier { name, contents}), id, location } => {
+        Statement { kind: StatementKind::Module(ModuleSpecifier { name, contents}), id } => {
-          self.source_map.add_location(id, *location);
+          seen_modules.try_register(&name, &id, &self.source_map_handle.borrow())
-          seen_modules.try_register(&name, &id, &self.source_map)
             .map_err(|line| format!("Duplicate module definition: {}. It's already defined at {}", name, line))?;
           scope_name_stack.push(ScopeSegment { name: name.clone() });
           let output = self.add_symbols_from_scope(contents, scope_name_stack);

schala-lang/language/src/symbol_table/source_map.rs

@@ -1,27 +0,0 @@
-use std::collections::HashMap;
-
-use crate::ast::ItemId;
-use crate::tokenizing::Location;
-
-
-//TODO rename this type to make its purpose clearer
-pub struct SourceMap {
-  map: HashMap<ItemId, Location>
-}
-
-impl SourceMap {
-  pub fn new() -> SourceMap {
-    SourceMap { map: HashMap::new() }
-  }
-
-  pub(crate) fn add_location(&mut self, id: &ItemId, loc: Location) {
-    self.map.insert(id.clone(), loc);
-  }
-
-  pub(crate) fn lookup(&self, id: &ItemId) -> Option<Location> {
-    match self.map.get(id) {
-      Some(loc) => Some(loc.clone()),
-      None => None
-    }
-  }
-}

schala-lang/language/src/symbol_table/test.rs

@@ -1,10 +1,14 @@
 #![cfg(test)]
+use std::cell::RefCell;
+use std::rc::Rc;
+
 use super::*;
 use crate::util::quick_ast;
 
 fn add_symbols_from_source(src: &str) -> (SymbolTable, Result<(), String>) {
-  let ast = quick_ast(src);
+  let (ast, source_map) = quick_ast(src);
-  let mut symbol_table = SymbolTable::new();
+  let source_map = Rc::new(RefCell::new(source_map));
+  let mut symbol_table = SymbolTable::new(source_map);
   let result = symbol_table.add_top_level_symbols(&ast);
   (symbol_table, result)
 }

schala-lang/language/src/tokenizing.rs

@@ -1,20 +1,7 @@
 use itertools::Itertools;
 use std::{iter::{Iterator, Peekable}, convert::TryFrom, rc::Rc, fmt};
-use std::convert::TryInto;
 
-pub type LineNumber = u32;
+use crate::source_map::Location;
-
-#[derive(Debug, Clone, Copy, PartialEq, Default)]
-pub struct Location {
-  pub(crate) line_num: LineNumber,
-  pub(crate) char_num: usize,
-}
-
-impl fmt::Display for Location {
-  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-    write!(f, "{}:{}", self.line_num, self.char_num)
-  }
-}
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum TokenKind {
@@ -108,7 +95,7 @@ impl TryFrom<&str> for Kw {
 #[derive(Debug, Clone, PartialEq)]
 pub struct Token {
   pub kind: TokenKind,
-  pub(crate) location: Location,
+  pub location: Location,
 }
 
 impl Token {
@@ -184,7 +171,7 @@ pub fn tokenize(input: &str) -> Vec<Token> {
       c if is_operator(&c) => handle_operator(c, &mut input),
       unknown => Error(format!("Unexpected character: {}", unknown)),
     };
-    let location = Location { line_num: line_num.try_into().unwrap(), char_num };
+    let location = Location { line_num, char_num };
     tokens.push(Token { kind: cur_tok_kind, location });
   }
   tokens

schala-lang/language/src/typechecking.rs

@@ -464,7 +464,7 @@ mod typechecking_tests {
   macro_rules! assert_type_in_fresh_context {
     ($string:expr, $type:expr) => {
       let mut tc = TypeContext::new();
-      let ref ast = crate::util::quick_ast($string);
+      let (ref ast, _) = crate::util::quick_ast($string);
       let ty = tc.typecheck(ast).unwrap();
       assert_eq!(ty, $type)
     }

schala-lang/language/src/util.rs

@@ -48,13 +48,18 @@ impl<'a, T, V> ScopeStack<'a, T, V> where T: Hash + Eq {
 
 /// this is intended for use in tests, and does no error-handling whatsoever
 #[allow(dead_code)]
-pub fn quick_ast(input: &str) -> crate::ast::AST {
+pub fn quick_ast(input: &str) -> (crate::ast::AST, crate::source_map::SourceMap) {
+  use std::cell::RefCell;
+  use std::rc::Rc;
+
+  let source_map = crate::source_map::SourceMap::new();
+  let source_map_handle = Rc::new(RefCell::new(source_map));
   let tokens = crate::tokenizing::tokenize(input);
-  let mut parser = crate::parsing::Parser::new();
+  let mut parser = crate::parsing::Parser::new(source_map_handle.clone());
   parser.add_new_tokens(tokens);
   let output = parser.parse();
   std::mem::drop(parser);
-  output.unwrap()
+  (output.unwrap(), Rc::try_unwrap(source_map_handle).map_err(|_| ()).unwrap().into_inner())
 }
 
 #[allow(unused_macros)]