A bunch of infrastructure for keeping track of AST node locations

Plus a failing test to illustrate the reason we care
This commit is contained in:
greg 2019-10-23 00:48:59 -07:00
parent 7825ef1eb9
commit 129d9ec673
7 changed files with 46 additions and 18 deletions

View File

@ -32,6 +32,7 @@ impl Fold for RecursiveDescentFn {
if self.parse_level != 0 { if self.parse_level != 0 {
self.parse_level -= 1; self.parse_level -= 1;
} }
result.map_err(|mut parse_error: ParseError| { result.map_err(|mut parse_error: ParseError| {
parse_error.production_name = Some(stringify!(#ident).to_string()); parse_error.production_name = Some(stringify!(#ident).to_string());
parse_error parse_error

View File

@ -40,6 +40,7 @@ mod scope_resolution;
mod builtin; mod builtin;
mod reduced_ast; mod reduced_ast;
mod eval; mod eval;
mod source_map;
mod schala; mod schala;

View File

@ -162,12 +162,12 @@ mod test;
use std::rc::Rc; use std::rc::Rc;
use std::str::FromStr; use std::str::FromStr;
use std::collections::HashMap;
use crate::tokenizing::*; use crate::tokenizing::*;
use crate::tokenizing::Kw::*; use crate::tokenizing::Kw::*;
use crate::tokenizing::TokenKind::*; use crate::tokenizing::TokenKind::*;
use crate::source_map::{SourceMap, Location};
use crate::ast::*; use crate::ast::*;
/// Represents a parsing error /// Represents a parsing error
@ -204,9 +204,6 @@ pub struct Parser {
source_map: SourceMap, source_map: SourceMap,
} }
struct SourceMap {
map: HashMap<ItemId,
}
struct ParserRestrictions { struct ParserRestrictions {
no_struct_literal: bool no_struct_literal: bool
@ -215,14 +212,14 @@ struct ParserRestrictions {
struct TokenHandler { struct TokenHandler {
tokens: Vec<Token>, tokens: Vec<Token>,
idx: usize, idx: usize,
end_of_file: (usize, usize), end_of_file: Location
} }
impl TokenHandler { impl TokenHandler {
fn new(tokens: Vec<Token>) -> TokenHandler { fn new(tokens: Vec<Token>) -> TokenHandler {
let end_of_file = match tokens.last() { let end_of_file = match tokens.last() {
None => (0, 0), None => Location { line_num: 0, char_num : 0 },
Some(t) => (t.line_num, t.char_num) Some(t) => t.location,
}; };
TokenHandler { idx: 0, tokens, end_of_file } TokenHandler { idx: 0, tokens, end_of_file }
} }
@ -235,15 +232,15 @@ impl TokenHandler {
self.peek_n(n).kind self.peek_n(n).kind
} }
fn peek(&mut self) -> Token { fn peek(&mut self) -> Token {
self.tokens.get(self.idx).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1}) self.tokens.get(self.idx).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, location: self.end_of_file })
} }
/// calling peek_n(0) is the same thing as peek() /// calling peek_n(0) is the same thing as peek()
fn peek_n(&mut self, n: usize) -> Token { fn peek_n(&mut self, n: usize) -> Token {
self.tokens.get(self.idx + n).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1}) self.tokens.get(self.idx + n).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, location: self.end_of_file })
} }
fn next(&mut self) -> Token { fn next(&mut self) -> Token {
self.idx += 1; self.idx += 1;
self.tokens.get(self.idx - 1).map(|t: &Token| { t.clone() }).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1}) self.tokens.get(self.idx - 1).map(|t: &Token| { t.clone() }).unwrap_or(Token { kind: TokenKind::EOF, location: self.end_of_file })
} }
} }

View File

@ -126,8 +126,8 @@ fn parsing(input: Vec<tokenizing::Token>, handle: &mut Schala, comp: Option<&mut
} }
fn format_parse_error(error: parsing::ParseError, handle: &mut Schala) -> String { fn format_parse_error(error: parsing::ParseError, handle: &mut Schala) -> String {
let line_num = error.token.line_num; let line_num = error.token.location.line_num;
let ch = error.token.char_num; let ch = error.token.location.char_num;
let line_from_program = handle.source_reference.get_line(line_num); let line_from_program = handle.source_reference.get_line(line_num);
let location_pointer = format!("{}^", " ".repeat(ch)); let location_pointer = format!("{}^", " ".repeat(ch));

View File

@ -0,0 +1,26 @@
use std::collections::HashMap;
use std::fmt;
use crate::ast::ItemId;
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Location {
pub line_num: usize,
pub char_num: usize
}
impl fmt::Display for Location {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}:{}", self.line_num, self.char_num)
}
}
pub struct SourceMap {
map: HashMap<ItemId, Location>
}
impl SourceMap {
pub fn new() -> SourceMap {
SourceMap { map: HashMap::new() }
}
}

View File

@ -42,7 +42,7 @@ fn no_duplicates() {
let ast = quick_ast(source); let ast = quick_ast(source);
let output = symbol_table.add_top_level_symbols(&ast).unwrap_err(); let output = symbol_table.add_top_level_symbols(&ast).unwrap_err();
println!("OUTPUT: {}", output); println!("OUTPUT: {}", output);
assert!(output.contains("Duplicateq")) assert!(output.contains("Duplicate"))
} }
#[test] #[test]
@ -55,7 +55,8 @@ fn no_duplicates_2() {
let mut symbol_table = SymbolTable::new(); let mut symbol_table = SymbolTable::new();
let ast = quick_ast(source); let ast = quick_ast(source);
let output = symbol_table.add_top_level_symbols(&ast).unwrap_err(); let output = symbol_table.add_top_level_symbols(&ast).unwrap_err();
assert!(output.contains("Duplicate")) assert!(output.contains("Duplicate"));
assert!(output.contains("Line 3"));
} }
#[test] #[test]

View File

@ -4,6 +4,8 @@ use std::rc::Rc;
use std::iter::{Iterator, Peekable}; use std::iter::{Iterator, Peekable};
use std::fmt; use std::fmt;
use crate::source_map::Location;
#[derive(Debug, PartialEq, Clone)] #[derive(Debug, PartialEq, Clone)]
pub enum TokenKind { pub enum TokenKind {
Newline, Semicolon, Newline, Semicolon,
@ -89,8 +91,7 @@ lazy_static! {
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub struct Token { pub struct Token {
pub kind: TokenKind, pub kind: TokenKind,
pub line_num: usize, pub location: Location,
pub char_num: usize
} }
impl Token { impl Token {
@ -101,7 +102,7 @@ impl Token {
} }
} }
pub fn to_string_with_metadata(&self) -> String { pub fn to_string_with_metadata(&self) -> String {
format!("{}(L:{},c:{})", self.kind, self.line_num, self.char_num) format!("{}({})", self.kind, self.location)
} }
pub fn get_kind(&self) -> TokenKind { pub fn get_kind(&self) -> TokenKind {
@ -169,7 +170,8 @@ pub fn tokenize(input: &str) -> Vec<Token> {
c if is_operator(&c) => handle_operator(c, &mut input), c if is_operator(&c) => handle_operator(c, &mut input),
unknown => Error(format!("Unexpected character: {}", unknown)), unknown => Error(format!("Unexpected character: {}", unknown)),
}; };
tokens.push(Token { kind: cur_tok_kind, line_num, char_num }); let location = Location { line_num, char_num };
tokens.push(Token { kind: cur_tok_kind, location });
} }
tokens tokens
} }