A bunch of infrastructure for keeping track of AST node locations
Plus a failing test to illustrate the reason we care
This commit is contained in:
parent
7825ef1eb9
commit
129d9ec673
@ -32,6 +32,7 @@ impl Fold for RecursiveDescentFn {
|
|||||||
if self.parse_level != 0 {
|
if self.parse_level != 0 {
|
||||||
self.parse_level -= 1;
|
self.parse_level -= 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
result.map_err(|mut parse_error: ParseError| {
|
result.map_err(|mut parse_error: ParseError| {
|
||||||
parse_error.production_name = Some(stringify!(#ident).to_string());
|
parse_error.production_name = Some(stringify!(#ident).to_string());
|
||||||
parse_error
|
parse_error
|
||||||
|
@ -40,6 +40,7 @@ mod scope_resolution;
|
|||||||
mod builtin;
|
mod builtin;
|
||||||
mod reduced_ast;
|
mod reduced_ast;
|
||||||
mod eval;
|
mod eval;
|
||||||
|
mod source_map;
|
||||||
|
|
||||||
mod schala;
|
mod schala;
|
||||||
|
|
||||||
|
@ -162,12 +162,12 @@ mod test;
|
|||||||
|
|
||||||
use std::rc::Rc;
|
use std::rc::Rc;
|
||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
use std::collections::HashMap;
|
|
||||||
|
|
||||||
use crate::tokenizing::*;
|
use crate::tokenizing::*;
|
||||||
use crate::tokenizing::Kw::*;
|
use crate::tokenizing::Kw::*;
|
||||||
use crate::tokenizing::TokenKind::*;
|
use crate::tokenizing::TokenKind::*;
|
||||||
|
|
||||||
|
use crate::source_map::{SourceMap, Location};
|
||||||
use crate::ast::*;
|
use crate::ast::*;
|
||||||
|
|
||||||
/// Represents a parsing error
|
/// Represents a parsing error
|
||||||
@ -204,9 +204,6 @@ pub struct Parser {
|
|||||||
source_map: SourceMap,
|
source_map: SourceMap,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct SourceMap {
|
|
||||||
map: HashMap<ItemId,
|
|
||||||
}
|
|
||||||
|
|
||||||
struct ParserRestrictions {
|
struct ParserRestrictions {
|
||||||
no_struct_literal: bool
|
no_struct_literal: bool
|
||||||
@ -215,14 +212,14 @@ struct ParserRestrictions {
|
|||||||
struct TokenHandler {
|
struct TokenHandler {
|
||||||
tokens: Vec<Token>,
|
tokens: Vec<Token>,
|
||||||
idx: usize,
|
idx: usize,
|
||||||
end_of_file: (usize, usize),
|
end_of_file: Location
|
||||||
}
|
}
|
||||||
|
|
||||||
impl TokenHandler {
|
impl TokenHandler {
|
||||||
fn new(tokens: Vec<Token>) -> TokenHandler {
|
fn new(tokens: Vec<Token>) -> TokenHandler {
|
||||||
let end_of_file = match tokens.last() {
|
let end_of_file = match tokens.last() {
|
||||||
None => (0, 0),
|
None => Location { line_num: 0, char_num : 0 },
|
||||||
Some(t) => (t.line_num, t.char_num)
|
Some(t) => t.location,
|
||||||
};
|
};
|
||||||
TokenHandler { idx: 0, tokens, end_of_file }
|
TokenHandler { idx: 0, tokens, end_of_file }
|
||||||
}
|
}
|
||||||
@ -235,15 +232,15 @@ impl TokenHandler {
|
|||||||
self.peek_n(n).kind
|
self.peek_n(n).kind
|
||||||
}
|
}
|
||||||
fn peek(&mut self) -> Token {
|
fn peek(&mut self) -> Token {
|
||||||
self.tokens.get(self.idx).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1})
|
self.tokens.get(self.idx).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, location: self.end_of_file })
|
||||||
}
|
}
|
||||||
/// calling peek_n(0) is the same thing as peek()
|
/// calling peek_n(0) is the same thing as peek()
|
||||||
fn peek_n(&mut self, n: usize) -> Token {
|
fn peek_n(&mut self, n: usize) -> Token {
|
||||||
self.tokens.get(self.idx + n).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1})
|
self.tokens.get(self.idx + n).map(|t: &Token| { t.clone()}).unwrap_or(Token { kind: TokenKind::EOF, location: self.end_of_file })
|
||||||
}
|
}
|
||||||
fn next(&mut self) -> Token {
|
fn next(&mut self) -> Token {
|
||||||
self.idx += 1;
|
self.idx += 1;
|
||||||
self.tokens.get(self.idx - 1).map(|t: &Token| { t.clone() }).unwrap_or(Token { kind: TokenKind::EOF, line_num: self.end_of_file.0, char_num: self.end_of_file.1})
|
self.tokens.get(self.idx - 1).map(|t: &Token| { t.clone() }).unwrap_or(Token { kind: TokenKind::EOF, location: self.end_of_file })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -126,8 +126,8 @@ fn parsing(input: Vec<tokenizing::Token>, handle: &mut Schala, comp: Option<&mut
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn format_parse_error(error: parsing::ParseError, handle: &mut Schala) -> String {
|
fn format_parse_error(error: parsing::ParseError, handle: &mut Schala) -> String {
|
||||||
let line_num = error.token.line_num;
|
let line_num = error.token.location.line_num;
|
||||||
let ch = error.token.char_num;
|
let ch = error.token.location.char_num;
|
||||||
let line_from_program = handle.source_reference.get_line(line_num);
|
let line_from_program = handle.source_reference.get_line(line_num);
|
||||||
let location_pointer = format!("{}^", " ".repeat(ch));
|
let location_pointer = format!("{}^", " ".repeat(ch));
|
||||||
|
|
||||||
|
26
schala-lang/language/src/source_map.rs
Normal file
26
schala-lang/language/src/source_map.rs
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
use std::collections::HashMap;
|
||||||
|
use std::fmt;
|
||||||
|
|
||||||
|
use crate::ast::ItemId;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||||
|
pub struct Location {
|
||||||
|
pub line_num: usize,
|
||||||
|
pub char_num: usize
|
||||||
|
}
|
||||||
|
|
||||||
|
impl fmt::Display for Location {
|
||||||
|
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||||
|
write!(f, "{}:{}", self.line_num, self.char_num)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct SourceMap {
|
||||||
|
map: HashMap<ItemId, Location>
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SourceMap {
|
||||||
|
pub fn new() -> SourceMap {
|
||||||
|
SourceMap { map: HashMap::new() }
|
||||||
|
}
|
||||||
|
}
|
@ -42,7 +42,7 @@ fn no_duplicates() {
|
|||||||
let ast = quick_ast(source);
|
let ast = quick_ast(source);
|
||||||
let output = symbol_table.add_top_level_symbols(&ast).unwrap_err();
|
let output = symbol_table.add_top_level_symbols(&ast).unwrap_err();
|
||||||
println!("OUTPUT: {}", output);
|
println!("OUTPUT: {}", output);
|
||||||
assert!(output.contains("Duplicateq"))
|
assert!(output.contains("Duplicate"))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@ -55,7 +55,8 @@ fn no_duplicates_2() {
|
|||||||
let mut symbol_table = SymbolTable::new();
|
let mut symbol_table = SymbolTable::new();
|
||||||
let ast = quick_ast(source);
|
let ast = quick_ast(source);
|
||||||
let output = symbol_table.add_top_level_symbols(&ast).unwrap_err();
|
let output = symbol_table.add_top_level_symbols(&ast).unwrap_err();
|
||||||
assert!(output.contains("Duplicate"))
|
assert!(output.contains("Duplicate"));
|
||||||
|
assert!(output.contains("Line 3"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
@ -4,6 +4,8 @@ use std::rc::Rc;
|
|||||||
use std::iter::{Iterator, Peekable};
|
use std::iter::{Iterator, Peekable};
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
|
|
||||||
|
use crate::source_map::Location;
|
||||||
|
|
||||||
#[derive(Debug, PartialEq, Clone)]
|
#[derive(Debug, PartialEq, Clone)]
|
||||||
pub enum TokenKind {
|
pub enum TokenKind {
|
||||||
Newline, Semicolon,
|
Newline, Semicolon,
|
||||||
@ -89,8 +91,7 @@ lazy_static! {
|
|||||||
#[derive(Debug, Clone, PartialEq)]
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
pub struct Token {
|
pub struct Token {
|
||||||
pub kind: TokenKind,
|
pub kind: TokenKind,
|
||||||
pub line_num: usize,
|
pub location: Location,
|
||||||
pub char_num: usize
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Token {
|
impl Token {
|
||||||
@ -101,7 +102,7 @@ impl Token {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
pub fn to_string_with_metadata(&self) -> String {
|
pub fn to_string_with_metadata(&self) -> String {
|
||||||
format!("{}(L:{},c:{})", self.kind, self.line_num, self.char_num)
|
format!("{}({})", self.kind, self.location)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn get_kind(&self) -> TokenKind {
|
pub fn get_kind(&self) -> TokenKind {
|
||||||
@ -169,7 +170,8 @@ pub fn tokenize(input: &str) -> Vec<Token> {
|
|||||||
c if is_operator(&c) => handle_operator(c, &mut input),
|
c if is_operator(&c) => handle_operator(c, &mut input),
|
||||||
unknown => Error(format!("Unexpected character: {}", unknown)),
|
unknown => Error(format!("Unexpected character: {}", unknown)),
|
||||||
};
|
};
|
||||||
tokens.push(Token { kind: cur_tok_kind, line_num, char_num });
|
let location = Location { line_num, char_num };
|
||||||
|
tokens.push(Token { kind: cur_tok_kind, location });
|
||||||
}
|
}
|
||||||
tokens
|
tokens
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user