greg/schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Nested comments

Browse Source
This commit is contained in:
greg 2018-03-17 22:25:43 -07:00
parent e67b22d109
commit 95a2620754
1 changed files with 24 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
src/schala_lang/tokenizing.rs
View File

@ -117,8 +117,8 @@ pub fn tokenize(input: &str) -> Vec<Token> {
while let Some((line_idx, ch_idx, c)) = input.next() { while let Some((line_idx, ch_idx, c)) = input.next() {
let cur_tok_type = match c { let cur_tok_type = match c {
'/' => match input.peek() { '/' => match input.peek().map(|t| t.2) {
Some(&(_, _, '/')) => { Some('/') => {
while let Some((_, _, c)) = input.next() { while let Some((_, _, c)) = input.next() {
if c == '\n' { if c == '\n' {
break; break;
@ -126,8 +126,22 @@ pub fn tokenize(input: &str) -> Vec<Token> {
} }
continue; continue;
}, },
Some(&(_, _, '*')) => { Some('*') => {
continue input.next();
let mut comment_level = 1;
while let Some((_, _, c)) = input.next() {
if c == '*' && input.peek().map(|t| t.2) == Some('/') {
input.next();
comment_level -= 1;
} else if c == '/' && input.peek().map(|t| t.2) == Some('*') {
input.next();
comment_level += 1;
}
if comment_level == 0 {
break;
}
}
continue;
}, },
_ => Slash _ => Slash
}, },
@ -251,7 +265,6 @@ mod schala_tokenizer_tests {
macro_rules! ident { ($ident:expr) => { Identifier(Rc::new($ident.to_string())) } } macro_rules! ident { ($ident:expr) => { Identifier(Rc::new($ident.to_string())) } }
macro_rules! op { ($ident:expr) => { Operator(Rc::new($ident.to_string())) } } macro_rules! op { ($ident:expr) => { Operator(Rc::new($ident.to_string())) } }
#[test] #[test]
fn tokens() { fn tokens() {
let a = tokenize("let a: A<B> = c ++ d"); let a = tokenize("let a: A<B> = c ++ d");
@ -265,4 +278,10 @@ mod schala_tokenizer_tests {
let token_types: Vec<TokenType> = tokenize("4_8").into_iter().map(move |t| t.token_type).collect(); let token_types: Vec<TokenType> = tokenize("4_8").into_iter().map(move |t| t.token_type).collect();
assert_eq!(token_types, vec![digit!("4"), Underscore, digit!("8")]); assert_eq!(token_types, vec![digit!("4"), Underscore, digit!("8")]);
} }
#[test]
fn comments() {
let token_types: Vec<TokenType> = tokenize("1 + /* hella /* bro */ */ 2").into_iter().map(move |t| t.token_type).collect();
assert_eq!(token_types, vec![digit!("1"), op!("+"), digit!("2")]);
}
} }