greg/schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Tokenize periods separately

Browse Source
This commit is contained in:
greg 2015-07-26 01:51:15 -07:00
parent 71d2428e57
commit 9d6dc5a5f2
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/tokenizer.rs
View File

@ -5,6 +5,7 @@ pub enum Token {
LParen, LParen,
RParen, RParen,
Comma, Comma,
Period,
NumLiteral(f64), NumLiteral(f64),
StrLiteral(String), StrLiteral(String),
Identifier(String), Identifier(String),
@ -34,6 +35,7 @@ pub fn tokenize(input: &str) -> Vec<Token> {
';' => true, ';' => true,
'(' => true, '(' => true,
')' => true, ')' => true,
'.' => true,
_ => false _ => false
} }
} }
@ -69,6 +71,8 @@ pub fn tokenize(input: &str) -> Vec<Token> {
tokens.push(Token::RParen); tokens.push(Token::RParen);
} else if c == ',' { } else if c == ',' {
tokens.push(Token::Comma); tokens.push(Token::Comma);
} else if c == '.' {
tokens.push(Token::Period);
} else { } else {
let mut buffer = String::with_capacity(20); let mut buffer = String::with_capacity(20);
buffer.push(c); buffer.push(c);