greg/schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Finish tokenizing Op separately

Browse Source
This commit is contained in:
greg 2016-01-15 00:31:00 -08:00
parent f09a6e14ba
commit e1aa7ecb17
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/tokenizer.rs
View File

@ -98,11 +98,11 @@ pub fn tokenize(input: &str) -> Option<Vec<Token>> {
Ok(f) => NumLiteral(f), Ok(f) => NumLiteral(f),
Err(_) => return None Err(_) => return None
} }
} else if !char::is_alphanumeric(c) { //TODO see if this what I want } else if !char::is_alphanumeric(c) {
let mut buffer = String::with_capacity(20); let mut buffer = String::with_capacity(20);
buffer.push(c); buffer.push(c);
loop { loop {
if iter.peek().map_or(false, |x| is_digit(x) || *x == '.') { if iter.peek().map_or(false, |x| !char::is_alphanumeric(*x)) {
let n = iter.next().unwrap(); let n = iter.next().unwrap();
buffer.push(n); buffer.push(n);
} else { } else {