greg/schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Use itertools peeking_take_while

Browse Source 
Cuts down on lines in the tokenizer
This commit is contained in:
greg 2017-01-01 21:08:08 -08:00
parent b2e453a9de
commit 1b59c264b4
2 changed files with 9 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.toml
View File

@ -8,5 +8,6 @@ authors = ["greg <greg.shuflin@protonmail.com>"]
simplerepl = { path = "../simplerepl" } simplerepl = { path = "../simplerepl" }
llvm-sys = "*" llvm-sys = "*"
take_mut = "0.1.3" take_mut = "0.1.3"
itertools = "0.5.8"
getopts = "*" getopts = "*"

32
src/tokenizer.rs
View File

@ -1,5 +1,8 @@
extern crate itertools;
use std::iter::Peekable; use std::iter::Peekable;
use std::str::Chars; use std::str::Chars;
use self::itertools::Itertools;
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub enum Token { pub enum Token {
@ -95,15 +98,7 @@ fn tokenize_str(iter: &mut Peekable<Chars>) -> Result<Token, TokenizeError> {
fn tokenize_operator(c: char, iter: &mut Peekable<Chars>) -> Result<Token, TokenizeError> { fn tokenize_operator(c: char, iter: &mut Peekable<Chars>) -> Result<Token, TokenizeError> {
let mut buffer = String::new(); let mut buffer = String::new();
buffer.push(c); buffer.push(c);
loop { buffer.extend(iter.peeking_take_while(|x| !char::is_alphanumeric(*x) && !char::is_whitespace(*x)));
if iter.peek().map_or(false,
|x| !char::is_alphanumeric(*x) && !char::is_whitespace(*x)) {
let n = iter.next().unwrap();
buffer.push(n);
} else {
break;
}
}
Ok(Token::Operator(Op(buffer))) Ok(Token::Operator(Op(buffer)))
} }
@ -114,14 +109,8 @@ fn tokenize_number_or_period(c: char, iter: &mut Peekable<Chars>) -> Result<Toke
let mut buffer = String::new(); let mut buffer = String::new();
buffer.push(c); buffer.push(c);
loop { buffer.extend(iter.peeking_take_while(|x| is_digit(x) || *x == '.'));
if iter.peek().map_or(false, |x| is_digit(x) || *x == '.') {
let n = iter.next().unwrap();
buffer.push(n);
} else {
break;
}
}
match buffer.parse::<f64>() { match buffer.parse::<f64>() {
Ok(f) => Ok(Token::NumLiteral(f)), Ok(f) => Ok(Token::NumLiteral(f)),
Err(_) => Err(TokenizeError::new("Failed to parse digit")), Err(_) => Err(TokenizeError::new("Failed to parse digit")),
@ -138,13 +127,8 @@ fn tokenize_identifier(c: char, iter: &mut Peekable<Chars>) -> Result<Token, Tok
use self::Token::*; use self::Token::*;
let mut buffer = String::new(); let mut buffer = String::new();
buffer.push(c); buffer.push(c);
loop { buffer.extend(iter.peeking_take_while(ends_identifier));
if iter.peek().map_or(true, |x| ends_identifier(x)) {
break;
} else {
buffer.push(iter.next().unwrap());
}
}
Ok(match &buffer[..] { Ok(match &buffer[..] {
"if" => Keyword(Kw::If), "if" => Keyword(Kw::If),
"then" => Keyword(Kw::Then), "then" => Keyword(Kw::Then),