diff --git a/schala-lang/language/src/tokenizing.rs b/schala-lang/language/src/tokenizing.rs index c883456..2aff7e5 100644 --- a/schala-lang/language/src/tokenizing.rs +++ b/schala-lang/language/src/tokenizing.rs @@ -135,7 +135,7 @@ pub fn tokenize(input: &str) -> Vec { let mut tokens: Vec = Vec::new(); let mut input = Iterator::intersperse(input.lines().enumerate(), (0, "\n")) - .flat_map(|(line_idx, ref line)| { + .flat_map(|(line_idx, line)| { line.chars().enumerate().map(move |(ch_idx, ch)| (line_idx, ch_idx, ch)) }) .peekable(); @@ -144,11 +144,11 @@ pub fn tokenize(input: &str) -> Vec { let cur_tok_kind = match c { '/' => match input.peek().map(|t| t.2) { Some('/') => { - while let Some((_, _, c)) = input.next() { - if c == '\n' { - break; + for (_, _, c) in input.by_ref() { + if c == '\n' { + break; + } } - } continue; }, Some('*') => { @@ -194,18 +194,20 @@ pub fn tokenize(input: &str) -> Vec { } fn handle_digit(c: char, input: &mut Peekable>) -> TokenKind { - if c == '0' && input.peek().map_or(false, |&(_, _, c)| { c == 'x' }) { - input.next(); - let rest: String = input.peeking_take_while(|&(_, _, ref c)| c.is_digit(16) || *c == '_').map(|(_, _, c)| { c }).collect(); - HexLiteral(Rc::new(rest)) - } else if c == '0' && input.peek().map_or(false, |&(_, _, c)| { c == 'b' }) { - input.next(); - BinNumberSigil - } else { - let mut buf = c.to_string(); - buf.extend(input.peeking_take_while(|&(_, _, ref c)| c.is_digit(10)).map(|(_, _, c)| { c })); - DigitGroup(Rc::new(buf)) - } + let next_ch = input.peek().map(|&(_, _, c)| c); + + if c == '0' && next_ch == Some('x') { + input.next(); + let rest: String = input.peeking_take_while(|&(_, _, ref c)| c.is_digit(16) || *c == '_').map(|(_, _, c)| { c }).collect(); + HexLiteral(Rc::new(rest)) + } else if c == '0' && next_ch == Some('b') { + input.next(); + BinNumberSigil + } else { + let mut buf = c.to_string(); + buf.extend(input.peeking_take_while(|&(_, _, ref c)| c.is_digit(10)).map(|(_, _, c)| { c })); + DigitGroup(Rc::new(buf)) + } } fn handle_quote(input: &mut Peekable>, quote_prefix: Option<&str>) -> TokenKind { @@ -236,7 +238,8 @@ fn handle_quote(input: &mut Peekable>, quote_prefix fn handle_alphabetic(c: char, input: &mut Peekable>) -> TokenKind { let mut buf = String::new(); buf.push(c); - if c == '_' && input.peek().map(|&(_, _, c)| { !c.is_alphabetic() }).unwrap_or(true) { + let next_is_alphabetic = input.peek().map(|&(_, _, c)| !c.is_alphabetic()).unwrap_or(true); + if c == '_' && next_is_alphabetic { return TokenKind::Underscore } @@ -263,8 +266,9 @@ fn handle_alphabetic(c: char, input: &mut Peekable> fn handle_operator(c: char, input: &mut Peekable>) -> TokenKind { match c { '<' | '>' | '|' | '.' | '=' => { - let ref next = input.peek().map(|&(_, _, c)| { c }); - if !next.map(|n| { is_operator(&n) }).unwrap_or(false) { + let next = &input.peek().map(|&(_, _, c)| { c }); + let next_is_op = next.map(|n| { is_operator(&n) }).unwrap_or(false); + if !next_is_op { return match c { '<' => LAngleBracket, '>' => RAngleBracket,