greg/schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Clippy lints for tokenizing.rs

Browse Source
This commit is contained in:
Greg Shuflin 2021-10-19 21:27:05 -07:00
parent 0c6c4ef47e
commit 91a7abf4cd
1 changed files with 24 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
schala-lang/language/src/tokenizing.rs
View File

@ -135,7 +135,7 @@ pub fn tokenize(input: &str) -> Vec<Token> {
let mut tokens: Vec<Token> = Vec::new(); let mut tokens: Vec<Token> = Vec::new();
let mut input = Iterator::intersperse(input.lines().enumerate(), (0, "\n")) let mut input = Iterator::intersperse(input.lines().enumerate(), (0, "\n"))
.flat_map(|(line_idx, ref line)| { .flat_map(|(line_idx, line)| {
line.chars().enumerate().map(move |(ch_idx, ch)| (line_idx, ch_idx, ch)) line.chars().enumerate().map(move |(ch_idx, ch)| (line_idx, ch_idx, ch))
}) })
.peekable(); .peekable();
@ -144,7 +144,7 @@ pub fn tokenize(input: &str) -> Vec<Token> {
let cur_tok_kind = match c { let cur_tok_kind = match c {
'/' => match input.peek().map(|t| t.2) { '/' => match input.peek().map(|t| t.2) {
Some('/') => { Some('/') => {
while let Some((_, _, c)) = input.next() { for (_, _, c) in input.by_ref() {
if c == '\n' { if c == '\n' {
break; break;
} }
@ -194,11 +194,13 @@ pub fn tokenize(input: &str) -> Vec<Token> {
} }
fn handle_digit(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind { fn handle_digit(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind {
if c == '0' && input.peek().map_or(false, |&(_, _, c)| { c == 'x' }) { let next_ch = input.peek().map(|&(_, _, c)| c);
if c == '0' && next_ch == Some('x') {
input.next(); input.next();
let rest: String = input.peeking_take_while(|&(_, _, ref c)| c.is_digit(16) || *c == '_').map(|(_, _, c)| { c }).collect(); let rest: String = input.peeking_take_while(|&(_, _, ref c)| c.is_digit(16) || *c == '_').map(|(_, _, c)| { c }).collect();
HexLiteral(Rc::new(rest)) HexLiteral(Rc::new(rest))
} else if c == '0' && input.peek().map_or(false, |&(_, _, c)| { c == 'b' }) { } else if c == '0' && next_ch == Some('b') {
input.next(); input.next();
BinNumberSigil BinNumberSigil
} else { } else {
@ -236,7 +238,8 @@ fn handle_quote(input: &mut Peekable<impl Iterator<Item=CharData>>, quote_prefix
fn handle_alphabetic(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind { fn handle_alphabetic(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind {
let mut buf = String::new(); let mut buf = String::new();
buf.push(c); buf.push(c);
if c == '_' && input.peek().map(|&(_, _, c)| { !c.is_alphabetic() }).unwrap_or(true) { let next_is_alphabetic = input.peek().map(|&(_, _, c)| !c.is_alphabetic()).unwrap_or(true);
if c == '_' && next_is_alphabetic {
return TokenKind::Underscore return TokenKind::Underscore
} }
@ -263,8 +266,9 @@ fn handle_alphabetic(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>
fn handle_operator(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind { fn handle_operator(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind {
match c { match c {
'<' | '>' | '|' | '.' | '=' => { '<' | '>' | '|' | '.' | '=' => {
let ref next = input.peek().map(|&(_, _, c)| { c }); let next = &input.peek().map(|&(_, _, c)| { c });
if !next.map(|n| { is_operator(&n) }).unwrap_or(false) { let next_is_op = next.map(|n| { is_operator(&n) }).unwrap_or(false);
if !next_is_op {
return match c { return match c {
'<' => LAngleBracket, '<' => LAngleBracket,
'>' => RAngleBracket, '>' => RAngleBracket,