greg/schala
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Clippy lints for tokenizing.rs

Browse Source
This commit is contained in:
Greg Shuflin 2021-10-19 21:27:05 -07:00
parent 0c6c4ef47e
commit 91a7abf4cd
1 changed files with 24 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
schala-lang/language/src/tokenizing.rs
View File

@ -135,7 +135,7 @@ pub fn tokenize(input: &str) -> Vec<Token> {
let mut tokens: Vec<Token> = Vec::new(); let mut tokens: Vec<Token> = Vec::new();
let mut input = Iterator::intersperse(input.lines().enumerate(), (0, "\n")) let mut input = Iterator::intersperse(input.lines().enumerate(), (0, "\n"))
.flat_map(|(line_idx, ref line)| { .flat_map(|(line_idx, line)| {
line.chars().enumerate().map(move |(ch_idx, ch)| (line_idx, ch_idx, ch)) line.chars().enumerate().map(move |(ch_idx, ch)| (line_idx, ch_idx, ch))
}) })
.peekable(); .peekable();
@ -144,11 +144,11 @@ pub fn tokenize(input: &str) -> Vec<Token> {
let cur_tok_kind = match c { let cur_tok_kind = match c {
'/' => match input.peek().map(|t| t.2) { '/' => match input.peek().map(|t| t.2) {
Some('/') => { Some('/') => {
while let Some((_, _, c)) = input.next() { for (_, _, c) in input.by_ref() {
if c == '\n' { if c == '\n' {
break; break;
}
} }
}
continue; continue;
}, },
Some('*') => { Some('*') => {
@ -194,18 +194,20 @@ pub fn tokenize(input: &str) -> Vec<Token> {
} }
fn handle_digit(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind { fn handle_digit(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind {
if c == '0' && input.peek().map_or(false, |&(_, _, c)| { c == 'x' }) { let next_ch = input.peek().map(|&(_, _, c)| c);
input.next();
let rest: String = input.peeking_take_while(|&(_, _, ref c)| c.is_digit(16) || *c == '_').map(|(_, _, c)| { c }).collect(); if c == '0' && next_ch == Some('x') {
HexLiteral(Rc::new(rest)) input.next();
} else if c == '0' && input.peek().map_or(false, |&(_, _, c)| { c == 'b' }) { let rest: String = input.peeking_take_while(|&(_, _, ref c)| c.is_digit(16) || *c == '_').map(|(_, _, c)| { c }).collect();
input.next(); HexLiteral(Rc::new(rest))
BinNumberSigil } else if c == '0' && next_ch == Some('b') {
} else { input.next();
let mut buf = c.to_string(); BinNumberSigil
buf.extend(input.peeking_take_while(|&(_, _, ref c)| c.is_digit(10)).map(|(_, _, c)| { c })); } else {
DigitGroup(Rc::new(buf)) let mut buf = c.to_string();
} buf.extend(input.peeking_take_while(|&(_, _, ref c)| c.is_digit(10)).map(|(_, _, c)| { c }));
DigitGroup(Rc::new(buf))
}
} }
fn handle_quote(input: &mut Peekable<impl Iterator<Item=CharData>>, quote_prefix: Option<&str>) -> TokenKind { fn handle_quote(input: &mut Peekable<impl Iterator<Item=CharData>>, quote_prefix: Option<&str>) -> TokenKind {
@ -236,7 +238,8 @@ fn handle_quote(input: &mut Peekable<impl Iterator<Item=CharData>>, quote_prefix
fn handle_alphabetic(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind { fn handle_alphabetic(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind {
let mut buf = String::new(); let mut buf = String::new();
buf.push(c); buf.push(c);
if c == '_' && input.peek().map(|&(_, _, c)| { !c.is_alphabetic() }).unwrap_or(true) { let next_is_alphabetic = input.peek().map(|&(_, _, c)| !c.is_alphabetic()).unwrap_or(true);
if c == '_' && next_is_alphabetic {
return TokenKind::Underscore return TokenKind::Underscore
} }
@ -263,8 +266,9 @@ fn handle_alphabetic(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>
fn handle_operator(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind { fn handle_operator(c: char, input: &mut Peekable<impl Iterator<Item=CharData>>) -> TokenKind {
match c { match c {
'<' | '>' | '|' | '.' | '=' => { '<' | '>' | '|' | '.' | '=' => {
let ref next = input.peek().map(|&(_, _, c)| { c }); let next = &input.peek().map(|&(_, _, c)| { c });
if !next.map(|n| { is_operator(&n) }).unwrap_or(false) { let next_is_op = next.map(|n| { is_operator(&n) }).unwrap_or(false);
if !next_is_op {
return match c { return match c {
'<' => LAngleBracket, '<' => LAngleBracket,
'>' => RAngleBracket, '>' => RAngleBracket,