Add number tokenizing
This commit is contained in:
parent
8e3774ffca
commit
4f17d5a0dc
25
src/main.rs
25
src/main.rs
@ -15,7 +15,7 @@ enum Token {
|
||||
LParen,
|
||||
RParen,
|
||||
Comma,
|
||||
NumLiteral(i32),
|
||||
NumLiteral(f64),
|
||||
StrLiteral(String),
|
||||
Identifier(String)
|
||||
/* Keyword(Keyword) */ //implement in future
|
||||
@ -29,7 +29,7 @@ enum ASTNode {
|
||||
|
||||
|
||||
fn repl() {
|
||||
let mut stdin = io::stdin();
|
||||
let stdin = io::stdin();
|
||||
let mut stdout = io::stdout();
|
||||
let mut buf = String::with_capacity(20);
|
||||
loop {
|
||||
@ -37,7 +37,7 @@ fn repl() {
|
||||
stdout.flush().ok();
|
||||
let line = stdin.lock().read_line(&mut buf);
|
||||
match line {
|
||||
Ok(n) => {
|
||||
Ok(_) => {
|
||||
let tokens = tokenize(&buf);
|
||||
buf.clear();
|
||||
println!("Tokens: {:?}", tokens);
|
||||
@ -60,6 +60,17 @@ fn tokenize(input: &str) -> Vec<Token> {
|
||||
let mut tokens = Vec::new();
|
||||
let mut iterator = input.chars().peekable();
|
||||
|
||||
fn ends_identifier(c: char) -> bool {
|
||||
match c {
|
||||
c if char::is_whitespace(c) => true,
|
||||
',' => true,
|
||||
';' => true,
|
||||
'(' => true,
|
||||
')' => true,
|
||||
_ => false
|
||||
}
|
||||
}
|
||||
|
||||
while let Some(c) = iterator.next() {
|
||||
if char::is_whitespace(c) {
|
||||
continue;
|
||||
@ -93,12 +104,16 @@ fn tokenize(input: &str) -> Vec<Token> {
|
||||
buffer.push(c);
|
||||
|
||||
while let Some(x) = iterator.peek().cloned() {
|
||||
if !char::is_alphanumeric(x) {
|
||||
if ends_identifier(x) {
|
||||
break;
|
||||
}
|
||||
buffer.push(iterator.next().unwrap());
|
||||
}
|
||||
tokens.push(Token::Identifier(buffer));
|
||||
|
||||
match buffer.parse::<f64>() {
|
||||
Ok(f) => tokens.push(Token::NumLiteral(f)),
|
||||
_ => tokens.push(Token::Identifier(buffer))
|
||||
}
|
||||
}
|
||||
}
|
||||
tokens.push(Token::EOF);
|
||||
|
Loading…
Reference in New Issue
Block a user