Move tokenizing into separate module
This commit is contained in:
parent
044e7a6a26
commit
429ace73bd
17
src/main.rs
17
src/main.rs
@ -4,7 +4,10 @@ use std::io::BufRead;
|
||||
use std::char;
|
||||
use std::slice::Iter;
|
||||
|
||||
use Token::*;
|
||||
use tokenizer::Token;
|
||||
use tokenizer::Token::*;
|
||||
|
||||
mod tokenizer;
|
||||
|
||||
|
||||
fn main() {
|
||||
@ -12,18 +15,6 @@ fn main() {
|
||||
repl();
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
enum Token {
|
||||
EOF,
|
||||
Separator,
|
||||
LParen,
|
||||
RParen,
|
||||
Comma,
|
||||
NumLiteral(f64),
|
||||
StrLiteral(String),
|
||||
Identifier(String)
|
||||
/* Keyword(Keyword) */ //implement in future
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum AST {
|
||||
|
13
src/tokenizer.rs
Normal file
13
src/tokenizer.rs
Normal file
@ -0,0 +1,13 @@
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Token {
|
||||
EOF,
|
||||
Separator,
|
||||
LParen,
|
||||
RParen,
|
||||
Comma,
|
||||
NumLiteral(f64),
|
||||
StrLiteral(String),
|
||||
Identifier(String)
|
||||
/* Keyword(Keyword) */ //implement in future
|
||||
}
|
Loading…
Reference in New Issue
Block a user