Move tokenizing into separate module

This commit is contained in:
greg 2015-07-22 03:02:55 -07:00
parent 044e7a6a26
commit 429ace73bd
2 changed files with 17 additions and 13 deletions

View File

@ -4,7 +4,10 @@ use std::io::BufRead;
use std::char;
use std::slice::Iter;
use Token::*;
use tokenizer::Token;
use tokenizer::Token::*;
mod tokenizer;
fn main() {
@ -12,18 +15,6 @@ fn main() {
repl();
}
#[derive(Debug, Clone)]
enum Token {
EOF,
Separator,
LParen,
RParen,
Comma,
NumLiteral(f64),
StrLiteral(String),
Identifier(String)
/* Keyword(Keyword) */ //implement in future
}
#[derive(Debug)]
enum AST {

13
src/tokenizer.rs Normal file
View File

@ -0,0 +1,13 @@
#[derive(Debug, Clone)]
pub enum Token {
EOF,
Separator,
LParen,
RParen,
Comma,
NumLiteral(f64),
StrLiteral(String),
Identifier(String)
/* Keyword(Keyword) */ //implement in future
}