Separate parsing into module

src/main.rs

@@ -1,13 +1,12 @@
 use std::io;
 use std::io::Write;
 use std::io::BufRead;
-use std::char;
-use std::slice::Iter;
 
-use tokenizer::Token;
+use tokenizer::tokenize;
-use tokenizer::Token::*;
+use parser::{parse, ParseResult};
 
 mod tokenizer;
+mod parser;
 
 
 fn main() {
@@ -15,21 +14,6 @@ fn main() {
     repl();
 }
 
-
-#[derive(Debug)]
-enum AST {
-    Name(String),
-    LangString(String),
-    Number(f64),
-    BinOp(Box<AST>, Box<AST>, Box<AST>),
-    Binding(String, Box<AST>)
-}
-
-enum ParseResult {
-    Ok(AST),
-    Err(String)
-}
-
 fn repl() {
     let stdin = io::stdin();
     let mut stdout = io::stdout();
@@ -63,129 +47,3 @@ fn repl() {
         }
     }
 }
-
-
-fn tokenize(input: &str) -> Vec<Token> {
-    let mut tokens = Vec::new();
-    let mut iterator = input.chars().peekable();
-
-    fn ends_identifier(c: char) -> bool {
-        match c {
-            c if char::is_whitespace(c) => true,
-            ',' => true,
-            ';' => true,
-            '(' => true,
-            ')' => true,
-            _ => false
-        }
-    }
-
-    while let Some(c) = iterator.next() {
-        if char::is_whitespace(c) {
-            continue;
-        } else if c == '"' {
-
-            let mut buffer = String::with_capacity(20);
-            while let Some(x) = iterator.next() {
-                if x == '"' {
-                    break;
-                }
-                buffer.push(x);
-            }
-            tokens.push(Token::StrLiteral(buffer));
-
-        } else if c == '#' {
-            while let Some(x) = iterator.next() {
-                if x == '\n' {
-                    break;
-                }
-            }
-        } else if c == ';' || c == '\n' {
-            tokens.push(Token::Separator);
-        } else if c == '(' {
-            tokens.push(Token::LParen);
-        } else if c == ')' {
-            tokens.push(Token::RParen);
-        } else if c == ',' {
-            tokens.push(Token::Comma);
-        } else {
-            let mut buffer = String::with_capacity(20);
-            buffer.push(c);
-
-            while let Some(x) = iterator.peek().cloned() {
-                if ends_identifier(x) {
-                    break;
-                }
-                buffer.push(iterator.next().unwrap());
-            }
-
-            match buffer.parse::<f64>() {
-                Ok(f) => tokens.push(Token::NumLiteral(f)),
-                _ => tokens.push(Token::Identifier(buffer))
-            }
-        }
-    }
-    tokens.push(Token::EOF);
-    tokens
-}
-
-fn parse(input: Vec<Token>) -> ParseResult {
-
-    let mut tokens = input.iter();
-
-    if let ParseResult::Ok(ast) = let_expression(&mut tokens) {
-        if expect(EOF, &mut tokens) {
-            return ParseResult::Ok(ast);
-        }
-    }
-
-    return ParseResult::Err("Bad parse".to_string());
-}
-
-fn expect(tok: Token, tokens: &mut Iter<Token>) -> bool {
-    if let Some(n) = tokens.next() {
-        let next = (*n).clone();
-        return match (tok, next) {
-            (EOF, EOF) => true,
-            (Separator, Separator) => true,
-            (LParen, LParen) => true,
-            (RParen, RParen) => true,
-            (Comma, Comma) => true,
-            (NumLiteral(_), NumLiteral(_)) => true,
-            (StrLiteral(_), StrLiteral(_)) => true,
-            (Identifier(ref i1), Identifier(ref i2)) => i1 == i2,
-            _ => false
-        }
-    }
-
-    return false;
-}
-
-fn let_expression<'a>(input: &mut Iter<Token>) -> ParseResult {
-    if expect(Identifier("let".to_string()), input) {
-        if let Some(&Identifier(ref name)) = input.next() {
-            if let Some(&Identifier(ref s)) = input.next() {
-                if s == "=" {
-                    let next = input.next();
-                    if let Some(&Identifier(ref value)) = next {
-                        let ast = AST::Binding(name.clone(), Box::new(AST::Name(value.clone())));
-                        return ParseResult::Ok(ast);
-                    }
-
-                    if let Some(&StrLiteral(ref value)) = next {
-                        let ast = AST::Binding(name.clone(), Box::new(AST::LangString(value.clone())));
-                        return ParseResult::Ok(ast);
-                    }
-
-                    if let Some(&NumLiteral(n)) = next {
-                        let ast = AST::Binding(name.clone(), Box::new(AST::Number(n)));
-                        return ParseResult::Ok(ast);
-                    }
-                }
-            }
-        }
-    }
-
-    return ParseResult::Err("Bad parse".to_string());
-}
-

src/parser.rs

@@ -0,0 +1,78 @@
+use std::slice::Iter;
+
+use tokenizer::{Token};
+use tokenizer::Token::*;
+
+#[derive(Debug)]
+pub enum AST {
+    Name(String),
+    LangString(String),
+    Number(f64),
+    BinOp(Box<AST>, Box<AST>, Box<AST>),
+    Binding(String, Box<AST>)
+}
+
+pub enum ParseResult {
+    Ok(AST),
+    Err(String)
+}
+
+pub fn parse(input: Vec<Token>) -> ParseResult {
+
+    let mut tokens = input.iter();
+
+    if let ParseResult::Ok(ast) = let_expression(&mut tokens) {
+        if expect(EOF, &mut tokens) {
+            return ParseResult::Ok(ast);
+        }
+    }
+
+    return ParseResult::Err("Bad parse".to_string());
+}
+
+fn expect(tok: Token, tokens: &mut Iter<Token>) -> bool {
+    if let Some(n) = tokens.next() {
+        let next = (*n).clone();
+        return match (tok, next) {
+            (EOF, EOF) => true,
+            (Separator, Separator) => true,
+            (LParen, LParen) => true,
+            (RParen, RParen) => true,
+            (Comma, Comma) => true,
+            (NumLiteral(_), NumLiteral(_)) => true,
+            (StrLiteral(_), StrLiteral(_)) => true,
+            (Identifier(ref i1), Identifier(ref i2)) => i1 == i2,
+            _ => false
+        }
+    }
+
+    return false;
+}
+
+fn let_expression<'a>(input: &mut Iter<Token>) -> ParseResult {
+    if expect(Identifier("let".to_string()), input) {
+        if let Some(&Identifier(ref name)) = input.next() {
+            if let Some(&Identifier(ref s)) = input.next() {
+                if s == "=" {
+                    let next = input.next();
+                    if let Some(&Identifier(ref value)) = next {
+                        let ast = AST::Binding(name.clone(), Box::new(AST::Name(value.clone())));
+                        return ParseResult::Ok(ast);
+                    }
+
+                    if let Some(&StrLiteral(ref value)) = next {
+                        let ast = AST::Binding(name.clone(), Box::new(AST::LangString(value.clone())));
+                        return ParseResult::Ok(ast);
+                    }
+
+                    if let Some(&NumLiteral(n)) = next {
+                        let ast = AST::Binding(name.clone(), Box::new(AST::Number(n)));
+                        return ParseResult::Ok(ast);
+                    }
+                }
+            }
+        }
+    }
+
+    return ParseResult::Err("Bad parse".to_string());
+}

src/tokenizer.rs

@@ -1,4 +1,3 @@
-
 #[derive(Debug, Clone)]
 pub enum Token {
     EOF,
@@ -11,3 +10,67 @@ pub enum Token {
     Identifier(String)
     /* Keyword(Keyword) */ //implement in future
 }
+
+pub fn tokenize(input: &str) -> Vec<Token> {
+    let mut tokens = Vec::new();
+    let mut iterator = input.chars().peekable();
+
+    fn ends_identifier(c: char) -> bool {
+        match c {
+            c if char::is_whitespace(c) => true,
+            ',' => true,
+            ';' => true,
+            '(' => true,
+            ')' => true,
+            _ => false
+        }
+    }
+
+    while let Some(c) = iterator.next() {
+        if char::is_whitespace(c) {
+            continue;
+        } else if c == '"' {
+
+            let mut buffer = String::with_capacity(20);
+            while let Some(x) = iterator.next() {
+                if x == '"' {
+                    break;
+                }
+                buffer.push(x);
+            }
+            tokens.push(Token::StrLiteral(buffer));
+
+        } else if c == '#' {
+            while let Some(x) = iterator.next() {
+                if x == '\n' {
+                    break;
+                }
+            }
+        } else if c == ';' || c == '\n' {
+            tokens.push(Token::Separator);
+        } else if c == '(' {
+            tokens.push(Token::LParen);
+        } else if c == ')' {
+            tokens.push(Token::RParen);
+        } else if c == ',' {
+            tokens.push(Token::Comma);
+        } else {
+            let mut buffer = String::with_capacity(20);
+            buffer.push(c);
+
+            while let Some(x) = iterator.peek().cloned() {
+                if ends_identifier(x) {
+                    break;
+                }
+                buffer.push(iterator.next().unwrap());
+            }
+
+            match buffer.parse::<f64>() {
+                Ok(f) => tokens.push(Token::NumLiteral(f)),
+                _ => tokens.push(Token::Identifier(buffer))
+            }
+        }
+    }
+    tokens.push(Token::EOF);
+    tokens
+}