Added Keyword lexical class

This commit is contained in:
greg 2015-07-22 04:01:56 -07:00
parent 25f5188d8c
commit bfa16fd6fb
2 changed files with 27 additions and 5 deletions

View File

@ -1,7 +1,7 @@
use std::slice::Iter; use std::slice::Iter;
use std::iter::Peekable; use std::iter::Peekable;
use tokenizer::{Token}; use tokenizer::{Token, Kw};
use tokenizer::Token::*; use tokenizer::Token::*;
#[derive(Debug)] #[derive(Debug)]
@ -39,6 +39,7 @@ fn expect_token(tok: Token, tokens: &mut Tokens) -> bool {
(NumLiteral(_), NumLiteral(_)) => true, (NumLiteral(_), NumLiteral(_)) => true,
(StrLiteral(_), StrLiteral(_)) => true, (StrLiteral(_), StrLiteral(_)) => true,
(Identifier(ref i1), Identifier(ref i2)) => i1 == i2, (Identifier(ref i1), Identifier(ref i2)) => i1 == i2,
(Keyword(k1), Keyword(k2)) => k1 == k2,
_ => false _ => false
} }
} }
@ -60,7 +61,7 @@ pub fn parse(input: Vec<Token>) -> ParseResult {
fn let_expression(input: &mut Tokens) -> ParseResult { fn let_expression(input: &mut Tokens) -> ParseResult {
expect!(Identifier("let".to_string()), input); expect!(Keyword(Kw::Let), input);
if let Some(&Identifier(ref name)) = input.next() { if let Some(&Identifier(ref name)) = input.next() {
if let Some(&Identifier(ref s)) = input.next() { if let Some(&Identifier(ref s)) = input.next() {
if s == "=" { if s == "=" {

View File

@ -7,8 +7,21 @@ pub enum Token {
Comma, Comma,
NumLiteral(f64), NumLiteral(f64),
StrLiteral(String), StrLiteral(String),
Identifier(String) Identifier(String),
/* Keyword(Keyword) */ //implement in future Keyword(Kw)
}
#[derive(Debug, Clone, PartialEq)]
pub enum Kw {
If,
Then,
Else,
While,
Do,
End,
Let,
Fn,
Null
} }
pub fn tokenize(input: &str) -> Vec<Token> { pub fn tokenize(input: &str) -> Vec<Token> {
@ -67,10 +80,18 @@ pub fn tokenize(input: &str) -> Vec<Token> {
match buffer.parse::<f64>() { match buffer.parse::<f64>() {
Ok(f) => tokens.push(Token::NumLiteral(f)), Ok(f) => tokens.push(Token::NumLiteral(f)),
_ => tokens.push(Token::Identifier(buffer)) _ => tokens.push(handle_identifier(buffer))
} }
} }
} }
tokens.push(Token::EOF); tokens.push(Token::EOF);
tokens tokens
} }
fn handle_identifier(identifier: String) -> Token {
if identifier == "let" {
return Token::Keyword(Kw::Let);
}
return Token::Identifier(identifier);
}