Identifiers and keywords

This commit is contained in:
greg 2017-09-07 19:38:22 -07:00
parent 7fe0a6589e
commit 88d1896281
4 changed files with 33 additions and 5 deletions

View File

@ -10,4 +10,6 @@ take_mut = "0.1.3"
itertools = "0.5.8" itertools = "0.5.8"
getopts = "*" getopts = "*"
linefeed = "0.2.2" linefeed = "0.2.2"
lazy_static = "0.2.8"
maplit = "*"

View File

@ -1,6 +1,10 @@
#![feature(advanced_slice_patterns, slice_patterns, box_patterns)] #![feature(advanced_slice_patterns, slice_patterns, box_patterns)]
extern crate getopts; extern crate getopts;
extern crate linefeed; extern crate linefeed;
#[macro_use]
extern crate lazy_static;
#[macro_use]
extern crate maplit;
use std::path::Path; use std::path::Path;
use std::fs::File; use std::fs::File;

View File

@ -2,8 +2,6 @@ use language::{ProgrammingLanguageInterface, EvalOptions, TraceArtifact, ReplOut
mod parsing; mod parsing;
use self::parsing::TokenType;
pub struct Schala { pub struct Schala {
} }

View File

@ -1,6 +1,7 @@
extern crate itertools; extern crate itertools;
use language::{TokenError, ParseError}; use language::{TokenError, ParseError};
use std::collections::HashMap;
use std::rc::Rc; use std::rc::Rc;
use std::iter::{Enumerate, Peekable}; use std::iter::{Enumerate, Peekable};
use self::itertools::Itertools; use self::itertools::Itertools;
@ -27,7 +28,7 @@ pub enum TokenType {
Error(String), Error(String),
} }
#[derive(Debug)] #[derive(Debug, Clone, Copy)]
pub enum Kw { pub enum Kw {
If, If,
Else, Else,
@ -36,6 +37,14 @@ pub enum Kw {
Loop, Loop,
} }
lazy_static! {
static ref KEYWORDS: HashMap<&'static str, Kw> =
hashmap! {
"if" => Kw::If,
"else" => Kw::Else,
};
}
#[derive(Debug)] #[derive(Debug)]
pub struct Token { pub struct Token {
token_type: TokenType, token_type: TokenType,
@ -85,7 +94,7 @@ pub fn tokenize(input: &str) -> Vec<Token> {
'[' => LSquareBracket, ']' => RSquareBracket, '[' => LSquareBracket, ']' => RSquareBracket,
'"' => handle_quote(&mut input), '"' => handle_quote(&mut input),
c if is_digit(&c) => handle_digit(c, &mut input), c if is_digit(&c) => handle_digit(c, &mut input),
c @ '_' | c if c.is_alphabetic() => handle_alphabetic(c, &mut input), c @ '_' | c if c.is_alphabetic() => handle_alphabetic(c, &mut input), //TODO I'll probably have to rewrite this if I care about types being uppercase, also type parameterization
c => handle_operator(c, &mut input), c => handle_operator(c, &mut input),
}; };
tokens.push(Token { token_type: cur_tok_type, offset: idx }); tokens.push(Token { token_type: cur_tok_type, offset: idx });
@ -135,7 +144,22 @@ fn handle_quote(input: &mut CharIter) -> TokenType {
} }
fn handle_alphabetic(c: char, input: &mut CharIter) -> TokenType { fn handle_alphabetic(c: char, input: &mut CharIter) -> TokenType {
unimplemented!() let mut buf = String::new();
buf.push(c);
loop {
match input.peek().map(|&(_, c)| { c }) {
Some(c) if c.is_alphanumeric() => {
input.next();
buf.push(c);
},
_ => break,
}
}
match KEYWORDS.get(buf.as_str()) {
Some(kw) => TokenType::Keyword(kw.clone()),
None => TokenType::Identifier(Rc::new(buf)),
}
} }
fn handle_operator(c: char, input: &mut CharIter) -> TokenType { fn handle_operator(c: char, input: &mut CharIter) -> TokenType {