Identifiers and keywords

This commit is contained in:
greg 2017-09-07 19:38:22 -07:00
parent 7fe0a6589e
commit 88d1896281
4 changed files with 33 additions and 5 deletions

View File

@ -10,4 +10,6 @@ take_mut = "0.1.3"
itertools = "0.5.8"
getopts = "*"
linefeed = "0.2.2"
lazy_static = "0.2.8"
maplit = "*"

View File

@ -1,6 +1,10 @@
#![feature(advanced_slice_patterns, slice_patterns, box_patterns)]
extern crate getopts;
extern crate linefeed;
#[macro_use]
extern crate lazy_static;
#[macro_use]
extern crate maplit;
use std::path::Path;
use std::fs::File;

View File

@ -2,8 +2,6 @@ use language::{ProgrammingLanguageInterface, EvalOptions, TraceArtifact, ReplOut
mod parsing;
use self::parsing::TokenType;
pub struct Schala {
}

View File

@ -1,6 +1,7 @@
extern crate itertools;
use language::{TokenError, ParseError};
use std::collections::HashMap;
use std::rc::Rc;
use std::iter::{Enumerate, Peekable};
use self::itertools::Itertools;
@ -27,7 +28,7 @@ pub enum TokenType {
Error(String),
}
#[derive(Debug)]
#[derive(Debug, Clone, Copy)]
pub enum Kw {
If,
Else,
@ -36,6 +37,14 @@ pub enum Kw {
Loop,
}
lazy_static! {
static ref KEYWORDS: HashMap<&'static str, Kw> =
hashmap! {
"if" => Kw::If,
"else" => Kw::Else,
};
}
#[derive(Debug)]
pub struct Token {
token_type: TokenType,
@ -85,7 +94,7 @@ pub fn tokenize(input: &str) -> Vec<Token> {
'[' => LSquareBracket, ']' => RSquareBracket,
'"' => handle_quote(&mut input),
c if is_digit(&c) => handle_digit(c, &mut input),
c @ '_' | c if c.is_alphabetic() => handle_alphabetic(c, &mut input),
c @ '_' | c if c.is_alphabetic() => handle_alphabetic(c, &mut input), //TODO I'll probably have to rewrite this if I care about types being uppercase, also type parameterization
c => handle_operator(c, &mut input),
};
tokens.push(Token { token_type: cur_tok_type, offset: idx });
@ -135,7 +144,22 @@ fn handle_quote(input: &mut CharIter) -> TokenType {
}
fn handle_alphabetic(c: char, input: &mut CharIter) -> TokenType {
unimplemented!()
let mut buf = String::new();
buf.push(c);
loop {
match input.peek().map(|&(_, c)| { c }) {
Some(c) if c.is_alphanumeric() => {
input.next();
buf.push(c);
},
_ => break,
}
}
match KEYWORDS.get(buf.as_str()) {
Some(kw) => TokenType::Keyword(kw.clone()),
None => TokenType::Identifier(Rc::new(buf)),
}
}
fn handle_operator(c: char, input: &mut CharIter) -> TokenType {