Identifiers and keywords
This commit is contained in:
parent
7fe0a6589e
commit
88d1896281
@ -10,4 +10,6 @@ take_mut = "0.1.3"
|
||||
itertools = "0.5.8"
|
||||
getopts = "*"
|
||||
linefeed = "0.2.2"
|
||||
lazy_static = "0.2.8"
|
||||
maplit = "*"
|
||||
|
||||
|
@ -1,6 +1,10 @@
|
||||
#![feature(advanced_slice_patterns, slice_patterns, box_patterns)]
|
||||
extern crate getopts;
|
||||
extern crate linefeed;
|
||||
#[macro_use]
|
||||
extern crate lazy_static;
|
||||
#[macro_use]
|
||||
extern crate maplit;
|
||||
|
||||
use std::path::Path;
|
||||
use std::fs::File;
|
||||
|
@ -2,8 +2,6 @@ use language::{ProgrammingLanguageInterface, EvalOptions, TraceArtifact, ReplOut
|
||||
|
||||
mod parsing;
|
||||
|
||||
use self::parsing::TokenType;
|
||||
|
||||
pub struct Schala {
|
||||
}
|
||||
|
||||
|
@ -1,6 +1,7 @@
|
||||
extern crate itertools;
|
||||
|
||||
use language::{TokenError, ParseError};
|
||||
use std::collections::HashMap;
|
||||
use std::rc::Rc;
|
||||
use std::iter::{Enumerate, Peekable};
|
||||
use self::itertools::Itertools;
|
||||
@ -27,7 +28,7 @@ pub enum TokenType {
|
||||
Error(String),
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum Kw {
|
||||
If,
|
||||
Else,
|
||||
@ -36,6 +37,14 @@ pub enum Kw {
|
||||
Loop,
|
||||
}
|
||||
|
||||
lazy_static! {
|
||||
static ref KEYWORDS: HashMap<&'static str, Kw> =
|
||||
hashmap! {
|
||||
"if" => Kw::If,
|
||||
"else" => Kw::Else,
|
||||
};
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Token {
|
||||
token_type: TokenType,
|
||||
@ -85,7 +94,7 @@ pub fn tokenize(input: &str) -> Vec<Token> {
|
||||
'[' => LSquareBracket, ']' => RSquareBracket,
|
||||
'"' => handle_quote(&mut input),
|
||||
c if is_digit(&c) => handle_digit(c, &mut input),
|
||||
c @ '_' | c if c.is_alphabetic() => handle_alphabetic(c, &mut input),
|
||||
c @ '_' | c if c.is_alphabetic() => handle_alphabetic(c, &mut input), //TODO I'll probably have to rewrite this if I care about types being uppercase, also type parameterization
|
||||
c => handle_operator(c, &mut input),
|
||||
};
|
||||
tokens.push(Token { token_type: cur_tok_type, offset: idx });
|
||||
@ -135,7 +144,22 @@ fn handle_quote(input: &mut CharIter) -> TokenType {
|
||||
}
|
||||
|
||||
fn handle_alphabetic(c: char, input: &mut CharIter) -> TokenType {
|
||||
unimplemented!()
|
||||
let mut buf = String::new();
|
||||
buf.push(c);
|
||||
loop {
|
||||
match input.peek().map(|&(_, c)| { c }) {
|
||||
Some(c) if c.is_alphanumeric() => {
|
||||
input.next();
|
||||
buf.push(c);
|
||||
},
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
|
||||
match KEYWORDS.get(buf.as_str()) {
|
||||
Some(kw) => TokenType::Keyword(kw.clone()),
|
||||
None => TokenType::Identifier(Rc::new(buf)),
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_operator(c: char, input: &mut CharIter) -> TokenType {
|
||||
|
Loading…
Reference in New Issue
Block a user