Identifiers and keywords
This commit is contained in:
parent
7fe0a6589e
commit
88d1896281
@ -10,4 +10,6 @@ take_mut = "0.1.3"
|
|||||||
itertools = "0.5.8"
|
itertools = "0.5.8"
|
||||||
getopts = "*"
|
getopts = "*"
|
||||||
linefeed = "0.2.2"
|
linefeed = "0.2.2"
|
||||||
|
lazy_static = "0.2.8"
|
||||||
|
maplit = "*"
|
||||||
|
|
||||||
|
@ -1,6 +1,10 @@
|
|||||||
#![feature(advanced_slice_patterns, slice_patterns, box_patterns)]
|
#![feature(advanced_slice_patterns, slice_patterns, box_patterns)]
|
||||||
extern crate getopts;
|
extern crate getopts;
|
||||||
extern crate linefeed;
|
extern crate linefeed;
|
||||||
|
#[macro_use]
|
||||||
|
extern crate lazy_static;
|
||||||
|
#[macro_use]
|
||||||
|
extern crate maplit;
|
||||||
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
|
@ -2,8 +2,6 @@ use language::{ProgrammingLanguageInterface, EvalOptions, TraceArtifact, ReplOut
|
|||||||
|
|
||||||
mod parsing;
|
mod parsing;
|
||||||
|
|
||||||
use self::parsing::TokenType;
|
|
||||||
|
|
||||||
pub struct Schala {
|
pub struct Schala {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
extern crate itertools;
|
extern crate itertools;
|
||||||
|
|
||||||
use language::{TokenError, ParseError};
|
use language::{TokenError, ParseError};
|
||||||
|
use std::collections::HashMap;
|
||||||
use std::rc::Rc;
|
use std::rc::Rc;
|
||||||
use std::iter::{Enumerate, Peekable};
|
use std::iter::{Enumerate, Peekable};
|
||||||
use self::itertools::Itertools;
|
use self::itertools::Itertools;
|
||||||
@ -27,7 +28,7 @@ pub enum TokenType {
|
|||||||
Error(String),
|
Error(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug, Clone, Copy)]
|
||||||
pub enum Kw {
|
pub enum Kw {
|
||||||
If,
|
If,
|
||||||
Else,
|
Else,
|
||||||
@ -36,6 +37,14 @@ pub enum Kw {
|
|||||||
Loop,
|
Loop,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
lazy_static! {
|
||||||
|
static ref KEYWORDS: HashMap<&'static str, Kw> =
|
||||||
|
hashmap! {
|
||||||
|
"if" => Kw::If,
|
||||||
|
"else" => Kw::Else,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct Token {
|
pub struct Token {
|
||||||
token_type: TokenType,
|
token_type: TokenType,
|
||||||
@ -85,7 +94,7 @@ pub fn tokenize(input: &str) -> Vec<Token> {
|
|||||||
'[' => LSquareBracket, ']' => RSquareBracket,
|
'[' => LSquareBracket, ']' => RSquareBracket,
|
||||||
'"' => handle_quote(&mut input),
|
'"' => handle_quote(&mut input),
|
||||||
c if is_digit(&c) => handle_digit(c, &mut input),
|
c if is_digit(&c) => handle_digit(c, &mut input),
|
||||||
c @ '_' | c if c.is_alphabetic() => handle_alphabetic(c, &mut input),
|
c @ '_' | c if c.is_alphabetic() => handle_alphabetic(c, &mut input), //TODO I'll probably have to rewrite this if I care about types being uppercase, also type parameterization
|
||||||
c => handle_operator(c, &mut input),
|
c => handle_operator(c, &mut input),
|
||||||
};
|
};
|
||||||
tokens.push(Token { token_type: cur_tok_type, offset: idx });
|
tokens.push(Token { token_type: cur_tok_type, offset: idx });
|
||||||
@ -135,7 +144,22 @@ fn handle_quote(input: &mut CharIter) -> TokenType {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn handle_alphabetic(c: char, input: &mut CharIter) -> TokenType {
|
fn handle_alphabetic(c: char, input: &mut CharIter) -> TokenType {
|
||||||
unimplemented!()
|
let mut buf = String::new();
|
||||||
|
buf.push(c);
|
||||||
|
loop {
|
||||||
|
match input.peek().map(|&(_, c)| { c }) {
|
||||||
|
Some(c) if c.is_alphanumeric() => {
|
||||||
|
input.next();
|
||||||
|
buf.push(c);
|
||||||
|
},
|
||||||
|
_ => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
match KEYWORDS.get(buf.as_str()) {
|
||||||
|
Some(kw) => TokenType::Keyword(kw.clone()),
|
||||||
|
None => TokenType::Identifier(Rc::new(buf)),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn handle_operator(c: char, input: &mut CharIter) -> TokenType {
|
fn handle_operator(c: char, input: &mut CharIter) -> TokenType {
|
||||||
|
Loading…
Reference in New Issue
Block a user