mii/catalysa
1
0
Fork 0
mirror of https://github.com/mii443/catalysa.git synced 2025-08-22 15:05:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add IDENT and NUMBER tokenizer

Browse Source
This commit is contained in:
mii
2021-12-19 22:26:52 +09:00
parent 1882c6649c
commit cb0aa27b4a
1 changed files with 52 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
catalysa-parser/src/tokenizer.rs
View File

@ -1,6 +1,6 @@
use nom::{IResult, branch::alt, bytes::complete::tag, character::complete::digit1};
use nom::{IResult, branch::alt, bytes::complete::{tag, take_while, take_while1}, character::{complete::digit1, is_alphabetic, is_alphanumeric}};
use crate::token::Token;
use crate::token::*;
pub struct Tokenizer {
pub code: str
@ -11,9 +11,58 @@ impl Tokenizer {
let mut tokens: Vec<Token> = vec![];
let mut code = &self.code;
while code != "" {
match Tokenizer::reserved(code) {
Ok((input, reserved)) => {
code = input;
tokens.push(Token {
kind: TokenKind::RESERVED,
num: 0,
str: reserved.to_string()
});
continue;
},
Err(_) => {},
}
match Tokenizer::number(code) {
Ok((input, number)) => {
code = input;
tokens.push(Token {
kind: TokenKind::NUMBER,
num: number,
str: String::default()
});
continue;
},
Err(_) => {},
}
match Tokenizer::ident(code) {
Ok((input, ident)) => {
code = input;
tokens.push(Token {
kind: TokenKind::IDENT,
num: 0,
str: ident.to_string()
});
continue;
},
Err(_) => {},
}
break;
}
vec![]
tokens
}
fn is_ident(ch: char) -> bool {
is_alphabetic(ch as u8)
}
fn ident(input: &str) -> IResult<&str, &str> {
take_while1(Tokenizer::is_ident)(input)
}
fn number(input: &str) -> IResult<&str, usize> {