|
|
|
|
@ -17,6 +17,8 @@ pub(crate) mod cursor;
|
|
|
|
|
|
|
|
|
|
use self::TokenKind::*; |
|
|
|
|
use cursor::Cursor; |
|
|
|
|
use lazy_static::lazy_static; |
|
|
|
|
use regex::Regex; |
|
|
|
|
|
|
|
|
|
#[cfg(test)] |
|
|
|
|
mod tests; |
|
|
|
|
@ -178,24 +180,48 @@ pub fn first_token(input: &str, pos: &mut Position) -> Token {
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
pub fn is_whitespace(c: char) -> bool { |
|
|
|
|
match c { |
|
|
|
|
' ' | '\n' | '\r' | '\t' => true, |
|
|
|
|
'\u{00A0}' => { |
|
|
|
|
dbg!("Non-standard unicode character found: '\u{00A0}'"); |
|
|
|
|
true |
|
|
|
|
} |
|
|
|
|
_ => false, |
|
|
|
|
} |
|
|
|
|
// https://doc.rust-lang.org/reference/whitespace.html
|
|
|
|
|
matches!( |
|
|
|
|
c, |
|
|
|
|
// Usual ASCII suspects
|
|
|
|
|
'\u{0009}' // \t
|
|
|
|
|
| '\u{000A}' // \n
|
|
|
|
|
| '\u{000B}' // vertical tab
|
|
|
|
|
| '\u{000C}' // form feed
|
|
|
|
|
| '\u{000D}' // \r
|
|
|
|
|
| '\u{0020}' // space
|
|
|
|
|
|
|
|
|
|
// NEXT LINE from latin1
|
|
|
|
|
| '\u{0085}' |
|
|
|
|
|
|
|
|
|
// Bidi markers
|
|
|
|
|
| '\u{200E}' // LEFT-TO-RIGHT MARK
|
|
|
|
|
| '\u{200F}' // RIGHT-TO-LEFT MARK
|
|
|
|
|
|
|
|
|
|
// Dedicated whitespace characters from Unicode
|
|
|
|
|
| '\u{2028}' // LINE SEPARATOR
|
|
|
|
|
| '\u{2029}' // PARAGRAPH SEPARATOR
|
|
|
|
|
) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// True if `c` is valid as a first character of an identifier.
|
|
|
|
|
/// True if `c` is a valid first character of an identifier
|
|
|
|
|
/// See [Antimony specification](https://antimony-lang.github.io/antimony/developers/specification.html#identifiers) for
|
|
|
|
|
/// a formal definition of valid identifier name.
|
|
|
|
|
pub fn is_id_start(c: char) -> bool { |
|
|
|
|
('a'..='z').contains(&c) || ('A'..='Z').contains(&c) || c == '_' |
|
|
|
|
lazy_static! { |
|
|
|
|
static ref ID_START: Regex = Regex::new(r"[\pL_]").unwrap(); |
|
|
|
|
} |
|
|
|
|
ID_START.is_match(&c.to_string()) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/// True if `c` is valid as a non-first character of an identifier.
|
|
|
|
|
/// True if `c` is a valid continuation of an identifier
|
|
|
|
|
/// See [Antimony specification](https://antimony-lang.github.io/antimony/developers/specification.html#identifiers) for
|
|
|
|
|
/// a formal definition of valid identifier name.
|
|
|
|
|
pub fn is_id_continue(c: char) -> bool { |
|
|
|
|
('a'..='z').contains(&c) || ('A'..='Z').contains(&c) || ('0'..='9').contains(&c) || c == '_' |
|
|
|
|
lazy_static! { |
|
|
|
|
static ref ID_CONTINUE: Regex = Regex::new(r"[\pL\p{Nd}_]").unwrap(); |
|
|
|
|
} |
|
|
|
|
ID_CONTINUE.is_match(&c.to_string()) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
impl Cursor<'_> { |
|
|
|
|
@ -330,7 +356,7 @@ impl Cursor<'_> {
|
|
|
|
|
{ |
|
|
|
|
let mut eaten: usize = 0; |
|
|
|
|
while predicate(self.first()) && !self.is_eof() { |
|
|
|
|
eaten += 1; |
|
|
|
|
eaten += self.first().len_utf8(); |
|
|
|
|
self.bump(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
Garrit Franke
3 years ago
committed by
GitHub