mathieu/granite-rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

simplify

Browse source
This commit is contained in:
Aleksey Kladov 2021-12-12 19:31:32 +03:00
parent 57e6ef0bfb
commit 6e4bb57014
2 changed files with 23 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
crates/parser/src/parser.rs
View file

@ -56,7 +56,7 @@ impl<'t> Parser<'t> {
assert!(PARSER_STEP_LIMIT.check(steps as usize).is_ok(), "the parser seems stuck");
self.steps.set(steps + 1);
self.tokens.get(self.pos + n).kind
self.tokens.kind(self.pos + n)
}
/// Checks if the current token is `kind`.
@ -92,7 +92,7 @@ impl<'t> Parser<'t> {
T![<<=] => self.at_composite3(n, T![<], T![<], T![=]),
T![>>=] => self.at_composite3(n, T![>], T![>], T![=]),
_ => self.tokens.get(self.pos + n).kind == kind,
_ => self.tokens.kind(self.pos + n) == kind,
}
}
@ -131,25 +131,17 @@ impl<'t> Parser<'t> {
}
fn at_composite2(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind) -> bool {
let t1 = self.tokens.get(self.pos + n);
if t1.kind != k1 || !t1.is_jointed_to_next {
return false;
}
let t2 = self.tokens.get(self.pos + n + 1);
t2.kind == k2
self.tokens.kind(self.pos + n) == k1
&& self.tokens.kind(self.pos + n + 1) == k2
&& self.tokens.is_joint(self.pos + n)
}
fn at_composite3(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind, k3: SyntaxKind) -> bool {
let t1 = self.tokens.get(self.pos + n);
if t1.kind != k1 || !t1.is_jointed_to_next {
return false;
}
let t2 = self.tokens.get(self.pos + n + 1);
if t2.kind != k2 || !t2.is_jointed_to_next {
return false;
}
let t3 = self.tokens.get(self.pos + n + 2);
t3.kind == k3
self.tokens.kind(self.pos + n) == k1
&& self.tokens.kind(self.pos + n + 1) == k2
&& self.tokens.kind(self.pos + n + 2) == k3
&& self.tokens.is_joint(self.pos + n)
&& self.tokens.is_joint(self.pos + n + 1)
}
/// Checks if the current token is in `kinds`.
@ -159,7 +151,7 @@ impl<'t> Parser<'t> {
/// Checks if the current token is contextual keyword with text `t`.
pub(crate) fn at_contextual_kw(&self, kw: SyntaxKind) -> bool {
self.tokens.get(self.pos).contextual_kw == kw
self.tokens.contextual_kind(self.pos) == kw
}
/// Starts a new node in the syntax tree. All nodes and tokens

40
crates/parser/src/tokens.rs
View file

@ -8,17 +8,6 @@ use crate::SyntaxKind;
#[allow(non_camel_case_types)]
type bits = u64;
/// `Token` abstracts the cursor of `TokenSource` operates on.
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub(crate) struct Token {
/// What is the current token?
pub(crate) kind: SyntaxKind,
/// Is the current token joined to the next one (`> >` vs `>>`).
pub(crate) is_jointed_to_next: bool,
pub(crate) contextual_kw: SyntaxKind,
}
/// Main input to the parser.
///
/// A sequence of tokens represented internally as a struct of arrays.
@ -71,10 +60,6 @@ impl Tokens {
let (idx, b_idx) = self.bit_index(n);
self.joint[idx] |= 1 << b_idx;
}
fn get_joint(&self, n: usize) -> bool {
let (idx, b_idx) = self.bit_index(n);
self.joint[idx] & 1 << b_idx != 0
}
fn bit_index(&self, n: usize) -> (usize, usize) {
let idx = n / (bits::BITS as usize);
let b_idx = n % (bits::BITS as usize);
@ -84,19 +69,18 @@ impl Tokens {
fn len(&self) -> usize {
self.kind.len()
}
pub(crate) fn get(&self, idx: usize) -> Token {
if idx < self.len() {
let kind = self.kind[idx];
let is_jointed_to_next = self.get_joint(idx);
let contextual_kw = self.contextual_kw[idx];
Token { kind, is_jointed_to_next, contextual_kw }
} else {
self.eof()
}
}
}
#[cold]
fn eof(&self) -> Token {
Token { kind: SyntaxKind::EOF, is_jointed_to_next: false, contextual_kw: SyntaxKind::EOF }
/// pub(crate) impl used by the parser.
impl Tokens {
pub(crate) fn kind(&self, idx: usize) -> SyntaxKind {
self.kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
}
pub(crate) fn contextual_kind(&self, idx: usize) -> SyntaxKind {
self.contextual_kw.get(idx).copied().unwrap_or(SyntaxKind::EOF)
}
pub(crate) fn is_joint(&self, n: usize) -> bool {
let (idx, b_idx) = self.bit_index(n);
self.joint[idx] & 1 << b_idx != 0
}
}