mathieu/granite-rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

tighten up invariants

Browse source
This commit is contained in:
Aleksey Kladov 2021-12-12 19:22:37 +03:00
parent 18d4737fb9
commit 57e6ef0bfb
3 changed files with 17 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
crates/mbe/src/to_parser_tokens.rs
View file

@ -62,7 +62,9 @@ pub(crate) fn to_parser_tokens(buffer: &TokenBuffer) -> parser::Tokens {
let kind = SyntaxKind::from_char(punct.char)
.unwrap_or_else(|| panic!("{:#?} is not a valid punct", punct));
res.push(kind);
res.was_joint(punct.spacing == tt::Spacing::Joint);
if punct.spacing == tt::Spacing::Joint {
res.was_joint();
}
}
}
cursor.bump()

7
crates/parser/src/tokens.rs
View file

@ -51,11 +51,8 @@ impl Tokens {
/// tokens.push(curr_joint)
/// ```
#[inline]
pub fn was_joint(&mut self, yes: bool) {
let idx = self.len();
if yes && idx > 0 {
self.set_joint(idx - 1);
}
pub fn was_joint(&mut self) {
self.set_joint(self.len() - 1);
}
#[inline]
pub fn push_ident(&mut self, contextual_kw: SyntaxKind) {

20
crates/syntax/src/parsing.rs
View file

@ -58,18 +58,22 @@ pub(crate) fn parse_text_as<T: AstNode>(
pub(crate) fn to_parser_tokens(text: &str, lexer_tokens: &[lexer::Token]) -> ::parser::Tokens {
let mut off = 0;
let mut res = parser::Tokens::default();
let mut was_joint = true;
let mut was_joint = false;
for t in lexer_tokens {
if t.kind.is_trivia() {
was_joint = false;
} else if t.kind == SyntaxKind::IDENT {
let token_text = &text[off..][..usize::from(t.len)];
let contextual_kw =
SyntaxKind::from_contextual_keyword(token_text).unwrap_or(SyntaxKind::IDENT);
res.push_ident(contextual_kw);
} else {
res.was_joint(was_joint);
res.push(t.kind);
if t.kind == SyntaxKind::IDENT {
let token_text = &text[off..][..usize::from(t.len)];
let contextual_kw =
SyntaxKind::from_contextual_keyword(token_text).unwrap_or(SyntaxKind::IDENT);
res.push_ident(contextual_kw);
} else {
if was_joint {
res.was_joint();
}
res.push(t.kind);
}
was_joint = true;
}
off += usize::from(t.len);