mathieu/os-rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add relaxed_delim_match parameter

Browse source
This commit is contained in:
Aaron Hill 2020-09-26 13:12:49 -04:00
parent ea468f4270
commit 820953819c
No known key found for this signature in database
GPG key ID: B4087E510E98B164
1 changed files with 53 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
compiler/rustc_parse/src/lib.rs
View file

@ -7,8 +7,8 @@
#![feature(or_patterns)]
use rustc_ast as ast;
use rustc_ast::token::{self, Nonterminal, Token, TokenKind};
use rustc_ast::tokenstream::{self, TokenStream, TokenTree};
use rustc_ast::token::{self, DelimToken, Nonterminal, Token, TokenKind};
use rustc_ast::tokenstream::{self, Spacing, TokenStream, TokenTree};
use rustc_ast_pretty::pprust;
use rustc_data_structures::sync::Lrc;
use rustc_errors::{Diagnostic, FatalError, Level, PResult};
@ -329,9 +329,8 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
// modifications, including adding/removing typically non-semantic
// tokens such as extra braces and commas, don't happen.
if let Some(tokens) = tokens {
// If the streams match, then the AST hasn't been modified. Return the captured
// `TokenStream`.
if tokenstream_probably_equal_for_proc_macro(&tokens, &reparsed_tokens, sess) {
// Compare with a non-relaxed delim match to start.
if tokenstream_probably_equal_for_proc_macro(&tokens, &reparsed_tokens, sess, false) {
return tokens;
}
@ -340,14 +339,21 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
// token stream to match up with inserted parenthesis in the reparsed stream.
let source_with_parens = pprust::nonterminal_to_string(nt);
let filename_with_parens = FileName::macro_expansion_source_code(&source_with_parens);
let tokens_with_parens = parse_stream_from_source_str(
let reparsed_tokens_with_parens = parse_stream_from_source_str(
filename_with_parens,
source_with_parens,
sess,
Some(span),
);
if tokenstream_probably_equal_for_proc_macro(&tokens, &tokens_with_parens, sess) {
// Compare with a relaxed delim match - we want inserted parenthesis in the
// reparsed stream to match `None`-delimiters in the original stream.
if tokenstream_probably_equal_for_proc_macro(
&tokens,
&reparsed_tokens_with_parens,
sess,
true,
) {
return tokens;
}
@ -355,8 +361,11 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
"cached tokens found, but they're not \"probably equal\", \
going with stringified version"
);
info!("cached tokens: {:?}", tokens);
info!("reparsed tokens: {:?}", reparsed_tokens);
info!("cached tokens: {}", pprust::tts_to_string(&tokens));
info!("reparsed tokens: {}", pprust::tts_to_string(&reparsed_tokens_with_parens));
info!("cached tokens debug: {:?}", tokens);
info!("reparsed tokens debug: {:?}", reparsed_tokens_with_parens);
}
reparsed_tokens
}
@ -370,6 +379,7 @@ pub fn tokenstream_probably_equal_for_proc_macro(
tokens: &TokenStream,
reparsed_tokens: &TokenStream,
sess: &ParseSess,
relaxed_delim_match: bool,
) -> bool {
// When checking for `probably_eq`, we ignore certain tokens that aren't
// preserved in the AST. Because they are not preserved, the pretty
@ -495,7 +505,9 @@ pub fn tokenstream_probably_equal_for_proc_macro(
let tokens = tokens.trees().flat_map(|t| expand_token(t, sess));
let reparsed_tokens = reparsed_tokens.trees().flat_map(|t| expand_token(t, sess));
tokens.eq_by(reparsed_tokens, |t, rt| tokentree_probably_equal_for_proc_macro(&t, &rt, sess))
tokens.eq_by(reparsed_tokens, |t, rt| {
tokentree_probably_equal_for_proc_macro(&t, &rt, sess, relaxed_delim_match)
})
}
// See comments in `Nonterminal::to_tokenstream` for why we care about
@ -507,17 +519,42 @@ pub fn tokentree_probably_equal_for_proc_macro(
token: &TokenTree,
reparsed_token: &TokenTree,
sess: &ParseSess,
relaxed_delim_match: bool,
) -> bool {
match (token, reparsed_token) {
(TokenTree::Token(token), TokenTree::Token(reparsed_token)) => {
token_probably_equal_for_proc_macro(token, reparsed_token)
}
(TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => {
// `NoDelim` delimiters can appear in the captured tokenstream, but not
// in the reparsed tokenstream. Allow them to match with anything, so
// that we check if the two streams are structurally equivalent.
(delim == delim2 || *delim == DelimToken::NoDelim || *delim2 == DelimToken::NoDelim)
&& tokenstream_probably_equal_for_proc_macro(&tts, &tts2, sess)
(
TokenTree::Delimited(_, delim, tokens),
TokenTree::Delimited(_, reparsed_delim, reparsed_tokens),
) if delim == reparsed_delim => tokenstream_probably_equal_for_proc_macro(
tokens,
reparsed_tokens,
sess,
relaxed_delim_match,
),
(TokenTree::Delimited(_, DelimToken::NoDelim, tokens), reparsed_token) => {
if relaxed_delim_match {
if let TokenTree::Delimited(_, DelimToken::Paren, reparsed_tokens) = reparsed_token
{
if tokenstream_probably_equal_for_proc_macro(
tokens,
reparsed_tokens,
sess,
relaxed_delim_match,
) {
return true;
}
}
}
tokens.len() == 1
&& tokentree_probably_equal_for_proc_macro(
&tokens.trees().next().unwrap(),
reparsed_token,
sess,
relaxed_delim_match,
)
}
_ => false,
}