mathieu/granite-rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Auto merge of #114353 - nnethercote:parser-ast-cleanups, r=petrochenkov

Browse source 
Some parser and AST cleanups

Things I found while looking closely at this code.

r? `@petrochenkov`
This commit is contained in:
bors 2023-08-03 04:26:42 +00:00
commit fb31b3c34e
17 changed files with 82 additions and 146 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
compiler/rustc_ast/src/ast.rs
View file

@ -14,7 +14,7 @@
//! - [`Generics`], [`GenericParam`], [`WhereClause`]: Metadata associated with generic parameters.
//! - [`EnumDef`] and [`Variant`]: Enum declaration.
//! - [`MetaItemLit`] and [`LitKind`]: Literal expressions.
//! - [`MacroDef`], [`MacStmtStyle`], [`MacCall`], [`MacDelimiter`]: Macro definition and invocation.
//! - [`MacroDef`], [`MacStmtStyle`], [`MacCall`]: Macro definition and invocation.
//! - [`Attribute`]: Metadata associated with item.
//! - [`UnOp`], [`BinOp`], and [`BinOpKind`]: Unary and binary operators.
@ -1693,7 +1693,7 @@ where
#[derive(Clone, Encodable, Decodable, Debug)]
pub struct DelimArgs {
pub dspan: DelimSpan,
pub delim: MacDelimiter,
pub delim: Delimiter, // Note: `Delimiter::Invisible` never occurs
pub tokens: TokenStream,
}
@ -1701,7 +1701,7 @@ impl DelimArgs {
/// Whether a macro with these arguments needs a semicolon
/// when used as a standalone item or statement.
pub fn need_semicolon(&self) -> bool {
!matches!(self, DelimArgs { delim: MacDelimiter::Brace, .. })
!matches!(self, DelimArgs { delim: Delimiter::Brace, .. })
}
}
@ -1717,32 +1717,6 @@ where
}
}
#[derive(Copy, Clone, PartialEq, Eq, Encodable, Decodable, Debug, HashStable_Generic)]
pub enum MacDelimiter {
Parenthesis,
Bracket,
Brace,
}
impl MacDelimiter {
pub fn to_token(self) -> Delimiter {
match self {
MacDelimiter::Parenthesis => Delimiter::Parenthesis,
MacDelimiter::Bracket => Delimiter::Bracket,
MacDelimiter::Brace => Delimiter::Brace,
}
}
pub fn from_token(delim: Delimiter) -> Option<MacDelimiter> {
match delim {
Delimiter::Parenthesis => Some(MacDelimiter::Parenthesis),
Delimiter::Bracket => Some(MacDelimiter::Bracket),
Delimiter::Brace => Some(MacDelimiter::Brace),
Delimiter::Invisible => None,
}
}
}
/// Represents a macro definition.
#[derive(Clone, Encodable, Decodable, Debug, HashStable_Generic)]
pub struct MacroDef {

14
compiler/rustc_ast/src/attr/mod.rs
View file

@ -2,7 +2,7 @@
use crate::ast::{AttrArgs, AttrArgsEq, AttrId, AttrItem, AttrKind, AttrStyle, AttrVec, Attribute};
use crate::ast::{DelimArgs, Expr, ExprKind, LitKind, MetaItemLit};
use crate::ast::{MacDelimiter, MetaItem, MetaItemKind, NestedMetaItem, NormalAttr};
use crate::ast::{MetaItem, MetaItemKind, NestedMetaItem, NormalAttr};
use crate::ast::{Path, PathSegment, DUMMY_NODE_ID};
use crate::ptr::P;
use crate::token::{self, CommentKind, Delimiter, Token};
@ -196,7 +196,7 @@ impl AttrItem {
fn meta_item_list(&self) -> Option<ThinVec<NestedMetaItem>> {
match &self.args {
AttrArgs::Delimited(args) if args.delim == MacDelimiter::Parenthesis => {
AttrArgs::Delimited(args) if args.delim == Delimiter::Parenthesis => {
MetaItemKind::list_from_tokens(args.tokens.clone())
}
AttrArgs::Delimited(_) | AttrArgs::Eq(..) | AttrArgs::Empty => None,
@ -402,11 +402,9 @@ impl MetaItemKind {
fn from_attr_args(args: &AttrArgs) -> Option<MetaItemKind> {
match args {
AttrArgs::Empty => Some(MetaItemKind::Word),
AttrArgs::Delimited(DelimArgs {
dspan: _,
delim: MacDelimiter::Parenthesis,
tokens,
}) => MetaItemKind::list_from_tokens(tokens.clone()).map(MetaItemKind::List),
AttrArgs::Delimited(DelimArgs { dspan: _, delim: Delimiter::Parenthesis, tokens }) => {
MetaItemKind::list_from_tokens(tokens.clone()).map(MetaItemKind::List)
}
AttrArgs::Delimited(..) => None,
AttrArgs::Eq(_, AttrArgsEq::Ast(expr)) => match expr.kind {
ExprKind::Lit(token_lit) => {
@ -578,7 +576,7 @@ pub fn mk_attr_nested_word(
let path = Path::from_ident(outer_ident);
let attr_args = AttrArgs::Delimited(DelimArgs {
dspan: DelimSpan::from_single(span),
delim: MacDelimiter::Parenthesis,
delim: Delimiter::Parenthesis,
tokens: inner_tokens,
});
mk_attr(g, style, path, attr_args, span)

2
compiler/rustc_ast/src/token.rs
View file

@ -41,8 +41,6 @@ pub enum BinOpToken {
/// Describes how a sequence of token trees is delimited.
/// Cannot use `proc_macro::Delimiter` directly because this
/// structure should implement some additional traits.
/// The `None` variant is also renamed to `Invisible` to be
/// less confusing and better convey the semantics.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
#[derive(Encodable, Decodable, Hash, HashStable_Generic)]
pub enum Delimiter {

6
compiler/rustc_ast_pretty/src/pprust/state.rs
View file

@ -476,7 +476,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
Some(MacHeader::Path(&item.path)),
false,
None,
delim.to_token(),
*delim,
tokens,
true,
span,
@ -640,7 +640,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
Some(MacHeader::Keyword(kw)),
has_bang,
Some(*ident),
macro_def.body.delim.to_token(),
macro_def.body.delim,
&macro_def.body.tokens.clone(),
true,
sp,
@ -1240,7 +1240,7 @@ impl<'a> State<'a> {
Some(MacHeader::Path(&m.path)),
true,
None,
m.args.delim.to_token(),
m.args.delim,
&m.args.tokens.clone(),
true,
m.span(),

5
compiler/rustc_builtin_macros/src/assert.rs
View file

@ -4,8 +4,9 @@ use crate::edition_panic::use_panic_2021;
use crate::errors;
use rustc_ast::ptr::P;
use rustc_ast::token;
use rustc_ast::token::Delimiter;
use rustc_ast::tokenstream::{DelimSpan, TokenStream};
use rustc_ast::{DelimArgs, Expr, ExprKind, MacCall, MacDelimiter, Path, PathSegment, UnOp};
use rustc_ast::{DelimArgs, Expr, ExprKind, MacCall, Path, PathSegment, UnOp};
use rustc_ast_pretty::pprust;
use rustc_errors::PResult;
use rustc_expand::base::{DummyResult, ExtCtxt, MacEager, MacResult};
@ -58,7 +59,7 @@ pub fn expand_assert<'cx>(
path: panic_path(),
args: P(DelimArgs {
dspan: DelimSpan::from_single(call_site_span),
delim: MacDelimiter::Parenthesis,
delim: Delimiter::Parenthesis,
tokens,
}),
})),

7
compiler/rustc_builtin_macros/src/assert/context.rs
View file

@ -1,9 +1,10 @@
use rustc_ast::{
ptr::P,
token,
token::Delimiter,
tokenstream::{DelimSpan, TokenStream, TokenTree},
BinOpKind, BorrowKind, DelimArgs, Expr, ExprKind, ItemKind, MacCall, MacDelimiter, MethodCall,
Mutability, Path, PathSegment, Stmt, StructRest, UnOp, UseTree, UseTreeKind, DUMMY_NODE_ID,
BinOpKind, BorrowKind, DelimArgs, Expr, ExprKind, ItemKind, MacCall, MethodCall, Mutability,
Path, PathSegment, Stmt, StructRest, UnOp, UseTree, UseTreeKind, DUMMY_NODE_ID,
};
use rustc_ast_pretty::pprust;
use rustc_data_structures::fx::FxHashSet;
@ -179,7 +180,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
path: panic_path,
args: P(DelimArgs {
dspan: DelimSpan::from_single(self.span),
delim: MacDelimiter::Parenthesis,
delim: Delimiter::Parenthesis,
tokens: initial.into_iter().chain(captures).collect::<TokenStream>(),
}),
})),

3
compiler/rustc_builtin_macros/src/edition_panic.rs
View file

@ -1,4 +1,5 @@
use rustc_ast::ptr::P;
use rustc_ast::token::Delimiter;
use rustc_ast::tokenstream::{DelimSpan, TokenStream};
use rustc_ast::*;
use rustc_expand::base::*;
@ -60,7 +61,7 @@ fn expand<'cx>(
},
args: P(DelimArgs {
dspan: DelimSpan::from_single(sp),
delim: MacDelimiter::Parenthesis,
delim: Delimiter::Parenthesis,
tokens: tts,
}),
})),

3
compiler/rustc_expand/src/placeholders.rs
View file

@ -2,6 +2,7 @@ use crate::expand::{AstFragment, AstFragmentKind};
use rustc_ast as ast;
use rustc_ast::mut_visit::*;
use rustc_ast::ptr::P;
use rustc_ast::token::Delimiter;
use rustc_data_structures::fx::FxHashMap;
use rustc_span::source_map::DUMMY_SP;
use rustc_span::symbol::Ident;
@ -18,7 +19,7 @@ pub fn placeholder(
path: ast::Path { span: DUMMY_SP, segments: ThinVec::new(), tokens: None },
args: P(ast::DelimArgs {
dspan: ast::tokenstream::DelimSpan::dummy(),
delim: ast::MacDelimiter::Parenthesis,
delim: Delimiter::Parenthesis,
tokens: ast::tokenstream::TokenStream::new(Vec::new()),
}),
})

6
compiler/rustc_parse/src/parser/attr.rs
View file

@ -36,7 +36,7 @@ impl<'a> Parser<'a> {
pub(super) fn parse_outer_attributes(&mut self) -> PResult<'a, AttrWrapper> {
let mut outer_attrs = ast::AttrVec::new();
let mut just_parsed_doc_comment = false;
let start_pos = self.token_cursor.num_next_calls;
let start_pos = self.num_bump_calls;
loop {
let attr = if self.check(&token::Pound) {
let prev_outer_attr_sp = outer_attrs.last().map(|attr| attr.span);
@ -277,7 +277,7 @@ impl<'a> Parser<'a> {
pub(crate) fn parse_inner_attributes(&mut self) -> PResult<'a, ast::AttrVec> {
let mut attrs = ast::AttrVec::new();
loop {
let start_pos: u32 = self.token_cursor.num_next_calls.try_into().unwrap();
let start_pos: u32 = self.num_bump_calls.try_into().unwrap();
// Only try to parse if it is an inner attribute (has `!`).
let attr = if self.check(&token::Pound) && self.look_ahead(1, |t| t == &token::Not) {
Some(self.parse_attribute(InnerAttrPolicy::Permitted)?)
@ -298,7 +298,7 @@ impl<'a> Parser<'a> {
None
};
if let Some(attr) = attr {
let end_pos: u32 = self.token_cursor.num_next_calls.try_into().unwrap();
let end_pos: u32 = self.num_bump_calls.try_into().unwrap();
// If we are currently capturing tokens, mark the location of this inner attribute.
// If capturing ends up creating a `LazyAttrTokenStream`, we will include
// this replace range with it, removing the inner attribute from the final

22
compiler/rustc_parse/src/parser/attr_wrapper.rs
View file

@ -213,6 +213,7 @@ impl<'a> Parser<'a> {
let start_token = (self.token.clone(), self.token_spacing);
let cursor_snapshot = self.token_cursor.clone();
let start_pos = self.num_bump_calls;
let has_outer_attrs = !attrs.attrs.is_empty();
let prev_capturing = std::mem::replace(&mut self.capture_state.capturing, Capturing::Yes);
@ -273,8 +274,7 @@ impl<'a> Parser<'a> {
let replace_ranges_end = self.capture_state.replace_ranges.len();
let cursor_snapshot_next_calls = cursor_snapshot.num_next_calls;
let mut end_pos = self.token_cursor.num_next_calls;
let mut end_pos = self.num_bump_calls;
let mut captured_trailing = false;
@ -301,12 +301,12 @@ impl<'a> Parser<'a> {
// then extend the range of captured tokens to include it, since the parser
// was not actually bumped past it. When the `LazyAttrTokenStream` gets converted
// into an `AttrTokenStream`, we will create the proper token.
if self.token_cursor.break_last_token {
if self.break_last_token {
assert!(!captured_trailing, "Cannot set break_last_token and have trailing token");
end_pos += 1;
}
let num_calls = end_pos - cursor_snapshot_next_calls;
let num_calls = end_pos - start_pos;
// If we have no attributes, then we will never need to
// use any replace ranges.
@ -316,7 +316,7 @@ impl<'a> Parser<'a> {
// Grab any replace ranges that occur *inside* the current AST node.
// We will perform the actual replacement when we convert the `LazyAttrTokenStream`
// to an `AttrTokenStream`.
let start_calls: u32 = cursor_snapshot_next_calls.try_into().unwrap();
let start_calls: u32 = start_pos.try_into().unwrap();
self.capture_state.replace_ranges[replace_ranges_start..replace_ranges_end]
.iter()
.cloned()
@ -331,7 +331,7 @@ impl<'a> Parser<'a> {
start_token,
num_calls,
cursor_snapshot,
break_last_token: self.token_cursor.break_last_token,
break_last_token: self.break_last_token,
replace_ranges,
});
@ -359,14 +359,10 @@ impl<'a> Parser<'a> {
// with a `FlatToken::AttrTarget`. If this AST node is inside an item
// that has `#[derive]`, then this will allow us to cfg-expand this
// AST node.
let start_pos =
if has_outer_attrs { attrs.start_pos } else { cursor_snapshot_next_calls };
let start_pos = if has_outer_attrs { attrs.start_pos } else { start_pos };
let new_tokens = vec![(FlatToken::AttrTarget(attr_data), Spacing::Alone)];
assert!(
!self.token_cursor.break_last_token,
"Should not have unglued last token with cfg attr"
);
assert!(!self.break_last_token, "Should not have unglued last token with cfg attr");
let range: Range<u32> = (start_pos.try_into().unwrap())..(end_pos.try_into().unwrap());
self.capture_state.replace_ranges.push((range, new_tokens));
self.capture_state.replace_ranges.extend(inner_attr_replace_ranges);
@ -464,6 +460,6 @@ mod size_asserts {
use rustc_data_structures::static_assert_size;
// tidy-alphabetical-start
static_assert_size!(AttrWrapper, 16);
static_assert_size!(LazyAttrTokenStreamImpl, 120);
static_assert_size!(LazyAttrTokenStreamImpl, 104);
// tidy-alphabetical-end
}

4
compiler/rustc_parse/src/parser/expr.rs
View file

@ -1167,7 +1167,7 @@ impl<'a> Parser<'a> {
DestructuredFloat::TrailingDot(sym, sym_span, dot_span) => {
assert!(suffix.is_none());
// Analogous to `Self::break_and_eat`
self.token_cursor.break_last_token = true;
self.break_last_token = true;
// This might work, in cases like `1. 2`, and might not,
// in cases like `offset_of!(Ty, 1.)`. It depends on what comes
// after the float-like token, and therefore we have to make
@ -2599,7 +2599,7 @@ impl<'a> Parser<'a> {
// Recover from missing expression in `for` loop
if matches!(expr.kind, ExprKind::Block(..))
&& !matches!(self.token.kind, token::OpenDelim(token::Delimiter::Brace))
&& !matches!(self.token.kind, token::OpenDelim(Delimiter::Brace))
&& self.may_recover()
{
self.sess

4
compiler/rustc_parse/src/parser/item.rs
View file

@ -9,12 +9,12 @@ use rustc_ast::ptr::P;
use rustc_ast::token::{self, Delimiter, TokenKind};
use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
use rustc_ast::util::case::Case;
use rustc_ast::MacCall;
use rustc_ast::{self as ast, AttrVec, Attribute, DUMMY_NODE_ID};
use rustc_ast::{Async, Const, Defaultness, IsAuto, Mutability, Unsafe, UseTree, UseTreeKind};
use rustc_ast::{BindingAnnotation, Block, FnDecl, FnSig, Param, SelfKind};
use rustc_ast::{EnumDef, FieldDef, Generics, TraitRef, Ty, TyKind, Variant, VariantData};
use rustc_ast::{FnHeader, ForeignItem, Path, PathSegment, Visibility, VisibilityKind};
use rustc_ast::{MacCall, MacDelimiter};
use rustc_ast_pretty::pprust;
use rustc_errors::{
struct_span_err, Applicability, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic, PResult,
@ -1968,7 +1968,7 @@ impl<'a> Parser<'a> {
let arrow = TokenTree::token_alone(token::FatArrow, pspan.between(bspan)); // `=>`
let tokens = TokenStream::new(vec![params, arrow, body]);
let dspan = DelimSpan::from_pair(pspan.shrink_to_lo(), bspan.shrink_to_hi());
P(DelimArgs { dspan, delim: MacDelimiter::Brace, tokens })
P(DelimArgs { dspan, delim: Delimiter::Brace, tokens })
} else {
return self.unexpected();
};

84
compiler/rustc_parse/src/parser/mod.rs
View file

@ -25,7 +25,7 @@ use rustc_ast::util::case::Case;
use rustc_ast::AttrId;
use rustc_ast::DUMMY_NODE_ID;
use rustc_ast::{self as ast, AnonConst, Const, DelimArgs, Extern};
use rustc_ast::{Async, AttrArgs, AttrArgsEq, Expr, ExprKind, MacDelimiter, Mutability, StrLit};
use rustc_ast::{Async, AttrArgs, AttrArgsEq, Expr, ExprKind, Mutability, StrLit};
use rustc_ast::{HasAttrs, HasTokens, Unsafe, Visibility, VisibilityKind};
use rustc_ast_pretty::pprust;
use rustc_data_structures::fx::FxHashMap;
@ -135,9 +135,24 @@ pub struct Parser<'a> {
pub capture_cfg: bool,
restrictions: Restrictions,
expected_tokens: Vec<TokenType>,
// Important: This must only be advanced from `bump` to ensure that
// `token_cursor.num_next_calls` is updated properly.
token_cursor: TokenCursor,
// The number of calls to `bump`, i.e. the position in the token stream.
num_bump_calls: usize,
// During parsing we may sometimes need to 'unglue' a glued token into two
// component tokens (e.g. '>>' into '>' and '>), so the parser can consume
// them one at a time. This process bypasses the normal capturing mechanism
// (e.g. `num_bump_calls` will not be incremented), since the 'unglued'
// tokens due not exist in the original `TokenStream`.
//
// If we end up consuming both unglued tokens, this is not an issue. We'll
// end up capturing the single 'glued' token.
//
// However, sometimes we may want to capture just the first 'unglued'
// token. For example, capturing the `Vec<u8>` in `Option<Vec<u8>>`
// requires us to unglue the trailing `>>` token. The `break_last_token`
// field is used to track this token. It gets appended to the captured
// stream when we evaluate a `LazyAttrTokenStream`.
break_last_token: bool,
/// This field is used to keep track of how many left angle brackets we have seen. This is
/// required in order to detect extra leading left angle brackets (`<` characters) and error
/// appropriately.
@ -161,7 +176,7 @@ pub struct Parser<'a> {
// This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure
// it doesn't unintentionally get bigger.
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
rustc_data_structures::static_assert_size!(Parser<'_>, 272);
rustc_data_structures::static_assert_size!(Parser<'_>, 264);
/// Stores span information about a closure.
#[derive(Clone)]
@ -223,32 +238,6 @@ struct TokenCursor {
// tokens are in `stack[n-1]`. `stack[0]` (when present) has no delimiters
// because it's the outermost token stream which never has delimiters.
stack: Vec<(TokenTreeCursor, Delimiter, DelimSpan)>,
// Counts the number of calls to `{,inlined_}next`.
num_next_calls: usize,
// During parsing, we may sometimes need to 'unglue' a
// glued token into two component tokens
// (e.g. '>>' into '>' and '>), so that the parser
// can consume them one at a time. This process
// bypasses the normal capturing mechanism
// (e.g. `num_next_calls` will not be incremented),
// since the 'unglued' tokens due not exist in
// the original `TokenStream`.
//
// If we end up consuming both unglued tokens,
// then this is not an issue - we'll end up
// capturing the single 'glued' token.
//
// However, in certain circumstances, we may
// want to capture just the first 'unglued' token.
// For example, capturing the `Vec<u8>`
// in `Option<Vec<u8>>` requires us to unglue
// the trailing `>>` token. The `break_last_token`
// field is used to track this token - it gets
// appended to the captured stream when
// we evaluate a `LazyAttrTokenStream`.
break_last_token: bool,
}
impl TokenCursor {
@ -399,12 +388,9 @@ impl<'a> Parser<'a> {
capture_cfg: false,
restrictions: Restrictions::empty(),
expected_tokens: Vec::new(),
token_cursor: TokenCursor {
tree_cursor: stream.into_trees(),
stack: Vec::new(),
num_next_calls: 0,
break_last_token: false,
},
token_cursor: TokenCursor { tree_cursor: stream.into_trees(), stack: Vec::new() },
num_bump_calls: 0,
break_last_token: false,
unmatched_angle_bracket_count: 0,
max_angle_bracket_count: 0,
last_unexpected_token_span: None,
@ -707,7 +693,7 @@ impl<'a> Parser<'a> {
// If we consume any additional tokens, then this token
// is not needed (we'll capture the entire 'glued' token),
// and `bump` will set this field to `None`
self.token_cursor.break_last_token = true;
self.break_last_token = true;
// Use the spacing of the glued token as the spacing
// of the unglued second token.
self.bump_with((Token::new(second, second_span), self.token_spacing));
@ -1049,11 +1035,11 @@ impl<'a> Parser<'a> {
// Note: destructuring here would give nicer code, but it was found in #96210 to be slower
// than `.0`/`.1` access.
let mut next = self.token_cursor.inlined_next();
self.token_cursor.num_next_calls += 1;
self.num_bump_calls += 1;
// We've retrieved an token from the underlying
// cursor, so we no longer need to worry about
// an unglued token. See `break_and_eat` for more details
self.token_cursor.break_last_token = false;
self.break_last_token = false;
if next.0.span.is_dummy() {
// Tweak the location for better diagnostics, but keep syntactic context intact.
let fallback_span = self.token.span;
@ -1230,12 +1216,10 @@ impl<'a> Parser<'a> {
|| self.check(&token::OpenDelim(Delimiter::Brace));
delimited.then(|| {
// We've confirmed above that there is a delimiter so unwrapping is OK.
let TokenTree::Delimited(dspan, delim, tokens) = self.parse_token_tree() else {
unreachable!()
};
DelimArgs { dspan, delim: MacDelimiter::from_token(delim).unwrap(), tokens }
DelimArgs { dspan, delim, tokens }
})
}
@ -1251,12 +1235,11 @@ impl<'a> Parser<'a> {
}
/// Parses a single token tree from the input.
pub(crate) fn parse_token_tree(&mut self) -> TokenTree {
pub fn parse_token_tree(&mut self) -> TokenTree {
match self.token.kind {
token::OpenDelim(..) => {
// Grab the tokens within the delimiters.
let tree_cursor = &self.token_cursor.tree_cursor;
let stream = tree_cursor.stream.clone();
let stream = self.token_cursor.tree_cursor.stream.clone();
let (_, delim, span) = *self.token_cursor.stack.last().unwrap();
// Advance the token cursor through the entire delimited
@ -1287,15 +1270,6 @@ impl<'a> Parser<'a> {
}
}
/// Parses a stream of tokens into a list of `TokenTree`s, up to EOF.
pub fn parse_all_token_trees(&mut self) -> PResult<'a, Vec<TokenTree>> {
let mut tts = Vec::new();
while self.token != token::Eof {
tts.push(self.parse_token_tree());
}
Ok(tts)
}
pub fn parse_tokens(&mut self) -> TokenStream {
let mut result = Vec::new();
loop {
@ -1455,7 +1429,7 @@ impl<'a> Parser<'a> {
}
pub fn approx_token_stream_pos(&self) -> usize {
self.token_cursor.num_next_calls
self.num_bump_calls
}
}

3
compiler/rustc_parse/src/parser/stmt.rs
View file

@ -193,10 +193,9 @@ impl<'a> Parser<'a> {
/// At this point, the `!` token after the path has already been eaten.
fn parse_stmt_mac(&mut self, lo: Span, attrs: AttrVec, path: ast::Path) -> PResult<'a, Stmt> {
let args = self.parse_delim_args()?;
let delim = args.delim.to_token();
let hi = self.prev_token.span;
let style = match delim {
let style = match args.delim {
Delimiter::Brace => MacStmtStyle::Braces,
_ => MacStmtStyle::NoBraces,
};

11
compiler/rustc_parse/src/validate_attr.rs
View file

@ -2,9 +2,10 @@
use crate::{errors, parse_in};
use rustc_ast::token::Delimiter;
use rustc_ast::tokenstream::DelimSpan;
use rustc_ast::MetaItemKind;
use rustc_ast::{self as ast, AttrArgs, AttrArgsEq, Attribute, DelimArgs, MacDelimiter, MetaItem};
use rustc_ast::{self as ast, AttrArgs, AttrArgsEq, Attribute, DelimArgs, MetaItem};
use rustc_ast_pretty::pprust;
use rustc_errors::{Applicability, FatalError, PResult};
use rustc_feature::{AttributeTemplate, BuiltinAttribute, BUILTIN_ATTRIBUTE_MAP};
@ -84,8 +85,8 @@ pub fn parse_meta<'a>(sess: &'a ParseSess, attr: &Attribute) -> PResult<'a, Meta
})
}
pub fn check_meta_bad_delim(sess: &ParseSess, span: DelimSpan, delim: MacDelimiter) {
if let ast::MacDelimiter::Parenthesis = delim {
pub fn check_meta_bad_delim(sess: &ParseSess, span: DelimSpan, delim: Delimiter) {
if let Delimiter::Parenthesis = delim {
return;
}
sess.emit_err(errors::MetaBadDelim {
@ -94,8 +95,8 @@ pub fn check_meta_bad_delim(sess: &ParseSess, span: DelimSpan, delim: MacDelimit
});
}
pub fn check_cfg_attr_bad_delim(sess: &ParseSess, span: DelimSpan, delim: MacDelimiter) {
if let ast::MacDelimiter::Parenthesis = delim {
pub fn check_cfg_attr_bad_delim(sess: &ParseSess, span: DelimSpan, delim: Delimiter) {
if let Delimiter::Parenthesis = delim {
return;
}
sess.emit_err(errors::CfgAttrBadDelim {

14
src/librustdoc/clean/render_macro_matchers.rs
View file

@ -76,17 +76,13 @@ fn snippet_equal_to_token(tcx: TyCtxt<'_>, matcher: &TokenTree) -> Option<String
};
// Reparse a single token tree.
let mut reparsed_trees = match parser.parse_all_token_trees() {
Ok(reparsed_trees) => reparsed_trees,
Err(diagnostic) => {
diagnostic.cancel();
return None;
}
};
if reparsed_trees.len() != 1 {
if parser.token == token::Eof {
return None;
}
let reparsed_tree = parser.parse_token_tree();
if parser.token != token::Eof {
return None;
}
let reparsed_tree = reparsed_trees.pop().unwrap();
// Compare against the original tree.
if reparsed_tree.eq_unspanned(matcher) { Some(snippet) } else { None }

8
src/tools/rustfmt/src/expr.rs
View file

@ -1382,12 +1382,8 @@ pub(crate) fn can_be_overflowed_expr(
|| (context.use_block_indent() && args_len == 1)
}
ast::ExprKind::MacCall(ref mac) => {
match (
rustc_ast::ast::MacDelimiter::from_token(mac.args.delim.to_token()),
context.config.overflow_delimited_expr(),
) {
(Some(ast::MacDelimiter::Bracket), true)
| (Some(ast::MacDelimiter::Brace), true) => true,
match (mac.args.delim, context.config.overflow_delimited_expr()) {
(Delimiter::Bracket, true) | (Delimiter::Brace, true) => true,
_ => context.use_block_indent() && args_len == 1,
}
}