From 7cdd95e1a683c89baff762a6c7755951bcd32717 Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Fri, 3 May 2024 15:47:23 +0300 Subject: [PATCH] Optimize async drop glue for some old types --- compiler/rustc_hir/src/lang_items.rs | 1 + compiler/rustc_middle/src/ty/adt.rs | 4 +- compiler/rustc_middle/src/ty/sty.rs | 48 +++--- compiler/rustc_middle/src/ty/util.rs | 139 +++++++++++------- .../src/shim/async_destructor_ctor.rs | 37 ++++- compiler/rustc_span/src/symbol.rs | 1 + .../src/solve/normalizes_to/mod.rs | 2 +- .../src/traits/project.rs | 2 +- compiler/rustc_ty_utils/src/instance.rs | 3 +- compiler/rustc_ty_utils/src/needs_drop.rs | 2 +- library/core/src/future/async_drop.rs | 11 ++ 11 files changed, 161 insertions(+), 89 deletions(-) diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs index c4be67cdd88..4c9d143a63b 100644 --- a/compiler/rustc_hir/src/lang_items.rs +++ b/compiler/rustc_hir/src/lang_items.rs @@ -176,6 +176,7 @@ language_item_table! { AsyncDropSlice, sym::async_drop_slice, async_drop_slice_fn, Target::Fn, GenericRequirement::Exact(1); AsyncDropChain, sym::async_drop_chain, async_drop_chain_fn, Target::Fn, GenericRequirement::Exact(2); AsyncDropNoop, sym::async_drop_noop, async_drop_noop_fn, Target::Fn, GenericRequirement::Exact(0); + AsyncDropDeferredDropInPlace, sym::async_drop_deferred_drop_in_place, async_drop_deferred_drop_in_place_fn, Target::Fn, GenericRequirement::Exact(1); AsyncDropFuse, sym::async_drop_fuse, async_drop_fuse_fn, Target::Fn, GenericRequirement::Exact(1); AsyncDropDefer, sym::async_drop_defer, async_drop_defer_fn, Target::Fn, GenericRequirement::Exact(1); AsyncDropEither, sym::async_drop_either, async_drop_either_fn, Target::Fn, GenericRequirement::Exact(3); diff --git a/compiler/rustc_middle/src/ty/adt.rs b/compiler/rustc_middle/src/ty/adt.rs index 988ce35484d..5f9b870331c 100644 --- a/compiler/rustc_middle/src/ty/adt.rs +++ b/compiler/rustc_middle/src/ty/adt.rs @@ -579,8 +579,8 @@ impl<'tcx> AdtDef<'tcx> { tcx.adt_destructor(self.did()) } - // FIXME(zetanumbers): consider supporting this method in same places where - // `destructor` is referenced + // FIXME: consider combining this method with `AdtDef::destructor` and removing + // this version pub fn async_destructor(self, tcx: TyCtxt<'tcx>) -> Option { tcx.adt_async_destructor(self.did()) } diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index 00f603a8818..5f7385fccc9 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -25,7 +25,7 @@ use std::assert_matches::debug_assert_matches; use std::borrow::Cow; use std::iter; use std::ops::{ControlFlow, Range}; -use ty::util::IntTypeExt; +use ty::util::{AsyncDropGlueMorphology, IntTypeExt}; use rustc_type_ir::TyKind::*; use rustc_type_ir::{self as ir, BoundVar, CollectAndApply, DynKind}; @@ -1951,11 +1951,22 @@ impl<'tcx> Ty<'tcx> { } /// Returns the type of the async destructor of this type. - pub fn async_destructor_ty(self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>) -> Ty<'tcx> { - if self.is_async_destructor_noop(tcx, param_env) || matches!(self.kind(), ty::Error(_)) { - return Ty::async_destructor_combinator(tcx, LangItem::AsyncDropNoop) - .instantiate_identity(); + pub fn async_destructor_ty(self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> { + match self.async_drop_glue_morphology(tcx) { + AsyncDropGlueMorphology::Noop => { + return Ty::async_destructor_combinator(tcx, LangItem::AsyncDropNoop) + .instantiate_identity(); + } + AsyncDropGlueMorphology::DeferredDropInPlace => { + let drop_in_place = + Ty::async_destructor_combinator(tcx, LangItem::AsyncDropDeferredDropInPlace) + .instantiate(tcx, &[self.into()]); + return Ty::async_destructor_combinator(tcx, LangItem::AsyncDropFuse) + .instantiate(tcx, &[drop_in_place.into()]); + } + AsyncDropGlueMorphology::Custom => (), } + match *self.kind() { ty::Param(_) | ty::Alias(..) | ty::Infer(ty::TyVar(_)) => { let assoc_items = tcx @@ -1974,19 +1985,13 @@ impl<'tcx> Ty<'tcx> { .adt_async_destructor_ty( tcx, adt_def.variants().iter().map(|v| v.fields.iter().map(|f| f.ty(tcx, args))), - param_env, ), - ty::Tuple(tys) => self.adt_async_destructor_ty(tcx, iter::once(tys), param_env), - ty::Closure(_, args) => self.adt_async_destructor_ty( - tcx, - iter::once(args.as_closure().upvar_tys()), - param_env, - ), - ty::CoroutineClosure(_, args) => self.adt_async_destructor_ty( - tcx, - iter::once(args.as_coroutine_closure().upvar_tys()), - param_env, - ), + ty::Tuple(tys) => self.adt_async_destructor_ty(tcx, iter::once(tys)), + ty::Closure(_, args) => { + self.adt_async_destructor_ty(tcx, iter::once(args.as_closure().upvar_tys())) + } + ty::CoroutineClosure(_, args) => self + .adt_async_destructor_ty(tcx, iter::once(args.as_coroutine_closure().upvar_tys())), ty::Adt(adt_def, _) => { assert!(adt_def.is_union()); @@ -2008,17 +2013,12 @@ impl<'tcx> Ty<'tcx> { } } - fn adt_async_destructor_ty( - self, - tcx: TyCtxt<'tcx>, - variants: I, - param_env: ParamEnv<'tcx>, - ) -> Ty<'tcx> + fn adt_async_destructor_ty(self, tcx: TyCtxt<'tcx>, variants: I) -> Ty<'tcx> where I: Iterator + ExactSizeIterator, I::Item: IntoIterator>, { - debug_assert!(!self.is_async_destructor_noop(tcx, param_env)); + debug_assert_eq!(self.async_drop_glue_morphology(tcx), AsyncDropGlueMorphology::Custom); let defer = Ty::async_destructor_combinator(tcx, LangItem::AsyncDropDefer); let chain = Ty::async_destructor_combinator(tcx, LangItem::AsyncDropChain); diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs index 905ada3832d..2f2d03e9ec8 100644 --- a/compiler/rustc_middle/src/ty/util.rs +++ b/compiler/rustc_middle/src/ty/util.rs @@ -421,6 +421,25 @@ impl<'tcx> TyCtxt<'tcx> { Some(ty::AsyncDestructor { future, ctor }) } + /// Returns async drop glue morphology for a definition. To get async drop + /// glue morphology for a type see [`Ty::async_drop_glue_morphology`]. + // + // FIXME: consider making this a query + pub fn async_drop_glue_morphology(self, did: DefId) -> AsyncDropGlueMorphology { + let ty: Ty<'tcx> = self.type_of(did).instantiate_identity(); + + // Async drop glue morphology is an internal detail, so reveal_all probably + // should be fine + let param_env = ty::ParamEnv::reveal_all(); + if ty.needs_async_drop(self, param_env) { + AsyncDropGlueMorphology::Custom + } else if ty.needs_drop(self, param_env) { + AsyncDropGlueMorphology::DeferredDropInPlace + } else { + AsyncDropGlueMorphology::Noop + } + } + /// Returns the set of types that are required to be alive in /// order to run the destructor of `def` (see RFCs 769 and /// 1238). @@ -1177,6 +1196,18 @@ impl<'tcx> TypeFolder> for WeakAliasTypeExpander<'tcx> { } } +/// Indicates the form of `AsyncDestruct::Destructor`. Used to simplify async +/// drop glue for types not using async drop. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum AsyncDropGlueMorphology { + /// Async destructor simply does nothing + Noop, + /// Async destructor simply runs `drop_in_place` + DeferredDropInPlace, + /// Async destructor has custom logic + Custom, +} + impl<'tcx> Ty<'tcx> { /// Returns the `Size` for primitive types (bool, uint, int, char, float). pub fn primitive_size(self, tcx: TyCtxt<'tcx>) -> Size { @@ -1342,27 +1373,16 @@ impl<'tcx> Ty<'tcx> { } } - /// Checks whether values of this type `T` implement has noop async destructor. + /// Get morphology of the async drop glue, needed for types which do not + /// use async drop. To get async drop glue morphology for a definition see + /// [`TyCtxt::async_drop_glue_morphology`]. Used for `AsyncDestruct::Destructor` + /// type construction. // - // FIXME: implement optimization to make ADTs, which do not need drop, - // to skip fields or to have noop async destructor, use `needs_(async_)drop` - pub fn is_async_destructor_noop( - self, - tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, - ) -> bool { - // TODO: check on the most generic version of your type - self.is_async_destructor_trivially_noop() - || self.needs_async_drop(tcx, param_env) - || self.needs_drop(tcx, param_env) - } - - /// Fast path helper for testing if a type has noop async destructor. - /// - /// Returning `true` means the type is known to have noop async destructor - /// implementation. Returning `true` means nothing -- could be - /// `Drop`, might not be. - fn is_async_destructor_trivially_noop(self) -> bool { + // FIXME: implement optimization to not instantiate a certain morphology of + // async drop glue too soon to allow per type optimizations, see array case + // for more info. Perhaps then remove this method and use `needs_(async_)drop` + // instead. + pub fn async_drop_glue_morphology(self, tcx: TyCtxt<'tcx>) -> AsyncDropGlueMorphology { match self.kind() { ty::Int(_) | ty::Uint(_) @@ -1374,37 +1394,43 @@ impl<'tcx> Ty<'tcx> { | ty::Ref(..) | ty::RawPtr(..) | ty::FnDef(..) - | ty::FnPtr(_) => true, - ty::Tuple(tys) => tys.is_empty(), - ty::Adt(adt_def, _) => adt_def.is_manually_drop(), - ty::Bool => todo!(), - ty::Char => todo!(), - ty::Int(_) => todo!(), - ty::Uint(_) => todo!(), - ty::Float(_) => todo!(), - ty::Adt(_, _) => todo!(), - ty::Foreign(_) => todo!(), - ty::Str => todo!(), - ty::Array(_, _) => todo!(), - ty::Pat(_, _) => todo!(), - ty::Slice(_) => todo!(), - ty::RawPtr(_, _) => todo!(), - ty::Ref(_, _, _) => todo!(), - ty::FnDef(_, _) => todo!(), - ty::FnPtr(_) => todo!(), - ty::Dynamic(_, _, _) => todo!(), - ty::Closure(_, _) => todo!(), - ty::CoroutineClosure(_, _) => todo!(), - ty::Coroutine(_, _) => todo!(), - ty::CoroutineWitness(_, _) => todo!(), - ty::Never => todo!(), - ty::Tuple(_) => todo!(), - ty::Alias(_, _) => todo!(), - ty::Param(_) => todo!(), - ty::Bound(_, _) => todo!(), - ty::Placeholder(_) => todo!(), - ty::Infer(_) => todo!(), - ty::Error(_) => todo!(), + | ty::FnPtr(_) + | ty::Infer(ty::FreshIntTy(_)) + | ty::Infer(ty::FreshFloatTy(_)) => AsyncDropGlueMorphology::Noop, + + ty::Tuple(tys) if tys.is_empty() => AsyncDropGlueMorphology::Noop, + ty::Adt(adt_def, _) if adt_def.is_manually_drop() => AsyncDropGlueMorphology::Noop, + + // Foreign types can never have destructors. + ty::Foreign(_) => AsyncDropGlueMorphology::Noop, + + // FIXME: implement dynamic types async drops + ty::Error(_) | ty::Dynamic(..) => AsyncDropGlueMorphology::DeferredDropInPlace, + + ty::Tuple(_) | ty::Array(_, _) | ty::Slice(_) => { + // Assume worst-case scenario, because we can instantiate async + // destructors in different orders: + // + // 1. Instantiate [T; N] with T = String and N = 0 + // 2. Instantiate <[String; 0] as AsyncDestruct>::Destructor + // + // And viceversa, thus we cannot rely on String not using async + // drop or array having zero (0) elements + AsyncDropGlueMorphology::Custom + } + ty::Pat(ty, _) => ty.async_drop_glue_morphology(tcx), + + ty::Adt(adt_def, _) => tcx.async_drop_glue_morphology(adt_def.did()), + + ty::Closure(did, _) + | ty::CoroutineClosure(did, _) + | ty::Coroutine(did, _) + | ty::CoroutineWitness(did, _) => tcx.async_drop_glue_morphology(*did), + + ty::Alias(..) | ty::Param(_) | ty::Bound(..) | ty::Placeholder(..) | ty::Infer(_) => { + // No specifics, but would usually mean forwarding async drop glue + AsyncDropGlueMorphology::Custom + } } } @@ -1451,7 +1477,11 @@ impl<'tcx> Ty<'tcx> { /// (Note that this implies that if `ty` has an async destructor attached, /// then `needs_async_drop` will definitely return `true` for `ty`.) /// - /// Note that this method is used to check eligible types in unions. + /// When constructing `AsyncDestruct::Destructor` type, use + /// [`Ty::async_drop_glue_morphology`] instead. + // + // FIXME(zetanumbers): Note that this method is used to check eligible types + // in unions. #[inline] pub fn needs_async_drop(self, tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> bool { // Avoid querying in simple cases. @@ -1647,10 +1677,13 @@ impl<'tcx> ExplicitSelf<'tcx> { } } -// FIXME(zetanumbers): make specifying asyncness explicit /// Returns a list of types such that the given type needs drop if and only if /// *any* of the returned types need drop. Returns `Err(AlwaysRequiresDrop)` if /// this type always needs drop. +// +// FIXME(zetanumbers): consider replacing this with only +// `needs_drop_components_with_async` +#[inline] pub fn needs_drop_components<'tcx>( tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, diff --git a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs index f4481c22fc1..aa9c87d8f80 100644 --- a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs +++ b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs @@ -12,7 +12,7 @@ use rustc_middle::mir::{ Terminator, TerminatorKind, UnwindAction, UnwindTerminateReason, RETURN_PLACE, }; use rustc_middle::ty::adjustment::PointerCoercion; -use rustc_middle::ty::util::Discr; +use rustc_middle::ty::util::{AsyncDropGlueMorphology, Discr}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; use rustc_span::source_map::respan; @@ -116,15 +116,25 @@ impl<'tcx> AsyncDestructorCtorShimBuilder<'tcx> { } fn build(self) -> Body<'tcx> { - let (tcx, def_id, Some(self_ty)) = (self.tcx, self.def_id, self.self_ty) else { + let (tcx, Some(self_ty)) = (self.tcx, self.self_ty) else { return self.build_zst_output(); }; + match self_ty.async_drop_glue_morphology(tcx) { + AsyncDropGlueMorphology::Noop => span_bug!( + self.span, + "async drop glue shim generator encountered type with noop async drop glue morphology" + ), + AsyncDropGlueMorphology::DeferredDropInPlace => { + return self.build_deferred_drop_in_place(); + } + AsyncDropGlueMorphology::Custom => (), + } let surface_drop_kind = || { - let param_env = tcx.param_env_reveal_all_normalized(def_id); - if self_ty.has_surface_async_drop(tcx, param_env) { + let adt_def = self_ty.ty_adt_def()?; + if adt_def.async_destructor(tcx).is_some() { Some(SurfaceDropKind::Async) - } else if self_ty.has_surface_drop(tcx, param_env) { + } else if adt_def.destructor(tcx).is_some() { Some(SurfaceDropKind::Sync) } else { None @@ -267,6 +277,13 @@ impl<'tcx> AsyncDestructorCtorShimBuilder<'tcx> { self.return_() } + fn build_deferred_drop_in_place(mut self) -> Body<'tcx> { + self.put_self(); + let deferred = self.combine_deferred_drop_in_place(); + self.combine_fuse(deferred); + self.return_() + } + fn build_fused_async_surface(mut self) -> Body<'tcx> { self.put_self(); let surface = self.combine_async_surface(); @@ -441,6 +458,14 @@ impl<'tcx> AsyncDestructorCtorShimBuilder<'tcx> { ) } + fn combine_deferred_drop_in_place(&mut self) -> Ty<'tcx> { + self.apply_combinator( + 1, + LangItem::AsyncDropDeferredDropInPlace, + &[self.self_ty.unwrap().into()], + ) + } + fn combine_fuse(&mut self, inner_future_ty: Ty<'tcx>) -> Ty<'tcx> { self.apply_combinator(1, LangItem::AsyncDropFuse, &[inner_future_ty.into()]) } @@ -481,7 +506,7 @@ impl<'tcx> AsyncDestructorCtorShimBuilder<'tcx> { if let Some(ty) = self.self_ty { debug_assert_eq!( output.ty(&self.locals, self.tcx), - ty.async_destructor_ty(self.tcx, self.param_env), + ty.async_destructor_ty(self.tcx), "output async destructor types did not match for type: {ty:?}", ); } diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 90da220b3f5..f771b468690 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -430,6 +430,7 @@ symbols! { async_drop, async_drop_chain, async_drop_defer, + async_drop_deferred_drop_in_place, async_drop_either, async_drop_fuse, async_drop_in_place, diff --git a/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs b/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs index 7fd2a3801cc..9d4c03375fe 100644 --- a/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs +++ b/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs @@ -839,7 +839,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for NormalizesTo<'tcx> { | ty::Str | ty::Slice(_) | ty::Tuple(_) - | ty::Error(_) => self_ty.async_destructor_ty(ecx.interner(), goal.param_env), + | ty::Error(_) => self_ty.async_destructor_ty(ecx.interner()), // We do not call `Ty::async_destructor_ty` on alias, param, or placeholder // types, which return `::AsyncDestructor` diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs index 87c8b1cda50..0f8b003c077 100644 --- a/compiler/rustc_trait_selection/src/traits/project.rs +++ b/compiler/rustc_trait_selection/src/traits/project.rs @@ -1560,7 +1560,7 @@ fn confirm_builtin_candidate<'cx, 'tcx>( let destructor_def_id = tcx.associated_item_def_ids(trait_def_id)[0]; assert_eq!(destructor_def_id, item_def_id); - (self_ty.async_destructor_ty(tcx, obligation.param_env).into(), Vec::new()) + (self_ty.async_destructor_ty(tcx).into(), Vec::new()) } else if lang_items.pointee_trait() == Some(trait_def_id) { let metadata_def_id = tcx.require_lang_item(LangItem::Metadata, None); assert_eq!(metadata_def_id, item_def_id); diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs index 0d089205c1e..e4dcea785d4 100644 --- a/compiler/rustc_ty_utils/src/instance.rs +++ b/compiler/rustc_ty_utils/src/instance.rs @@ -4,6 +4,7 @@ use rustc_infer::infer::TyCtxtInferExt; use rustc_middle::bug; use rustc_middle::query::Providers; use rustc_middle::traits::{BuiltinImplSource, CodegenObligationError}; +use rustc_middle::ty::util::AsyncDropGlueMorphology; use rustc_middle::ty::GenericArgsRef; use rustc_middle::ty::{self, Instance, TyCtxt, TypeVisitableExt}; use rustc_span::sym; @@ -59,7 +60,7 @@ fn resolve_instance<'tcx>( } else if Some(def_id) == tcx.lang_items().async_drop_in_place_fn() { let ty = args.type_at(0); - if !ty.is_async_destructor_noop(tcx, param_env) { + if ty.async_drop_glue_morphology(tcx) != AsyncDropGlueMorphology::Noop { match *ty.kind() { ty::Closure(..) | ty::CoroutineClosure(..) diff --git a/compiler/rustc_ty_utils/src/needs_drop.rs b/compiler/rustc_ty_utils/src/needs_drop.rs index 7fc3543ff66..205b3f2760f 100644 --- a/compiler/rustc_ty_utils/src/needs_drop.rs +++ b/compiler/rustc_ty_utils/src/needs_drop.rs @@ -36,7 +36,7 @@ fn needs_async_drop_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty // it needs async drop. let adt_has_async_dtor = |adt_def: ty::AdtDef<'tcx>| adt_def.async_destructor(tcx).map(|_| DtorType::Significant); - let res = drop_tys_helper(tcx, query.value, query.param_env, adt_has_dtor, false) + let res = drop_tys_helper(tcx, query.value, query.param_env, adt_has_async_dtor, false) .filter(filter_array_elements(tcx, query.param_env)) .next() .is_some(); diff --git a/library/core/src/future/async_drop.rs b/library/core/src/future/async_drop.rs index 0eb8d7bb328..ea7e2f52ba1 100644 --- a/library/core/src/future/async_drop.rs +++ b/library/core/src/future/async_drop.rs @@ -161,6 +161,11 @@ async unsafe fn surface_drop_in_place(ptr: *mut T) { /// wrapped future completes by returning `Poll::Ready(())` on poll. This /// is useful for constructing async destructors to guarantee this /// "fuse" property +// +// FIXME: Consider optimizing combinators to not have to use fuse in majority +// of cases, perhaps by adding `#[(rustc_)idempotent(_future)]` attribute for +// async functions and blocks with the unit return type. However current layout +// optimizations currently encode `None` case into the async block's discriminant. struct Fuse { inner: Option, } @@ -251,6 +256,12 @@ async unsafe fn either, M: IntoFuture, T } } +#[cfg(not(bootstrap))] +#[lang = "async_drop_deferred_drop_in_place"] +async unsafe fn deferred_drop_in_place(to_drop: *mut T) { + unsafe { crate::ptr::drop_in_place(to_drop) } +} + /// Used for noop async destructors. We don't use [`core::future::Ready`] /// because it panics after its second poll, which could be potentially /// bad if that would happen during the cleanup.