Merge remote-tracking branch 'upstream/master' into HEAD

Cargo.lock

@@ -469,9 +469,9 @@ version = "0.1.0"
 
 [[package]]
 name = "cc"
-version = "1.0.79"
+version = "1.0.90"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
+checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5"
 
 [[package]]
 name = "cfg-if"
@@ -3970,6 +3970,7 @@ version = "0.0.0"
 dependencies = [
  "itertools 0.11.0",
  "rustc_ast",
+ "rustc_ast_ir",
  "rustc_attr",
  "rustc_data_structures",
  "rustc_errors",
@@ -4038,6 +4039,7 @@ dependencies = [
 name = "rustc_infer"
 version = "0.0.0"
 dependencies = [
+ "rustc_ast_ir",
  "rustc_data_structures",
  "rustc_errors",
  "rustc_fluent_macro",
@@ -4570,6 +4572,7 @@ name = "rustc_span"
 version = "0.0.0"
 dependencies = [
  "indexmap",
+ "itoa",
  "md-5",
  "rustc_arena",
  "rustc_data_structures",
@@ -4632,6 +4635,7 @@ dependencies = [
  "bitflags 2.4.2",
  "itertools 0.11.0",
  "rustc_ast",
+ "rustc_ast_ir",
  "rustc_attr",
  "rustc_data_structures",
  "rustc_errors",
@@ -4670,6 +4674,7 @@ name = "rustc_transmute"
 version = "0.0.0"
 dependencies = [
  "itertools 0.11.0",
+ "rustc_ast_ir",
  "rustc_data_structures",
  "rustc_hir",
  "rustc_infer",
@@ -4685,6 +4690,7 @@ name = "rustc_ty_utils"
 version = "0.0.0"
 dependencies = [
  "itertools 0.11.0",
+ "rustc_ast_ir",
  "rustc_data_structures",
  "rustc_errors",
  "rustc_fluent_macro",

compiler/rustc_abi/src/layout.rs

@@ -1,6 +1,9 @@
+use std::borrow::{Borrow, Cow};
+use std::cmp;
 use std::fmt::{self, Write};
+use std::iter;
+use std::ops::Bound;
 use std::ops::Deref;
-use std::{borrow::Borrow, cmp, iter, ops::Bound};
 
 use rustc_index::Idx;
 use tracing::debug;
@@ -32,7 +35,7 @@ where
 pub trait LayoutCalculator {
     type TargetDataLayoutRef: Borrow<TargetDataLayout>;
 
-    fn delayed_bug(&self, txt: String);
+    fn delayed_bug(&self, txt: impl Into<Cow<'static, str>>);
     fn current_data_layout(&self) -> Self::TargetDataLayoutRef;
 
     fn scalar_pair<FieldIdx: Idx, VariantIdx: Idx>(

compiler/rustc_abi/src/lib.rs

@@ -1612,8 +1612,9 @@ pub enum PointerKind {
     SharedRef { frozen: bool },
     /// Mutable reference. `unpin` indicates the absence of any pinned data.
     MutableRef { unpin: bool },
-    /// Box. `unpin` indicates the absence of any pinned data.
-    Box { unpin: bool },
+    /// Box. `unpin` indicates the absence of any pinned data. `global` indicates whether this box
+    /// uses the global allocator or a custom one.
+    Box { unpin: bool, global: bool },
 }
 
 /// Note that this information is advisory only, and backends are free to ignore it.
@@ -1622,6 +1623,8 @@ pub enum PointerKind {
 pub struct PointeeInfo {
     pub size: Size,
     pub align: Align,
+    /// If this is `None`, then this is a raw pointer, so size and alignment are not guaranteed to
+    /// be reliable.
     pub safe: Option<PointerKind>,
 }
 

compiler/rustc_ast/src/ast.rs

@@ -2302,6 +2302,9 @@ pub enum InlineAsmOperand {
     Sym {
         sym: InlineAsmSym,
     },
+    Label {
+        block: P<Block>,
+    },
 }
 
 impl InlineAsmOperand {
@@ -2311,7 +2314,7 @@ impl InlineAsmOperand {
             | Self::Out { reg, .. }
             | Self::InOut { reg, .. }
             | Self::SplitInOut { reg, .. } => Some(reg),
-            Self::Const { .. } | Self::Sym { .. } => None,
+            Self::Const { .. } | Self::Sym { .. } | Self::Label { .. } => None,
         }
     }
 }

compiler/rustc_ast/src/mut_visit.rs

@@ -1331,6 +1331,7 @@ pub fn noop_visit_inline_asm<T: MutVisitor>(asm: &mut InlineAsm, vis: &mut T) {
             }
             InlineAsmOperand::Const { anon_const } => vis.visit_anon_const(anon_const),
             InlineAsmOperand::Sym { sym } => vis.visit_inline_asm_sym(sym),
+            InlineAsmOperand::Label { block } => vis.visit_block(block),
         }
     }
 }
@@ -1604,7 +1605,10 @@ pub fn noop_visit_capture_by<T: MutVisitor>(capture_by: &mut CaptureBy, vis: &mu
     }
 }
 
-/// Some value for the AST node that is valid but possibly meaningless.
+/// Some value for the AST node that is valid but possibly meaningless. Similar
+/// to `Default` but not intended for wide use. The value will never be used
+/// meaningfully, it exists just to support unwinding in `visit_clobber` in the
+/// case where its closure panics.
 pub trait DummyAstNode {
     fn dummy() -> Self;
 }
@@ -1679,19 +1683,6 @@ impl DummyAstNode for Stmt {
     }
 }
 
-impl DummyAstNode for Block {
-    fn dummy() -> Self {
-        Block {
-            stmts: Default::default(),
-            id: DUMMY_NODE_ID,
-            rules: BlockCheckMode::Default,
-            span: Default::default(),
-            tokens: Default::default(),
-            could_be_bare_literal: Default::default(),
-        }
-    }
-}
-
 impl DummyAstNode for Crate {
     fn dummy() -> Self {
         Crate {

compiler/rustc_ast/src/visit.rs

@@ -15,11 +15,12 @@
 
 use crate::ast::*;
 
-use core::ops::ControlFlow;
-
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
 
+pub use rustc_ast_ir::visit::VisitorResult;
+pub use rustc_ast_ir::{try_visit, visit_opt, walk_list, walk_visitable_list};
+
 #[derive(Copy, Clone, Debug, PartialEq)]
 pub enum AssocCtxt {
     Trait,
@@ -101,51 +102,6 @@ pub enum LifetimeCtxt {
     GenericArg,
 }
 
-/// Similar to the `Try` trait, but also implemented for `()`.
-pub trait VisitorResult {
-    type Residual;
-    fn output() -> Self;
-    fn from_residual(residual: Self::Residual) -> Self;
-    fn branch(self) -> ControlFlow<Self::Residual>;
-}
-
-impl VisitorResult for () {
-    type Residual = !;
-
-    fn output() -> Self {}
-    fn from_residual(_: !) -> Self {}
-    fn branch(self) -> ControlFlow<!> {
-        ControlFlow::Continue(())
-    }
-}
-
-impl<T> VisitorResult for ControlFlow<T> {
-    type Residual = T;
-
-    fn output() -> Self {
-        ControlFlow::Continue(())
-    }
-    fn from_residual(residual: Self::Residual) -> Self {
-        ControlFlow::Break(residual)
-    }
-    fn branch(self) -> ControlFlow<T> {
-        self
-    }
-}
-
-#[macro_export]
-macro_rules! try_visit {
-    ($e:expr) => {
-        match $crate::visit::VisitorResult::branch($e) {
-            core::ops::ControlFlow::Continue(()) => (),
-            #[allow(unreachable_code)]
-            core::ops::ControlFlow::Break(r) => {
-                return $crate::visit::VisitorResult::from_residual(r);
-            }
-        }
-    };
-}
-
 /// Each method of the `Visitor` trait is a hook to be potentially
 /// overridden. Each method's default implementation recursively visits
 /// the substructure of the input via the corresponding `walk` method;
@@ -316,24 +272,6 @@ pub trait Visitor<'ast>: Sized {
     }
 }
 
-#[macro_export]
-macro_rules! walk_list {
-    ($visitor: expr, $method: ident, $list: expr $(, $($extra_args: expr),* )?) => {
-        for elem in $list {
-            $crate::try_visit!($visitor.$method(elem $(, $($extra_args,)* )?));
-        }
-    }
-}
-
-#[macro_export]
-macro_rules! visit_opt {
-    ($visitor: expr, $method: ident, $opt: expr $(, $($extra_args: expr),* )?) => {
-        if let Some(x) = $opt {
-            $crate::try_visit!($visitor.$method(x $(, $($extra_args,)* )?));
-        }
-    }
-}
-
 pub fn walk_crate<'a, V: Visitor<'a>>(visitor: &mut V, krate: &'a Crate) -> V::Result {
     walk_list!(visitor, visit_item, &krate.items);
     walk_list!(visitor, visit_attribute, &krate.attrs);
@@ -885,6 +823,7 @@ pub fn walk_inline_asm<'a, V: Visitor<'a>>(visitor: &mut V, asm: &'a InlineAsm)
                 try_visit!(visitor.visit_anon_const(anon_const))
             }
             InlineAsmOperand::Sym { sym } => try_visit!(visitor.visit_inline_asm_sym(sym)),
+            InlineAsmOperand::Label { block } => try_visit!(visitor.visit_block(block)),
         }
     }
     V::Result::output()

compiler/rustc_ast_ir/src/lib.rs

@@ -1,3 +1,4 @@
+#![cfg_attr(feature = "nightly", feature(never_type))]
 #![cfg_attr(feature = "nightly", feature(rustc_attrs))]
 #![cfg_attr(feature = "nightly", allow(internal_features))]
 
@@ -5,6 +6,8 @@
 #[macro_use]
 extern crate rustc_macros;
 
+pub mod visit;
+
 /// The movability of a coroutine / closure literal:
 /// whether a coroutine contains self-references, causing it to be `!Unpin`.
 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Copy)]

compiler/rustc_ast_ir/src/visit.rs

@@ -0,0 +1,82 @@
+use core::ops::ControlFlow;
+
+/// Similar to the `Try` trait, but also implemented for `()`.
+pub trait VisitorResult {
+    type Residual;
+    fn output() -> Self;
+    fn from_residual(residual: Self::Residual) -> Self;
+    fn from_branch(b: ControlFlow<Self::Residual>) -> Self;
+    fn branch(self) -> ControlFlow<Self::Residual>;
+}
+
+impl VisitorResult for () {
+    #[cfg(feature = "nightly")]
+    type Residual = !;
+
+    #[cfg(not(feature = "nightly"))]
+    type Residual = core::ops::Infallible;
+
+    fn output() -> Self {}
+    fn from_residual(_: Self::Residual) -> Self {}
+    fn from_branch(_: ControlFlow<Self::Residual>) -> Self {}
+    fn branch(self) -> ControlFlow<Self::Residual> {
+        ControlFlow::Continue(())
+    }
+}
+
+impl<T> VisitorResult for ControlFlow<T> {
+    type Residual = T;
+
+    fn output() -> Self {
+        ControlFlow::Continue(())
+    }
+    fn from_residual(residual: Self::Residual) -> Self {
+        ControlFlow::Break(residual)
+    }
+    fn from_branch(b: Self) -> Self {
+        b
+    }
+    fn branch(self) -> Self {
+        self
+    }
+}
+
+#[macro_export]
+macro_rules! try_visit {
+    ($e:expr) => {
+        match $crate::visit::VisitorResult::branch($e) {
+            core::ops::ControlFlow::Continue(()) => (),
+            #[allow(unreachable_code)]
+            core::ops::ControlFlow::Break(r) => {
+                return $crate::visit::VisitorResult::from_residual(r);
+            }
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! visit_opt {
+    ($visitor: expr, $method: ident, $opt: expr $(, $($extra_args: expr),* )?) => {
+        if let Some(x) = $opt {
+            $crate::try_visit!($visitor.$method(x $(, $($extra_args,)* )?));
+        }
+    }
+}
+
+#[macro_export]
+macro_rules! walk_list {
+    ($visitor: expr, $method: ident, $list: expr $(, $($extra_args: expr),* )?) => {
+        for elem in $list {
+            $crate::try_visit!($visitor.$method(elem $(, $($extra_args,)* )?));
+        }
+    }
+}
+
+#[macro_export]
+macro_rules! walk_visitable_list {
+    ($visitor: expr, $list: expr $(, $($extra_args: expr),* )?) => {
+        for elem in $list {
+            $crate::try_visit!(elem.visit_with($visitor $(, $($extra_args,)* )?));
+        }
+    }
+}

compiler/rustc_ast_lowering/messages.ftl

@@ -88,6 +88,9 @@ ast_lowering_invalid_abi_suggestion = did you mean
 ast_lowering_invalid_asm_template_modifier_const =
     asm template modifiers are not allowed for `const` arguments
 
+ast_lowering_invalid_asm_template_modifier_label =
+    asm template modifiers are not allowed for `label` arguments
+
 ast_lowering_invalid_asm_template_modifier_reg_class =
     invalid asm template modifier for this register class
 

compiler/rustc_ast_lowering/src/asm.rs

@@ -3,9 +3,9 @@ use crate::{ImplTraitContext, ImplTraitPosition, ParamMode, ResolverAstLoweringE
 use super::errors::{
     AbiSpecifiedMultipleTimes, AttSyntaxOnlyX86, ClobberAbiNotSupported,
     InlineAsmUnsupportedTarget, InvalidAbiClobberAbi, InvalidAsmTemplateModifierConst,
-    InvalidAsmTemplateModifierRegClass, InvalidAsmTemplateModifierRegClassSub,
-    InvalidAsmTemplateModifierSym, InvalidRegister, InvalidRegisterClass, RegisterClassOnlyClobber,
-    RegisterConflict,
+    InvalidAsmTemplateModifierLabel, InvalidAsmTemplateModifierRegClass,
+    InvalidAsmTemplateModifierRegClassSub, InvalidAsmTemplateModifierSym, InvalidRegister,
+    InvalidRegisterClass, RegisterClassOnlyClobber, RegisterConflict,
 };
 use super::LoweringContext;
 
@@ -22,6 +22,7 @@ use std::collections::hash_map::Entry;
 use std::fmt::Write;
 
 impl<'a, 'hir> LoweringContext<'a, 'hir> {
+    #[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
     pub(crate) fn lower_inline_asm(
         &mut self,
         sp: Span,
@@ -236,6 +237,18 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             }
                         }
                     }
+                    InlineAsmOperand::Label { block } => {
+                        if !self.tcx.features().asm_goto {
+                            feature_err(
+                                sess,
+                                sym::asm_goto,
+                                *op_sp,
+                                "label operands for inline assembly are unstable",
+                            )
+                            .emit();
+                        }
+                        hir::InlineAsmOperand::Label { block: self.lower_block(block, false) }
+                    }
                 };
                 (op, self.lower_span(*op_sp))
             })
@@ -295,6 +308,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             op_span: op_sp,
                         });
                     }
+                    hir::InlineAsmOperand::Label { .. } => {
+                        self.dcx().emit_err(InvalidAsmTemplateModifierLabel {
+                            placeholder_span,
+                            op_span: op_sp,
+                        });
+                    }
                 }
             }
         }
@@ -334,7 +353,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
                         hir::InlineAsmOperand::Const { .. }
                         | hir::InlineAsmOperand::SymFn { .. }
-                        | hir::InlineAsmOperand::SymStatic { .. } => {
+                        | hir::InlineAsmOperand::SymStatic { .. }
+                        | hir::InlineAsmOperand::Label { .. } => {
                             unreachable!("{op:?} is not a register operand");
                         }
                     };

compiler/rustc_ast_lowering/src/errors.rs

@@ -1,5 +1,5 @@
 use rustc_errors::{
-    codes::*, AddToDiagnostic, Diag, DiagArgFromDisplay, EmissionGuarantee, SubdiagnosticMessageOp,
+    codes::*, AddToDiagnostic, Diag, DiagArgFromDisplay, EmissionGuarantee, SubdiagMessageOp,
 };
 use rustc_macros::{Diagnostic, Subdiagnostic};
 use rustc_span::{symbol::Ident, Span, Symbol};
@@ -41,7 +41,7 @@ pub struct InvalidAbi {
 pub struct InvalidAbiReason(pub &'static str);
 
 impl AddToDiagnostic for InvalidAbiReason {
-    fn add_to_diagnostic_with<G: EmissionGuarantee, F: SubdiagnosticMessageOp<G>>(
+    fn add_to_diagnostic_with<G: EmissionGuarantee, F: SubdiagMessageOp<G>>(
         self,
         diag: &mut Diag<'_, G>,
         _: F,
@@ -261,6 +261,16 @@ pub struct InvalidAsmTemplateModifierSym {
     pub op_span: Span,
 }
 
+#[derive(Diagnostic, Clone, Copy)]
+#[diag(ast_lowering_invalid_asm_template_modifier_label)]
+pub struct InvalidAsmTemplateModifierLabel {
+    #[primary_span]
+    #[label(ast_lowering_template_modifier)]
+    pub placeholder_span: Span,
+    #[label(ast_lowering_argument)]
+    pub op_span: Span,
+}
+
 #[derive(Diagnostic, Clone, Copy)]
 #[diag(ast_lowering_register_class_only_clobber)]
 pub struct RegisterClassOnlyClobber {

compiler/rustc_ast_lowering/src/expr.rs

@@ -125,12 +125,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     let lit_kind = match LitKind::from_token_lit(*token_lit) {
                         Ok(lit_kind) => lit_kind,
                         Err(err) => {
-                            let guar = report_lit_error(
-                                &self.tcx.sess.parse_sess,
-                                err,
-                                *token_lit,
-                                e.span,
-                            );
+                            let guar =
+                                report_lit_error(&self.tcx.sess.psess, err, *token_lit, e.span);
                             LitKind::Err(guar)
                         }
                     };
@@ -721,7 +717,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         sym::track_caller,
                         span,
                     )))),
-                    id: self.tcx.sess.parse_sess.attr_id_generator.mk_attr_id(),
+                    id: self.tcx.sess.psess.attr_id_generator.mk_attr_id(),
                     style: AttrStyle::Outer,
                     span: unstable_span,
                 }],
@@ -764,10 +760,28 @@ impl<'hir> LoweringContext<'_, 'hir> {
             Some(hir::CoroutineKind::Coroutine(_))
             | Some(hir::CoroutineKind::Desugared(hir::CoroutineDesugaring::Gen, _))
             | None => {
-                return hir::ExprKind::Err(self.dcx().emit_err(AwaitOnlyInAsyncFnAndBlocks {
-                    await_kw_span,
-                    item_span: self.current_item,
-                }));
+                // Lower to a block `{ EXPR; <error> }` so that the awaited expr
+                // is not accidentally orphaned.
+                let stmt_id = self.next_id();
+                let expr_err = self.expr(
+                    expr.span,
+                    hir::ExprKind::Err(self.dcx().emit_err(AwaitOnlyInAsyncFnAndBlocks {
+                        await_kw_span,
+                        item_span: self.current_item,
+                    })),
+                );
+                return hir::ExprKind::Block(
+                    self.block_all(
+                        expr.span,
+                        arena_vec![self; hir::Stmt {
+                            hir_id: stmt_id,
+                            kind: hir::StmtKind::Semi(expr),
+                            span: expr.span,
+                        }],
+                        Some(self.arena.alloc(expr_err)),
+                    ),
+                    None,
+                );
             }
         };
 
@@ -1499,13 +1513,33 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }
     }
 
+    #[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
     fn lower_expr_yield(&mut self, span: Span, opt_expr: Option<&Expr>) -> hir::ExprKind<'hir> {
+        let yielded =
+            opt_expr.as_ref().map(|x| self.lower_expr(x)).unwrap_or_else(|| self.expr_unit(span));
+
         let is_async_gen = match self.coroutine_kind {
             Some(hir::CoroutineKind::Desugared(hir::CoroutineDesugaring::Gen, _)) => false,
             Some(hir::CoroutineKind::Desugared(hir::CoroutineDesugaring::AsyncGen, _)) => true,
             Some(hir::CoroutineKind::Desugared(hir::CoroutineDesugaring::Async, _)) => {
-                return hir::ExprKind::Err(
-                    self.dcx().emit_err(AsyncCoroutinesNotSupported { span }),
+                // Lower to a block `{ EXPR; <error> }` so that the awaited expr
+                // is not accidentally orphaned.
+                let stmt_id = self.next_id();
+                let expr_err = self.expr(
+                    yielded.span,
+                    hir::ExprKind::Err(self.dcx().emit_err(AsyncCoroutinesNotSupported { span })),
+                );
+                return hir::ExprKind::Block(
+                    self.block_all(
+                        yielded.span,
+                        arena_vec![self; hir::Stmt {
+                            hir_id: stmt_id,
+                            kind: hir::StmtKind::Semi(yielded),
+                            span: yielded.span,
+                        }],
+                        Some(self.arena.alloc(expr_err)),
+                    ),
+                    None,
                 );
             }
             Some(hir::CoroutineKind::Coroutine(_)) => {
@@ -1535,9 +1569,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
             }
         };
 
-        let yielded =
-            opt_expr.as_ref().map(|x| self.lower_expr(x)).unwrap_or_else(|| self.expr_unit(span));
-
         if is_async_gen {
             // `yield $expr` is transformed into `task_context = yield async_gen_ready($expr)`.
             // This ensures that we store our resumed `ResumeContext` correctly, and also that
@@ -1756,7 +1787,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
         // `#[allow(unreachable_code)]`
         let attr = attr::mk_attr_nested_word(
-            &self.tcx.sess.parse_sess.attr_id_generator,
+            &self.tcx.sess.psess.attr_id_generator,
             AttrStyle::Outer,
             sym::allow,
             sym::unreachable_code,

compiler/rustc_ast_lowering/src/format.rs

@@ -1,4 +1,5 @@
 use super::LoweringContext;
+use core::ops::ControlFlow;
 use rustc_ast as ast;
 use rustc_ast::visit::Visitor;
 use rustc_ast::*;
@@ -594,30 +595,32 @@ fn expand_format_args<'hir>(
 }
 
 fn may_contain_yield_point(e: &ast::Expr) -> bool {
-    struct MayContainYieldPoint(bool);
+    struct MayContainYieldPoint;
 
     impl Visitor<'_> for MayContainYieldPoint {
-        fn visit_expr(&mut self, e: &ast::Expr) {
+        type Result = ControlFlow<()>;
+
+        fn visit_expr(&mut self, e: &ast::Expr) -> ControlFlow<()> {
             if let ast::ExprKind::Await(_, _) | ast::ExprKind::Yield(_) = e.kind {
-                self.0 = true;
+                ControlFlow::Break(())
             } else {
                 visit::walk_expr(self, e);
+                ControlFlow::Continue(())
             }
         }
 
-        fn visit_mac_call(&mut self, _: &ast::MacCall) {
+        fn visit_mac_call(&mut self, _: &ast::MacCall) -> ControlFlow<()> {
             // Macros should be expanded at this point.
             unreachable!("unexpanded macro in ast lowering");
         }
 
-        fn visit_item(&mut self, _: &ast::Item) {
+        fn visit_item(&mut self, _: &ast::Item) -> ControlFlow<()> {
             // Do not recurse into nested items.
+            ControlFlow::Continue(())
         }
     }
 
-    let mut visitor = MayContainYieldPoint(false);
-    visitor.visit_expr(e);
-    visitor.0
+    MayContainYieldPoint.visit_expr(e).is_break()
 }
 
 fn for_all_argument_indexes(template: &mut [FormatArgsPiece], mut f: impl FnMut(&mut usize)) {

compiler/rustc_ast_lowering/src/item.rs

@@ -275,7 +275,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         }
                         Some(ty) => this.lower_ty(
                             ty,
-                            ImplTraitContext::TypeAliasesOpaqueTy { in_assoc_ty: false },
+                            ImplTraitContext::OpaqueTy {
+                                origin: hir::OpaqueTyOrigin::TyAlias {
+                                    parent: this.local_def_id(id),
+                                    in_assoc_ty: false,
+                                },
+                                fn_kind: None,
+                            },
                         ),
                     },
                 );
@@ -936,7 +942,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         Some(ty) => {
                             let ty = this.lower_ty(
                                 ty,
-                                ImplTraitContext::TypeAliasesOpaqueTy { in_assoc_ty: true },
+                                ImplTraitContext::OpaqueTy {
+                                    origin: hir::OpaqueTyOrigin::TyAlias {
+                                        parent: this.local_def_id(i.id),
+                                        in_assoc_ty: true,
+                                    },
+                                    fn_kind: None,
+                                },
                             );
                             hir::ImplItemKind::Type(ty)
                         }

compiler/rustc_ast_lowering/src/lib.rs

@@ -265,13 +265,12 @@ enum ImplTraitContext {
     /// Example: `fn foo() -> impl Debug`, where `impl Debug` is conceptually
     /// equivalent to a new opaque type like `type T = impl Debug; fn foo() -> T`.
     ///
-    ReturnPositionOpaqueTy {
-        /// Origin: Either OpaqueTyOrigin::FnReturn or OpaqueTyOrigin::AsyncFn,
+    OpaqueTy {
         origin: hir::OpaqueTyOrigin,
-        fn_kind: FnDeclKind,
+        /// Only used to change the lifetime capture rules, since
+        /// RPITIT captures all in scope, RPIT does not.
+        fn_kind: Option<FnDeclKind>,
     },
-    /// Impl trait in type aliases.
-    TypeAliasesOpaqueTy { in_assoc_ty: bool },
     /// `impl Trait` is unstably accepted in this position.
     FeatureGated(ImplTraitPosition, Symbol),
     /// `impl Trait` is not accepted in this position.
@@ -427,7 +426,7 @@ pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> hir::Crate<'_> {
     tcx.ensure_with_value().early_lint_checks(());
     tcx.ensure_with_value().debugger_visualizers(LOCAL_CRATE);
     tcx.ensure_with_value().get_lang_items(());
-    let (mut resolver, krate) = tcx.resolver_for_lowering(()).steal();
+    let (mut resolver, krate) = tcx.resolver_for_lowering().steal();
 
     let ast_index = index_crate(&resolver.node_id_to_def_id, &krate);
     let mut owners = IndexVec::from_fn_n(
@@ -1075,9 +1074,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 // Disallow ATB in dyn types
                 if self.is_in_dyn_type {
                     let suggestion = match itctx {
-                        ImplTraitContext::ReturnPositionOpaqueTy { .. }
-                        | ImplTraitContext::TypeAliasesOpaqueTy { .. }
-                        | ImplTraitContext::Universal => {
+                        ImplTraitContext::OpaqueTy { .. } | ImplTraitContext::Universal => {
                             let bound_end_span = constraint
                                 .gen_args
                                 .as_ref()
@@ -1417,24 +1414,14 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             TyKind::ImplTrait(def_node_id, bounds) => {
                 let span = t.span;
                 match itctx {
-                    ImplTraitContext::ReturnPositionOpaqueTy { origin, fn_kind } => self
-                        .lower_opaque_impl_trait(
-                            span,
-                            origin,
-                            *def_node_id,
-                            bounds,
-                            Some(fn_kind),
-                            itctx,
-                        ),
-                    ImplTraitContext::TypeAliasesOpaqueTy { in_assoc_ty } => self
-                        .lower_opaque_impl_trait(
-                            span,
-                            hir::OpaqueTyOrigin::TyAlias { in_assoc_ty },
-                            *def_node_id,
-                            bounds,
-                            None,
-                            itctx,
-                        ),
+                    ImplTraitContext::OpaqueTy { origin, fn_kind } => self.lower_opaque_impl_trait(
+                        span,
+                        origin,
+                        *def_node_id,
+                        bounds,
+                        fn_kind,
+                        itctx,
+                    ),
                     ImplTraitContext::Universal => {
                         let span = t.span;
 
@@ -1553,9 +1540,14 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
         let captured_lifetimes_to_duplicate = match origin {
             hir::OpaqueTyOrigin::TyAlias { .. } => {
-                // in a TAIT like `type Foo<'a> = impl Foo<'a>`, we don't duplicate any
-                // lifetimes, since we don't have the issue that any are late-bound.
-                Vec::new()
+                // type alias impl trait and associated type position impl trait were
+                // decided to capture all in-scope lifetimes, which we collect for
+                // all opaques during resolution.
+                self.resolver
+                    .take_extra_lifetime_params(opaque_ty_node_id)
+                    .into_iter()
+                    .map(|(ident, id, _)| Lifetime { id, ident })
+                    .collect()
             }
             hir::OpaqueTyOrigin::FnReturn(..) => {
                 if matches!(
@@ -1823,9 +1815,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         FnDeclKind::Fn
                         | FnDeclKind::Inherent
                         | FnDeclKind::Trait
-                        | FnDeclKind::Impl => ImplTraitContext::ReturnPositionOpaqueTy {
+                        | FnDeclKind::Impl => ImplTraitContext::OpaqueTy {
                             origin: hir::OpaqueTyOrigin::FnReturn(self.local_def_id(fn_node_id)),
-                            fn_kind: kind,
+                            fn_kind: Some(kind),
                         },
                         FnDeclKind::ExternFn => {
                             ImplTraitContext::Disallowed(ImplTraitPosition::ExternFnReturn)
@@ -1919,9 +1911,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     output,
                     coro,
                     opaque_ty_span,
-                    ImplTraitContext::ReturnPositionOpaqueTy {
+                    ImplTraitContext::OpaqueTy {
                         origin: hir::OpaqueTyOrigin::FnReturn(fn_def_id),
-                        fn_kind,
+                        fn_kind: Some(fn_kind),
                     },
                 );
                 arena_vec![this; bound]
@@ -2272,6 +2264,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         self.expr_block(block)
     }
 
+    #[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
     fn lower_array_length(&mut self, c: &AnonConst) -> hir::ArrayLen {
         match c.value.kind {
             ExprKind::Underscore => {

compiler/rustc_ast_lowering/src/path.rs

@@ -423,7 +423,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             // fn f(_: impl Fn() -> impl Debug) -> impl Fn() -> impl Debug
             // //      disallowed --^^^^^^^^^^        allowed --^^^^^^^^^^
             // ```
-            FnRetTy::Ty(ty) if matches!(itctx, ImplTraitContext::ReturnPositionOpaqueTy { .. }) => {
+            FnRetTy::Ty(ty) if matches!(itctx, ImplTraitContext::OpaqueTy { .. }) => {
                 if self.tcx.features().impl_trait_in_fn_trait_return {
                     self.lower_ty(ty, itctx)
                 } else {

compiler/rustc_ast_passes/src/ast_validation.rs

@@ -8,8 +8,7 @@
 
 use itertools::{Either, Itertools};
 use rustc_ast::ptr::P;
-use rustc_ast::visit::{AssocCtxt, BoundKind, FnCtxt, FnKind, Visitor};
-use rustc_ast::walk_list;
+use rustc_ast::visit::{walk_list, AssocCtxt, BoundKind, FnCtxt, FnKind, Visitor};
 use rustc_ast::*;
 use rustc_ast_pretty::pprust::{self, State};
 use rustc_data_structures::fx::FxIndexMap;
@@ -18,7 +17,7 @@ use rustc_parse::validate_attr;
 use rustc_session::lint::builtin::{
     DEPRECATED_WHERE_CLAUSE_LOCATION, MISSING_ABI, PATTERNS_IN_FNS_WITHOUT_BODY,
 };
-use rustc_session::lint::{BuiltinLintDiagnostics, LintBuffer};
+use rustc_session::lint::{BuiltinLintDiag, LintBuffer};
 use rustc_session::Session;
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::Span;
@@ -750,7 +749,7 @@ impl<'a> AstValidator<'a> {
                 id,
                 span,
                 fluent::ast_passes_extern_without_abi,
-                BuiltinLintDiagnostics::MissingAbi(span, abi::Abi::FALLBACK),
+                BuiltinLintDiag::MissingAbi(span, abi::Abi::FALLBACK),
             )
         }
     }
@@ -835,7 +834,7 @@ fn validate_generic_param_order(
 
 impl<'a> Visitor<'a> for AstValidator<'a> {
     fn visit_attribute(&mut self, attr: &Attribute) {
-        validate_attr::check_attr(&self.session.parse_sess, attr);
+        validate_attr::check_attr(&self.session.psess, attr);
     }
 
     fn visit_ty(&mut self, ty: &'a Ty) {
@@ -930,35 +929,38 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                         only_trait: only_trait.then_some(()),
                     };
 
-                self.visibility_not_permitted(
-                    &item.vis,
-                    errors::VisibilityNotPermittedNote::IndividualImplItems,
-                );
-                if let &Unsafe::Yes(span) = unsafety {
-                    self.dcx().emit_err(errors::InherentImplCannotUnsafe {
-                        span: self_ty.span,
-                        annotation_span: span,
-                        annotation: "unsafe",
-                        self_ty: self_ty.span,
-                    });
-                }
-                if let &ImplPolarity::Negative(span) = polarity {
-                    self.dcx().emit_err(error(span, "negative", false));
-                }
-                if let &Defaultness::Default(def_span) = defaultness {
-                    self.dcx().emit_err(error(def_span, "`default`", true));
-                }
-                if let &Const::Yes(span) = constness {
-                    self.dcx().emit_err(error(span, "`const`", true));
-                }
+                self.with_in_trait_impl(None, |this| {
+                    this.visibility_not_permitted(
+                        &item.vis,
+                        errors::VisibilityNotPermittedNote::IndividualImplItems,
+                    );
+                    if let &Unsafe::Yes(span) = unsafety {
+                        this.dcx().emit_err(errors::InherentImplCannotUnsafe {
+                            span: self_ty.span,
+                            annotation_span: span,
+                            annotation: "unsafe",
+                            self_ty: self_ty.span,
+                        });
+                    }
+                    if let &ImplPolarity::Negative(span) = polarity {
+                        this.dcx().emit_err(error(span, "negative", false));
+                    }
+                    if let &Defaultness::Default(def_span) = defaultness {
+                        this.dcx().emit_err(error(def_span, "`default`", true));
+                    }
+                    if let &Const::Yes(span) = constness {
+                        this.dcx().emit_err(error(span, "`const`", true));
+                    }
 
-                self.visit_vis(&item.vis);
-                self.visit_ident(item.ident);
-                self.with_tilde_const(Some(DisallowTildeConstContext::Impl(item.span)), |this| {
-                    this.visit_generics(generics)
+                    this.visit_vis(&item.vis);
+                    this.visit_ident(item.ident);
+                    this.with_tilde_const(
+                        Some(DisallowTildeConstContext::Impl(item.span)),
+                        |this| this.visit_generics(generics),
+                    );
+                    this.visit_ty(self_ty);
+                    walk_list!(this, visit_assoc_item, items, AssocCtxt::Impl);
                 });
-                self.visit_ty(self_ty);
-                walk_list!(self, visit_assoc_item, items, AssocCtxt::Impl);
                 walk_list!(self, visit_attribute, &item.attrs);
                 return; // Avoid visiting again.
             }
@@ -1408,7 +1410,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                             FnCtxt::Foreign => fluent::ast_passes_pattern_in_foreign,
                             _ => fluent::ast_passes_pattern_in_bodiless,
                         };
-                        let diag = BuiltinLintDiagnostics::PatternsInFnsWithoutBody(span, ident);
+                        let diag = BuiltinLintDiag::PatternsInFnsWithoutBody(span, ident);
                         self.lint_buffer.buffer_lint_with_diagnostic(
                             PATTERNS_IN_FNS_WITHOUT_BODY,
                             id,
@@ -1491,7 +1493,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 item.id,
                 err.span,
                 fluent::ast_passes_deprecated_where_clause_location,
-                BuiltinLintDiagnostics::DeprecatedWhereclauseLocation(sugg),
+                BuiltinLintDiag::DeprecatedWhereclauseLocation(sugg),
             );
         }
 

compiler/rustc_ast_passes/src/errors.rs

@@ -2,7 +2,7 @@
 
 use rustc_ast::ParamKindOrd;
 use rustc_errors::{
-    codes::*, AddToDiagnostic, Applicability, Diag, EmissionGuarantee, SubdiagnosticMessageOp,
+    codes::*, AddToDiagnostic, Applicability, Diag, EmissionGuarantee, SubdiagMessageOp,
 };
 use rustc_macros::{Diagnostic, Subdiagnostic};
 use rustc_span::{symbol::Ident, Span, Symbol};
@@ -374,7 +374,7 @@ pub struct EmptyLabelManySpans(pub Vec<Span>);
 
 // The derive for `Vec<Span>` does multiple calls to `span_label`, adding commas between each
 impl AddToDiagnostic for EmptyLabelManySpans {
-    fn add_to_diagnostic_with<G: EmissionGuarantee, F: SubdiagnosticMessageOp<G>>(
+    fn add_to_diagnostic_with<G: EmissionGuarantee, F: SubdiagMessageOp<G>>(
         self,
         diag: &mut Diag<'_, G>,
         _: F,
@@ -743,7 +743,7 @@ pub struct StableFeature {
 }
 
 impl AddToDiagnostic for StableFeature {
-    fn add_to_diagnostic_with<G: EmissionGuarantee, F: SubdiagnosticMessageOp<G>>(
+    fn add_to_diagnostic_with<G: EmissionGuarantee, F: SubdiagMessageOp<G>>(
         self,
         diag: &mut Diag<'_, G>,
         _: F,

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -17,6 +17,7 @@ use crate::errors;
 macro_rules! gate {
     ($visitor:expr, $feature:ident, $span:expr, $explain:expr) => {{
         if !$visitor.features.$feature && !$span.allows_unstable(sym::$feature) {
+            #[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
             feature_err(&$visitor.sess, sym::$feature, $span, $explain).emit();
         }
     }};
@@ -34,6 +35,7 @@ macro_rules! gate {
 macro_rules! gate_alt {
     ($visitor:expr, $has_feature:expr, $name:expr, $span:expr, $explain:expr) => {{
         if !$has_feature && !$span.allows_unstable($name) {
+            #[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
             feature_err(&$visitor.sess, $name, $span, $explain).emit();
         }
     }};
@@ -73,6 +75,7 @@ struct PostExpansionVisitor<'a> {
 }
 
 impl<'a> PostExpansionVisitor<'a> {
+    #[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
     fn check_abi(&self, abi: ast::StrLit, constness: ast::Const) {
         let ast::StrLit { symbol_unescaped, span, .. } = abi;
 
@@ -203,14 +206,6 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                 );
             }
         }
-        if !attr.is_doc_comment()
-            && let [seg, _] = attr.get_normal_item().path.segments.as_slice()
-            && seg.ident.name == sym::diagnostic
-            && !self.features.diagnostic_namespace
-        {
-            let msg = "`#[diagnostic]` attribute name space is experimental";
-            gate!(self, diagnostic_namespace, seg.ident.span, msg);
-        }
 
         // Emit errors for non-staged-api crates.
         if !self.features.staged_api {
@@ -507,7 +502,7 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session, features: &Features) {
     check_incompatible_features(sess, features);
     let mut visitor = PostExpansionVisitor { sess, features };
 
-    let spans = sess.parse_sess.gated_spans.spans.borrow();
+    let spans = sess.psess.gated_spans.spans.borrow();
     macro_rules! gate_all {
         ($gate:ident, $msg:literal) => {
             if let Some(spans) = spans.get(&sym::$gate) {
@@ -579,6 +574,7 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session, features: &Features) {
                 if let Ok(snippet) = sm.span_to_snippet(span)
                     && snippet == "!"
                 {
+                    #[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
                     feature_err(sess, sym::never_patterns, span, "`!` patterns are experimental")
                         .emit();
                 } else {

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -1452,6 +1452,10 @@ impl<'a> State<'a> {
                             s.print_path(&sym.path, true, 0);
                         }
                     }
+                    InlineAsmOperand::Label { block } => {
+                        s.head("label");
+                        s.print_block(block);
+                    }
                 }
             }
             AsmArg::ClobberAbi(abi) => {

compiler/rustc_attr/src/builtin.rs

@@ -8,7 +8,7 @@ use rustc_feature::{find_gated_cfg, is_builtin_attr_name, Features, GatedCfg};
 use rustc_macros::HashStable_Generic;
 use rustc_session::config::ExpectedValues;
 use rustc_session::lint::builtin::UNEXPECTED_CFGS;
-use rustc_session::lint::BuiltinLintDiagnostics;
+use rustc_session::lint::BuiltinLintDiag;
 use rustc_session::parse::feature_err;
 use rustc_session::{RustcVersion, Session};
 use rustc_span::hygiene::Transparency;
@@ -516,6 +516,7 @@ pub struct Condition {
 }
 
 /// Tests if a cfg-pattern matches the cfg set
+#[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
 pub fn cfg_matches(
     cfg: &ast::MetaItem,
     sess: &Session,
@@ -524,9 +525,9 @@ pub fn cfg_matches(
 ) -> bool {
     eval_condition(cfg, sess, features, &mut |cfg| {
         try_gate_cfg(cfg.name, cfg.span, sess, features);
-        match sess.parse_sess.check_config.expecteds.get(&cfg.name) {
+        match sess.psess.check_config.expecteds.get(&cfg.name) {
             Some(ExpectedValues::Some(values)) if !values.contains(&cfg.value) => {
-                sess.parse_sess.buffer_lint_with_diagnostic(
+                sess.psess.buffer_lint_with_diagnostic(
                     UNEXPECTED_CFGS,
                     cfg.span,
                     lint_node_id,
@@ -535,19 +536,19 @@ pub fn cfg_matches(
                     } else {
                         format!("unexpected `cfg` condition value: (none)")
                     },
-                    BuiltinLintDiagnostics::UnexpectedCfgValue(
+                    BuiltinLintDiag::UnexpectedCfgValue(
                         (cfg.name, cfg.name_span),
                         cfg.value.map(|v| (v, cfg.value_span.unwrap())),
                     ),
                 );
             }
-            None if sess.parse_sess.check_config.exhaustive_names => {
-                sess.parse_sess.buffer_lint_with_diagnostic(
+            None if sess.psess.check_config.exhaustive_names => {
+                sess.psess.buffer_lint_with_diagnostic(
                     UNEXPECTED_CFGS,
                     cfg.span,
                     lint_node_id,
                     format!("unexpected `cfg` condition name: `{}`", cfg.name),
-                    BuiltinLintDiagnostics::UnexpectedCfgName(
+                    BuiltinLintDiag::UnexpectedCfgName(
                         (cfg.name, cfg.name_span),
                         cfg.value.map(|v| (v, cfg.value_span.unwrap())),
                     ),
@@ -555,7 +556,7 @@ pub fn cfg_matches(
             }
             _ => { /* not unexpected */ }
         }
-        sess.parse_sess.config.contains(&(cfg.name, cfg.value))
+        sess.psess.config.contains(&(cfg.name, cfg.value))
     })
 }
 
@@ -566,6 +567,7 @@ fn try_gate_cfg(name: Symbol, span: Span, sess: &Session, features: Option<&Feat
     }
 }
 
+#[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
 fn gate_cfg(gated_cfg: &GatedCfg, cfg_span: Span, sess: &Session, features: &Features) {
     let (cfg, feature, has_feature) = gated_cfg;
     if !has_feature(features) && !cfg_span.allows_unstable(*feature) {
@@ -592,13 +594,14 @@ fn parse_version(s: Symbol) -> Option<RustcVersion> {
 
 /// Evaluate a cfg-like condition (with `any` and `all`), using `eval` to
 /// evaluate individual items.
+#[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
 pub fn eval_condition(
     cfg: &ast::MetaItem,
     sess: &Session,
     features: Option<&Features>,
     eval: &mut impl FnMut(Condition) -> bool,
 ) -> bool {
-    let dcx = &sess.parse_sess.dcx;
+    let dcx = &sess.psess.dcx;
     match &cfg.kind {
         ast::MetaItemKind::List(mis) if cfg.name_or_empty() == sym::version => {
             try_gate_cfg(sym::version, cfg.span, sess, features);
@@ -626,7 +629,7 @@ pub fn eval_condition(
             };
 
             // See https://github.com/rust-lang/rust/issues/64796#issuecomment-640851454 for details
-            if sess.parse_sess.assume_incomplete_release {
+            if sess.psess.assume_incomplete_release {
                 RustcVersion::CURRENT > min_version
             } else {
                 RustcVersion::CURRENT >= min_version

compiler/rustc_borrowck/src/diagnostics/mod.rs

@@ -76,6 +76,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
     /// LL |         for (key, value) in dict {
     ///    |                             ^^^^
     /// ```
+    #[allow(rustc::diagnostic_outside_of_impl)] // FIXME
     pub(super) fn add_moved_or_invoked_closure_note(
         &self,
         location: Location,
@@ -585,6 +586,7 @@ impl UseSpans<'_> {
     }
 
     /// Add a span label to the arguments of the closure, if it exists.
+    #[allow(rustc::diagnostic_outside_of_impl)]
     pub(super) fn args_subdiag(
         self,
         dcx: &rustc_errors::DiagCtxt,
@@ -598,6 +600,7 @@ impl UseSpans<'_> {
 
     /// Add a span label to the use of the captured variable, if it exists.
     /// only adds label to the `path_span`
+    #[allow(rustc::diagnostic_outside_of_impl)]
     pub(super) fn var_path_only_subdiag(
         self,
         dcx: &rustc_errors::DiagCtxt,
@@ -635,6 +638,7 @@ impl UseSpans<'_> {
     }
 
     /// Add a subdiagnostic to the use of the captured variable, if it exists.
+    #[allow(rustc::diagnostic_outside_of_impl)]
     pub(super) fn var_subdiag(
         self,
         dcx: &rustc_errors::DiagCtxt,
@@ -1008,6 +1012,8 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         self.borrow_spans(span, borrow.reserve_location)
     }
 
+    #[allow(rustc::diagnostic_outside_of_impl)]
+    #[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
     fn explain_captures(
         &mut self,
         err: &mut Diag<'_>,

compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs

@@ -1,6 +1,7 @@
 #![allow(rustc::diagnostic_outside_of_impl)]
 #![allow(rustc::untranslatable_diagnostic)]
 
+use core::ops::ControlFlow;
 use hir::ExprKind;
 use rustc_errors::{Applicability, Diag};
 use rustc_hir as hir;
@@ -727,30 +728,12 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             _ => local_decl.source_info.span,
         };
 
-        struct BindingFinder {
-            span: Span,
-            hir_id: Option<hir::HirId>,
-        }
-
-        impl<'tcx> Visitor<'tcx> for BindingFinder {
-            fn visit_stmt(&mut self, s: &'tcx hir::Stmt<'tcx>) {
-                if let hir::StmtKind::Local(local) = s.kind {
-                    if local.pat.span == self.span {
-                        self.hir_id = Some(local.hir_id);
-                    }
-                }
-                hir::intravisit::walk_stmt(self, s);
-            }
-        }
-
         let def_id = self.body.source.def_id();
         let hir_id = if let Some(local_def_id) = def_id.as_local()
             && let Some(body_id) = self.infcx.tcx.hir().maybe_body_owned_by(local_def_id)
         {
             let body = self.infcx.tcx.hir().body(body_id);
-            let mut v = BindingFinder { span: pat_span, hir_id: None };
-            v.visit_body(body);
-            v.hir_id
+            BindingFinder { span: pat_span }.visit_body(body).break_value()
         } else {
             None
         };
@@ -859,17 +842,18 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
         };
 
         let hir_map = self.infcx.tcx.hir();
-        struct Finder<'tcx> {
+        struct Finder {
             span: Span,
-            expr: Option<&'tcx Expr<'tcx>>,
         }
 
-        impl<'tcx> Visitor<'tcx> for Finder<'tcx> {
-            fn visit_expr(&mut self, e: &'tcx hir::Expr<'tcx>) {
-                if e.span == self.span && self.expr.is_none() {
-                    self.expr = Some(e);
+        impl<'tcx> Visitor<'tcx> for Finder {
+            type Result = ControlFlow<&'tcx Expr<'tcx>>;
+            fn visit_expr(&mut self, e: &'tcx hir::Expr<'tcx>) -> Self::Result {
+                if e.span == self.span {
+                    ControlFlow::Break(e)
+                } else {
+                    hir::intravisit::walk_expr(self, e)
                 }
-                hir::intravisit::walk_expr(self, e);
             }
         }
         if let Some(body_id) = hir_map.maybe_body_owned_by(self.mir_def_id())
@@ -878,9 +862,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             // `span` corresponds to the expression being iterated, find the `for`-loop desugared
             // expression with that span in order to identify potential fixes when encountering a
             // read-only iterator that should be mutable.
-            let mut v = Finder { span, expr: None };
-            v.visit_block(block);
-            if let Some(expr) = v.expr
+            if let ControlFlow::Break(expr) = (Finder { span }).visit_block(block)
                 && let Call(_, [expr]) = expr.kind
             {
                 match expr.kind {
@@ -1179,29 +1161,12 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                 );
             }
             Some((false, err_label_span, message)) => {
-                struct BindingFinder {
-                    span: Span,
-                    hir_id: Option<hir::HirId>,
-                }
-
-                impl<'tcx> Visitor<'tcx> for BindingFinder {
-                    fn visit_stmt(&mut self, s: &'tcx hir::Stmt<'tcx>) {
-                        if let hir::StmtKind::Local(local) = s.kind {
-                            if local.pat.span == self.span {
-                                self.hir_id = Some(local.hir_id);
-                            }
-                        }
-                        hir::intravisit::walk_stmt(self, s);
-                    }
-                }
                 let def_id = self.body.source.def_id();
                 let hir_id = if let Some(local_def_id) = def_id.as_local()
                     && let Some(body_id) = self.infcx.tcx.hir().maybe_body_owned_by(local_def_id)
                 {
                     let body = self.infcx.tcx.hir().body(body_id);
-                    let mut v = BindingFinder { span: err_label_span, hir_id: None };
-                    v.visit_body(body);
-                    v.hir_id
+                    BindingFinder { span: err_label_span }.visit_body(body).break_value()
                 } else {
                     None
                 };
@@ -1333,6 +1298,23 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
     }
 }
 
+struct BindingFinder {
+    span: Span,
+}
+
+impl<'tcx> Visitor<'tcx> for BindingFinder {
+    type Result = ControlFlow<hir::HirId>;
+    fn visit_stmt(&mut self, s: &'tcx hir::Stmt<'tcx>) -> Self::Result {
+        if let hir::StmtKind::Local(local) = s.kind
+            && local.pat.span == self.span
+        {
+            ControlFlow::Break(local.hir_id)
+        } else {
+            hir::intravisit::walk_stmt(self, s)
+        }
+    }
+}
+
 pub fn mut_borrow_of_mutable_ref(local_decl: &LocalDecl<'_>, local_name: Option<Symbol>) -> bool {
     debug!("local_info: {:?}, ty.kind(): {:?}", local_decl.local_info, local_decl.ty.kind());
 

compiler/rustc_borrowck/src/diagnostics/region_errors.rs

@@ -201,6 +201,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
     // For generic associated types (GATs) which implied 'static requirement
     // from higher-ranked trait bounds (HRTB). Try to locate span of the trait
     // and the span which bounded to the trait for adding 'static lifetime suggestion
+    #[allow(rustc::diagnostic_outside_of_impl)]
+    #[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
     fn suggest_static_lifetime_for_gat_from_hrtb(
         &self,
         diag: &mut Diag<'_>,
@@ -254,9 +256,6 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
 
         hrtb_bounds.iter().for_each(|bound| {
             let Trait(PolyTraitRef { trait_ref, span: trait_span, .. }, _) = bound else { return; };
-            // FIXME: make this translatable
-            #[allow(rustc::diagnostic_outside_of_impl)]
-            #[allow(rustc::untranslatable_diagnostic)]
             diag.span_note(
                 *trait_span,
                 "due to current limitations in the borrow checker, this implies a `'static` lifetime"
@@ -580,6 +579,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
     ///            executing...
     ///    = note: ...therefore, returned references to captured variables will escape the closure
     /// ```
+    #[allow(rustc::diagnostic_outside_of_impl)] // FIXME
     fn report_fnmut_error(
         &self,
         errci: &ErrorConstraintInfo<'tcx>,
@@ -761,6 +761,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
     ///    |     ^^^^^^^^^^^^^^ function was supposed to return data with lifetime `'a` but it
     ///    |                    is returning data with lifetime `'b`
     /// ```
+    #[allow(rustc::diagnostic_outside_of_impl)] // FIXME
     fn report_general_error(&self, errci: &ErrorConstraintInfo<'tcx>) -> Diag<'tcx> {
         let ErrorConstraintInfo {
             fr,
@@ -822,6 +823,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
     /// LL |     fn iter_values_anon(&self) -> impl Iterator<Item=u32> + 'a {
     ///    |                                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
     /// ```
+    #[allow(rustc::diagnostic_outside_of_impl)]
+    #[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
     fn add_static_impl_trait_suggestion(
         &self,
         diag: &mut Diag<'_>,
@@ -972,6 +975,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
         self.suggest_constrain_dyn_trait_in_impl(diag, &visitor.0, ident, self_ty);
     }
 
+    #[allow(rustc::diagnostic_outside_of_impl)]
+    #[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
     #[instrument(skip(self, err), level = "debug")]
     fn suggest_constrain_dyn_trait_in_impl(
         &self,
@@ -1034,6 +1039,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
         suggest_adding_lifetime_params(self.infcx.tcx, sub, ty_sup, ty_sub, diag);
     }
 
+    #[allow(rustc::diagnostic_outside_of_impl)]
+    #[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
     fn suggest_move_on_borrowing_closure(&self, diag: &mut Diag<'_>) {
         let map = self.infcx.tcx.hir();
         let body_id = map.body_owned_by(self.mir_def_id());

compiler/rustc_borrowck/src/lib.rs

@@ -6,6 +6,7 @@
 #![feature(assert_matches)]
 #![feature(associated_type_bounds)]
 #![feature(box_patterns)]
+#![feature(control_flow_enum)]
 #![feature(let_chains)]
 #![feature(min_specialization)]
 #![feature(never_type)]
@@ -723,7 +724,7 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
                 operands,
                 options: _,
                 line_spans: _,
-                destination: _,
+                targets: _,
                 unwind: _,
             } => {
                 for op in operands {
@@ -749,7 +750,8 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
                         }
                         InlineAsmOperand::Const { value: _ }
                         | InlineAsmOperand::SymFn { value: _ }
-                        | InlineAsmOperand::SymStatic { def_id: _ } => {}
+                        | InlineAsmOperand::SymStatic { def_id: _ }
+                        | InlineAsmOperand::Label { target_index: _ } => {}
                     }
                 }
             }

compiler/rustc_borrowck/src/polonius/loan_invalidations.rs

@@ -161,7 +161,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
                 operands,
                 options: _,
                 line_spans: _,
-                destination: _,
+                targets: _,
                 unwind: _,
             } => {
                 for op in operands {
@@ -182,7 +182,8 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
                         }
                         InlineAsmOperand::Const { value: _ }
                         | InlineAsmOperand::SymFn { value: _ }
-                        | InlineAsmOperand::SymStatic { def_id: _ } => {}
+                        | InlineAsmOperand::SymStatic { def_id: _ }
+                        | InlineAsmOperand::Label { target_index: _ } => {}
                     }
                 }
             }

compiler/rustc_borrowck/src/region_infer/opaque_types.rs

@@ -367,14 +367,17 @@ fn check_opaque_type_parameter_valid(
     span: Span,
 ) -> Result<(), ErrorGuaranteed> {
     let opaque_ty_hir = tcx.hir().expect_item(opaque_type_key.def_id);
-    let is_ty_alias = match opaque_ty_hir.expect_opaque_ty().origin {
-        OpaqueTyOrigin::TyAlias { .. } => true,
-        OpaqueTyOrigin::AsyncFn(..) | OpaqueTyOrigin::FnReturn(..) => false,
+    let (parent, is_ty_alias) = match opaque_ty_hir.expect_opaque_ty().origin {
+        OpaqueTyOrigin::TyAlias { parent, .. } => (parent, true),
+        OpaqueTyOrigin::AsyncFn(parent) | OpaqueTyOrigin::FnReturn(parent) => (parent, false),
     };
 
-    let opaque_generics = tcx.generics_of(opaque_type_key.def_id);
+    let parent_generics = tcx.generics_of(parent);
     let mut seen_params: FxIndexMap<_, Vec<_>> = FxIndexMap::default();
-    for (i, arg) in opaque_type_key.args.iter().enumerate() {
+
+    // Only check the parent generics, which will ignore any of the
+    // duplicated lifetime args that come from reifying late-bounds.
+    for (i, arg) in opaque_type_key.args.iter().take(parent_generics.count()).enumerate() {
         if let Err(guar) = arg.error_reported() {
             return Err(guar);
         }
@@ -395,7 +398,7 @@ fn check_opaque_type_parameter_valid(
             seen_params.entry(arg).or_default().push(i);
         } else {
             // Prevent `fn foo() -> Foo<u32>` from being defining.
-            let opaque_param = opaque_generics.param_at(i, tcx);
+            let opaque_param = parent_generics.param_at(i, tcx);
             let kind = opaque_param.kind.descr();
 
             return Err(tcx.dcx().emit_err(NonGenericOpaqueTypeParam {
@@ -409,10 +412,10 @@ fn check_opaque_type_parameter_valid(
 
     for (_, indices) in seen_params {
         if indices.len() > 1 {
-            let descr = opaque_generics.param_at(indices[0], tcx).kind.descr();
+            let descr = parent_generics.param_at(indices[0], tcx).kind.descr();
             let spans: Vec<_> = indices
                 .into_iter()
-                .map(|i| tcx.def_span(opaque_generics.param_at(i, tcx).def_id))
+                .map(|i| tcx.def_span(parent_generics.param_at(i, tcx).def_id))
                 .collect();
             #[allow(rustc::diagnostic_outside_of_impl)]
             #[allow(rustc::untranslatable_diagnostic)]

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -1667,7 +1667,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             // (Eventually this should use const-generics, but those are not up for the task yet:
             // https://github.com/rust-lang/rust/issues/85229.)
             if let Some(name @ (sym::simd_shuffle | sym::simd_insert | sym::simd_extract)) =
-                self.tcx().intrinsic(def_id)
+                self.tcx().intrinsic(def_id).map(|i| i.name)
             {
                 let idx = match name {
                     sym::simd_shuffle => 2,
@@ -1770,8 +1770,8 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 self.assert_iscleanup(body, block_data, real_target, is_cleanup);
                 self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);
             }
-            TerminatorKind::InlineAsm { destination, unwind, .. } => {
-                if let Some(target) = destination {
+            TerminatorKind::InlineAsm { ref targets, unwind, .. } => {
+                for &target in targets {
                     self.assert_iscleanup(body, block_data, target, is_cleanup);
                 }
                 self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);

compiler/rustc_builtin_macros/messages.ftl

@@ -19,6 +19,8 @@ builtin_macros_asm_expected_other = expected operand, {$is_global_asm ->
 
 builtin_macros_asm_explicit_register_name = explicit register arguments cannot have names
 
+builtin_macros_asm_mayunwind = asm labels are not allowed with the `may_unwind` option
+
 builtin_macros_asm_modifier_invalid = asm template modifier must be a single character
 
 builtin_macros_asm_mutually_exclusive = the `{$opt1}` and `{$opt2}` options are mutually exclusive

compiler/rustc_builtin_macros/src/asm.rs

@@ -10,7 +10,6 @@ use rustc_index::bit_set::GrowableBitSet;
 use rustc_parse::parser::Parser;
 use rustc_parse_format as parse;
 use rustc_session::lint;
-use rustc_session::parse::ParseSess;
 use rustc_span::symbol::Ident;
 use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::{ErrorGuaranteed, InnerSpan, Span};
@@ -36,19 +35,17 @@ fn parse_args<'a>(
     is_global_asm: bool,
 ) -> PResult<'a, AsmArgs> {
     let mut p = ecx.new_parser_from_tts(tts);
-    let sess = &ecx.sess.parse_sess;
-    parse_asm_args(&mut p, sess, sp, is_global_asm)
+    parse_asm_args(&mut p, sp, is_global_asm)
 }
 
 // Primarily public for rustfmt consumption.
 // Internal consumers should continue to leverage `expand_asm`/`expand__global_asm`
 pub fn parse_asm_args<'a>(
     p: &mut Parser<'a>,
-    sess: &'a ParseSess,
     sp: Span,
     is_global_asm: bool,
 ) -> PResult<'a, AsmArgs> {
-    let dcx = &sess.dcx;
+    let dcx = &p.psess.dcx;
 
     if p.token == token::Eof {
         return Err(dcx.create_err(errors::AsmRequiresTemplate { span: sp }));
@@ -167,6 +164,9 @@ pub fn parse_asm_args<'a>(
                 path: path.clone(),
             };
             ast::InlineAsmOperand::Sym { sym }
+        } else if !is_global_asm && p.eat_keyword(sym::label) {
+            let block = p.parse_block()?;
+            ast::InlineAsmOperand::Label { block }
         } else if allow_templates {
             let template = p.parse_expr()?;
             // If it can't possibly expand to a string, provide diagnostics here to include other
@@ -243,6 +243,7 @@ pub fn parse_asm_args<'a>(
     let mut have_real_output = false;
     let mut outputs_sp = vec![];
     let mut regclass_outputs = vec![];
+    let mut labels_sp = vec![];
     for (op, op_sp) in &args.operands {
         match op {
             ast::InlineAsmOperand::Out { reg, expr, .. }
@@ -260,6 +261,9 @@ pub fn parse_asm_args<'a>(
                     regclass_outputs.push(*op_sp);
                 }
             }
+            ast::InlineAsmOperand::Label { .. } => {
+                labels_sp.push(*op_sp);
+            }
             _ => {}
         }
     }
@@ -271,6 +275,9 @@ pub fn parse_asm_args<'a>(
         // Bail out now since this is likely to confuse MIR
         return Err(err);
     }
+    if args.options.contains(ast::InlineAsmOptions::MAY_UNWIND) && !labels_sp.is_empty() {
+        dcx.emit_err(errors::AsmMayUnwind { labels_sp });
+    }
 
     if args.clobber_abis.len() > 0 {
         if is_global_asm {
@@ -299,7 +306,7 @@ pub fn parse_asm_args<'a>(
 fn err_duplicate_option(p: &mut Parser<'_>, symbol: Symbol, span: Span) {
     // Tool-only output
     let full_span = if p.token.kind == token::Comma { span.to(p.token.span) } else { span };
-    p.sess.dcx.emit_err(errors::AsmOptAlreadyprovided { span, symbol, full_span });
+    p.psess.dcx.emit_err(errors::AsmOptAlreadyprovided { span, symbol, full_span });
 }
 
 /// Try to set the provided option in the provided `AsmArgs`.
@@ -371,7 +378,7 @@ fn parse_clobber_abi<'a>(p: &mut Parser<'a>, args: &mut AsmArgs) -> PResult<'a,
     p.expect(&token::OpenDelim(Delimiter::Parenthesis))?;
 
     if p.eat(&token::CloseDelim(Delimiter::Parenthesis)) {
-        return Err(p.sess.dcx.create_err(errors::NonABI { span: p.token.span }));
+        return Err(p.psess.dcx.create_err(errors::NonABI { span: p.token.span }));
     }
 
     let mut new_abis = Vec::new();
@@ -382,7 +389,7 @@ fn parse_clobber_abi<'a>(p: &mut Parser<'a>, args: &mut AsmArgs) -> PResult<'a,
             }
             Err(opt_lit) => {
                 let span = opt_lit.map_or(p.token.span, |lit| lit.span);
-                let mut err = p.sess.dcx.struct_span_err(span, "expected string literal");
+                let mut err = p.psess.dcx.struct_span_err(span, "expected string literal");
                 err.span_label(span, "not a string literal");
                 return Err(err);
             }
@@ -498,7 +505,7 @@ fn expand_preparsed_asm(
             };
 
             if template_str.contains(".intel_syntax") {
-                ecx.parse_sess().buffer_lint(
+                ecx.psess().buffer_lint(
                     lint::builtin::BAD_ASM_STYLE,
                     find_span(".intel_syntax"),
                     ecx.current_expansion.lint_node_id,
@@ -506,7 +513,7 @@ fn expand_preparsed_asm(
                 );
             }
             if template_str.contains(".att_syntax") {
-                ecx.parse_sess().buffer_lint(
+                ecx.psess().buffer_lint(
                     lint::builtin::BAD_ASM_STYLE,
                     find_span(".att_syntax"),
                     ecx.current_expansion.lint_node_id,

compiler/rustc_builtin_macros/src/cfg_accessible.rs

@@ -46,7 +46,7 @@ impl MultiItemModifier for Expander {
     ) -> ExpandResult<Vec<Annotatable>, Annotatable> {
         let template = AttributeTemplate { list: Some("path"), ..Default::default() };
         validate_attr::check_builtin_meta_item(
-            &ecx.sess.parse_sess,
+            &ecx.sess.psess,
             meta_item,
             ast::AttrStyle::Outer,
             sym::cfg_accessible,

compiler/rustc_builtin_macros/src/cfg_eval.rs

@@ -1,5 +1,6 @@
 use crate::util::{check_builtin_macro_attribute, warn_on_duplicate_attribute};
 
+use core::ops::ControlFlow;
 use rustc_ast as ast;
 use rustc_ast::mut_visit::MutVisitor;
 use rustc_ast::ptr::P;
@@ -87,41 +88,40 @@ fn flat_map_annotatable(
     }
 }
 
-struct CfgFinder {
-    has_cfg_or_cfg_attr: bool,
-}
+fn has_cfg_or_cfg_attr(annotatable: &Annotatable) -> bool {
+    struct CfgFinder;
 
-impl CfgFinder {
-    fn has_cfg_or_cfg_attr(annotatable: &Annotatable) -> bool {
-        let mut finder = CfgFinder { has_cfg_or_cfg_attr: false };
-        match annotatable {
-            Annotatable::Item(item) => finder.visit_item(item),
-            Annotatable::TraitItem(item) => finder.visit_assoc_item(item, visit::AssocCtxt::Trait),
-            Annotatable::ImplItem(item) => finder.visit_assoc_item(item, visit::AssocCtxt::Impl),
-            Annotatable::ForeignItem(item) => finder.visit_foreign_item(item),
-            Annotatable::Stmt(stmt) => finder.visit_stmt(stmt),
-            Annotatable::Expr(expr) => finder.visit_expr(expr),
-            Annotatable::Arm(arm) => finder.visit_arm(arm),
-            Annotatable::ExprField(field) => finder.visit_expr_field(field),
-            Annotatable::PatField(field) => finder.visit_pat_field(field),
-            Annotatable::GenericParam(param) => finder.visit_generic_param(param),
-            Annotatable::Param(param) => finder.visit_param(param),
-            Annotatable::FieldDef(field) => finder.visit_field_def(field),
-            Annotatable::Variant(variant) => finder.visit_variant(variant),
-            Annotatable::Crate(krate) => finder.visit_crate(krate),
-        };
-        finder.has_cfg_or_cfg_attr
-    }
-}
-
-impl<'ast> visit::Visitor<'ast> for CfgFinder {
-    fn visit_attribute(&mut self, attr: &'ast Attribute) {
-        // We want short-circuiting behavior, so don't use the '|=' operator.
-        self.has_cfg_or_cfg_attr = self.has_cfg_or_cfg_attr
-            || attr
+    impl<'ast> visit::Visitor<'ast> for CfgFinder {
+        type Result = ControlFlow<()>;
+        fn visit_attribute(&mut self, attr: &'ast Attribute) -> ControlFlow<()> {
+            if attr
                 .ident()
-                .is_some_and(|ident| ident.name == sym::cfg || ident.name == sym::cfg_attr);
+                .is_some_and(|ident| ident.name == sym::cfg || ident.name == sym::cfg_attr)
+            {
+                ControlFlow::Break(())
+            } else {
+                ControlFlow::Continue(())
+            }
+        }
     }
+
+    let res = match annotatable {
+        Annotatable::Item(item) => CfgFinder.visit_item(item),
+        Annotatable::TraitItem(item) => CfgFinder.visit_assoc_item(item, visit::AssocCtxt::Trait),
+        Annotatable::ImplItem(item) => CfgFinder.visit_assoc_item(item, visit::AssocCtxt::Impl),
+        Annotatable::ForeignItem(item) => CfgFinder.visit_foreign_item(item),
+        Annotatable::Stmt(stmt) => CfgFinder.visit_stmt(stmt),
+        Annotatable::Expr(expr) => CfgFinder.visit_expr(expr),
+        Annotatable::Arm(arm) => CfgFinder.visit_arm(arm),
+        Annotatable::ExprField(field) => CfgFinder.visit_expr_field(field),
+        Annotatable::PatField(field) => CfgFinder.visit_pat_field(field),
+        Annotatable::GenericParam(param) => CfgFinder.visit_generic_param(param),
+        Annotatable::Param(param) => CfgFinder.visit_param(param),
+        Annotatable::FieldDef(field) => CfgFinder.visit_field_def(field),
+        Annotatable::Variant(variant) => CfgFinder.visit_variant(variant),
+        Annotatable::Crate(krate) => CfgFinder.visit_crate(krate),
+    };
+    res.is_break()
 }
 
 impl CfgEval<'_, '_> {
@@ -132,7 +132,7 @@ impl CfgEval<'_, '_> {
     fn configure_annotatable(&mut self, mut annotatable: Annotatable) -> Option<Annotatable> {
         // Tokenizing and re-parsing the `Annotatable` can have a significant
         // performance impact, so try to avoid it if possible
-        if !CfgFinder::has_cfg_or_cfg_attr(&annotatable) {
+        if !has_cfg_or_cfg_attr(&annotatable) {
             return Some(annotatable);
         }
 
@@ -195,8 +195,7 @@ impl CfgEval<'_, '_> {
         // Re-parse the tokens, setting the `capture_cfg` flag to save extra information
         // to the captured `AttrTokenStream` (specifically, we capture
         // `AttrTokenTree::AttributesData` for all occurrences of `#[cfg]` and `#[cfg_attr]`)
-        let mut parser =
-            rustc_parse::stream_to_parser(&self.cfg.sess.parse_sess, orig_tokens, None);
+        let mut parser = rustc_parse::stream_to_parser(&self.cfg.sess.psess, orig_tokens, None);
         parser.capture_cfg = true;
         match parse_annotatable_with(&mut parser) {
             Ok(a) => annotatable = a,

compiler/rustc_builtin_macros/src/cmdline_attrs.rs

@@ -7,10 +7,10 @@ use rustc_ast::{self as ast, AttrItem, AttrStyle};
 use rustc_session::parse::ParseSess;
 use rustc_span::FileName;
 
-pub fn inject(krate: &mut ast::Crate, parse_sess: &ParseSess, attrs: &[String]) {
+pub fn inject(krate: &mut ast::Crate, psess: &ParseSess, attrs: &[String]) {
     for raw_attr in attrs {
         let mut parser = rustc_parse::new_parser_from_source_str(
-            parse_sess,
+            psess,
             FileName::cli_crate_attr_source_code(raw_attr),
             raw_attr.clone(),
         );
@@ -25,12 +25,12 @@ pub fn inject(krate: &mut ast::Crate, parse_sess: &ParseSess, attrs: &[String])
         };
         let end_span = parser.token.span;
         if parser.token != token::Eof {
-            parse_sess.dcx.emit_err(errors::InvalidCrateAttr { span: start_span.to(end_span) });
+            psess.dcx.emit_err(errors::InvalidCrateAttr { span: start_span.to(end_span) });
             continue;
         }
 
         krate.attrs.push(mk_attr(
-            &parse_sess.attr_id_generator,
+            &psess.attr_id_generator,
             AttrStyle::Inner,
             path,
             args,

compiler/rustc_builtin_macros/src/concat.rs

@@ -43,7 +43,7 @@ pub fn expand_concat(
                     guar = Some(guarantee);
                 }
                 Err(err) => {
-                    guar = Some(report_lit_error(&cx.sess.parse_sess, err, token_lit, e.span));
+                    guar = Some(report_lit_error(&cx.sess.psess, err, token_lit, e.span));
                 }
             },
             // We also want to allow negative numeric literals.
@@ -52,7 +52,7 @@ pub fn expand_concat(
                     Ok(LitKind::Int(i, _)) => accumulator.push_str(&format!("-{i}")),
                     Ok(LitKind::Float(f, _)) => accumulator.push_str(&format!("-{f}")),
                     Err(err) => {
-                        guar = Some(report_lit_error(&cx.sess.parse_sess, err, token_lit, e.span));
+                        guar = Some(report_lit_error(&cx.sess.psess, err, token_lit, e.span));
                     }
                     _ => missing_literal.push(e.span),
                 }

compiler/rustc_builtin_macros/src/concat_bytes.rs

@@ -55,7 +55,7 @@ fn invalid_type_err(
         Ok(LitKind::Int(_, _)) => dcx.emit_err(ConcatBytesNonU8 { span }),
         Ok(LitKind::ByteStr(..) | LitKind::Byte(_)) => unreachable!(),
         Ok(LitKind::Err(guar)) => guar,
-        Err(err) => report_lit_error(&cx.sess.parse_sess, err, token_lit, span),
+        Err(err) => report_lit_error(&cx.sess.psess, err, token_lit, span),
     }
 }
 

compiler/rustc_builtin_macros/src/derive.rs

@@ -34,7 +34,7 @@ impl MultiItemModifier for Expander {
                 let template =
                     AttributeTemplate { list: Some("Trait1, Trait2, ..."), ..Default::default() };
                 validate_attr::check_builtin_meta_item(
-                    &sess.parse_sess,
+                    &sess.psess,
                     meta_item,
                     ast::AttrStyle::Outer,
                     sym::derive,

compiler/rustc_builtin_macros/src/deriving/default.rs

@@ -1,8 +1,10 @@
 use crate::deriving::generic::ty::*;
 use crate::deriving::generic::*;
 use crate::errors;
+use core::ops::ControlFlow;
 use rustc_ast as ast;
-use rustc_ast::{attr, walk_list, EnumDef, VariantData};
+use rustc_ast::visit::walk_list;
+use rustc_ast::{attr, EnumDef, VariantData};
 use rustc_expand::base::{Annotatable, DummyResult, ExtCtxt};
 use rustc_span::symbol::Ident;
 use rustc_span::symbol::{kw, sym};
@@ -230,20 +232,19 @@ impl<'a, 'b> rustc_ast::visit::Visitor<'a> for DetectNonVariantDefaultAttr<'a, '
 }
 
 fn has_a_default_variant(item: &Annotatable) -> bool {
-    struct HasDefaultAttrOnVariant {
-        found: bool,
-    }
+    struct HasDefaultAttrOnVariant;
 
     impl<'ast> rustc_ast::visit::Visitor<'ast> for HasDefaultAttrOnVariant {
-        fn visit_variant(&mut self, v: &'ast rustc_ast::Variant) {
+        type Result = ControlFlow<()>;
+        fn visit_variant(&mut self, v: &'ast rustc_ast::Variant) -> ControlFlow<()> {
             if v.attrs.iter().any(|attr| attr.has_name(kw::Default)) {
-                self.found = true;
+                ControlFlow::Break(())
+            } else {
+                // no need to subrecurse.
+                ControlFlow::Continue(())
             }
-            // no need to subrecurse.
         }
     }
 
-    let mut visitor = HasDefaultAttrOnVariant { found: false };
-    item.visit_with(&mut visitor);
-    visitor.found
+    item.visit_with(&mut HasDefaultAttrOnVariant).is_break()
 }

compiler/rustc_builtin_macros/src/deriving/generic/mod.rs

@@ -1603,9 +1603,10 @@ impl<'a> TraitDef<'a> {
                         // Once use of `icu4x-0.9.0` has dropped sufficiently, this
                         // exception should be removed.
                         let is_simple_path = |ty: &P<ast::Ty>, sym| {
-                            if let TyKind::Path(None, ast::Path { segments, .. }) = &ty.kind &&
-                                let [seg] = segments.as_slice() &&
-                                seg.ident.name == sym && seg.args.is_none()
+                            if let TyKind::Path(None, ast::Path { segments, .. }) = &ty.kind
+                                && let [seg] = segments.as_slice()
+                                && seg.ident.name == sym
+                                && seg.args.is_none()
                             {
                                 true
                             } else {
@@ -1613,8 +1614,8 @@ impl<'a> TraitDef<'a> {
                             }
                         };
 
-                        let exception = if let TyKind::Slice(ty) = &struct_field.ty.kind &&
-                            is_simple_path(ty, sym::u8)
+                        let exception = if let TyKind::Slice(ty) = &struct_field.ty.kind
+                            && is_simple_path(ty, sym::u8)
                         {
                             Some("byte")
                         } else if is_simple_path(&struct_field.ty, sym::str) {
@@ -1624,14 +1625,14 @@ impl<'a> TraitDef<'a> {
                         };
 
                         if let Some(ty) = exception {
-                            cx.sess.parse_sess.buffer_lint_with_diagnostic(
+                            cx.sess.psess.buffer_lint_with_diagnostic(
                                 BYTE_SLICE_IN_PACKED_STRUCT_WITH_DERIVE,
                                 sp,
                                 ast::CRATE_NODE_ID,
                                 format!(
                                     "{ty} slice in a packed struct that derives a built-in trait"
                                 ),
-                                rustc_lint_defs::BuiltinLintDiagnostics::ByteSliceInPackedStructWithDerive
+                                rustc_lint_defs::BuiltinLintDiag::ByteSliceInPackedStructWithDerive,
                             );
                         } else {
                             // Wrap the expression in `{...}`, causing a copy.

compiler/rustc_builtin_macros/src/env.rs

@@ -39,7 +39,7 @@ pub fn expand_option_env<'cx>(
 
     let sp = cx.with_def_site_ctxt(sp);
     let value = lookup_env(cx, var);
-    cx.sess.parse_sess.env_depinfo.borrow_mut().insert((var, value));
+    cx.sess.psess.env_depinfo.borrow_mut().insert((var, value));
     let e = match value {
         None => {
             let lt = cx.lifetime(sp, Ident::new(kw::StaticLifetime, sp));
@@ -94,7 +94,7 @@ pub fn expand_env<'cx>(
 
     let span = cx.with_def_site_ctxt(sp);
     let value = lookup_env(cx, var);
-    cx.sess.parse_sess.env_depinfo.borrow_mut().insert((var, value));
+    cx.sess.psess.env_depinfo.borrow_mut().insert((var, value));
     let e = match value {
         None => {
             let ExprKind::Lit(token::Lit {

compiler/rustc_builtin_macros/src/errors.rs

@@ -1,6 +1,6 @@
 use rustc_errors::{
     codes::*, AddToDiagnostic, Diag, DiagCtxt, EmissionGuarantee, IntoDiagnostic, Level, MultiSpan,
-    SingleLabelManySpans, SubdiagnosticMessageOp,
+    SingleLabelManySpans, SubdiagMessageOp,
 };
 use rustc_macros::{Diagnostic, Subdiagnostic};
 use rustc_span::{symbol::Ident, Span, Symbol};
@@ -590,7 +590,7 @@ pub(crate) struct FormatUnusedArg {
 // Allow the singular form to be a subdiagnostic of the multiple-unused
 // form of diagnostic.
 impl AddToDiagnostic for FormatUnusedArg {
-    fn add_to_diagnostic_with<G: EmissionGuarantee, F: SubdiagnosticMessageOp<G>>(
+    fn add_to_diagnostic_with<G: EmissionGuarantee, F: SubdiagMessageOp<G>>(
         self,
         diag: &mut Diag<'_, G>,
         f: F,
@@ -766,6 +766,13 @@ pub(crate) struct AsmNoReturn {
     pub(crate) outputs_sp: Vec<Span>,
 }
 
+#[derive(Diagnostic)]
+#[diag(builtin_macros_asm_mayunwind)]
+pub(crate) struct AsmMayUnwind {
+    #[primary_span]
+    pub(crate) labels_sp: Vec<Span>,
+}
+
 #[derive(Diagnostic)]
 #[diag(builtin_macros_global_asm_clobber_abi)]
 pub(crate) struct GlobalAsmClobberAbi {

compiler/rustc_builtin_macros/src/format.rs

@@ -16,7 +16,7 @@ use rustc_span::symbol::{Ident, Symbol};
 use rustc_span::{BytePos, ErrorGuaranteed, InnerSpan, Span};
 
 use rustc_lint_defs::builtin::NAMED_ARGUMENTS_USED_POSITIONALLY;
-use rustc_lint_defs::{BufferedEarlyLint, BuiltinLintDiagnostics, LintId};
+use rustc_lint_defs::{BufferedEarlyLint, BuiltinLintDiag, LintId};
 
 // The format_args!() macro is expanded in three steps:
 //  1. First, `parse_args` will parse the `(literal, arg, arg, name=arg, name=arg)` syntax,
@@ -553,7 +553,7 @@ fn make_format_args(
                 msg: format!("named argument `{}` is not used by name", arg_name.name).into(),
                 node_id: rustc_ast::CRATE_NODE_ID,
                 lint_id: LintId::of(NAMED_ARGUMENTS_USED_POSITIONALLY),
-                diagnostic: BuiltinLintDiagnostics::NamedArgumentUsedPositionally {
+                diagnostic: BuiltinLintDiag::NamedArgumentUsedPositionally {
                     position_sp_to_replace,
                     position_sp_for_msg,
                     named_arg_sp: arg_name.span,

compiler/rustc_builtin_macros/src/source_util.rs

@@ -118,7 +118,7 @@ pub fn expand_include<'cx>(
             return DummyResult::any(sp, guar);
         }
     };
-    let p = new_parser_from_file(cx.parse_sess(), &file, Some(sp));
+    let p = new_parser_from_file(cx.psess(), &file, Some(sp));
 
     // If in the included file we have e.g., `mod bar;`,
     // then the path of `bar.rs` should be relative to the directory of `file`.
@@ -136,7 +136,7 @@ pub fn expand_include<'cx>(
         fn make_expr(mut self: Box<ExpandResult<'a>>) -> Option<P<ast::Expr>> {
             let expr = parse_expr(&mut self.p).ok()?;
             if self.p.token != token::Eof {
-                self.p.sess.buffer_lint(
+                self.p.psess.buffer_lint(
                     INCOMPLETE_INCLUDE,
                     self.p.token.span,
                     self.node_id,

compiler/rustc_builtin_macros/src/standard_library_imports.rs

@@ -17,7 +17,7 @@ pub fn inject(
     features: &Features,
 ) -> usize {
     let orig_num_items = krate.items.len();
-    let edition = sess.parse_sess.edition;
+    let edition = sess.psess.edition;
 
     // the first name in this list is the crate name of the crate with the prelude
     let names: &[Symbol] = if attr::contains_name(pre_configured_attrs, sym::no_core) {

compiler/rustc_builtin_macros/src/test_harness.rs

@@ -159,7 +159,7 @@ struct InnerItemLinter<'a> {
 impl<'a> Visitor<'a> for InnerItemLinter<'_> {
     fn visit_item(&mut self, i: &'a ast::Item) {
         if let Some(attr) = attr::find_by_name(&i.attrs, sym::rustc_test_marker) {
-            self.sess.parse_sess.buffer_lint(
+            self.sess.psess.buffer_lint(
                 UNNAMEABLE_TEST_ITEMS,
                 attr.span,
                 i.id,
@@ -200,7 +200,7 @@ impl<'a> MutVisitor for EntryPointCleaner<'a> {
             EntryPointType::MainNamed | EntryPointType::RustcMainAttr | EntryPointType::Start => {
                 item.map(|ast::Item { id, ident, attrs, kind, vis, span, tokens }| {
                     let allow_dead_code = attr::mk_attr_nested_word(
-                        &self.sess.parse_sess.attr_id_generator,
+                        &self.sess.psess.attr_id_generator,
                         ast::AttrStyle::Outer,
                         sym::allow,
                         sym::dead_code,

compiler/rustc_builtin_macros/src/util.rs

@@ -9,7 +9,7 @@ pub fn check_builtin_macro_attribute(ecx: &ExtCtxt<'_>, meta_item: &MetaItem, na
     // All the built-in macro attributes are "words" at the moment.
     let template = AttributeTemplate { word: true, ..Default::default() };
     validate_attr::check_builtin_meta_item(
-        &ecx.sess.parse_sess,
+        &ecx.sess.psess,
         meta_item,
         AttrStyle::Outer,
         name,
@@ -37,7 +37,7 @@ pub fn warn_on_duplicate_attribute(ecx: &ExtCtxt<'_>, item: &Annotatable, name:
     };
     if let Some(attrs) = attrs {
         if let Some(attr) = attr::find_by_name(attrs, name) {
-            ecx.parse_sess().buffer_lint(
+            ecx.psess().buffer_lint(
                 DUPLICATE_MACRO_ATTRIBUTES,
                 attr.span,
                 ecx.current_expansion.lint_node_id,

compiler/rustc_codegen_cranelift/.github/workflows/main.yml

@@ -44,9 +44,10 @@ jobs:
             env:
               TARGET_TRIPLE: x86_64-apple-darwin
           # cross-compile from Linux to Windows using mingw
-          - os: ubuntu-latest
-            env:
-              TARGET_TRIPLE: x86_64-pc-windows-gnu
+          # FIXME The wine version in Ubuntu 22.04 is missing ProcessPrng
+          #- os: ubuntu-latest
+          #  env:
+          #    TARGET_TRIPLE: x86_64-pc-windows-gnu
           - os: ubuntu-latest
             env:
               TARGET_TRIPLE: aarch64-unknown-linux-gnu
@@ -80,11 +81,11 @@ jobs:
       if: matrix.os == 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
       run: rustup set default-host x86_64-pc-windows-gnu
 
-    - name: Install MinGW toolchain and wine
-      if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
+    #- name: Install MinGW toolchain and wine
+    #  if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+    #  run: |
+    #    sudo apt-get update
+    #    sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
 
     - name: Install AArch64 toolchain and qemu
       if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'aarch64-unknown-linux-gnu'

compiler/rustc_codegen_cranelift/.gitignore

@@ -1,18 +1,21 @@
-/target
-/build_system/target
-**/*.rs.bk
-*.rlib
-*.o
-perf.data
-perf.data.old
-*.events
-*.string*
+# Build artifacts during normal use
 /y.bin
 /y.bin.dSYM
 /y.exe
 /y.pdb
+/download
 /build
 /dist
+/target
+/build_system/target
+
+# Downloaded by certain scripts
 /rust
-/download
 /git-fixed-subtree.sh
+
+# Various things that can be created during development
+*.rlib
+*.o
+perf.data
+perf.data.old
+*.mm_profdata

compiler/rustc_codegen_cranelift/Cargo.lock

@@ -46,18 +46,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d819feeda4c420a18f1e28236ca0ce1177b22bf7c8a44ddee92dfe40de15bcf0"
+checksum = "9515fcc42b6cb5137f76b84c1a6f819782d0cf12473d145d3bc5cd67eedc8bc2"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9b8d03d5bdbca7e5f72b0e0a0f69933ed1f09e24be6c075aa6fe3f802b0cc0c"
+checksum = "1ad827c6071bfe6d22de1bc331296a29f9ddc506ff926d8415b435ec6a6efce0"
 dependencies = [
  "bumpalo",
  "cranelift-bforest",
@@ -76,39 +76,39 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3fd3664e38e51649b17dc30cfdd561273fe2f590dcd013fb75d9eabc6272dfb"
+checksum = "10e6b36237a9ca2ce2fb4cc7741d418a080afa1327402138412ef85d5367bef1"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b031ec5e605828975952622b5a77d49126f20ffe88d33719a0af66b23a0fc36"
+checksum = "c36bf4bfb86898a94ccfa773a1f86e8a5346b1983ff72059bdd2db4600325251"
 
 [[package]]
 name = "cranelift-control"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fada054d017cf2ed8f7ed2336e0517fc1b19e6825be1790de9eb00c94788362b"
+checksum = "7cbf36560e7a6bd1409ca91e7b43b2cc7ed8429f343d7605eadf9046e8fac0d0"
 dependencies = [
  "arbitrary",
 ]
 
 [[package]]
 name = "cranelift-entity"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "177b6f94ae8de6348eb45bf977c79ab9e3c40fc3ac8cb7ed8109560ea39bee7d"
+checksum = "a71e11061a75b1184c09bea97c026a88f08b59ade96a7bb1f259d4ea0df2e942"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebebd23a69a23e3ddea78e98ff3a2de222e88c8e045d81ef4a72f042e0d79dbd"
+checksum = "af5d4da63143ee3485c7bcedde0a818727d737d1083484a0ceedb8950c89e495"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -118,15 +118,15 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1571bfc14df8966d12c6121b5325026591a4b4009e22fea0fe3765ab7cd33b96"
+checksum = "457a9832b089e26f5eea70dcf49bed8ec6edafed630ce7c83161f24d46ab8085"
 
 [[package]]
 name = "cranelift-jit"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f61e236d7622c3c43016e8b0f3ba27136e21ac7de328c7fda902e61db1de851"
+checksum = "0af95fe68d5a10919012c8db82b1d59820405b8001c8c6d05f94b08031334fa9"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -144,9 +144,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f30c6820342015c5009070e3e48d1da7b13521399de904663f1c84f5ee839657"
+checksum = "11b0b201fa10a4014062d4c56c307c8d18fdf9a84cb5279efe6080241f42c7a7"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -155,9 +155,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35a69c37e0c10b46fe5527f2397ac821046efbf5f7ec112c8b84df25712f465b"
+checksum = "9b490d579df1ce365e1ea359e24ed86d82289fa785153327c2f6a69a59a731e4"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -166,9 +166,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24425a329b4343177d5f1852243841dcec17f929d72c0e7f41262140155e55e7"
+checksum = "fb7e821ac6db471bcdbd004e5a4fa0d374f1046bd3a2ce278c332e0b0c01ca63"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -241,9 +241,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.148"
+version = "0.2.153"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b"
+checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
 
 [[package]]
 name = "libloading"
@@ -410,9 +410,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "17.0.0"
+version = "18.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bdc26415bb89e9ccd3bdc498fef63aabf665c4c0dd710c107691deb9694955da"
+checksum = "33f4121cb29dda08139b2824a734dd095d83ce843f2d613a84eb580b9cfc17ac"
 dependencies = [
  "cfg-if",
  "libc",

compiler/rustc_codegen_cranelift/Cargo.toml

@@ -8,12 +8,12 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.104", default-features = false, features = ["std", "unwind", "all-arch"] }
-cranelift-frontend = { version = "0.104" }
-cranelift-module = { version = "0.104" }
-cranelift-native = { version = "0.104" }
-cranelift-jit = { version = "0.104", optional = true }
-cranelift-object = { version = "0.104" }
+cranelift-codegen = { version = "0.105.2", default-features = false, features = ["std", "unwind", "all-arch"] }
+cranelift-frontend = { version = "0.105.2" }
+cranelift-module = { version = "0.105.2" }
+cranelift-native = { version = "0.105.2" }
+cranelift-jit = { version = "0.105.2", optional = true }
+cranelift-object = { version = "0.105.2" }
 target-lexicon = "0.12.0"
 gimli = { version = "0.28", default-features = false, features = ["write"]}
 object = { version = "0.32", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }

compiler/rustc_codegen_cranelift/Readme.md

@@ -123,11 +123,6 @@ You need to do this steps to successfully compile and use the cranelift backend
 
 You can also set `rust-analyzer.rustc.source` to your rust workspace to get rust-analyzer to understand your changes.
 
-## Configuration
-
-See the documentation on the `BackendConfig` struct in [config.rs](src/config.rs) for all
-configuration options.
-
 ## Not yet supported
 
 * SIMD ([tracked here](https://github.com/rust-lang/rustc_codegen_cranelift/issues/171), `std::simd` fully works, `std::arch` is partially supported)

compiler/rustc_codegen_cranelift/build_system/prepare.rs

@@ -1,5 +1,6 @@
 use std::ffi::OsStr;
 use std::fs;
+use std::hash::{Hash, Hasher};
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
@@ -71,7 +72,11 @@ fn hash_file(file: &std::path::Path) -> u64 {
     let contents = std::fs::read(file).unwrap();
     #[allow(deprecated)]
     let mut hasher = std::hash::SipHasher::new();
-    std::hash::Hash::hash(&contents, &mut hasher);
+    // The following is equivalent to
+    //   std::hash::Hash::hash(&contents, &mut hasher);
+    // but gives the same result independent of host byte order.
+    hasher.write_usize(contents.len().to_le());
+    Hash::hash_slice(&contents, &mut hasher);
     std::hash::Hasher::finish(&hasher)
 }
 
@@ -80,16 +85,26 @@ fn hash_dir(dir: &std::path::Path) -> u64 {
     for entry in std::fs::read_dir(dir).unwrap() {
         let entry = entry.unwrap();
         if entry.file_type().unwrap().is_dir() {
-            sub_hashes
-                .insert(entry.file_name().to_str().unwrap().to_owned(), hash_dir(&entry.path()));
+            sub_hashes.insert(
+                entry.file_name().to_str().unwrap().to_owned(),
+                hash_dir(&entry.path()).to_le(),
+            );
         } else {
-            sub_hashes
-                .insert(entry.file_name().to_str().unwrap().to_owned(), hash_file(&entry.path()));
+            sub_hashes.insert(
+                entry.file_name().to_str().unwrap().to_owned(),
+                hash_file(&entry.path()).to_le(),
+            );
         }
     }
     #[allow(deprecated)]
     let mut hasher = std::hash::SipHasher::new();
-    std::hash::Hash::hash(&sub_hashes, &mut hasher);
+    // The following is equivalent to
+    //   std::hash::Hash::hash(&sub_hashes, &mut hasher);
+    // but gives the same result independent of host byte order.
+    hasher.write_usize(sub_hashes.len().to_le());
+    for elt in sub_hashes {
+        elt.hash(&mut hasher);
+    }
     std::hash::Hasher::finish(&hasher)
 }
 

compiler/rustc_codegen_cranelift/build_system/tests.rs

@@ -133,8 +133,8 @@ pub(crate) static REGEX: CargoProject = CargoProject::new(&REGEX_REPO.source_dir
 pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github(
     "rust-lang",
     "portable-simd",
-    "97007cc2e70df8c97326ce896a79e2f0ce4dd98b",
-    "e54a16035cedf205",
+    "5794c837bc605c4cd9dbb884285976dfdb293cce",
+    "a64d8fdd0ed0d9c4",
     "portable-simd",
 );
 

compiler/rustc_codegen_cranelift/example/mini_core.rs

@@ -525,8 +525,11 @@ pub struct Unique<T: ?Sized> {
 impl<T: ?Sized, U: ?Sized> CoerceUnsized<Unique<U>> for Unique<T> where T: Unsize<U> {}
 impl<T: ?Sized, U: ?Sized> DispatchFromDyn<Unique<U>> for Unique<T> where T: Unsize<U> {}
 
+#[lang = "global_alloc_ty"]
+pub struct Global;
+
 #[lang = "owned_box"]
-pub struct Box<T: ?Sized, A = ()>(Unique<T>, A);
+pub struct Box<T: ?Sized, A = Global>(Unique<T>, A);
 
 impl<T: ?Sized + Unsize<U>, U: ?Sized> CoerceUnsized<Box<U>> for Box<T> {}
 
@@ -536,7 +539,7 @@ impl<T> Box<T> {
             let size = intrinsics::size_of::<T>();
             let ptr = libc::malloc(size);
             intrinsics::copy(&val as *const T as *const u8, ptr, size);
-            Box(Unique { pointer: NonNull(ptr as *const T), _marker: PhantomData }, ())
+            Box(Unique { pointer: NonNull(ptr as *const T), _marker: PhantomData }, Global)
         }
     }
 }

compiler/rustc_codegen_cranelift/patches/0022-coretests-Disable-not-compiling-tests.patch

@@ -39,6 +39,6 @@ index 42a26ae..5ac1042 100644
 +#![cfg(test)]
  #![feature(alloc_layout_extra)]
  #![feature(array_chunks)]
- #![feature(array_methods)]
+ #![feature(array_windows)]
 --
 2.21.0 (Apple Git-122)

compiler/rustc_codegen_cranelift/patches/0023-coretests-Ignore-failing-tests.patch

@@ -1,26 +0,0 @@
-From dd82e95c9de212524e14fc60155de1ae40156dfc Mon Sep 17 00:00:00 2001
-From: bjorn3 <bjorn3@users.noreply.github.com>
-Date: Sun, 24 Nov 2019 15:34:06 +0100
-Subject: [PATCH] [core] Ignore failing tests
-
----
- library/core/tests/iter.rs       |  4 ++++
- library/core/tests/num/bignum.rs | 10 ++++++++++
- library/core/tests/num/mod.rs    |  5 +++--
- library/core/tests/time.rs       |  1 +
- 4 files changed, 18 insertions(+), 2 deletions(-)
-
-diff --git a/atomic.rs b/atomic.rs
-index 13b12db..96fe4b9 100644
---- a/atomic.rs
-+++ b/atomic.rs
-@@ -185,6 +185,7 @@ fn ptr_bitops() {
- }
- 
- #[test]
-+#[cfg_attr(target_arch = "s390x", ignore)] // s390x backend doesn't support stack alignment >8 bytes
- #[cfg(any(not(target_arch = "arm"), target_os = "linux"))] // Missing intrinsic in compiler-builtins
- fn ptr_bitops_tagging() {
-     #[repr(align(16))]
--- 
-2.21.0 (Apple Git-122)

compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml

@@ -150,9 +150,9 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.3.3"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
+checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-alloc",
@@ -161,9 +161,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.150"
+version = "0.2.153"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c"
+checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
 dependencies = [
  "rustc-std-workspace-core",
 ]
@@ -398,6 +398,7 @@ version = "0.0.0"
 dependencies = [
  "core",
  "getopts",
+ "libc",
  "panic_abort",
  "panic_unwind",
  "std",

compiler/rustc_codegen_cranelift/rust-toolchain

@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2024-01-26"
+channel = "nightly-2024-03-08"
 components = ["rust-src", "rustc-dev", "llvm-tools"]

compiler/rustc_codegen_cranelift/rustfmt.toml

@@ -1,5 +1,4 @@
 ignore = [
-    "y.rs",
     "example/gen_block_iterate.rs", # uses edition 2024
 ]
 

compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh

@@ -10,12 +10,26 @@ pushd rust
 
 command -v rg >/dev/null 2>&1 || cargo install ripgrep
 
+# FIXME(rust-lang/rust#122196) fix stage0 rmake.rs run-make tests and remove
+# this workaround
+for test in $(ls tests/run-make); do
+  if [[ -e "tests/run-make/$test/rmake.rs" ]]; then
+    rm -r "tests/run-make/$test"
+  fi
+done
+
+# FIXME remove this workaround once ICE tests no longer emit an outdated nightly message
+for test in $(rg -i --files-with-matches "//@(\[.*\])? failure-status: 101" tests/ui); do
+  echo "rm $test"
+  rm $test
+done
+
 rm -r tests/ui/{unsized-locals/,lto/,linkage*} || true
 for test in $(rg --files-with-matches "lto" tests/{codegen-units,ui,incremental}); do
   rm $test
 done
 
-for test in $(rg -i --files-with-matches "//(\[\w+\])?~[^\|]*\s*ERR|// error-pattern:|// build-fail|// run-fail|-Cllvm-args" tests/ui); do
+for test in $(rg -i --files-with-matches "//(\[\w+\])?~[^\|]*\s*ERR|//@ error-pattern:|//@(\[.*\])? build-fail|//@(\[.*\])? run-fail|-Cllvm-args" tests/ui); do
   rm $test
 done
 
@@ -43,8 +57,8 @@ rm tests/ui/proc-macro/allowed-signatures.rs
 rm tests/ui/proc-macro/no-mangle-in-proc-macro-issue-111888.rs
 
 # vendor intrinsics
-rm tests/ui/sse2.rs # CodegenBackend::target_features not yet implemented
 rm tests/ui/simd/array-type.rs # "Index argument for `simd_insert` is not a constant"
+rm tests/ui/asm/x86_64/evex512-implicit-feature.rs # unimplemented AVX512 x86 vendor intrinsic
 
 # exotic linkages
 rm tests/ui/issues/issue-33992.rs # unsupported linkages
@@ -62,14 +76,12 @@ rm -r tests/run-pass-valgrind/unsized-locals
 # misc unimplemented things
 rm tests/ui/intrinsics/intrinsic-nearby.rs # unimplemented nearbyintf32 and nearbyintf64 intrinsics
 rm tests/ui/target-feature/missing-plusminus.rs # error not implemented
-rm tests/ui/fn/dyn-fn-alignment.rs # wants a 256 byte alignment
 rm -r tests/run-make/emit-named-files # requires full --emit support
 rm -r tests/run-make/repr128-dwarf # debuginfo test
 rm -r tests/run-make/split-debuginfo # same
 rm -r tests/run-make/symbols-include-type-name # --emit=asm not supported
 rm -r tests/run-make/target-specs # i686 not supported by Cranelift
 rm -r tests/run-make/mismatching-target-triples # same
-rm tests/ui/asm/x86_64/issue-82869.rs # vector regs in inline asm not yet supported
 rm tests/ui/asm/x86_64/issue-96797.rs # const and sym inline asm operands don't work entirely correctly
 
 # requires LTO
@@ -109,8 +121,6 @@ rm -r tests/run-make/optimization-remarks-dir # remarks are LLVM specific
 rm tests/ui/mir/mir_misc_casts.rs # depends on deduplication of constants
 rm tests/ui/mir/mir_raw_fat_ptr.rs # same
 rm tests/ui/consts/issue-33537.rs # same
-rm tests/ui/layout/valid_range_oob.rs # different ICE message
-rm tests/ui/const-generics/generic_const_exprs/issue-80742.rs # gives error instead of ICE with cg_clif
 
 # rustdoc-clif passes extra args, suppressing the help message when no args are passed
 rm -r tests/run-make/issue-88756-default-output
@@ -119,15 +129,12 @@ rm -r tests/run-make/issue-88756-default-output
 # should work when using ./x.py test the way it is intended
 # ============================================================
 rm -r tests/run-make/remap-path-prefix-dwarf # requires llvm-dwarfdump
-rm -r tests/ui/consts/missing_span_in_backtrace.rs # expects sysroot source to be elsewhere
 
 # genuine bugs
 # ============
 rm tests/incremental/spike-neg1.rs # errors out for some reason
 rm tests/incremental/spike-neg2.rs # same
 
-rm tests/ui/simd/simd-bitmask.rs # simd_bitmask doesn't implement [u*; N] return type
-
 rm -r tests/run-make/issue-51671 # wrong filename given in case of --emit=obj
 rm -r tests/run-make/issue-30063 # same
 rm -r tests/run-make/multiple-emits # same
@@ -145,6 +152,7 @@ rm tests/ui/codegen/subtyping-enforces-type-equality.rs # assert_assignable bug
 # ======================
 rm tests/ui/backtrace.rs # TODO warning
 rm tests/ui/process/nofile-limit.rs # TODO some AArch64 linking issue
+rm tests/ui/async-await/async-closures/once.rs # FIXME bug in the rustc FnAbi calculation code
 
 rm tests/ui/stdio-is-blocking.rs # really slow with unoptimized libstd
 

compiler/rustc_codegen_cranelift/src/base.rs

@@ -445,7 +445,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                 template,
                 operands,
                 options,
-                destination,
+                targets,
                 line_spans: _,
                 unwind: _,
             } => {
@@ -456,13 +456,25 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                     );
                 }
 
+                let have_labels = if options.contains(InlineAsmOptions::NORETURN) {
+                    !targets.is_empty()
+                } else {
+                    targets.len() > 1
+                };
+                if have_labels {
+                    fx.tcx.dcx().span_fatal(
+                        source_info.span,
+                        "cranelift doesn't support labels in inline assembly.",
+                    );
+                }
+
                 crate::inline_asm::codegen_inline_asm_terminator(
                     fx,
                     source_info.span,
                     template,
                     operands,
                     *options,
-                    *destination,
+                    targets.get(0).copied(),
                 );
             }
             TerminatorKind::UnwindTerminate(reason) => {

compiler/rustc_codegen_cranelift/src/common.rs

@@ -392,18 +392,25 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
     }
 
     pub(crate) fn create_stack_slot(&mut self, size: u32, align: u32) -> Pointer {
-        if align <= 16 {
+        let abi_align = if self.tcx.sess.target.arch == "s390x" { 8 } else { 16 };
+        if align <= abi_align {
             let stack_slot = self.bcx.create_sized_stack_slot(StackSlotData {
                 kind: StackSlotKind::ExplicitSlot,
-                // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
-                // specify stack slot alignment.
-                size: (size + 15) / 16 * 16,
+                // FIXME Don't force the size to a multiple of <abi_align> bytes once Cranelift gets
+                // a way to specify stack slot alignment.
+                size: (size + abi_align - 1) / abi_align * abi_align,
             });
             Pointer::stack_slot(stack_slot)
         } else {
             // Alignment is too big to handle using the above hack. Dynamically realign a stack slot
             // instead. This wastes some space for the realignment.
-            let base_ptr = self.create_stack_slot(size + align, 16).get_addr(self);
+            let stack_slot = self.bcx.create_sized_stack_slot(StackSlotData {
+                kind: StackSlotKind::ExplicitSlot,
+                // FIXME Don't force the size to a multiple of <abi_align> bytes once Cranelift gets
+                // a way to specify stack slot alignment.
+                size: (size + align) / abi_align * abi_align,
+            });
+            let base_ptr = self.bcx.ins().stack_addr(self.pointer_type, stack_slot, 0);
             let misalign_offset = self.bcx.ins().urem_imm(base_ptr, i64::from(align));
             let realign_offset = self.bcx.ins().irsub_imm(misalign_offset, i64::from(align));
             Pointer::new(self.bcx.ins().iadd(base_ptr, realign_offset))

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -372,7 +372,13 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
         }
 
         let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec();
-        data.define(bytes.into_boxed_slice());
+        if bytes.is_empty() {
+            // FIXME(bytecodealliance/wasmtime#7918) cranelift-jit has a bug where it causes UB on
+            // empty data objects
+            data.define(Box::new([0]));
+        } else {
+            data.define(bytes.into_boxed_slice());
+        }
 
         for &(offset, prov) in alloc.provenance().ptrs().iter() {
             let alloc_id = prov.alloc_id();

compiler/rustc_codegen_cranelift/src/global_asm.rs

@@ -78,7 +78,8 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String,
                         InlineAsmOperand::In { .. }
                         | InlineAsmOperand::Out { .. }
                         | InlineAsmOperand::InOut { .. }
-                        | InlineAsmOperand::SplitInOut { .. } => {
+                        | InlineAsmOperand::SplitInOut { .. }
+                        | InlineAsmOperand::Label { .. } => {
                             span_bug!(op_sp, "invalid operand type for global_asm!")
                         }
                     }

compiler/rustc_codegen_cranelift/src/inline_asm.rs

@@ -129,6 +129,9 @@ pub(crate) fn codegen_inline_asm_terminator<'tcx>(
                 let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
                 CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance).name.to_owned() }
             }
+            InlineAsmOperand::Label { .. } => {
+                span_bug!(span, "asm! label operands are not yet supported");
+            }
         })
         .collect::<Vec<_>>();
 

compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs

@@ -170,6 +170,65 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             }
         }
 
+        "llvm.x86.sse.add.ss" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_add_ss&ig_expand=171
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            assert_eq!(a.layout(), b.layout());
+            assert_eq!(a.layout(), ret.layout());
+            let layout = a.layout();
+
+            let (_, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+            assert!(lane_ty.is_floating_point());
+            let ret_lane_layout = fx.layout_of(lane_ty);
+
+            ret.write_cvalue(fx, a);
+
+            let a_lane = a.value_lane(fx, 0).load_scalar(fx);
+            let b_lane = b.value_lane(fx, 0).load_scalar(fx);
+
+            let res = fx.bcx.ins().fadd(a_lane, b_lane);
+
+            let res_lane = CValue::by_val(res, ret_lane_layout);
+            ret.place_lane(fx, 0).write_cvalue(fx, res_lane);
+        }
+
+        "llvm.x86.sse.sqrt.ps" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sqrt_ps&ig_expand=6245
+            intrinsic_args!(fx, args => (a); intrinsic);
+
+            // FIXME use vector instructions when possible
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| {
+                fx.bcx.ins().sqrt(lane)
+            });
+        }
+
+        "llvm.x86.sse.max.ps" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_ps&ig_expand=4357
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            simd_pair_for_each_lane(
+                fx,
+                a,
+                b,
+                ret,
+                &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| fx.bcx.ins().fmax(a_lane, b_lane),
+            );
+        }
+
+        "llvm.x86.sse.min.ps" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_ps&ig_expand=4489
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            simd_pair_for_each_lane(
+                fx,
+                a,
+                b,
+                ret,
+                &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| fx.bcx.ins().fmin(a_lane, b_lane),
+            );
+        }
+
         "llvm.x86.sse.cmp.ps" | "llvm.x86.sse2.cmp.pd" => {
             let (x, y, kind) = match args {
                 [x, y, kind] => (x, y, kind),
@@ -1067,6 +1126,122 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             );
         }
 
+        "llvm.x86.sha1rnds4" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha1rnds4_epu32&ig_expand=5877
+            intrinsic_args!(fx, args => (a, b, _func); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            let func = if let Some(func) =
+                crate::constant::mir_operand_get_const_val(fx, &args[2].node)
+            {
+                func
+            } else {
+                fx.tcx
+                    .dcx()
+                    .span_fatal(span, "Func argument for `_mm_sha1rnds4_epu32` is not a constant");
+            };
+
+            let func = func.try_to_u8().unwrap_or_else(|_| panic!("kind not scalar: {:?}", func));
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String(format!("sha1rnds4 xmm1, xmm2, {func}"))],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)),
+                        value: b,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.sha1msg1" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha1msg1_epu32&ig_expand=5874
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String("sha1msg1 xmm1, xmm2".to_string())],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)),
+                        value: b,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.sha1msg2" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha1msg2_epu32&ig_expand=5875
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String("sha1msg2 xmm1, xmm2".to_string())],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)),
+                        value: b,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.sha1nexte" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha1nexte_epu32&ig_expand=5876
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String("sha1nexte xmm1, xmm2".to_string())],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)),
+                        value: b,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
         "llvm.x86.sha256rnds2" => {
             // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha256rnds2_epu32&ig_expand=5977
             intrinsic_args!(fx, args => (a, b, k); intrinsic);

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

@@ -391,12 +391,15 @@ fn codegen_float_intrinsic_call<'tcx>(
         | sym::ceilf32
         | sym::ceilf64
         | sym::truncf32
-        | sym::truncf64 => {
+        | sym::truncf64
+        | sym::sqrtf32
+        | sym::sqrtf64 => {
             let val = match intrinsic {
                 sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(args[0]),
                 sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(args[0]),
                 sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(args[0]),
                 sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(args[0]),
+                sym::sqrtf32 | sym::sqrtf64 => fx.bcx.ins().sqrt(args[0]),
                 _ => unreachable!(),
             };
 
@@ -1255,7 +1258,17 @@ fn codegen_regular_intrinsic_call<'tcx>(
 
         // Unimplemented intrinsics must have a fallback body. The fallback body is obtained
         // by converting the `InstanceDef::Intrinsic` to an `InstanceDef::Item`.
-        _ => return Err(Instance::new(instance.def_id(), instance.args)),
+        _ => {
+            let intrinsic = fx.tcx.intrinsic(instance.def_id()).unwrap();
+            if intrinsic.must_be_overridden {
+                span_bug!(
+                    source_info.span,
+                    "intrinsic {} must be overridden by codegen_cranelift, but isn't",
+                    intrinsic.name,
+                );
+            }
+            return Err(Instance::new(instance.def_id(), instance.args));
+        }
     }
 
     let ret_block = fx.get_block(destination.unwrap());

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -853,7 +853,13 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             };
 
             for lane in 0..lane_count {
-                let m_lane = fx.bcx.ins().ushr_imm(m, u64::from(lane) as i64);
+                // The bit order of the mask depends on the byte endianness, LSB-first for
+                // little endian and MSB-first for big endian.
+                let mask_lane = match fx.tcx.sess.target.endian {
+                    Endian::Big => lane_count - 1 - lane,
+                    Endian::Little => lane,
+                };
+                let m_lane = fx.bcx.ins().ushr_imm(m, u64::from(mask_lane) as i64);
                 let m_lane = fx.bcx.ins().band_imm(m_lane, 1);
                 let a_lane = a.value_lane(fx, lane).load_scalar(fx);
                 let b_lane = b.value_lane(fx, lane).load_scalar(fx);

compiler/rustc_codegen_cranelift/src/unsize.rs

@@ -74,10 +74,6 @@ fn unsize_ptr<'tcx>(
         | (&ty::RawPtr(ty::TypeAndMut { ty: a, .. }), &ty::RawPtr(ty::TypeAndMut { ty: b, .. })) => {
             (src, unsized_info(fx, *a, *b, old_info))
         }
-        (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) if def_a.is_box() && def_b.is_box() => {
-            let (a, b) = (src_layout.ty.boxed_ty(), dst_layout.ty.boxed_ty());
-            (src, unsized_info(fx, a, b, old_info))
-        }
         (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => {
             assert_eq!(def_a, def_b);
 

compiler/rustc_codegen_cranelift/y.rs

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-#![deny(unsafe_code)] /*This line is ignored by bash
-# This block is ignored by rustc
-echo "Warning: y.rs is a deprecated alias for y.sh" 1>&2
-exec ./y.sh "$@"
-*/

compiler/rustc_codegen_gcc/example/mini_core.rs

@@ -472,6 +472,7 @@ pub trait Allocator {
 
 impl Allocator for () {}
 
+#[lang = "global_alloc_ty"]
 pub struct Global;
 
 impl Allocator for Global {}

compiler/rustc_codegen_gcc/src/asm.rs

@@ -114,7 +114,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         options: InlineAsmOptions,
         span: &[Span],
         instance: Instance<'_>,
-        _dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
+        _catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
     ) {
         if options.contains(InlineAsmOptions::MAY_UNWIND) {
             self.sess().dcx().create_err(UnwindingInlineAsm { span: span[0] }).emit();
@@ -132,6 +132,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         // added to `outputs.len()`
         let mut inputs = vec![];
 
+        // GCC index of a label equals its position in the array added to
+        // `outputs.len() + inputs.len()`.
+        let mut labels = vec![];
+
         // Clobbers collected from `out("explicit register") _` and `inout("expl_reg") var => _`
         let mut clobbers = vec![];
 
@@ -283,6 +287,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                     constants_len +=
                         self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len();
                 }
+
+                InlineAsmOperandRef::Label { label } => {
+                    labels.push(label);
+                }
             }
         }
 
@@ -381,6 +389,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                 InlineAsmOperandRef::Const { .. } => {
                     // processed in the previous pass
                 }
+
+                InlineAsmOperandRef::Label { .. } => {
+                    // processed in the previous pass
+                }
             }
         }
 
@@ -470,6 +482,14 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                         InlineAsmOperandRef::Const { ref string } => {
                             template_str.push_str(string);
                         }
+
+                        InlineAsmOperandRef::Label { label } => {
+                            let label_gcc_index = labels.iter()
+                                .position(|&l| l == label)
+                                .expect("wrong rust index");
+                            let gcc_index = label_gcc_index + outputs.len() + inputs.len();
+                            push_to_template(Some('l'), gcc_index);
+                        }
                     }
                 }
             }
@@ -482,7 +502,12 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         // 4. Generate Extended Asm block
 
         let block = self.llbb();
-        let extended_asm = block.add_extended_asm(None, &template_str);
+        let extended_asm = if let Some(dest) = dest {
+            assert!(!labels.is_empty());
+            block.end_with_extended_asm_goto(None, &template_str, &labels, Some(dest))
+        } else {
+            block.add_extended_asm(None, &template_str)
+        };
 
         for op in &outputs {
             extended_asm.add_output_operand(None, &op.to_constraint(), op.tmp_var);
@@ -510,7 +535,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         if !options.contains(InlineAsmOptions::NOSTACK) {
             // TODO(@Commeownist): figure out how to align stack
         }
-        if options.contains(InlineAsmOptions::NORETURN) {
+        if dest.is_none() && options.contains(InlineAsmOptions::NORETURN) {
             let builtin_unreachable = self.context.get_builtin_function("__builtin_unreachable");
             let builtin_unreachable: RValue<'gcc> =
                 unsafe { std::mem::transmute(builtin_unreachable) };

compiler/rustc_codegen_llvm/src/asm.rs

@@ -28,7 +28,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         options: InlineAsmOptions,
         line_spans: &[Span],
         instance: Instance<'_>,
-        dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
+        dest: Option<Self::BasicBlock>,
+        catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
     ) {
         let asm_arch = self.tcx.sess.asm_arch.unwrap();
 
@@ -165,6 +166,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         }
 
         // Build the template string
+        let mut labels = vec![];
         let mut template_str = String::new();
         for piece in template {
             match *piece {
@@ -205,6 +207,11 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                             // Only emit the raw symbol name
                             template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx]));
                         }
+                        InlineAsmOperandRef::Label { label } => {
+                            template_str.push_str(&format!("${{{}:l}}", constraints.len()));
+                            constraints.push("!i".to_owned());
+                            labels.push(label);
+                        }
                     }
                 }
             }
@@ -292,12 +299,14 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             &constraints.join(","),
             &inputs,
             output_type,
+            &labels,
             volatile,
             alignstack,
             dialect,
             line_spans,
             options.contains(InlineAsmOptions::MAY_UNWIND),
-            dest_catch_funclet,
+            dest,
+            catch_funclet,
         )
         .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
 
@@ -317,7 +326,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs });
 
         // Switch to the 'normal' basic block if we did an `invoke` instead of a `call`
-        if let Some((dest, _, _)) = dest_catch_funclet {
+        if let Some(dest) = dest {
             self.switch_to_block(dest);
         }
 
@@ -415,16 +424,14 @@ pub(crate) fn inline_asm_call<'ll>(
     cons: &str,
     inputs: &[&'ll Value],
     output: &'ll llvm::Type,
+    labels: &[&'ll llvm::BasicBlock],
     volatile: bool,
     alignstack: bool,
     dia: llvm::AsmDialect,
     line_spans: &[Span],
     unwind: bool,
-    dest_catch_funclet: Option<(
-        &'ll llvm::BasicBlock,
-        &'ll llvm::BasicBlock,
-        Option<&Funclet<'ll>>,
-    )>,
+    dest: Option<&'ll llvm::BasicBlock>,
+    catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>,
 ) -> Option<&'ll Value> {
     let volatile = if volatile { llvm::True } else { llvm::False };
     let alignstack = if alignstack { llvm::True } else { llvm::False };
@@ -457,8 +464,11 @@ pub(crate) fn inline_asm_call<'ll>(
                 can_throw,
             );
 
-            let call = if let Some((dest, catch, funclet)) = dest_catch_funclet {
-                bx.invoke(fty, None, None, v, inputs, dest, catch, funclet)
+            let call = if !labels.is_empty() {
+                assert!(catch_funclet.is_none());
+                bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None)
+            } else if let Some((catch, funclet)) = catch_funclet {
+                bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet)
             } else {
                 bx.call(fty, None, None, v, inputs, None)
             };

compiler/rustc_codegen_llvm/src/back/archive.rs

@@ -55,6 +55,7 @@ fn llvm_machine_type(cpu: &str) -> LLVMMachineType {
         "x86_64" => LLVMMachineType::AMD64,
         "x86" => LLVMMachineType::I386,
         "aarch64" => LLVMMachineType::ARM64,
+        "arm64ec" => LLVMMachineType::ARM64EC,
         "arm" => LLVMMachineType::ARM,
         _ => panic!("unsupported cpu type {cpu}"),
     }

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1538,6 +1538,58 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         }
     }
 
+    pub(crate) fn callbr(
+        &mut self,
+        llty: &'ll Type,
+        fn_attrs: Option<&CodegenFnAttrs>,
+        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
+        llfn: &'ll Value,
+        args: &[&'ll Value],
+        default_dest: &'ll BasicBlock,
+        indirect_dest: &[&'ll BasicBlock],
+        funclet: Option<&Funclet<'ll>>,
+    ) -> &'ll Value {
+        debug!("invoke {:?} with args ({:?})", llfn, args);
+
+        let args = self.check_call("callbr", llty, llfn, args);
+        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
+        let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw);
+        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
+        if let Some(funclet_bundle) = funclet_bundle {
+            bundles.push(funclet_bundle);
+        }
+
+        // Emit CFI pointer type membership test
+        self.cfi_type_test(fn_attrs, fn_abi, llfn);
+
+        // Emit KCFI operand bundle
+        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn);
+        let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw);
+        if let Some(kcfi_bundle) = kcfi_bundle {
+            bundles.push(kcfi_bundle);
+        }
+
+        let callbr = unsafe {
+            llvm::LLVMRustBuildCallBr(
+                self.llbuilder,
+                llty,
+                llfn,
+                default_dest,
+                indirect_dest.as_ptr(),
+                indirect_dest.len() as c_uint,
+                args.as_ptr(),
+                args.len() as c_uint,
+                bundles.as_ptr(),
+                bundles.len() as c_uint,
+                UNNAMED,
+            )
+        };
+        if let Some(fn_abi) = fn_abi {
+            fn_abi.apply_attrs_callsite(self, callbr);
+        }
+        callbr
+    }
+
     // Emits CFI pointer type membership tests.
     fn cfi_type_test(
         &mut self,

compiler/rustc_codegen_llvm/src/context.rs

@@ -260,35 +260,29 @@ pub unsafe fn create_module<'ll>(
     }
 
     if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection {
-        let behavior = if llvm_version >= (15, 0, 0) {
-            llvm::LLVMModFlagBehavior::Min
-        } else {
-            llvm::LLVMModFlagBehavior::Error
-        };
-
         if sess.target.arch == "aarch64" {
             llvm::LLVMRustAddModuleFlag(
                 llmod,
-                behavior,
+                llvm::LLVMModFlagBehavior::Min,
                 c"branch-target-enforcement".as_ptr().cast(),
                 bti.into(),
             );
             llvm::LLVMRustAddModuleFlag(
                 llmod,
-                behavior,
+                llvm::LLVMModFlagBehavior::Min,
                 c"sign-return-address".as_ptr().cast(),
                 pac_ret.is_some().into(),
             );
             let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A });
             llvm::LLVMRustAddModuleFlag(
                 llmod,
-                behavior,
+                llvm::LLVMModFlagBehavior::Min,
                 c"sign-return-address-all".as_ptr().cast(),
                 pac_opts.leaf.into(),
             );
             llvm::LLVMRustAddModuleFlag(
                 llmod,
-                behavior,
+                llvm::LLVMModFlagBehavior::Min,
                 c"sign-return-address-with-bkey".as_ptr().cast(),
                 u32::from(pac_opts.key == PAuthKey::B),
             );

compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs

@@ -454,9 +454,13 @@ pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll D
         ty::RawPtr(ty::TypeAndMut { ty: pointee_type, .. }) | ty::Ref(_, pointee_type, _) => {
             build_pointer_or_reference_di_node(cx, t, pointee_type, unique_type_id)
         }
-        // Box<T, A> may have a non-1-ZST allocator A. In that case, we
-        // cannot treat Box<T, A> as just an owned alias of `*mut T`.
-        ty::Adt(def, args) if def.is_box() && cx.layout_of(args.type_at(1)).is_1zst() => {
+        // Some `Box` are newtyped pointers, make debuginfo aware of that.
+        // Only works if the allocator argument is a 1-ZST and hence irrelevant for layout
+        // (or if there is no allocator argument).
+        ty::Adt(def, args)
+            if def.is_box()
+                && args.get(1).map_or(true, |arg| cx.layout_of(arg.expect_ty()).is_1zst()) =>
+        {
             build_pointer_or_reference_di_node(cx, t, t.boxed_ty(), unique_type_id)
         }
         ty::FnDef(..) | ty::FnPtr(_) => build_subroutine_type_di_node(cx, unique_type_id),

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -448,12 +448,14 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     constraint,
                     inputs,
                     self.type_void(),
+                    &[],
                     true,
                     false,
                     llvm::AsmDialect::Att,
                     &[span],
                     false,
                     None,
+                    None,
                 )
                 .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`"));
 

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -56,6 +56,7 @@ pub enum LLVMMachineType {
     AMD64 = 0x8664,
     I386 = 0x14c,
     ARM64 = 0xaa64,
+    ARM64EC = 0xa641,
     ARM = 0x01c0,
 }
 
@@ -1615,6 +1616,20 @@ extern "C" {
         Name: *const c_char,
     ) -> &'a Value;
 
+    pub fn LLVMRustBuildCallBr<'a>(
+        B: &Builder<'a>,
+        Ty: &'a Type,
+        Fn: &'a Value,
+        DefaultDest: &'a BasicBlock,
+        IndirectDests: *const &'a BasicBlock,
+        NumIndirectDests: c_uint,
+        Args: *const &'a Value,
+        NumArgs: c_uint,
+        OpBundles: *const &OperandBundleDef<'a>,
+        NumOpBundles: c_uint,
+        Name: *const c_char,
+    ) -> &'a Value;
+
     pub fn LLVMRustSetFastMath(Instr: &Value);
     pub fn LLVMRustSetAlgebraicMath(Instr: &Value);
     pub fn LLVMRustSetAllowReassoc(Instr: &Value);

compiler/rustc_codegen_llvm/src/llvm_util.rs

@@ -201,7 +201,13 @@ impl<'a> IntoIterator for LLVMFeature<'a> {
 // which might lead to failures if the oldest tested / supported LLVM version
 // doesn't yet support the relevant intrinsics
 pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> {
-    let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch };
+    let arch = if sess.target.arch == "x86_64" {
+        "x86"
+    } else if sess.target.arch == "arm64ec" {
+        "aarch64"
+    } else {
+        &*sess.target.arch
+    };
     match (arch, s) {
         ("x86", "sse4.2") => {
             LLVMFeature::with_dependency("sse4.2", TargetFeatureFoldStrength::EnableOnly("crc32"))

compiler/rustc_codegen_llvm/src/va_arg.rs

@@ -288,7 +288,7 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
         // Generic x86
         "x86" => emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), true),
         // Windows AArch64
-        "aarch64" if target.is_like_windows => {
+        "aarch64" | "arm64ec" if target.is_like_windows => {
             emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), false)
         }
         // macOS / iOS AArch64

compiler/rustc_codegen_ssa/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 # tidy-alphabetical-start
 ar_archive_writer = "0.1.5"
 bitflags = "2.4.1"
-cc = "1.0.69"
+cc = "1.0.90"
 itertools = "0.11"
 jobserver = "0.1.28"
 pathdiff = "0.2.0"

compiler/rustc_codegen_ssa/src/assert_module_sources.rs

@@ -176,7 +176,7 @@ impl<'tcx> AssertModuleSource<'tcx> {
     /// Scan for a `cfg="foo"` attribute and check whether we have a
     /// cfg flag called `foo`.
     fn check_config(&self, attr: &ast::Attribute) -> bool {
-        let config = &self.tcx.sess.parse_sess.config;
+        let config = &self.tcx.sess.psess.config;
         let value = self.field(attr, sym::cfg);
         debug!("check_config(config={:?}, value={:?})", config, value);
         if config.iter().any(|&(name, _)| name == value) {

compiler/rustc_codegen_ssa/src/back/metadata.rs

@@ -8,7 +8,7 @@ use std::path::Path;
 use object::write::{self, StandardSegment, Symbol, SymbolSection};
 use object::{
     elf, pe, xcoff, Architecture, BinaryFormat, Endianness, FileFlags, Object, ObjectSection,
-    ObjectSymbol, SectionFlags, SectionKind, SymbolFlags, SymbolKind, SymbolScope,
+    ObjectSymbol, SectionFlags, SectionKind, SubArchitecture, SymbolFlags, SymbolKind, SymbolScope,
 };
 
 use rustc_data_structures::memmap::Mmap;
@@ -182,37 +182,40 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static
         Endian::Little => Endianness::Little,
         Endian::Big => Endianness::Big,
     };
-    let architecture = match &sess.target.arch[..] {
-        "arm" => Architecture::Arm,
-        "aarch64" => {
+    let (architecture, sub_architecture) = match &sess.target.arch[..] {
+        "arm" => (Architecture::Arm, None),
+        "aarch64" => (
             if sess.target.pointer_width == 32 {
                 Architecture::Aarch64_Ilp32
             } else {
                 Architecture::Aarch64
-            }
-        }
-        "x86" => Architecture::I386,
-        "s390x" => Architecture::S390x,
-        "mips" | "mips32r6" => Architecture::Mips,
-        "mips64" | "mips64r6" => Architecture::Mips64,
-        "x86_64" => {
+            },
+            None,
+        ),
+        "x86" => (Architecture::I386, None),
+        "s390x" => (Architecture::S390x, None),
+        "mips" | "mips32r6" => (Architecture::Mips, None),
+        "mips64" | "mips64r6" => (Architecture::Mips64, None),
+        "x86_64" => (
             if sess.target.pointer_width == 32 {
                 Architecture::X86_64_X32
             } else {
                 Architecture::X86_64
-            }
-        }
-        "powerpc" => Architecture::PowerPc,
-        "powerpc64" => Architecture::PowerPc64,
-        "riscv32" => Architecture::Riscv32,
-        "riscv64" => Architecture::Riscv64,
-        "sparc64" => Architecture::Sparc64,
-        "avr" => Architecture::Avr,
-        "msp430" => Architecture::Msp430,
-        "hexagon" => Architecture::Hexagon,
-        "bpf" => Architecture::Bpf,
-        "loongarch64" => Architecture::LoongArch64,
-        "csky" => Architecture::Csky,
+            },
+            None,
+        ),
+        "powerpc" => (Architecture::PowerPc, None),
+        "powerpc64" => (Architecture::PowerPc64, None),
+        "riscv32" => (Architecture::Riscv32, None),
+        "riscv64" => (Architecture::Riscv64, None),
+        "sparc64" => (Architecture::Sparc64, None),
+        "avr" => (Architecture::Avr, None),
+        "msp430" => (Architecture::Msp430, None),
+        "hexagon" => (Architecture::Hexagon, None),
+        "bpf" => (Architecture::Bpf, None),
+        "loongarch64" => (Architecture::LoongArch64, None),
+        "csky" => (Architecture::Csky, None),
+        "arm64ec" => (Architecture::Aarch64, Some(SubArchitecture::Arm64EC)),
         // Unsupported architecture.
         _ => return None,
     };
@@ -227,6 +230,7 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static
     };
 
     let mut file = write::Object::new(binary_format, architecture, endianness);
+    file.set_sub_architecture(sub_architecture);
     if sess.target.is_like_osx {
         if macho_is_arm64e(&sess.target) {
             file.set_macho_cpu_subtype(object::macho::CPU_SUBTYPE_ARM64E);
@@ -335,7 +339,7 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static
                 "ilp32s" | "lp64s" => e_flags |= elf::EF_LARCH_ABI_SOFT_FLOAT,
                 "ilp32f" | "lp64f" => e_flags |= elf::EF_LARCH_ABI_SINGLE_FLOAT,
                 "ilp32d" | "lp64d" => e_flags |= elf::EF_LARCH_ABI_DOUBLE_FLOAT,
-                _ => bug!("unknown RISC-V ABI name"),
+                _ => bug!("unknown LoongArch ABI name"),
             }
 
             e_flags

compiler/rustc_codegen_ssa/src/back/symbol_export.rs

@@ -81,6 +81,10 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, _: LocalCrate) -> DefIdMap<S
                 return library.kind.is_statically_included().then_some(def_id);
             }
 
+            if tcx.intrinsic(def_id).is_some_and(|i| i.must_be_overridden) {
+                return None;
+            }
+
             // Only consider nodes that actually have exported symbols.
             match tcx.def_kind(def_id) {
                 DefKind::Fn | DefKind::Static(_) => {}
@@ -563,9 +567,10 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>(
         return undecorated;
     }
 
-    let x86 = match &target.arch[..] {
-        "x86" => true,
-        "x86_64" => false,
+    let prefix = match &target.arch[..] {
+        "x86" => Some('_'),
+        "x86_64" => None,
+        "arm64ec" => Some('#'),
         // Only x86/64 use symbol decorations.
         _ => return undecorated,
     };
@@ -602,8 +607,8 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>(
         Conv::X86Stdcall => ("_", "@"),
         Conv::X86VectorCall => ("", "@@"),
         _ => {
-            if x86 {
-                undecorated.insert(0, '_');
+            if let Some(prefix) = prefix {
+                undecorated.insert(0, prefix);
             }
             return undecorated;
         }

compiler/rustc_codegen_ssa/src/back/write.rs

@@ -16,8 +16,8 @@ use rustc_data_structures::sync::Lrc;
 use rustc_errors::emitter::Emitter;
 use rustc_errors::translation::Translate;
 use rustc_errors::{
-    Diag, DiagArgMap, DiagCtxt, DiagnosticMessage, ErrCode, FatalError, FluentBundle, Level,
-    MultiSpan, Style,
+    Diag, DiagArgMap, DiagCtxt, DiagMessage, ErrCode, FatalError, FluentBundle, Level, MultiSpan,
+    Style,
 };
 use rustc_fs_util::link_or_copy;
 use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
@@ -1004,9 +1004,9 @@ pub(crate) enum Message<B: WriteBackendMethods> {
 /// process another codegen unit.
 pub struct CguMessage;
 
-// A cut-down version of `rustc_errors::Diagnostic` that impls `Send`, which
+// A cut-down version of `rustc_errors::DiagInner` that impls `Send`, which
 // can be used to send diagnostics from codegen threads to the main thread.
-// It's missing the following fields from `rustc_errors::Diagnostic`.
+// It's missing the following fields from `rustc_errors::DiagInner`.
 // - `span`: it doesn't impl `Send`.
 // - `suggestions`: it doesn't impl `Send`, and isn't used for codegen
 //   diagnostics.
@@ -1015,18 +1015,18 @@ pub struct CguMessage;
 // - `emitted_at`: not used for codegen diagnostics.
 struct Diagnostic {
     level: Level,
-    messages: Vec<(DiagnosticMessage, Style)>,
+    messages: Vec<(DiagMessage, Style)>,
     code: Option<ErrCode>,
     children: Vec<Subdiagnostic>,
     args: DiagArgMap,
 }
 
-// A cut-down version of `rustc_errors::SubDiagnostic` that impls `Send`. It's
-// missing the following fields from `rustc_errors::SubDiagnostic`.
+// A cut-down version of `rustc_errors::Subdiag` that impls `Send`. It's
+// missing the following fields from `rustc_errors::Subdiag`.
 // - `span`: it doesn't impl `Send`.
 pub struct Subdiagnostic {
     level: Level,
-    messages: Vec<(DiagnosticMessage, Style)>,
+    messages: Vec<(DiagMessage, Style)>,
 }
 
 #[derive(PartialEq, Clone, Copy, Debug)]

compiler/rustc_codegen_ssa/src/base.rs

@@ -510,11 +510,13 @@ fn get_argc_argv<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         // Params for UEFI
         let param_handle = bx.get_param(0);
         let param_system_table = bx.get_param(1);
+        let ptr_size = bx.tcx().data_layout.pointer_size;
+        let ptr_align = bx.tcx().data_layout.pointer_align.abi;
         let arg_argc = bx.const_int(cx.type_isize(), 2);
-        let arg_argv = bx.alloca(cx.type_array(cx.type_ptr(), 2), Align::ONE);
-        bx.store(param_handle, arg_argv, Align::ONE);
-        let arg_argv_el1 = bx.gep(cx.type_ptr(), arg_argv, &[bx.const_int(cx.type_int(), 1)]);
-        bx.store(param_system_table, arg_argv_el1, Align::ONE);
+        let arg_argv = bx.alloca(cx.type_array(cx.type_ptr(), 2), ptr_align);
+        bx.store(param_handle, arg_argv, ptr_align);
+        let arg_argv_el1 = bx.inbounds_ptradd(arg_argv, bx.const_usize(ptr_size.bytes()));
+        bx.store(param_system_table, arg_argv_el1, ptr_align);
         (arg_argc, arg_argv)
     } else if cx.sess().target.main_needs_argc_argv {
         // Params from native `main()` used as args for rust start function
@@ -907,7 +909,11 @@ impl CrateInfo {
                     lang_items::required(tcx, l).then_some(name)
                 })
                 .collect();
-            let prefix = if target.is_like_windows && target.arch == "x86" { "_" } else { "" };
+            let prefix = match (target.is_like_windows, target.arch.as_ref()) {
+                (true, "x86") => "_",
+                (true, "arm64ec") => "#",
+                _ => "",
+            };
 
             // This loop only adds new items to values of the hash map, so the order in which we
             // iterate over the values is not important.

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -12,12 +12,12 @@ use crate::MemFlags;
 use rustc_ast as ast;
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_hir::lang_items::LangItem;
-use rustc_middle::mir::{self, AssertKind, SwitchTargets, UnwindTerminateReason};
+use rustc_middle::mir::{self, AssertKind, BasicBlock, SwitchTargets, UnwindTerminateReason};
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement};
 use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 use rustc_middle::ty::{self, Instance, Ty};
 use rustc_session::config::OptLevel;
-use rustc_span::{source_map::Spanned, sym, Span, Symbol};
+use rustc_span::{source_map::Spanned, sym, Span};
 use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode, Reg};
 use rustc_target::abi::{self, HasDataLayout, WrappingRange};
 use rustc_target::spec::abi::Abi;
@@ -264,7 +264,8 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
             mir::UnwindAction::Unreachable => None,
         };
 
-        if let Some(cleanup) = unwind_target {
+        if operands.iter().any(|x| matches!(x, InlineAsmOperandRef::Label { .. })) {
+            assert!(unwind_target.is_none());
             let ret_llbb = if let Some(target) = destination {
                 fx.llbb(target)
             } else {
@@ -277,11 +278,29 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
                 options,
                 line_spans,
                 instance,
-                Some((ret_llbb, cleanup, self.funclet(fx))),
+                Some(ret_llbb),
+                None,
+            );
+            MergingSucc::False
+        } else if let Some(cleanup) = unwind_target {
+            let ret_llbb = if let Some(target) = destination {
+                fx.llbb(target)
+            } else {
+                fx.unreachable_block()
+            };
+
+            bx.codegen_inline_asm(
+                template,
+                operands,
+                options,
+                line_spans,
+                instance,
+                Some(ret_llbb),
+                Some((cleanup, self.funclet(fx))),
             );
             MergingSucc::False
         } else {
-            bx.codegen_inline_asm(template, operands, options, line_spans, instance, None);
+            bx.codegen_inline_asm(template, operands, options, line_spans, instance, None, None);
 
             if let Some(target) = destination {
                 self.funclet_br(fx, bx, target, mergeable_succ)
@@ -680,7 +699,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         &mut self,
         helper: &TerminatorCodegenHelper<'tcx>,
         bx: &mut Bx,
-        intrinsic: Option<Symbol>,
+        intrinsic: Option<ty::IntrinsicDef>,
         instance: Option<Instance<'tcx>>,
         source_info: mir::SourceInfo,
         target: Option<mir::BasicBlock>,
@@ -690,7 +709,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // Emit a panic or a no-op for `assert_*` intrinsics.
         // These are intrinsics that compile to panics so that we can get a message
         // which mentions the offending type, even from a const context.
-        let panic_intrinsic = intrinsic.and_then(|s| ValidityRequirement::from_intrinsic(s));
+        let panic_intrinsic = intrinsic.and_then(|i| ValidityRequirement::from_intrinsic(i.name));
         if let Some(requirement) = panic_intrinsic {
             let ty = instance.unwrap().args.type_at(0);
 
@@ -826,14 +845,20 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // The arguments we'll be passing. Plus one to account for outptr, if used.
         let arg_count = fn_abi.args.len() + fn_abi.ret.is_indirect() as usize;
 
-        if intrinsic == Some(sym::caller_location) {
+        if matches!(intrinsic, Some(ty::IntrinsicDef { name: sym::caller_location, .. })) {
             return if let Some(target) = target {
                 let location =
                     self.get_caller_location(bx, mir::SourceInfo { span: fn_span, ..source_info });
 
                 let mut llargs = Vec::with_capacity(arg_count);
-                let ret_dest =
-                    self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs, true, true);
+                let ret_dest = self.make_return_dest(
+                    bx,
+                    destination,
+                    &fn_abi.ret,
+                    &mut llargs,
+                    intrinsic,
+                    Some(target),
+                );
                 assert_eq!(llargs, []);
                 if let ReturnDest::IndirectOperand(tmp, _) = ret_dest {
                     location.val.store(bx, tmp);
@@ -846,7 +871,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         }
 
         let instance = match intrinsic {
-            None | Some(sym::drop_in_place) => instance,
+            None => instance,
             Some(intrinsic) => {
                 let mut llargs = Vec::with_capacity(1);
                 let ret_dest = self.make_return_dest(
@@ -854,8 +879,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     destination,
                     &fn_abi.ret,
                     &mut llargs,
-                    true,
-                    target.is_some(),
+                    Some(intrinsic),
+                    target,
                 );
                 let dest = match ret_dest {
                     _ if fn_abi.ret.is_indirect() => llargs[0],
@@ -873,7 +898,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         // The indices passed to simd_shuffle in the
                         // third argument must be constant. This is
                         // checked by the type-checker.
-                        if i == 2 && intrinsic == sym::simd_shuffle {
+                        if i == 2 && intrinsic.name == sym::simd_shuffle {
                             if let mir::Operand::Constant(constant) = &arg.node {
                                 let (llval, ty) = self.simd_shuffle_indices(bx, constant);
                                 return OperandRef {
@@ -903,14 +928,33 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             MergingSucc::False
                         };
                     }
-                    Err(instance) => Some(instance),
+                    Err(instance) => {
+                        if intrinsic.must_be_overridden {
+                            span_bug!(
+                                span,
+                                "intrinsic {} must be overridden by codegen backend, but isn't",
+                                intrinsic.name,
+                            );
+                        }
+                        Some(instance)
+                    }
                 }
             }
         };
 
         let mut llargs = Vec::with_capacity(arg_count);
         let destination = target.as_ref().map(|&target| {
-            (self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs, false, true), target)
+            (
+                self.make_return_dest(
+                    bx,
+                    destination,
+                    &fn_abi.ret,
+                    &mut llargs,
+                    None,
+                    Some(target),
+                ),
+                target,
+            )
         });
 
         // Split the rust-call tupled arguments off.
@@ -1075,7 +1119,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         operands: &[mir::InlineAsmOperand<'tcx>],
         options: ast::InlineAsmOptions,
         line_spans: &[Span],
-        destination: Option<mir::BasicBlock>,
+        targets: &[mir::BasicBlock],
         unwind: mir::UnwindAction,
         instance: Instance<'_>,
         mergeable_succ: bool,
@@ -1127,6 +1171,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 mir::InlineAsmOperand::SymStatic { def_id } => {
                     InlineAsmOperandRef::SymStatic { def_id }
                 }
+                mir::InlineAsmOperand::Label { target_index } => {
+                    InlineAsmOperandRef::Label { label: self.llbb(targets[target_index]) }
+                }
             })
             .collect();
 
@@ -1137,7 +1184,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             &operands,
             options,
             line_spans,
-            destination,
+            if options.contains(InlineAsmOptions::NORETURN) {
+                None
+            } else {
+                targets.get(0).copied()
+            },
             unwind,
             instance,
             mergeable_succ,
@@ -1293,7 +1344,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 ref operands,
                 options,
                 line_spans,
-                destination,
+                ref targets,
                 unwind,
             } => self.codegen_asm_terminator(
                 helper,
@@ -1303,7 +1354,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 operands,
                 options,
                 line_spans,
-                destination,
+                targets,
                 unwind,
                 self.instance,
                 mergeable_succ(),
@@ -1643,10 +1694,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         dest: mir::Place<'tcx>,
         fn_ret: &ArgAbi<'tcx, Ty<'tcx>>,
         llargs: &mut Vec<Bx::Value>,
-        is_intrinsic: bool,
-        has_target: bool,
+        intrinsic: Option<ty::IntrinsicDef>,
+        target: Option<BasicBlock>,
     ) -> ReturnDest<'tcx, Bx::Value> {
-        if !has_target {
+        if target.is_none() {
             return ReturnDest::Nothing;
         }
         // If the return is ignored, we can just return a do-nothing `ReturnDest`.
@@ -1667,7 +1718,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         tmp.storage_live(bx);
                         llargs.push(tmp.llval);
                         ReturnDest::IndirectOperand(tmp, index)
-                    } else if is_intrinsic {
+                    } else if intrinsic.is_some() {
                         // Currently, intrinsics always need a location to store
                         // the result, so we create a temporary `alloca` for the
                         // result.

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -204,6 +204,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 
     pub fn deref<Cx: LayoutTypeMethods<'tcx>>(self, cx: &Cx) -> PlaceRef<'tcx, V> {
         if self.layout.ty.is_box() {
+            // Derefer should have removed all Box derefs
             bug!("dereferencing {:?} in codegen", self.layout.ty);
         }
 

compiler/rustc_codegen_ssa/src/mir/place.rs

@@ -104,10 +104,6 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
         let mut simple = || {
             let llval = if offset.bytes() == 0 {
                 self.llval
-            } else if field.is_zst() {
-                // FIXME(erikdesjardins): it should be fine to use inbounds for ZSTs too;
-                // keeping this logic for now to preserve previous behavior.
-                bx.ptradd(self.llval, bx.const_usize(offset.bytes()))
             } else {
                 bx.inbounds_ptradd(self.llval, bx.const_usize(offset.bytes()))
             };
@@ -168,8 +164,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
         debug!("struct_field_ptr: DST field offset: {:?}", offset);
 
         // Adjust pointer.
-        // FIXME(erikdesjardins): should be able to use inbounds here too.
-        let ptr = bx.ptradd(self.llval, offset);
+        let ptr = bx.inbounds_ptradd(self.llval, offset);
 
         PlaceRef { llval: ptr, llextra: self.llextra, layout: field, align: effective_field_align }
     }

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -94,8 +94,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
 
                 if let OperandValue::Immediate(v) = cg_elem.val {
-                    let zero = bx.const_usize(0);
-                    let start = dest.project_index(bx, zero).llval;
+                    let start = dest.llval;
                     let size = bx.const_usize(dest.layout.size.bytes());
 
                     // Use llvm.memset.p0i8.* to initialize all zero arrays

compiler/rustc_codegen_ssa/src/mono_item.rs

@@ -76,7 +76,8 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
                             hir::InlineAsmOperand::In { .. }
                             | hir::InlineAsmOperand::Out { .. }
                             | hir::InlineAsmOperand::InOut { .. }
-                            | hir::InlineAsmOperand::SplitInOut { .. } => {
+                            | hir::InlineAsmOperand::SplitInOut { .. }
+                            | hir::InlineAsmOperand::Label { .. } => {
                                 span_bug!(*op_sp, "invalid operand type for global_asm!")
                             }
                         })

compiler/rustc_codegen_ssa/src/traits/asm.rs

@@ -33,6 +33,9 @@ pub enum InlineAsmOperandRef<'tcx, B: BackendTypes + ?Sized> {
     SymStatic {
         def_id: DefId,
     },
+    Label {
+        label: B::BasicBlock,
+    },
 }
 
 #[derive(Debug)]
@@ -51,7 +54,8 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes {
         options: InlineAsmOptions,
         line_spans: &[Span],
         instance: Instance<'_>,
-        dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
+        dest: Option<Self::BasicBlock>,
+        catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
     );
 }
 

compiler/rustc_const_eval/messages.ftl

@@ -232,9 +232,6 @@ const_eval_non_const_fn_call =
 const_eval_non_const_impl =
     impl defined here, but it is not `const`
 
-const_eval_noreturn_asm_returned =
-    returned from noreturn inline assembly
-
 const_eval_not_enough_caller_args =
     calling a function with fewer arguments than it requires
 

compiler/rustc_const_eval/src/const_eval/error.rs

@@ -1,8 +1,6 @@
 use std::mem;
 
-use rustc_errors::{
-    DiagArgName, DiagArgValue, DiagnosticMessage, IntoDiagnostic, IntoDiagnosticArg,
-};
+use rustc_errors::{DiagArgName, DiagArgValue, DiagMessage, IntoDiagnostic, IntoDiagnosticArg};
 use rustc_hir::CRATE_HIR_ID;
 use rustc_middle::mir::AssertKind;
 use rustc_middle::query::TyCtxtAt;
@@ -25,7 +23,7 @@ pub enum ConstEvalErrKind {
 }
 
 impl MachineStopType for ConstEvalErrKind {
-    fn diagnostic_message(&self) -> DiagnosticMessage {
+    fn diagnostic_message(&self) -> DiagMessage {
         use crate::fluent_generated::*;
         use ConstEvalErrKind::*;
         match self {

compiler/rustc_const_eval/src/const_eval/machine.rs

@@ -24,7 +24,7 @@ use crate::errors::{LongRunning, LongRunningWarn};
 use crate::fluent_generated as fluent;
 use crate::interpret::{
     self, compile_time_machine, AllocId, AllocRange, ConstAllocation, CtfeProvenance, FnArg, FnVal,
-    Frame, ImmTy, InterpCx, InterpResult, OpTy, PlaceTy, Pointer, PointerArithmetic, Scalar,
+    Frame, ImmTy, InterpCx, InterpResult, MPlaceTy, OpTy, Pointer, PointerArithmetic, Scalar,
 };
 
 use super::error::*;
@@ -219,7 +219,7 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
         &mut self,
         instance: ty::Instance<'tcx>,
         args: &[FnArg<'tcx>],
-        dest: &PlaceTy<'tcx>,
+        dest: &MPlaceTy<'tcx>,
         ret: Option<mir::BasicBlock>,
     ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
         let def_id = instance.def_id();
@@ -227,7 +227,7 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
         if self.tcx.has_attr(def_id, sym::rustc_const_panic_str)
             || Some(def_id) == self.tcx.lang_items().begin_panic_fn()
         {
-            let args = self.copy_fn_args(args)?;
+            let args = self.copy_fn_args(args);
             // &str or &&str
             assert!(args.len() == 1);
 
@@ -254,7 +254,7 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
 
             return Ok(Some(new_instance));
         } else if Some(def_id) == self.tcx.lang_items().align_offset_fn() {
-            let args = self.copy_fn_args(args)?;
+            let args = self.copy_fn_args(args);
             // For align_offset, we replace the function call if the pointer has no address.
             match self.align_offset(instance, &args, dest, ret)? {
                 ControlFlow::Continue(()) => return Ok(Some(instance)),
@@ -280,7 +280,7 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
         &mut self,
         instance: ty::Instance<'tcx>,
         args: &[OpTy<'tcx>],
-        dest: &PlaceTy<'tcx>,
+        dest: &MPlaceTy<'tcx>,
         ret: Option<mir::BasicBlock>,
     ) -> InterpResult<'tcx, ControlFlow<()>> {
         assert_eq!(args.len(), 2);
@@ -410,7 +410,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
         orig_instance: ty::Instance<'tcx>,
         _abi: CallAbi,
         args: &[FnArg<'tcx>],
-        dest: &PlaceTy<'tcx>,
+        dest: &MPlaceTy<'tcx>,
         ret: Option<mir::BasicBlock>,
         _unwind: mir::UnwindAction, // unwinding is not supported in consts
     ) -> InterpResult<'tcx, Option<(&'mir mir::Body<'tcx>, ty::Instance<'tcx>)>> {
@@ -455,7 +455,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
         ecx: &mut InterpCx<'mir, 'tcx, Self>,
         instance: ty::Instance<'tcx>,
         args: &[OpTy<'tcx>],
-        dest: &PlaceTy<'tcx, Self::Provenance>,
+        dest: &MPlaceTy<'tcx, Self::Provenance>,
         target: Option<mir::BasicBlock>,
         _unwind: mir::UnwindAction,
     ) -> InterpResult<'tcx> {

compiler/rustc_const_eval/src/errors.rs

@@ -1,8 +1,7 @@
 use std::borrow::Cow;
 
 use rustc_errors::{
-    codes::*, Diag, DiagArgValue, DiagCtxt, DiagnosticMessage, EmissionGuarantee, IntoDiagnostic,
-    Level,
+    codes::*, Diag, DiagArgValue, DiagCtxt, DiagMessage, EmissionGuarantee, IntoDiagnostic, Level,
 };
 use rustc_hir::ConstContext;
 use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
@@ -424,7 +423,7 @@ pub struct UndefinedBehavior {
 
 pub trait ReportErrorExt {
     /// Returns the diagnostic message for this error.
-    fn diagnostic_message(&self) -> DiagnosticMessage;
+    fn diagnostic_message(&self) -> DiagMessage;
     fn add_args<G: EmissionGuarantee>(self, diag: &mut Diag<'_, G>);
 
     fn debug(self) -> String
@@ -433,7 +432,7 @@ pub trait ReportErrorExt {
     {
         ty::tls::with(move |tcx| {
             let dcx = tcx.dcx();
-            let mut diag = dcx.struct_allow(DiagnosticMessage::Str(String::new().into()));
+            let mut diag = dcx.struct_allow(DiagMessage::Str(String::new().into()));
             let message = self.diagnostic_message();
             self.add_args(&mut diag);
             let s = dcx.eagerly_translate_to_string(message, diag.args.iter());
@@ -457,7 +456,7 @@ fn bad_pointer_message(msg: CheckInAllocMsg, dcx: &DiagCtxt) -> String {
 }
 
 impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> {
-    fn diagnostic_message(&self) -> DiagnosticMessage {
+    fn diagnostic_message(&self) -> DiagMessage {
         use crate::fluent_generated::*;
         use UndefinedBehaviorInfo::*;
         match self {
@@ -595,7 +594,7 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> {
 }
 
 impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
-    fn diagnostic_message(&self) -> DiagnosticMessage {
+    fn diagnostic_message(&self) -> DiagMessage {
         use crate::fluent_generated::*;
         use rustc_middle::mir::interpret::ValidationErrorKind::*;
         match self.kind {
@@ -783,7 +782,7 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
 }
 
 impl ReportErrorExt for UnsupportedOpInfo {
-    fn diagnostic_message(&self) -> DiagnosticMessage {
+    fn diagnostic_message(&self) -> DiagMessage {
         use crate::fluent_generated::*;
         match self {
             UnsupportedOpInfo::Unsupported(s) => s.clone().into(),
@@ -819,7 +818,7 @@ impl ReportErrorExt for UnsupportedOpInfo {
 }
 
 impl<'tcx> ReportErrorExt for InterpError<'tcx> {
-    fn diagnostic_message(&self) -> DiagnosticMessage {
+    fn diagnostic_message(&self) -> DiagMessage {
         match self {
             InterpError::UndefinedBehavior(ub) => ub.diagnostic_message(),
             InterpError::Unsupported(e) => e.diagnostic_message(),
@@ -842,7 +841,7 @@ impl<'tcx> ReportErrorExt for InterpError<'tcx> {
 }
 
 impl<'tcx> ReportErrorExt for InvalidProgramInfo<'tcx> {
-    fn diagnostic_message(&self) -> DiagnosticMessage {
+    fn diagnostic_message(&self) -> DiagMessage {
         use crate::fluent_generated::*;
         match self {
             InvalidProgramInfo::TooGeneric => const_eval_too_generic,
@@ -877,7 +876,7 @@ impl<'tcx> ReportErrorExt for InvalidProgramInfo<'tcx> {
 }
 
 impl ReportErrorExt for ResourceExhaustionInfo {
-    fn diagnostic_message(&self) -> DiagnosticMessage {
+    fn diagnostic_message(&self) -> DiagMessage {
         use crate::fluent_generated::*;
         match self {
             ResourceExhaustionInfo::StackFrameLimitReached => const_eval_stack_frame_limit_reached,

compiler/rustc_const_eval/src/interpret/cast.rs

@@ -182,13 +182,14 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
         use rustc_type_ir::TyKind::*;
 
-        let val = match src.layout.ty.kind() {
-            // Floating point
-            Float(FloatTy::F32) => self.cast_from_float(src.to_scalar().to_f32()?, cast_to.ty),
-            Float(FloatTy::F64) => self.cast_from_float(src.to_scalar().to_f64()?, cast_to.ty),
-            _ => {
-                bug!("Can't cast 'Float' type into {}", cast_to.ty);
-            }
+        let Float(fty) = src.layout.ty.kind() else {
+            bug!("FloatToFloat/FloatToInt cast: source type {} is not a float type", src.layout.ty)
+        };
+        let val = match fty {
+            FloatTy::F16 => unimplemented!("f16_f128"),
+            FloatTy::F32 => self.cast_from_float(src.to_scalar().to_f32()?, cast_to.ty),
+            FloatTy::F64 => self.cast_from_float(src.to_scalar().to_f64()?, cast_to.ty),
+            FloatTy::F128 => unimplemented!("f16_f128"),
         };
         Ok(ImmTy::from_scalar(val, cast_to))
     }
@@ -275,6 +276,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         trace!("cast_from_scalar: {}, {} -> {}", v, src_layout.ty, cast_ty);
 
         Ok(match *cast_ty.kind() {
+            // int -> int
             Int(_) | Uint(_) => {
                 let size = match *cast_ty.kind() {
                     Int(t) => Integer::from_int_ty(self, t).size(),
@@ -285,15 +287,26 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 Scalar::from_uint(v, size)
             }
 
-            Float(FloatTy::F32) if signed => Scalar::from_f32(Single::from_i128(v as i128).value),
-            Float(FloatTy::F64) if signed => Scalar::from_f64(Double::from_i128(v as i128).value),
-            Float(FloatTy::F32) => Scalar::from_f32(Single::from_u128(v).value),
-            Float(FloatTy::F64) => Scalar::from_f64(Double::from_u128(v).value),
-
-            Char => {
-                // `u8` to `char` cast
-                Scalar::from_u32(u8::try_from(v).unwrap().into())
+            // signed int -> float
+            Float(fty) if signed => {
+                let v = v as i128;
+                match fty {
+                    FloatTy::F16 => unimplemented!("f16_f128"),
+                    FloatTy::F32 => Scalar::from_f32(Single::from_i128(v).value),
+                    FloatTy::F64 => Scalar::from_f64(Double::from_i128(v).value),
+                    FloatTy::F128 => unimplemented!("f16_f128"),
+                }
             }
+            // unsigned int -> float
+            Float(fty) => match fty {
+                FloatTy::F16 => unimplemented!("f16_f128"),
+                FloatTy::F32 => Scalar::from_f32(Single::from_u128(v).value),
+                FloatTy::F64 => Scalar::from_f64(Double::from_u128(v).value),
+                FloatTy::F128 => unimplemented!("f16_f128"),
+            },
+
+            // u8 -> char
+            Char => Scalar::from_u32(u8::try_from(v).unwrap().into()),
 
             // Casts to bool are not permitted by rustc, no need to handle them here.
             _ => span_bug!(self.cur_span(), "invalid int to {} cast", cast_ty),
@@ -339,14 +352,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 let v = f.to_i128(size.bits_usize()).value;
                 Scalar::from_int(v, size)
             }
-            // float -> f32
-            Float(FloatTy::F32) => {
-                Scalar::from_f32(adjust_nan(self, f, f.convert(&mut false).value))
-            }
-            // float -> f64
-            Float(FloatTy::F64) => {
-                Scalar::from_f64(adjust_nan(self, f, f.convert(&mut false).value))
-            }
+            // float -> float
+            Float(fty) => match fty {
+                FloatTy::F16 => unimplemented!("f16_f128"),
+                FloatTy::F32 => Scalar::from_f32(adjust_nan(self, f, f.convert(&mut false).value)),
+                FloatTy::F64 => Scalar::from_f64(adjust_nan(self, f, f.convert(&mut false).value)),
+                FloatTy::F128 => unimplemented!("f16_f128"),
+            },
             // That's it.
             _ => span_bug!(self.cur_span(), "invalid float to {} cast", dest_ty),
         }

compiler/rustc_const_eval/src/interpret/discriminant.rs

@@ -10,6 +10,8 @@ use super::{ImmTy, InterpCx, InterpResult, Machine, Readable, Scalar, Writeable}
 
 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     /// Writes the discriminant of the given variant.
+    ///
+    /// If the variant is uninhabited, this is UB.
     #[instrument(skip(self), level = "trace")]
     pub fn write_discriminant(
         &mut self,
@@ -102,6 +104,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
     /// Read discriminant, return the runtime value as well as the variant index.
     /// Can also legally be called on non-enums (e.g. through the discriminant_value intrinsic)!
+    ///
+    /// Will never return an uninhabited variant.
     #[instrument(skip(self), level = "trace")]
     pub fn read_discriminant(
         &self,
@@ -244,7 +248,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 variant
             }
         };
-        // For consistency with `write_discriminant`, and to make sure that `project_downcast` cannot fail due to strange layouts, we declare immediate UB for uninhabited variants.
+        // Reading the discriminant of an uninhabited variant is UB. This is the basis for the
+        // `uninhabited_enum_branching` MIR pass. It also ensures consistency with
+        // `write_discriminant`.
         if op.layout().for_variant(self, index).abi.is_uninhabited() {
             throw_ub!(UninhabitedEnumVariantRead(index))
         }

compiler/rustc_const_eval/src/interpret/eval_context.rs

@@ -108,7 +108,7 @@ pub struct Frame<'mir, 'tcx, Prov: Provenance = CtfeProvenance, Extra = ()> {
 
     /// The location where the result of the current stack frame should be written to,
     /// and its layout in the caller.
-    pub return_place: PlaceTy<'tcx, Prov>,
+    pub return_place: MPlaceTy<'tcx, Prov>,
 
     /// The list of locals for this stack frame, stored in order as
     /// `[return_ptr, arguments..., variables..., temporaries...]`.
@@ -771,7 +771,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         &mut self,
         instance: ty::Instance<'tcx>,
         body: &'mir mir::Body<'tcx>,
-        return_place: &PlaceTy<'tcx, M::Provenance>,
+        return_place: &MPlaceTy<'tcx, M::Provenance>,
         return_to_block: StackPopCleanup,
     ) -> InterpResult<'tcx> {
         trace!("body: {:#?}", body);
@@ -912,7 +912,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             } else {
                 self.copy_op_allow_transmute(&op, &dest)
             };
-            trace!("return value: {:?}", self.dump_place(&dest));
+            trace!("return value: {:?}", self.dump_place(&dest.into()));
             // We delay actually short-circuiting on this error until *after* the stack frame is
             // popped, since we want this error to be attributed to the caller, whose type defines
             // this transmute.

compiler/rustc_const_eval/src/interpret/intrinsics.rs

@@ -21,7 +21,7 @@ use rustc_span::symbol::{sym, Symbol};
 use rustc_target::abi::Size;
 
 use super::{
-    util::ensure_monomorphic_enough, CheckInAllocMsg, ImmTy, InterpCx, Machine, OpTy, PlaceTy,
+    util::ensure_monomorphic_enough, CheckInAllocMsg, ImmTy, InterpCx, MPlaceTy, Machine, OpTy,
     Pointer,
 };
 
@@ -104,7 +104,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         &mut self,
         instance: ty::Instance<'tcx>,
         args: &[OpTy<'tcx, M::Provenance>],
-        dest: &PlaceTy<'tcx, M::Provenance>,
+        dest: &MPlaceTy<'tcx, M::Provenance>,
         ret: Option<mir::BasicBlock>,
     ) -> InterpResult<'tcx, bool> {
         let instance_args = instance.args;
@@ -377,7 +377,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
                 let elem = &args[2];
                 let (input, input_len) = self.operand_to_simd(&args[0])?;
-                let (dest, dest_len) = self.place_to_simd(dest)?;
+                let (dest, dest_len) = self.mplace_to_simd(dest)?;
                 assert_eq!(input_len, dest_len, "Return vector length must match input length");
                 // Bounds are not checked by typeck so we have to do it ourselves.
                 if index >= input_len {
@@ -430,7 +430,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             _ => return Ok(false),
         }
 
-        trace!("{:?}", self.dump_place(dest));
+        trace!("{:?}", self.dump_place(&dest.clone().into()));
         self.go_to_block(ret);
         Ok(true)
     }
@@ -488,7 +488,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         &mut self,
         a: &ImmTy<'tcx, M::Provenance>,
         b: &ImmTy<'tcx, M::Provenance>,
-        dest: &PlaceTy<'tcx, M::Provenance>,
+        dest: &MPlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx> {
         assert_eq!(a.layout.ty, b.layout.ty);
         assert!(matches!(a.layout.ty.kind(), ty::Int(..) | ty::Uint(..)));
@@ -506,7 +506,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             )
         }
         // `Rem` says this is all right, so we can let `Div` do its job.
-        self.binop_ignore_overflow(BinOp::Div, a, b, dest)
+        self.binop_ignore_overflow(BinOp::Div, a, b, &dest.clone().into())
     }
 
     pub fn saturating_arith(