Merge from rustc

compiler/rustc_ast/src/ast.rs

@@ -2302,6 +2302,9 @@ pub enum InlineAsmOperand {
     Sym {
         sym: InlineAsmSym,
     },
+    Label {
+        block: P<Block>,
+    },
 }
 
 impl InlineAsmOperand {
@@ -2311,7 +2314,7 @@ impl InlineAsmOperand {
             | Self::Out { reg, .. }
             | Self::InOut { reg, .. }
             | Self::SplitInOut { reg, .. } => Some(reg),
-            Self::Const { .. } | Self::Sym { .. } => None,
+            Self::Const { .. } | Self::Sym { .. } | Self::Label { .. } => None,
         }
     }
 }

compiler/rustc_ast/src/mut_visit.rs

@@ -1331,6 +1331,7 @@ pub fn noop_visit_inline_asm<T: MutVisitor>(asm: &mut InlineAsm, vis: &mut T) {
             }
             InlineAsmOperand::Const { anon_const } => vis.visit_anon_const(anon_const),
             InlineAsmOperand::Sym { sym } => vis.visit_inline_asm_sym(sym),
+            InlineAsmOperand::Label { block } => vis.visit_block(block),
         }
     }
 }

compiler/rustc_ast/src/visit.rs

@@ -823,6 +823,7 @@ pub fn walk_inline_asm<'a, V: Visitor<'a>>(visitor: &mut V, asm: &'a InlineAsm)
                 try_visit!(visitor.visit_anon_const(anon_const))
             }
             InlineAsmOperand::Sym { sym } => try_visit!(visitor.visit_inline_asm_sym(sym)),
+            InlineAsmOperand::Label { block } => try_visit!(visitor.visit_block(block)),
         }
     }
     V::Result::output()

compiler/rustc_ast_lowering/messages.ftl

@@ -88,6 +88,9 @@ ast_lowering_invalid_abi_suggestion = did you mean
 ast_lowering_invalid_asm_template_modifier_const =
     asm template modifiers are not allowed for `const` arguments
 
+ast_lowering_invalid_asm_template_modifier_label =
+    asm template modifiers are not allowed for `label` arguments
+
 ast_lowering_invalid_asm_template_modifier_reg_class =
     invalid asm template modifier for this register class
 

compiler/rustc_ast_lowering/src/asm.rs

@@ -3,9 +3,9 @@ use crate::{ImplTraitContext, ImplTraitPosition, ParamMode, ResolverAstLoweringE
 use super::errors::{
     AbiSpecifiedMultipleTimes, AttSyntaxOnlyX86, ClobberAbiNotSupported,
     InlineAsmUnsupportedTarget, InvalidAbiClobberAbi, InvalidAsmTemplateModifierConst,
-    InvalidAsmTemplateModifierRegClass, InvalidAsmTemplateModifierRegClassSub,
+    InvalidAsmTemplateModifierLabel, InvalidAsmTemplateModifierRegClass,
-    InvalidAsmTemplateModifierSym, InvalidRegister, InvalidRegisterClass, RegisterClassOnlyClobber,
+    InvalidAsmTemplateModifierRegClassSub, InvalidAsmTemplateModifierSym, InvalidRegister,
-    RegisterConflict,
+    InvalidRegisterClass, RegisterClassOnlyClobber, RegisterConflict,
 };
 use super::LoweringContext;
 
@@ -237,6 +237,18 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             }
                         }
                     }
+                    InlineAsmOperand::Label { block } => {
+                        if !self.tcx.features().asm_goto {
+                            feature_err(
+                                sess,
+                                sym::asm_goto,
+                                *op_sp,
+                                "label operands for inline assembly are unstable",
+                            )
+                            .emit();
+                        }
+                        hir::InlineAsmOperand::Label { block: self.lower_block(block, false) }
+                    }
                 };
                 (op, self.lower_span(*op_sp))
             })
@@ -296,6 +308,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             op_span: op_sp,
                         });
                     }
+                    hir::InlineAsmOperand::Label { .. } => {
+                        self.dcx().emit_err(InvalidAsmTemplateModifierLabel {
+                            placeholder_span,
+                            op_span: op_sp,
+                        });
+                    }
                 }
             }
         }
@@ -335,7 +353,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
                         hir::InlineAsmOperand::Const { .. }
                         | hir::InlineAsmOperand::SymFn { .. }
-                        | hir::InlineAsmOperand::SymStatic { .. } => {
+                        | hir::InlineAsmOperand::SymStatic { .. }
+                        | hir::InlineAsmOperand::Label { .. } => {
                             unreachable!("{op:?} is not a register operand");
                         }
                     };

compiler/rustc_ast_lowering/src/errors.rs

@@ -261,6 +261,16 @@ pub struct InvalidAsmTemplateModifierSym {
     pub op_span: Span,
 }
 
+#[derive(Diagnostic, Clone, Copy)]
+#[diag(ast_lowering_invalid_asm_template_modifier_label)]
+pub struct InvalidAsmTemplateModifierLabel {
+    #[primary_span]
+    #[label(ast_lowering_template_modifier)]
+    pub placeholder_span: Span,
+    #[label(ast_lowering_argument)]
+    pub op_span: Span,
+}
+
 #[derive(Diagnostic, Clone, Copy)]
 #[diag(ast_lowering_register_class_only_clobber)]
 pub struct RegisterClassOnlyClobber {

compiler/rustc_ast_lowering/src/format.rs

@@ -1,4 +1,5 @@
 use super::LoweringContext;
+use core::ops::ControlFlow;
 use rustc_ast as ast;
 use rustc_ast::visit::Visitor;
 use rustc_ast::*;
@@ -594,30 +595,32 @@ fn expand_format_args<'hir>(
 }
 
 fn may_contain_yield_point(e: &ast::Expr) -> bool {
-    struct MayContainYieldPoint(bool);
+    struct MayContainYieldPoint;
 
     impl Visitor<'_> for MayContainYieldPoint {
-        fn visit_expr(&mut self, e: &ast::Expr) {
+        type Result = ControlFlow<()>;
+
+        fn visit_expr(&mut self, e: &ast::Expr) -> ControlFlow<()> {
             if let ast::ExprKind::Await(_, _) | ast::ExprKind::Yield(_) = e.kind {
-                self.0 = true;
+                ControlFlow::Break(())
             } else {
                 visit::walk_expr(self, e);
+                ControlFlow::Continue(())
             }
         }
 
-        fn visit_mac_call(&mut self, _: &ast::MacCall) {
+        fn visit_mac_call(&mut self, _: &ast::MacCall) -> ControlFlow<()> {
             // Macros should be expanded at this point.
             unreachable!("unexpanded macro in ast lowering");
         }
 
-        fn visit_item(&mut self, _: &ast::Item) {
+        fn visit_item(&mut self, _: &ast::Item) -> ControlFlow<()> {
             // Do not recurse into nested items.
+            ControlFlow::Continue(())
         }
     }
 
-    let mut visitor = MayContainYieldPoint(false);
+    MayContainYieldPoint.visit_expr(e).is_break()
-    visitor.visit_expr(e);
-    visitor.0
 }
 
 fn for_all_argument_indexes(template: &mut [FormatArgsPiece], mut f: impl FnMut(&mut usize)) {

compiler/rustc_ast_lowering/src/item.rs

@@ -275,7 +275,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         }
                         Some(ty) => this.lower_ty(
                             ty,
-                            ImplTraitContext::TypeAliasesOpaqueTy { in_assoc_ty: false },
+                            ImplTraitContext::OpaqueTy {
+                                origin: hir::OpaqueTyOrigin::TyAlias {
+                                    parent: this.local_def_id(id),
+                                    in_assoc_ty: false,
+                                },
+                                fn_kind: None,
+                            },
                         ),
                     },
                 );
@@ -936,7 +942,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         Some(ty) => {
                             let ty = this.lower_ty(
                                 ty,
-                                ImplTraitContext::TypeAliasesOpaqueTy { in_assoc_ty: true },
+                                ImplTraitContext::OpaqueTy {
+                                    origin: hir::OpaqueTyOrigin::TyAlias {
+                                        parent: this.local_def_id(i.id),
+                                        in_assoc_ty: true,
+                                    },
+                                    fn_kind: None,
+                                },
                             );
                             hir::ImplItemKind::Type(ty)
                         }

compiler/rustc_ast_lowering/src/lib.rs

@@ -265,13 +265,12 @@ enum ImplTraitContext {
     /// Example: `fn foo() -> impl Debug`, where `impl Debug` is conceptually
     /// equivalent to a new opaque type like `type T = impl Debug; fn foo() -> T`.
     ///
-    ReturnPositionOpaqueTy {
+    OpaqueTy {
-        /// Origin: Either OpaqueTyOrigin::FnReturn or OpaqueTyOrigin::AsyncFn,
         origin: hir::OpaqueTyOrigin,
-        fn_kind: FnDeclKind,
+        /// Only used to change the lifetime capture rules, since
+        /// RPITIT captures all in scope, RPIT does not.
+        fn_kind: Option<FnDeclKind>,
     },
-    /// Impl trait in type aliases.
-    TypeAliasesOpaqueTy { in_assoc_ty: bool },
     /// `impl Trait` is unstably accepted in this position.
     FeatureGated(ImplTraitPosition, Symbol),
     /// `impl Trait` is not accepted in this position.
@@ -1075,9 +1074,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 // Disallow ATB in dyn types
                 if self.is_in_dyn_type {
                     let suggestion = match itctx {
-                        ImplTraitContext::ReturnPositionOpaqueTy { .. }
+                        ImplTraitContext::OpaqueTy { .. } | ImplTraitContext::Universal => {
-                        | ImplTraitContext::TypeAliasesOpaqueTy { .. }
-                        | ImplTraitContext::Universal => {
                             let bound_end_span = constraint
                                 .gen_args
                                 .as_ref()
@@ -1417,22 +1414,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             TyKind::ImplTrait(def_node_id, bounds) => {
                 let span = t.span;
                 match itctx {
-                    ImplTraitContext::ReturnPositionOpaqueTy { origin, fn_kind } => self
+                    ImplTraitContext::OpaqueTy { origin, fn_kind } => self.lower_opaque_impl_trait(
-                        .lower_opaque_impl_trait(
                         span,
                         origin,
                         *def_node_id,
                         bounds,
-                            Some(fn_kind),
+                        fn_kind,
-                            itctx,
-                        ),
-                    ImplTraitContext::TypeAliasesOpaqueTy { in_assoc_ty } => self
-                        .lower_opaque_impl_trait(
-                            span,
-                            hir::OpaqueTyOrigin::TyAlias { in_assoc_ty },
-                            *def_node_id,
-                            bounds,
-                            None,
                         itctx,
                     ),
                     ImplTraitContext::Universal => {
@@ -1553,9 +1540,14 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
         let captured_lifetimes_to_duplicate = match origin {
             hir::OpaqueTyOrigin::TyAlias { .. } => {
-                // in a TAIT like `type Foo<'a> = impl Foo<'a>`, we don't duplicate any
+                // type alias impl trait and associated type position impl trait were
-                // lifetimes, since we don't have the issue that any are late-bound.
+                // decided to capture all in-scope lifetimes, which we collect for
-                Vec::new()
+                // all opaques during resolution.
+                self.resolver
+                    .take_extra_lifetime_params(opaque_ty_node_id)
+                    .into_iter()
+                    .map(|(ident, id, _)| Lifetime { id, ident })
+                    .collect()
             }
             hir::OpaqueTyOrigin::FnReturn(..) => {
                 if matches!(
@@ -1823,9 +1815,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         FnDeclKind::Fn
                         | FnDeclKind::Inherent
                         | FnDeclKind::Trait
-                        | FnDeclKind::Impl => ImplTraitContext::ReturnPositionOpaqueTy {
+                        | FnDeclKind::Impl => ImplTraitContext::OpaqueTy {
                             origin: hir::OpaqueTyOrigin::FnReturn(self.local_def_id(fn_node_id)),
-                            fn_kind: kind,
+                            fn_kind: Some(kind),
                         },
                         FnDeclKind::ExternFn => {
                             ImplTraitContext::Disallowed(ImplTraitPosition::ExternFnReturn)
@@ -1919,9 +1911,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     output,
                     coro,
                     opaque_ty_span,
-                    ImplTraitContext::ReturnPositionOpaqueTy {
+                    ImplTraitContext::OpaqueTy {
                         origin: hir::OpaqueTyOrigin::FnReturn(fn_def_id),
-                        fn_kind,
+                        fn_kind: Some(fn_kind),
                     },
                 );
                 arena_vec![this; bound]

compiler/rustc_ast_lowering/src/path.rs

@@ -423,7 +423,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             // fn f(_: impl Fn() -> impl Debug) -> impl Fn() -> impl Debug
             // //      disallowed --^^^^^^^^^^        allowed --^^^^^^^^^^
             // ```
-            FnRetTy::Ty(ty) if matches!(itctx, ImplTraitContext::ReturnPositionOpaqueTy { .. }) => {
+            FnRetTy::Ty(ty) if matches!(itctx, ImplTraitContext::OpaqueTy { .. }) => {
                 if self.tcx.features().impl_trait_in_fn_trait_return {
                     self.lower_ty(ty, itctx)
                 } else {

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -1452,6 +1452,10 @@ impl<'a> State<'a> {
                             s.print_path(&sym.path, true, 0);
                         }
                     }
+                    InlineAsmOperand::Label { block } => {
+                        s.head("label");
+                        s.print_block(block);
+                    }
                 }
             }
             AsmArg::ClobberAbi(abi) => {

compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs

@@ -1,6 +1,7 @@
 #![allow(rustc::diagnostic_outside_of_impl)]
 #![allow(rustc::untranslatable_diagnostic)]
 
+use core::ops::ControlFlow;
 use hir::ExprKind;
 use rustc_errors::{Applicability, Diag};
 use rustc_hir as hir;
@@ -727,30 +728,12 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             _ => local_decl.source_info.span,
         };
 
-        struct BindingFinder {
-            span: Span,
-            hir_id: Option<hir::HirId>,
-        }
-
-        impl<'tcx> Visitor<'tcx> for BindingFinder {
-            fn visit_stmt(&mut self, s: &'tcx hir::Stmt<'tcx>) {
-                if let hir::StmtKind::Local(local) = s.kind {
-                    if local.pat.span == self.span {
-                        self.hir_id = Some(local.hir_id);
-                    }
-                }
-                hir::intravisit::walk_stmt(self, s);
-            }
-        }
-
         let def_id = self.body.source.def_id();
         let hir_id = if let Some(local_def_id) = def_id.as_local()
             && let Some(body_id) = self.infcx.tcx.hir().maybe_body_owned_by(local_def_id)
         {
             let body = self.infcx.tcx.hir().body(body_id);
-            let mut v = BindingFinder { span: pat_span, hir_id: None };
+            BindingFinder { span: pat_span }.visit_body(body).break_value()
-            v.visit_body(body);
-            v.hir_id
         } else {
             None
         };
@@ -859,17 +842,18 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
         };
 
         let hir_map = self.infcx.tcx.hir();
-        struct Finder<'tcx> {
+        struct Finder {
             span: Span,
-            expr: Option<&'tcx Expr<'tcx>>,
         }
 
-        impl<'tcx> Visitor<'tcx> for Finder<'tcx> {
+        impl<'tcx> Visitor<'tcx> for Finder {
-            fn visit_expr(&mut self, e: &'tcx hir::Expr<'tcx>) {
+            type Result = ControlFlow<&'tcx Expr<'tcx>>;
-                if e.span == self.span && self.expr.is_none() {
+            fn visit_expr(&mut self, e: &'tcx hir::Expr<'tcx>) -> Self::Result {
-                    self.expr = Some(e);
+                if e.span == self.span {
+                    ControlFlow::Break(e)
+                } else {
+                    hir::intravisit::walk_expr(self, e)
                 }
-                hir::intravisit::walk_expr(self, e);
             }
         }
         if let Some(body_id) = hir_map.maybe_body_owned_by(self.mir_def_id())
@@ -878,9 +862,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             // `span` corresponds to the expression being iterated, find the `for`-loop desugared
             // expression with that span in order to identify potential fixes when encountering a
             // read-only iterator that should be mutable.
-            let mut v = Finder { span, expr: None };
+            if let ControlFlow::Break(expr) = (Finder { span }).visit_block(block)
-            v.visit_block(block);
-            if let Some(expr) = v.expr
                 && let Call(_, [expr]) = expr.kind
             {
                 match expr.kind {
@@ -1179,29 +1161,12 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                 );
             }
             Some((false, err_label_span, message)) => {
-                struct BindingFinder {
-                    span: Span,
-                    hir_id: Option<hir::HirId>,
-                }
-
-                impl<'tcx> Visitor<'tcx> for BindingFinder {
-                    fn visit_stmt(&mut self, s: &'tcx hir::Stmt<'tcx>) {
-                        if let hir::StmtKind::Local(local) = s.kind {
-                            if local.pat.span == self.span {
-                                self.hir_id = Some(local.hir_id);
-                            }
-                        }
-                        hir::intravisit::walk_stmt(self, s);
-                    }
-                }
                 let def_id = self.body.source.def_id();
                 let hir_id = if let Some(local_def_id) = def_id.as_local()
                     && let Some(body_id) = self.infcx.tcx.hir().maybe_body_owned_by(local_def_id)
                 {
                     let body = self.infcx.tcx.hir().body(body_id);
-                    let mut v = BindingFinder { span: err_label_span, hir_id: None };
+                    BindingFinder { span: err_label_span }.visit_body(body).break_value()
-                    v.visit_body(body);
-                    v.hir_id
                 } else {
                     None
                 };
@@ -1333,6 +1298,23 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
     }
 }
 
+struct BindingFinder {
+    span: Span,
+}
+
+impl<'tcx> Visitor<'tcx> for BindingFinder {
+    type Result = ControlFlow<hir::HirId>;
+    fn visit_stmt(&mut self, s: &'tcx hir::Stmt<'tcx>) -> Self::Result {
+        if let hir::StmtKind::Local(local) = s.kind
+            && local.pat.span == self.span
+        {
+            ControlFlow::Break(local.hir_id)
+        } else {
+            hir::intravisit::walk_stmt(self, s)
+        }
+    }
+}
+
 pub fn mut_borrow_of_mutable_ref(local_decl: &LocalDecl<'_>, local_name: Option<Symbol>) -> bool {
     debug!("local_info: {:?}, ty.kind(): {:?}", local_decl.local_info, local_decl.ty.kind());
 

compiler/rustc_borrowck/src/lib.rs

@@ -6,6 +6,7 @@
 #![feature(assert_matches)]
 #![feature(associated_type_bounds)]
 #![feature(box_patterns)]
+#![feature(control_flow_enum)]
 #![feature(let_chains)]
 #![feature(min_specialization)]
 #![feature(never_type)]
@@ -723,7 +724,7 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
                 operands,
                 options: _,
                 line_spans: _,
-                destination: _,
+                targets: _,
                 unwind: _,
             } => {
                 for op in operands {
@@ -749,7 +750,8 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
                         }
                         InlineAsmOperand::Const { value: _ }
                         | InlineAsmOperand::SymFn { value: _ }
-                        | InlineAsmOperand::SymStatic { def_id: _ } => {}
+                        | InlineAsmOperand::SymStatic { def_id: _ }
+                        | InlineAsmOperand::Label { target_index: _ } => {}
                     }
                 }
             }

compiler/rustc_borrowck/src/polonius/loan_invalidations.rs

@@ -161,7 +161,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
                 operands,
                 options: _,
                 line_spans: _,
-                destination: _,
+                targets: _,
                 unwind: _,
             } => {
                 for op in operands {
@@ -182,7 +182,8 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
                         }
                         InlineAsmOperand::Const { value: _ }
                         | InlineAsmOperand::SymFn { value: _ }
-                        | InlineAsmOperand::SymStatic { def_id: _ } => {}
+                        | InlineAsmOperand::SymStatic { def_id: _ }
+                        | InlineAsmOperand::Label { target_index: _ } => {}
                     }
                 }
             }

compiler/rustc_borrowck/src/region_infer/opaque_types.rs

@@ -367,14 +367,17 @@ fn check_opaque_type_parameter_valid(
     span: Span,
 ) -> Result<(), ErrorGuaranteed> {
     let opaque_ty_hir = tcx.hir().expect_item(opaque_type_key.def_id);
-    let is_ty_alias = match opaque_ty_hir.expect_opaque_ty().origin {
+    let (parent, is_ty_alias) = match opaque_ty_hir.expect_opaque_ty().origin {
-        OpaqueTyOrigin::TyAlias { .. } => true,
+        OpaqueTyOrigin::TyAlias { parent, .. } => (parent, true),
-        OpaqueTyOrigin::AsyncFn(..) | OpaqueTyOrigin::FnReturn(..) => false,
+        OpaqueTyOrigin::AsyncFn(parent) | OpaqueTyOrigin::FnReturn(parent) => (parent, false),
     };
 
-    let opaque_generics = tcx.generics_of(opaque_type_key.def_id);
+    let parent_generics = tcx.generics_of(parent);
     let mut seen_params: FxIndexMap<_, Vec<_>> = FxIndexMap::default();
-    for (i, arg) in opaque_type_key.args.iter().enumerate() {
+
+    // Only check the parent generics, which will ignore any of the
+    // duplicated lifetime args that come from reifying late-bounds.
+    for (i, arg) in opaque_type_key.args.iter().take(parent_generics.count()).enumerate() {
         if let Err(guar) = arg.error_reported() {
             return Err(guar);
         }
@@ -395,7 +398,7 @@ fn check_opaque_type_parameter_valid(
             seen_params.entry(arg).or_default().push(i);
         } else {
             // Prevent `fn foo() -> Foo<u32>` from being defining.
-            let opaque_param = opaque_generics.param_at(i, tcx);
+            let opaque_param = parent_generics.param_at(i, tcx);
             let kind = opaque_param.kind.descr();
 
             return Err(tcx.dcx().emit_err(NonGenericOpaqueTypeParam {
@@ -409,10 +412,10 @@ fn check_opaque_type_parameter_valid(
 
     for (_, indices) in seen_params {
         if indices.len() > 1 {
-            let descr = opaque_generics.param_at(indices[0], tcx).kind.descr();
+            let descr = parent_generics.param_at(indices[0], tcx).kind.descr();
             let spans: Vec<_> = indices
                 .into_iter()
-                .map(|i| tcx.def_span(opaque_generics.param_at(i, tcx).def_id))
+                .map(|i| tcx.def_span(parent_generics.param_at(i, tcx).def_id))
                 .collect();
             #[allow(rustc::diagnostic_outside_of_impl)]
             #[allow(rustc::untranslatable_diagnostic)]

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -1770,8 +1770,8 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 self.assert_iscleanup(body, block_data, real_target, is_cleanup);
                 self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);
             }
-            TerminatorKind::InlineAsm { destination, unwind, .. } => {
+            TerminatorKind::InlineAsm { ref targets, unwind, .. } => {
-                if let Some(target) = destination {
+                for &target in targets {
                     self.assert_iscleanup(body, block_data, target, is_cleanup);
                 }
                 self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);

compiler/rustc_builtin_macros/messages.ftl

@@ -19,6 +19,8 @@ builtin_macros_asm_expected_other = expected operand, {$is_global_asm ->
 
 builtin_macros_asm_explicit_register_name = explicit register arguments cannot have names
 
+builtin_macros_asm_mayunwind = asm labels are not allowed with the `may_unwind` option
+
 builtin_macros_asm_modifier_invalid = asm template modifier must be a single character
 
 builtin_macros_asm_mutually_exclusive = the `{$opt1}` and `{$opt2}` options are mutually exclusive

compiler/rustc_builtin_macros/src/asm.rs

@@ -164,6 +164,9 @@ pub fn parse_asm_args<'a>(
                 path: path.clone(),
             };
             ast::InlineAsmOperand::Sym { sym }
+        } else if !is_global_asm && p.eat_keyword(sym::label) {
+            let block = p.parse_block()?;
+            ast::InlineAsmOperand::Label { block }
         } else if allow_templates {
             let template = p.parse_expr()?;
             // If it can't possibly expand to a string, provide diagnostics here to include other
@@ -240,6 +243,7 @@ pub fn parse_asm_args<'a>(
     let mut have_real_output = false;
     let mut outputs_sp = vec![];
     let mut regclass_outputs = vec![];
+    let mut labels_sp = vec![];
     for (op, op_sp) in &args.operands {
         match op {
             ast::InlineAsmOperand::Out { reg, expr, .. }
@@ -257,6 +261,9 @@ pub fn parse_asm_args<'a>(
                     regclass_outputs.push(*op_sp);
                 }
             }
+            ast::InlineAsmOperand::Label { .. } => {
+                labels_sp.push(*op_sp);
+            }
             _ => {}
         }
     }
@@ -268,6 +275,9 @@ pub fn parse_asm_args<'a>(
         // Bail out now since this is likely to confuse MIR
         return Err(err);
     }
+    if args.options.contains(ast::InlineAsmOptions::MAY_UNWIND) && !labels_sp.is_empty() {
+        dcx.emit_err(errors::AsmMayUnwind { labels_sp });
+    }
 
     if args.clobber_abis.len() > 0 {
         if is_global_asm {

compiler/rustc_builtin_macros/src/cfg_eval.rs

@@ -1,5 +1,6 @@
 use crate::util::{check_builtin_macro_attribute, warn_on_duplicate_attribute};
 
+use core::ops::ControlFlow;
 use rustc_ast as ast;
 use rustc_ast::mut_visit::MutVisitor;
 use rustc_ast::ptr::P;
@@ -87,41 +88,40 @@ fn flat_map_annotatable(
     }
 }
 
-struct CfgFinder {
+fn has_cfg_or_cfg_attr(annotatable: &Annotatable) -> bool {
-    has_cfg_or_cfg_attr: bool,
+    struct CfgFinder;
-}
 
-impl CfgFinder {
+    impl<'ast> visit::Visitor<'ast> for CfgFinder {
-    fn has_cfg_or_cfg_attr(annotatable: &Annotatable) -> bool {
+        type Result = ControlFlow<()>;
-        let mut finder = CfgFinder { has_cfg_or_cfg_attr: false };
+        fn visit_attribute(&mut self, attr: &'ast Attribute) -> ControlFlow<()> {
-        match annotatable {
+            if attr
-            Annotatable::Item(item) => finder.visit_item(item),
-            Annotatable::TraitItem(item) => finder.visit_assoc_item(item, visit::AssocCtxt::Trait),
-            Annotatable::ImplItem(item) => finder.visit_assoc_item(item, visit::AssocCtxt::Impl),
-            Annotatable::ForeignItem(item) => finder.visit_foreign_item(item),
-            Annotatable::Stmt(stmt) => finder.visit_stmt(stmt),
-            Annotatable::Expr(expr) => finder.visit_expr(expr),
-            Annotatable::Arm(arm) => finder.visit_arm(arm),
-            Annotatable::ExprField(field) => finder.visit_expr_field(field),
-            Annotatable::PatField(field) => finder.visit_pat_field(field),
-            Annotatable::GenericParam(param) => finder.visit_generic_param(param),
-            Annotatable::Param(param) => finder.visit_param(param),
-            Annotatable::FieldDef(field) => finder.visit_field_def(field),
-            Annotatable::Variant(variant) => finder.visit_variant(variant),
-            Annotatable::Crate(krate) => finder.visit_crate(krate),
-        };
-        finder.has_cfg_or_cfg_attr
-    }
-}
-
-impl<'ast> visit::Visitor<'ast> for CfgFinder {
-    fn visit_attribute(&mut self, attr: &'ast Attribute) {
-        // We want short-circuiting behavior, so don't use the '|=' operator.
-        self.has_cfg_or_cfg_attr = self.has_cfg_or_cfg_attr
-            || attr
                 .ident()
-                .is_some_and(|ident| ident.name == sym::cfg || ident.name == sym::cfg_attr);
+                .is_some_and(|ident| ident.name == sym::cfg || ident.name == sym::cfg_attr)
+            {
+                ControlFlow::Break(())
+            } else {
+                ControlFlow::Continue(())
             }
+        }
+    }
+
+    let res = match annotatable {
+        Annotatable::Item(item) => CfgFinder.visit_item(item),
+        Annotatable::TraitItem(item) => CfgFinder.visit_assoc_item(item, visit::AssocCtxt::Trait),
+        Annotatable::ImplItem(item) => CfgFinder.visit_assoc_item(item, visit::AssocCtxt::Impl),
+        Annotatable::ForeignItem(item) => CfgFinder.visit_foreign_item(item),
+        Annotatable::Stmt(stmt) => CfgFinder.visit_stmt(stmt),
+        Annotatable::Expr(expr) => CfgFinder.visit_expr(expr),
+        Annotatable::Arm(arm) => CfgFinder.visit_arm(arm),
+        Annotatable::ExprField(field) => CfgFinder.visit_expr_field(field),
+        Annotatable::PatField(field) => CfgFinder.visit_pat_field(field),
+        Annotatable::GenericParam(param) => CfgFinder.visit_generic_param(param),
+        Annotatable::Param(param) => CfgFinder.visit_param(param),
+        Annotatable::FieldDef(field) => CfgFinder.visit_field_def(field),
+        Annotatable::Variant(variant) => CfgFinder.visit_variant(variant),
+        Annotatable::Crate(krate) => CfgFinder.visit_crate(krate),
+    };
+    res.is_break()
 }
 
 impl CfgEval<'_, '_> {
@@ -132,7 +132,7 @@ impl CfgEval<'_, '_> {
     fn configure_annotatable(&mut self, mut annotatable: Annotatable) -> Option<Annotatable> {
         // Tokenizing and re-parsing the `Annotatable` can have a significant
         // performance impact, so try to avoid it if possible
-        if !CfgFinder::has_cfg_or_cfg_attr(&annotatable) {
+        if !has_cfg_or_cfg_attr(&annotatable) {
             return Some(annotatable);
         }
 

compiler/rustc_builtin_macros/src/deriving/default.rs

@@ -1,6 +1,7 @@
 use crate::deriving::generic::ty::*;
 use crate::deriving::generic::*;
 use crate::errors;
+use core::ops::ControlFlow;
 use rustc_ast as ast;
 use rustc_ast::visit::walk_list;
 use rustc_ast::{attr, EnumDef, VariantData};
@@ -231,20 +232,19 @@ impl<'a, 'b> rustc_ast::visit::Visitor<'a> for DetectNonVariantDefaultAttr<'a, '
 }
 
 fn has_a_default_variant(item: &Annotatable) -> bool {
-    struct HasDefaultAttrOnVariant {
+    struct HasDefaultAttrOnVariant;
-        found: bool,
-    }
 
     impl<'ast> rustc_ast::visit::Visitor<'ast> for HasDefaultAttrOnVariant {
-        fn visit_variant(&mut self, v: &'ast rustc_ast::Variant) {
+        type Result = ControlFlow<()>;
+        fn visit_variant(&mut self, v: &'ast rustc_ast::Variant) -> ControlFlow<()> {
             if v.attrs.iter().any(|attr| attr.has_name(kw::Default)) {
-                self.found = true;
+                ControlFlow::Break(())
-            }
+            } else {
                 // no need to subrecurse.
+                ControlFlow::Continue(())
+            }
         }
     }
 
-    let mut visitor = HasDefaultAttrOnVariant { found: false };
+    item.visit_with(&mut HasDefaultAttrOnVariant).is_break()
-    item.visit_with(&mut visitor);
-    visitor.found
 }

compiler/rustc_builtin_macros/src/errors.rs

@@ -766,6 +766,13 @@ pub(crate) struct AsmNoReturn {
     pub(crate) outputs_sp: Vec<Span>,
 }
 
+#[derive(Diagnostic)]
+#[diag(builtin_macros_asm_mayunwind)]
+pub(crate) struct AsmMayUnwind {
+    #[primary_span]
+    pub(crate) labels_sp: Vec<Span>,
+}
+
 #[derive(Diagnostic)]
 #[diag(builtin_macros_global_asm_clobber_abi)]
 pub(crate) struct GlobalAsmClobberAbi {

compiler/rustc_codegen_cranelift/.github/workflows/main.yml

@@ -44,9 +44,10 @@ jobs:
             env:
               TARGET_TRIPLE: x86_64-apple-darwin
           # cross-compile from Linux to Windows using mingw
-          - os: ubuntu-latest
+          # FIXME The wine version in Ubuntu 22.04 is missing ProcessPrng
-            env:
+          #- os: ubuntu-latest
-              TARGET_TRIPLE: x86_64-pc-windows-gnu
+          #  env:
+          #    TARGET_TRIPLE: x86_64-pc-windows-gnu
           - os: ubuntu-latest
             env:
               TARGET_TRIPLE: aarch64-unknown-linux-gnu
@@ -80,11 +81,11 @@ jobs:
       if: matrix.os == 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
       run: rustup set default-host x86_64-pc-windows-gnu
 
-    - name: Install MinGW toolchain and wine
+    #- name: Install MinGW toolchain and wine
-      if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
+    #  if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
-      run: |
+    #  run: |
-        sudo apt-get update
+    #    sudo apt-get update
-        sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
+    #    sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable
 
     - name: Install AArch64 toolchain and qemu
       if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'aarch64-unknown-linux-gnu'

compiler/rustc_codegen_cranelift/.gitignore

@@ -1,18 +1,21 @@
-/target
+# Build artifacts during normal use
-/build_system/target
-**/*.rs.bk
-*.rlib
-*.o
-perf.data
-perf.data.old
-*.events
-*.string*
 /y.bin
 /y.bin.dSYM
 /y.exe
 /y.pdb
+/download
 /build
 /dist
+/target
+/build_system/target
+
+# Downloaded by certain scripts
 /rust
-/download
 /git-fixed-subtree.sh
+
+# Various things that can be created during development
+*.rlib
+*.o
+perf.data
+perf.data.old
+*.mm_profdata

compiler/rustc_codegen_cranelift/Cargo.lock

@@ -46,18 +46,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d819feeda4c420a18f1e28236ca0ce1177b22bf7c8a44ddee92dfe40de15bcf0"
+checksum = "9515fcc42b6cb5137f76b84c1a6f819782d0cf12473d145d3bc5cd67eedc8bc2"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9b8d03d5bdbca7e5f72b0e0a0f69933ed1f09e24be6c075aa6fe3f802b0cc0c"
+checksum = "1ad827c6071bfe6d22de1bc331296a29f9ddc506ff926d8415b435ec6a6efce0"
 dependencies = [
  "bumpalo",
  "cranelift-bforest",
@@ -76,39 +76,39 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3fd3664e38e51649b17dc30cfdd561273fe2f590dcd013fb75d9eabc6272dfb"
+checksum = "10e6b36237a9ca2ce2fb4cc7741d418a080afa1327402138412ef85d5367bef1"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b031ec5e605828975952622b5a77d49126f20ffe88d33719a0af66b23a0fc36"
+checksum = "c36bf4bfb86898a94ccfa773a1f86e8a5346b1983ff72059bdd2db4600325251"
 
 [[package]]
 name = "cranelift-control"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fada054d017cf2ed8f7ed2336e0517fc1b19e6825be1790de9eb00c94788362b"
+checksum = "7cbf36560e7a6bd1409ca91e7b43b2cc7ed8429f343d7605eadf9046e8fac0d0"
 dependencies = [
  "arbitrary",
 ]
 
 [[package]]
 name = "cranelift-entity"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "177b6f94ae8de6348eb45bf977c79ab9e3c40fc3ac8cb7ed8109560ea39bee7d"
+checksum = "a71e11061a75b1184c09bea97c026a88f08b59ade96a7bb1f259d4ea0df2e942"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebebd23a69a23e3ddea78e98ff3a2de222e88c8e045d81ef4a72f042e0d79dbd"
+checksum = "af5d4da63143ee3485c7bcedde0a818727d737d1083484a0ceedb8950c89e495"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -118,15 +118,15 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1571bfc14df8966d12c6121b5325026591a4b4009e22fea0fe3765ab7cd33b96"
+checksum = "457a9832b089e26f5eea70dcf49bed8ec6edafed630ce7c83161f24d46ab8085"
 
 [[package]]
 name = "cranelift-jit"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f61e236d7622c3c43016e8b0f3ba27136e21ac7de328c7fda902e61db1de851"
+checksum = "0af95fe68d5a10919012c8db82b1d59820405b8001c8c6d05f94b08031334fa9"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -144,9 +144,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f30c6820342015c5009070e3e48d1da7b13521399de904663f1c84f5ee839657"
+checksum = "11b0b201fa10a4014062d4c56c307c8d18fdf9a84cb5279efe6080241f42c7a7"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -155,9 +155,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35a69c37e0c10b46fe5527f2397ac821046efbf5f7ec112c8b84df25712f465b"
+checksum = "9b490d579df1ce365e1ea359e24ed86d82289fa785153327c2f6a69a59a731e4"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -166,9 +166,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.104.0"
+version = "0.105.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24425a329b4343177d5f1852243841dcec17f929d72c0e7f41262140155e55e7"
+checksum = "fb7e821ac6db471bcdbd004e5a4fa0d374f1046bd3a2ce278c332e0b0c01ca63"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -241,9 +241,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.148"
+version = "0.2.153"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b"
+checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
 
 [[package]]
 name = "libloading"
@@ -410,9 +410,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "17.0.0"
+version = "18.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bdc26415bb89e9ccd3bdc498fef63aabf665c4c0dd710c107691deb9694955da"
+checksum = "33f4121cb29dda08139b2824a734dd095d83ce843f2d613a84eb580b9cfc17ac"
 dependencies = [
  "cfg-if",
  "libc",

compiler/rustc_codegen_cranelift/Cargo.toml

@@ -8,12 +8,12 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.104", default-features = false, features = ["std", "unwind", "all-arch"] }
+cranelift-codegen = { version = "0.105.2", default-features = false, features = ["std", "unwind", "all-arch"] }
-cranelift-frontend = { version = "0.104" }
+cranelift-frontend = { version = "0.105.2" }
-cranelift-module = { version = "0.104" }
+cranelift-module = { version = "0.105.2" }
-cranelift-native = { version = "0.104" }
+cranelift-native = { version = "0.105.2" }
-cranelift-jit = { version = "0.104", optional = true }
+cranelift-jit = { version = "0.105.2", optional = true }
-cranelift-object = { version = "0.104" }
+cranelift-object = { version = "0.105.2" }
 target-lexicon = "0.12.0"
 gimli = { version = "0.28", default-features = false, features = ["write"]}
 object = { version = "0.32", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }

compiler/rustc_codegen_cranelift/Readme.md

@@ -123,11 +123,6 @@ You need to do this steps to successfully compile and use the cranelift backend
 
 You can also set `rust-analyzer.rustc.source` to your rust workspace to get rust-analyzer to understand your changes.
 
-## Configuration
-
-See the documentation on the `BackendConfig` struct in [config.rs](src/config.rs) for all
-configuration options.
-
 ## Not yet supported
 
 * SIMD ([tracked here](https://github.com/rust-lang/rustc_codegen_cranelift/issues/171), `std::simd` fully works, `std::arch` is partially supported)

compiler/rustc_codegen_cranelift/build_system/prepare.rs

@@ -1,5 +1,6 @@
 use std::ffi::OsStr;
 use std::fs;
+use std::hash::{Hash, Hasher};
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
@@ -71,7 +72,11 @@ fn hash_file(file: &std::path::Path) -> u64 {
     let contents = std::fs::read(file).unwrap();
     #[allow(deprecated)]
     let mut hasher = std::hash::SipHasher::new();
-    std::hash::Hash::hash(&contents, &mut hasher);
+    // The following is equivalent to
+    //   std::hash::Hash::hash(&contents, &mut hasher);
+    // but gives the same result independent of host byte order.
+    hasher.write_usize(contents.len().to_le());
+    Hash::hash_slice(&contents, &mut hasher);
     std::hash::Hasher::finish(&hasher)
 }
 
@@ -80,16 +85,26 @@ fn hash_dir(dir: &std::path::Path) -> u64 {
     for entry in std::fs::read_dir(dir).unwrap() {
         let entry = entry.unwrap();
         if entry.file_type().unwrap().is_dir() {
-            sub_hashes
+            sub_hashes.insert(
-                .insert(entry.file_name().to_str().unwrap().to_owned(), hash_dir(&entry.path()));
+                entry.file_name().to_str().unwrap().to_owned(),
+                hash_dir(&entry.path()).to_le(),
+            );
         } else {
-            sub_hashes
+            sub_hashes.insert(
-                .insert(entry.file_name().to_str().unwrap().to_owned(), hash_file(&entry.path()));
+                entry.file_name().to_str().unwrap().to_owned(),
+                hash_file(&entry.path()).to_le(),
+            );
         }
     }
     #[allow(deprecated)]
     let mut hasher = std::hash::SipHasher::new();
-    std::hash::Hash::hash(&sub_hashes, &mut hasher);
+    // The following is equivalent to
+    //   std::hash::Hash::hash(&sub_hashes, &mut hasher);
+    // but gives the same result independent of host byte order.
+    hasher.write_usize(sub_hashes.len().to_le());
+    for elt in sub_hashes {
+        elt.hash(&mut hasher);
+    }
     std::hash::Hasher::finish(&hasher)
 }
 

compiler/rustc_codegen_cranelift/build_system/tests.rs

@@ -133,8 +133,8 @@ pub(crate) static REGEX: CargoProject = CargoProject::new(&REGEX_REPO.source_dir
 pub(crate) static PORTABLE_SIMD_REPO: GitRepo = GitRepo::github(
     "rust-lang",
     "portable-simd",
-    "97007cc2e70df8c97326ce896a79e2f0ce4dd98b",
+    "5794c837bc605c4cd9dbb884285976dfdb293cce",
-    "e54a16035cedf205",
+    "a64d8fdd0ed0d9c4",
     "portable-simd",
 );
 

compiler/rustc_codegen_cranelift/patches/0022-coretests-Disable-not-compiling-tests.patch

@@ -39,6 +39,6 @@ index 42a26ae..5ac1042 100644
 +#![cfg(test)]
  #![feature(alloc_layout_extra)]
  #![feature(array_chunks)]
- #![feature(array_methods)]
+ #![feature(array_windows)]
 --
 2.21.0 (Apple Git-122)

compiler/rustc_codegen_cranelift/patches/0023-coretests-Ignore-failing-tests.patch

@@ -1,26 +0,0 @@
-From dd82e95c9de212524e14fc60155de1ae40156dfc Mon Sep 17 00:00:00 2001
-From: bjorn3 <bjorn3@users.noreply.github.com>
-Date: Sun, 24 Nov 2019 15:34:06 +0100
-Subject: [PATCH] [core] Ignore failing tests
-
----
- library/core/tests/iter.rs       |  4 ++++
- library/core/tests/num/bignum.rs | 10 ++++++++++
- library/core/tests/num/mod.rs    |  5 +++--
- library/core/tests/time.rs       |  1 +
- 4 files changed, 18 insertions(+), 2 deletions(-)
-
-diff --git a/atomic.rs b/atomic.rs
-index 13b12db..96fe4b9 100644
---- a/atomic.rs
-+++ b/atomic.rs
-@@ -185,6 +185,7 @@ fn ptr_bitops() {
- }
- 
- #[test]
-+#[cfg_attr(target_arch = "s390x", ignore)] // s390x backend doesn't support stack alignment >8 bytes
- #[cfg(any(not(target_arch = "arm"), target_os = "linux"))] // Missing intrinsic in compiler-builtins
- fn ptr_bitops_tagging() {
-     #[repr(align(16))]
--- 
-2.21.0 (Apple Git-122)

compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml

@@ -150,9 +150,9 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.3.3"
+version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
+checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-alloc",
@@ -161,9 +161,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.150"
+version = "0.2.153"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c"
+checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
 dependencies = [
  "rustc-std-workspace-core",
 ]
@@ -398,6 +398,7 @@ version = "0.0.0"
 dependencies = [
  "core",
  "getopts",
+ "libc",
  "panic_abort",
  "panic_unwind",
  "std",

compiler/rustc_codegen_cranelift/rust-toolchain

@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2024-01-26"
+channel = "nightly-2024-03-08"
 components = ["rust-src", "rustc-dev", "llvm-tools"]

compiler/rustc_codegen_cranelift/rustfmt.toml

@@ -1,5 +1,4 @@
 ignore = [
-    "y.rs",
     "example/gen_block_iterate.rs", # uses edition 2024
 ]
 

compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh

@@ -10,12 +10,26 @@ pushd rust
 
 command -v rg >/dev/null 2>&1 || cargo install ripgrep
 
+# FIXME(rust-lang/rust#122196) fix stage0 rmake.rs run-make tests and remove
+# this workaround
+for test in $(ls tests/run-make); do
+  if [[ -e "tests/run-make/$test/rmake.rs" ]]; then
+    rm -r "tests/run-make/$test"
+  fi
+done
+
+# FIXME remove this workaround once ICE tests no longer emit an outdated nightly message
+for test in $(rg -i --files-with-matches "//@(\[.*\])? failure-status: 101" tests/ui); do
+  echo "rm $test"
+  rm $test
+done
+
 rm -r tests/ui/{unsized-locals/,lto/,linkage*} || true
 for test in $(rg --files-with-matches "lto" tests/{codegen-units,ui,incremental}); do
   rm $test
 done
 
-for test in $(rg -i --files-with-matches "//(\[\w+\])?~[^\|]*\s*ERR|// error-pattern:|// build-fail|// run-fail|-Cllvm-args" tests/ui); do
+for test in $(rg -i --files-with-matches "//(\[\w+\])?~[^\|]*\s*ERR|//@ error-pattern:|//@(\[.*\])? build-fail|//@(\[.*\])? run-fail|-Cllvm-args" tests/ui); do
   rm $test
 done
 
@@ -43,8 +57,8 @@ rm tests/ui/proc-macro/allowed-signatures.rs
 rm tests/ui/proc-macro/no-mangle-in-proc-macro-issue-111888.rs
 
 # vendor intrinsics
-rm tests/ui/sse2.rs # CodegenBackend::target_features not yet implemented
 rm tests/ui/simd/array-type.rs # "Index argument for `simd_insert` is not a constant"
+rm tests/ui/asm/x86_64/evex512-implicit-feature.rs # unimplemented AVX512 x86 vendor intrinsic
 
 # exotic linkages
 rm tests/ui/issues/issue-33992.rs # unsupported linkages
@@ -62,14 +76,12 @@ rm -r tests/run-pass-valgrind/unsized-locals
 # misc unimplemented things
 rm tests/ui/intrinsics/intrinsic-nearby.rs # unimplemented nearbyintf32 and nearbyintf64 intrinsics
 rm tests/ui/target-feature/missing-plusminus.rs # error not implemented
-rm tests/ui/fn/dyn-fn-alignment.rs # wants a 256 byte alignment
 rm -r tests/run-make/emit-named-files # requires full --emit support
 rm -r tests/run-make/repr128-dwarf # debuginfo test
 rm -r tests/run-make/split-debuginfo # same
 rm -r tests/run-make/symbols-include-type-name # --emit=asm not supported
 rm -r tests/run-make/target-specs # i686 not supported by Cranelift
 rm -r tests/run-make/mismatching-target-triples # same
-rm tests/ui/asm/x86_64/issue-82869.rs # vector regs in inline asm not yet supported
 rm tests/ui/asm/x86_64/issue-96797.rs # const and sym inline asm operands don't work entirely correctly
 
 # requires LTO
@@ -109,8 +121,6 @@ rm -r tests/run-make/optimization-remarks-dir # remarks are LLVM specific
 rm tests/ui/mir/mir_misc_casts.rs # depends on deduplication of constants
 rm tests/ui/mir/mir_raw_fat_ptr.rs # same
 rm tests/ui/consts/issue-33537.rs # same
-rm tests/ui/layout/valid_range_oob.rs # different ICE message
-rm tests/ui/const-generics/generic_const_exprs/issue-80742.rs # gives error instead of ICE with cg_clif
 
 # rustdoc-clif passes extra args, suppressing the help message when no args are passed
 rm -r tests/run-make/issue-88756-default-output
@@ -119,15 +129,12 @@ rm -r tests/run-make/issue-88756-default-output
 # should work when using ./x.py test the way it is intended
 # ============================================================
 rm -r tests/run-make/remap-path-prefix-dwarf # requires llvm-dwarfdump
-rm -r tests/ui/consts/missing_span_in_backtrace.rs # expects sysroot source to be elsewhere
 
 # genuine bugs
 # ============
 rm tests/incremental/spike-neg1.rs # errors out for some reason
 rm tests/incremental/spike-neg2.rs # same
 
-rm tests/ui/simd/simd-bitmask.rs # simd_bitmask doesn't implement [u*; N] return type
-
 rm -r tests/run-make/issue-51671 # wrong filename given in case of --emit=obj
 rm -r tests/run-make/issue-30063 # same
 rm -r tests/run-make/multiple-emits # same
@@ -145,6 +152,7 @@ rm tests/ui/codegen/subtyping-enforces-type-equality.rs # assert_assignable bug
 # ======================
 rm tests/ui/backtrace.rs # TODO warning
 rm tests/ui/process/nofile-limit.rs # TODO some AArch64 linking issue
+rm tests/ui/async-await/async-closures/once.rs # FIXME bug in the rustc FnAbi calculation code
 
 rm tests/ui/stdio-is-blocking.rs # really slow with unoptimized libstd
 

compiler/rustc_codegen_cranelift/src/base.rs

@@ -445,7 +445,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                 template,
                 operands,
                 options,
-                destination,
+                targets,
                 line_spans: _,
                 unwind: _,
             } => {
@@ -456,13 +456,25 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                     );
                 }
 
+                let have_labels = if options.contains(InlineAsmOptions::NORETURN) {
+                    !targets.is_empty()
+                } else {
+                    targets.len() > 1
+                };
+                if have_labels {
+                    fx.tcx.dcx().span_fatal(
+                        source_info.span,
+                        "cranelift doesn't support labels in inline assembly.",
+                    );
+                }
+
                 crate::inline_asm::codegen_inline_asm_terminator(
                     fx,
                     source_info.span,
                     template,
                     operands,
                     *options,
-                    *destination,
+                    targets.get(0).copied(),
                 );
             }
             TerminatorKind::UnwindTerminate(reason) => {

compiler/rustc_codegen_cranelift/src/common.rs

@@ -392,18 +392,25 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
     }
 
     pub(crate) fn create_stack_slot(&mut self, size: u32, align: u32) -> Pointer {
-        if align <= 16 {
+        let abi_align = if self.tcx.sess.target.arch == "s390x" { 8 } else { 16 };
+        if align <= abi_align {
             let stack_slot = self.bcx.create_sized_stack_slot(StackSlotData {
                 kind: StackSlotKind::ExplicitSlot,
-                // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
+                // FIXME Don't force the size to a multiple of <abi_align> bytes once Cranelift gets
-                // specify stack slot alignment.
+                // a way to specify stack slot alignment.
-                size: (size + 15) / 16 * 16,
+                size: (size + abi_align - 1) / abi_align * abi_align,
             });
             Pointer::stack_slot(stack_slot)
         } else {
             // Alignment is too big to handle using the above hack. Dynamically realign a stack slot
             // instead. This wastes some space for the realignment.
-            let base_ptr = self.create_stack_slot(size + align, 16).get_addr(self);
+            let stack_slot = self.bcx.create_sized_stack_slot(StackSlotData {
+                kind: StackSlotKind::ExplicitSlot,
+                // FIXME Don't force the size to a multiple of <abi_align> bytes once Cranelift gets
+                // a way to specify stack slot alignment.
+                size: (size + align) / abi_align * abi_align,
+            });
+            let base_ptr = self.bcx.ins().stack_addr(self.pointer_type, stack_slot, 0);
             let misalign_offset = self.bcx.ins().urem_imm(base_ptr, i64::from(align));
             let realign_offset = self.bcx.ins().irsub_imm(misalign_offset, i64::from(align));
             Pointer::new(self.bcx.ins().iadd(base_ptr, realign_offset))

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -372,7 +372,13 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
         }
 
         let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec();
+        if bytes.is_empty() {
+            // FIXME(bytecodealliance/wasmtime#7918) cranelift-jit has a bug where it causes UB on
+            // empty data objects
+            data.define(Box::new([0]));
+        } else {
             data.define(bytes.into_boxed_slice());
+        }
 
         for &(offset, prov) in alloc.provenance().ptrs().iter() {
             let alloc_id = prov.alloc_id();

compiler/rustc_codegen_cranelift/src/global_asm.rs

@@ -78,7 +78,8 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String,
                         InlineAsmOperand::In { .. }
                         | InlineAsmOperand::Out { .. }
                         | InlineAsmOperand::InOut { .. }
-                        | InlineAsmOperand::SplitInOut { .. } => {
+                        | InlineAsmOperand::SplitInOut { .. }
+                        | InlineAsmOperand::Label { .. } => {
                             span_bug!(op_sp, "invalid operand type for global_asm!")
                         }
                     }

compiler/rustc_codegen_cranelift/src/inline_asm.rs

@@ -129,6 +129,9 @@ pub(crate) fn codegen_inline_asm_terminator<'tcx>(
                 let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
                 CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance).name.to_owned() }
             }
+            InlineAsmOperand::Label { .. } => {
+                span_bug!(span, "asm! label operands are not yet supported");
+            }
         })
         .collect::<Vec<_>>();
 

compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs

@@ -170,6 +170,65 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             }
         }
 
+        "llvm.x86.sse.add.ss" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_add_ss&ig_expand=171
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            assert_eq!(a.layout(), b.layout());
+            assert_eq!(a.layout(), ret.layout());
+            let layout = a.layout();
+
+            let (_, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+            assert!(lane_ty.is_floating_point());
+            let ret_lane_layout = fx.layout_of(lane_ty);
+
+            ret.write_cvalue(fx, a);
+
+            let a_lane = a.value_lane(fx, 0).load_scalar(fx);
+            let b_lane = b.value_lane(fx, 0).load_scalar(fx);
+
+            let res = fx.bcx.ins().fadd(a_lane, b_lane);
+
+            let res_lane = CValue::by_val(res, ret_lane_layout);
+            ret.place_lane(fx, 0).write_cvalue(fx, res_lane);
+        }
+
+        "llvm.x86.sse.sqrt.ps" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sqrt_ps&ig_expand=6245
+            intrinsic_args!(fx, args => (a); intrinsic);
+
+            // FIXME use vector instructions when possible
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| {
+                fx.bcx.ins().sqrt(lane)
+            });
+        }
+
+        "llvm.x86.sse.max.ps" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_max_ps&ig_expand=4357
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            simd_pair_for_each_lane(
+                fx,
+                a,
+                b,
+                ret,
+                &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| fx.bcx.ins().fmax(a_lane, b_lane),
+            );
+        }
+
+        "llvm.x86.sse.min.ps" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_min_ps&ig_expand=4489
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            simd_pair_for_each_lane(
+                fx,
+                a,
+                b,
+                ret,
+                &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| fx.bcx.ins().fmin(a_lane, b_lane),
+            );
+        }
+
         "llvm.x86.sse.cmp.ps" | "llvm.x86.sse2.cmp.pd" => {
             let (x, y, kind) = match args {
                 [x, y, kind] => (x, y, kind),
@@ -1067,6 +1126,122 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             );
         }
 
+        "llvm.x86.sha1rnds4" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha1rnds4_epu32&ig_expand=5877
+            intrinsic_args!(fx, args => (a, b, _func); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            let func = if let Some(func) =
+                crate::constant::mir_operand_get_const_val(fx, &args[2].node)
+            {
+                func
+            } else {
+                fx.tcx
+                    .dcx()
+                    .span_fatal(span, "Func argument for `_mm_sha1rnds4_epu32` is not a constant");
+            };
+
+            let func = func.try_to_u8().unwrap_or_else(|_| panic!("kind not scalar: {:?}", func));
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String(format!("sha1rnds4 xmm1, xmm2, {func}"))],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)),
+                        value: b,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.sha1msg1" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha1msg1_epu32&ig_expand=5874
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String("sha1msg1 xmm1, xmm2".to_string())],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)),
+                        value: b,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.sha1msg2" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha1msg2_epu32&ig_expand=5875
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String("sha1msg2 xmm1, xmm2".to_string())],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)),
+                        value: b,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.sha1nexte" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha1nexte_epu32&ig_expand=5876
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String("sha1nexte xmm1, xmm2".to_string())],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)),
+                        value: b,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
         "llvm.x86.sha256rnds2" => {
             // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha256rnds2_epu32&ig_expand=5977
             intrinsic_args!(fx, args => (a, b, k); intrinsic);

compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

@@ -391,12 +391,15 @@ fn codegen_float_intrinsic_call<'tcx>(
         | sym::ceilf32
         | sym::ceilf64
         | sym::truncf32
-        | sym::truncf64 => {
+        | sym::truncf64
+        | sym::sqrtf32
+        | sym::sqrtf64 => {
             let val = match intrinsic {
                 sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(args[0]),
                 sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(args[0]),
                 sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(args[0]),
                 sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(args[0]),
+                sym::sqrtf32 | sym::sqrtf64 => fx.bcx.ins().sqrt(args[0]),
                 _ => unreachable!(),
             };
 

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -853,7 +853,13 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             };
 
             for lane in 0..lane_count {
-                let m_lane = fx.bcx.ins().ushr_imm(m, u64::from(lane) as i64);
+                // The bit order of the mask depends on the byte endianness, LSB-first for
+                // little endian and MSB-first for big endian.
+                let mask_lane = match fx.tcx.sess.target.endian {
+                    Endian::Big => lane_count - 1 - lane,
+                    Endian::Little => lane,
+                };
+                let m_lane = fx.bcx.ins().ushr_imm(m, u64::from(mask_lane) as i64);
                 let m_lane = fx.bcx.ins().band_imm(m_lane, 1);
                 let a_lane = a.value_lane(fx, lane).load_scalar(fx);
                 let b_lane = b.value_lane(fx, lane).load_scalar(fx);

compiler/rustc_codegen_cranelift/y.rs

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-#![deny(unsafe_code)] /*This line is ignored by bash
-# This block is ignored by rustc
-echo "Warning: y.rs is a deprecated alias for y.sh" 1>&2
-exec ./y.sh "$@"
-*/

compiler/rustc_codegen_gcc/src/asm.rs

@@ -107,7 +107,7 @@ enum ConstraintOrRegister {
 
 
 impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
-    fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], instance: Instance<'_>, _dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>) {
+    fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], instance: Instance<'_>, dest: Option<Self::BasicBlock>, _catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>) {
         if options.contains(InlineAsmOptions::MAY_UNWIND) {
             self.sess().dcx()
                 .create_err(UnwindingInlineAsm { span: span[0] })
@@ -126,6 +126,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         // added to `outputs.len()`
         let mut inputs = vec![];
 
+        // GCC index of a label equals its position in the array added to
+        // `outputs.len() + inputs.len()`.
+        let mut labels = vec![];
+
         // Clobbers collected from `out("explicit register") _` and `inout("expl_reg") var => _`
         let mut clobbers = vec![];
 
@@ -269,6 +273,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                     // some targets to add a leading underscore (Mach-O).
                     constants_len += self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len();
                 }
+
+                InlineAsmOperandRef::Label { label } => {
+                    labels.push(label);
+                }
             }
         }
 
@@ -368,6 +376,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                 InlineAsmOperandRef::Const { .. } => {
                     // processed in the previous pass
                 }
+
+                InlineAsmOperandRef::Label { .. } => {
+                    // processed in the previous pass
+                }
             }
         }
 
@@ -454,6 +466,14 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                         InlineAsmOperandRef::Const { ref string } => {
                             template_str.push_str(string);
                         }
+
+                        InlineAsmOperandRef::Label { label } => {
+                            let label_gcc_index = labels.iter()
+                                .position(|&l| l == label)
+                                .expect("wrong rust index");
+                            let gcc_index = label_gcc_index + outputs.len() + inputs.len();
+                            push_to_template(Some('l'), gcc_index);
+                        }
                     }
                 }
             }
@@ -466,7 +486,12 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         // 4. Generate Extended Asm block
 
         let block = self.llbb();
-        let extended_asm = block.add_extended_asm(None, &template_str);
+        let extended_asm = if let Some(dest) = dest {
+            assert!(!labels.is_empty());
+            block.end_with_extended_asm_goto(None, &template_str, &labels, Some(dest))
+        } else {
+            block.add_extended_asm(None, &template_str)
+        };
 
         for op in &outputs {
             extended_asm.add_output_operand(None, &op.to_constraint(), op.tmp_var);
@@ -494,7 +519,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         if !options.contains(InlineAsmOptions::NOSTACK) {
             // TODO(@Commeownist): figure out how to align stack
         }
-        if options.contains(InlineAsmOptions::NORETURN) {
+        if dest.is_none() && options.contains(InlineAsmOptions::NORETURN) {
             let builtin_unreachable = self.context.get_builtin_function("__builtin_unreachable");
             let builtin_unreachable: RValue<'gcc> = unsafe { std::mem::transmute(builtin_unreachable) };
             self.call(self.type_void(), None, None, builtin_unreachable, &[], None);

compiler/rustc_codegen_llvm/src/asm.rs

@@ -28,7 +28,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         options: InlineAsmOptions,
         line_spans: &[Span],
         instance: Instance<'_>,
-        dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
+        dest: Option<Self::BasicBlock>,
+        catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
     ) {
         let asm_arch = self.tcx.sess.asm_arch.unwrap();
 
@@ -165,6 +166,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         }
 
         // Build the template string
+        let mut labels = vec![];
         let mut template_str = String::new();
         for piece in template {
             match *piece {
@@ -205,6 +207,11 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                             // Only emit the raw symbol name
                             template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx]));
                         }
+                        InlineAsmOperandRef::Label { label } => {
+                            template_str.push_str(&format!("${{{}:l}}", constraints.len()));
+                            constraints.push("!i".to_owned());
+                            labels.push(label);
+                        }
                     }
                 }
             }
@@ -292,12 +299,14 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             &constraints.join(","),
             &inputs,
             output_type,
+            &labels,
             volatile,
             alignstack,
             dialect,
             line_spans,
             options.contains(InlineAsmOptions::MAY_UNWIND),
-            dest_catch_funclet,
+            dest,
+            catch_funclet,
         )
         .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
 
@@ -317,7 +326,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs });
 
         // Switch to the 'normal' basic block if we did an `invoke` instead of a `call`
-        if let Some((dest, _, _)) = dest_catch_funclet {
+        if let Some(dest) = dest {
             self.switch_to_block(dest);
         }
 
@@ -415,16 +424,14 @@ pub(crate) fn inline_asm_call<'ll>(
     cons: &str,
     inputs: &[&'ll Value],
     output: &'ll llvm::Type,
+    labels: &[&'ll llvm::BasicBlock],
     volatile: bool,
     alignstack: bool,
     dia: llvm::AsmDialect,
     line_spans: &[Span],
     unwind: bool,
-    dest_catch_funclet: Option<(
+    dest: Option<&'ll llvm::BasicBlock>,
-        &'ll llvm::BasicBlock,
+    catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>,
-        &'ll llvm::BasicBlock,
-        Option<&Funclet<'ll>>,
-    )>,
 ) -> Option<&'ll Value> {
     let volatile = if volatile { llvm::True } else { llvm::False };
     let alignstack = if alignstack { llvm::True } else { llvm::False };
@@ -457,8 +464,11 @@ pub(crate) fn inline_asm_call<'ll>(
                 can_throw,
             );
 
-            let call = if let Some((dest, catch, funclet)) = dest_catch_funclet {
+            let call = if !labels.is_empty() {
-                bx.invoke(fty, None, None, v, inputs, dest, catch, funclet)
+                assert!(catch_funclet.is_none());
+                bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None)
+            } else if let Some((catch, funclet)) = catch_funclet {
+                bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet)
             } else {
                 bx.call(fty, None, None, v, inputs, None)
             };

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1538,6 +1538,58 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         }
     }
 
+    pub(crate) fn callbr(
+        &mut self,
+        llty: &'ll Type,
+        fn_attrs: Option<&CodegenFnAttrs>,
+        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
+        llfn: &'ll Value,
+        args: &[&'ll Value],
+        default_dest: &'ll BasicBlock,
+        indirect_dest: &[&'ll BasicBlock],
+        funclet: Option<&Funclet<'ll>>,
+    ) -> &'ll Value {
+        debug!("invoke {:?} with args ({:?})", llfn, args);
+
+        let args = self.check_call("callbr", llty, llfn, args);
+        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
+        let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw);
+        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
+        if let Some(funclet_bundle) = funclet_bundle {
+            bundles.push(funclet_bundle);
+        }
+
+        // Emit CFI pointer type membership test
+        self.cfi_type_test(fn_attrs, fn_abi, llfn);
+
+        // Emit KCFI operand bundle
+        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn);
+        let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw);
+        if let Some(kcfi_bundle) = kcfi_bundle {
+            bundles.push(kcfi_bundle);
+        }
+
+        let callbr = unsafe {
+            llvm::LLVMRustBuildCallBr(
+                self.llbuilder,
+                llty,
+                llfn,
+                default_dest,
+                indirect_dest.as_ptr(),
+                indirect_dest.len() as c_uint,
+                args.as_ptr(),
+                args.len() as c_uint,
+                bundles.as_ptr(),
+                bundles.len() as c_uint,
+                UNNAMED,
+            )
+        };
+        if let Some(fn_abi) = fn_abi {
+            fn_abi.apply_attrs_callsite(self, callbr);
+        }
+        callbr
+    }
+
     // Emits CFI pointer type membership tests.
     fn cfi_type_test(
         &mut self,

compiler/rustc_codegen_llvm/src/context.rs

@@ -260,35 +260,29 @@ pub unsafe fn create_module<'ll>(
     }
 
     if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection {
-        let behavior = if llvm_version >= (15, 0, 0) {
-            llvm::LLVMModFlagBehavior::Min
-        } else {
-            llvm::LLVMModFlagBehavior::Error
-        };
-
         if sess.target.arch == "aarch64" {
             llvm::LLVMRustAddModuleFlag(
                 llmod,
-                behavior,
+                llvm::LLVMModFlagBehavior::Min,
                 c"branch-target-enforcement".as_ptr().cast(),
                 bti.into(),
             );
             llvm::LLVMRustAddModuleFlag(
                 llmod,
-                behavior,
+                llvm::LLVMModFlagBehavior::Min,
                 c"sign-return-address".as_ptr().cast(),
                 pac_ret.is_some().into(),
             );
             let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A });
             llvm::LLVMRustAddModuleFlag(
                 llmod,
-                behavior,
+                llvm::LLVMModFlagBehavior::Min,
                 c"sign-return-address-all".as_ptr().cast(),
                 pac_opts.leaf.into(),
             );
             llvm::LLVMRustAddModuleFlag(
                 llmod,
-                behavior,
+                llvm::LLVMModFlagBehavior::Min,
                 c"sign-return-address-with-bkey".as_ptr().cast(),
                 u32::from(pac_opts.key == PAuthKey::B),
             );

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -448,12 +448,14 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     constraint,
                     inputs,
                     self.type_void(),
+                    &[],
                     true,
                     false,
                     llvm::AsmDialect::Att,
                     &[span],
                     false,
                     None,
+                    None,
                 )
                 .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`"));
 

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1616,6 +1616,20 @@ extern "C" {
         Name: *const c_char,
     ) -> &'a Value;
 
+    pub fn LLVMRustBuildCallBr<'a>(
+        B: &Builder<'a>,
+        Ty: &'a Type,
+        Fn: &'a Value,
+        DefaultDest: &'a BasicBlock,
+        IndirectDests: *const &'a BasicBlock,
+        NumIndirectDests: c_uint,
+        Args: *const &'a Value,
+        NumArgs: c_uint,
+        OpBundles: *const &OperandBundleDef<'a>,
+        NumOpBundles: c_uint,
+        Name: *const c_char,
+    ) -> &'a Value;
+
     pub fn LLVMRustSetFastMath(Instr: &Value);
     pub fn LLVMRustSetAlgebraicMath(Instr: &Value);
     pub fn LLVMRustSetAllowReassoc(Instr: &Value);

compiler/rustc_codegen_ssa/src/base.rs

@@ -510,11 +510,13 @@ fn get_argc_argv<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         // Params for UEFI
         let param_handle = bx.get_param(0);
         let param_system_table = bx.get_param(1);
+        let ptr_size = bx.tcx().data_layout.pointer_size;
+        let ptr_align = bx.tcx().data_layout.pointer_align.abi;
         let arg_argc = bx.const_int(cx.type_isize(), 2);
-        let arg_argv = bx.alloca(cx.type_array(cx.type_ptr(), 2), Align::ONE);
+        let arg_argv = bx.alloca(cx.type_array(cx.type_ptr(), 2), ptr_align);
-        bx.store(param_handle, arg_argv, Align::ONE);
+        bx.store(param_handle, arg_argv, ptr_align);
-        let arg_argv_el1 = bx.gep(cx.type_ptr(), arg_argv, &[bx.const_int(cx.type_int(), 1)]);
+        let arg_argv_el1 = bx.inbounds_ptradd(arg_argv, bx.const_usize(ptr_size.bytes()));
-        bx.store(param_system_table, arg_argv_el1, Align::ONE);
+        bx.store(param_system_table, arg_argv_el1, ptr_align);
         (arg_argc, arg_argv)
     } else if cx.sess().target.main_needs_argc_argv {
         // Params from native `main()` used as args for rust start function

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -264,7 +264,8 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
             mir::UnwindAction::Unreachable => None,
         };
 
-        if let Some(cleanup) = unwind_target {
+        if operands.iter().any(|x| matches!(x, InlineAsmOperandRef::Label { .. })) {
+            assert!(unwind_target.is_none());
             let ret_llbb = if let Some(target) = destination {
                 fx.llbb(target)
             } else {
@@ -277,11 +278,29 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
                 options,
                 line_spans,
                 instance,
-                Some((ret_llbb, cleanup, self.funclet(fx))),
+                Some(ret_llbb),
+                None,
+            );
+            MergingSucc::False
+        } else if let Some(cleanup) = unwind_target {
+            let ret_llbb = if let Some(target) = destination {
+                fx.llbb(target)
+            } else {
+                fx.unreachable_block()
+            };
+
+            bx.codegen_inline_asm(
+                template,
+                operands,
+                options,
+                line_spans,
+                instance,
+                Some(ret_llbb),
+                Some((cleanup, self.funclet(fx))),
             );
             MergingSucc::False
         } else {
-            bx.codegen_inline_asm(template, operands, options, line_spans, instance, None);
+            bx.codegen_inline_asm(template, operands, options, line_spans, instance, None, None);
 
             if let Some(target) = destination {
                 self.funclet_br(fx, bx, target, mergeable_succ)
@@ -1100,7 +1119,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         operands: &[mir::InlineAsmOperand<'tcx>],
         options: ast::InlineAsmOptions,
         line_spans: &[Span],
-        destination: Option<mir::BasicBlock>,
+        targets: &[mir::BasicBlock],
         unwind: mir::UnwindAction,
         instance: Instance<'_>,
         mergeable_succ: bool,
@@ -1152,6 +1171,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 mir::InlineAsmOperand::SymStatic { def_id } => {
                     InlineAsmOperandRef::SymStatic { def_id }
                 }
+                mir::InlineAsmOperand::Label { target_index } => {
+                    InlineAsmOperandRef::Label { label: self.llbb(targets[target_index]) }
+                }
             })
             .collect();
 
@@ -1162,7 +1184,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             &operands,
             options,
             line_spans,
-            destination,
+            if options.contains(InlineAsmOptions::NORETURN) {
+                None
+            } else {
+                targets.get(0).copied()
+            },
             unwind,
             instance,
             mergeable_succ,
@@ -1318,7 +1344,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 ref operands,
                 options,
                 line_spans,
-                destination,
+                ref targets,
                 unwind,
             } => self.codegen_asm_terminator(
                 helper,
@@ -1328,7 +1354,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 operands,
                 options,
                 line_spans,
-                destination,
+                targets,
                 unwind,
                 self.instance,
                 mergeable_succ(),

compiler/rustc_codegen_ssa/src/mono_item.rs

@@ -76,7 +76,8 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
                             hir::InlineAsmOperand::In { .. }
                             | hir::InlineAsmOperand::Out { .. }
                             | hir::InlineAsmOperand::InOut { .. }
-                            | hir::InlineAsmOperand::SplitInOut { .. } => {
+                            | hir::InlineAsmOperand::SplitInOut { .. }
+                            | hir::InlineAsmOperand::Label { .. } => {
                                 span_bug!(*op_sp, "invalid operand type for global_asm!")
                             }
                         })

compiler/rustc_codegen_ssa/src/traits/asm.rs

@@ -33,6 +33,9 @@ pub enum InlineAsmOperandRef<'tcx, B: BackendTypes + ?Sized> {
     SymStatic {
         def_id: DefId,
     },
+    Label {
+        label: B::BasicBlock,
+    },
 }
 
 #[derive(Debug)]
@@ -51,7 +54,8 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes {
         options: InlineAsmOptions,
         line_spans: &[Span],
         instance: Instance<'_>,
-        dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
+        dest: Option<Self::BasicBlock>,
+        catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
     );
 }
 

compiler/rustc_const_eval/messages.ftl

@@ -232,9 +232,6 @@ const_eval_non_const_fn_call =
 const_eval_non_const_impl =
     impl defined here, but it is not `const`
 
-const_eval_noreturn_asm_returned =
-    returned from noreturn inline assembly
-
 const_eval_not_enough_caller_args =
     calling a function with fewer arguments than it requires
 

compiler/rustc_const_eval/src/const_eval/machine.rs

@@ -227,7 +227,7 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
         if self.tcx.has_attr(def_id, sym::rustc_const_panic_str)
             || Some(def_id) == self.tcx.lang_items().begin_panic_fn()
         {
-            let args = self.copy_fn_args(args)?;
+            let args = self.copy_fn_args(args);
             // &str or &&str
             assert!(args.len() == 1);
 
@@ -254,7 +254,7 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
 
             return Ok(Some(new_instance));
         } else if Some(def_id) == self.tcx.lang_items().align_offset_fn() {
-            let args = self.copy_fn_args(args)?;
+            let args = self.copy_fn_args(args);
             // For align_offset, we replace the function call if the pointer has no address.
             match self.align_offset(instance, &args, dest, ret)? {
                 ControlFlow::Continue(()) => return Ok(Some(instance)),

compiler/rustc_const_eval/src/interpret/discriminant.rs

@@ -10,6 +10,8 @@ use super::{ImmTy, InterpCx, InterpResult, Machine, Readable, Scalar, Writeable}
 
 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     /// Writes the discriminant of the given variant.
+    ///
+    /// If the variant is uninhabited, this is UB.
     #[instrument(skip(self), level = "trace")]
     pub fn write_discriminant(
         &mut self,
@@ -102,6 +104,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
     /// Read discriminant, return the runtime value as well as the variant index.
     /// Can also legally be called on non-enums (e.g. through the discriminant_value intrinsic)!
+    ///
+    /// Will never return an uninhabited variant.
     #[instrument(skip(self), level = "trace")]
     pub fn read_discriminant(
         &self,
@@ -244,7 +248,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 variant
             }
         };
-        // For consistency with `write_discriminant`, and to make sure that `project_downcast` cannot fail due to strange layouts, we declare immediate UB for uninhabited variants.
+        // Reading the discriminant of an uninhabited variant is UB. This is the basis for the
+        // `uninhabited_enum_branching` MIR pass. It also ensures consistency with
+        // `write_discriminant`.
         if op.layout().for_variant(self, index).abi.is_uninhabited() {
             throw_ub!(UninhabitedEnumVariantRead(index))
         }

compiler/rustc_const_eval/src/interpret/machine.rs

@@ -374,11 +374,17 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized {
         kind: Option<MemoryKind<Self::MemoryKind>>,
     ) -> InterpResult<'tcx, Cow<'b, Allocation<Self::Provenance, Self::AllocExtra, Self::Bytes>>>;
 
+    /// Evaluate the inline assembly.
+    ///
+    /// This should take care of jumping to the next block (one of `targets`) when asm goto
+    /// is triggered, `targets[0]` when the assembly falls through, or diverge in case of
+    /// `InlineAsmOptions::NORETURN` being set.
     fn eval_inline_asm(
         _ecx: &mut InterpCx<'mir, 'tcx, Self>,
         _template: &'tcx [InlineAsmTemplatePiece],
         _operands: &[mir::InlineAsmOperand<'tcx>],
         _options: InlineAsmOptions,
+        _targets: &[mir::BasicBlock],
     ) -> InterpResult<'tcx> {
         throw_unsup_format!("inline assembly is not supported")
     }
@@ -466,11 +472,11 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized {
     /// argument/return value was actually copied or passed in-place..
     fn protect_in_place_function_argument(
         ecx: &mut InterpCx<'mir, 'tcx, Self>,
-        place: &PlaceTy<'tcx, Self::Provenance>,
+        mplace: &MPlaceTy<'tcx, Self::Provenance>,
     ) -> InterpResult<'tcx> {
         // Without an aliasing model, all we can do is put `Uninit` into the place.
         // Conveniently this also ensures that the place actually points to suitable memory.
-        ecx.write_uninit(place)
+        ecx.write_uninit(mplace)
     }
 
     /// Called immediately before a new stack frame gets pushed.

compiler/rustc_const_eval/src/interpret/terminator.rs

@@ -1,6 +1,7 @@
 use std::borrow::Cow;
 
-use rustc_ast::ast::InlineAsmOptions;
+use either::Either;
+
 use rustc_middle::{
     mir,
     ty::{
@@ -30,14 +31,14 @@ pub enum FnArg<'tcx, Prov: Provenance = CtfeProvenance> {
     Copy(OpTy<'tcx, Prov>),
     /// Allow for the argument to be passed in-place: destroy the value originally stored at that place and
     /// make the place inaccessible for the duration of the function call.
-    InPlace(PlaceTy<'tcx, Prov>),
+    InPlace(MPlaceTy<'tcx, Prov>),
 }
 
 impl<'tcx, Prov: Provenance> FnArg<'tcx, Prov> {
     pub fn layout(&self) -> &TyAndLayout<'tcx> {
         match self {
             FnArg::Copy(op) => &op.layout,
-            FnArg::InPlace(place) => &place.layout,
+            FnArg::InPlace(mplace) => &mplace.layout,
         }
     }
 }
@@ -45,13 +46,10 @@ impl<'tcx, Prov: Provenance> FnArg<'tcx, Prov> {
 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     /// Make a copy of the given fn_arg. Any `InPlace` are degenerated to copies, no protection of the
     /// original memory occurs.
-    pub fn copy_fn_arg(
+    pub fn copy_fn_arg(&self, arg: &FnArg<'tcx, M::Provenance>) -> OpTy<'tcx, M::Provenance> {
-        &self,
-        arg: &FnArg<'tcx, M::Provenance>,
-    ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
         match arg {
-            FnArg::Copy(op) => Ok(op.clone()),
+            FnArg::Copy(op) => op.clone(),
-            FnArg::InPlace(place) => self.place_to_op(place),
+            FnArg::InPlace(mplace) => mplace.clone().into(),
         }
     }
 
@@ -60,7 +58,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     pub fn copy_fn_args(
         &self,
         args: &[FnArg<'tcx, M::Provenance>],
-    ) -> InterpResult<'tcx, Vec<OpTy<'tcx, M::Provenance>>> {
+    ) -> Vec<OpTy<'tcx, M::Provenance>> {
         args.iter().map(|fn_arg| self.copy_fn_arg(fn_arg)).collect()
     }
 
@@ -71,7 +69,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     ) -> InterpResult<'tcx, FnArg<'tcx, M::Provenance>> {
         Ok(match arg {
             FnArg::Copy(op) => FnArg::Copy(self.project_field(op, field)?),
-            FnArg::InPlace(place) => FnArg::InPlace(self.project_field(place, field)?),
+            FnArg::InPlace(mplace) => FnArg::InPlace(self.project_field(mplace, field)?),
         })
     }
 
@@ -224,15 +222,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 terminator.kind
             ),
 
-            InlineAsm { template, ref operands, options, destination, .. } => {
+            InlineAsm { template, ref operands, options, ref targets, .. } => {
-                M::eval_inline_asm(self, template, operands, options)?;
+                M::eval_inline_asm(self, template, operands, options, targets)?;
-                if options.contains(InlineAsmOptions::NORETURN) {
-                    throw_ub_custom!(fluent::const_eval_noreturn_asm_returned);
-                }
-                self.go_to_block(
-                    destination
-                        .expect("InlineAsm terminators without noreturn must have a destination"),
-                )
             }
         }
 
@@ -246,10 +237,36 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     ) -> InterpResult<'tcx, Vec<FnArg<'tcx, M::Provenance>>> {
         ops.iter()
             .map(|op| {
-                Ok(match &op.node {
+                let arg = match &op.node {
-                    mir::Operand::Move(place) => FnArg::InPlace(self.eval_place(*place)?),
+                    mir::Operand::Copy(_) | mir::Operand::Constant(_) => {
-                    _ => FnArg::Copy(self.eval_operand(&op.node, None)?),
+                        // Make a regular copy.
-                })
+                        let op = self.eval_operand(&op.node, None)?;
+                        FnArg::Copy(op)
+                    }
+                    mir::Operand::Move(place) => {
+                        // If this place lives in memory, preserve its location.
+                        // We call `place_to_op` which will be an `MPlaceTy` whenever there exists
+                        // an mplace for this place. (This is in contrast to `PlaceTy::as_mplace_or_local`
+                        // which can return a local even if that has an mplace.)
+                        let place = self.eval_place(*place)?;
+                        let op = self.place_to_op(&place)?;
+
+                        match op.as_mplace_or_imm() {
+                            Either::Left(mplace) => FnArg::InPlace(mplace),
+                            Either::Right(_imm) => {
+                                // This argument doesn't live in memory, so there's no place
+                                // to make inaccessible during the call.
+                                // We rely on there not being any stray `PlaceTy` that would let the
+                                // caller directly access this local!
+                                // This is also crucial for tail calls, where we want the `FnArg` to
+                                // stay valid when the old stack frame gets popped.
+                                FnArg::Copy(op)
+                            }
+                        }
+                    }
+                };
+
+                Ok(arg)
             })
             .collect()
     }
@@ -459,7 +476,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         // We work with a copy of the argument for now; if this is in-place argument passing, we
         // will later protect the source it comes from. This means the callee cannot observe if we
         // did in-place of by-copy argument passing, except for pointer equality tests.
-        let caller_arg_copy = self.copy_fn_arg(caller_arg)?;
+        let caller_arg_copy = self.copy_fn_arg(caller_arg);
         if !already_live {
             let local = callee_arg.as_local().unwrap();
             let meta = caller_arg_copy.meta();
@@ -477,8 +494,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         // specifically.)
         self.copy_op_allow_transmute(&caller_arg_copy, &callee_arg)?;
         // If this was an in-place pass, protect the place it comes from for the duration of the call.
-        if let FnArg::InPlace(place) = caller_arg {
+        if let FnArg::InPlace(mplace) = caller_arg {
-            M::protect_in_place_function_argument(self, place)?;
+            M::protect_in_place_function_argument(self, mplace)?;
         }
         Ok(())
     }
@@ -525,7 +542,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 M::call_intrinsic(
                     self,
                     instance,
-                    &self.copy_fn_args(args)?,
+                    &self.copy_fn_args(args),
                     destination,
                     target,
                     unwind,
@@ -602,8 +619,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                             .map(|arg| (
                                 arg.layout().ty,
                                 match arg {
-                                    FnArg::Copy(op) => format!("copy({:?})", *op),
+                                    FnArg::Copy(op) => format!("copy({op:?})"),
-                                    FnArg::InPlace(place) => format!("in-place({:?})", *place),
+                                    FnArg::InPlace(mplace) => format!("in-place({mplace:?})"),
                                 }
                             ))
                             .collect::<Vec<_>>()
@@ -725,8 +742,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                             callee_ty: callee_fn_abi.ret.layout.ty
                         });
                     }
+
                     // Protect return place for in-place return value passing.
-                    M::protect_in_place_function_argument(self, &destination.clone().into())?;
+                    M::protect_in_place_function_argument(self, &destination)?;
 
                     // Don't forget to mark "initially live" locals as live.
                     self.storage_live_for_always_live_locals()?;
@@ -749,7 +767,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 // An `InPlace` does nothing here, we keep the original receiver intact. We can't
                 // really pass the argument in-place anyway, and we are constructing a new
                 // `Immediate` receiver.
-                let mut receiver = self.copy_fn_arg(&args[0])?;
+                let mut receiver = self.copy_fn_arg(&args[0]);
                 let receiver_place = loop {
                     match receiver.layout.ty.kind() {
                         ty::Ref(..) | ty::RawPtr(..) => {

compiler/rustc_const_eval/src/transform/check_consts/check.rs

@@ -581,7 +581,6 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                 if is_int_bool_or_char(lhs_ty) && is_int_bool_or_char(rhs_ty) {
                     // Int, bool, and char operations are fine.
                 } else if lhs_ty.is_fn_ptr() || lhs_ty.is_unsafe_ptr() {
-                    assert_eq!(lhs_ty, rhs_ty);
                     assert!(matches!(
                         op,
                         BinOp::Eq

compiler/rustc_const_eval/src/transform/validate.rs

@@ -471,9 +471,9 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
                 self.check_edge(location, *real_target, EdgeKind::Normal);
                 self.check_unwind_edge(location, *unwind);
             }
-            TerminatorKind::InlineAsm { destination, unwind, .. } => {
+            TerminatorKind::InlineAsm { targets, unwind, .. } => {
-                if let Some(destination) = destination {
+                for &target in targets {
-                    self.check_edge(location, *destination, EdgeKind::Normal);
+                    self.check_edge(location, target, EdgeKind::Normal);
                 }
                 self.check_unwind_edge(location, *unwind);
             }

compiler/rustc_error_codes/src/error_codes/E0657.md

@@ -1,57 +1,26 @@
-A lifetime bound on a trait implementation was captured at an incorrect place.
+An `impl Trait` captured a higher-ranked lifetime, which is not supported.
+
+Currently, `impl Trait` types are only allowed to capture lifetimes from
+their parent items, and not from any `for<'a>` binders in scope.
 
 Erroneous code example:
 
 ```compile_fail,E0657
-trait Id<T> {}
+trait BorrowInto<'a> {
-trait Lt<'a> {}
+    type Target;
 
-impl<'a> Lt<'a> for () {}
+    fn borrow_into(&'a self) -> Self::Target;
-impl<T> Id<T> for T {}
-
-fn free_fn_capture_hrtb_in_impl_trait()
-    -> Box<for<'a> Id<impl Lt<'a>>> // error!
-{
-    Box::new(())
 }
 
-struct Foo;
+impl<'a> BorrowInto<'a> for () {
-impl Foo {
+    type Target = &'a ();
-    fn impl_fn_capture_hrtb_in_impl_trait()
+
-        -> Box<for<'a> Id<impl Lt<'a>>> // error!
+    fn borrow_into(&'a self) -> Self::Target {
-    {
+        self
-        Box::new(())
     }
 }
-```
 
-Here, you have used the inappropriate lifetime in the `impl Trait`,
+fn opaque() -> impl for<'a> BorrowInto<'a, Target = impl Sized + 'a> {
-The `impl Trait` can only capture lifetimes bound at the fn or impl
+    ()
-level.
-
-To fix this we have to define the lifetime at the function or impl
-level and use that lifetime in the `impl Trait`. For example you can
-define the lifetime at the function:
-
-```
-trait Id<T> {}
-trait Lt<'a> {}
-
-impl<'a> Lt<'a> for () {}
-impl<T> Id<T> for T {}
-
-fn free_fn_capture_hrtb_in_impl_trait<'b>()
-    -> Box<for<'a> Id<impl Lt<'b>>> // ok!
-{
-    Box::new(())
-}
-
-struct Foo;
-impl Foo {
-    fn impl_fn_capture_hrtb_in_impl_trait<'b>()
-        -> Box<for<'a> Id<impl Lt<'b>>> // ok!
-    {
-        Box::new(())
-    }
 }
 ```

compiler/rustc_feature/src/unstable.rs

@@ -345,6 +345,8 @@ declare_features! (
     (unstable, asm_const, "1.58.0", Some(93332)),
     /// Enables experimental inline assembly support for additional architectures.
     (unstable, asm_experimental_arch, "1.58.0", Some(93335)),
+    /// Allows using `label` operands in inline assembly.
+    (unstable, asm_goto, "CURRENT_RUSTC_VERSION", Some(119364)),
     /// Allows the `may_unwind` option in inline assembly.
     (unstable, asm_unwind, "1.58.0", Some(93334)),
     /// Allows users to enforce equality of associated constants `TraitImpl<AssocConst=3>`.

compiler/rustc_hir/src/hir.rs

@@ -2562,6 +2562,8 @@ pub enum OpaqueTyOrigin {
     AsyncFn(LocalDefId),
     /// type aliases: `type Foo = impl Trait;`
     TyAlias {
+        /// The type alias or associated type parent of the TAIT/ATPIT
+        parent: LocalDefId,
         /// associated types in impl blocks for traits.
         in_assoc_ty: bool,
     },
@@ -2650,6 +2652,9 @@ pub enum InlineAsmOperand<'hir> {
         path: QPath<'hir>,
         def_id: DefId,
     },
+    Label {
+        block: &'hir Block<'hir>,
+    },
 }
 
 impl<'hir> InlineAsmOperand<'hir> {
@@ -2659,7 +2664,10 @@ impl<'hir> InlineAsmOperand<'hir> {
             | Self::Out { reg, .. }
             | Self::InOut { reg, .. }
             | Self::SplitInOut { reg, .. } => Some(reg),
-            Self::Const { .. } | Self::SymFn { .. } | Self::SymStatic { .. } => None,
+            Self::Const { .. }
+            | Self::SymFn { .. }
+            | Self::SymStatic { .. }
+            | Self::Label { .. } => None,
         }
     }
 
@@ -2680,6 +2688,12 @@ pub struct InlineAsm<'hir> {
     pub line_spans: &'hir [Span],
 }
 
+impl InlineAsm<'_> {
+    pub fn contains_label(&self) -> bool {
+        self.operands.iter().any(|x| matches!(x.0, InlineAsmOperand::Label { .. }))
+    }
+}
+
 /// Represents a parameter in a function header.
 #[derive(Debug, Clone, Copy, HashStable_Generic)]
 pub struct Param<'hir> {

compiler/rustc_hir/src/intravisit.rs

@@ -1289,6 +1289,7 @@ pub fn walk_inline_asm<'v, V: Visitor<'v>>(
             InlineAsmOperand::SymStatic { path, .. } => {
                 try_visit!(visitor.visit_qpath(path, id, *op_sp));
             }
+            InlineAsmOperand::Label { block } => try_visit!(visitor.visit_block(block)),
         }
     }
     V::Result::output()

compiler/rustc_hir_analysis/messages.ftl

@@ -312,6 +312,10 @@ hir_analysis_only_current_traits_primitive = only traits defined in the current
 
 hir_analysis_only_current_traits_ty = `{$ty}` is not defined in the current crate
 
+hir_analysis_opaque_captures_higher_ranked_lifetime = `impl Trait` cannot capture {$bad_place}
+    .label = `impl Trait` implicitly captures all lifetimes in scope
+    .note = lifetime declared here
+
 hir_analysis_paren_sugar_attribute = the `#[rustc_paren_sugar]` attribute is a temporary means of controlling which traits can use parenthetical notation
     .help = add `#![feature(unboxed_closures)]` to the crate attributes to use it
 

compiler/rustc_hir_analysis/src/check/check.rs

@@ -339,8 +339,9 @@ fn check_opaque_meets_bounds<'tcx>(
     origin: &hir::OpaqueTyOrigin,
 ) -> Result<(), ErrorGuaranteed> {
     let defining_use_anchor = match *origin {
-        hir::OpaqueTyOrigin::FnReturn(did) | hir::OpaqueTyOrigin::AsyncFn(did) => did,
+        hir::OpaqueTyOrigin::FnReturn(did)
-        hir::OpaqueTyOrigin::TyAlias { .. } => tcx.impl_trait_parent(def_id),
+        | hir::OpaqueTyOrigin::AsyncFn(did)
+        | hir::OpaqueTyOrigin::TyAlias { parent: did, .. } => did,
     };
     let param_env = tcx.param_env(defining_use_anchor);
 
@@ -351,14 +352,14 @@ fn check_opaque_meets_bounds<'tcx>(
     let ocx = ObligationCtxt::new(&infcx);
 
     let args = match *origin {
-        hir::OpaqueTyOrigin::FnReturn(parent) | hir::OpaqueTyOrigin::AsyncFn(parent) => {
+        hir::OpaqueTyOrigin::FnReturn(parent)
-            GenericArgs::identity_for_item(tcx, parent).extend_to(
+        | hir::OpaqueTyOrigin::AsyncFn(parent)
-                tcx,
+        | hir::OpaqueTyOrigin::TyAlias { parent, .. } => GenericArgs::identity_for_item(
-                def_id.to_def_id(),
+            tcx, parent,
-                |param, _| tcx.map_rpit_lifetime_to_fn_lifetime(param.def_id.expect_local()).into(),
         )
-        }
+        .extend_to(tcx, def_id.to_def_id(), |param, _| {
-        hir::OpaqueTyOrigin::TyAlias { .. } => GenericArgs::identity_for_item(tcx, def_id),
+            tcx.map_opaque_lifetime_to_parent_lifetime(param.def_id.expect_local()).into()
+        }),
     };
 
     let opaque_ty = Ty::new_opaque(tcx, def_id.to_def_id(), args);

compiler/rustc_hir_analysis/src/check/compare_impl_item.rs

@@ -1,5 +1,6 @@
 use super::potentially_plural_count;
 use crate::errors::{LifetimesOrBoundsMismatchOnTrait, MethodShouldReturnFuture};
+use core::ops::ControlFlow;
 use hir::def_id::{DefId, DefIdMap, LocalDefId};
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_errors::{codes::*, pluralize, struct_span_code_err, Applicability, ErrorGuaranteed};
@@ -520,14 +521,6 @@ pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>(
         )
         .fold_with(&mut collector);
 
-    if !unnormalized_trait_sig.output().references_error() {
-        debug_assert_ne!(
-            collector.types.len(),
-            0,
-            "expect >1 RPITITs in call to `collect_return_position_impl_trait_in_trait_tys`"
-        );
-    }
-
     let trait_sig = ocx.normalize(&misc_cause, param_env, unnormalized_trait_sig);
     trait_sig.error_reported()?;
     let trait_return_ty = trait_sig.output();
@@ -646,6 +639,13 @@ pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>(
         }
     }
 
+    if !unnormalized_trait_sig.output().references_error() {
+        debug_assert!(
+            !collector.types.is_empty(),
+            "expect >0 RPITITs in call to `collect_return_position_impl_trait_in_trait_tys`"
+        );
+    }
+
     // FIXME: This has the same issue as #108544, but since this isn't breaking
     // existing code, I'm not particularly inclined to do the same hack as above
     // where we process wf obligations manually. This can be fixed in a forward-
@@ -1565,24 +1565,24 @@ fn compare_synthetic_generics<'tcx>(
                     let (sig, _) = impl_m.expect_fn();
                     let input_tys = sig.decl.inputs;
 
-                    struct Visitor(Option<Span>, hir::def_id::LocalDefId);
+                    struct Visitor(hir::def_id::LocalDefId);
                     impl<'v> intravisit::Visitor<'v> for Visitor {
-                        fn visit_ty(&mut self, ty: &'v hir::Ty<'v>) {
+                        type Result = ControlFlow<Span>;
-                            intravisit::walk_ty(self, ty);
+                        fn visit_ty(&mut self, ty: &'v hir::Ty<'v>) -> Self::Result {
                             if let hir::TyKind::Path(hir::QPath::Resolved(None, path)) = ty.kind
                                 && let Res::Def(DefKind::TyParam, def_id) = path.res
-                                && def_id == self.1.to_def_id()
+                                && def_id == self.0.to_def_id()
                             {
-                                self.0 = Some(ty.span);
+                                ControlFlow::Break(ty.span)
+                            } else {
+                                intravisit::walk_ty(self, ty)
                             }
                         }
                     }
 
-                    let mut visitor = Visitor(None, impl_def_id);
+                    let span = input_tys.iter().find_map(|ty| {
-                    for ty in input_tys {
+                        intravisit::Visitor::visit_ty(&mut Visitor(impl_def_id), ty).break_value()
-                        intravisit::Visitor::visit_ty(&mut visitor, ty);
+                    })?;
-                    }
-                    let span = visitor.0?;
 
                     let bounds = impl_m.generics.bounds_for_param(impl_def_id).next()?.bounds;
                     let bounds = bounds.first()?.span().to(bounds.last()?.span());

compiler/rustc_hir_analysis/src/check/intrinsicck.rs

@@ -470,6 +470,8 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                         }
                     };
                 }
+                // No special checking is needed for labels.
+                hir::InlineAsmOperand::Label { .. } => {}
             }
         }
     }

compiler/rustc_hir_analysis/src/check/wfcheck.rs

@@ -1,7 +1,9 @@
 use crate::autoderef::Autoderef;
+use crate::collect::CollectItemTypesVisitor;
 use crate::constrained_generic_params::{identify_constrained_generic_params, Parameter};
 use crate::errors;
 
+use hir::intravisit::Visitor;
 use rustc_ast as ast;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
 use rustc_errors::{codes::*, pluralize, struct_span_code_err, Applicability, ErrorGuaranteed};
@@ -225,6 +227,7 @@ fn check_item<'tcx>(tcx: TyCtxt<'tcx>, item: &'tcx hir::Item<'tcx>) -> Result<()
         ?item.owner_id,
         item.name = ? tcx.def_path_str(def_id)
     );
+    CollectItemTypesVisitor { tcx }.visit_item(item);
 
     let res = match item.kind {
         // Right now we check that every default trait implementation
@@ -336,9 +339,14 @@ fn check_item<'tcx>(tcx: TyCtxt<'tcx>, item: &'tcx hir::Item<'tcx>) -> Result<()
     res
 }
 
-fn check_foreign_item(tcx: TyCtxt<'_>, item: &hir::ForeignItem<'_>) -> Result<(), ErrorGuaranteed> {
+fn check_foreign_item<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    item: &'tcx hir::ForeignItem<'tcx>,
+) -> Result<(), ErrorGuaranteed> {
     let def_id = item.owner_id.def_id;
 
+    CollectItemTypesVisitor { tcx }.visit_foreign_item(item);
+
     debug!(
         ?item.owner_id,
         item.name = ? tcx.def_path_str(def_id)
@@ -355,12 +363,14 @@ fn check_foreign_item(tcx: TyCtxt<'_>, item: &hir::ForeignItem<'_>) -> Result<()
     }
 }
 
-fn check_trait_item(
+fn check_trait_item<'tcx>(
-    tcx: TyCtxt<'_>,
+    tcx: TyCtxt<'tcx>,
-    trait_item: &hir::TraitItem<'_>,
+    trait_item: &'tcx hir::TraitItem<'tcx>,
 ) -> Result<(), ErrorGuaranteed> {
     let def_id = trait_item.owner_id.def_id;
 
+    CollectItemTypesVisitor { tcx }.visit_trait_item(trait_item);
+
     let (method_sig, span) = match trait_item.kind {
         hir::TraitItemKind::Fn(ref sig, _) => (Some(sig), trait_item.span),
         hir::TraitItemKind::Type(_bounds, Some(ty)) => (None, ty.span),
@@ -895,7 +905,12 @@ fn check_object_unsafe_self_trait_by_name(tcx: TyCtxt<'_>, item: &hir::TraitItem
     }
 }
 
-fn check_impl_item(tcx: TyCtxt<'_>, impl_item: &hir::ImplItem<'_>) -> Result<(), ErrorGuaranteed> {
+fn check_impl_item<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    impl_item: &'tcx hir::ImplItem<'tcx>,
+) -> Result<(), ErrorGuaranteed> {
+    CollectItemTypesVisitor { tcx }.visit_impl_item(impl_item);
+
     let (method_sig, span) = match impl_item.kind {
         hir::ImplItemKind::Fn(ref sig, _) => (Some(sig), impl_item.span),
         // Constrain binding and overflow error spans to `<Ty>` in `type foo = <Ty>`.

compiler/rustc_hir_analysis/src/collect.rs

@@ -20,7 +20,7 @@ use rustc_data_structures::unord::UnordMap;
 use rustc_errors::{Applicability, Diag, ErrorGuaranteed, StashKey};
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
-use rustc_hir::def_id::{DefId, LocalDefId, LocalModDefId};
+use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::{GenericParamKind, Node};
 use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
@@ -52,11 +52,6 @@ mod resolve_bound_vars;
 mod type_of;
 
 ///////////////////////////////////////////////////////////////////////////
-// Main entry point
-
-fn collect_mod_item_types(tcx: TyCtxt<'_>, module_def_id: LocalModDefId) {
-    tcx.hir().visit_item_likes_in_module(module_def_id, &mut CollectItemTypesVisitor { tcx });
-}
 
 pub fn provide(providers: &mut Providers) {
     resolve_bound_vars::provide(providers);
@@ -82,7 +77,6 @@ pub fn provide(providers: &mut Providers) {
         impl_trait_header,
         coroutine_kind,
         coroutine_for_closure,
-        collect_mod_item_types,
         is_type_alias_impl_trait,
         find_field,
         ..*providers
@@ -155,8 +149,8 @@ impl<'v> Visitor<'v> for HirPlaceholderCollector {
     }
 }
 
-struct CollectItemTypesVisitor<'tcx> {
+pub struct CollectItemTypesVisitor<'tcx> {
-    tcx: TyCtxt<'tcx>,
+    pub tcx: TyCtxt<'tcx>,
 }
 
 /// If there are any placeholder types (`_`), emit an error explaining that this is not allowed

compiler/rustc_hir_analysis/src/collect/predicates_of.rs

@@ -339,7 +339,7 @@ fn compute_bidirectional_outlives_predicates<'tcx>(
     predicates: &mut Vec<(ty::Clause<'tcx>, Span)>,
 ) {
     for param in opaque_own_params {
-        let orig_lifetime = tcx.map_rpit_lifetime_to_fn_lifetime(param.def_id.expect_local());
+        let orig_lifetime = tcx.map_opaque_lifetime_to_parent_lifetime(param.def_id.expect_local());
         if let ty::ReEarlyParam(..) = *orig_lifetime {
             let dup_lifetime = ty::Region::new_early_param(
                 tcx,

compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs

@@ -6,9 +6,9 @@
 //! the types in HIR to identify late-bound lifetimes and assign their Debruijn indices. This file
 //! is also responsible for assigning their semantics to implicit lifetimes in trait objects.
 
+use core::ops::ControlFlow;
 use rustc_ast::visit::walk_list;
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet};
-use rustc_errors::{codes::*, struct_span_code_err};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::LocalDefId;
@@ -417,23 +417,18 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
         {
             if let &hir::ClosureBinder::For { span: for_sp, .. } = binder {
                 fn span_of_infer(ty: &hir::Ty<'_>) -> Option<Span> {
-                    struct V(Option<Span>);
+                    struct V;
-
                     impl<'v> Visitor<'v> for V {
-                        fn visit_ty(&mut self, t: &'v hir::Ty<'v>) {
+                        type Result = ControlFlow<Span>;
-                            match t.kind {
+                        fn visit_ty(&mut self, t: &'v hir::Ty<'v>) -> Self::Result {
-                                _ if self.0.is_some() => (),
+                            if matches!(t.kind, hir::TyKind::Infer) {
-                                hir::TyKind::Infer => {
+                                ControlFlow::Break(t.span)
-                                    self.0 = Some(t.span);
+                            } else {
-                                }
+                                intravisit::walk_ty(self, t)
-                                _ => intravisit::walk_ty(self, t),
                             }
                         }
                     }
-
+                    V.visit_ty(ty).break_value()
-                    let mut v = V(None);
-                    v.visit_ty(ty);
-                    v.0
                 }
 
                 let infer_in_rt_sp = match fn_decl.output {
@@ -514,38 +509,11 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
                 // These sorts of items have no lifetime parameters at all.
                 intravisit::walk_item(self, item);
             }
-            hir::ItemKind::OpaqueTy(hir::OpaqueTy {
-                origin: hir::OpaqueTyOrigin::TyAlias { .. },
-                ..
-            }) => {
-                // Opaque types are visited when we visit the
-                // `TyKind::OpaqueDef`, so that they have the lifetimes from
-                // their parent opaque_ty in scope.
-                //
-                // The core idea here is that since OpaqueTys are generated with the impl Trait as
-                // their owner, we can keep going until we find the Item that owns that. We then
-                // conservatively add all resolved lifetimes. Otherwise we run into problems in
-                // cases like `type Foo<'a> = impl Bar<As = impl Baz + 'a>`.
-                let parent_item = self.tcx.hir().get_parent_item(item.hir_id());
-                let resolved_lifetimes: &ResolveBoundVars =
-                    self.tcx.resolve_bound_vars(parent_item);
-                // We need to add *all* deps, since opaque tys may want them from *us*
-                for (&owner, defs) in resolved_lifetimes.defs.iter() {
-                    defs.iter().for_each(|(&local_id, region)| {
-                        self.map.defs.insert(hir::HirId { owner, local_id }, *region);
-                    });
-                }
-                for (&owner, late_bound_vars) in resolved_lifetimes.late_bound_vars.iter() {
-                    late_bound_vars.iter().for_each(|(&local_id, late_bound_vars)| {
-                        self.record_late_bound_vars(
-                            hir::HirId { owner, local_id },
-                            late_bound_vars.clone(),
-                        );
-                    });
-                }
-            }
             hir::ItemKind::OpaqueTy(&hir::OpaqueTy {
-                origin: hir::OpaqueTyOrigin::FnReturn(parent) | hir::OpaqueTyOrigin::AsyncFn(parent),
+                origin:
+                    hir::OpaqueTyOrigin::FnReturn(parent)
+                    | hir::OpaqueTyOrigin::AsyncFn(parent)
+                    | hir::OpaqueTyOrigin::TyAlias { parent, .. },
                 generics,
                 ..
             }) => {
@@ -683,26 +651,7 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
                 //                                      the opaque_ty generics
                 let opaque_ty = self.tcx.hir().item(item_id);
                 match &opaque_ty.kind {
-                    hir::ItemKind::OpaqueTy(hir::OpaqueTy {
+                    hir::ItemKind::OpaqueTy(hir::OpaqueTy { origin: _, .. }) => {}
-                        origin: hir::OpaqueTyOrigin::TyAlias { .. },
-                        ..
-                    }) => {
-                        intravisit::walk_ty(self, ty);
-
-                        // Elided lifetimes and late-bound lifetimes (from the parent)
-                        // are not allowed in non-return position impl Trait
-                        let scope = Scope::LateBoundary {
-                            s: &Scope::TraitRefBoundary { s: self.scope },
-                            what: "type alias impl trait",
-                        };
-                        self.with(scope, |this| intravisit::walk_item(this, opaque_ty));
-
-                        return;
-                    }
-                    hir::ItemKind::OpaqueTy(hir::OpaqueTy {
-                        origin: hir::OpaqueTyOrigin::FnReturn(..) | hir::OpaqueTyOrigin::AsyncFn(..),
-                        ..
-                    }) => {}
                     i => bug!("`impl Trait` pointed to non-opaque type?? {:#?}", i),
                 };
 
@@ -719,34 +668,47 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
                     // and ban them. Type variables instantiated inside binders aren't
                     // well-supported at the moment, so this doesn't work.
                     // In the future, this should be fixed and this error should be removed.
-                    let def = self.map.defs.get(&lifetime.hir_id).cloned();
+                    let def = self.map.defs.get(&lifetime.hir_id).copied();
-                    let Some(ResolvedArg::LateBound(_, _, def_id)) = def else { continue };
+                    let Some(ResolvedArg::LateBound(_, _, lifetime_def_id)) = def else { continue };
-                    let Some(def_id) = def_id.as_local() else { continue };
+                    let Some(lifetime_def_id) = lifetime_def_id.as_local() else { continue };
-                    let hir_id = self.tcx.local_def_id_to_hir_id(def_id);
+                    let lifetime_hir_id = self.tcx.local_def_id_to_hir_id(lifetime_def_id);
-                    // Ensure that the parent of the def is an item, not HRTB
+
-                    let parent_id = self.tcx.parent_hir_id(hir_id);
+                    let bad_place = match self.tcx.hir_node(self.tcx.parent_hir_id(lifetime_hir_id))
-                    if !parent_id.is_owner() {
-                        struct_span_code_err!(
-                            self.tcx.dcx(),
-                            lifetime.ident.span,
-                            E0657,
-                            "`impl Trait` can only capture lifetimes bound at the fn or impl level"
-                        )
-                        .emit();
-                        self.uninsert_lifetime_on_error(lifetime, def.unwrap());
-                    }
-                    if let hir::Node::Item(hir::Item {
-                        kind: hir::ItemKind::OpaqueTy { .. }, ..
-                    }) = self.tcx.hir_node(parent_id)
                     {
-                        self.tcx.dcx().struct_span_err(
+                        // Opaques do not declare their own lifetimes, so if a lifetime comes from an opaque
-                            lifetime.ident.span,
+                        // it must be a reified late-bound lifetime from a trait goal.
-                            "higher kinded lifetime bounds on nested opaque types are not supported yet",
+                        hir::Node::Item(hir::Item {
-                        )
+                            kind: hir::ItemKind::OpaqueTy { .. }, ..
-                        .with_span_note(self.tcx.def_span(def_id), "lifetime declared here")
+                        }) => "higher-ranked lifetime from outer `impl Trait`",
-                        .emit();
+                        // Other items are fine.
-                        self.uninsert_lifetime_on_error(lifetime, def.unwrap());
+                        hir::Node::Item(_) | hir::Node::TraitItem(_) | hir::Node::ImplItem(_) => {
+                            continue;
                         }
+                        hir::Node::Ty(hir::Ty { kind: hir::TyKind::BareFn(_), .. }) => {
+                            "higher-ranked lifetime from function pointer"
+                        }
+                        hir::Node::Ty(hir::Ty { kind: hir::TyKind::TraitObject(..), .. }) => {
+                            "higher-ranked lifetime from `dyn` type"
+                        }
+                        _ => "higher-ranked lifetime",
+                    };
+
+                    let (span, label) = if lifetime.ident.span == self.tcx.def_span(lifetime_def_id)
+                    {
+                        let opaque_span = self.tcx.def_span(item_id.owner_id);
+                        (opaque_span, Some(opaque_span))
+                    } else {
+                        (lifetime.ident.span, None)
+                    };
+
+                    // Ensure that the parent of the def is an item, not HRTB
+                    self.tcx.dcx().emit_err(errors::OpaqueCapturesHigherRankedLifetime {
+                        span,
+                        label,
+                        decl_span: self.tcx.def_span(lifetime_def_id),
+                        bad_place,
+                    });
+                    self.uninsert_lifetime_on_error(lifetime, def.unwrap());
                 }
             }
             _ => intravisit::walk_ty(self, ty),

compiler/rustc_hir_analysis/src/collect/type_of.rs

@@ -1,3 +1,4 @@
+use core::ops::ControlFlow;
 use rustc_errors::{Applicability, StashKey};
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
@@ -553,11 +554,11 @@ pub(super) fn type_of_opaque(
         Ok(ty::EarlyBinder::bind(match tcx.hir_node_by_def_id(def_id) {
             Node::Item(item) => match item.kind {
                 ItemKind::OpaqueTy(OpaqueTy {
-                    origin: hir::OpaqueTyOrigin::TyAlias { in_assoc_ty: false },
+                    origin: hir::OpaqueTyOrigin::TyAlias { in_assoc_ty: false, .. },
                     ..
                 }) => opaque::find_opaque_ty_constraints_for_tait(tcx, def_id),
                 ItemKind::OpaqueTy(OpaqueTy {
-                    origin: hir::OpaqueTyOrigin::TyAlias { in_assoc_ty: true },
+                    origin: hir::OpaqueTyOrigin::TyAlias { in_assoc_ty: true, .. },
                     ..
                 }) => opaque::find_opaque_ty_constraints_for_impl_trait_in_assoc_type(tcx, def_id),
                 // Opaque types desugared from `impl Trait`.
@@ -675,19 +676,16 @@ pub fn type_alias_is_lazy<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> bool {
     if tcx.features().lazy_type_alias {
         return true;
     }
-    struct HasTait {
+    struct HasTait;
-        has_type_alias_impl_trait: bool,
-    }
     impl<'tcx> Visitor<'tcx> for HasTait {
-        fn visit_ty(&mut self, t: &'tcx hir::Ty<'tcx>) {
+        type Result = ControlFlow<()>;
+        fn visit_ty(&mut self, t: &'tcx hir::Ty<'tcx>) -> Self::Result {
             if let hir::TyKind::OpaqueDef(..) = t.kind {
-                self.has_type_alias_impl_trait = true;
+                ControlFlow::Break(())
             } else {
-                hir::intravisit::walk_ty(self, t);
+                hir::intravisit::walk_ty(self, t)
             }
         }
     }
-    let mut has_tait = HasTait { has_type_alias_impl_trait: false };
+    HasTait.visit_ty(tcx.hir().expect_item(def_id).expect_ty_alias().0).is_break()
-    has_tait.visit_ty(tcx.hir().expect_item(def_id).expect_ty_alias().0);
-    has_tait.has_type_alias_impl_trait
 }

compiler/rustc_hir_analysis/src/errors.rs

@@ -1607,3 +1607,15 @@ pub struct UnnamedFieldsReprFieldDefined {
     #[primary_span]
     pub span: Span,
 }
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_opaque_captures_higher_ranked_lifetime, code = E0657)]
+pub struct OpaqueCapturesHigherRankedLifetime {
+    #[primary_span]
+    pub span: Span,
+    #[label]
+    pub label: Option<Span>,
+    #[note]
+    pub decl_span: Span,
+    pub bad_place: &'static str,
+}

compiler/rustc_hir_analysis/src/lib.rs

@@ -159,12 +159,6 @@ pub fn provide(providers: &mut Providers) {
 pub fn check_crate(tcx: TyCtxt<'_>) -> Result<(), ErrorGuaranteed> {
     let _prof_timer = tcx.sess.timer("type_check_crate");
 
-    // this ensures that later parts of type checking can assume that items
-    // have valid types and not error
-    tcx.sess.time("type_collecting", || {
-        tcx.hir().for_each_module(|module| tcx.ensure().collect_mod_item_types(module))
-    });
-
     if tcx.features().rustc_attrs {
         tcx.sess.time("outlives_testing", || outlives::test::test_inferred_outlives(tcx))?;
     }

compiler/rustc_hir_analysis/src/variance/mod.rs

@@ -125,15 +125,7 @@ fn variance_of_opaque(tcx: TyCtxt<'_>, item_def_id: LocalDefId) -> &[ty::Varianc
 
     // By default, RPIT are invariant wrt type and const generics, but they are bivariant wrt
     // lifetime generics.
-    let variances = std::iter::repeat(ty::Invariant).take(generics.count());
+    let mut variances = vec![ty::Invariant; generics.count()];
-
-    let mut variances: Vec<_> = match tcx.opaque_type_origin(item_def_id) {
-        rustc_hir::OpaqueTyOrigin::FnReturn(_) | rustc_hir::OpaqueTyOrigin::AsyncFn(_) => {
-            variances.collect()
-        }
-        // But TAIT are invariant for all generics
-        rustc_hir::OpaqueTyOrigin::TyAlias { .. } => return tcx.arena.alloc_from_iter(variances),
-    };
 
     // Mark all lifetimes from parent generics as unused (Bivariant).
     // This will be overridden later if required.

compiler/rustc_hir_pretty/src/lib.rs

@@ -1265,6 +1265,10 @@ impl<'a> State<'a> {
                     s.space();
                     s.print_qpath(path, true);
                 }
+                hir::InlineAsmOperand::Label { block } => {
+                    s.head("label");
+                    s.print_block(block);
+                }
             },
             AsmArg::Options(opts) => {
                 s.word("options");

compiler/rustc_hir_typeck/src/expr.rs

@@ -3232,6 +3232,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 
     fn check_expr_asm(&self, asm: &'tcx hir::InlineAsm<'tcx>) -> Ty<'tcx> {
+        let mut diverge = asm.options.contains(ast::InlineAsmOptions::NORETURN);
+
         for (op, _op_sp) in asm.operands {
             match op {
                 hir::InlineAsmOperand::In { expr, .. } => {
@@ -3253,13 +3255,24 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // be well-formed.
                 hir::InlineAsmOperand::Const { .. } | hir::InlineAsmOperand::SymFn { .. } => {}
                 hir::InlineAsmOperand::SymStatic { .. } => {}
+                hir::InlineAsmOperand::Label { block } => {
+                    let previous_diverges = self.diverges.get();
+
+                    // The label blocks should have unit return value or diverge.
+                    let ty =
+                        self.check_block_with_expected(block, ExpectHasType(self.tcx.types.unit));
+                    if !ty.is_never() {
+                        self.demand_suptype(block.span, self.tcx.types.unit, ty);
+                        diverge = false;
+                    }
+
+                    // We need this to avoid false unreachable warning when a label diverges.
+                    self.diverges.set(previous_diverges);
                 }
             }
-        if asm.options.contains(ast::InlineAsmOptions::NORETURN) {
-            self.tcx.types.never
-        } else {
-            Ty::new_unit(self.tcx)
         }
+
+        if diverge { self.tcx.types.never } else { self.tcx.types.unit }
     }
 
     fn check_offset_of(

compiler/rustc_hir_typeck/src/expr_use_visitor.rs

@@ -293,6 +293,9 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
                         | hir::InlineAsmOperand::Const { .. }
                         | hir::InlineAsmOperand::SymFn { .. }
                         | hir::InlineAsmOperand::SymStatic { .. } => {}
+                        hir::InlineAsmOperand::Label { block } => {
+                            self.walk_block(block);
+                        }
                     }
                 }
             }

compiler/rustc_hir_typeck/src/lib.rs

@@ -61,7 +61,7 @@ use rustc_hir::{HirIdMap, Node};
 use rustc_hir_analysis::astconv::AstConv;
 use rustc_hir_analysis::check::check_abi;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
-use rustc_infer::traits::ObligationInspector;
+use rustc_infer::traits::{ObligationCauseCode, ObligationInspector, WellFormedLoc};
 use rustc_middle::query::Providers;
 use rustc_middle::traits;
 use rustc_middle::ty::{self, Ty, TyCtxt};
@@ -253,6 +253,10 @@ fn typeck_with_fallback<'tcx>(
         let expected_type = expected_type.unwrap_or_else(fallback);
 
         let expected_type = fcx.normalize(body.value.span, expected_type);
+
+        let wf_code = ObligationCauseCode::WellFormed(Some(WellFormedLoc::Ty(def_id)));
+        fcx.register_wf_obligation(expected_type.into(), body.value.span, wf_code);
+
         fcx.require_type_is_sized(expected_type, body.value.span, traits::ConstSized);
 
         // Gather locals in statics (because of block expressions).

compiler/rustc_hir_typeck/src/method/suggest.rs

@@ -6,6 +6,7 @@
 use crate::errors::{self, CandidateTraitNote, NoAssociatedItem};
 use crate::Expectation;
 use crate::FnCtxt;
+use core::ops::ControlFlow;
 use rustc_ast::ast::Mutability;
 use rustc_attr::parse_confusables;
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
@@ -2212,30 +2213,28 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 let map = self.infcx.tcx.hir();
                 let body_id = self.tcx.hir().body_owned_by(self.body_id);
                 let body = map.body(body_id);
-                struct LetVisitor<'a> {
+                struct LetVisitor {
-                    result: Option<&'a hir::Expr<'a>>,
                     ident_name: Symbol,
                 }
 
                 // FIXME: This really should be taking scoping, etc into account.
-                impl<'v> Visitor<'v> for LetVisitor<'v> {
+                impl<'v> Visitor<'v> for LetVisitor {
-                    fn visit_stmt(&mut self, ex: &'v hir::Stmt<'v>) {
+                    type Result = ControlFlow<Option<&'v hir::Expr<'v>>>;
-                        if let hir::StmtKind::Local(hir::Local { pat, init, .. }) = &ex.kind
+                    fn visit_stmt(&mut self, ex: &'v hir::Stmt<'v>) -> Self::Result {
+                        if let hir::StmtKind::Local(&hir::Local { pat, init, .. }) = ex.kind
                             && let Binding(_, _, ident, ..) = pat.kind
                             && ident.name == self.ident_name
                         {
-                            self.result = *init;
+                            ControlFlow::Break(init)
                         } else {
-                            hir::intravisit::walk_stmt(self, ex);
+                            hir::intravisit::walk_stmt(self, ex)
                         }
                     }
                 }
 
-                let mut visitor = LetVisitor { result: None, ident_name: seg1.ident.name };
-                visitor.visit_body(body);
-
                 if let Node::Expr(call_expr) = self.tcx.parent_hir_node(seg1.hir_id)
-                    && let Some(expr) = visitor.result
+                    && let ControlFlow::Break(Some(expr)) =
+                        (LetVisitor { ident_name: seg1.ident.name }).visit_body(body)
                     && let Some(self_ty) = self.node_ty_opt(expr.hir_id)
                 {
                     let probe = self.lookup_probe_for_diagnostic(

compiler/rustc_infer/src/infer/error_reporting/mod.rs

@@ -79,7 +79,7 @@ use rustc_middle::ty::{
 use rustc_span::{sym, symbol::kw, BytePos, DesugaringKind, Pos, Span};
 use rustc_target::spec::abi;
 use std::borrow::Cow;
-use std::ops::Deref;
+use std::ops::{ControlFlow, Deref};
 use std::path::PathBuf;
 use std::{cmp, fmt, iter};
 
@@ -2129,15 +2129,12 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         let tykind = match self.tcx.opt_hir_node_by_def_id(trace.cause.body_id) {
             Some(hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn(_, _, body_id), .. })) => {
                 let body = hir.body(*body_id);
-                struct LetVisitor<'v> {
+                struct LetVisitor {
                     span: Span,
-                    result: Option<&'v hir::Ty<'v>>,
-                }
-                impl<'v> Visitor<'v> for LetVisitor<'v> {
-                    fn visit_stmt(&mut self, s: &'v hir::Stmt<'v>) {
-                        if self.result.is_some() {
-                            return;
                 }
+                impl<'v> Visitor<'v> for LetVisitor {
+                    type Result = ControlFlow<&'v hir::TyKind<'v>>;
+                    fn visit_stmt(&mut self, s: &'v hir::Stmt<'v>) -> Self::Result {
                         // Find a local statement where the initializer has
                         // the same span as the error and the type is specified.
                         if let hir::Stmt {
@@ -2151,13 +2148,13 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                         } = s
                             && init_span == &self.span
                         {
-                            self.result = Some(*array_ty);
+                            ControlFlow::Break(&array_ty.peel_refs().kind)
+                        } else {
+                            ControlFlow::Continue(())
                         }
                     }
                 }
-                let mut visitor = LetVisitor { span, result: None };
+                LetVisitor { span }.visit_body(body).break_value()
-                visitor.visit_body(body);
-                visitor.result.map(|r| &r.peel_refs().kind)
             }
             Some(hir::Node::Item(hir::Item { kind: hir::ItemKind::Const(ty, _, _), .. })) => {
                 Some(&ty.peel_refs().kind)

compiler/rustc_infer/src/infer/error_reporting/nice_region_error/find_anon_type.rs

@@ -1,3 +1,4 @@
+use core::ops::ControlFlow;
 use rustc_hir as hir;
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_middle::hir::map::Map;
@@ -43,14 +44,9 @@ fn find_component_for_bound_region<'tcx>(
     arg: &'tcx hir::Ty<'tcx>,
     br: &ty::BoundRegionKind,
 ) -> Option<&'tcx hir::Ty<'tcx>> {
-    let mut nested_visitor = FindNestedTypeVisitor {
+    FindNestedTypeVisitor { tcx, bound_region: *br, current_index: ty::INNERMOST }
-        tcx,
+        .visit_ty(arg)
-        bound_region: *br,
+        .break_value()
-        found_type: None,
-        current_index: ty::INNERMOST,
-    };
-    nested_visitor.visit_ty(arg);
-    nested_visitor.found_type
 }
 
 // The FindNestedTypeVisitor captures the corresponding `hir::Ty` of the
@@ -65,26 +61,24 @@ struct FindNestedTypeVisitor<'tcx> {
     // The bound_region corresponding to the Refree(freeregion)
     // associated with the anonymous region we are looking for.
     bound_region: ty::BoundRegionKind,
-    // The type where the anonymous lifetime appears
-    // for e.g., Vec<`&u8`> and <`&u8`>
-    found_type: Option<&'tcx hir::Ty<'tcx>>,
     current_index: ty::DebruijnIndex,
 }
 
 impl<'tcx> Visitor<'tcx> for FindNestedTypeVisitor<'tcx> {
+    type Result = ControlFlow<&'tcx hir::Ty<'tcx>>;
     type NestedFilter = nested_filter::OnlyBodies;
 
     fn nested_visit_map(&mut self) -> Self::Map {
         self.tcx.hir()
     }
 
-    fn visit_ty(&mut self, arg: &'tcx hir::Ty<'tcx>) {
+    fn visit_ty(&mut self, arg: &'tcx hir::Ty<'tcx>) -> Self::Result {
         match arg.kind {
             hir::TyKind::BareFn(_) => {
                 self.current_index.shift_in(1);
                 intravisit::walk_ty(self, arg);
                 self.current_index.shift_out(1);
-                return;
+                return ControlFlow::Continue(());
             }
 
             hir::TyKind::TraitObject(bounds, ..) => {
@@ -105,8 +99,7 @@ impl<'tcx> Visitor<'tcx> for FindNestedTypeVisitor<'tcx> {
                     (Some(rbv::ResolvedArg::EarlyBound(id)), ty::BrNamed(def_id, _)) => {
                         debug!("EarlyBound id={:?} def_id={:?}", id, def_id);
                         if id == def_id {
-                            self.found_type = Some(arg);
+                            return ControlFlow::Break(arg);
-                            return; // we can stop visiting now
                         }
                     }
 
@@ -123,8 +116,7 @@ impl<'tcx> Visitor<'tcx> for FindNestedTypeVisitor<'tcx> {
                         );
                         debug!("LateBound id={:?} def_id={:?}", id, def_id);
                         if debruijn_index == self.current_index && id == def_id {
-                            self.found_type = Some(arg);
+                            return ControlFlow::Break(arg);
-                            return; // we can stop visiting now
                         }
                     }
 
@@ -145,23 +137,30 @@ impl<'tcx> Visitor<'tcx> for FindNestedTypeVisitor<'tcx> {
             }
             // Checks if it is of type `hir::TyKind::Path` which corresponds to a struct.
             hir::TyKind::Path(_) => {
-                let subvisitor = &mut TyPathVisitor {
+                // Prefer using the lifetime in type arguments rather than lifetime arguments.
+                intravisit::walk_ty(self, arg)?;
+
+                // Call `walk_ty` as `visit_ty` is empty.
+                return if intravisit::walk_ty(
+                    &mut TyPathVisitor {
                         tcx: self.tcx,
-                    found_it: false,
                         bound_region: self.bound_region,
                         current_index: self.current_index,
+                    },
+                    arg,
+                )
+                .is_break()
+                {
+                    ControlFlow::Break(arg)
+                } else {
+                    ControlFlow::Continue(())
                 };
-                intravisit::walk_ty(subvisitor, arg); // call walk_ty; as visit_ty is empty,
-                // this will visit only outermost type
-                if subvisitor.found_it {
-                    self.found_type = Some(arg);
-                }
             }
             _ => {}
         }
         // walk the embedded contents: e.g., if we are visiting `Vec<&Foo>`,
         // go on to visit `&Foo`
-        intravisit::walk_ty(self, arg);
+        intravisit::walk_ty(self, arg)
     }
 }
 
@@ -173,26 +172,25 @@ impl<'tcx> Visitor<'tcx> for FindNestedTypeVisitor<'tcx> {
 // specific part of the type in the error message.
 struct TyPathVisitor<'tcx> {
     tcx: TyCtxt<'tcx>,
-    found_it: bool,
     bound_region: ty::BoundRegionKind,
     current_index: ty::DebruijnIndex,
 }
 
 impl<'tcx> Visitor<'tcx> for TyPathVisitor<'tcx> {
+    type Result = ControlFlow<()>;
     type NestedFilter = nested_filter::OnlyBodies;
 
     fn nested_visit_map(&mut self) -> Map<'tcx> {
         self.tcx.hir()
     }
 
-    fn visit_lifetime(&mut self, lifetime: &hir::Lifetime) {
+    fn visit_lifetime(&mut self, lifetime: &hir::Lifetime) -> Self::Result {
         match (self.tcx.named_bound_var(lifetime.hir_id), self.bound_region) {
             // the lifetime of the TyPath!
             (Some(rbv::ResolvedArg::EarlyBound(id)), ty::BrNamed(def_id, _)) => {
                 debug!("EarlyBound id={:?} def_id={:?}", id, def_id);
                 if id == def_id {
-                    self.found_it = true;
+                    return ControlFlow::Break(());
-                    return; // we can stop visiting now
                 }
             }
 
@@ -201,8 +199,7 @@ impl<'tcx> Visitor<'tcx> for TyPathVisitor<'tcx> {
                 debug!("id={:?}", id);
                 debug!("def_id={:?}", def_id);
                 if debruijn_index == self.current_index && id == def_id {
-                    self.found_it = true;
+                    return ControlFlow::Break(());
-                    return; // we can stop visiting now
                 }
             }
 
@@ -220,9 +217,10 @@ impl<'tcx> Visitor<'tcx> for TyPathVisitor<'tcx> {
                 debug!("no arg found");
             }
         }
+        ControlFlow::Continue(())
     }
 
-    fn visit_ty(&mut self, arg: &'tcx hir::Ty<'tcx>) {
+    fn visit_ty(&mut self, arg: &'tcx hir::Ty<'tcx>) -> Self::Result {
         // ignore nested types
         //
         // If you have a type like `Foo<'a, &Ty>` we
@@ -231,5 +229,6 @@ impl<'tcx> Visitor<'tcx> for TyPathVisitor<'tcx> {
         // Making `visit_ty` empty will ignore the `&Ty` embedded
         // inside, it will get reached by the outer visitor.
         debug!("`Ty` corresponding to a struct is {:?}", arg);
+        ControlFlow::Continue(())
     }
 }

compiler/rustc_infer/src/infer/error_reporting/suggest.rs

@@ -1,4 +1,5 @@
 use crate::infer::error_reporting::hir::Path;
+use core::ops::ControlFlow;
 use hir::def::CtorKind;
 use hir::intravisit::{walk_expr, walk_stmt, Visitor};
 use hir::{Local, QPath};
@@ -563,33 +564,27 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         cause: &ObligationCause<'_>,
         span: Span,
     ) -> Option<TypeErrorAdditionalDiags> {
-        let hir = self.tcx.hir();
-        if let Some(body_id) = self.tcx.hir().maybe_body_owned_by(cause.body_id) {
-            let body = hir.body(body_id);
-
         /// Find the if expression with given span
         struct IfVisitor {
-                pub result: bool,
             pub found_if: bool,
             pub err_span: Span,
         }
 
         impl<'v> Visitor<'v> for IfVisitor {
-                fn visit_expr(&mut self, ex: &'v hir::Expr<'v>) {
+            type Result = ControlFlow<()>;
-                    if self.result {
+            fn visit_expr(&mut self, ex: &'v hir::Expr<'v>) -> Self::Result {
-                        return;
-                    }
                 match ex.kind {
                     hir::ExprKind::If(cond, _, _) => {
                         self.found_if = true;
-                            walk_expr(self, cond);
+                        walk_expr(self, cond)?;
                         self.found_if = false;
+                        ControlFlow::Continue(())
                     }
                     _ => walk_expr(self, ex),
                 }
             }
 
-                fn visit_stmt(&mut self, ex: &'v hir::Stmt<'v>) {
+            fn visit_stmt(&mut self, ex: &'v hir::Stmt<'v>) -> Self::Result {
                 if let hir::StmtKind::Local(hir::Local {
                     span,
                     pat: hir::Pat { .. },
@@ -600,25 +595,24 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     && self.found_if
                     && span.eq(&self.err_span)
                 {
-                        self.result = true;
+                    ControlFlow::Break(())
-                    }
+                } else {
-                    walk_stmt(self, ex);
+                    walk_stmt(self, ex)
-                }
-
-                fn visit_body(&mut self, body: &'v hir::Body<'v>) {
-                    hir::intravisit::walk_body(self, body);
                 }
             }
 
-            let mut visitor = IfVisitor { err_span: span, found_if: false, result: false };
+            fn visit_body(&mut self, body: &'v hir::Body<'v>) -> Self::Result {
-            visitor.visit_body(body);
+                hir::intravisit::walk_body(self, body)
-            if visitor.result {
-                return Some(TypeErrorAdditionalDiags::AddLetForLetChains {
-                    span: span.shrink_to_lo(),
-                });
             }
         }
-        None
+
+        self.tcx.hir().maybe_body_owned_by(cause.body_id).and_then(|body_id| {
+            let body = self.tcx.hir().body(body_id);
+            IfVisitor { err_span: span, found_if: false }
+                .visit_body(body)
+                .is_break()
+                .then(|| TypeErrorAdditionalDiags::AddLetForLetChains { span: span.shrink_to_lo() })
+        })
     }
 
     /// For "one type is more general than the other" errors on closures, suggest changing the lifetime

compiler/rustc_infer/src/infer/opaque_types/mod.rs

@@ -391,7 +391,7 @@ impl<'tcx> InferCtxt<'tcx> {
             // Anonymous `impl Trait`
             hir::OpaqueTyOrigin::FnReturn(parent) => parent == parent_def_id,
             // Named `type Foo = impl Bar;`
-            hir::OpaqueTyOrigin::TyAlias { in_assoc_ty } => {
+            hir::OpaqueTyOrigin::TyAlias { in_assoc_ty, .. } => {
                 if in_assoc_ty {
                     self.tcx.opaque_types_defined_by(parent_def_id).contains(&def_id)
                 } else {

compiler/rustc_infer/src/lib.rs

@@ -20,6 +20,7 @@
 #![allow(rustc::untranslatable_diagnostic)]
 #![feature(associated_type_bounds)]
 #![feature(box_patterns)]
+#![feature(control_flow_enum)]
 #![feature(extend_one)]
 #![feature(let_chains)]
 #![feature(if_let_guard)]

compiler/rustc_lint/src/internal.rs

@@ -379,13 +379,15 @@ impl LateLintPass<'_> for Diagnostics {
                     _ => return, // occurs for fns passed as args
                 }
             }
-            ExprKind::MethodCall(segment, _recv, args, _span) => {
+            ExprKind::MethodCall(_segment, _recv, args, _span) => {
-                let def_id = cx.typeck_results().type_dependent_def_id(expr.hir_id).unwrap();
+                let Some((span, def_id, fn_gen_args)) = typeck_results_of_method_fn(cx, expr)
-                let fn_gen_args = cx.typeck_results().node_args(expr.hir_id);
+                else {
+                    return;
+                };
                 let mut call_tys: Vec<_> =
                     args.iter().map(|arg| cx.typeck_results().expr_ty(arg)).collect();
                 call_tys.insert(0, cx.tcx.types.self_param); // dummy inserted for `self`
-                (segment.ident.span, def_id, fn_gen_args, call_tys)
+                (span, def_id, fn_gen_args, call_tys)
             }
             _ => return,
         };

compiler/rustc_lint/src/unused.rs

@@ -20,6 +20,7 @@ use rustc_span::symbol::Symbol;
 use rustc_span::symbol::{kw, sym};
 use rustc_span::{BytePos, Span};
 use std::iter;
+use std::ops::ControlFlow;
 
 declare_lint! {
     /// The `unused_must_use` lint detects unused result of a type flagged as
@@ -753,21 +754,18 @@ trait UnusedDelimLint {
         // fn f(){(print!(á
         // ```
         use rustc_ast::visit::{walk_expr, Visitor};
-        struct ErrExprVisitor {
+        struct ErrExprVisitor;
-            has_error: bool,
-        }
         impl<'ast> Visitor<'ast> for ErrExprVisitor {
-            fn visit_expr(&mut self, expr: &'ast ast::Expr) {
+            type Result = ControlFlow<()>;
+            fn visit_expr(&mut self, expr: &'ast ast::Expr) -> ControlFlow<()> {
                 if let ExprKind::Err(_) = expr.kind {
-                    self.has_error = true;
+                    ControlFlow::Break(())
-                    return;
+                } else {
-                }
                     walk_expr(self, expr)
                 }
             }
-        let mut visitor = ErrExprVisitor { has_error: false };
+        }
-        visitor.visit_expr(value);
+        if ErrExprVisitor.visit_expr(value).is_break() {
-        if visitor.has_error {
             return;
         }
         let spans = match value.kind {

compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp

@@ -451,14 +451,30 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
       Options.ObjectFilenameForDebug = OutputObjFile;
   }
   if (!strcmp("zlib", DebugInfoCompression) && llvm::compression::zlib::isAvailable()) {
+#if LLVM_VERSION_GE(19, 0)
+    Options.MCOptions.CompressDebugSections = DebugCompressionType::Zlib;
+#else
     Options.CompressDebugSections = DebugCompressionType::Zlib;
+#endif
   } else if (!strcmp("zstd", DebugInfoCompression) && llvm::compression::zstd::isAvailable()) {
+#if LLVM_VERSION_GE(19, 0)
+    Options.MCOptions.CompressDebugSections = DebugCompressionType::Zstd;
+#else
     Options.CompressDebugSections = DebugCompressionType::Zstd;
+#endif
   } else if (!strcmp("none", DebugInfoCompression)) {
+#if LLVM_VERSION_GE(19, 0)
+    Options.MCOptions.CompressDebugSections = DebugCompressionType::None;
+#else
     Options.CompressDebugSections = DebugCompressionType::None;
+#endif
   }
 
+#if LLVM_VERSION_GE(19, 0)
+  Options.MCOptions.X86RelaxRelocations = RelaxELFRelocations;
+#else
   Options.RelaxELFRelocations = RelaxELFRelocations;
+#endif
   Options.UseInitArray = UseInitArray;
 
 #if LLVM_VERSION_LT(17, 0)

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -1549,6 +1549,31 @@ LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
                                       Name));
 }
 
+extern "C" LLVMValueRef
+LLVMRustBuildCallBr(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
+                    LLVMBasicBlockRef DefaultDest,
+                    LLVMBasicBlockRef *IndirectDests, unsigned NumIndirectDests,
+                    LLVMValueRef *Args, unsigned NumArgs,
+                    OperandBundleDef **OpBundles, unsigned NumOpBundles,
+                    const char *Name) {
+  Value *Callee = unwrap(Fn);
+  FunctionType *FTy = unwrap<FunctionType>(Ty);
+
+  // FIXME: Is there a way around this?
+  std::vector<BasicBlock*> IndirectDestsUnwrapped;
+  IndirectDestsUnwrapped.reserve(NumIndirectDests);
+  for (unsigned i = 0; i < NumIndirectDests; ++i) {
+    IndirectDestsUnwrapped.push_back(unwrap(IndirectDests[i]));
+  }
+
+  return wrap(unwrap(B)->CreateCallBr(
+      FTy, Callee, unwrap(DefaultDest),
+      ArrayRef<BasicBlock*>(IndirectDestsUnwrapped),
+      ArrayRef<Value*>(unwrap(Args), NumArgs),
+      ArrayRef<OperandBundleDef>(*OpBundles, NumOpBundles),
+      Name));
+}
+
 extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B,
                                                LLVMBasicBlockRef BB) {
   auto Point = unwrap(BB)->getFirstInsertionPt();

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -356,7 +356,7 @@ provide! { tcx, def_id, other, cdata,
         cdata.get_stability_implications(tcx).iter().copied().collect()
     }
     stripped_cfg_items => { cdata.get_stripped_cfg_items(cdata.cnum, tcx) }
-    intrinsic => { cdata.get_intrinsic(def_id.index) }
+    intrinsic_raw => { cdata.get_intrinsic(def_id.index) }
     defined_lang_items => { cdata.get_lang_items(tcx) }
     diagnostic_items => { cdata.get_diagnostic_items() }
     missing_lang_items => { cdata.get_missing_lang_items(tcx) }

compiler/rustc_middle/src/mir/mod.rs

@@ -1716,13 +1716,13 @@ mod size_asserts {
     use super::*;
     use rustc_data_structures::static_assert_size;
     // tidy-alphabetical-start
-    static_assert_size!(BasicBlockData<'_>, 136);
+    static_assert_size!(BasicBlockData<'_>, 144);
     static_assert_size!(LocalDecl<'_>, 40);
     static_assert_size!(SourceScopeData<'_>, 72);
     static_assert_size!(Statement<'_>, 32);
     static_assert_size!(StatementKind<'_>, 16);
-    static_assert_size!(Terminator<'_>, 104);
+    static_assert_size!(Terminator<'_>, 112);
-    static_assert_size!(TerminatorKind<'_>, 88);
+    static_assert_size!(TerminatorKind<'_>, 96);
     static_assert_size!(VarDebugInfo<'_>, 88);
     // tidy-alphabetical-end
 }

compiler/rustc_middle/src/mir/patch.rs

@@ -11,6 +11,8 @@ pub struct MirPatch<'tcx> {
     resume_block: Option<BasicBlock>,
     // Only for unreachable in cleanup path.
     unreachable_cleanup_block: Option<BasicBlock>,
+    // Only for unreachable not in cleanup path.
+    unreachable_no_cleanup_block: Option<BasicBlock>,
     // Cached block for UnwindTerminate (with reason)
     terminate_block: Option<(BasicBlock, UnwindTerminateReason)>,
     body_span: Span,
@@ -27,6 +29,7 @@ impl<'tcx> MirPatch<'tcx> {
             next_local: body.local_decls.len(),
             resume_block: None,
             unreachable_cleanup_block: None,
+            unreachable_no_cleanup_block: None,
             terminate_block: None,
             body_span: body.span,
         };
@@ -43,9 +46,12 @@ impl<'tcx> MirPatch<'tcx> {
             // Check if we already have an unreachable block
             if matches!(block.terminator().kind, TerminatorKind::Unreachable)
                 && block.statements.is_empty()
-                && block.is_cleanup
             {
+                if block.is_cleanup {
                     result.unreachable_cleanup_block = Some(bb);
+                } else {
+                    result.unreachable_no_cleanup_block = Some(bb);
+                }
                 continue;
             }
 
@@ -95,6 +101,23 @@ impl<'tcx> MirPatch<'tcx> {
         bb
     }
 
+    pub fn unreachable_no_cleanup_block(&mut self) -> BasicBlock {
+        if let Some(bb) = self.unreachable_no_cleanup_block {
+            return bb;
+        }
+
+        let bb = self.new_block(BasicBlockData {
+            statements: vec![],
+            terminator: Some(Terminator {
+                source_info: SourceInfo::outermost(self.body_span),
+                kind: TerminatorKind::Unreachable,
+            }),
+            is_cleanup: false,
+        });
+        self.unreachable_no_cleanup_block = Some(bb);
+        bb
+    }
+
     pub fn terminate_block(&mut self, reason: UnwindTerminateReason) -> BasicBlock {
         if let Some((cached_bb, cached_reason)) = self.terminate_block
             && reason == cached_reason

compiler/rustc_middle/src/mir/pretty.rs

@@ -7,7 +7,7 @@ use std::path::{Path, PathBuf};
 use crate::mir::interpret::ConstAllocation;
 
 use super::graphviz::write_mir_fn_graphviz;
-use rustc_ast::InlineAsmTemplatePiece;
+use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_middle::mir::interpret::{
     alloc_range, read_target_uint, AllocBytes, AllocId, Allocation, GlobalAlloc, Pointer,
     Provenance,
@@ -830,6 +830,9 @@ impl<'tcx> TerminatorKind<'tcx> {
                         InlineAsmOperand::SymStatic { def_id } => {
                             write!(fmt, "sym_static {def_id:?}")?;
                         }
+                        InlineAsmOperand::Label { target_index } => {
+                            write!(fmt, "label {target_index}")?;
+                        }
                     }
                 }
                 write!(fmt, ", options({options:?}))")
@@ -868,16 +871,19 @@ impl<'tcx> TerminatorKind<'tcx> {
                 vec!["real".into(), "unwind".into()]
             }
             FalseUnwind { unwind: _, .. } => vec!["real".into()],
-            InlineAsm { destination: Some(_), unwind: UnwindAction::Cleanup(_), .. } => {
+            InlineAsm { options, ref targets, unwind, .. } => {
-                vec!["return".into(), "unwind".into()]
+                let mut vec = Vec::with_capacity(targets.len() + 1);
+                if !options.contains(InlineAsmOptions::NORETURN) {
+                    vec.push("return".into());
                 }
-            InlineAsm { destination: Some(_), unwind: _, .. } => {
+                vec.resize(targets.len(), "label".into());
-                vec!["return".into()]
+
+                if let UnwindAction::Cleanup(_) = unwind {
+                    vec.push("unwind".into());
                 }
-            InlineAsm { destination: None, unwind: UnwindAction::Cleanup(_), .. } => {
+
-                vec!["unwind".into()]
+                vec
             }
-            InlineAsm { destination: None, unwind: _, .. } => vec![],
         }
     }
 }

compiler/rustc_middle/src/mir/syntax.rs

@@ -793,9 +793,10 @@ pub enum TerminatorKind<'tcx> {
         /// used to map assembler errors back to the line in the source code.
         line_spans: &'tcx [Span],
 
-        /// Destination block after the inline assembly returns, unless it is
+        /// Valid targets for the inline assembly.
-        /// diverging (InlineAsmOptions::NORETURN).
+        /// The first element is the fallthrough destination, unless
-        destination: Option<BasicBlock>,
+        /// InlineAsmOptions::NORETURN is set.
+        targets: Vec<BasicBlock>,
 
         /// Action to be taken if the inline assembly unwinds. This is present
         /// if and only if InlineAsmOptions::MAY_UNWIND is set.
@@ -918,6 +919,10 @@ pub enum InlineAsmOperand<'tcx> {
     SymStatic {
         def_id: DefId,
     },
+    Label {
+        /// This represents the index into the `targets` array in `TerminatorKind::InlineAsm`.
+        target_index: usize,
+    },
 }
 
 /// Type for MIR `Assert` terminator error messages.

compiler/rustc_middle/src/mir/terminator.rs

@@ -74,6 +74,17 @@ impl SwitchTargets {
     pub fn target_for_value(&self, value: u128) -> BasicBlock {
         self.iter().find_map(|(v, t)| (v == value).then_some(t)).unwrap_or_else(|| self.otherwise())
     }
+
+    /// Adds a new target to the switch. But You cannot add an already present value.
+    #[inline]
+    pub fn add_target(&mut self, value: u128, bb: BasicBlock) {
+        let value = Pu128(value);
+        if self.values.contains(&value) {
+            bug!("target value {:?} already present", value);
+        }
+        self.values.push(value);
+        self.targets.insert(self.targets.len() - 1, bb);
+    }
 }
 
 pub struct SwitchTargetsIter<'a> {
@@ -336,8 +347,7 @@ pub struct Terminator<'tcx> {
 }
 
 pub type Successors<'a> = impl DoubleEndedIterator<Item = BasicBlock> + 'a;
-pub type SuccessorsMut<'a> =
+pub type SuccessorsMut<'a> = impl DoubleEndedIterator<Item = &'a mut BasicBlock> + 'a;
-    iter::Chain<std::option::IntoIter<&'a mut BasicBlock>, slice::IterMut<'a, BasicBlock>>;
 
 impl<'tcx> Terminator<'tcx> {
     #[inline]
@@ -371,40 +381,36 @@ impl<'tcx> TerminatorKind<'tcx> {
     pub fn successors(&self) -> Successors<'_> {
         use self::TerminatorKind::*;
         match *self {
-            Call { target: Some(t), unwind: UnwindAction::Cleanup(ref u), .. }
+            Call { target: Some(ref t), unwind: UnwindAction::Cleanup(u), .. }
-            | Yield { resume: t, drop: Some(ref u), .. }
+            | Yield { resume: ref t, drop: Some(u), .. }
-            | Drop { target: t, unwind: UnwindAction::Cleanup(ref u), .. }
+            | Drop { target: ref t, unwind: UnwindAction::Cleanup(u), .. }
-            | Assert { target: t, unwind: UnwindAction::Cleanup(ref u), .. }
+            | Assert { target: ref t, unwind: UnwindAction::Cleanup(u), .. }
-            | FalseUnwind { real_target: t, unwind: UnwindAction::Cleanup(ref u) }
+            | FalseUnwind { real_target: ref t, unwind: UnwindAction::Cleanup(u) } => {
-            | InlineAsm { destination: Some(t), unwind: UnwindAction::Cleanup(ref u), .. } => {
+                slice::from_ref(t).into_iter().copied().chain(Some(u))
-                Some(t).into_iter().chain(slice::from_ref(u).into_iter().copied())
             }
-            Goto { target: t }
+            Goto { target: ref t }
-            | Call { target: None, unwind: UnwindAction::Cleanup(t), .. }
+            | Call { target: None, unwind: UnwindAction::Cleanup(ref t), .. }
-            | Call { target: Some(t), unwind: _, .. }
+            | Call { target: Some(ref t), unwind: _, .. }
-            | Yield { resume: t, drop: None, .. }
+            | Yield { resume: ref t, drop: None, .. }
-            | Drop { target: t, unwind: _, .. }
+            | Drop { target: ref t, unwind: _, .. }
-            | Assert { target: t, unwind: _, .. }
+            | Assert { target: ref t, unwind: _, .. }
-            | FalseUnwind { real_target: t, unwind: _ }
+            | FalseUnwind { real_target: ref t, unwind: _ } => {
-            | InlineAsm { destination: None, unwind: UnwindAction::Cleanup(t), .. }
+                slice::from_ref(t).into_iter().copied().chain(None)
-            | InlineAsm { destination: Some(t), unwind: _, .. } => {
-                Some(t).into_iter().chain((&[]).into_iter().copied())
             }
             UnwindResume
             | UnwindTerminate(_)
             | CoroutineDrop
             | Return
             | Unreachable
-            | Call { target: None, unwind: _, .. }
+            | Call { target: None, unwind: _, .. } => (&[]).into_iter().copied().chain(None),
-            | InlineAsm { destination: None, unwind: _, .. } => {
+            InlineAsm { ref targets, unwind: UnwindAction::Cleanup(u), .. } => {
-                None.into_iter().chain((&[]).into_iter().copied())
+                targets.iter().copied().chain(Some(u))
             }
-            SwitchInt { ref targets, .. } => {
+            InlineAsm { ref targets, unwind: _, .. } => targets.iter().copied().chain(None),
-                None.into_iter().chain(targets.targets.iter().copied())
+            SwitchInt { ref targets, .. } => targets.targets.iter().copied().chain(None),
+            FalseEdge { ref real_target, imaginary_target } => {
+                slice::from_ref(real_target).into_iter().copied().chain(Some(imaginary_target))
             }
-            FalseEdge { real_target, ref imaginary_target } => Some(real_target)
-                .into_iter()
-                .chain(slice::from_ref(imaginary_target).into_iter().copied()),
         }
     }
 
@@ -416,33 +422,31 @@ impl<'tcx> TerminatorKind<'tcx> {
             | Yield { resume: ref mut t, drop: Some(ref mut u), .. }
             | Drop { target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u), .. }
             | Assert { target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u), .. }
-            | FalseUnwind { real_target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u) }
+            | FalseUnwind { real_target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u) } => {
-            | InlineAsm {
+                slice::from_mut(t).into_iter().chain(Some(u))
-                destination: Some(ref mut t),
+            }
-                unwind: UnwindAction::Cleanup(ref mut u),
-                ..
-            } => Some(t).into_iter().chain(slice::from_mut(u)),
             Goto { target: ref mut t }
             | Call { target: None, unwind: UnwindAction::Cleanup(ref mut t), .. }
             | Call { target: Some(ref mut t), unwind: _, .. }
             | Yield { resume: ref mut t, drop: None, .. }
             | Drop { target: ref mut t, unwind: _, .. }
             | Assert { target: ref mut t, unwind: _, .. }
-            | FalseUnwind { real_target: ref mut t, unwind: _ }
+            | FalseUnwind { real_target: ref mut t, unwind: _ } => {
-            | InlineAsm { destination: None, unwind: UnwindAction::Cleanup(ref mut t), .. }
+                slice::from_mut(t).into_iter().chain(None)
-            | InlineAsm { destination: Some(ref mut t), unwind: _, .. } => {
-                Some(t).into_iter().chain(&mut [])
             }
             UnwindResume
             | UnwindTerminate(_)
             | CoroutineDrop
             | Return
             | Unreachable
-            | Call { target: None, unwind: _, .. }
+            | Call { target: None, unwind: _, .. } => (&mut []).into_iter().chain(None),
-            | InlineAsm { destination: None, unwind: _, .. } => None.into_iter().chain(&mut []),
+            InlineAsm { ref mut targets, unwind: UnwindAction::Cleanup(ref mut u), .. } => {
-            SwitchInt { ref mut targets, .. } => None.into_iter().chain(&mut targets.targets),
+                targets.iter_mut().chain(Some(u))
+            }
+            InlineAsm { ref mut targets, unwind: _, .. } => targets.iter_mut().chain(None),
+            SwitchInt { ref mut targets, .. } => targets.targets.iter_mut().chain(None),
             FalseEdge { ref mut real_target, ref mut imaginary_target } => {
-                Some(real_target).into_iter().chain(slice::from_mut(imaginary_target))
+                slice::from_mut(real_target).into_iter().chain(Some(imaginary_target))
             }
         }
     }
@@ -514,7 +518,7 @@ pub enum TerminatorEdges<'mir, 'tcx> {
     Double(BasicBlock, BasicBlock),
     /// Special action for `Yield`, `Call` and `InlineAsm` terminators.
     AssignOnReturn {
-        return_: Option<BasicBlock>,
+        return_: &'mir [BasicBlock],
         /// The cleanup block, if it exists.
         cleanup: Option<BasicBlock>,
         place: CallReturnPlaces<'mir, 'tcx>,
@@ -578,31 +582,37 @@ impl<'tcx> TerminatorKind<'tcx> {
                 TerminatorEdges::Double(real_target, imaginary_target)
             }
 
-            Yield { resume: target, drop, resume_arg, value: _ } => {
+            Yield { resume: ref target, drop, resume_arg, value: _ } => {
                 TerminatorEdges::AssignOnReturn {
-                    return_: Some(target),
+                    return_: slice::from_ref(target),
                     cleanup: drop,
                     place: CallReturnPlaces::Yield(resume_arg),
                 }
             }
 
-            Call { unwind, destination, target, func: _, args: _, fn_span: _, call_source: _ } => {
+            Call {
-                TerminatorEdges::AssignOnReturn {
+                unwind,
-                    return_: target,
+                destination,
+                ref target,
+                func: _,
+                args: _,
+                fn_span: _,
+                call_source: _,
+            } => TerminatorEdges::AssignOnReturn {
+                return_: target.as_ref().map(slice::from_ref).unwrap_or_default(),
                 cleanup: unwind.cleanup_block(),
                 place: CallReturnPlaces::Call(destination),
-                }
+            },
-            }
 
             InlineAsm {
                 template: _,
                 ref operands,
                 options: _,
                 line_spans: _,
-                destination,
+                ref targets,
                 unwind,
             } => TerminatorEdges::AssignOnReturn {
-                return_: destination,
+                return_: targets,
                 cleanup: unwind.cleanup_block(),
                 place: CallReturnPlaces::InlineAsm(operands),
             },

compiler/rustc_middle/src/mir/visit.rs

@@ -565,7 +565,7 @@ macro_rules! make_mir_visitor {
                         operands,
                         options: _,
                         line_spans: _,
-                        destination: _,
+                        targets: _,
                         unwind: _,
                     } => {
                         for op in operands {
@@ -595,7 +595,8 @@ macro_rules! make_mir_visitor {
                                     self.visit_constant(value, location);
                                 }
                                 InlineAsmOperand::Out { place: None, .. }
-                                | InlineAsmOperand::SymStatic { def_id: _ } => {}
+                                | InlineAsmOperand::SymStatic { def_id: _ }
+                                | InlineAsmOperand::Label { target_index: _ } => {}
                             }
                         }
                     }

compiler/rustc_middle/src/query/mod.rs

@@ -762,6 +762,7 @@ rustc_queries! {
         desc { |tcx| "computing the variances of `{}`", tcx.def_path_str(def_id) }
         cache_on_disk_if { def_id.is_local() }
         separate_provide_extern
+        cycle_delay_bug
     }
 
     /// Maps from thee `DefId` of a type to its (inferred) outlives.
@@ -959,10 +960,6 @@ rustc_queries! {
         ensure_forwards_result_if_red
     }
 
-    query collect_mod_item_types(key: LocalModDefId) {
-        desc { |tcx| "collecting item types in {}", describe_as_module(key, tcx) }
-    }
-
     /// Caches `CoerceUnsized` kinds for impls on custom types.
     query coerce_unsized_info(key: DefId) -> Result<ty::adjustment::CoerceUnsizedInfo, ErrorGuaranteed> {
         desc { |tcx| "computing CoerceUnsized info for `{}`", tcx.def_path_str(key) }
@@ -1745,7 +1742,7 @@ rustc_queries! {
         separate_provide_extern
     }
     /// Whether the function is an intrinsic
-    query intrinsic(def_id: DefId) -> Option<rustc_middle::ty::IntrinsicDef> {
+    query intrinsic_raw(def_id: DefId) -> Option<rustc_middle::ty::IntrinsicDef> {
         desc { |tcx| "fetch intrinsic name if `{}` is an intrinsic", tcx.def_path_str(def_id) }
         separate_provide_extern
     }

compiler/rustc_middle/src/thir.rs

@@ -565,6 +565,9 @@ pub enum InlineAsmOperand<'tcx> {
     SymStatic {
         def_id: DefId,
     },
+    Label {
+        block: BlockId,
+    },
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, HashStable)]

compiler/rustc_middle/src/thir/visit.rs

@@ -162,6 +162,7 @@ pub fn walk_expr<'thir, 'tcx: 'thir, V: Visitor<'thir, 'tcx>>(
                     | Const { value: _, span: _ }
                     | SymFn { value: _, span: _ }
                     | SymStatic { def_id: _ } => {}
+                    Label { block } => visitor.visit_block(&visitor.thir()[*block]),
                 }
             }
         }