Rollup merge of #119365 - nbdd0121:asm-goto, r=Amanieu

Add asm goto support to `asm!`

Tracking issue: #119364

This PR implements asm-goto support, using the syntax described in "future possibilities" section of [RFC2873](https://rust-lang.github.io/rfcs/2873-inline-asm.html#asm-goto).

Currently I have only implemented the `label` part, not the `fallthrough` part (i.e. fallthrough is implicit). This doesn't reduce the expressive though, since you can use label-break to get arbitrary control flow or simply set a value and rely on jump threading optimisation to get the desired control flow. I can add that later if deemed necessary.

r? ``@Amanieu``
cc ``@ojeda``

compiler/rustc_ast/src/ast.rs

@@ -2302,6 +2302,9 @@ pub enum InlineAsmOperand {
     Sym {
         sym: InlineAsmSym,
     },
+    Label {
+        block: P<Block>,
+    },
 }
 
 impl InlineAsmOperand {
@@ -2311,7 +2314,7 @@ impl InlineAsmOperand {
             | Self::Out { reg, .. }
             | Self::InOut { reg, .. }
             | Self::SplitInOut { reg, .. } => Some(reg),
-            Self::Const { .. } | Self::Sym { .. } => None,
+            Self::Const { .. } | Self::Sym { .. } | Self::Label { .. } => None,
         }
     }
 }

compiler/rustc_ast/src/mut_visit.rs

@@ -1331,6 +1331,7 @@ pub fn noop_visit_inline_asm<T: MutVisitor>(asm: &mut InlineAsm, vis: &mut T) {
             }
             InlineAsmOperand::Const { anon_const } => vis.visit_anon_const(anon_const),
             InlineAsmOperand::Sym { sym } => vis.visit_inline_asm_sym(sym),
+            InlineAsmOperand::Label { block } => vis.visit_block(block),
         }
     }
 }

compiler/rustc_ast/src/visit.rs

@@ -823,6 +823,7 @@ pub fn walk_inline_asm<'a, V: Visitor<'a>>(visitor: &mut V, asm: &'a InlineAsm)
                 try_visit!(visitor.visit_anon_const(anon_const))
             }
             InlineAsmOperand::Sym { sym } => try_visit!(visitor.visit_inline_asm_sym(sym)),
+            InlineAsmOperand::Label { block } => try_visit!(visitor.visit_block(block)),
         }
     }
     V::Result::output()

compiler/rustc_ast_lowering/messages.ftl

@@ -88,6 +88,9 @@ ast_lowering_invalid_abi_suggestion = did you mean
 ast_lowering_invalid_asm_template_modifier_const =
     asm template modifiers are not allowed for `const` arguments
 
+ast_lowering_invalid_asm_template_modifier_label =
+    asm template modifiers are not allowed for `label` arguments
+
 ast_lowering_invalid_asm_template_modifier_reg_class =
     invalid asm template modifier for this register class
 

compiler/rustc_ast_lowering/src/asm.rs

@@ -3,9 +3,9 @@ use crate::{ImplTraitContext, ImplTraitPosition, ParamMode, ResolverAstLoweringE
 use super::errors::{
     AbiSpecifiedMultipleTimes, AttSyntaxOnlyX86, ClobberAbiNotSupported,
     InlineAsmUnsupportedTarget, InvalidAbiClobberAbi, InvalidAsmTemplateModifierConst,
-    InvalidAsmTemplateModifierRegClass, InvalidAsmTemplateModifierRegClassSub,
-    InvalidAsmTemplateModifierSym, InvalidRegister, InvalidRegisterClass, RegisterClassOnlyClobber,
-    RegisterConflict,
+    InvalidAsmTemplateModifierLabel, InvalidAsmTemplateModifierRegClass,
+    InvalidAsmTemplateModifierRegClassSub, InvalidAsmTemplateModifierSym, InvalidRegister,
+    InvalidRegisterClass, RegisterClassOnlyClobber, RegisterConflict,
 };
 use super::LoweringContext;
 
@@ -237,6 +237,18 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             }
                         }
                     }
+                    InlineAsmOperand::Label { block } => {
+                        if !self.tcx.features().asm_goto {
+                            feature_err(
+                                sess,
+                                sym::asm_goto,
+                                *op_sp,
+                                "label operands for inline assembly are unstable",
+                            )
+                            .emit();
+                        }
+                        hir::InlineAsmOperand::Label { block: self.lower_block(block, false) }
+                    }
                 };
                 (op, self.lower_span(*op_sp))
             })
@@ -296,6 +308,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             op_span: op_sp,
                         });
                     }
+                    hir::InlineAsmOperand::Label { .. } => {
+                        self.dcx().emit_err(InvalidAsmTemplateModifierLabel {
+                            placeholder_span,
+                            op_span: op_sp,
+                        });
+                    }
                 }
             }
         }
@@ -335,7 +353,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
                         hir::InlineAsmOperand::Const { .. }
                         | hir::InlineAsmOperand::SymFn { .. }
-                        | hir::InlineAsmOperand::SymStatic { .. } => {
+                        | hir::InlineAsmOperand::SymStatic { .. }
+                        | hir::InlineAsmOperand::Label { .. } => {
                             unreachable!("{op:?} is not a register operand");
                         }
                     };

compiler/rustc_ast_lowering/src/errors.rs

@@ -261,6 +261,16 @@ pub struct InvalidAsmTemplateModifierSym {
     pub op_span: Span,
 }
 
+#[derive(Diagnostic, Clone, Copy)]
+#[diag(ast_lowering_invalid_asm_template_modifier_label)]
+pub struct InvalidAsmTemplateModifierLabel {
+    #[primary_span]
+    #[label(ast_lowering_template_modifier)]
+    pub placeholder_span: Span,
+    #[label(ast_lowering_argument)]
+    pub op_span: Span,
+}
+
 #[derive(Diagnostic, Clone, Copy)]
 #[diag(ast_lowering_register_class_only_clobber)]
 pub struct RegisterClassOnlyClobber {

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -1452,6 +1452,10 @@ impl<'a> State<'a> {
                             s.print_path(&sym.path, true, 0);
                         }
                     }
+                    InlineAsmOperand::Label { block } => {
+                        s.head("label");
+                        s.print_block(block);
+                    }
                 }
             }
             AsmArg::ClobberAbi(abi) => {

compiler/rustc_borrowck/src/lib.rs

@@ -723,7 +723,7 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
                 operands,
                 options: _,
                 line_spans: _,
-                destination: _,
+                targets: _,
                 unwind: _,
             } => {
                 for op in operands {
@@ -749,7 +749,8 @@ impl<'cx, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx, R> for MirBorro
                         }
                         InlineAsmOperand::Const { value: _ }
                         | InlineAsmOperand::SymFn { value: _ }
-                        | InlineAsmOperand::SymStatic { def_id: _ } => {}
+                        | InlineAsmOperand::SymStatic { def_id: _ }
+                        | InlineAsmOperand::Label { target_index: _ } => {}
                     }
                 }
             }

compiler/rustc_borrowck/src/polonius/loan_invalidations.rs

@@ -161,7 +161,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
                 operands,
                 options: _,
                 line_spans: _,
-                destination: _,
+                targets: _,
                 unwind: _,
             } => {
                 for op in operands {
@@ -182,7 +182,8 @@ impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
                         }
                         InlineAsmOperand::Const { value: _ }
                         | InlineAsmOperand::SymFn { value: _ }
-                        | InlineAsmOperand::SymStatic { def_id: _ } => {}
+                        | InlineAsmOperand::SymStatic { def_id: _ }
+                        | InlineAsmOperand::Label { target_index: _ } => {}
                     }
                 }
             }

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -1770,8 +1770,8 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 self.assert_iscleanup(body, block_data, real_target, is_cleanup);
                 self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);
             }
-            TerminatorKind::InlineAsm { destination, unwind, .. } => {
-                if let Some(target) = destination {
+            TerminatorKind::InlineAsm { ref targets, unwind, .. } => {
+                for &target in targets {
                     self.assert_iscleanup(body, block_data, target, is_cleanup);
                 }
                 self.assert_iscleanup_unwind(body, block_data, unwind, is_cleanup);

compiler/rustc_builtin_macros/messages.ftl

@@ -19,6 +19,8 @@ builtin_macros_asm_expected_other = expected operand, {$is_global_asm ->
 
 builtin_macros_asm_explicit_register_name = explicit register arguments cannot have names
 
+builtin_macros_asm_mayunwind = asm labels are not allowed with the `may_unwind` option
+
 builtin_macros_asm_modifier_invalid = asm template modifier must be a single character
 
 builtin_macros_asm_mutually_exclusive = the `{$opt1}` and `{$opt2}` options are mutually exclusive

compiler/rustc_builtin_macros/src/asm.rs

@@ -164,6 +164,9 @@ pub fn parse_asm_args<'a>(
                 path: path.clone(),
             };
             ast::InlineAsmOperand::Sym { sym }
+        } else if !is_global_asm && p.eat_keyword(sym::label) {
+            let block = p.parse_block()?;
+            ast::InlineAsmOperand::Label { block }
         } else if allow_templates {
             let template = p.parse_expr()?;
             // If it can't possibly expand to a string, provide diagnostics here to include other
@@ -240,6 +243,7 @@ pub fn parse_asm_args<'a>(
     let mut have_real_output = false;
     let mut outputs_sp = vec![];
     let mut regclass_outputs = vec![];
+    let mut labels_sp = vec![];
     for (op, op_sp) in &args.operands {
         match op {
             ast::InlineAsmOperand::Out { reg, expr, .. }
@@ -257,6 +261,9 @@ pub fn parse_asm_args<'a>(
                     regclass_outputs.push(*op_sp);
                 }
             }
+            ast::InlineAsmOperand::Label { .. } => {
+                labels_sp.push(*op_sp);
+            }
             _ => {}
         }
     }
@@ -268,6 +275,9 @@ pub fn parse_asm_args<'a>(
         // Bail out now since this is likely to confuse MIR
         return Err(err);
     }
+    if args.options.contains(ast::InlineAsmOptions::MAY_UNWIND) && !labels_sp.is_empty() {
+        dcx.emit_err(errors::AsmMayUnwind { labels_sp });
+    }
 
     if args.clobber_abis.len() > 0 {
         if is_global_asm {

compiler/rustc_builtin_macros/src/errors.rs

@@ -766,6 +766,13 @@ pub(crate) struct AsmNoReturn {
     pub(crate) outputs_sp: Vec<Span>,
 }
 
+#[derive(Diagnostic)]
+#[diag(builtin_macros_asm_mayunwind)]
+pub(crate) struct AsmMayUnwind {
+    #[primary_span]
+    pub(crate) labels_sp: Vec<Span>,
+}
+
 #[derive(Diagnostic)]
 #[diag(builtin_macros_global_asm_clobber_abi)]
 pub(crate) struct GlobalAsmClobberAbi {

compiler/rustc_codegen_cranelift/src/base.rs

@@ -445,7 +445,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                 template,
                 operands,
                 options,
-                destination,
+                targets,
                 line_spans: _,
                 unwind: _,
             } => {
@@ -456,13 +456,25 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                     );
                 }
 
+                let have_labels = if options.contains(InlineAsmOptions::NORETURN) {
+                    !targets.is_empty()
+                } else {
+                    targets.len() > 1
+                };
+                if have_labels {
+                    fx.tcx.dcx().span_fatal(
+                        source_info.span,
+                        "cranelift doesn't support labels in inline assembly.",
+                    );
+                }
+
                 crate::inline_asm::codegen_inline_asm_terminator(
                     fx,
                     source_info.span,
                     template,
                     operands,
                     *options,
-                    *destination,
+                    targets.get(0).copied(),
                 );
             }
             TerminatorKind::UnwindTerminate(reason) => {

compiler/rustc_codegen_cranelift/src/global_asm.rs

@@ -78,7 +78,8 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String,
                         InlineAsmOperand::In { .. }
                         | InlineAsmOperand::Out { .. }
                         | InlineAsmOperand::InOut { .. }
-                        | InlineAsmOperand::SplitInOut { .. } => {
+                        | InlineAsmOperand::SplitInOut { .. }
+                        | InlineAsmOperand::Label { .. } => {
                             span_bug!(op_sp, "invalid operand type for global_asm!")
                         }
                     }

compiler/rustc_codegen_cranelift/src/inline_asm.rs

@@ -129,6 +129,9 @@ pub(crate) fn codegen_inline_asm_terminator<'tcx>(
                 let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
                 CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance).name.to_owned() }
             }
+            InlineAsmOperand::Label { .. } => {
+                span_bug!(span, "asm! label operands are not yet supported");
+            }
         })
         .collect::<Vec<_>>();
 

compiler/rustc_codegen_gcc/src/asm.rs

@@ -107,7 +107,7 @@ enum ConstraintOrRegister {
 
 
 impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
-    fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], instance: Instance<'_>, _dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>) {
+    fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], instance: Instance<'_>, dest: Option<Self::BasicBlock>, _catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>) {
         if options.contains(InlineAsmOptions::MAY_UNWIND) {
             self.sess().dcx()
                 .create_err(UnwindingInlineAsm { span: span[0] })
@@ -126,6 +126,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         // added to `outputs.len()`
         let mut inputs = vec![];
 
+        // GCC index of a label equals its position in the array added to
+        // `outputs.len() + inputs.len()`.
+        let mut labels = vec![];
+
         // Clobbers collected from `out("explicit register") _` and `inout("expl_reg") var => _`
         let mut clobbers = vec![];
 
@@ -269,6 +273,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                     // some targets to add a leading underscore (Mach-O).
                     constants_len += self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len();
                 }
+
+                InlineAsmOperandRef::Label { label } => {
+                    labels.push(label);
+                }
             }
         }
 
@@ -368,6 +376,10 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                 InlineAsmOperandRef::Const { .. } => {
                     // processed in the previous pass
                 }
+
+                InlineAsmOperandRef::Label { .. } => {
+                    // processed in the previous pass
+                }
             }
         }
 
@@ -454,6 +466,14 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                         InlineAsmOperandRef::Const { ref string } => {
                             template_str.push_str(string);
                         }
+
+                        InlineAsmOperandRef::Label { label } => {
+                            let label_gcc_index = labels.iter()
+                                .position(|&l| l == label)
+                                .expect("wrong rust index");
+                            let gcc_index = label_gcc_index + outputs.len() + inputs.len();
+                            push_to_template(Some('l'), gcc_index);
+                        }
                     }
                 }
             }
@@ -466,7 +486,12 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         // 4. Generate Extended Asm block
 
         let block = self.llbb();
-        let extended_asm = block.add_extended_asm(None, &template_str);
+        let extended_asm = if let Some(dest) = dest {
+            assert!(!labels.is_empty());
+            block.end_with_extended_asm_goto(None, &template_str, &labels, Some(dest))
+        } else {
+            block.add_extended_asm(None, &template_str)
+        };
 
         for op in &outputs {
             extended_asm.add_output_operand(None, &op.to_constraint(), op.tmp_var);
@@ -494,7 +519,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         if !options.contains(InlineAsmOptions::NOSTACK) {
             // TODO(@Commeownist): figure out how to align stack
         }
-        if options.contains(InlineAsmOptions::NORETURN) {
+        if dest.is_none() && options.contains(InlineAsmOptions::NORETURN) {
             let builtin_unreachable = self.context.get_builtin_function("__builtin_unreachable");
             let builtin_unreachable: RValue<'gcc> = unsafe { std::mem::transmute(builtin_unreachable) };
             self.call(self.type_void(), None, None, builtin_unreachable, &[], None);

compiler/rustc_codegen_llvm/src/asm.rs

@@ -28,7 +28,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         options: InlineAsmOptions,
         line_spans: &[Span],
         instance: Instance<'_>,
-        dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
+        dest: Option<Self::BasicBlock>,
+        catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
     ) {
         let asm_arch = self.tcx.sess.asm_arch.unwrap();
 
@@ -165,6 +166,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         }
 
         // Build the template string
+        let mut labels = vec![];
         let mut template_str = String::new();
         for piece in template {
             match *piece {
@@ -205,6 +207,11 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                             // Only emit the raw symbol name
                             template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx]));
                         }
+                        InlineAsmOperandRef::Label { label } => {
+                            template_str.push_str(&format!("${{{}:l}}", constraints.len()));
+                            constraints.push("!i".to_owned());
+                            labels.push(label);
+                        }
                     }
                 }
             }
@@ -292,12 +299,14 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             &constraints.join(","),
             &inputs,
             output_type,
+            &labels,
             volatile,
             alignstack,
             dialect,
             line_spans,
             options.contains(InlineAsmOptions::MAY_UNWIND),
-            dest_catch_funclet,
+            dest,
+            catch_funclet,
         )
         .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
 
@@ -317,7 +326,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs });
 
         // Switch to the 'normal' basic block if we did an `invoke` instead of a `call`
-        if let Some((dest, _, _)) = dest_catch_funclet {
+        if let Some(dest) = dest {
             self.switch_to_block(dest);
         }
 
@@ -415,16 +424,14 @@ pub(crate) fn inline_asm_call<'ll>(
     cons: &str,
     inputs: &[&'ll Value],
     output: &'ll llvm::Type,
+    labels: &[&'ll llvm::BasicBlock],
     volatile: bool,
     alignstack: bool,
     dia: llvm::AsmDialect,
     line_spans: &[Span],
     unwind: bool,
-    dest_catch_funclet: Option<(
-        &'ll llvm::BasicBlock,
-        &'ll llvm::BasicBlock,
-        Option<&Funclet<'ll>>,
-    )>,
+    dest: Option<&'ll llvm::BasicBlock>,
+    catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>,
 ) -> Option<&'ll Value> {
     let volatile = if volatile { llvm::True } else { llvm::False };
     let alignstack = if alignstack { llvm::True } else { llvm::False };
@@ -457,8 +464,11 @@ pub(crate) fn inline_asm_call<'ll>(
                 can_throw,
             );
 
-            let call = if let Some((dest, catch, funclet)) = dest_catch_funclet {
-                bx.invoke(fty, None, None, v, inputs, dest, catch, funclet)
+            let call = if !labels.is_empty() {
+                assert!(catch_funclet.is_none());
+                bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None)
+            } else if let Some((catch, funclet)) = catch_funclet {
+                bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet)
             } else {
                 bx.call(fty, None, None, v, inputs, None)
             };

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1538,6 +1538,58 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
         }
     }
 
+    pub(crate) fn callbr(
+        &mut self,
+        llty: &'ll Type,
+        fn_attrs: Option<&CodegenFnAttrs>,
+        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
+        llfn: &'ll Value,
+        args: &[&'ll Value],
+        default_dest: &'ll BasicBlock,
+        indirect_dest: &[&'ll BasicBlock],
+        funclet: Option<&Funclet<'ll>>,
+    ) -> &'ll Value {
+        debug!("invoke {:?} with args ({:?})", llfn, args);
+
+        let args = self.check_call("callbr", llty, llfn, args);
+        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
+        let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw);
+        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
+        if let Some(funclet_bundle) = funclet_bundle {
+            bundles.push(funclet_bundle);
+        }
+
+        // Emit CFI pointer type membership test
+        self.cfi_type_test(fn_attrs, fn_abi, llfn);
+
+        // Emit KCFI operand bundle
+        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, llfn);
+        let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw);
+        if let Some(kcfi_bundle) = kcfi_bundle {
+            bundles.push(kcfi_bundle);
+        }
+
+        let callbr = unsafe {
+            llvm::LLVMRustBuildCallBr(
+                self.llbuilder,
+                llty,
+                llfn,
+                default_dest,
+                indirect_dest.as_ptr(),
+                indirect_dest.len() as c_uint,
+                args.as_ptr(),
+                args.len() as c_uint,
+                bundles.as_ptr(),
+                bundles.len() as c_uint,
+                UNNAMED,
+            )
+        };
+        if let Some(fn_abi) = fn_abi {
+            fn_abi.apply_attrs_callsite(self, callbr);
+        }
+        callbr
+    }
+
     // Emits CFI pointer type membership tests.
     fn cfi_type_test(
         &mut self,

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -448,12 +448,14 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     constraint,
                     inputs,
                     self.type_void(),
+                    &[],
                     true,
                     false,
                     llvm::AsmDialect::Att,
                     &[span],
                     false,
                     None,
+                    None,
                 )
                 .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`"));
 

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1616,6 +1616,20 @@ extern "C" {
         Name: *const c_char,
     ) -> &'a Value;
 
+    pub fn LLVMRustBuildCallBr<'a>(
+        B: &Builder<'a>,
+        Ty: &'a Type,
+        Fn: &'a Value,
+        DefaultDest: &'a BasicBlock,
+        IndirectDests: *const &'a BasicBlock,
+        NumIndirectDests: c_uint,
+        Args: *const &'a Value,
+        NumArgs: c_uint,
+        OpBundles: *const &OperandBundleDef<'a>,
+        NumOpBundles: c_uint,
+        Name: *const c_char,
+    ) -> &'a Value;
+
     pub fn LLVMRustSetFastMath(Instr: &Value);
     pub fn LLVMRustSetAlgebraicMath(Instr: &Value);
     pub fn LLVMRustSetAllowReassoc(Instr: &Value);

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -264,7 +264,8 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
             mir::UnwindAction::Unreachable => None,
         };
 
-        if let Some(cleanup) = unwind_target {
+        if operands.iter().any(|x| matches!(x, InlineAsmOperandRef::Label { .. })) {
+            assert!(unwind_target.is_none());
             let ret_llbb = if let Some(target) = destination {
                 fx.llbb(target)
             } else {
@@ -277,11 +278,29 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
                 options,
                 line_spans,
                 instance,
-                Some((ret_llbb, cleanup, self.funclet(fx))),
+                Some(ret_llbb),
+                None,
+            );
+            MergingSucc::False
+        } else if let Some(cleanup) = unwind_target {
+            let ret_llbb = if let Some(target) = destination {
+                fx.llbb(target)
+            } else {
+                fx.unreachable_block()
+            };
+
+            bx.codegen_inline_asm(
+                template,
+                operands,
+                options,
+                line_spans,
+                instance,
+                Some(ret_llbb),
+                Some((cleanup, self.funclet(fx))),
             );
             MergingSucc::False
         } else {
-            bx.codegen_inline_asm(template, operands, options, line_spans, instance, None);
+            bx.codegen_inline_asm(template, operands, options, line_spans, instance, None, None);
 
             if let Some(target) = destination {
                 self.funclet_br(fx, bx, target, mergeable_succ)
@@ -1100,7 +1119,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         operands: &[mir::InlineAsmOperand<'tcx>],
         options: ast::InlineAsmOptions,
         line_spans: &[Span],
-        destination: Option<mir::BasicBlock>,
+        targets: &[mir::BasicBlock],
         unwind: mir::UnwindAction,
         instance: Instance<'_>,
         mergeable_succ: bool,
@@ -1152,6 +1171,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 mir::InlineAsmOperand::SymStatic { def_id } => {
                     InlineAsmOperandRef::SymStatic { def_id }
                 }
+                mir::InlineAsmOperand::Label { target_index } => {
+                    InlineAsmOperandRef::Label { label: self.llbb(targets[target_index]) }
+                }
             })
             .collect();
 
@@ -1162,7 +1184,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             &operands,
             options,
             line_spans,
-            destination,
+            if options.contains(InlineAsmOptions::NORETURN) {
+                None
+            } else {
+                targets.get(0).copied()
+            },
             unwind,
             instance,
             mergeable_succ,
@@ -1318,7 +1344,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 ref operands,
                 options,
                 line_spans,
-                destination,
+                ref targets,
                 unwind,
             } => self.codegen_asm_terminator(
                 helper,
@@ -1328,7 +1354,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 operands,
                 options,
                 line_spans,
-                destination,
+                targets,
                 unwind,
                 self.instance,
                 mergeable_succ(),

compiler/rustc_codegen_ssa/src/mono_item.rs

@@ -76,7 +76,8 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
                             hir::InlineAsmOperand::In { .. }
                             | hir::InlineAsmOperand::Out { .. }
                             | hir::InlineAsmOperand::InOut { .. }
-                            | hir::InlineAsmOperand::SplitInOut { .. } => {
+                            | hir::InlineAsmOperand::SplitInOut { .. }
+                            | hir::InlineAsmOperand::Label { .. } => {
                                 span_bug!(*op_sp, "invalid operand type for global_asm!")
                             }
                         })

compiler/rustc_codegen_ssa/src/traits/asm.rs

@@ -33,6 +33,9 @@ pub enum InlineAsmOperandRef<'tcx, B: BackendTypes + ?Sized> {
     SymStatic {
         def_id: DefId,
     },
+    Label {
+        label: B::BasicBlock,
+    },
 }
 
 #[derive(Debug)]
@@ -51,7 +54,8 @@ pub trait AsmBuilderMethods<'tcx>: BackendTypes {
         options: InlineAsmOptions,
         line_spans: &[Span],
         instance: Instance<'_>,
-        dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
+        dest: Option<Self::BasicBlock>,
+        catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
     );
 }
 

compiler/rustc_const_eval/messages.ftl

@@ -232,9 +232,6 @@ const_eval_non_const_fn_call =
 const_eval_non_const_impl =
     impl defined here, but it is not `const`
 
-const_eval_noreturn_asm_returned =
-    returned from noreturn inline assembly
-
 const_eval_not_enough_caller_args =
     calling a function with fewer arguments than it requires
 

compiler/rustc_const_eval/src/interpret/machine.rs

@@ -374,11 +374,17 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized {
         kind: Option<MemoryKind<Self::MemoryKind>>,
     ) -> InterpResult<'tcx, Cow<'b, Allocation<Self::Provenance, Self::AllocExtra, Self::Bytes>>>;
 
+    /// Evaluate the inline assembly.
+    ///
+    /// This should take care of jumping to the next block (one of `targets`) when asm goto
+    /// is triggered, `targets[0]` when the assembly falls through, or diverge in case of
+    /// `InlineAsmOptions::NORETURN` being set.
     fn eval_inline_asm(
         _ecx: &mut InterpCx<'mir, 'tcx, Self>,
         _template: &'tcx [InlineAsmTemplatePiece],
         _operands: &[mir::InlineAsmOperand<'tcx>],
         _options: InlineAsmOptions,
+        _targets: &[mir::BasicBlock],
     ) -> InterpResult<'tcx> {
         throw_unsup_format!("inline assembly is not supported")
     }

compiler/rustc_const_eval/src/interpret/terminator.rs

@@ -1,6 +1,5 @@
 use std::borrow::Cow;
 
-use rustc_ast::ast::InlineAsmOptions;
 use rustc_middle::{
     mir,
     ty::{
@@ -224,15 +223,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 terminator.kind
             ),
 
-            InlineAsm { template, ref operands, options, destination, .. } => {
-                M::eval_inline_asm(self, template, operands, options)?;
-                if options.contains(InlineAsmOptions::NORETURN) {
-                    throw_ub_custom!(fluent::const_eval_noreturn_asm_returned);
-                }
-                self.go_to_block(
-                    destination
-                        .expect("InlineAsm terminators without noreturn must have a destination"),
-                )
+            InlineAsm { template, ref operands, options, ref targets, .. } => {
+                M::eval_inline_asm(self, template, operands, options, targets)?;
             }
         }
 

compiler/rustc_const_eval/src/transform/validate.rs

@@ -471,9 +471,9 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
                 self.check_edge(location, *real_target, EdgeKind::Normal);
                 self.check_unwind_edge(location, *unwind);
             }
-            TerminatorKind::InlineAsm { destination, unwind, .. } => {
-                if let Some(destination) = destination {
-                    self.check_edge(location, *destination, EdgeKind::Normal);
+            TerminatorKind::InlineAsm { targets, unwind, .. } => {
+                for &target in targets {
+                    self.check_edge(location, target, EdgeKind::Normal);
                 }
                 self.check_unwind_edge(location, *unwind);
             }

compiler/rustc_feature/src/unstable.rs

@@ -345,6 +345,8 @@ declare_features! (
     (unstable, asm_const, "1.58.0", Some(93332)),
     /// Enables experimental inline assembly support for additional architectures.
     (unstable, asm_experimental_arch, "1.58.0", Some(93335)),
+    /// Allows using `label` operands in inline assembly.
+    (unstable, asm_goto, "CURRENT_RUSTC_VERSION", Some(119364)),
     /// Allows the `may_unwind` option in inline assembly.
     (unstable, asm_unwind, "1.58.0", Some(93334)),
     /// Allows users to enforce equality of associated constants `TraitImpl<AssocConst=3>`.

compiler/rustc_hir/src/hir.rs

@@ -2650,6 +2650,9 @@ pub enum InlineAsmOperand<'hir> {
         path: QPath<'hir>,
         def_id: DefId,
     },
+    Label {
+        block: &'hir Block<'hir>,
+    },
 }
 
 impl<'hir> InlineAsmOperand<'hir> {
@@ -2659,7 +2662,10 @@ impl<'hir> InlineAsmOperand<'hir> {
             | Self::Out { reg, .. }
             | Self::InOut { reg, .. }
             | Self::SplitInOut { reg, .. } => Some(reg),
-            Self::Const { .. } | Self::SymFn { .. } | Self::SymStatic { .. } => None,
+            Self::Const { .. }
+            | Self::SymFn { .. }
+            | Self::SymStatic { .. }
+            | Self::Label { .. } => None,
         }
     }
 
@@ -2680,6 +2686,12 @@ pub struct InlineAsm<'hir> {
     pub line_spans: &'hir [Span],
 }
 
+impl InlineAsm<'_> {
+    pub fn contains_label(&self) -> bool {
+        self.operands.iter().any(|x| matches!(x.0, InlineAsmOperand::Label { .. }))
+    }
+}
+
 /// Represents a parameter in a function header.
 #[derive(Debug, Clone, Copy, HashStable_Generic)]
 pub struct Param<'hir> {

compiler/rustc_hir/src/intravisit.rs

@@ -1289,6 +1289,7 @@ pub fn walk_inline_asm<'v, V: Visitor<'v>>(
             InlineAsmOperand::SymStatic { path, .. } => {
                 try_visit!(visitor.visit_qpath(path, id, *op_sp));
             }
+            InlineAsmOperand::Label { block } => try_visit!(visitor.visit_block(block)),
         }
     }
     V::Result::output()

compiler/rustc_hir_analysis/src/check/intrinsicck.rs

@@ -470,6 +470,8 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                         }
                     };
                 }
+                // No special checking is needed for labels.
+                hir::InlineAsmOperand::Label { .. } => {}
             }
         }
     }

compiler/rustc_hir_pretty/src/lib.rs

@@ -1265,6 +1265,10 @@ impl<'a> State<'a> {
                     s.space();
                     s.print_qpath(path, true);
                 }
+                hir::InlineAsmOperand::Label { block } => {
+                    s.head("label");
+                    s.print_block(block);
+                }
             },
             AsmArg::Options(opts) => {
                 s.word("options");

compiler/rustc_hir_typeck/src/expr.rs

@@ -3232,6 +3232,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 
     fn check_expr_asm(&self, asm: &'tcx hir::InlineAsm<'tcx>) -> Ty<'tcx> {
+        let mut diverge = asm.options.contains(ast::InlineAsmOptions::NORETURN);
+
         for (op, _op_sp) in asm.operands {
             match op {
                 hir::InlineAsmOperand::In { expr, .. } => {
@@ -3253,13 +3255,24 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // be well-formed.
                 hir::InlineAsmOperand::Const { .. } | hir::InlineAsmOperand::SymFn { .. } => {}
                 hir::InlineAsmOperand::SymStatic { .. } => {}
+                hir::InlineAsmOperand::Label { block } => {
+                    let previous_diverges = self.diverges.get();
+
+                    // The label blocks should have unit return value or diverge.
+                    let ty =
+                        self.check_block_with_expected(block, ExpectHasType(self.tcx.types.unit));
+                    if !ty.is_never() {
+                        self.demand_suptype(block.span, self.tcx.types.unit, ty);
+                        diverge = false;
+                    }
+
+                    // We need this to avoid false unreachable warning when a label diverges.
+                    self.diverges.set(previous_diverges);
+                }
             }
         }
-        if asm.options.contains(ast::InlineAsmOptions::NORETURN) {
-            self.tcx.types.never
-        } else {
-            Ty::new_unit(self.tcx)
-        }
+
+        if diverge { self.tcx.types.never } else { self.tcx.types.unit }
     }
 
     fn check_offset_of(

compiler/rustc_hir_typeck/src/expr_use_visitor.rs

@@ -293,6 +293,9 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
                         | hir::InlineAsmOperand::Const { .. }
                         | hir::InlineAsmOperand::SymFn { .. }
                         | hir::InlineAsmOperand::SymStatic { .. } => {}
+                        hir::InlineAsmOperand::Label { block } => {
+                            self.walk_block(block);
+                        }
                     }
                 }
             }

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -1549,6 +1549,31 @@ LLVMRustBuildInvoke(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
                                       Name));
 }
 
+extern "C" LLVMValueRef
+LLVMRustBuildCallBr(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
+                    LLVMBasicBlockRef DefaultDest,
+                    LLVMBasicBlockRef *IndirectDests, unsigned NumIndirectDests,
+                    LLVMValueRef *Args, unsigned NumArgs,
+                    OperandBundleDef **OpBundles, unsigned NumOpBundles,
+                    const char *Name) {
+  Value *Callee = unwrap(Fn);
+  FunctionType *FTy = unwrap<FunctionType>(Ty);
+
+  // FIXME: Is there a way around this?
+  std::vector<BasicBlock*> IndirectDestsUnwrapped;
+  IndirectDestsUnwrapped.reserve(NumIndirectDests);
+  for (unsigned i = 0; i < NumIndirectDests; ++i) {
+    IndirectDestsUnwrapped.push_back(unwrap(IndirectDests[i]));
+  }
+
+  return wrap(unwrap(B)->CreateCallBr(
+      FTy, Callee, unwrap(DefaultDest),
+      ArrayRef<BasicBlock*>(IndirectDestsUnwrapped),
+      ArrayRef<Value*>(unwrap(Args), NumArgs),
+      ArrayRef<OperandBundleDef>(*OpBundles, NumOpBundles),
+      Name));
+}
+
 extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B,
                                                LLVMBasicBlockRef BB) {
   auto Point = unwrap(BB)->getFirstInsertionPt();

compiler/rustc_middle/src/mir/mod.rs

@@ -1716,13 +1716,13 @@ mod size_asserts {
     use super::*;
     use rustc_data_structures::static_assert_size;
     // tidy-alphabetical-start
-    static_assert_size!(BasicBlockData<'_>, 136);
+    static_assert_size!(BasicBlockData<'_>, 144);
     static_assert_size!(LocalDecl<'_>, 40);
     static_assert_size!(SourceScopeData<'_>, 72);
     static_assert_size!(Statement<'_>, 32);
     static_assert_size!(StatementKind<'_>, 16);
-    static_assert_size!(Terminator<'_>, 104);
-    static_assert_size!(TerminatorKind<'_>, 88);
+    static_assert_size!(Terminator<'_>, 112);
+    static_assert_size!(TerminatorKind<'_>, 96);
     static_assert_size!(VarDebugInfo<'_>, 88);
     // tidy-alphabetical-end
 }

compiler/rustc_middle/src/mir/pretty.rs

@@ -7,7 +7,7 @@ use std::path::{Path, PathBuf};
 use crate::mir::interpret::ConstAllocation;
 
 use super::graphviz::write_mir_fn_graphviz;
-use rustc_ast::InlineAsmTemplatePiece;
+use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_middle::mir::interpret::{
     alloc_range, read_target_uint, AllocBytes, AllocId, Allocation, GlobalAlloc, Pointer,
     Provenance,
@@ -830,6 +830,9 @@ impl<'tcx> TerminatorKind<'tcx> {
                         InlineAsmOperand::SymStatic { def_id } => {
                             write!(fmt, "sym_static {def_id:?}")?;
                         }
+                        InlineAsmOperand::Label { target_index } => {
+                            write!(fmt, "label {target_index}")?;
+                        }
                     }
                 }
                 write!(fmt, ", options({options:?}))")
@@ -868,16 +871,19 @@ impl<'tcx> TerminatorKind<'tcx> {
                 vec!["real".into(), "unwind".into()]
             }
             FalseUnwind { unwind: _, .. } => vec!["real".into()],
-            InlineAsm { destination: Some(_), unwind: UnwindAction::Cleanup(_), .. } => {
-                vec!["return".into(), "unwind".into()]
+            InlineAsm { options, ref targets, unwind, .. } => {
+                let mut vec = Vec::with_capacity(targets.len() + 1);
+                if !options.contains(InlineAsmOptions::NORETURN) {
+                    vec.push("return".into());
+                }
+                vec.resize(targets.len(), "label".into());
+
+                if let UnwindAction::Cleanup(_) = unwind {
+                    vec.push("unwind".into());
+                }
+
+                vec
             }
-            InlineAsm { destination: Some(_), unwind: _, .. } => {
-                vec!["return".into()]
-            }
-            InlineAsm { destination: None, unwind: UnwindAction::Cleanup(_), .. } => {
-                vec!["unwind".into()]
-            }
-            InlineAsm { destination: None, unwind: _, .. } => vec![],
         }
     }
 }

compiler/rustc_middle/src/mir/syntax.rs

@@ -793,9 +793,10 @@ pub enum TerminatorKind<'tcx> {
         /// used to map assembler errors back to the line in the source code.
         line_spans: &'tcx [Span],
 
-        /// Destination block after the inline assembly returns, unless it is
-        /// diverging (InlineAsmOptions::NORETURN).
-        destination: Option<BasicBlock>,
+        /// Valid targets for the inline assembly.
+        /// The first element is the fallthrough destination, unless
+        /// InlineAsmOptions::NORETURN is set.
+        targets: Vec<BasicBlock>,
 
         /// Action to be taken if the inline assembly unwinds. This is present
         /// if and only if InlineAsmOptions::MAY_UNWIND is set.
@@ -918,6 +919,10 @@ pub enum InlineAsmOperand<'tcx> {
     SymStatic {
         def_id: DefId,
     },
+    Label {
+        /// This represents the index into the `targets` array in `TerminatorKind::InlineAsm`.
+        target_index: usize,
+    },
 }
 
 /// Type for MIR `Assert` terminator error messages.

compiler/rustc_middle/src/mir/terminator.rs

@@ -336,8 +336,7 @@ pub struct Terminator<'tcx> {
 }
 
 pub type Successors<'a> = impl DoubleEndedIterator<Item = BasicBlock> + 'a;
-pub type SuccessorsMut<'a> =
-    iter::Chain<std::option::IntoIter<&'a mut BasicBlock>, slice::IterMut<'a, BasicBlock>>;
+pub type SuccessorsMut<'a> = impl DoubleEndedIterator<Item = &'a mut BasicBlock> + 'a;
 
 impl<'tcx> Terminator<'tcx> {
     #[inline]
@@ -371,40 +370,36 @@ impl<'tcx> TerminatorKind<'tcx> {
     pub fn successors(&self) -> Successors<'_> {
         use self::TerminatorKind::*;
         match *self {
-            Call { target: Some(t), unwind: UnwindAction::Cleanup(ref u), .. }
-            | Yield { resume: t, drop: Some(ref u), .. }
-            | Drop { target: t, unwind: UnwindAction::Cleanup(ref u), .. }
-            | Assert { target: t, unwind: UnwindAction::Cleanup(ref u), .. }
-            | FalseUnwind { real_target: t, unwind: UnwindAction::Cleanup(ref u) }
-            | InlineAsm { destination: Some(t), unwind: UnwindAction::Cleanup(ref u), .. } => {
-                Some(t).into_iter().chain(slice::from_ref(u).into_iter().copied())
+            Call { target: Some(ref t), unwind: UnwindAction::Cleanup(u), .. }
+            | Yield { resume: ref t, drop: Some(u), .. }
+            | Drop { target: ref t, unwind: UnwindAction::Cleanup(u), .. }
+            | Assert { target: ref t, unwind: UnwindAction::Cleanup(u), .. }
+            | FalseUnwind { real_target: ref t, unwind: UnwindAction::Cleanup(u) } => {
+                slice::from_ref(t).into_iter().copied().chain(Some(u))
             }
-            Goto { target: t }
-            | Call { target: None, unwind: UnwindAction::Cleanup(t), .. }
-            | Call { target: Some(t), unwind: _, .. }
-            | Yield { resume: t, drop: None, .. }
-            | Drop { target: t, unwind: _, .. }
-            | Assert { target: t, unwind: _, .. }
-            | FalseUnwind { real_target: t, unwind: _ }
-            | InlineAsm { destination: None, unwind: UnwindAction::Cleanup(t), .. }
-            | InlineAsm { destination: Some(t), unwind: _, .. } => {
-                Some(t).into_iter().chain((&[]).into_iter().copied())
+            Goto { target: ref t }
+            | Call { target: None, unwind: UnwindAction::Cleanup(ref t), .. }
+            | Call { target: Some(ref t), unwind: _, .. }
+            | Yield { resume: ref t, drop: None, .. }
+            | Drop { target: ref t, unwind: _, .. }
+            | Assert { target: ref t, unwind: _, .. }
+            | FalseUnwind { real_target: ref t, unwind: _ } => {
+                slice::from_ref(t).into_iter().copied().chain(None)
             }
             UnwindResume
             | UnwindTerminate(_)
             | CoroutineDrop
             | Return
             | Unreachable
-            | Call { target: None, unwind: _, .. }
-            | InlineAsm { destination: None, unwind: _, .. } => {
-                None.into_iter().chain((&[]).into_iter().copied())
+            | Call { target: None, unwind: _, .. } => (&[]).into_iter().copied().chain(None),
+            InlineAsm { ref targets, unwind: UnwindAction::Cleanup(u), .. } => {
+                targets.iter().copied().chain(Some(u))
             }
-            SwitchInt { ref targets, .. } => {
-                None.into_iter().chain(targets.targets.iter().copied())
+            InlineAsm { ref targets, unwind: _, .. } => targets.iter().copied().chain(None),
+            SwitchInt { ref targets, .. } => targets.targets.iter().copied().chain(None),
+            FalseEdge { ref real_target, imaginary_target } => {
+                slice::from_ref(real_target).into_iter().copied().chain(Some(imaginary_target))
             }
-            FalseEdge { real_target, ref imaginary_target } => Some(real_target)
-                .into_iter()
-                .chain(slice::from_ref(imaginary_target).into_iter().copied()),
         }
     }
 
@@ -416,33 +411,31 @@ impl<'tcx> TerminatorKind<'tcx> {
             | Yield { resume: ref mut t, drop: Some(ref mut u), .. }
             | Drop { target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u), .. }
             | Assert { target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u), .. }
-            | FalseUnwind { real_target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u) }
-            | InlineAsm {
-                destination: Some(ref mut t),
-                unwind: UnwindAction::Cleanup(ref mut u),
-                ..
-            } => Some(t).into_iter().chain(slice::from_mut(u)),
+            | FalseUnwind { real_target: ref mut t, unwind: UnwindAction::Cleanup(ref mut u) } => {
+                slice::from_mut(t).into_iter().chain(Some(u))
+            }
             Goto { target: ref mut t }
             | Call { target: None, unwind: UnwindAction::Cleanup(ref mut t), .. }
             | Call { target: Some(ref mut t), unwind: _, .. }
             | Yield { resume: ref mut t, drop: None, .. }
             | Drop { target: ref mut t, unwind: _, .. }
             | Assert { target: ref mut t, unwind: _, .. }
-            | FalseUnwind { real_target: ref mut t, unwind: _ }
-            | InlineAsm { destination: None, unwind: UnwindAction::Cleanup(ref mut t), .. }
-            | InlineAsm { destination: Some(ref mut t), unwind: _, .. } => {
-                Some(t).into_iter().chain(&mut [])
+            | FalseUnwind { real_target: ref mut t, unwind: _ } => {
+                slice::from_mut(t).into_iter().chain(None)
             }
             UnwindResume
             | UnwindTerminate(_)
             | CoroutineDrop
             | Return
             | Unreachable
-            | Call { target: None, unwind: _, .. }
-            | InlineAsm { destination: None, unwind: _, .. } => None.into_iter().chain(&mut []),
-            SwitchInt { ref mut targets, .. } => None.into_iter().chain(&mut targets.targets),
+            | Call { target: None, unwind: _, .. } => (&mut []).into_iter().chain(None),
+            InlineAsm { ref mut targets, unwind: UnwindAction::Cleanup(ref mut u), .. } => {
+                targets.iter_mut().chain(Some(u))
+            }
+            InlineAsm { ref mut targets, unwind: _, .. } => targets.iter_mut().chain(None),
+            SwitchInt { ref mut targets, .. } => targets.targets.iter_mut().chain(None),
             FalseEdge { ref mut real_target, ref mut imaginary_target } => {
-                Some(real_target).into_iter().chain(slice::from_mut(imaginary_target))
+                slice::from_mut(real_target).into_iter().chain(Some(imaginary_target))
             }
         }
     }
@@ -514,7 +507,7 @@ pub enum TerminatorEdges<'mir, 'tcx> {
     Double(BasicBlock, BasicBlock),
     /// Special action for `Yield`, `Call` and `InlineAsm` terminators.
     AssignOnReturn {
-        return_: Option<BasicBlock>,
+        return_: &'mir [BasicBlock],
         /// The cleanup block, if it exists.
         cleanup: Option<BasicBlock>,
         place: CallReturnPlaces<'mir, 'tcx>,
@@ -578,31 +571,37 @@ impl<'tcx> TerminatorKind<'tcx> {
                 TerminatorEdges::Double(real_target, imaginary_target)
             }
 
-            Yield { resume: target, drop, resume_arg, value: _ } => {
+            Yield { resume: ref target, drop, resume_arg, value: _ } => {
                 TerminatorEdges::AssignOnReturn {
-                    return_: Some(target),
+                    return_: slice::from_ref(target),
                     cleanup: drop,
                     place: CallReturnPlaces::Yield(resume_arg),
                 }
             }
 
-            Call { unwind, destination, target, func: _, args: _, fn_span: _, call_source: _ } => {
-                TerminatorEdges::AssignOnReturn {
-                    return_: target,
-                    cleanup: unwind.cleanup_block(),
-                    place: CallReturnPlaces::Call(destination),
-                }
-            }
+            Call {
+                unwind,
+                destination,
+                ref target,
+                func: _,
+                args: _,
+                fn_span: _,
+                call_source: _,
+            } => TerminatorEdges::AssignOnReturn {
+                return_: target.as_ref().map(slice::from_ref).unwrap_or_default(),
+                cleanup: unwind.cleanup_block(),
+                place: CallReturnPlaces::Call(destination),
+            },
 
             InlineAsm {
                 template: _,
                 ref operands,
                 options: _,
                 line_spans: _,
-                destination,
+                ref targets,
                 unwind,
             } => TerminatorEdges::AssignOnReturn {
-                return_: destination,
+                return_: targets,
                 cleanup: unwind.cleanup_block(),
                 place: CallReturnPlaces::InlineAsm(operands),
             },

compiler/rustc_middle/src/mir/visit.rs

@@ -565,7 +565,7 @@ macro_rules! make_mir_visitor {
                         operands,
                         options: _,
                         line_spans: _,
-                        destination: _,
+                        targets: _,
                         unwind: _,
                     } => {
                         for op in operands {
@@ -595,7 +595,8 @@ macro_rules! make_mir_visitor {
                                     self.visit_constant(value, location);
                                 }
                                 InlineAsmOperand::Out { place: None, .. }
-                                | InlineAsmOperand::SymStatic { def_id: _ } => {}
+                                | InlineAsmOperand::SymStatic { def_id: _ }
+                                | InlineAsmOperand::Label { target_index: _ } => {}
                             }
                         }
                     }

compiler/rustc_middle/src/thir.rs

@@ -565,6 +565,9 @@ pub enum InlineAsmOperand<'tcx> {
     SymStatic {
         def_id: DefId,
     },
+    Label {
+        block: BlockId,
+    },
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, HashStable)]

compiler/rustc_middle/src/thir/visit.rs

@@ -162,6 +162,7 @@ pub fn walk_expr<'thir, 'tcx: 'thir, V: Visitor<'thir, 'tcx>>(
                     | Const { value: _, span: _ }
                     | SymFn { value: _, span: _ }
                     | SymStatic { def_id: _ } => {}
+                    Label { block } => visitor.visit_block(&visitor.thir()[*block]),
                 }
             }
         }

compiler/rustc_mir_build/src/build/expr/into.rs

@@ -383,6 +383,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 line_spans,
             }) => {
                 use rustc_middle::{mir, thir};
+
+                let destination_block = this.cfg.start_new_block();
+                let mut targets = if options.contains(InlineAsmOptions::NORETURN) {
+                    vec![]
+                } else {
+                    vec![destination_block]
+                };
+
                 let operands = operands
                     .into_iter()
                     .map(|op| match *op {
@@ -438,14 +446,28 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         thir::InlineAsmOperand::SymStatic { def_id } => {
                             mir::InlineAsmOperand::SymStatic { def_id }
                         }
+                        thir::InlineAsmOperand::Label { block } => {
+                            let target = this.cfg.start_new_block();
+                            let target_index = targets.len();
+                            targets.push(target);
+
+                            let tmp = this.get_unit_temp();
+                            let target = unpack!(this.ast_block(tmp, target, block, source_info));
+                            this.cfg.terminate(
+                                target,
+                                source_info,
+                                TerminatorKind::Goto { target: destination_block },
+                            );
+
+                            mir::InlineAsmOperand::Label { target_index }
+                        }
                     })
                     .collect();
 
-                if !options.contains(InlineAsmOptions::NORETURN) {
+                if !expr.ty.is_never() {
                     this.cfg.push_assign_unit(block, source_info, destination, this.tcx);
                 }
 
-                let destination_block = this.cfg.start_new_block();
                 this.cfg.terminate(
                     block,
                     source_info,
@@ -454,11 +476,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         operands,
                         options,
                         line_spans,
-                        destination: if options.contains(InlineAsmOptions::NORETURN) {
-                            None
-                        } else {
-                            Some(destination_block)
-                        },
+                        targets,
                         unwind: if options.contains(InlineAsmOptions::MAY_UNWIND) {
                             UnwindAction::Continue
                         } else {

compiler/rustc_mir_build/src/lints.rs

@@ -199,9 +199,10 @@ impl<'mir, 'tcx, C: TerminatorClassifier<'tcx>> TriColorVisitor<BasicBlocks<'tcx
             | TerminatorKind::Unreachable
             | TerminatorKind::Yield { .. } => ControlFlow::Break(NonRecursive),
 
-            // A diverging InlineAsm is treated as non-recursing
-            TerminatorKind::InlineAsm { destination, .. } => {
-                if destination.is_some() {
+            // A InlineAsm without targets (diverging and contains no labels)
+            // is treated as non-recursing.
+            TerminatorKind::InlineAsm { ref targets, .. } => {
+                if !targets.is_empty() {
                     ControlFlow::Continue(())
                 } else {
                     ControlFlow::Break(NonRecursive)

compiler/rustc_mir_build/src/thir/cx/expr.rs

@@ -656,6 +656,9 @@ impl<'tcx> Cx<'tcx> {
                         hir::InlineAsmOperand::SymStatic { path: _, def_id } => {
                             InlineAsmOperand::SymStatic { def_id }
                         }
+                        hir::InlineAsmOperand::Label { block } => {
+                            InlineAsmOperand::Label { block: self.mirror_block(block) }
+                        }
                     })
                     .collect(),
                 options: asm.options,

compiler/rustc_mir_build/src/thir/print.rs

@@ -889,6 +889,12 @@ impl<'a, 'tcx> ThirPrinter<'a, 'tcx> {
                 print_indented!(self, format!("def_id: {:?}", def_id), depth_lvl + 1);
                 print_indented!(self, "}", depth_lvl + 1);
             }
+            InlineAsmOperand::Label { block } => {
+                print_indented!(self, "InlineAsmOperand::Block {", depth_lvl);
+                print_indented!(self, "block:", depth_lvl + 1);
+                self.print_block(*block, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl + 1);
+            }
         }
     }
 }

compiler/rustc_mir_dataflow/src/framework/direction.rs

@@ -242,9 +242,9 @@ impl Direction for Backward {
                     propagate(pred, &tmp);
                 }
 
-                mir::TerminatorKind::InlineAsm {
-                    destination: Some(dest), ref operands, ..
-                } if dest == bb => {
+                mir::TerminatorKind::InlineAsm { ref targets, ref operands, .. }
+                    if targets.contains(&bb) =>
+                {
                     let mut tmp = exit_state.clone();
                     analysis.apply_call_return_effect(
                         &mut tmp,
@@ -491,9 +491,12 @@ impl Direction for Forward {
                 if let Some(cleanup) = cleanup {
                     propagate(cleanup, exit_state);
                 }
-                if let Some(return_) = return_ {
+
+                if !return_.is_empty() {
                     analysis.apply_call_return_effect(exit_state, bb, place);
-                    propagate(return_, exit_state);
+                    for &target in return_ {
+                        propagate(target, exit_state);
+                    }
                 }
             }
             TerminatorEdges::SwitchInt { targets, discr } => {

compiler/rustc_mir_dataflow/src/framework/graphviz.rs

@@ -299,7 +299,9 @@ where
                 })?;
             }
 
-            mir::TerminatorKind::InlineAsm { destination: Some(_), ref operands, .. } => {
+            mir::TerminatorKind::InlineAsm { ref targets, ref operands, .. }
+                if !targets.is_empty() =>
+            {
                 self.write_row(w, "", "(on successful return)", |this, w, fmt| {
                     let state_on_unwind = this.results.get().clone();
                     this.results.apply_custom_effect(|analysis, state| {

compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs

@@ -271,7 +271,8 @@ impl<'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'_, 'tcx> {
                         InlineAsmOperand::In { .. }
                         | InlineAsmOperand::Const { .. }
                         | InlineAsmOperand::SymFn { .. }
-                        | InlineAsmOperand::SymStatic { .. } => {}
+                        | InlineAsmOperand::SymStatic { .. }
+                        | InlineAsmOperand::Label { .. } => {}
                     }
                 }
             }

compiler/rustc_mir_dataflow/src/move_paths/builder.rs

@@ -491,7 +491,7 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
                 ref operands,
                 options: _,
                 line_spans: _,
-                destination: _,
+                targets: _,
                 unwind: _,
             } => {
                 for op in operands {
@@ -515,7 +515,8 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
                         }
                         InlineAsmOperand::Const { value: _ }
                         | InlineAsmOperand::SymFn { value: _ }
-                        | InlineAsmOperand::SymStatic { def_id: _ } => {}
+                        | InlineAsmOperand::SymStatic { def_id: _ }
+                        | InlineAsmOperand::Label { target_index: _ } => {}
                     }
                 }
             }

compiler/rustc_mir_transform/src/coverage/graph.rs

@@ -349,12 +349,20 @@ fn bcb_filtered_successors<'a, 'tcx>(terminator: &'a Terminator<'tcx>) -> Covera
         | FalseUnwind { real_target: target, .. }
         | Goto { target } => CoverageSuccessors::Chainable(target),
 
-        // These terminators can normally be chained, except when they have no
+        // A call terminator can normally be chained, except when they have no
         // successor because they are known to diverge.
-        Call { target: maybe_target, .. } | InlineAsm { destination: maybe_target, .. } => {
-            match maybe_target {
-                Some(target) => CoverageSuccessors::Chainable(target),
-                None => CoverageSuccessors::NotChainable(&[]),
+        Call { target: maybe_target, .. } => match maybe_target {
+            Some(target) => CoverageSuccessors::Chainable(target),
+            None => CoverageSuccessors::NotChainable(&[]),
+        },
+
+        // An inline asm terminator can normally be chained, except when it diverges or uses asm
+        // goto.
+        InlineAsm { ref targets, .. } => {
+            if targets.len() == 1 {
+                CoverageSuccessors::Chainable(targets[0])
+            } else {
+                CoverageSuccessors::NotChainable(targets)
             }
         }
 

compiler/rustc_mir_transform/src/coverage/tests.rs

@@ -88,7 +88,6 @@ impl<'tcx> MockBlocks<'tcx> {
             | TerminatorKind::FalseEdge { real_target: ref mut target, .. }
             | TerminatorKind::FalseUnwind { real_target: ref mut target, .. }
             | TerminatorKind::Goto { ref mut target }
-            | TerminatorKind::InlineAsm { destination: Some(ref mut target), .. }
             | TerminatorKind::Yield { resume: ref mut target, .. } => *target = to_block,
             ref invalid => bug!("Invalid from_block: {:?}", invalid),
         }
@@ -185,10 +184,12 @@ fn debug_basic_blocks(mir_body: &Body<'_>) -> String {
                     | TerminatorKind::FalseEdge { real_target: target, .. }
                     | TerminatorKind::FalseUnwind { real_target: target, .. }
                     | TerminatorKind::Goto { target }
-                    | TerminatorKind::InlineAsm { destination: Some(target), .. }
                     | TerminatorKind::Yield { resume: target, .. } => {
                         format!("{}{:?}:{} -> {:?}", sp, bb, kind.name(), target)
                     }
+                    TerminatorKind::InlineAsm { targets, .. } => {
+                        format!("{}{:?}:{} -> {:?}", sp, bb, kind.name(), targets)
+                    }
                     TerminatorKind::SwitchInt { targets, .. } => {
                         format!("{}{:?}:{} -> {:?}", sp, bb, kind.name(), targets)
                     }

compiler/rustc_mir_transform/src/dest_prop.rs

@@ -648,7 +648,8 @@ impl WriteInfo {
                         }
                         InlineAsmOperand::Const { .. }
                         | InlineAsmOperand::SymFn { .. }
-                        | InlineAsmOperand::SymStatic { .. } => (),
+                        | InlineAsmOperand::SymStatic { .. }
+                        | InlineAsmOperand::Label { .. } => {}
                     }
                 }
             }

compiler/rustc_mir_transform/src/inline.rs

@@ -1036,8 +1036,8 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> {
             {
                 bug!("False unwinds should have been removed before inlining")
             }
-            TerminatorKind::InlineAsm { ref mut destination, ref mut unwind, .. } => {
-                if let Some(ref mut tgt) = *destination {
+            TerminatorKind::InlineAsm { ref mut targets, ref mut unwind, .. } => {
+                for tgt in targets.iter_mut() {
                     *tgt = self.map_block(*tgt);
                 }
                 *unwind = self.map_unwind(*unwind);

compiler/rustc_monomorphize/src/collector.rs

@@ -446,7 +446,8 @@ fn collect_items_rec<'tcx>(
                         hir::InlineAsmOperand::In { .. }
                         | hir::InlineAsmOperand::Out { .. }
                         | hir::InlineAsmOperand::InOut { .. }
-                        | hir::InlineAsmOperand::SplitInOut { .. } => {
+                        | hir::InlineAsmOperand::SplitInOut { .. }
+                        | hir::InlineAsmOperand::Label { .. } => {
                             span_bug!(*op_sp, "invalid operand type for global_asm!")
                         }
                     }

compiler/rustc_parse/src/parser/stmt.rs

@@ -448,7 +448,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses a block. No inner attributes are allowed.
-    pub(super) fn parse_block(&mut self) -> PResult<'a, P<Block>> {
+    pub fn parse_block(&mut self) -> PResult<'a, P<Block>> {
         let (attrs, block) = self.parse_inner_attrs_and_block()?;
         if let [.., last] = &*attrs {
             self.error_on_forbidden_inner_attr(

compiler/rustc_passes/src/liveness.rs

@@ -86,7 +86,6 @@ use crate::errors;
 use self::LiveNodeKind::*;
 use self::VarKind::*;
 
-use rustc_ast::InlineAsmOptions;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir as hir;
 use rustc_hir::def::*;
@@ -416,6 +415,12 @@ impl<'tcx> Visitor<'tcx> for IrMaps<'tcx> {
                 self.add_live_node_for_node(expr.hir_id, ExprNode(expr.span, expr.hir_id));
             }
 
+            // Inline assembly may contain labels.
+            hir::ExprKind::InlineAsm(asm) if asm.contains_label() => {
+                self.add_live_node_for_node(expr.hir_id, ExprNode(expr.span, expr.hir_id));
+                intravisit::walk_expr(self, expr);
+            }
+
             // otherwise, live nodes are not required:
             hir::ExprKind::Index(..)
             | hir::ExprKind::Field(..)
@@ -1045,20 +1050,53 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
             | hir::ExprKind::Repeat(ref e, _) => self.propagate_through_expr(e, succ),
 
             hir::ExprKind::InlineAsm(asm) => {
-                // Handle non-returning asm
-                let mut succ = if asm.options.contains(InlineAsmOptions::NORETURN) {
-                    self.exit_ln
-                } else {
-                    succ
-                };
+                //
+                //     (inputs)
+                //        |
+                //        v
+                //     (outputs)
+                //    /         \
+                //    |         |
+                //    v         v
+                // (labels)(fallthrough)
+                //    |         |
+                //    v         v
+                // ( succ / exit_ln )
 
-                // Do a first pass for writing outputs only
+                // Handle non-returning asm
+                let mut succ =
+                    if self.typeck_results.expr_ty(expr).is_never() { self.exit_ln } else { succ };
+
+                // Do a first pass for labels only
+                if asm.contains_label() {
+                    let ln = self.live_node(expr.hir_id, expr.span);
+                    self.init_from_succ(ln, succ);
+                    for (op, _op_sp) in asm.operands.iter().rev() {
+                        match op {
+                            hir::InlineAsmOperand::Label { block } => {
+                                let label_ln = self.propagate_through_block(block, succ);
+                                self.merge_from_succ(ln, label_ln);
+                            }
+                            hir::InlineAsmOperand::In { .. }
+                            | hir::InlineAsmOperand::Out { .. }
+                            | hir::InlineAsmOperand::InOut { .. }
+                            | hir::InlineAsmOperand::SplitInOut { .. }
+                            | hir::InlineAsmOperand::Const { .. }
+                            | hir::InlineAsmOperand::SymFn { .. }
+                            | hir::InlineAsmOperand::SymStatic { .. } => {}
+                        }
+                    }
+                    succ = ln;
+                }
+
+                // Do a second pass for writing outputs only
                 for (op, _op_sp) in asm.operands.iter().rev() {
                     match op {
                         hir::InlineAsmOperand::In { .. }
                         | hir::InlineAsmOperand::Const { .. }
                         | hir::InlineAsmOperand::SymFn { .. }
-                        | hir::InlineAsmOperand::SymStatic { .. } => {}
+                        | hir::InlineAsmOperand::SymStatic { .. }
+                        | hir::InlineAsmOperand::Label { .. } => {}
                         hir::InlineAsmOperand::Out { expr, .. } => {
                             if let Some(expr) = expr {
                                 succ = self.write_place(expr, succ, ACC_WRITE);
@@ -1075,7 +1113,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
                     }
                 }
 
-                // Then do a second pass for inputs
+                // Then do a third pass for inputs
                 for (op, _op_sp) in asm.operands.iter().rev() {
                     match op {
                         hir::InlineAsmOperand::In { expr, .. } => {
@@ -1097,7 +1135,8 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
                         }
                         hir::InlineAsmOperand::Const { .. }
                         | hir::InlineAsmOperand::SymFn { .. }
-                        | hir::InlineAsmOperand::SymStatic { .. } => {}
+                        | hir::InlineAsmOperand::SymStatic { .. }
+                        | hir::InlineAsmOperand::Label { .. } => {}
                     }
                 }
                 succ

compiler/rustc_passes/src/naked_functions.rs

@@ -237,7 +237,8 @@ impl<'tcx> CheckInlineAssembly<'tcx> {
                 InlineAsmOperand::In { .. }
                 | InlineAsmOperand::Out { .. }
                 | InlineAsmOperand::InOut { .. }
-                | InlineAsmOperand::SplitInOut { .. } => Some(op_sp),
+                | InlineAsmOperand::SplitInOut { .. }
+                | InlineAsmOperand::Label { .. } => Some(op_sp),
             })
             .collect();
         if !unsupported_operands.is_empty() {

compiler/rustc_resolve/src/late.rs

@@ -1242,6 +1242,7 @@ impl<'a: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast,
                     self.resolve_anon_const(anon_const, AnonConstKind::InlineConst);
                 }
                 InlineAsmOperand::Sym { sym } => self.visit_inline_asm_sym(sym),
+                InlineAsmOperand::Label { block } => self.visit_block(block),
             }
         }
     }

compiler/rustc_smir/src/rustc_smir/convert/mir.rs

@@ -559,7 +559,8 @@ impl<'tcx> Stable<'tcx> for mir::InlineAsmOperand<'tcx> {
             }
             InlineAsmOperand::Const { .. }
             | InlineAsmOperand::SymFn { .. }
-            | InlineAsmOperand::SymStatic { .. } => (None, None),
+            | InlineAsmOperand::SymStatic { .. }
+            | InlineAsmOperand::Label { .. } => (None, None),
         };
 
         stable_mir::mir::InlineAsmOperand { in_value, out_place, raw_rpr: format!("{self:?}") }
@@ -632,14 +633,15 @@ impl<'tcx> Stable<'tcx> for mir::TerminatorKind<'tcx> {
                 operands,
                 options,
                 line_spans,
-                destination,
+                targets,
                 unwind,
             } => TerminatorKind::InlineAsm {
                 template: format!("{template:?}"),
                 operands: operands.iter().map(|operand| operand.stable(tables)).collect(),
                 options: format!("{options:?}"),
                 line_spans: format!("{line_spans:?}"),
-                destination: destination.map(|d| d.as_usize()),
+                // FIXME: Figure out how to do labels in SMIR
+                destination: targets.first().map(|d| d.as_usize()),
                 unwind: unwind.stable(tables),
             },
             mir::TerminatorKind::Yield { .. }

compiler/rustc_span/src/symbol.rs

@@ -399,6 +399,7 @@ symbols! {
         asm,
         asm_const,
         asm_experimental_arch,
+        asm_goto,
         asm_sym,
         asm_unwind,
         assert,

src/doc/unstable-book/src/language-features/asm-goto.md

@@ -0,0 +1,30 @@
+# `asm_goto`
+
+The tracking issue for this feature is: [#119364]
+
+[#119364]: https://github.com/rust-lang/rust/issues/119364
+
+------------------------
+
+This feature adds a `label <block>` operand type to `asm!`.
+
+Example:
+```rust,ignore (partial-example, x86-only)
+
+unsafe {
+    asm!(
+        "jmp {}",
+        label {
+            println!("Jumped from asm!");
+        }
+    );
+}
+```
+
+The block must have unit type or diverge.
+
+When `label <block>` is used together with `noreturn` option, it means that the
+assembly will not fallthrough. It's allowed to jump to a label within the
+assembly. In this case, the entire `asm!` expression will have an unit type as
+opposed to diverging, if not all label blocks diverge. The `asm!` expression
+still diverges if `noreturn` option is used and all label blocks diverge.

src/tools/clippy/clippy_lints/src/loops/never_loop.rs

@@ -255,6 +255,9 @@ fn never_loop_expr<'tcx>(
             InlineAsmOperand::Const { .. } | InlineAsmOperand::SymFn { .. } | InlineAsmOperand::SymStatic { .. } => {
                 NeverLoopResult::Normal
             },
+            InlineAsmOperand::Label { block } => {
+                never_loop_block(cx, block, local_labels, main_loop_id)
+            }
         })),
         ExprKind::OffsetOf(_, _)
         | ExprKind::Yield(_, _)

src/tools/clippy/clippy_utils/src/hir_utils.rs

@@ -833,6 +833,7 @@ impl<'a, 'tcx> SpanlessHash<'a, 'tcx> {
                             self.hash_body(anon_const.body);
                         },
                         InlineAsmOperand::SymStatic { path, def_id: _ } => self.hash_qpath(path),
+                        InlineAsmOperand::Label { block } => self.hash_block(block),
                     }
                 }
             },

tests/codegen/asm-goto.rs

@@ -0,0 +1,51 @@
+//@ compile-flags: -O
+//@ only-x86_64
+
+#![crate_type = "rlib"]
+#![feature(asm_goto)]
+
+use std::arch::asm;
+
+#[no_mangle]
+pub extern "C" fn panicky() {}
+
+struct Foo;
+
+impl Drop for Foo {
+    fn drop(&mut self) {
+        println!();
+    }
+}
+
+// CHECK-LABEL: @asm_goto
+#[no_mangle]
+pub unsafe fn asm_goto() {
+    // CHECK: callbr void asm sideeffect alignstack inteldialect "
+    // CHECK-NEXT: to label %[[FALLTHROUGHBB:[a-b0-9]+]] [label %[[JUMPBB:[a-b0-9]+]]]
+    asm!("jmp {}", label {});
+}
+
+// CHECK-LABEL: @asm_goto_with_outputs
+#[no_mangle]
+pub unsafe fn asm_goto_with_outputs() -> u64 {
+    let out: u64;
+    // CHECK: [[RES:%[0-9]+]] = callbr i64 asm sideeffect alignstack inteldialect "
+    // CHECK-NEXT: to label %[[FALLTHROUGHBB:[a-b0-9]+]] [label %[[JUMPBB:[a-b0-9]+]]]
+    asm!("{} /* {} */", out(reg) out, label { return 1; });
+    // CHECK: [[JUMPBB]]:
+    // CHECK-NEXT: [[RET:%.+]] = phi i64 [ [[RES]], %[[FALLTHROUGHBB]] ], [ 1, %start ]
+    // CHECK-NEXT: ret i64 [[RET]]
+    out
+}
+
+// CHECK-LABEL: @asm_goto_noreturn
+#[no_mangle]
+pub unsafe fn asm_goto_noreturn() -> u64 {
+    let out: u64;
+    // CHECK: callbr void asm sideeffect alignstack inteldialect "
+    // CHECK-NEXT: to label %unreachable [label %[[JUMPBB:[a-b0-9]+]]]
+    asm!("jmp {}", label { return 1; }, options(noreturn));
+    // CHECK: [[JUMPBB]]:
+    // CHECK-NEXT: ret i64 1
+    out
+}

tests/ui/asm/aarch64/parse-error.stderr

@@ -130,17 +130,17 @@ LL |         asm!("{1}", in("x0") foo, const bar);
    |                     |
    |                     explicit register argument
 
-error: expected one of `clobber_abi`, `const`, `in`, `inlateout`, `inout`, `lateout`, `options`, `out`, or `sym`, found `""`
+error: expected one of `clobber_abi`, `const`, `in`, `inlateout`, `inout`, `label`, `lateout`, `options`, `out`, or `sym`, found `""`
   --> $DIR/parse-error.rs:66:29
    |
 LL |         asm!("", options(), "");
-   |                             ^^ expected one of 9 possible tokens
+   |                             ^^ expected one of 10 possible tokens
 
-error: expected one of `clobber_abi`, `const`, `in`, `inlateout`, `inout`, `lateout`, `options`, `out`, or `sym`, found `"{}"`
+error: expected one of `clobber_abi`, `const`, `in`, `inlateout`, `inout`, `label`, `lateout`, `options`, `out`, or `sym`, found `"{}"`
   --> $DIR/parse-error.rs:68:33
    |
 LL |         asm!("{}", in(reg) foo, "{}", out(reg) foo);
-   |                                 ^^^^ expected one of 9 possible tokens
+   |                                 ^^^^ expected one of 10 possible tokens
 
 error: asm template must be a string literal
   --> $DIR/parse-error.rs:70:14

tests/ui/asm/parse-error.rs

@@ -142,3 +142,5 @@ global_asm!(format!("{{{}}}", 0), const FOO);
 //~^ ERROR asm template must be a string literal
 global_asm!("{1}", format!("{{{}}}", 0), const FOO, const BAR);
 //~^ ERROR asm template must be a string literal
+global_asm!("{}", label {});
+//~^ ERROR expected operand, options, or additional template string

tests/ui/asm/parse-error.stderr

@@ -176,17 +176,17 @@ LL |         asm!("{a}", a = const foo, a = const bar);
    |
    = help: if this argument is intentionally unused, consider using it in an asm comment: `"/* {1} */"`
 
-error: expected one of `clobber_abi`, `const`, `in`, `inlateout`, `inout`, `lateout`, `options`, `out`, or `sym`, found `""`
+error: expected one of `clobber_abi`, `const`, `in`, `inlateout`, `inout`, `label`, `lateout`, `options`, `out`, or `sym`, found `""`
   --> $DIR/parse-error.rs:82:29
    |
 LL |         asm!("", options(), "");
-   |                             ^^ expected one of 9 possible tokens
+   |                             ^^ expected one of 10 possible tokens
 
-error: expected one of `clobber_abi`, `const`, `in`, `inlateout`, `inout`, `lateout`, `options`, `out`, or `sym`, found `"{}"`
+error: expected one of `clobber_abi`, `const`, `in`, `inlateout`, `inout`, `label`, `lateout`, `options`, `out`, or `sym`, found `"{}"`
   --> $DIR/parse-error.rs:84:33
    |
 LL |         asm!("{}", in(reg) foo, "{}", out(reg) foo);
-   |                                 ^^^^ expected one of 9 possible tokens
+   |                                 ^^^^ expected one of 10 possible tokens
 
 error: asm template must be a string literal
   --> $DIR/parse-error.rs:86:14
@@ -362,6 +362,12 @@ LL | global_asm!("{1}", format!("{{{}}}", 0), const FOO, const BAR);
    |
    = note: this error originates in the macro `format` (in Nightly builds, run with -Z macro-backtrace for more info)
 
+error: expected operand, options, or additional template string
+  --> $DIR/parse-error.rs:145:19
+   |
+LL | global_asm!("{}", label {});
+   |                   ^^^^^^^^ expected operand, options, or additional template string
+
 error[E0435]: attempt to use a non-constant value in a constant
   --> $DIR/parse-error.rs:39:37
    |
@@ -407,6 +413,6 @@ LL |     let mut bar = 0;
 LL |         asm!("{a}", a = const foo, a = const bar);
    |                                              ^^^ non-constant value
 
-error: aborting due to 63 previous errors
+error: aborting due to 64 previous errors
 
 For more information about this error, try `rustc --explain E0435`.

tests/ui/asm/x86_64/bad-options.rs

@@ -1,5 +1,7 @@
 //@ only-x86_64
 
+#![feature(asm_unwind, asm_goto)]
+
 use std::arch::{asm, global_asm};
 
 fn main() {
@@ -14,6 +16,8 @@ fn main() {
         //~^ ERROR asm with the `pure` option must have at least one output
         asm!("{}", out(reg) foo, options(noreturn));
         //~^ ERROR asm outputs are not allowed with the `noreturn` option
+        asm!("{}", label {}, options(may_unwind));
+        //~^ ERROR asm labels are not allowed with the `may_unwind` option
     }
 
     unsafe {

tests/ui/asm/x86_64/bad-options.stderr

@@ -1,35 +1,41 @@
 error: the `nomem` and `readonly` options are mutually exclusive
-  --> $DIR/bad-options.rs:8:18
+  --> $DIR/bad-options.rs:10:18
    |
 LL |         asm!("", options(nomem, readonly));
    |                  ^^^^^^^^^^^^^^^^^^^^^^^^
 
 error: the `pure` and `noreturn` options are mutually exclusive
-  --> $DIR/bad-options.rs:10:18
+  --> $DIR/bad-options.rs:12:18
    |
 LL |         asm!("", options(pure, nomem, noreturn));
    |                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error: asm with the `pure` option must have at least one output
-  --> $DIR/bad-options.rs:10:18
+  --> $DIR/bad-options.rs:12:18
    |
 LL |         asm!("", options(pure, nomem, noreturn));
    |                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error: asm with the `pure` option must have at least one output
-  --> $DIR/bad-options.rs:13:33
+  --> $DIR/bad-options.rs:15:33
    |
 LL |         asm!("{}", in(reg) foo, options(pure, nomem));
    |                                 ^^^^^^^^^^^^^^^^^^^^
 
 error: asm outputs are not allowed with the `noreturn` option
-  --> $DIR/bad-options.rs:15:20
+  --> $DIR/bad-options.rs:17:20
    |
 LL |         asm!("{}", out(reg) foo, options(noreturn));
    |                    ^^^^^^^^^^^^
 
+error: asm labels are not allowed with the `may_unwind` option
+  --> $DIR/bad-options.rs:19:20
+   |
+LL |         asm!("{}", label {}, options(may_unwind));
+   |                    ^^^^^^^^
+
 error: asm with `clobber_abi` must specify explicit registers for outputs
-  --> $DIR/bad-options.rs:22:20
+  --> $DIR/bad-options.rs:26:20
    |
 LL |         asm!("{}", out(reg) foo, clobber_abi("C"));
    |                    ^^^^^^^^^^^^  ---------------- clobber_abi
@@ -37,7 +43,7 @@ LL |         asm!("{}", out(reg) foo, clobber_abi("C"));
    |                    generic outputs
 
 error: asm with `clobber_abi` must specify explicit registers for outputs
-  --> $DIR/bad-options.rs:24:20
+  --> $DIR/bad-options.rs:28:20
    |
 LL |         asm!("{}", out(reg) foo, clobber_abi("C"), clobber_abi("C"));
    |                    ^^^^^^^^^^^^  ----------------  ---------------- clobber_abi
@@ -46,43 +52,43 @@ LL |         asm!("{}", out(reg) foo, clobber_abi("C"), clobber_abi("C"));
    |                    generic outputs
 
 error: expected one of `)`, `att_syntax`, or `raw`, found `nomem`
-  --> $DIR/bad-options.rs:31:25
+  --> $DIR/bad-options.rs:35:25
    |
 LL | global_asm!("", options(nomem));
    |                         ^^^^^ expected one of `)`, `att_syntax`, or `raw`
 
 error: expected one of `)`, `att_syntax`, or `raw`, found `readonly`
-  --> $DIR/bad-options.rs:33:25
+  --> $DIR/bad-options.rs:37:25
    |
 LL | global_asm!("", options(readonly));
    |                         ^^^^^^^^ expected one of `)`, `att_syntax`, or `raw`
 
 error: expected one of `)`, `att_syntax`, or `raw`, found `noreturn`
-  --> $DIR/bad-options.rs:35:25
+  --> $DIR/bad-options.rs:39:25
    |
 LL | global_asm!("", options(noreturn));
    |                         ^^^^^^^^ expected one of `)`, `att_syntax`, or `raw`
 
 error: expected one of `)`, `att_syntax`, or `raw`, found `pure`
-  --> $DIR/bad-options.rs:37:25
+  --> $DIR/bad-options.rs:41:25
    |
 LL | global_asm!("", options(pure));
    |                         ^^^^ expected one of `)`, `att_syntax`, or `raw`
 
 error: expected one of `)`, `att_syntax`, or `raw`, found `nostack`
-  --> $DIR/bad-options.rs:39:25
+  --> $DIR/bad-options.rs:43:25
    |
 LL | global_asm!("", options(nostack));
    |                         ^^^^^^^ expected one of `)`, `att_syntax`, or `raw`
 
 error: expected one of `)`, `att_syntax`, or `raw`, found `preserves_flags`
-  --> $DIR/bad-options.rs:41:25
+  --> $DIR/bad-options.rs:45:25
    |
 LL | global_asm!("", options(preserves_flags));
    |                         ^^^^^^^^^^^^^^^ expected one of `)`, `att_syntax`, or `raw`
 
 error: invalid ABI for `clobber_abi`
-  --> $DIR/bad-options.rs:20:18
+  --> $DIR/bad-options.rs:24:18
    |
 LL |         asm!("", clobber_abi("foo"));
    |                  ^^^^^^^^^^^^^^^^^^
@@ -90,12 +96,12 @@ LL |         asm!("", clobber_abi("foo"));
    = note: the following ABIs are supported on this target: `C`, `system`, `efiapi`, `win64`, `sysv64`
 
 error: `C` ABI specified multiple times
-  --> $DIR/bad-options.rs:24:52
+  --> $DIR/bad-options.rs:28:52
    |
 LL |         asm!("{}", out(reg) foo, clobber_abi("C"), clobber_abi("C"));
    |                                  ----------------  ^^^^^^^^^^^^^^^^
    |                                  |
    |                                  previously specified here
 
-error: aborting due to 15 previous errors
+error: aborting due to 16 previous errors
 

tests/ui/asm/x86_64/goto.mirunsafeck.stderr

@@ -0,0 +1,23 @@
+warning: unreachable statement
+  --> $DIR/goto.rs:99:9
+   |
+LL | /         asm!(
+LL | |             "jmp {}",
+LL | |             label {
+LL | |                 return;
+LL | |             },
+LL | |             options(noreturn)
+LL | |         );
+   | |_________- any code following this expression is unreachable
+LL |           unreachable!();
+   |           ^^^^^^^^^^^^^^ unreachable statement
+   |
+note: the lint level is defined here
+  --> $DIR/goto.rs:89:8
+   |
+LL | #[warn(unreachable_code)]
+   |        ^^^^^^^^^^^^^^^^
+   = note: this warning originates in the macro `unreachable` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+warning: 1 warning emitted
+

tests/ui/asm/x86_64/goto.rs

@@ -0,0 +1,111 @@
+//@ only-x86_64
+//@ run-pass
+//@ needs-asm-support
+//@ revisions: mirunsafeck thirunsafeck
+//@ [thirunsafeck]compile-flags: -Z thir-unsafeck
+
+#![deny(unreachable_code)]
+#![feature(asm_goto)]
+
+use std::arch::asm;
+
+fn goto_fallthough() {
+    unsafe {
+        asm!(
+            "/* {} */",
+            label {
+                unreachable!();
+            }
+        )
+    }
+}
+
+fn goto_jump() {
+    unsafe {
+        let mut value = false;
+        asm!(
+            "jmp {}",
+            label {
+                value = true;
+            }
+        );
+        assert!(value);
+    }
+}
+
+// asm goto with outputs cause miscompilation in LLVM. UB can be triggered
+// when outputs are used inside the label block when optimisation is enabled.
+// See: https://github.com/llvm/llvm-project/issues/74483
+/*
+fn goto_out_fallthrough() {
+    unsafe {
+        let mut out: usize;
+        asm!(
+            "lea {}, [{} + 1]",
+            "/* {} */",
+            out(reg) out,
+            in(reg) 0x12345678usize,
+            label {
+                unreachable!();
+            }
+        );
+        assert_eq!(out, 0x12345679);
+    }
+}
+
+fn goto_out_jump() {
+    unsafe {
+        let mut value = false;
+        let mut out: usize;
+        asm!(
+            "lea {}, [{} + 1]",
+            "jmp {}",
+            out(reg) out,
+            in(reg) 0x12345678usize,
+            label {
+                value = true;
+                assert_eq!(out, 0x12345679);
+            }
+        );
+        assert!(value);
+    }
+}
+*/
+
+fn goto_noreturn() {
+    unsafe {
+        let a;
+        asm!(
+            "jmp {}",
+            label {
+                a = 1;
+            },
+            options(noreturn)
+        );
+        assert_eq!(a, 1);
+    }
+}
+
+#[warn(unreachable_code)]
+fn goto_noreturn_diverge() {
+    unsafe {
+        asm!(
+            "jmp {}",
+            label {
+                return;
+            },
+            options(noreturn)
+        );
+        unreachable!();
+        //~^ WARN unreachable statement
+    }
+}
+
+fn main() {
+    goto_fallthough();
+    goto_jump();
+    // goto_out_fallthrough();
+    // goto_out_jump();
+    goto_noreturn();
+    goto_noreturn_diverge();
+}

tests/ui/asm/x86_64/goto.thirunsafeck.stderr

@@ -0,0 +1,23 @@
+warning: unreachable statement
+  --> $DIR/goto.rs:99:9
+   |
+LL | /         asm!(
+LL | |             "jmp {}",
+LL | |             label {
+LL | |                 return;
+LL | |             },
+LL | |             options(noreturn)
+LL | |         );
+   | |_________- any code following this expression is unreachable
+LL |           unreachable!();
+   |           ^^^^^^^^^^^^^^ unreachable statement
+   |
+note: the lint level is defined here
+  --> $DIR/goto.rs:89:8
+   |
+LL | #[warn(unreachable_code)]
+   |        ^^^^^^^^^^^^^^^^
+   = note: this warning originates in the macro `unreachable` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+warning: 1 warning emitted
+

tests/ui/feature-gates/feature-gate-asm_goto.rs

@@ -0,0 +1,10 @@
+//@ only-x86_64
+
+use std::arch::asm;
+
+fn main() {
+    unsafe {
+        asm!("jmp {}", label {});
+        //~^ ERROR label operands for inline assembly are unstable
+    }
+}

tests/ui/feature-gates/feature-gate-asm_goto.stderr

@@ -0,0 +1,13 @@
+error[E0658]: label operands for inline assembly are unstable
+  --> $DIR/feature-gate-asm_goto.rs:7:24
+   |
+LL |         asm!("jmp {}", label {});
+   |                        ^^^^^^^^
+   |
+   = note: see issue #119364 <https://github.com/rust-lang/rust/issues/119364> for more information
+   = help: add `#![feature(asm_goto)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0658`.