Merge from rustc

Cargo.lock

@@ -4525,6 +4525,7 @@ dependencies = [
  "rustc_middle",
  "rustc_span",
  "rustc_target",
+ "scoped-tls",
  "stable_mir",
  "tracing",
 ]

compiler/rustc_attr/messages.ftl

@@ -58,6 +58,9 @@ attr_invalid_repr_hint_no_paren =
 attr_invalid_repr_hint_no_value =
     invalid representation hint: `{$name}` does not take a value
 
+attr_invalid_since =
+    'since' must be a Rust version number, such as "1.31.0"
+
 attr_missing_feature =
     missing 'feature'
 

compiler/rustc_attr/src/builtin.rs

@@ -3,6 +3,7 @@
 use rustc_ast::{self as ast, attr};
 use rustc_ast::{Attribute, LitKind, MetaItem, MetaItemKind, MetaItemLit, NestedMetaItem, NodeId};
 use rustc_ast_pretty::pprust;
+use rustc_errors::ErrorGuaranteed;
 use rustc_feature::{find_gated_cfg, is_builtin_attr_name, Features, GatedCfg};
 use rustc_macros::HashStable_Generic;
 use rustc_session::config::ExpectedValues;
@@ -361,25 +362,32 @@ fn parse_stability(sess: &Session, attr: &Attribute) -> Option<(Symbol, Stabilit
         }
     }
 
-    if let Some(s) = since
-        && s.as_str() == VERSION_PLACEHOLDER
-    {
-        since = Some(rust_version_symbol());
-    }
+    let feature = match feature {
+        Some(feature) if rustc_lexer::is_ident(feature.as_str()) => Ok(feature),
+        Some(_bad_feature) => {
+            Err(sess.emit_err(session_diagnostics::NonIdentFeature { span: attr.span }))
+        }
+        None => Err(sess.emit_err(session_diagnostics::MissingFeature { span: attr.span })),
+    };
+
+    let since = if let Some(since) = since {
+        if since.as_str() == VERSION_PLACEHOLDER {
+            Ok(rust_version_symbol())
+        } else if parse_version(since.as_str(), false).is_some() {
+            Ok(since)
+        } else {
+            Err(sess.emit_err(session_diagnostics::InvalidSince { span: attr.span }))
+        }
+    } else {
+        Err(sess.emit_err(session_diagnostics::MissingSince { span: attr.span }))
+    };
 
     match (feature, since) {
-        (Some(feature), Some(since)) => {
+        (Ok(feature), Ok(since)) => {
             let level = StabilityLevel::Stable { since, allowed_through_unstable_modules: false };
             Some((feature, level))
         }
-        (None, _) => {
-            sess.emit_err(session_diagnostics::MissingFeature { span: attr.span });
-            None
-        }
-        _ => {
-            sess.emit_err(session_diagnostics::MissingSince { span: attr.span });
-            None
-        }
+        (Err(ErrorGuaranteed { .. }), _) | (_, Err(ErrorGuaranteed { .. })) => None,
     }
 }
 
@@ -451,12 +459,19 @@ fn parse_unstability(sess: &Session, attr: &Attribute) -> Option<(Symbol, Stabil
         }
     }
 
-    match (feature, reason, issue) {
-        (Some(feature), reason, Some(_)) => {
-            if !rustc_lexer::is_ident(feature.as_str()) {
-                sess.emit_err(session_diagnostics::NonIdentFeature { span: attr.span });
-                return None;
-            }
+    let feature = match feature {
+        Some(feature) if rustc_lexer::is_ident(feature.as_str()) => Ok(feature),
+        Some(_bad_feature) => {
+            Err(sess.emit_err(session_diagnostics::NonIdentFeature { span: attr.span }))
+        }
+        None => Err(sess.emit_err(session_diagnostics::MissingFeature { span: attr.span })),
+    };
+
+    let issue =
+        issue.ok_or_else(|| sess.emit_err(session_diagnostics::MissingIssue { span: attr.span }));
+
+    match (feature, issue) {
+        (Ok(feature), Ok(_)) => {
             let level = StabilityLevel::Unstable {
                 reason: UnstableReason::from_opt_reason(reason),
                 issue: issue_num,
@@ -465,14 +480,7 @@ fn parse_unstability(sess: &Session, attr: &Attribute) -> Option<(Symbol, Stabil
             };
             Some((feature, level))
         }
-        (None, _, _) => {
-            sess.emit_err(session_diagnostics::MissingFeature { span: attr.span });
-            return None;
-        }
-        _ => {
-            sess.emit_err(session_diagnostics::MissingIssue { span: attr.span });
-            return None;
-        }
+        (Err(ErrorGuaranteed { .. }), _) | (_, Err(ErrorGuaranteed { .. })) => None,
     }
 }
 

compiler/rustc_attr/src/session_diagnostics.rs

@@ -370,6 +370,13 @@ pub(crate) struct ExpectsFeatures {
     pub name: String,
 }
 
+#[derive(Diagnostic)]
+#[diag(attr_invalid_since)]
+pub(crate) struct InvalidSince {
+    #[primary_span]
+    pub span: Span,
+}
+
 #[derive(Diagnostic)]
 #[diag(attr_soft_no_args)]
 pub(crate) struct SoftNoArgs {

compiler/rustc_borrowck/src/renumber.rs

@@ -81,6 +81,10 @@ impl<'a, 'tcx> MutVisitor<'tcx> for RegionRenumberer<'a, 'tcx> {
 
     #[instrument(skip(self), level = "debug")]
     fn visit_ty(&mut self, ty: &mut Ty<'tcx>, ty_context: TyContext) {
+        if matches!(ty_context, TyContext::ReturnTy(_)) {
+            // We will renumber the return ty when called again with `TyContext::LocalDecl`
+            return;
+        }
         *ty = self.renumber_regions(*ty, || RegionCtxt::TyContext(ty_context));
 
         debug!(?ty);

compiler/rustc_codegen_cranelift/.vscode/settings.json

@@ -33,7 +33,7 @@
             ]
         },
         {
-            "sysroot_src": "./download/sysroot/sysroot_src/library",
+            "sysroot_src": "./build/stdlib/library",
             "crates": [
                 {
                     "root_module": "./example/std_example.rs",

compiler/rustc_codegen_cranelift/Cargo.lock

@@ -45,18 +45,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e5e1df0da8488dd03b34afc134ba84b754d61862cc465932a9e5d07952f661e"
+checksum = "c1512c3bb6b13018e7109fc3ac964bc87b329eaf3a77825d337558d0c7f6f1be"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77a17ca4e699a0aaf49a0c88f6311a864f321048aa63f6b787cab20eb5f93f10"
+checksum = "16cb8fb9220a6ea7a226705a273ab905309ee546267bdf34948d57932d7f0396"
 dependencies = [
  "bumpalo",
  "cranelift-bforest",
@@ -75,39 +75,39 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "022f2793cdade1d37a1f755ac42938a3f832f533eac6cafc8b26b209544c3c06"
+checksum = "ab3a8d3b0d4745b183da5ea0792b13d79f5c23d6e69ac04761728e2532b56649"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4d72dbb83c2ad788dec4ad0843070973cb48c35a3ca19b1e7437ac40834fd9c"
+checksum = "524141c8e68f2abc2043de4c2b31f6d9dd42432738c246431d0572a1422a4a84"
 
 [[package]]
 name = "cranelift-control"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae07cf26dcc90d546826d747ac63b6c40c916f34b03e92a6ae0422c28d771b8a"
+checksum = "97513b57c961c713789a03886a57b43e14ebcd204cbaa8ae50ca6c70a8e716b3"
 dependencies = [
  "arbitrary",
 ]
 
 [[package]]
 name = "cranelift-entity"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2fe6b7e49820893691aea497f36257e9d6f52061d8c4758d61d802d5f101a3d"
+checksum = "e3f23d3cf3afa7e45f239702612c76d87964f652a55e28d13ed6d7e20f3479dd"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44f497576ca3674581581601b6a55ccc1b43447217648c880e5bce70db3cf659"
+checksum = "554cd4947ec9209b58bf9ae5bf83581b5ddf9128bd967208e334b504a57db54e"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -117,15 +117,15 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b96aa02eac00fffee13b0cd37d17874ccdb3d5458983041accd825ef78ce6454"
+checksum = "6c1892a439696b6413cb54083806f5fd9fc431768b8de74864b3d9e8b93b124f"
 
 [[package]]
 name = "cranelift-jit"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1d6e0e308c873eefc185745a6b21daec2a10f7554c9fb67e334c2d7d756d979"
+checksum = "32209252fb38acaf1662ccd0397907bbe0e92bdb13b6ddbfd2f74e437f83e685"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -143,9 +143,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1aa8ebb06eced4e478c3f94f1d65d4e7c93493f4640057912b27a3e34b84841"
+checksum = "bf42656f5f6df7bfafc4dd7b63a1888b0627c07b43b2cb9aa54e13843fed39eb"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -154,9 +154,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2870170ca44054b202c737626607b87be6e35655084bd94a6ff807a5812ba7df"
+checksum = "e0c2d3badd4b9690865f5bb68a71fa94de592fa2df3f3d11a5a062c60c0a107a"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -165,9 +165,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20647761742d17dabac8205da958910ede78599550e06418a16711a3ee2fc897"
+checksum = "88eca54bbecea3170035168357306e9c779d4a63d8bf036c9e16bd21fdaa69b5"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -374,9 +374,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "14.0.0"
+version = "14.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3a5dda53ad6993f9b0a2d65fb49e0348a7232a27a8794064122870d6ee19eb2"
+checksum = "9aaf2fa8fd2d6b65abae9b92edfe69254cc5d6b166e342364036c3e347de8da9"
 dependencies = [
  "cfg-if",
  "libc",

compiler/rustc_codegen_cranelift/Cargo.toml

@@ -8,12 +8,12 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.101", features = ["unwind", "all-arch"] }
-cranelift-frontend = { version = "0.101" }
-cranelift-module = { version = "0.101" }
-cranelift-native = { version = "0.101" }
-cranelift-jit = { version = "0.101", optional = true }
-cranelift-object = { version = "0.101" }
+cranelift-codegen = { version = "0.101.1", features = ["unwind", "all-arch"] }
+cranelift-frontend = { version = "0.101.1" }
+cranelift-module = { version = "0.101.1" }
+cranelift-native = { version = "0.101.1" }
+cranelift-jit = { version = "0.101.1", optional = true }
+cranelift-object = { version = "0.101.1" }
 target-lexicon = "0.12.0"
 gimli = { version = "0.28", default-features = false, features = ["write"]}
 object = { version = "0.32", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }

compiler/rustc_codegen_cranelift/Readme.md

@@ -8,7 +8,7 @@ If not please open an issue.
 ## Building and testing
 
 ```bash
-$ git clone https://github.com/bjorn3/rustc_codegen_cranelift
+$ git clone https://github.com/rust-lang/rustc_codegen_cranelift
 $ cd rustc_codegen_cranelift
 $ ./y.sh prepare
 $ ./y.sh build
@@ -29,7 +29,7 @@ Extract the `dist` directory in the archive anywhere you want.
 If you want to use `cargo clif build` instead of having to specify the full path to the `cargo-clif` executable, you can add the `bin` subdirectory of the extracted `dist` directory to your `PATH`.
 (tutorial [for Windows](https://stackoverflow.com/a/44272417), and [for Linux/MacOS](https://unix.stackexchange.com/questions/26047/how-to-correctly-add-a-path-to-path/26059#26059)).
 
-[releases]: https://github.com/bjorn3/rustc_codegen_cranelift/releases/tag/dev
+[releases]: https://github.com/rust-lang/rustc_codegen_cranelift/releases/tag/dev
 
 ## Usage
 
@@ -78,7 +78,7 @@ configuration options.
 
 * Inline assembly ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1041))
     * On UNIX there is support for invoking an external assembler for `global_asm!` and `asm!`.
-* SIMD ([tracked here](https://github.com/bjorn3/rustc_codegen_cranelift/issues/171), `std::simd` fully works, `std::arch` is partially supported)
+* SIMD ([tracked here](https://github.com/rust-lang/rustc_codegen_cranelift/issues/171), `std::simd` fully works, `std::arch` is partially supported)
 * Unwinding on panics ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1677), `-Cpanic=abort` is enabled by default)
 
 ## License

compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs

@@ -353,6 +353,17 @@ fn main() {
 
     let f = V([0.0, 1.0]);
     let _a = f.0[0];
+
+    stack_val_align();
+}
+
+#[inline(never)]
+fn stack_val_align() {
+    #[repr(align(8192))]
+    struct Foo(u8);
+
+    let a = Foo(0);
+    assert_eq!(&a as *const Foo as usize % 8192, 0);
 }
 
 #[cfg(all(

compiler/rustc_codegen_cranelift/patches/0001-regex-Ignore-test-which-gets-miscompiled-with-llvm-sysroot.patch

@@ -1,25 +0,0 @@
-From 5d4afb8d807d181038b6a004d17ed055a8d191b2 Mon Sep 17 00:00:00 2001
-From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
-Date: Mon, 2 Oct 2023 13:59:00 +0000
-Subject: [PATCH] Ignore test which gets miscompiled with llvm sysroot
-
----
- regex-automata/src/util/pool.rs | 2 ++
- 1 file changed, 2 insertions(+)
-
-diff --git a/regex-automata/src/util/pool.rs b/regex-automata/src/util/pool.rs
-index c03d7b0..28b233b 100644
---- a/regex-automata/src/util/pool.rs
-+++ b/regex-automata/src/util/pool.rs
-@@ -1081,6 +1081,8 @@ mod tests {
-     // into the pool. This in turn resulted in this test producing a data race.
-     #[cfg(feature = "std")]
-     #[test]
-+    // FIXME(rustc_codegen_cranelift#1395) miscompilation of thread::scope with LLVM sysroot
-+    #[ignore]
-     fn thread_owner_sync() {
-         let pool = Pool::new(|| vec!['a']);
-         {
--- 
-2.34.1
-

compiler/rustc_codegen_cranelift/src/abi/mod.rs

@@ -120,32 +120,25 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         args: &[Value],
     ) -> Cow<'_, [Value]> {
         if self.tcx.sess.target.is_like_windows {
-            let (mut params, mut args): (Vec<_>, Vec<_>) =
-                params
-                    .into_iter()
-                    .zip(args)
-                    .map(|(param, &arg)| {
-                        if param.value_type == types::I128 {
-                            let arg_ptr = Pointer::stack_slot(self.bcx.create_sized_stack_slot(
-                                StackSlotData { kind: StackSlotKind::ExplicitSlot, size: 16 },
-                            ));
-                            arg_ptr.store(self, arg, MemFlags::trusted());
-                            (AbiParam::new(self.pointer_type), arg_ptr.get_addr(self))
-                        } else {
-                            (param, arg)
-                        }
-                    })
-                    .unzip();
+            let (mut params, mut args): (Vec<_>, Vec<_>) = params
+                .into_iter()
+                .zip(args)
+                .map(|(param, &arg)| {
+                    if param.value_type == types::I128 {
+                        let arg_ptr = self.create_stack_slot(16, 16);
+                        arg_ptr.store(self, arg, MemFlags::trusted());
+                        (AbiParam::new(self.pointer_type), arg_ptr.get_addr(self))
+                    } else {
+                        (param, arg)
+                    }
+                })
+                .unzip();
 
             let indirect_ret_val = returns.len() == 1 && returns[0].value_type == types::I128;
 
             if indirect_ret_val {
                 params.insert(0, AbiParam::new(self.pointer_type));
-                let ret_ptr =
-                    Pointer::stack_slot(self.bcx.create_sized_stack_slot(StackSlotData {
-                        kind: StackSlotKind::ExplicitSlot,
-                        size: 16,
-                    }));
+                let ret_ptr = self.create_stack_slot(16, 16);
                 args.insert(0, ret_ptr.get_addr(self));
                 self.lib_call_unadjusted(name, params, vec![], &args);
                 return Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())]);

compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs

@@ -189,16 +189,13 @@ pub(super) fn from_casted_value<'tcx>(
     let abi_params = cast_target_to_abi_params(cast);
     let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum();
     let layout_size = u32::try_from(layout.size.bytes()).unwrap();
-    let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
-        kind: StackSlotKind::ExplicitSlot,
-        // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
-        // specify stack slot alignment.
+    let ptr = fx.create_stack_slot(
         // Stack slot size may be bigger for example `[u8; 3]` which is packed into an `i32`.
         // It may also be smaller for example when the type is a wrapper around an integer with a
         // larger alignment than the integer.
-        size: (std::cmp::max(abi_param_size, layout_size) + 15) / 16 * 16,
-    });
-    let ptr = Pointer::stack_slot(stack_slot);
+        std::cmp::max(abi_param_size, layout_size),
+        u32::try_from(layout.align.pref.bytes()).unwrap(),
+    );
     let mut offset = 0;
     let mut block_params_iter = block_params.iter().copied();
     for param in abi_params {

compiler/rustc_codegen_cranelift/src/cast.rs

@@ -104,11 +104,7 @@ pub(crate) fn clif_int_or_float_cast(
                     &[from],
                 )[0];
                 // FIXME(bytecodealliance/wasmtime#6104) use bitcast instead of store to get from i64x2 to i128
-                let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
-                    kind: StackSlotKind::ExplicitSlot,
-                    size: 16,
-                });
-                let ret_ptr = Pointer::stack_slot(stack_slot);
+                let ret_ptr = fx.create_stack_slot(16, 16);
                 ret_ptr.store(fx, ret, MemFlags::trusted());
                 ret_ptr.load(fx, types::I128, MemFlags::trusted())
             } else {

compiler/rustc_codegen_cranelift/src/common.rs

@@ -383,6 +383,25 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         })
     }
 
+    pub(crate) fn create_stack_slot(&mut self, size: u32, align: u32) -> Pointer {
+        if align <= 16 {
+            let stack_slot = self.bcx.create_sized_stack_slot(StackSlotData {
+                kind: StackSlotKind::ExplicitSlot,
+                // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
+                // specify stack slot alignment.
+                size: (size + 15) / 16 * 16,
+            });
+            Pointer::stack_slot(stack_slot)
+        } else {
+            // Alignment is too big to handle using the above hack. Dynamically realign a stack slot
+            // instead. This wastes some space for the realignment.
+            let base_ptr = self.create_stack_slot(size + align, 16).get_addr(self);
+            let misalign_offset = self.bcx.ins().urem_imm(base_ptr, i64::from(align));
+            let realign_offset = self.bcx.ins().irsub_imm(misalign_offset, i64::from(align));
+            Pointer::new(self.bcx.ins().iadd(base_ptr, realign_offset))
+        }
+    }
+
     pub(crate) fn set_debug_loc(&mut self, source_info: mir::SourceInfo) {
         if let Some(debug_context) = &mut self.cx.debug_context {
             let (file, line, column) =

compiler/rustc_codegen_cranelift/src/driver/aot.rs

@@ -361,12 +361,26 @@ pub(crate) fn run_aot(
     metadata: EncodedMetadata,
     need_metadata_module: bool,
 ) -> Box<OngoingCodegen> {
+    // FIXME handle `-Ctarget-cpu=native`
+    let target_cpu = match tcx.sess.opts.cg.target_cpu {
+        Some(ref name) => name,
+        None => tcx.sess.target.cpu.as_ref(),
+    }
+    .to_owned();
+
     let cgus = if tcx.sess.opts.output_types.should_codegen() {
         tcx.collect_and_partition_mono_items(()).1
     } else {
         // If only `--emit metadata` is used, we shouldn't perform any codegen.
         // Also `tcx.collect_and_partition_mono_items` may panic in that case.
-        &[]
+        return Box::new(OngoingCodegen {
+            modules: vec![],
+            allocator_module: None,
+            metadata_module: None,
+            metadata,
+            crate_info: CrateInfo::new(tcx, target_cpu),
+            concurrency_limiter: ConcurrencyLimiter::new(tcx.sess, 0),
+        });
     };
 
     if tcx.dep_graph.is_fully_enabled() {
@@ -481,13 +495,6 @@ pub(crate) fn run_aot(
         None
     };
 
-    // FIXME handle `-Ctarget-cpu=native`
-    let target_cpu = match tcx.sess.opts.cg.target_cpu {
-        Some(ref name) => name,
-        None => tcx.sess.target.cpu.as_ref(),
-    }
-    .to_owned();
-
     Box::new(OngoingCodegen {
         modules,
         allocator_module,

compiler/rustc_codegen_cranelift/src/inline_asm.rs

@@ -878,13 +878,7 @@ fn call_inline_asm<'tcx>(
     inputs: Vec<(Size, Value)>,
     outputs: Vec<(Size, CPlace<'tcx>)>,
 ) {
-    let stack_slot = fx.bcx.func.create_sized_stack_slot(StackSlotData {
-        kind: StackSlotKind::ExplicitSlot,
-        size: u32::try_from(slot_size.bytes()).unwrap(),
-    });
-    if fx.clif_comments.enabled() {
-        fx.add_comment(stack_slot, "inline asm scratch slot");
-    }
+    let stack_slot = fx.create_stack_slot(u32::try_from(slot_size.bytes()).unwrap(), 16);
 
     let inline_asm_func = fx
         .module
@@ -904,15 +898,23 @@ fn call_inline_asm<'tcx>(
     }
 
     for (offset, value) in inputs {
-        fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());
+        stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).store(
+            fx,
+            value,
+            MemFlags::trusted(),
+        );
     }
 
-    let stack_slot_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0);
+    let stack_slot_addr = stack_slot.get_addr(fx);
     fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]);
 
     for (offset, place) in outputs {
         let ty = fx.clif_type(place.layout().ty).unwrap();
-        let value = fx.bcx.ins().stack_load(ty, stack_slot, i32::try_from(offset.bytes()).unwrap());
+        let value = stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).load(
+            fx,
+            ty,
+            MemFlags::trusted(),
+        );
         place.write_cvalue(fx, CValue::by_val(value, place.layout()));
     }
 }

compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs

@@ -310,6 +310,143 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             let val = CValue::by_val_pair(cb_out, c, layout);
             ret.write_cvalue(fx, val);
         }
+        "llvm.x86.sse2.pavg.b" | "llvm.x86.sse2.pavg.w" => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            // FIXME use vector instructions when possible
+            simd_pair_for_each_lane(
+                fx,
+                a,
+                b,
+                ret,
+                &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| {
+                    // (a + b + 1) >> 1
+                    let lane_ty = fx.bcx.func.dfg.value_type(a_lane);
+                    let a_lane = fx.bcx.ins().uextend(lane_ty.double_width().unwrap(), a_lane);
+                    let b_lane = fx.bcx.ins().uextend(lane_ty.double_width().unwrap(), b_lane);
+                    let sum = fx.bcx.ins().iadd(a_lane, b_lane);
+                    let num_plus_one = fx.bcx.ins().iadd_imm(sum, 1);
+                    let res = fx.bcx.ins().ushr_imm(num_plus_one, 1);
+                    fx.bcx.ins().ireduce(lane_ty, res)
+                },
+            );
+        }
+        "llvm.x86.sse2.psra.w" => {
+            intrinsic_args!(fx, args => (a, count); intrinsic);
+
+            let count_lane = count.force_stack(fx).0.load(fx, types::I64, MemFlags::trusted());
+            let lane_ty = fx.clif_type(a.layout().ty.simd_size_and_type(fx.tcx).1).unwrap();
+            let max_count = fx.bcx.ins().iconst(types::I64, i64::from(lane_ty.bits() - 1));
+            let saturated_count = fx.bcx.ins().umin(count_lane, max_count);
+
+            // FIXME use vector instructions when possible
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, a_lane| {
+                fx.bcx.ins().sshr(a_lane, saturated_count)
+            });
+        }
+        "llvm.x86.sse2.psad.bw" => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            assert_eq!(a.layout(), b.layout());
+            let layout = a.layout();
+
+            let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+            assert_eq!(lane_ty, fx.tcx.types.u8);
+            assert_eq!(ret_lane_ty, fx.tcx.types.u64);
+            assert_eq!(lane_count, ret_lane_count * 8);
+
+            let ret_lane_layout = fx.layout_of(fx.tcx.types.u64);
+            for out_lane_idx in 0..lane_count / 8 {
+                let mut lane_diff_acc = fx.bcx.ins().iconst(types::I64, 0);
+
+                for lane_idx in out_lane_idx * 8..out_lane_idx * 8 + 1 {
+                    let a_lane = a.value_lane(fx, lane_idx).load_scalar(fx);
+                    let b_lane = b.value_lane(fx, lane_idx).load_scalar(fx);
+
+                    let lane_diff = fx.bcx.ins().isub(a_lane, b_lane);
+                    let abs_lane_diff = fx.bcx.ins().iabs(lane_diff);
+                    let abs_lane_diff = fx.bcx.ins().uextend(types::I64, abs_lane_diff);
+                    lane_diff_acc = fx.bcx.ins().iadd(lane_diff_acc, abs_lane_diff);
+                }
+
+                let res_lane = CValue::by_val(lane_diff_acc, ret_lane_layout);
+
+                ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane);
+            }
+        }
+        "llvm.x86.ssse3.pmadd.ub.sw.128" => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+            assert_eq!(lane_ty, fx.tcx.types.u8);
+            assert_eq!(ret_lane_ty, fx.tcx.types.i16);
+            assert_eq!(lane_count, ret_lane_count * 2);
+
+            let ret_lane_layout = fx.layout_of(fx.tcx.types.i16);
+            for out_lane_idx in 0..lane_count / 2 {
+                let a_lane0 = a.value_lane(fx, out_lane_idx * 2).load_scalar(fx);
+                let a_lane0 = fx.bcx.ins().uextend(types::I16, a_lane0);
+                let b_lane0 = b.value_lane(fx, out_lane_idx * 2).load_scalar(fx);
+                let b_lane0 = fx.bcx.ins().sextend(types::I16, b_lane0);
+
+                let a_lane1 = a.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx);
+                let a_lane1 = fx.bcx.ins().uextend(types::I16, a_lane1);
+                let b_lane1 = b.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx);
+                let b_lane1 = fx.bcx.ins().sextend(types::I16, b_lane1);
+
+                let mul0: Value = fx.bcx.ins().imul(a_lane0, b_lane0);
+                let mul1 = fx.bcx.ins().imul(a_lane1, b_lane1);
+
+                let (val, has_overflow) = fx.bcx.ins().sadd_overflow(mul0, mul1);
+
+                let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, mul1, 0);
+
+                let min = fx.bcx.ins().iconst(types::I16, i64::from(i16::MIN as u16));
+                let max = fx.bcx.ins().iconst(types::I16, i64::from(i16::MAX as u16));
+
+                let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
+                let res_lane = fx.bcx.ins().select(has_overflow, sat_val, val);
+
+                let res_lane = CValue::by_val(res_lane, ret_lane_layout);
+
+                ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane);
+            }
+        }
+        "llvm.x86.sse2.pmadd.wd" => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            assert_eq!(a.layout(), b.layout());
+            let layout = a.layout();
+
+            let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+            assert_eq!(lane_ty, fx.tcx.types.i16);
+            assert_eq!(ret_lane_ty, fx.tcx.types.i32);
+            assert_eq!(lane_count, ret_lane_count * 2);
+
+            let ret_lane_layout = fx.layout_of(fx.tcx.types.i32);
+            for out_lane_idx in 0..lane_count / 2 {
+                let a_lane0 = a.value_lane(fx, out_lane_idx * 2).load_scalar(fx);
+                let a_lane0 = fx.bcx.ins().uextend(types::I32, a_lane0);
+                let b_lane0 = b.value_lane(fx, out_lane_idx * 2).load_scalar(fx);
+                let b_lane0 = fx.bcx.ins().sextend(types::I32, b_lane0);
+
+                let a_lane1 = a.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx);
+                let a_lane1 = fx.bcx.ins().uextend(types::I32, a_lane1);
+                let b_lane1 = b.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx);
+                let b_lane1 = fx.bcx.ins().sextend(types::I32, b_lane1);
+
+                let mul0: Value = fx.bcx.ins().imul(a_lane0, b_lane0);
+                let mul1 = fx.bcx.ins().imul(a_lane1, b_lane1);
+
+                let res_lane = fx.bcx.ins().iadd(mul0, mul1);
+                let res_lane = CValue::by_val(res_lane, ret_lane_layout);
+
+                ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane);
+            }
+        }
         _ => {
             fx.tcx
                 .sess

compiler/rustc_codegen_cranelift/src/value_and_place.rs

@@ -132,18 +132,11 @@ impl<'tcx> CValue<'tcx> {
                 (ptr.get_addr(fx), vtable)
             }
             CValueInner::ByValPair(data, vtable) => {
-                let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
-                    kind: StackSlotKind::ExplicitSlot,
-                    // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
-                    // specify stack slot alignment.
-                    size: (u32::try_from(fx.target_config.pointer_type().bytes()).unwrap() + 15)
-                        / 16
-                        * 16,
-                });
-                let data_ptr = Pointer::stack_slot(stack_slot);
-                let mut flags = MemFlags::new();
-                flags.set_notrap();
-                data_ptr.store(fx, data, flags);
+                let data_ptr = fx.create_stack_slot(
+                    u32::try_from(fx.target_config.pointer_type().bytes()).unwrap(),
+                    u32::try_from(fx.target_config.pointer_type().bytes()).unwrap(),
+                );
+                data_ptr.store(fx, data, MemFlags::trusted());
 
                 (data_ptr.get_addr(fx), vtable)
             }
@@ -372,13 +365,11 @@ impl<'tcx> CPlace<'tcx> {
                 .fatal(format!("values of type {} are too big to store on the stack", layout.ty));
         }
 
-        let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
-            kind: StackSlotKind::ExplicitSlot,
-            // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
-            // specify stack slot alignment.
-            size: (u32::try_from(layout.size.bytes()).unwrap() + 15) / 16 * 16,
-        });
-        CPlace { inner: CPlaceInner::Addr(Pointer::stack_slot(stack_slot), None), layout }
+        let stack_slot = fx.create_stack_slot(
+            u32::try_from(layout.size.bytes()).unwrap(),
+            u32::try_from(layout.align.pref.bytes()).unwrap(),
+        );
+        CPlace { inner: CPlaceInner::Addr(stack_slot, None), layout }
     }
 
     pub(crate) fn new_var(
@@ -543,13 +534,7 @@ impl<'tcx> CPlace<'tcx> {
                 _ if src_ty.is_vector() && dst_ty.is_vector() => codegen_bitcast(fx, dst_ty, data),
                 _ if src_ty.is_vector() || dst_ty.is_vector() => {
                     // FIXME(bytecodealliance/wasmtime#6104) do something more efficient for transmutes between vectors and integers.
-                    let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
-                        kind: StackSlotKind::ExplicitSlot,
-                        // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
-                        // specify stack slot alignment.
-                        size: (src_ty.bytes() + 15) / 16 * 16,
-                    });
-                    let ptr = Pointer::stack_slot(stack_slot);
+                    let ptr = fx.create_stack_slot(src_ty.bytes(), src_ty.bytes());
                     ptr.store(fx, data, MemFlags::trusted());
                     ptr.load(fx, dst_ty, MemFlags::trusted())
                 }

compiler/rustc_codegen_ssa/src/target_features.rs

@@ -342,6 +342,19 @@ const CSKY_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     ("hard-float-abi", Some(sym::csky_target_feature)),
     // tidy-alphabetical-end
 ];
+
+const LOONGARCH_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
+    // tidy-alphabetical-start
+    ("d", Some(sym::loongarch_target_feature)),
+    ("f", Some(sym::loongarch_target_feature)),
+    ("lasx", Some(sym::loongarch_target_feature)),
+    ("lbt", Some(sym::loongarch_target_feature)),
+    ("lsx", Some(sym::loongarch_target_feature)),
+    ("lvz", Some(sym::loongarch_target_feature)),
+    ("ual", Some(sym::loongarch_target_feature)),
+    // tidy-alphabetical-end
+];
+
 /// When rustdoc is running, provide a list of all known features so that all their respective
 /// primitives may be documented.
 ///
@@ -358,6 +371,7 @@ pub fn all_known_features() -> impl Iterator<Item = (&'static str, Option<Symbol
         .chain(WASM_ALLOWED_FEATURES.iter())
         .chain(BPF_ALLOWED_FEATURES.iter())
         .chain(CSKY_ALLOWED_FEATURES)
+        .chain(LOONGARCH_ALLOWED_FEATURES)
         .cloned()
 }
 
@@ -373,6 +387,7 @@ pub fn supported_target_features(sess: &Session) -> &'static [(&'static str, Opt
         "wasm32" | "wasm64" => WASM_ALLOWED_FEATURES,
         "bpf" => BPF_ALLOWED_FEATURES,
         "csky" => CSKY_ALLOWED_FEATURES,
+        "loongarch64" => LOONGARCH_ALLOWED_FEATURES,
         _ => &[],
     }
 }
@@ -445,6 +460,7 @@ pub fn from_target_feature(
                 Some(sym::bpf_target_feature) => rust_features.bpf_target_feature,
                 Some(sym::aarch64_ver_target_feature) => rust_features.aarch64_ver_target_feature,
                 Some(sym::csky_target_feature) => rust_features.csky_target_feature,
+                Some(sym::loongarch_target_feature) => rust_features.loongarch_target_feature,
                 Some(name) => bug!("unknown target feature gate {}", name),
                 None => true,
             };

compiler/rustc_const_eval/src/interpret/eval_context.rs

@@ -1010,7 +1010,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             // Just make this an efficient immediate.
             // Note that not calling `layout_of` here does have one real consequence:
             // if the type is too big, we'll only notice this when the local is actually initialized,
-            // which is a bit too late -- we should ideally notice this alreayd here, when the memory
+            // which is a bit too late -- we should ideally notice this already here, when the memory
             // is conceptually allocated. But given how rare that error is and that this is a hot function,
             // we accept this downside for now.
             Operand::Immediate(Immediate::Uninit)

compiler/rustc_feature/src/builtin_attrs.rs

@@ -825,6 +825,7 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
     rustc_attr!(TEST, rustc_strict_coherence, Normal, template!(Word), WarnFollowing),
     rustc_attr!(TEST, rustc_variance, Normal, template!(Word), WarnFollowing),
     rustc_attr!(TEST, rustc_variance_of_opaques, Normal, template!(Word), WarnFollowing),
+    rustc_attr!(TEST, rustc_hidden_type_of_opaques, Normal, template!(Word), WarnFollowing),
     rustc_attr!(TEST, rustc_layout, Normal, template!(List: "field1, field2, ..."), WarnFollowing),
     rustc_attr!(TEST, rustc_abi, Normal, template!(List: "field1, field2, ..."), WarnFollowing),
     rustc_attr!(TEST, rustc_regions, Normal, template!(Word), WarnFollowing),

compiler/rustc_feature/src/unstable.rs

@@ -288,6 +288,7 @@ declare_features! (
     (unstable, csky_target_feature, "1.73.0", Some(44839), None),
     (unstable, ermsb_target_feature, "1.49.0", Some(44839), None),
     (unstable, hexagon_target_feature, "1.27.0", Some(44839), None),
+    (unstable, loongarch_target_feature, "1.73.0", Some(44839), None),
     (unstable, mips_target_feature, "1.27.0", Some(44839), None),
     (unstable, powerpc_target_feature, "1.27.0", Some(44839), None),
     (unstable, riscv_target_feature, "1.45.0", Some(44839), None),

compiler/rustc_hir_analysis/messages.ftl

@@ -72,6 +72,12 @@ hir_analysis_copy_impl_on_type_with_dtor =
     the trait `Copy` cannot be implemented for this type; the type has a destructor
     .label = `Copy` not allowed on types with destructors
 
+hir_analysis_cross_crate_traits = cross-crate traits with a default impl, like `{$traits}`, can only be implemented for a struct/enum type, not `{$self_ty}`
+    .label = can't implement cross-crate trait with a default impl for non-struct/enum type
+
+hir_analysis_cross_crate_traits_defined = cross-crate traits with a default impl, like `{$traits}`, can only be implemented for a struct/enum type defined in the current crate
+    .label = can't implement cross-crate trait for type in another crate
+
 hir_analysis_dispatch_from_dyn_multi = implementing the `DispatchFromDyn` trait requires multiple coercions
     .note = the trait `DispatchFromDyn` may only be implemented for a coercion between structures with a single field being coerced
     .coercions_note = currently, {$number} fields need coercions: {$coercions}
@@ -237,6 +243,28 @@ hir_analysis_must_implement_not_function_span_note = required by this annotation
 
 hir_analysis_must_implement_one_of_attribute = the `#[rustc_must_implement_one_of]` attribute must be used with at least 2 args
 
+hir_analysis_only_current_traits_arbitrary = only traits defined in the current crate can be implemented for arbitrary types
+
+hir_analysis_only_current_traits_foreign = this is not defined in the current crate because this is a foreign trait
+
+hir_analysis_only_current_traits_label = impl doesn't use only types from inside the current crate
+
+hir_analysis_only_current_traits_name = this is not defined in the current crate because {$name} are always foreign
+
+hir_analysis_only_current_traits_note = define and implement a trait or new type instead
+
+hir_analysis_only_current_traits_opaque = type alias impl trait is treated as if it were foreign, because its hidden type could be from a foreign crate
+
+hir_analysis_only_current_traits_outside = only traits defined in the current crate can be implemented for types defined outside of the crate
+
+hir_analysis_only_current_traits_pointer = `{$pointer}` is not defined in the current crate because raw pointers are always foreign
+
+hir_analysis_only_current_traits_pointer_sugg = consider introducing a new wrapper type
+
+hir_analysis_only_current_traits_primitive = only traits defined in the current crate can be implemented for primitive types
+
+hir_analysis_only_current_traits_ty = `{$ty}` is not defined in the current crate
+
 hir_analysis_paren_sugar_attribute = the `#[rustc_paren_sugar]` attribute is a temporary means of controlling which traits can use parenthetical notation
     .help = add `#![feature(unboxed_closures)]` to the crate attributes to use it
 
@@ -326,6 +354,9 @@ hir_analysis_trait_object_declared_with_no_traits =
     at least one trait is required for an object type
     .alias_span = this alias does not contain a trait
 
+hir_analysis_traits_with_defualt_impl = traits with a default impl, like `{$traits}`, cannot be implemented for {$problematic_kind} `{$self_ty}`
+    .note = a trait object implements `{$traits}` if and only if `{$traits}` is one of the trait object's trait bounds
+
 hir_analysis_transparent_enum_variant = transparent enum needs exactly one variant, but has {$number}
     .label = needs exactly one variant, but has {$number}
     .many_label = too many variants in `{$path}`
@@ -339,6 +370,18 @@ hir_analysis_transparent_non_zero_sized_enum = the variant of a transparent {$de
     .label = needs at most one field with non-trivial size or alignment, but has {$field_count}
     .labels = this field has non-zero size or requires alignment
 
+hir_analysis_ty_param_first_local = type parameter `{$param_ty}` must be covered by another type when it appears before the first local type (`{$local_type}`)
+    .label = type parameter `{$param_ty}` must be covered by another type when it appears before the first local type (`{$local_type}`)
+    .note = implementing a foreign trait is only possible if at least one of the types for which it is implemented is local, and no uncovered type parameters appear before that first local type
+    .case_note = in this case, 'before' refers to the following order: `impl<..> ForeignTrait<T1, ..., Tn> for T0`, where `T0` is the first and `Tn` is the last
+
+hir_analysis_ty_param_some = type parameter `{$param_ty}` must be used as the type parameter for some local type (e.g., `MyStruct<{$param_ty}>`)
+    .label = type parameter `{$param_ty}` must be used as the type parameter for some local type
+    .note = implementing a foreign trait is only possible if at least one of the types for which it is implemented is local
+    .only_note = only traits defined in the current crate can be implemented for a type parameter
+
+hir_analysis_type_of = {$type_of}
+
 hir_analysis_typeof_reserved_keyword_used =
     `typeof` is a reserved keyword but unimplemented
     .suggestion = consider replacing `typeof(...)` with an actual type

compiler/rustc_hir_analysis/src/coherence/orphan.rs

@@ -2,8 +2,7 @@
 //! crate or pertains to a type defined in this crate.
 
 use rustc_data_structures::fx::FxHashSet;
-use rustc_errors::{struct_span_err, DelayDm};
-use rustc_errors::{Diagnostic, ErrorGuaranteed};
+use rustc_errors::{DelayDm, ErrorGuaranteed};
 use rustc_hir as hir;
 use rustc_middle::ty::util::CheckRegions;
 use rustc_middle::ty::GenericArgs;
@@ -17,6 +16,8 @@ use rustc_span::Span;
 use rustc_trait_selection::traits;
 use std::ops::ControlFlow;
 
+use crate::errors;
+
 #[instrument(skip(tcx), level = "debug")]
 pub(crate) fn orphan_check_impl(
     tcx: TyCtxt<'_>,
@@ -259,49 +260,30 @@ fn do_orphan_check_impl<'tcx>(
             match local_impl {
                 LocalImpl::Allow => {}
                 LocalImpl::Disallow { problematic_kind } => {
-                    let msg = format!(
-                        "traits with a default impl, like `{trait}`, \
-                                cannot be implemented for {problematic_kind} `{self_ty}`",
-                        trait = tcx.def_path_str(trait_def_id),
-                    );
-                    let label = format!(
-                        "a trait object implements `{trait}` if and only if `{trait}` \
-                                is one of the trait object's trait bounds",
-                        trait = tcx.def_path_str(trait_def_id),
-                    );
-                    let sp = tcx.def_span(def_id);
-                    let reported =
-                        struct_span_err!(tcx.sess, sp, E0321, "{}", msg).note(label).emit();
-                    return Err(reported);
+                    return Err(tcx.sess.emit_err(errors::TraitsWithDefaultImpl {
+                        span: tcx.def_span(def_id),
+                        traits: tcx.def_path_str(trait_def_id),
+                        problematic_kind,
+                        self_ty,
+                    }));
                 }
             }
         } else {
-            if let Some((msg, label)) = match nonlocal_impl {
-                NonlocalImpl::Allow => None,
-                NonlocalImpl::DisallowBecauseNonlocal => Some((
-                    format!(
-                        "cross-crate traits with a default impl, like `{}`, \
-                                can only be implemented for a struct/enum type \
-                                defined in the current crate",
-                        tcx.def_path_str(trait_def_id)
-                    ),
-                    "can't implement cross-crate trait for type in another crate",
-                )),
-                NonlocalImpl::DisallowOther => Some((
-                    format!(
-                        "cross-crate traits with a default impl, like `{}`, can \
-                                only be implemented for a struct/enum type, not `{}`",
-                        tcx.def_path_str(trait_def_id),
-                        self_ty
-                    ),
-                    "can't implement cross-crate trait with a default impl for \
-                            non-struct/enum type",
-                )),
-            } {
-                let sp = tcx.def_span(def_id);
-                let reported =
-                    struct_span_err!(tcx.sess, sp, E0321, "{}", msg).span_label(sp, label).emit();
-                return Err(reported);
+            match nonlocal_impl {
+                NonlocalImpl::Allow => {}
+                NonlocalImpl::DisallowBecauseNonlocal => {
+                    return Err(tcx.sess.emit_err(errors::CrossCrateTraitsDefined {
+                        span: tcx.def_span(def_id),
+                        traits: tcx.def_path_str(trait_def_id),
+                    }));
+                }
+                NonlocalImpl::DisallowOther => {
+                    return Err(tcx.sess.emit_err(errors::CrossCrateTraits {
+                        span: tcx.def_span(def_id),
+                        traits: tcx.def_path_str(trait_def_id),
+                        self_ty,
+                    }));
+                }
             }
         }
     }
@@ -322,19 +304,18 @@ fn emit_orphan_check_error<'tcx>(
     let self_ty = trait_ref.self_ty();
     Err(match err {
         traits::OrphanCheckErr::NonLocalInputType(tys) => {
-            let msg = match self_ty.kind() {
-                ty::Adt(..) => "can be implemented for types defined outside of the crate",
-                _ if self_ty.is_primitive() => "can be implemented for primitive types",
-                _ => "can be implemented for arbitrary types",
-            };
-            let mut err = struct_span_err!(
-                tcx.sess,
-                sp,
-                E0117,
-                "only traits defined in the current crate {msg}"
-            );
-            err.span_label(sp, "impl doesn't use only types from inside the current crate");
+            let (mut opaque, mut foreign, mut name, mut pointer, mut ty_diag) =
+                (Vec::new(), Vec::new(), Vec::new(), Vec::new(), Vec::new());
+            let mut sugg = None;
             for &(mut ty, is_target_ty) in &tys {
+                let span = if is_target_ty {
+                    // Point at `D<A>` in `impl<A, B> for C<B> in D<A>`
+                    self_ty_span
+                } else {
+                    // Point at `C<B>` in `impl<A, B> for C<B> in D<A>`
+                    trait_span
+                };
+
                 ty = tcx.erase_regions(ty);
                 ty = match ty.kind() {
                     // Remove the type arguments from the output, as they are not relevant.
@@ -345,50 +326,103 @@ fn emit_orphan_check_error<'tcx>(
                     ty::Adt(def, _) => Ty::new_adt(tcx, *def, ty::List::empty()),
                     _ => ty,
                 };
-                let msg = |ty: &str, postfix: &str| {
-                    format!("{ty} is not defined in the current crate{postfix}")
-                };
 
-                let this = |name: &str| {
-                    if !trait_ref.def_id.is_local() && !is_target_ty {
-                        msg("this", " because this is a foreign trait")
+                fn push_to_foreign_or_name<'tcx>(
+                    is_foreign: bool,
+                    foreign: &mut Vec<errors::OnlyCurrentTraitsForeign>,
+                    name: &mut Vec<errors::OnlyCurrentTraitsName<'tcx>>,
+                    span: Span,
+                    sname: &'tcx str,
+                ) {
+                    if is_foreign {
+                        foreign.push(errors::OnlyCurrentTraitsForeign { span })
                     } else {
-                        msg("this", &format!(" because {name} are always foreign"))
+                        name.push(errors::OnlyCurrentTraitsName { span, name: sname });
+                    }
+                }
+
+                let is_foreign = !trait_ref.def_id.is_local() && !is_target_ty;
+
+                match &ty.kind() {
+                    ty::Slice(_) => {
+                        push_to_foreign_or_name(
+                            is_foreign,
+                            &mut foreign,
+                            &mut name,
+                            span,
+                            "slices",
+                        );
+                    }
+                    ty::Array(..) => {
+                        push_to_foreign_or_name(
+                            is_foreign,
+                            &mut foreign,
+                            &mut name,
+                            span,
+                            "arrays",
+                        );
+                    }
+                    ty::Tuple(..) => {
+                        push_to_foreign_or_name(
+                            is_foreign,
+                            &mut foreign,
+                            &mut name,
+                            span,
+                            "tuples",
+                        );
                     }
-                };
-                let msg = match &ty.kind() {
-                    ty::Slice(_) => this("slices"),
-                    ty::Array(..) => this("arrays"),
-                    ty::Tuple(..) => this("tuples"),
                     ty::Alias(ty::Opaque, ..) => {
-                        "type alias impl trait is treated as if it were foreign, \
-                        because its hidden type could be from a foreign crate"
-                            .to_string()
+                        opaque.push(errors::OnlyCurrentTraitsOpaque { span })
                     }
                     ty::RawPtr(ptr_ty) => {
-                        emit_newtype_suggestion_for_raw_ptr(
-                            full_impl_span,
-                            self_ty,
-                            self_ty_span,
-                            ptr_ty,
-                            &mut err,
-                        );
-
-                        msg(&format!("`{ty}`"), " because raw pointers are always foreign")
+                        if !self_ty.has_param() {
+                            let mut_key = ptr_ty.mutbl.prefix_str();
+                            sugg = Some(errors::OnlyCurrentTraitsPointerSugg {
+                                wrapper_span: self_ty_span,
+                                struct_span: full_impl_span.shrink_to_lo(),
+                                mut_key,
+                                ptr_ty: ptr_ty.ty,
+                            });
+                        }
+                        pointer.push(errors::OnlyCurrentTraitsPointer { span, pointer: ty });
                     }
-                    _ => msg(&format!("`{ty}`"), ""),
-                };
-
-                if is_target_ty {
-                    // Point at `D<A>` in `impl<A, B> for C<B> in D<A>`
-                    err.span_label(self_ty_span, msg);
-                } else {
-                    // Point at `C<B>` in `impl<A, B> for C<B> in D<A>`
-                    err.span_label(trait_span, msg);
+                    _ => ty_diag.push(errors::OnlyCurrentTraitsTy { span, ty }),
                 }
             }
-            err.note("define and implement a trait or new type instead");
-            err.emit()
+
+            let err_struct = match self_ty.kind() {
+                ty::Adt(..) => errors::OnlyCurrentTraits::Outside {
+                    span: sp,
+                    note: (),
+                    opaque,
+                    foreign,
+                    name,
+                    pointer,
+                    ty: ty_diag,
+                    sugg,
+                },
+                _ if self_ty.is_primitive() => errors::OnlyCurrentTraits::Primitive {
+                    span: sp,
+                    note: (),
+                    opaque,
+                    foreign,
+                    name,
+                    pointer,
+                    ty: ty_diag,
+                    sugg,
+                },
+                _ => errors::OnlyCurrentTraits::Arbitrary {
+                    span: sp,
+                    note: (),
+                    opaque,
+                    foreign,
+                    name,
+                    pointer,
+                    ty: ty_diag,
+                    sugg,
+                },
+            };
+            tcx.sess.emit_err(err_struct)
         }
         traits::OrphanCheckErr::UncoveredTy(param_ty, local_type) => {
             let mut sp = sp;
@@ -399,85 +433,18 @@ fn emit_orphan_check_error<'tcx>(
             }
 
             match local_type {
-                Some(local_type) => struct_span_err!(
-                    tcx.sess,
-                    sp,
-                    E0210,
-                    "type parameter `{}` must be covered by another type \
-                    when it appears before the first local type (`{}`)",
+                Some(local_type) => tcx.sess.emit_err(errors::TyParamFirstLocal {
+                    span: sp,
+                    note: (),
                     param_ty,
-                    local_type
-                )
-                .span_label(
-                    sp,
-                    format!(
-                        "type parameter `{param_ty}` must be covered by another type \
-                    when it appears before the first local type (`{local_type}`)"
-                    ),
-                )
-                .note(
-                    "implementing a foreign trait is only possible if at \
-                        least one of the types for which it is implemented is local, \
-                        and no uncovered type parameters appear before that first \
-                        local type",
-                )
-                .note(
-                    "in this case, 'before' refers to the following order: \
-                        `impl<..> ForeignTrait<T1, ..., Tn> for T0`, \
-                        where `T0` is the first and `Tn` is the last",
-                )
-                .emit(),
-                None => struct_span_err!(
-                    tcx.sess,
-                    sp,
-                    E0210,
-                    "type parameter `{}` must be used as the type parameter for some \
-                    local type (e.g., `MyStruct<{}>`)",
-                    param_ty,
-                    param_ty
-                )
-                .span_label(
-                    sp,
-                    format!(
-                        "type parameter `{param_ty}` must be used as the type parameter for some \
-                    local type",
-                    ),
-                )
-                .note(
-                    "implementing a foreign trait is only possible if at \
-                        least one of the types for which it is implemented is local",
-                )
-                .note(
-                    "only traits defined in the current crate can be \
-                        implemented for a type parameter",
-                )
-                .emit(),
+                    local_type,
+                }),
+                None => tcx.sess.emit_err(errors::TyParamSome { span: sp, note: (), param_ty }),
             }
         }
     })
 }
 
-fn emit_newtype_suggestion_for_raw_ptr(
-    full_impl_span: Span,
-    self_ty: Ty<'_>,
-    self_ty_span: Span,
-    ptr_ty: &ty::TypeAndMut<'_>,
-    diag: &mut Diagnostic,
-) {
-    if !self_ty.has_param() {
-        let mut_key = ptr_ty.mutbl.prefix_str();
-        let msg_sugg = "consider introducing a new wrapper type".to_owned();
-        let sugg = vec![
-            (
-                full_impl_span.shrink_to_lo(),
-                format!("struct WrapperType(*{}{});\n\n", mut_key, ptr_ty.ty),
-            ),
-            (self_ty_span, "WrapperType".to_owned()),
-        ];
-        diag.multipart_suggestion(msg_sugg, sugg, rustc_errors::Applicability::MaybeIncorrect);
-    }
-}
-
 /// Lint impls of auto traits if they are likely to have
 /// unsound or surprising effects on auto impls.
 fn lint_auto_trait_impl<'tcx>(

compiler/rustc_hir_analysis/src/collect.rs

@@ -14,14 +14,11 @@
 //! At present, however, we do run collection across all items in the
 //! crate as a kind of pass. This should eventually be factored away.
 
-use crate::astconv::AstConv;
-use crate::check::intrinsic::intrinsic_operation_unsafety;
-use crate::errors;
-use hir::def::DefKind;
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_errors::{Applicability, DiagnosticBuilder, ErrorGuaranteed, StashKey};
 use rustc_hir as hir;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId, LocalModDefId};
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::{GenericParamKind, Node};
@@ -40,6 +37,11 @@ use rustc_trait_selection::traits::ObligationCtxt;
 use std::iter;
 use std::ops::Bound;
 
+use crate::astconv::AstConv;
+use crate::check::intrinsic::intrinsic_operation_unsafety;
+use crate::errors;
+pub use type_of::test_opaque_hidden_types;
+
 mod generics_of;
 mod item_bounds;
 mod predicates_of;

compiler/rustc_hir_analysis/src/collect/type_of.rs

@@ -11,6 +11,7 @@ use rustc_span::{Span, DUMMY_SP};
 
 use super::ItemCtxt;
 use super::{bad_placeholder, is_suggestable_infer_ty};
+pub use opaque::test_opaque_hidden_types;
 
 mod opaque;
 

compiler/rustc_hir_analysis/src/collect/type_of/opaque.rs

@@ -1,12 +1,25 @@
 use rustc_errors::StashKey;
-use rustc_hir::def_id::LocalDefId;
+use rustc_hir::def::DefKind;
+use rustc_hir::def_id::{LocalDefId, CRATE_DEF_ID};
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::{self as hir, def, Expr, ImplItem, Item, Node, TraitItem};
 use rustc_middle::hir::nested_filter;
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
-use rustc_span::DUMMY_SP;
+use rustc_span::{sym, DUMMY_SP};
 
-use crate::errors::{TaitForwardCompat, UnconstrainedOpaqueType};
+use crate::errors::{TaitForwardCompat, TypeOf, UnconstrainedOpaqueType};
+
+pub fn test_opaque_hidden_types(tcx: TyCtxt<'_>) {
+    if tcx.has_attr(CRATE_DEF_ID, sym::rustc_hidden_type_of_opaques) {
+        for id in tcx.hir().items() {
+            if matches!(tcx.def_kind(id.owner_id), DefKind::OpaqueTy) {
+                let type_of = tcx.type_of(id.owner_id).instantiate_identity();
+
+                tcx.sess.emit_err(TypeOf { span: tcx.def_span(id.owner_id), type_of });
+            }
+        }
+    }
+}
 
 /// Checks "defining uses" of opaque `impl Trait` types to ensure that they meet the restrictions
 /// laid for "higher-order pattern unification".

compiler/rustc_hir_analysis/src/errors.rs

@@ -467,6 +467,14 @@ pub(crate) struct VariancesOf {
     pub variances_of: String,
 }
 
+#[derive(Diagnostic)]
+#[diag(hir_analysis_type_of)]
+pub(crate) struct TypeOf<'tcx> {
+    #[primary_span]
+    pub span: Span,
+    pub type_of: Ty<'tcx>,
+}
+
 #[derive(Diagnostic)]
 #[diag(hir_analysis_pass_to_variadic_function, code = "E0617")]
 pub(crate) struct PassToVariadicFunction<'tcx, 'a> {
@@ -675,7 +683,6 @@ pub(crate) struct SIMDFFIHighlyExperimental {
 }
 
 #[derive(Diagnostic)]
-
 pub enum ImplNotMarkedDefault {
     #[diag(hir_analysis_impl_not_marked_default, code = "E0520")]
     #[note]
@@ -1151,3 +1158,174 @@ pub struct ImplForTyRequires {
     pub trait_name: String,
     pub ty: String,
 }
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_traits_with_defualt_impl, code = "E0321")]
+#[note]
+pub struct TraitsWithDefaultImpl<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub traits: String,
+    pub problematic_kind: &'a str,
+    pub self_ty: Ty<'a>,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_cross_crate_traits, code = "E0321")]
+pub struct CrossCrateTraits<'a> {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub traits: String,
+    pub self_ty: Ty<'a>,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_cross_crate_traits_defined, code = "E0321")]
+pub struct CrossCrateTraitsDefined {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub traits: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_ty_param_first_local, code = "E0210")]
+#[note]
+pub struct TyParamFirstLocal<'a> {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    #[note(hir_analysis_case_note)]
+    pub note: (),
+    pub param_ty: Ty<'a>,
+    pub local_type: Ty<'a>,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_ty_param_some, code = "E0210")]
+#[note]
+pub struct TyParamSome<'a> {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    #[note(hir_analysis_only_note)]
+    pub note: (),
+    pub param_ty: Ty<'a>,
+}
+
+#[derive(Diagnostic)]
+pub enum OnlyCurrentTraits<'a> {
+    #[diag(hir_analysis_only_current_traits_outside, code = "E0117")]
+    Outside {
+        #[primary_span]
+        #[label(hir_analysis_only_current_traits_label)]
+        span: Span,
+        #[note(hir_analysis_only_current_traits_note)]
+        note: (),
+        #[subdiagnostic]
+        opaque: Vec<OnlyCurrentTraitsOpaque>,
+        #[subdiagnostic]
+        foreign: Vec<OnlyCurrentTraitsForeign>,
+        #[subdiagnostic]
+        name: Vec<OnlyCurrentTraitsName<'a>>,
+        #[subdiagnostic]
+        pointer: Vec<OnlyCurrentTraitsPointer<'a>>,
+        #[subdiagnostic]
+        ty: Vec<OnlyCurrentTraitsTy<'a>>,
+        #[subdiagnostic]
+        sugg: Option<OnlyCurrentTraitsPointerSugg<'a>>,
+    },
+    #[diag(hir_analysis_only_current_traits_primitive, code = "E0117")]
+    Primitive {
+        #[primary_span]
+        #[label(hir_analysis_only_current_traits_label)]
+        span: Span,
+        #[note(hir_analysis_only_current_traits_note)]
+        note: (),
+        #[subdiagnostic]
+        opaque: Vec<OnlyCurrentTraitsOpaque>,
+        #[subdiagnostic]
+        foreign: Vec<OnlyCurrentTraitsForeign>,
+        #[subdiagnostic]
+        name: Vec<OnlyCurrentTraitsName<'a>>,
+        #[subdiagnostic]
+        pointer: Vec<OnlyCurrentTraitsPointer<'a>>,
+        #[subdiagnostic]
+        ty: Vec<OnlyCurrentTraitsTy<'a>>,
+        #[subdiagnostic]
+        sugg: Option<OnlyCurrentTraitsPointerSugg<'a>>,
+    },
+    #[diag(hir_analysis_only_current_traits_arbitrary, code = "E0117")]
+    Arbitrary {
+        #[primary_span]
+        #[label(hir_analysis_only_current_traits_label)]
+        span: Span,
+        #[note(hir_analysis_only_current_traits_note)]
+        note: (),
+        #[subdiagnostic]
+        opaque: Vec<OnlyCurrentTraitsOpaque>,
+        #[subdiagnostic]
+        foreign: Vec<OnlyCurrentTraitsForeign>,
+        #[subdiagnostic]
+        name: Vec<OnlyCurrentTraitsName<'a>>,
+        #[subdiagnostic]
+        pointer: Vec<OnlyCurrentTraitsPointer<'a>>,
+        #[subdiagnostic]
+        ty: Vec<OnlyCurrentTraitsTy<'a>>,
+        #[subdiagnostic]
+        sugg: Option<OnlyCurrentTraitsPointerSugg<'a>>,
+    },
+}
+
+#[derive(Subdiagnostic)]
+#[label(hir_analysis_only_current_traits_opaque)]
+pub struct OnlyCurrentTraitsOpaque {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Subdiagnostic)]
+#[label(hir_analysis_only_current_traits_foreign)]
+pub struct OnlyCurrentTraitsForeign {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Subdiagnostic)]
+#[label(hir_analysis_only_current_traits_name)]
+pub struct OnlyCurrentTraitsName<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub name: &'a str,
+}
+
+#[derive(Subdiagnostic)]
+#[label(hir_analysis_only_current_traits_pointer)]
+pub struct OnlyCurrentTraitsPointer<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub pointer: Ty<'a>,
+}
+
+#[derive(Subdiagnostic)]
+#[label(hir_analysis_only_current_traits_ty)]
+pub struct OnlyCurrentTraitsTy<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub ty: Ty<'a>,
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion(
+    hir_analysis_only_current_traits_pointer_sugg,
+    applicability = "maybe-incorrect"
+)]
+pub struct OnlyCurrentTraitsPointerSugg<'a> {
+    #[suggestion_part(code = "WrapperType")]
+    pub wrapper_span: Span,
+    #[suggestion_part(code = "struct WrapperType(*{mut_key}{ptr_ty});\n\n")]
+    pub struct_span: Span,
+    pub mut_key: &'a str,
+    pub ptr_ty: Ty<'a>,
+}

compiler/rustc_hir_analysis/src/lib.rs

@@ -214,6 +214,10 @@ pub fn check_crate(tcx: TyCtxt<'_>) -> Result<(), ErrorGuaranteed> {
         tcx.hir().for_each_module(|module| tcx.ensure().check_mod_item_types(module))
     });
 
+    if tcx.features().rustc_attrs {
+        tcx.sess.track_errors(|| collect::test_opaque_hidden_types(tcx))?;
+    }
+
     // Freeze definitions as we don't add new ones at this point. This improves performance by
     // allowing lock-free access to them.
     tcx.untracked().definitions.freeze();

compiler/rustc_hir_typeck/src/demand.rs

@@ -58,7 +58,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             || self.suggest_into(err, expr, expr_ty, expected)
             || self.suggest_floating_point_literal(err, expr, expected)
             || self.suggest_null_ptr_for_literal_zero_given_to_ptr_arg(err, expr, expected)
-            || self.suggest_coercing_result_via_try_operator(err, expr, expected, expr_ty);
+            || self.suggest_coercing_result_via_try_operator(err, expr, expected, expr_ty)
+            || self.suggest_missing_unwrap_expect(err, expr, expected, expr_ty);
 
         if !suggested {
             self.note_source_of_type_mismatch_constraint(

compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs

@@ -6,6 +6,7 @@ use crate::method::probe::{IsSuggestion, Mode, ProbeScope};
 use rustc_ast::util::parser::{ExprPrecedence, PREC_POSTFIX};
 use rustc_errors::{Applicability, Diagnostic, MultiSpan};
 use rustc_hir as hir;
+use rustc_hir::def::Res;
 use rustc_hir::def::{CtorKind, CtorOf, DefKind};
 use rustc_hir::lang_items::LangItem;
 use rustc_hir::{
@@ -1738,4 +1739,83 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // If the field is hygienic it must come from the same syntax context.
             && self.tcx.def_ident_span(field.did).unwrap().normalize_to_macros_2_0().eq_ctxt(span)
     }
+
+    pub(crate) fn suggest_missing_unwrap_expect(
+        &self,
+        err: &mut Diagnostic,
+        expr: &hir::Expr<'tcx>,
+        expected: Ty<'tcx>,
+        found: Ty<'tcx>,
+    ) -> bool {
+        let ty::Adt(adt, args) = found.kind() else { return false };
+        let ret_ty_matches = |diagnostic_item| {
+            if let Some(ret_ty) = self
+                    .ret_coercion
+                    .as_ref()
+                    .map(|c| self.resolve_vars_if_possible(c.borrow().expected_ty()))
+                    && let ty::Adt(kind, _) = ret_ty.kind()
+                    && self.tcx.get_diagnostic_item(diagnostic_item) == Some(kind.did())
+                {
+                    true
+                } else {
+                    false
+                }
+        };
+
+        // don't suggest anything like `Ok(ok_val).unwrap()` , `Some(some_val).unwrap()`,
+        // `None.unwrap()` etc.
+        let is_ctor = matches!(
+            expr.kind,
+            hir::ExprKind::Call(
+                hir::Expr {
+                    kind: hir::ExprKind::Path(hir::QPath::Resolved(
+                        None,
+                        hir::Path { res: Res::Def(hir::def::DefKind::Ctor(_, _), _), .. },
+                    )),
+                    ..
+                },
+                ..,
+            ) | hir::ExprKind::Path(hir::QPath::Resolved(
+                None,
+                hir::Path { res: Res::Def(hir::def::DefKind::Ctor(_, _), _), .. },
+            )),
+        );
+
+        let (article, kind, variant, sugg_operator) =
+            if self.tcx.is_diagnostic_item(sym::Result, adt.did()) {
+                ("a", "Result", "Err", ret_ty_matches(sym::Result))
+            } else if self.tcx.is_diagnostic_item(sym::Option, adt.did()) {
+                ("an", "Option", "None", ret_ty_matches(sym::Option))
+            } else {
+                return false;
+            };
+        if is_ctor || !self.can_coerce(args.type_at(0), expected) {
+            return false;
+        }
+
+        let (msg, sugg) = if sugg_operator {
+            (
+                format!(
+                    "use the `?` operator to extract the `{found}` value, propagating \
+                            {article} `{kind}::{variant}` value to the caller"
+                ),
+                "?",
+            )
+        } else {
+            (
+                format!(
+                    "consider using `{kind}::expect` to unwrap the `{found}` value, \
+                                panicking if the value is {article} `{kind}::{variant}`"
+                ),
+                ".expect(\"REASON\")",
+            )
+        };
+        err.span_suggestion_verbose(
+            expr.span.shrink_to_hi(),
+            msg,
+            sugg,
+            Applicability::HasPlaceholders,
+        );
+        return true;
+    }
 }

compiler/rustc_infer/src/infer/canonical/mod.rs

@@ -152,7 +152,7 @@ impl<'tcx> InferCtxt<'tcx> {
                 )
                 .into(),
             CanonicalVarKind::Effect => {
-                let vid = self.inner.borrow_mut().effect_unification_table().new_key(None);
+                let vid = self.inner.borrow_mut().effect_unification_table().new_key(None).vid;
                 ty::Const::new_infer(self.tcx, ty::InferConst::EffectVar(vid), self.tcx.types.bool)
                     .into()
             }

compiler/rustc_infer/src/infer/combine.rs

@@ -320,7 +320,7 @@ impl<'tcx> InferCtxt<'tcx> {
     #[instrument(level = "debug", skip(self))]
     fn unify_const_variable(
         &self,
-        target_vid: ty::ConstVid<'tcx>,
+        target_vid: ty::ConstVid,
         ct: ty::Const<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
     ) -> RelateResult<'tcx, ty::Const<'tcx>> {
@@ -381,7 +381,7 @@ impl<'tcx> InferCtxt<'tcx> {
     fn unify_effect_variable(
         &self,
         vid_is_expected: bool,
-        vid: ty::EffectVid<'tcx>,
+        vid: ty::EffectVid,
         val: EffectVarValue<'tcx>,
     ) -> RelateResult<'tcx, ty::Const<'tcx>> {
         self.inner

compiler/rustc_infer/src/infer/freshen.rs

@@ -42,7 +42,7 @@ pub struct TypeFreshener<'a, 'tcx> {
     ty_freshen_count: u32,
     const_freshen_count: u32,
     ty_freshen_map: FxHashMap<ty::InferTy, Ty<'tcx>>,
-    const_freshen_map: FxHashMap<ty::InferConst<'tcx>, ty::Const<'tcx>>,
+    const_freshen_map: FxHashMap<ty::InferConst, ty::Const<'tcx>>,
 }
 
 impl<'a, 'tcx> TypeFreshener<'a, 'tcx> {
@@ -79,12 +79,12 @@ impl<'a, 'tcx> TypeFreshener<'a, 'tcx> {
     fn freshen_const<F>(
         &mut self,
         opt_ct: Option<ty::Const<'tcx>>,
-        key: ty::InferConst<'tcx>,
+        key: ty::InferConst,
         freshener: F,
         ty: Ty<'tcx>,
     ) -> ty::Const<'tcx>
     where
-        F: FnOnce(u32) -> ty::InferConst<'tcx>,
+        F: FnOnce(u32) -> ty::InferConst,
     {
         if let Some(ct) = opt_ct {
             return ct.fold_with(self);

compiler/rustc_infer/src/infer/fudge.rs

@@ -1,3 +1,4 @@
+use rustc_middle::infer::unify_key::ConstVidKey;
 use rustc_middle::ty::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
 use rustc_middle::ty::{self, ConstVid, FloatVid, IntVid, RegionVid, Ty, TyCtxt, TyVid};
 
@@ -23,14 +24,14 @@ where
 }
 
 fn const_vars_since_snapshot<'tcx>(
-    table: &mut UnificationTable<'_, 'tcx, ConstVid<'tcx>>,
+    table: &mut UnificationTable<'_, 'tcx, ConstVidKey<'tcx>>,
     snapshot_var_len: usize,
-) -> (Range<ConstVid<'tcx>>, Vec<ConstVariableOrigin>) {
+) -> (Range<ConstVid>, Vec<ConstVariableOrigin>) {
     let range = vars_since_snapshot(table, snapshot_var_len);
     (
-        range.start..range.end,
-        (range.start.index..range.end.index)
-            .map(|index| table.probe_value(ConstVid::from_index(index)).origin)
+        range.start.vid..range.end.vid,
+        (range.start.index()..range.end.index())
+            .map(|index| table.probe_value(ConstVid::from_u32(index)).origin)
             .collect(),
     )
 }
@@ -172,7 +173,7 @@ pub struct InferenceFudger<'a, 'tcx> {
     int_vars: Range<IntVid>,
     float_vars: Range<FloatVid>,
     region_vars: (Range<RegionVid>, Vec<RegionVariableOrigin>),
-    const_vars: (Range<ConstVid<'tcx>>, Vec<ConstVariableOrigin>),
+    const_vars: (Range<ConstVid>, Vec<ConstVariableOrigin>),
 }
 
 impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for InferenceFudger<'a, 'tcx> {
@@ -235,7 +236,7 @@ impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for InferenceFudger<'a, 'tcx> {
             if self.const_vars.0.contains(&vid) {
                 // This variable was created during the fudging.
                 // Recreate it with a fresh variable here.
-                let idx = (vid.index - self.const_vars.0.start.index) as usize;
+                let idx = (vid.index() - self.const_vars.0.start.index()) as usize;
                 let origin = self.const_vars.1[idx];
                 self.infcx.next_const_var(ct.ty(), origin)
             } else {

compiler/rustc_infer/src/infer/generalize.rs

@@ -17,7 +17,7 @@ pub(super) fn generalize<'tcx, D: GeneralizerDelegate<'tcx>, T: Into<Term<'tcx>>
     infcx: &InferCtxt<'tcx>,
     delegate: &mut D,
     term: T,
-    for_vid: impl Into<ty::TermVid<'tcx>>,
+    for_vid: impl Into<ty::TermVid>,
     ambient_variance: ty::Variance,
 ) -> RelateResult<'tcx, Generalization<T>> {
     let (for_universe, root_vid) = match for_vid.into() {
@@ -27,7 +27,7 @@ pub(super) fn generalize<'tcx, D: GeneralizerDelegate<'tcx>, T: Into<Term<'tcx>>
         ),
         ty::TermVid::Const(ct_vid) => (
             infcx.probe_const_var(ct_vid).unwrap_err(),
-            ty::TermVid::Const(infcx.inner.borrow_mut().const_unification_table().find(ct_vid)),
+            ty::TermVid::Const(infcx.inner.borrow_mut().const_unification_table().find(ct_vid).vid),
         ),
     };
 
@@ -127,7 +127,7 @@ struct Generalizer<'me, 'tcx, D> {
     /// The vid of the type variable that is in the process of being
     /// instantiated. If we find this within the value we are folding,
     /// that means we would have created a cyclic value.
-    root_vid: ty::TermVid<'tcx>,
+    root_vid: ty::TermVid,
 
     /// The universe of the type variable that is in the process of being
     /// instantiated. If we find anything that this universe cannot name,
@@ -376,7 +376,7 @@ where
                 // `vid` are related and we'd be inferring an infinitely
                 // deep const.
                 if ty::TermVid::Const(
-                    self.infcx.inner.borrow_mut().const_unification_table().find(vid),
+                    self.infcx.inner.borrow_mut().const_unification_table().find(vid).vid,
                 ) == self.root_vid
                 {
                     return Err(self.cyclic_term_error());
@@ -394,10 +394,14 @@ where
                         if self.for_universe.can_name(universe) {
                             Ok(c)
                         } else {
-                            let new_var_id = variable_table.new_key(ConstVarValue {
-                                origin: var_value.origin,
-                                val: ConstVariableValue::Unknown { universe: self.for_universe },
-                            });
+                            let new_var_id = variable_table
+                                .new_key(ConstVarValue {
+                                    origin: var_value.origin,
+                                    val: ConstVariableValue::Unknown {
+                                        universe: self.for_universe,
+                                    },
+                                })
+                                .vid;
                             Ok(ty::Const::new_var(self.tcx(), new_var_id, c.ty()))
                         }
                     }

compiler/rustc_infer/src/infer/mod.rs

@@ -6,7 +6,9 @@ pub use self::RegionVariableOrigin::*;
 pub use self::SubregionOrigin::*;
 pub use self::ValuePairs::*;
 pub use combine::ObligationEmittingRelation;
+use rustc_data_structures::captures::Captures;
 use rustc_data_structures::undo_log::UndoLogs;
+use rustc_middle::infer::unify_key::{ConstVidKey, EffectVidKey};
 
 use self::opaque_types::OpaqueTypeStorage;
 pub(crate) use self::undo_log::{InferCtxtUndoLogs, Snapshot, UndoLog};
@@ -40,7 +42,6 @@ use rustc_span::{Span, DUMMY_SP};
 
 use std::cell::{Cell, RefCell};
 use std::fmt;
-use std::marker::PhantomData;
 
 use self::combine::CombineFields;
 use self::error_reporting::TypeErrCtxt;
@@ -85,7 +86,7 @@ pub struct InferOk<'tcx, T> {
 pub type InferResult<'tcx, T> = Result<InferOk<'tcx, T>, TypeError<'tcx>>;
 
 pub type UnitResult<'tcx> = RelateResult<'tcx, ()>; // "unify result"
-pub type FixupResult<'tcx, T> = Result<T, FixupError<'tcx>>; // "fixup result"
+pub type FixupResult<T> = Result<T, FixupError>; // "fixup result"
 
 pub(crate) type UnificationTable<'a, 'tcx, T> = ut::UnificationTable<
     ut::InPlace<T, &'a mut ut::UnificationStorage<T>, &'a mut InferCtxtUndoLogs<'tcx>>,
@@ -108,7 +109,7 @@ pub struct InferCtxtInner<'tcx> {
     type_variable_storage: type_variable::TypeVariableStorage<'tcx>,
 
     /// Map from const parameter variable to the kind of const it represents.
-    const_unification_storage: ut::UnificationTableStorage<ty::ConstVid<'tcx>>,
+    const_unification_storage: ut::UnificationTableStorage<ConstVidKey<'tcx>>,
 
     /// Map from integral variable to the kind of integer it represents.
     int_unification_storage: ut::UnificationTableStorage<ty::IntVid>,
@@ -117,7 +118,7 @@ pub struct InferCtxtInner<'tcx> {
     float_unification_storage: ut::UnificationTableStorage<ty::FloatVid>,
 
     /// Map from effect variable to the effect param it represents.
-    effect_unification_storage: ut::UnificationTableStorage<ty::EffectVid<'tcx>>,
+    effect_unification_storage: ut::UnificationTableStorage<EffectVidKey<'tcx>>,
 
     /// Tracks the set of region variables and the constraints between them.
     ///
@@ -224,11 +225,11 @@ impl<'tcx> InferCtxtInner<'tcx> {
     }
 
     #[inline]
-    fn const_unification_table(&mut self) -> UnificationTable<'_, 'tcx, ty::ConstVid<'tcx>> {
+    fn const_unification_table(&mut self) -> UnificationTable<'_, 'tcx, ConstVidKey<'tcx>> {
         self.const_unification_storage.with_log(&mut self.undo_log)
     }
 
-    fn effect_unification_table(&mut self) -> UnificationTable<'_, 'tcx, ty::EffectVid<'tcx>> {
+    fn effect_unification_table(&mut self) -> UnificationTable<'_, 'tcx, EffectVidKey<'tcx>> {
         self.effect_unification_storage.with_log(&mut self.undo_log)
     }
 
@@ -341,7 +342,9 @@ pub struct InferCtxt<'tcx> {
     next_trait_solver: bool,
 }
 
-impl<'tcx> ty::InferCtxtLike<TyCtxt<'tcx>> for InferCtxt<'tcx> {
+impl<'tcx> ty::InferCtxtLike for InferCtxt<'tcx> {
+    type Interner = TyCtxt<'tcx>;
+
     fn universe_of_ty(&self, ty: ty::InferTy) -> Option<ty::UniverseIndex> {
         use InferTy::*;
         match ty {
@@ -357,7 +360,7 @@ impl<'tcx> ty::InferCtxtLike<TyCtxt<'tcx>> for InferCtxt<'tcx> {
         }
     }
 
-    fn universe_of_ct(&self, ct: ty::InferConst<'tcx>) -> Option<ty::UniverseIndex> {
+    fn universe_of_ct(&self, ct: ty::InferConst) -> Option<ty::UniverseIndex> {
         use ty::InferConst::*;
         match ct {
             // Same issue as with `universe_of_ty`
@@ -546,11 +549,11 @@ pub enum NllRegionVariableOrigin {
 
 // FIXME(eddyb) investigate overlap between this and `TyOrConstInferVar`.
 #[derive(Copy, Clone, Debug)]
-pub enum FixupError<'tcx> {
+pub enum FixupError {
     UnresolvedIntTy(IntVid),
     UnresolvedFloatTy(FloatVid),
     UnresolvedTy(TyVid),
-    UnresolvedConst(ConstVid<'tcx>),
+    UnresolvedConst(ConstVid),
 }
 
 /// See the `region_obligations` field for more information.
@@ -561,7 +564,7 @@ pub struct RegionObligation<'tcx> {
     pub origin: SubregionOrigin<'tcx>,
 }
 
-impl<'tcx> fmt::Display for FixupError<'tcx> {
+impl fmt::Display for FixupError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         use self::FixupError::*;
 
@@ -792,7 +795,7 @@ impl<'tcx> InferCtxt<'tcx> {
         let mut table = inner.effect_unification_table();
 
         (0..table.len())
-            .map(|i| ty::EffectVid { index: i as u32, phantom: PhantomData })
+            .map(|i| ty::EffectVid::from_usize(i))
             .filter(|&vid| table.probe_value(vid).is_none())
             .map(|v| {
                 ty::Const::new_infer(self.tcx, ty::InferConst::EffectVar(v), self.tcx.types.bool)
@@ -1070,15 +1073,20 @@ impl<'tcx> InferCtxt<'tcx> {
             .inner
             .borrow_mut()
             .const_unification_table()
-            .new_key(ConstVarValue { origin, val: ConstVariableValue::Unknown { universe } });
+            .new_key(ConstVarValue { origin, val: ConstVariableValue::Unknown { universe } })
+            .vid;
         ty::Const::new_var(self.tcx, vid, ty)
     }
 
-    pub fn next_const_var_id(&self, origin: ConstVariableOrigin) -> ConstVid<'tcx> {
-        self.inner.borrow_mut().const_unification_table().new_key(ConstVarValue {
-            origin,
-            val: ConstVariableValue::Unknown { universe: self.universe() },
-        })
+    pub fn next_const_var_id(&self, origin: ConstVariableOrigin) -> ConstVid {
+        self.inner
+            .borrow_mut()
+            .const_unification_table()
+            .new_key(ConstVarValue {
+                origin,
+                val: ConstVariableValue::Unknown { universe: self.universe() },
+            })
+            .vid
     }
 
     fn next_int_var_id(&self) -> IntVid {
@@ -1192,11 +1200,15 @@ impl<'tcx> InferCtxt<'tcx> {
                     ),
                     span,
                 };
-                let const_var_id =
-                    self.inner.borrow_mut().const_unification_table().new_key(ConstVarValue {
+                let const_var_id = self
+                    .inner
+                    .borrow_mut()
+                    .const_unification_table()
+                    .new_key(ConstVarValue {
                         origin,
                         val: ConstVariableValue::Unknown { universe: self.universe() },
-                    });
+                    })
+                    .vid;
                 ty::Const::new_var(
                     self.tcx,
                     const_var_id,
@@ -1211,7 +1223,7 @@ impl<'tcx> InferCtxt<'tcx> {
     }
 
     pub fn var_for_effect(&self, param: &ty::GenericParamDef) -> GenericArg<'tcx> {
-        let effect_vid = self.inner.borrow_mut().effect_unification_table().new_key(None);
+        let effect_vid = self.inner.borrow_mut().effect_unification_table().new_key(None).vid;
         let ty = self
             .tcx
             .type_of(param.def_id)
@@ -1331,12 +1343,12 @@ impl<'tcx> InferCtxt<'tcx> {
         self.inner.borrow_mut().type_variables().root_var(var)
     }
 
-    pub fn root_const_var(&self, var: ty::ConstVid<'tcx>) -> ty::ConstVid<'tcx> {
-        self.inner.borrow_mut().const_unification_table().find(var)
+    pub fn root_const_var(&self, var: ty::ConstVid) -> ty::ConstVid {
+        self.inner.borrow_mut().const_unification_table().find(var).vid
     }
 
-    pub fn root_effect_var(&self, var: ty::EffectVid<'tcx>) -> ty::EffectVid<'tcx> {
-        self.inner.borrow_mut().effect_unification_table().find(var)
+    pub fn root_effect_var(&self, var: ty::EffectVid) -> ty::EffectVid {
+        self.inner.borrow_mut().effect_unification_table().find(var).vid
     }
 
     /// Resolves an int var to a rigid int type, if it was constrained to one,
@@ -1400,17 +1412,14 @@ impl<'tcx> InferCtxt<'tcx> {
         value.visit_with(&mut resolve::UnresolvedTypeOrConstFinder::new(self)).break_value()
     }
 
-    pub fn probe_const_var(
-        &self,
-        vid: ty::ConstVid<'tcx>,
-    ) -> Result<ty::Const<'tcx>, ty::UniverseIndex> {
+    pub fn probe_const_var(&self, vid: ty::ConstVid) -> Result<ty::Const<'tcx>, ty::UniverseIndex> {
         match self.inner.borrow_mut().const_unification_table().probe_value(vid).val {
             ConstVariableValue::Known { value } => Ok(value),
             ConstVariableValue::Unknown { universe } => Err(universe),
         }
     }
 
-    pub fn probe_effect_var(&self, vid: EffectVid<'tcx>) -> Option<EffectVarValue<'tcx>> {
+    pub fn probe_effect_var(&self, vid: EffectVid) -> Option<EffectVarValue<'tcx>> {
         self.inner.borrow_mut().effect_unification_table().probe_value(vid)
     }
 
@@ -1421,7 +1430,7 @@ impl<'tcx> InferCtxt<'tcx> {
     ///
     /// This method is idempotent, but it not typically not invoked
     /// except during the writeback phase.
-    pub fn fully_resolve<T: TypeFoldable<TyCtxt<'tcx>>>(&self, value: T) -> FixupResult<'tcx, T> {
+    pub fn fully_resolve<T: TypeFoldable<TyCtxt<'tcx>>>(&self, value: T) -> FixupResult<T> {
         match resolve::fully_resolve(self, value) {
             Ok(value) => {
                 if value.has_non_region_infer() {
@@ -1645,11 +1654,11 @@ impl<'tcx> InferCtxt<'tcx> {
     #[inline]
     pub fn is_ty_infer_var_definitely_unchanged<'a>(
         &'a self,
-    ) -> (impl Fn(TyOrConstInferVar<'tcx>) -> bool + 'a) {
+    ) -> (impl Fn(TyOrConstInferVar) -> bool + Captures<'tcx> + 'a) {
         // This hoists the borrow/release out of the loop body.
         let inner = self.inner.try_borrow();
 
-        return move |infer_var: TyOrConstInferVar<'tcx>| match (infer_var, &inner) {
+        return move |infer_var: TyOrConstInferVar| match (infer_var, &inner) {
             (TyOrConstInferVar::Ty(ty_var), Ok(inner)) => {
                 use self::type_variable::TypeVariableValue;
 
@@ -1672,7 +1681,7 @@ impl<'tcx> InferCtxt<'tcx> {
     /// inference variables), and it handles both `Ty` and `ty::Const` without
     /// having to resort to storing full `GenericArg`s in `stalled_on`.
     #[inline(always)]
-    pub fn ty_or_const_infer_var_changed(&self, infer_var: TyOrConstInferVar<'tcx>) -> bool {
+    pub fn ty_or_const_infer_var_changed(&self, infer_var: TyOrConstInferVar) -> bool {
         match infer_var {
             TyOrConstInferVar::Ty(v) => {
                 use self::type_variable::TypeVariableValue;
@@ -1779,7 +1788,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
 /// Helper for [InferCtxt::ty_or_const_infer_var_changed] (see comment on that), currently
 /// used only for `traits::fulfill`'s list of `stalled_on` inference variables.
 #[derive(Copy, Clone, Debug)]
-pub enum TyOrConstInferVar<'tcx> {
+pub enum TyOrConstInferVar {
     /// Equivalent to `ty::Infer(ty::TyVar(_))`.
     Ty(TyVid),
     /// Equivalent to `ty::Infer(ty::IntVar(_))`.
@@ -1788,12 +1797,12 @@ pub enum TyOrConstInferVar<'tcx> {
     TyFloat(FloatVid),
 
     /// Equivalent to `ty::ConstKind::Infer(ty::InferConst::Var(_))`.
-    Const(ConstVid<'tcx>),
+    Const(ConstVid),
     /// Equivalent to `ty::ConstKind::Infer(ty::InferConst::EffectVar(_))`.
-    Effect(EffectVid<'tcx>),
+    Effect(EffectVid),
 }
 
-impl<'tcx> TyOrConstInferVar<'tcx> {
+impl<'tcx> TyOrConstInferVar {
     /// Tries to extract an inference variable from a type or a constant, returns `None`
     /// for types other than `ty::Infer(_)` (or `InferTy::Fresh*`) and
     /// for constants other than `ty::ConstKind::Infer(_)` (or `InferConst::Fresh`).

compiler/rustc_infer/src/infer/resolve.rs

@@ -192,7 +192,7 @@ impl<'a, 'tcx> TypeVisitor<TyCtxt<'tcx>> for UnresolvedTypeOrConstFinder<'a, 'tc
 /// Full type resolution replaces all type and region variables with
 /// their concrete results. If any variable cannot be replaced (never unified, etc)
 /// then an `Err` result is returned.
-pub fn fully_resolve<'tcx, T>(infcx: &InferCtxt<'tcx>, value: T) -> FixupResult<'tcx, T>
+pub fn fully_resolve<'tcx, T>(infcx: &InferCtxt<'tcx>, value: T) -> FixupResult<T>
 where
     T: TypeFoldable<TyCtxt<'tcx>>,
 {
@@ -206,7 +206,7 @@ struct FullTypeResolver<'a, 'tcx> {
 }
 
 impl<'a, 'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for FullTypeResolver<'a, 'tcx> {
-    type Error = FixupError<'tcx>;
+    type Error = FixupError;
 
     fn interner(&self) -> TyCtxt<'tcx> {
         self.infcx.tcx

compiler/rustc_infer/src/infer/undo_log.rs

@@ -3,7 +3,7 @@ use std::marker::PhantomData;
 use rustc_data_structures::snapshot_vec as sv;
 use rustc_data_structures::undo_log::{Rollback, UndoLogs};
 use rustc_data_structures::unify as ut;
-use rustc_middle::infer::unify_key::RegionVidKey;
+use rustc_middle::infer::unify_key::{ConstVidKey, EffectVidKey, RegionVidKey};
 use rustc_middle::ty::{self, OpaqueHiddenType, OpaqueTypeKey};
 
 use crate::{
@@ -21,10 +21,10 @@ pub struct Snapshot<'tcx> {
 pub(crate) enum UndoLog<'tcx> {
     OpaqueTypes(OpaqueTypeKey<'tcx>, Option<OpaqueHiddenType<'tcx>>),
     TypeVariables(type_variable::UndoLog<'tcx>),
-    ConstUnificationTable(sv::UndoLog<ut::Delegate<ty::ConstVid<'tcx>>>),
+    ConstUnificationTable(sv::UndoLog<ut::Delegate<ConstVidKey<'tcx>>>),
     IntUnificationTable(sv::UndoLog<ut::Delegate<ty::IntVid>>),
     FloatUnificationTable(sv::UndoLog<ut::Delegate<ty::FloatVid>>),
-    EffectUnificationTable(sv::UndoLog<ut::Delegate<ty::EffectVid<'tcx>>>),
+    EffectUnificationTable(sv::UndoLog<ut::Delegate<EffectVidKey<'tcx>>>),
     RegionConstraintCollector(region_constraints::UndoLog<'tcx>),
     RegionUnificationTable(sv::UndoLog<ut::Delegate<RegionVidKey<'tcx>>>),
     ProjectionCache(traits::UndoLog<'tcx>),
@@ -56,9 +56,9 @@ impl_from! {
     IntUnificationTable(sv::UndoLog<ut::Delegate<ty::IntVid>>),
 
     FloatUnificationTable(sv::UndoLog<ut::Delegate<ty::FloatVid>>),
-    EffectUnificationTable(sv::UndoLog<ut::Delegate<ty::EffectVid<'tcx>>>),
+    EffectUnificationTable(sv::UndoLog<ut::Delegate<EffectVidKey<'tcx>>>),
 
-    ConstUnificationTable(sv::UndoLog<ut::Delegate<ty::ConstVid<'tcx>>>),
+    ConstUnificationTable(sv::UndoLog<ut::Delegate<ConstVidKey<'tcx>>>),
 
     RegionUnificationTable(sv::UndoLog<ut::Delegate<RegionVidKey<'tcx>>>),
     ProjectionCache(traits::UndoLog<'tcx>),

compiler/rustc_interface/src/passes.rs

@@ -775,12 +775,16 @@ fn analysis(tcx: TyCtxt<'_>, (): ()) -> Result<()> {
     rustc_hir_analysis::check_crate(tcx)?;
 
     sess.time("MIR_borrow_checking", || {
-        tcx.hir().par_body_owners(|def_id| tcx.ensure().mir_borrowck(def_id));
+        tcx.hir().par_body_owners(|def_id| {
+            // Run THIR unsafety check because it's responsible for stealing
+            // and deallocating THIR when enabled.
+            tcx.ensure().thir_check_unsafety(def_id);
+            tcx.ensure().mir_borrowck(def_id)
+        });
     });
 
     sess.time("MIR_effect_checking", || {
         for def_id in tcx.hir().body_owners() {
-            tcx.ensure().thir_check_unsafety(def_id);
             if !tcx.sess.opts.unstable_opts.thir_unsafeck {
                 rustc_mir_transform::check_unsafety::check_unsafety(tcx, def_id);
             }

compiler/rustc_middle/src/infer/unify_key.rs

@@ -141,18 +141,30 @@ pub struct ConstVarValue<'tcx> {
     pub val: ConstVariableValue<'tcx>,
 }
 
-impl<'tcx> UnifyKey for ty::ConstVid<'tcx> {
+#[derive(PartialEq, Copy, Clone, Debug)]
+pub struct ConstVidKey<'tcx> {
+    pub vid: ty::ConstVid,
+    pub phantom: PhantomData<ty::Const<'tcx>>,
+}
+
+impl<'tcx> From<ty::ConstVid> for ConstVidKey<'tcx> {
+    fn from(vid: ty::ConstVid) -> Self {
+        ConstVidKey { vid, phantom: PhantomData }
+    }
+}
+
+impl<'tcx> UnifyKey for ConstVidKey<'tcx> {
     type Value = ConstVarValue<'tcx>;
     #[inline]
     fn index(&self) -> u32 {
-        self.index
+        self.vid.as_u32()
     }
     #[inline]
     fn from_index(i: u32) -> Self {
-        ty::ConstVid { index: i, phantom: PhantomData }
+        ConstVidKey::from(ty::ConstVid::from_u32(i))
     }
     fn tag() -> &'static str {
-        "ConstVid"
+        "ConstVidKey"
     }
 }
 
@@ -224,17 +236,29 @@ impl<'tcx> UnifyValue for EffectVarValue<'tcx> {
     }
 }
 
-impl<'tcx> UnifyKey for ty::EffectVid<'tcx> {
+#[derive(PartialEq, Copy, Clone, Debug)]
+pub struct EffectVidKey<'tcx> {
+    pub vid: ty::EffectVid,
+    pub phantom: PhantomData<EffectVarValue<'tcx>>,
+}
+
+impl<'tcx> From<ty::EffectVid> for EffectVidKey<'tcx> {
+    fn from(vid: ty::EffectVid) -> Self {
+        EffectVidKey { vid, phantom: PhantomData }
+    }
+}
+
+impl<'tcx> UnifyKey for EffectVidKey<'tcx> {
     type Value = Option<EffectVarValue<'tcx>>;
     #[inline]
     fn index(&self) -> u32 {
-        self.index
+        self.vid.as_u32()
     }
     #[inline]
     fn from_index(i: u32) -> Self {
-        ty::EffectVid { index: i, phantom: PhantomData }
+        EffectVidKey::from(ty::EffectVid::from_u32(i))
     }
     fn tag() -> &'static str {
-        "EffectVid"
+        "EffectVidKey"
     }
 }

compiler/rustc_middle/src/thir.rs

@@ -636,7 +636,8 @@ impl<'tcx> Pat<'tcx> {
             Wild | Range(..) | Binding { subpattern: None, .. } | Constant { .. } | Error(_) => {}
             AscribeUserType { subpattern, .. }
             | Binding { subpattern: Some(subpattern), .. }
-            | Deref { subpattern } => subpattern.walk_(it),
+            | Deref { subpattern }
+            | InlineConstant { subpattern, .. } => subpattern.walk_(it),
             Leaf { subpatterns } | Variant { subpatterns, .. } => {
                 subpatterns.iter().for_each(|field| field.pattern.walk_(it))
             }
@@ -764,6 +765,22 @@ pub enum PatKind<'tcx> {
         value: mir::Const<'tcx>,
     },
 
+    /// Inline constant found while lowering a pattern.
+    InlineConstant {
+        /// [LocalDefId] of the constant, we need this so that we have a
+        /// reference that can be used by unsafety checking to visit nested
+        /// unevaluated constants.
+        def: LocalDefId,
+        /// If the inline constant is used in a range pattern, this subpattern
+        /// represents the range (if both ends are inline constants, there will
+        /// be multiple InlineConstant wrappers).
+        ///
+        /// Otherwise, the actual pattern that the constant lowered to. As with
+        /// other constants, inline constants are matched structurally where
+        /// possible.
+        subpattern: Box<Pat<'tcx>>,
+    },
+
     Range(Box<PatRange<'tcx>>),
 
     /// Matches against a slice, checking the length and extracting elements.
@@ -924,6 +941,9 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
                 write!(f, "{subpattern}")
             }
             PatKind::Constant { value } => write!(f, "{value}"),
+            PatKind::InlineConstant { def: _, ref subpattern } => {
+                write!(f, "{} (from inline const)", subpattern)
+            }
             PatKind::Range(box PatRange { lo, hi, end }) => {
                 write!(f, "{lo}")?;
                 write!(f, "{end}")?;

compiler/rustc_middle/src/thir/visit.rs

@@ -233,16 +233,17 @@ pub fn walk_pat<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, pat: &Pat<'
             }
         }
         Constant { value: _ } => {}
+        InlineConstant { def: _, subpattern } => visitor.visit_pat(subpattern),
         Range(_) => {}
         Slice { prefix, slice, suffix } | Array { prefix, slice, suffix } => {
             for subpattern in prefix.iter() {
-                visitor.visit_pat(&subpattern);
+                visitor.visit_pat(subpattern);
             }
             if let Some(pat) = slice {
-                visitor.visit_pat(&pat);
+                visitor.visit_pat(pat);
             }
             for subpattern in suffix.iter() {
-                visitor.visit_pat(&subpattern);
+                visitor.visit_pat(subpattern);
             }
         }
         Or { pats } => {

compiler/rustc_middle/src/ty/consts.rs

@@ -57,7 +57,7 @@ impl<'tcx> Const<'tcx> {
     }
 
     #[inline]
-    pub fn new_var(tcx: TyCtxt<'tcx>, infer: ty::ConstVid<'tcx>, ty: Ty<'tcx>) -> Const<'tcx> {
+    pub fn new_var(tcx: TyCtxt<'tcx>, infer: ty::ConstVid, ty: Ty<'tcx>) -> Const<'tcx> {
         Const::new(tcx, ty::ConstKind::Infer(ty::InferConst::Var(infer)), ty)
     }
 
@@ -67,7 +67,7 @@ impl<'tcx> Const<'tcx> {
     }
 
     #[inline]
-    pub fn new_infer(tcx: TyCtxt<'tcx>, infer: ty::InferConst<'tcx>, ty: Ty<'tcx>) -> Const<'tcx> {
+    pub fn new_infer(tcx: TyCtxt<'tcx>, infer: ty::InferConst, ty: Ty<'tcx>) -> Const<'tcx> {
         Const::new(tcx, ty::ConstKind::Infer(infer), ty)
     }
 

compiler/rustc_middle/src/ty/consts/kind.rs

@@ -80,19 +80,19 @@ static_assert_size!(super::ConstKind<'_>, 32);
 
 /// An inference variable for a const, for use in const generics.
 #[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable, Hash)]
-pub enum InferConst<'tcx> {
+pub enum InferConst {
     /// Infer the value of the const.
-    Var(ty::ConstVid<'tcx>),
+    Var(ty::ConstVid),
     /// Infer the value of the effect.
     ///
     /// For why this is separate from the `Var` variant above, see the
     /// documentation on `EffectVid`.
-    EffectVar(ty::EffectVid<'tcx>),
+    EffectVar(ty::EffectVid),
     /// A fresh const variable. See `infer::freshen` for more details.
     Fresh(u32),
 }
 
-impl<CTX> HashStable<CTX> for InferConst<'_> {
+impl<CTX> HashStable<CTX> for InferConst {
     fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
         match self {
             InferConst::Var(_) | InferConst::EffectVar(_) => {

compiler/rustc_middle/src/ty/context.rs

@@ -100,7 +100,7 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
     type PolyFnSig = PolyFnSig<'tcx>;
     type AllocId = crate::mir::interpret::AllocId;
     type Const = ty::Const<'tcx>;
-    type InferConst = ty::InferConst<'tcx>;
+    type InferConst = ty::InferConst;
     type AliasConst = ty::UnevaluatedConst<'tcx>;
     type PlaceholderConst = ty::PlaceholderConst<'tcx>;
     type ParamConst = ty::ParamConst;

compiler/rustc_middle/src/ty/list.rs

@@ -1,7 +1,7 @@
 use crate::arena::Arena;
 use rustc_data_structures::aligned::{align_of, Aligned};
 use rustc_serialize::{Encodable, Encoder};
-use rustc_type_ir::{InferCtxtLike, OptWithInfcx};
+use rustc_type_ir::{InferCtxtLike, WithInfcx};
 use std::alloc::Layout;
 use std::cmp::Ordering;
 use std::fmt;
@@ -121,8 +121,8 @@ impl<T: fmt::Debug> fmt::Debug for List<T> {
     }
 }
 impl<'tcx, T: super::DebugWithInfcx<TyCtxt<'tcx>>> super::DebugWithInfcx<TyCtxt<'tcx>> for List<T> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         fmt::Debug::fmt(&this.map(|this| this.as_slice()), f)

compiler/rustc_middle/src/ty/mod.rs

@@ -54,7 +54,7 @@ use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{ExpnId, ExpnKind, Span};
 use rustc_target::abi::{Align, FieldIdx, Integer, IntegerType, VariantIdx};
 pub use rustc_target::abi::{ReprFlags, ReprOptions};
-pub use rustc_type_ir::{DebugWithInfcx, InferCtxtLike, OptWithInfcx};
+pub use rustc_type_ir::{DebugWithInfcx, InferCtxtLike, WithInfcx};
 pub use vtable::*;
 
 use std::fmt::Debug;
@@ -1084,19 +1084,19 @@ impl ParamTerm {
 }
 
 #[derive(Copy, Clone, Eq, PartialEq, Debug)]
-pub enum TermVid<'tcx> {
+pub enum TermVid {
     Ty(ty::TyVid),
-    Const(ty::ConstVid<'tcx>),
+    Const(ty::ConstVid),
 }
 
-impl From<ty::TyVid> for TermVid<'_> {
+impl From<ty::TyVid> for TermVid {
     fn from(value: ty::TyVid) -> Self {
         TermVid::Ty(value)
     }
 }
 
-impl<'tcx> From<ty::ConstVid<'tcx>> for TermVid<'tcx> {
-    fn from(value: ty::ConstVid<'tcx>) -> Self {
+impl From<ty::ConstVid> for TermVid {
+    fn from(value: ty::ConstVid) -> Self {
         TermVid::Const(value)
     }
 }

compiler/rustc_middle/src/ty/print/pretty.rs

@@ -1194,7 +1194,7 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
         None
     }
 
-    fn const_infer_name(&self, _: ty::ConstVid<'tcx>) -> Option<Symbol> {
+    fn const_infer_name(&self, _: ty::ConstVid) -> Option<Symbol> {
         None
     }
 
@@ -1742,7 +1742,7 @@ pub struct FmtPrinterData<'a, 'tcx> {
     pub region_highlight_mode: RegionHighlightMode<'tcx>,
 
     pub ty_infer_name_resolver: Option<Box<dyn Fn(ty::TyVid) -> Option<Symbol> + 'a>>,
-    pub const_infer_name_resolver: Option<Box<dyn Fn(ty::ConstVid<'tcx>) -> Option<Symbol> + 'a>>,
+    pub const_infer_name_resolver: Option<Box<dyn Fn(ty::ConstVid) -> Option<Symbol> + 'a>>,
 }
 
 impl<'a, 'tcx> Deref for FmtPrinter<'a, 'tcx> {
@@ -2082,7 +2082,7 @@ impl<'tcx> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx> {
         self.printed_type_count = 0;
     }
 
-    fn const_infer_name(&self, id: ty::ConstVid<'tcx>) -> Option<Symbol> {
+    fn const_infer_name(&self, id: ty::ConstVid) -> Option<Symbol> {
         self.0.const_infer_name_resolver.as_ref().and_then(|func| func(id))
     }
 

compiler/rustc_middle/src/ty/structural_impls.rs

@@ -10,7 +10,7 @@ use crate::ty::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitor};
 use crate::ty::{self, AliasTy, InferConst, Lift, Term, TermKind, Ty, TyCtxt};
 use rustc_hir::def::Namespace;
 use rustc_target::abi::TyAndLayout;
-use rustc_type_ir::{ConstKind, DebugWithInfcx, InferCtxtLike, OptWithInfcx};
+use rustc_type_ir::{ConstKind, DebugWithInfcx, InferCtxtLike, WithInfcx};
 
 use std::fmt::{self, Debug};
 use std::ops::ControlFlow;
@@ -87,12 +87,12 @@ impl fmt::Debug for ty::FreeRegion {
 
 impl<'tcx> fmt::Debug for ty::FnSig<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        OptWithInfcx::new_no_ctx(self).fmt(f)
+        WithInfcx::with_no_infcx(self).fmt(f)
     }
 }
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::FnSig<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         let sig = this.data;
@@ -128,18 +128,6 @@ impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::FnSig<'tcx> {
     }
 }
 
-impl<'tcx> fmt::Debug for ty::ConstVid<'tcx> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "?{}c", self.index)
-    }
-}
-
-impl fmt::Debug for ty::EffectVid<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "?{}e", self.index)
-    }
-}
-
 impl<'tcx> fmt::Debug for ty::TraitRef<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         with_no_trimmed_paths!(fmt::Display::fmt(self, f))
@@ -147,8 +135,8 @@ impl<'tcx> fmt::Debug for ty::TraitRef<'tcx> {
 }
 
 impl<'tcx> ty::DebugWithInfcx<TyCtxt<'tcx>> for Ty<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         this.data.fmt(f)
@@ -236,12 +224,12 @@ impl<'tcx> fmt::Debug for ty::PredicateKind<'tcx> {
 
 impl<'tcx> fmt::Debug for AliasTy<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        OptWithInfcx::new_no_ctx(self).fmt(f)
+        WithInfcx::with_no_infcx(self).fmt(f)
     }
 }
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for AliasTy<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         f.debug_struct("AliasTy")
@@ -251,7 +239,7 @@ impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for AliasTy<'tcx> {
     }
 }
 
-impl<'tcx> fmt::Debug for ty::InferConst<'tcx> {
+impl fmt::Debug for ty::InferConst {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             InferConst::Var(var) => write!(f, "{var:?}"),
@@ -260,17 +248,17 @@ impl<'tcx> fmt::Debug for ty::InferConst<'tcx> {
         }
     }
 }
-impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::InferConst<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::InferConst {
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         use ty::InferConst::*;
-        match this.infcx.and_then(|infcx| infcx.universe_of_ct(*this.data)) {
+        match this.infcx.universe_of_ct(*this.data) {
             None => write!(f, "{:?}", this.data),
             Some(universe) => match *this.data {
-                Var(vid) => write!(f, "?{}_{}c", vid.index, universe.index()),
-                EffectVar(vid) => write!(f, "?{}_{}e", vid.index, universe.index()),
+                Var(vid) => write!(f, "?{}_{}c", vid.index(), universe.index()),
+                EffectVar(vid) => write!(f, "?{}_{}e", vid.index(), universe.index()),
                 Fresh(_) => {
                     unreachable!()
                 }
@@ -281,12 +269,12 @@ impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::InferConst<'tcx> {
 
 impl<'tcx> fmt::Debug for ty::consts::Expr<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        OptWithInfcx::new_no_ctx(self).fmt(f)
+        WithInfcx::with_no_infcx(self).fmt(f)
     }
 }
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::consts::Expr<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         match this.data {
@@ -314,12 +302,12 @@ impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::consts::Expr<'tcx> {
 
 impl<'tcx> fmt::Debug for ty::UnevaluatedConst<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        OptWithInfcx::new_no_ctx(self).fmt(f)
+        WithInfcx::with_no_infcx(self).fmt(f)
     }
 }
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::UnevaluatedConst<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         f.debug_struct("UnevaluatedConst")
@@ -331,12 +319,12 @@ impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::UnevaluatedConst<'tcx> {
 
 impl<'tcx> fmt::Debug for ty::Const<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        OptWithInfcx::new_no_ctx(self).fmt(f)
+        WithInfcx::with_no_infcx(self).fmt(f)
     }
 }
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::Const<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         // If this is a value, we spend some effort to make it look nice.
@@ -392,8 +380,8 @@ impl<'tcx> fmt::Debug for GenericArg<'tcx> {
     }
 }
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for GenericArg<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         match this.data.unpack() {
@@ -410,8 +398,8 @@ impl<'tcx> fmt::Debug for Region<'tcx> {
     }
 }
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for Region<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         write!(f, "{:?}", &this.map(|data| data.kind()))
@@ -419,11 +407,11 @@ impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for Region<'tcx> {
 }
 
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::RegionVid {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
-        match this.infcx.and_then(|infcx| infcx.universe_of_lt(*this.data)) {
+        match this.infcx.universe_of_lt(*this.data) {
             Some(universe) => write!(f, "'?{}_{}", this.data.index(), universe.index()),
             None => write!(f, "{:?}", this.data),
         }
@@ -431,8 +419,8 @@ impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::RegionVid {
 }
 
 impl<'tcx, T: DebugWithInfcx<TyCtxt<'tcx>>> DebugWithInfcx<TyCtxt<'tcx>> for ty::Binder<'tcx, T> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         f.debug_tuple("Binder")
@@ -862,7 +850,7 @@ impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for ty::Const<'tcx> {
     }
 }
 
-impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for InferConst<'tcx> {
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for InferConst {
     fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
         _folder: &mut F,
@@ -871,7 +859,7 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for InferConst<'tcx> {
     }
 }
 
-impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for InferConst<'tcx> {
+impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for InferConst {
     fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(
         &self,
         _visitor: &mut V,

compiler/rustc_middle/src/ty/sty.rs

@@ -29,7 +29,6 @@ use std::assert_matches::debug_assert_matches;
 use std::borrow::Cow;
 use std::cmp::Ordering;
 use std::fmt;
-use std::marker::PhantomData;
 use std::ops::{ControlFlow, Deref, Range};
 use ty::util::IntTypeExt;
 
@@ -683,8 +682,8 @@ pub enum ExistentialPredicate<'tcx> {
 }
 
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ExistentialPredicate<'tcx> {
-    fn fmt<InfCtx: rustc_type_ir::InferCtxtLike<TyCtxt<'tcx>>>(
-        this: rustc_type_ir::OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: rustc_type_ir::InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: rustc_type_ir::WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         fmt::Debug::fmt(&this.data, f)
@@ -1583,26 +1582,22 @@ impl fmt::Debug for EarlyBoundRegion {
     }
 }
 
-/// A **`const`** **v**ariable **ID**.
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
-#[derive(HashStable, TyEncodable, TyDecodable)]
-pub struct ConstVid<'tcx> {
-    pub index: u32,
-    pub phantom: PhantomData<&'tcx ()>,
+rustc_index::newtype_index! {
+    /// A **`const`** **v**ariable **ID**.
+    #[debug_format = "?{}c"]
+    pub struct ConstVid {}
 }
 
-/// An **effect** **v**ariable **ID**.
-///
-/// Handling effect infer variables happens separately from const infer variables
-/// because we do not want to reuse any of the const infer machinery. If we try to
-/// relate an effect variable with a normal one, we would ICE, which can catch bugs
-/// where we are not correctly using the effect var for an effect param. Fallback
-/// is also implemented on top of having separate effect and normal const variables.
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
-#[derive(TyEncodable, TyDecodable)]
-pub struct EffectVid<'tcx> {
-    pub index: u32,
-    pub phantom: PhantomData<&'tcx ()>,
+rustc_index::newtype_index! {
+    /// An **effect** **v**ariable **ID**.
+    ///
+    /// Handling effect infer variables happens separately from const infer variables
+    /// because we do not want to reuse any of the const infer machinery. If we try to
+    /// relate an effect variable with a normal one, we would ICE, which can catch bugs
+    /// where we are not correctly using the effect var for an effect param. Fallback
+    /// is also implemented on top of having separate effect and normal const variables.
+    #[debug_format = "?{}e"]
+    pub struct EffectVid {}
 }
 
 rustc_index::newtype_index! {

compiler/rustc_mir_build/src/build/matches/mod.rs

@@ -847,6 +847,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 self.visit_primary_bindings(subpattern, subpattern_user_ty, f)
             }
 
+            PatKind::InlineConstant { ref subpattern, .. } => {
+                self.visit_primary_bindings(subpattern, pattern_user_ty.clone(), f)
+            }
+
             PatKind::Leaf { ref subpatterns } => {
                 for subpattern in subpatterns {
                     let subpattern_user_ty = pattern_user_ty.clone().leaf(subpattern.field);

compiler/rustc_mir_build/src/build/matches/simplify.rs

@@ -204,6 +204,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 Err(match_pair)
             }
 
+            PatKind::InlineConstant { subpattern: ref pattern, def: _ } => {
+                candidate.match_pairs.push(MatchPair::new(match_pair.place, pattern, self));
+
+                Ok(())
+            }
+
             PatKind::Range(box PatRange { lo, hi, end }) => {
                 let (range, bias) = match *lo.ty().kind() {
                     ty::Char => {
@@ -229,8 +235,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     // correct the comparison. This is achieved by XORing with a bias (see
                     // pattern/_match.rs for another pertinent example of this pattern).
                     //
-                    // Also, for performance, it's important to only do the second `try_to_bits` if
-                    // necessary.
+                    // Also, for performance, it's important to only do the second
+                    // `try_to_bits` if necessary.
                     let lo = lo.try_to_bits(sz).unwrap() ^ bias;
                     if lo <= min {
                         let hi = hi.try_to_bits(sz).unwrap() ^ bias;

compiler/rustc_mir_build/src/build/matches/test.rs

@@ -73,6 +73,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             PatKind::Or { .. } => bug!("or-patterns should have already been handled"),
 
             PatKind::AscribeUserType { .. }
+            | PatKind::InlineConstant { .. }
             | PatKind::Array { .. }
             | PatKind::Wild
             | PatKind::Binding { .. }
@@ -111,6 +112,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             | PatKind::Or { .. }
             | PatKind::Binding { .. }
             | PatKind::AscribeUserType { .. }
+            | PatKind::InlineConstant { .. }
             | PatKind::Leaf { .. }
             | PatKind::Deref { .. }
             | PatKind::Error(_) => {

compiler/rustc_mir_build/src/build/mod.rs

@@ -53,10 +53,7 @@ pub(crate) fn closure_saved_names_of_captured_variables<'tcx>(
 }
 
 /// Construct the MIR for a given `DefId`.
-fn mir_build(tcx: TyCtxt<'_>, def: LocalDefId) -> Body<'_> {
-    // Ensure unsafeck and abstract const building is ran before we steal the THIR.
-    tcx.ensure_with_value()
-        .thir_check_unsafety(tcx.typeck_root_def_id(def.to_def_id()).expect_local());
+fn mir_build<'tcx>(tcx: TyCtxt<'tcx>, def: LocalDefId) -> Body<'tcx> {
     tcx.ensure_with_value().thir_abstract_const(def);
     if let Err(e) = tcx.check_match(def) {
         return construct_error(tcx, def, e);
@@ -65,20 +62,27 @@ fn mir_build(tcx: TyCtxt<'_>, def: LocalDefId) -> Body<'_> {
     let body = match tcx.thir_body(def) {
         Err(error_reported) => construct_error(tcx, def, error_reported),
         Ok((thir, expr)) => {
-            // We ran all queries that depended on THIR at the beginning
-            // of `mir_build`, so now we can steal it
-            let thir = thir.steal();
+            let build_mir = |thir: &Thir<'tcx>| match thir.body_type {
+                thir::BodyTy::Fn(fn_sig) => construct_fn(tcx, def, thir, expr, fn_sig),
+                thir::BodyTy::Const(ty) => construct_const(tcx, def, thir, expr, ty),
+            };
 
-            tcx.ensure().check_match(def);
             // this must run before MIR dump, because
             // "not all control paths return a value" is reported here.
             //
             // maybe move the check to a MIR pass?
             tcx.ensure().check_liveness(def);
 
-            match thir.body_type {
-                thir::BodyTy::Fn(fn_sig) => construct_fn(tcx, def, &thir, expr, fn_sig),
-                thir::BodyTy::Const(ty) => construct_const(tcx, def, &thir, expr, ty),
+            if tcx.sess.opts.unstable_opts.thir_unsafeck {
+                // Don't steal here if THIR unsafeck is being used. Instead
+                // steal in unsafeck. This is so that pattern inline constants
+                // can be evaluated as part of building the THIR of the parent
+                // function without a cycle.
+                build_mir(&thir.borrow())
+            } else {
+                // We ran all queries that depended on THIR at the beginning
+                // of `mir_build`, so now we can steal it
+                build_mir(&thir.steal())
             }
         }
     };

compiler/rustc_mir_build/src/check_unsafety.rs

@@ -124,7 +124,9 @@ impl<'tcx> UnsafetyVisitor<'_, 'tcx> {
     /// Handle closures/coroutines/inline-consts, which is unsafecked with their parent body.
     fn visit_inner_body(&mut self, def: LocalDefId) {
         if let Ok((inner_thir, expr)) = self.tcx.thir_body(def) {
-            let inner_thir = &inner_thir.borrow();
+            // Runs all other queries that depend on THIR.
+            self.tcx.ensure_with_value().mir_built(def);
+            let inner_thir = &inner_thir.steal();
             let hir_context = self.tcx.hir().local_def_id_to_hir_id(def);
             let mut inner_visitor = UnsafetyVisitor { thir: inner_thir, hir_context, ..*self };
             inner_visitor.visit_expr(&inner_thir[expr]);
@@ -224,6 +226,7 @@ impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> {
                 PatKind::Wild |
                 // these just wrap other patterns
                 PatKind::Or { .. } |
+                PatKind::InlineConstant { .. } |
                 PatKind::AscribeUserType { .. } |
                 PatKind::Error(_) => {}
             }
@@ -277,6 +280,9 @@ impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> {
                 visit::walk_pat(self, pat);
                 self.inside_adt = old_inside_adt;
             }
+            PatKind::InlineConstant { def, .. } => {
+                self.visit_inner_body(*def);
+            }
             _ => {
                 visit::walk_pat(self, pat);
             }
@@ -788,7 +794,9 @@ pub fn thir_check_unsafety(tcx: TyCtxt<'_>, def: LocalDefId) {
     }
 
     let Ok((thir, expr)) = tcx.thir_body(def) else { return };
-    let thir = &thir.borrow();
+    // Runs all other queries that depend on THIR.
+    tcx.ensure_with_value().mir_built(def);
+    let thir = &thir.steal();
     // If `thir` is empty, a type error occurred, skip this body.
     if thir.exprs.is_empty() {
         return;

compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs

@@ -1326,7 +1326,8 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
         let ctor;
         let fields;
         match &pat.kind {
-            PatKind::AscribeUserType { subpattern, .. } => return mkpat(subpattern),
+            PatKind::AscribeUserType { subpattern, .. }
+            | PatKind::InlineConstant { subpattern, .. } => return mkpat(subpattern),
             PatKind::Binding { subpattern: Some(subpat), .. } => return mkpat(subpat),
             PatKind::Binding { subpattern: None, .. } | PatKind::Wild => {
                 ctor = Wildcard;

compiler/rustc_mir_build/src/thir/pattern/mod.rs

@@ -27,6 +27,7 @@ use rustc_middle::ty::{
     self, AdtDef, CanonicalUserTypeAnnotation, GenericArg, GenericArgsRef, Region, Ty, TyCtxt,
     TypeVisitableExt, UserType,
 };
+use rustc_span::def_id::LocalDefId;
 use rustc_span::{ErrorGuaranteed, Span, Symbol};
 use rustc_target::abi::{FieldIdx, Integer};
 
@@ -88,15 +89,21 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
     fn lower_pattern_range_endpoint(
         &mut self,
         expr: Option<&'tcx hir::Expr<'tcx>>,
-    ) -> Result<(Option<mir::Const<'tcx>>, Option<Ascription<'tcx>>), ErrorGuaranteed> {
+    ) -> Result<
+        (Option<mir::Const<'tcx>>, Option<Ascription<'tcx>>, Option<LocalDefId>),
+        ErrorGuaranteed,
+    > {
         match expr {
-            None => Ok((None, None)),
+            None => Ok((None, None, None)),
             Some(expr) => {
-                let (kind, ascr) = match self.lower_lit(expr) {
-                    PatKind::AscribeUserType { ascription, subpattern: box Pat { kind, .. } } => {
-                        (kind, Some(ascription))
+                let (kind, ascr, inline_const) = match self.lower_lit(expr) {
+                    PatKind::InlineConstant { subpattern, def } => {
+                        (subpattern.kind, None, Some(def))
                     }
-                    kind => (kind, None),
+                    PatKind::AscribeUserType { ascription, subpattern: box Pat { kind, .. } } => {
+                        (kind, Some(ascription), None)
+                    }
+                    kind => (kind, None, None),
                 };
                 let value = if let PatKind::Constant { value } = kind {
                     value
@@ -106,7 +113,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                     );
                     return Err(self.tcx.sess.delay_span_bug(expr.span, msg));
                 };
-                Ok((Some(value), ascr))
+                Ok((Some(value), ascr, inline_const))
             }
         }
     }
@@ -177,8 +184,8 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
             return Err(self.tcx.sess.delay_span_bug(span, msg));
         }
 
-        let (lo, lo_ascr) = self.lower_pattern_range_endpoint(lo_expr)?;
-        let (hi, hi_ascr) = self.lower_pattern_range_endpoint(hi_expr)?;
+        let (lo, lo_ascr, lo_inline) = self.lower_pattern_range_endpoint(lo_expr)?;
+        let (hi, hi_ascr, hi_inline) = self.lower_pattern_range_endpoint(hi_expr)?;
 
         let lo = lo.unwrap_or_else(|| {
             // Unwrap is ok because the type is known to be numeric.
@@ -237,6 +244,12 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                 };
             }
         }
+        for inline_const in [lo_inline, hi_inline] {
+            if let Some(def) = inline_const {
+                kind =
+                    PatKind::InlineConstant { def, subpattern: Box::new(Pat { span, ty, kind }) };
+            }
+        }
         Ok(kind)
     }
 
@@ -599,11 +612,9 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         // const eval path below.
         // FIXME: investigate the performance impact of removing this.
         let lit_input = match expr.kind {
-            hir::ExprKind::Lit(ref lit) => Some(LitToConstInput { lit: &lit.node, ty, neg: false }),
-            hir::ExprKind::Unary(hir::UnOp::Neg, ref expr) => match expr.kind {
-                hir::ExprKind::Lit(ref lit) => {
-                    Some(LitToConstInput { lit: &lit.node, ty, neg: true })
-                }
+            hir::ExprKind::Lit(lit) => Some(LitToConstInput { lit: &lit.node, ty, neg: false }),
+            hir::ExprKind::Unary(hir::UnOp::Neg, expr) => match expr.kind {
+                hir::ExprKind::Lit(lit) => Some(LitToConstInput { lit: &lit.node, ty, neg: true }),
                 _ => None,
             },
             _ => None,
@@ -633,13 +644,13 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         if let Ok(Some(valtree)) =
             self.tcx.const_eval_resolve_for_typeck(self.param_env, ct, Some(span))
         {
-            self.const_to_pat(
+            let subpattern = self.const_to_pat(
                 Const::Ty(ty::Const::new_value(self.tcx, valtree, ty)),
                 id,
                 span,
                 None,
-            )
-            .kind
+            );
+            PatKind::InlineConstant { subpattern, def: def_id }
         } else {
             // If that fails, convert it to an opaque constant pattern.
             match tcx.const_eval_resolve(self.param_env, uneval, Some(span)) {
@@ -822,6 +833,9 @@ impl<'tcx> PatternFoldable<'tcx> for PatKind<'tcx> {
                 PatKind::Deref { subpattern: subpattern.fold_with(folder) }
             }
             PatKind::Constant { value } => PatKind::Constant { value },
+            PatKind::InlineConstant { def, subpattern: ref pattern } => {
+                PatKind::InlineConstant { def, subpattern: pattern.fold_with(folder) }
+            }
             PatKind::Range(ref range) => PatKind::Range(range.clone()),
             PatKind::Slice { ref prefix, ref slice, ref suffix } => PatKind::Slice {
                 prefix: prefix.fold_with(folder),

compiler/rustc_mir_build/src/thir/print.rs

@@ -692,7 +692,7 @@ impl<'a, 'tcx> ThirPrinter<'a, 'tcx> {
             }
             PatKind::Deref { subpattern } => {
                 print_indented!(self, "Deref { ", depth_lvl + 1);
-                print_indented!(self, "subpattern: ", depth_lvl + 2);
+                print_indented!(self, "subpattern:", depth_lvl + 2);
                 self.print_pat(subpattern, depth_lvl + 2);
                 print_indented!(self, "}", depth_lvl + 1);
             }
@@ -701,6 +701,13 @@ impl<'a, 'tcx> ThirPrinter<'a, 'tcx> {
                 print_indented!(self, format!("value: {:?}", value), depth_lvl + 2);
                 print_indented!(self, "}", depth_lvl + 1);
             }
+            PatKind::InlineConstant { def, subpattern } => {
+                print_indented!(self, "InlineConstant {", depth_lvl + 1);
+                print_indented!(self, format!("def: {:?}", def), depth_lvl + 2);
+                print_indented!(self, "subpattern:", depth_lvl + 2);
+                self.print_pat(subpattern, depth_lvl + 2);
+                print_indented!(self, "}", depth_lvl + 1);
+            }
             PatKind::Range(pat_range) => {
                 print_indented!(self, format!("Range ( {:?} )", pat_range), depth_lvl + 1);
             }

compiler/rustc_mir_transform/src/dataflow_const_prop.rs

@@ -2,13 +2,13 @@
 //!
 //! Currently, this pass only propagates scalar values.
 
-use rustc_const_eval::interpret::{ImmTy, Immediate, InterpCx, OpTy, Projectable};
+use rustc_const_eval::interpret::{ImmTy, Immediate, InterpCx, OpTy, PlaceTy, Projectable};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def::DefKind;
 use rustc_middle::mir::interpret::{AllocId, ConstAllocation, InterpResult, Scalar};
 use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor};
 use rustc_middle::mir::*;
-use rustc_middle::ty::layout::TyAndLayout;
+use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_mir_dataflow::value_analysis::{
     Map, PlaceIndex, State, TrackElem, ValueAnalysis, ValueAnalysisWrapper, ValueOrPlace,
@@ -16,8 +16,9 @@ use rustc_mir_dataflow::value_analysis::{
 use rustc_mir_dataflow::{lattice::FlatSet, Analysis, Results, ResultsVisitor};
 use rustc_span::def_id::DefId;
 use rustc_span::DUMMY_SP;
-use rustc_target::abi::{FieldIdx, VariantIdx};
+use rustc_target::abi::{Abi, FieldIdx, Size, VariantIdx, FIRST_VARIANT};
 
+use crate::const_prop::throw_machine_stop_str;
 use crate::MirPass;
 
 // These constants are somewhat random guesses and have not been optimized.
@@ -553,18 +554,153 @@ impl<'tcx, 'locals> Collector<'tcx, 'locals> {
 
     fn try_make_constant(
         &self,
+        ecx: &mut InterpCx<'tcx, 'tcx, DummyMachine>,
         place: Place<'tcx>,
         state: &State<FlatSet<Scalar>>,
         map: &Map,
     ) -> Option<Const<'tcx>> {
-        let FlatSet::Elem(Scalar::Int(value)) = state.get(place.as_ref(), &map) else {
-            return None;
-        };
         let ty = place.ty(self.local_decls, self.patch.tcx).ty;
-        Some(Const::Val(ConstValue::Scalar(value.into()), ty))
+        let layout = ecx.layout_of(ty).ok()?;
+
+        if layout.is_zst() {
+            return Some(Const::zero_sized(ty));
+        }
+
+        if layout.is_unsized() {
+            return None;
+        }
+
+        let place = map.find(place.as_ref())?;
+        if layout.abi.is_scalar()
+            && let Some(value) = propagatable_scalar(place, state, map)
+        {
+            return Some(Const::Val(ConstValue::Scalar(value), ty));
+        }
+
+        if matches!(layout.abi, Abi::Scalar(..) | Abi::ScalarPair(..)) {
+            let alloc_id = ecx
+                .intern_with_temp_alloc(layout, |ecx, dest| {
+                    try_write_constant(ecx, dest, place, ty, state, map)
+                })
+                .ok()?;
+            return Some(Const::Val(ConstValue::Indirect { alloc_id, offset: Size::ZERO }, ty));
+        }
+
+        None
     }
 }
 
+fn propagatable_scalar(
+    place: PlaceIndex,
+    state: &State<FlatSet<Scalar>>,
+    map: &Map,
+) -> Option<Scalar> {
+    if let FlatSet::Elem(value) = state.get_idx(place, map) && value.try_to_int().is_ok() {
+        // Do not attempt to propagate pointers, as we may fail to preserve their identity.
+        Some(value)
+    } else {
+        None
+    }
+}
+
+#[instrument(level = "trace", skip(ecx, state, map))]
+fn try_write_constant<'tcx>(
+    ecx: &mut InterpCx<'_, 'tcx, DummyMachine>,
+    dest: &PlaceTy<'tcx>,
+    place: PlaceIndex,
+    ty: Ty<'tcx>,
+    state: &State<FlatSet<Scalar>>,
+    map: &Map,
+) -> InterpResult<'tcx> {
+    let layout = ecx.layout_of(ty)?;
+
+    // Fast path for ZSTs.
+    if layout.is_zst() {
+        return Ok(());
+    }
+
+    // Fast path for scalars.
+    if layout.abi.is_scalar()
+        && let Some(value) = propagatable_scalar(place, state, map)
+    {
+        return ecx.write_immediate(Immediate::Scalar(value), dest);
+    }
+
+    match ty.kind() {
+        // ZSTs. Nothing to do.
+        ty::FnDef(..) => {}
+
+        // Those are scalars, must be handled above.
+        ty::Bool | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Char => throw_machine_stop_str!("primitive type with provenance"),
+
+        ty::Tuple(elem_tys) => {
+            for (i, elem) in elem_tys.iter().enumerate() {
+                let Some(field) = map.apply(place, TrackElem::Field(FieldIdx::from_usize(i))) else {
+                    throw_machine_stop_str!("missing field in tuple")
+                };
+                let field_dest = ecx.project_field(dest, i)?;
+                try_write_constant(ecx, &field_dest, field, elem, state, map)?;
+            }
+        }
+
+        ty::Adt(def, args) => {
+            if def.is_union() {
+                throw_machine_stop_str!("cannot propagate unions")
+            }
+
+            let (variant_idx, variant_def, variant_place, variant_dest) = if def.is_enum() {
+                let Some(discr) = map.apply(place, TrackElem::Discriminant) else {
+                    throw_machine_stop_str!("missing discriminant for enum")
+                };
+                let FlatSet::Elem(Scalar::Int(discr)) = state.get_idx(discr, map) else {
+                    throw_machine_stop_str!("discriminant with provenance")
+                };
+                let discr_bits = discr.assert_bits(discr.size());
+                let Some((variant, _)) = def.discriminants(*ecx.tcx).find(|(_, var)| discr_bits == var.val) else {
+                    throw_machine_stop_str!("illegal discriminant for enum")
+                };
+                let Some(variant_place) = map.apply(place, TrackElem::Variant(variant)) else {
+                    throw_machine_stop_str!("missing variant for enum")
+                };
+                let variant_dest = ecx.project_downcast(dest, variant)?;
+                (variant, def.variant(variant), variant_place, variant_dest)
+            } else {
+                (FIRST_VARIANT, def.non_enum_variant(), place, dest.clone())
+            };
+
+            for (i, field) in variant_def.fields.iter_enumerated() {
+                let ty = field.ty(*ecx.tcx, args);
+                let Some(field) = map.apply(variant_place, TrackElem::Field(i)) else {
+                    throw_machine_stop_str!("missing field in ADT")
+                };
+                let field_dest = ecx.project_field(&variant_dest, i.as_usize())?;
+                try_write_constant(ecx, &field_dest, field, ty, state, map)?;
+            }
+            ecx.write_discriminant(variant_idx, dest)?;
+        }
+
+        // Unsupported for now.
+        ty::Array(_, _)
+
+        // Do not attempt to support indirection in constants.
+        | ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..) | ty::Str | ty::Slice(_)
+
+        | ty::Never
+        | ty::Foreign(..)
+        | ty::Alias(..)
+        | ty::Param(_)
+        | ty::Bound(..)
+        | ty::Placeholder(..)
+        | ty::Closure(..)
+        | ty::Coroutine(..)
+        | ty::Dynamic(..) => throw_machine_stop_str!("unsupported type"),
+
+        ty::Error(_) | ty::Infer(..) | ty::CoroutineWitness(..) => bug!(),
+    }
+
+    Ok(())
+}
+
 impl<'mir, 'tcx>
     ResultsVisitor<'mir, 'tcx, Results<'tcx, ValueAnalysisWrapper<ConstAnalysis<'_, 'tcx>>>>
     for Collector<'tcx, '_>
@@ -580,8 +716,13 @@ impl<'mir, 'tcx>
     ) {
         match &statement.kind {
             StatementKind::Assign(box (_, rvalue)) => {
-                OperandCollector { state, visitor: self, map: &results.analysis.0.map }
-                    .visit_rvalue(rvalue, location);
+                OperandCollector {
+                    state,
+                    visitor: self,
+                    ecx: &mut results.analysis.0.ecx,
+                    map: &results.analysis.0.map,
+                }
+                .visit_rvalue(rvalue, location);
             }
             _ => (),
         }
@@ -599,7 +740,12 @@ impl<'mir, 'tcx>
                 // Don't overwrite the assignment if it already uses a constant (to keep the span).
             }
             StatementKind::Assign(box (place, _)) => {
-                if let Some(value) = self.try_make_constant(place, state, &results.analysis.0.map) {
+                if let Some(value) = self.try_make_constant(
+                    &mut results.analysis.0.ecx,
+                    place,
+                    state,
+                    &results.analysis.0.map,
+                ) {
                     self.patch.assignments.insert(location, value);
                 }
             }
@@ -614,8 +760,13 @@ impl<'mir, 'tcx>
         terminator: &'mir Terminator<'tcx>,
         location: Location,
     ) {
-        OperandCollector { state, visitor: self, map: &results.analysis.0.map }
-            .visit_terminator(terminator, location);
+        OperandCollector {
+            state,
+            visitor: self,
+            ecx: &mut results.analysis.0.ecx,
+            map: &results.analysis.0.map,
+        }
+        .visit_terminator(terminator, location);
     }
 }
 
@@ -670,6 +821,7 @@ impl<'tcx> MutVisitor<'tcx> for Patch<'tcx> {
 struct OperandCollector<'tcx, 'map, 'locals, 'a> {
     state: &'a State<FlatSet<Scalar>>,
     visitor: &'a mut Collector<'tcx, 'locals>,
+    ecx: &'map mut InterpCx<'tcx, 'tcx, DummyMachine>,
     map: &'map Map,
 }
 
@@ -682,7 +834,7 @@ impl<'tcx> Visitor<'tcx> for OperandCollector<'tcx, '_, '_, '_> {
         location: Location,
     ) {
         if let PlaceElem::Index(local) = elem
-            && let Some(value) = self.visitor.try_make_constant(local.into(), self.state, self.map)
+            && let Some(value) = self.visitor.try_make_constant(self.ecx, local.into(), self.state, self.map)
         {
             self.visitor.patch.before_effect.insert((location, local.into()), value);
         }
@@ -690,7 +842,9 @@ impl<'tcx> Visitor<'tcx> for OperandCollector<'tcx, '_, '_, '_> {
 
     fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) {
         if let Some(place) = operand.place() {
-            if let Some(value) = self.visitor.try_make_constant(place, self.state, self.map) {
+            if let Some(value) =
+                self.visitor.try_make_constant(self.ecx, place, self.state, self.map)
+            {
                 self.visitor.patch.before_effect.insert((location, place), value);
             } else if !place.projection.is_empty() {
                 // Try to propagate into `Index` projections.
@@ -713,7 +867,7 @@ impl<'mir, 'tcx: 'mir> rustc_const_eval::interpret::Machine<'mir, 'tcx> for Dumm
     }
 
     fn enforce_validity(_ecx: &InterpCx<'mir, 'tcx, Self>, _layout: TyAndLayout<'tcx>) -> bool {
-        unimplemented!()
+        false
     }
 
     fn before_access_global(
@@ -725,13 +879,13 @@ impl<'mir, 'tcx: 'mir> rustc_const_eval::interpret::Machine<'mir, 'tcx> for Dumm
         is_write: bool,
     ) -> InterpResult<'tcx> {
         if is_write {
-            crate::const_prop::throw_machine_stop_str!("can't write to global");
+            throw_machine_stop_str!("can't write to global");
         }
 
         // If the static allocation is mutable, then we can't const prop it as its content
         // might be different at runtime.
         if alloc.inner().mutability.is_mut() {
-            crate::const_prop::throw_machine_stop_str!("can't access mutable globals in ConstProp");
+            throw_machine_stop_str!("can't access mutable globals in ConstProp");
         }
 
         Ok(())
@@ -781,7 +935,7 @@ impl<'mir, 'tcx: 'mir> rustc_const_eval::interpret::Machine<'mir, 'tcx> for Dumm
         _left: &rustc_const_eval::interpret::ImmTy<'tcx, Self::Provenance>,
         _right: &rustc_const_eval::interpret::ImmTy<'tcx, Self::Provenance>,
     ) -> interpret::InterpResult<'tcx, (ImmTy<'tcx, Self::Provenance>, bool)> {
-        crate::const_prop::throw_machine_stop_str!("can't do pointer arithmetic");
+        throw_machine_stop_str!("can't do pointer arithmetic");
     }
 
     fn expose_ptr(

compiler/rustc_session/src/config.rs

@@ -169,6 +169,9 @@ pub enum MirSpanview {
 pub enum InstrumentCoverage {
     /// Default `-C instrument-coverage` or `-C instrument-coverage=statement`
     All,
+    /// Additionally, instrument branches and output branch coverage.
+    /// `-Zunstable-options -C instrument-coverage=branch`
+    Branch,
     /// `-Zunstable-options -C instrument-coverage=except-unused-generics`
     ExceptUnusedGenerics,
     /// `-Zunstable-options -C instrument-coverage=except-unused-functions`
@@ -2747,7 +2750,10 @@ pub fn build_session_options(
         }
         (Some(InstrumentCoverage::Off | InstrumentCoverage::All), _) => {}
         (Some(_), _) if !unstable_opts.unstable_options => {
-            handler.early_error("`-C instrument-coverage=except-*` requires `-Z unstable-options`");
+            handler.early_error(
+                "`-C instrument-coverage=branch` and `-C instrument-coverage=except-*` \
+                require `-Z unstable-options`",
+            );
         }
         (None, None) => {}
         (None, ic) => {

compiler/rustc_session/src/options.rs

@@ -389,7 +389,7 @@ mod desc {
     pub const parse_mir_spanview: &str = "`statement` (default), `terminator`, or `block`";
     pub const parse_dump_mono_stats: &str = "`markdown` (default) or `json`";
     pub const parse_instrument_coverage: &str =
-        "`all` (default), `except-unused-generics`, `except-unused-functions`, or `off`";
+        "`all` (default), `branch`, `except-unused-generics`, `except-unused-functions`, or `off`";
     pub const parse_instrument_xray: &str = "either a boolean (`yes`, `no`, `on`, `off`, etc), or a comma separated list of settings: `always` or `never` (mutually exclusive), `ignore-loops`, `instruction-threshold=N`, `skip-entry`, `skip-exit`";
     pub const parse_unpretty: &str = "`string` or `string=string`";
     pub const parse_treat_err_as_bug: &str = "either no value or a non-negative number";
@@ -931,6 +931,7 @@ mod parse {
 
         *slot = Some(match v {
             "all" => InstrumentCoverage::All,
+            "branch" => InstrumentCoverage::Branch,
             "except-unused-generics" | "except_unused_generics" => {
                 InstrumentCoverage::ExceptUnusedGenerics
             }
@@ -1356,6 +1357,7 @@ options! {
         reports (note, the compiler build config must include `profiler = true`); \
         implies `-C symbol-mangling-version=v0`. Optional values are:
         `=all` (implicit value)
+        `=branch`
         `=except-unused-generics`
         `=except-unused-functions`
         `=off` (default)"),
@@ -1597,6 +1599,7 @@ options! {
         reports (note, the compiler build config must include `profiler = true`); \
         implies `-C symbol-mangling-version=v0`. Optional values are:
         `=all` (implicit value)
+        `=branch`
         `=except-unused-generics`
         `=except-unused-functions`
         `=off` (default)"),

compiler/rustc_session/src/session.rs

@@ -702,6 +702,10 @@ impl Session {
         self.opts.cg.instrument_coverage() != InstrumentCoverage::Off
     }
 
+    pub fn instrument_coverage_branch(&self) -> bool {
+        self.opts.cg.instrument_coverage() == InstrumentCoverage::Branch
+    }
+
     pub fn instrument_coverage_except_unused_generics(&self) -> bool {
         self.opts.cg.instrument_coverage() == InstrumentCoverage::ExceptUnusedGenerics
     }

compiler/rustc_smir/Cargo.toml

@@ -9,6 +9,7 @@ rustc_hir = { path = "../rustc_hir" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
+scoped-tls = "1.0"
 stable_mir = {path = "../stable_mir" }
 tracing = "0.1"
 

compiler/rustc_smir/src/rustc_internal/mod.rs

@@ -3,7 +3,7 @@
 //! For that, we define APIs that will temporarily be public to 3P that exposes rustc internal APIs
 //! until stable MIR is complete.
 
-use crate::rustc_smir::Tables;
+use crate::rustc_smir::{Stable, Tables, TablesWrapper};
 use rustc_data_structures::fx;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_middle::mir::interpret::AllocId;
@@ -11,13 +11,24 @@ use rustc_middle::ty;
 use rustc_middle::ty::TyCtxt;
 use rustc_span::def_id::{CrateNum, DefId};
 use rustc_span::Span;
+use scoped_tls::scoped_thread_local;
 use stable_mir::ty::IndexedVal;
+use std::cell::Cell;
+use std::cell::RefCell;
 use std::fmt::Debug;
 use std::hash::Hash;
 use std::ops::Index;
 
 mod internal;
 
+pub fn stable<'tcx, S: Stable<'tcx>>(item: &S) -> S::T {
+    with_tables(|tables| item.stable(tables))
+}
+
+pub fn internal<'tcx, S: RustcInternal<'tcx>>(item: &S) -> S::T {
+    with_tables(|tables| item.internal(tables))
+}
+
 impl<'tcx> Index<stable_mir::DefId> for Tables<'tcx> {
     type Output = DefId;
 
@@ -125,18 +136,41 @@ pub fn crate_num(item: &stable_mir::Crate) -> CrateNum {
     item.id.into()
 }
 
+// A thread local variable that stores a pointer to the tables mapping between TyCtxt
+// datastructures and stable MIR datastructures
+scoped_thread_local! (static TLV: Cell<*const ()>);
+
+pub(crate) fn init<'tcx>(tables: &TablesWrapper<'tcx>, f: impl FnOnce()) {
+    assert!(!TLV.is_set());
+    let ptr = tables as *const _ as *const ();
+    TLV.set(&Cell::new(ptr), || {
+        f();
+    });
+}
+
+/// Loads the current context and calls a function with it.
+/// Do not nest these, as that will ICE.
+pub(crate) fn with_tables<'tcx, R>(f: impl FnOnce(&mut Tables<'tcx>) -> R) -> R {
+    assert!(TLV.is_set());
+    TLV.with(|tlv| {
+        let ptr = tlv.get();
+        assert!(!ptr.is_null());
+        let wrapper = ptr as *const TablesWrapper<'tcx>;
+        let mut tables = unsafe { (*wrapper).0.borrow_mut() };
+        f(&mut *tables)
+    })
+}
+
 pub fn run(tcx: TyCtxt<'_>, f: impl FnOnce()) {
-    stable_mir::run(
-        Tables {
-            tcx,
-            def_ids: IndexMap::default(),
-            alloc_ids: IndexMap::default(),
-            spans: IndexMap::default(),
-            types: vec![],
-            instances: IndexMap::default(),
-        },
-        f,
-    );
+    let tables = TablesWrapper(RefCell::new(Tables {
+        tcx,
+        def_ids: IndexMap::default(),
+        alloc_ids: IndexMap::default(),
+        spans: IndexMap::default(),
+        types: vec![],
+        instances: IndexMap::default(),
+    }));
+    stable_mir::run(&tables, || init(&tables, f));
 }
 
 #[macro_export]
@@ -251,7 +285,7 @@ impl<K: PartialEq + Hash + Eq, V: Copy + Debug + PartialEq + IndexedVal> Index<V
 /// Trait used to translate a stable construct to its rustc counterpart.
 ///
 /// This is basically a mirror of [crate::rustc_smir::Stable].
-pub(crate) trait RustcInternal<'tcx> {
+pub trait RustcInternal<'tcx> {
     type T;
     fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T;
 }

compiler/rustc_smir/src/rustc_smir/mod.rs

@@ -23,27 +23,31 @@ use stable_mir::ty::{
     FloatTy, GenericParamDef, IntTy, LineInfo, Movability, RigidTy, Span, TyKind, UintTy,
 };
 use stable_mir::{self, opaque, Context, Filename};
+use std::cell::RefCell;
 use tracing::debug;
 
 mod alloc;
 mod builder;
 
-impl<'tcx> Context for Tables<'tcx> {
+impl<'tcx> Context for TablesWrapper<'tcx> {
     fn local_crate(&self) -> stable_mir::Crate {
-        smir_crate(self.tcx, LOCAL_CRATE)
+        let tables = self.0.borrow();
+        smir_crate(tables.tcx, LOCAL_CRATE)
     }
 
     fn external_crates(&self) -> Vec<stable_mir::Crate> {
-        self.tcx.crates(()).iter().map(|crate_num| smir_crate(self.tcx, *crate_num)).collect()
+        let tables = self.0.borrow();
+        tables.tcx.crates(()).iter().map(|crate_num| smir_crate(tables.tcx, *crate_num)).collect()
     }
 
     fn find_crates(&self, name: &str) -> Vec<stable_mir::Crate> {
+        let tables = self.0.borrow();
         let crates: Vec<stable_mir::Crate> = [LOCAL_CRATE]
             .iter()
-            .chain(self.tcx.crates(()).iter())
+            .chain(tables.tcx.crates(()).iter())
             .map(|crate_num| {
-                let crate_name = self.tcx.crate_name(*crate_num).to_string();
-                (name == crate_name).then(|| smir_crate(self.tcx, *crate_num))
+                let crate_name = tables.tcx.crate_name(*crate_num).to_string();
+                (name == crate_name).then(|| smir_crate(tables.tcx, *crate_num))
             })
             .into_iter()
             .filter_map(|c| c)
@@ -52,163 +56,197 @@ impl<'tcx> Context for Tables<'tcx> {
     }
 
     fn name_of_def_id(&self, def_id: stable_mir::DefId) -> String {
-        self.tcx.def_path_str(self[def_id])
+        let tables = self.0.borrow();
+        tables.tcx.def_path_str(tables[def_id])
     }
 
     fn span_to_string(&self, span: stable_mir::ty::Span) -> String {
-        self.tcx.sess.source_map().span_to_diagnostic_string(self[span])
+        let tables = self.0.borrow();
+        tables.tcx.sess.source_map().span_to_diagnostic_string(tables[span])
     }
 
     fn get_filename(&self, span: &Span) -> Filename {
+        let tables = self.0.borrow();
         opaque(
-            &self
+            &tables
                 .tcx
                 .sess
                 .source_map()
-                .span_to_filename(self[*span])
+                .span_to_filename(tables[*span])
                 .display(rustc_span::FileNameDisplayPreference::Local)
                 .to_string(),
         )
     }
 
     fn get_lines(&self, span: &Span) -> LineInfo {
-        let lines = &self.tcx.sess.source_map().span_to_location_info(self[*span]);
+        let tables = self.0.borrow();
+        let lines = &tables.tcx.sess.source_map().span_to_location_info(tables[*span]);
         LineInfo { start_line: lines.1, start_col: lines.2, end_line: lines.3, end_col: lines.4 }
     }
 
-    fn def_kind(&mut self, def_id: stable_mir::DefId) -> stable_mir::DefKind {
-        self.tcx.def_kind(self[def_id]).stable(self)
+    fn def_kind(&self, def_id: stable_mir::DefId) -> stable_mir::DefKind {
+        let mut tables = self.0.borrow_mut();
+        tables.tcx.def_kind(tables[def_id]).stable(&mut *tables)
     }
 
-    fn span_of_an_item(&mut self, def_id: stable_mir::DefId) -> Span {
-        self.tcx.def_span(self[def_id]).stable(self)
+    fn span_of_an_item(&self, def_id: stable_mir::DefId) -> Span {
+        let mut tables = self.0.borrow_mut();
+        tables.tcx.def_span(tables[def_id]).stable(&mut *tables)
     }
 
-    fn all_local_items(&mut self) -> stable_mir::CrateItems {
-        self.tcx.mir_keys(()).iter().map(|item| self.crate_item(item.to_def_id())).collect()
+    fn all_local_items(&self) -> stable_mir::CrateItems {
+        let mut tables = self.0.borrow_mut();
+        tables.tcx.mir_keys(()).iter().map(|item| tables.crate_item(item.to_def_id())).collect()
     }
 
-    fn entry_fn(&mut self) -> Option<stable_mir::CrateItem> {
-        Some(self.crate_item(self.tcx.entry_fn(())?.0))
+    fn entry_fn(&self) -> Option<stable_mir::CrateItem> {
+        let mut tables = self.0.borrow_mut();
+        let tcx = tables.tcx;
+        Some(tables.crate_item(tcx.entry_fn(())?.0))
     }
 
-    fn all_trait_decls(&mut self) -> stable_mir::TraitDecls {
-        self.tcx
+    fn all_trait_decls(&self) -> stable_mir::TraitDecls {
+        let mut tables = self.0.borrow_mut();
+        tables
+            .tcx
             .traits(LOCAL_CRATE)
             .iter()
-            .map(|trait_def_id| self.trait_def(*trait_def_id))
+            .map(|trait_def_id| tables.trait_def(*trait_def_id))
             .collect()
     }
 
-    fn trait_decl(&mut self, trait_def: &stable_mir::ty::TraitDef) -> stable_mir::ty::TraitDecl {
-        let def_id = self[trait_def.0];
-        let trait_def = self.tcx.trait_def(def_id);
-        trait_def.stable(self)
+    fn trait_decl(&self, trait_def: &stable_mir::ty::TraitDef) -> stable_mir::ty::TraitDecl {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[trait_def.0];
+        let trait_def = tables.tcx.trait_def(def_id);
+        trait_def.stable(&mut *tables)
     }
 
-    fn all_trait_impls(&mut self) -> stable_mir::ImplTraitDecls {
-        self.tcx
+    fn all_trait_impls(&self) -> stable_mir::ImplTraitDecls {
+        let mut tables = self.0.borrow_mut();
+        tables
+            .tcx
             .trait_impls_in_crate(LOCAL_CRATE)
             .iter()
-            .map(|impl_def_id| self.impl_def(*impl_def_id))
+            .map(|impl_def_id| tables.impl_def(*impl_def_id))
             .collect()
     }
 
-    fn trait_impl(&mut self, impl_def: &stable_mir::ty::ImplDef) -> stable_mir::ty::ImplTrait {
-        let def_id = self[impl_def.0];
-        let impl_trait = self.tcx.impl_trait_ref(def_id).unwrap();
-        impl_trait.stable(self)
+    fn trait_impl(&self, impl_def: &stable_mir::ty::ImplDef) -> stable_mir::ty::ImplTrait {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[impl_def.0];
+        let impl_trait = tables.tcx.impl_trait_ref(def_id).unwrap();
+        impl_trait.stable(&mut *tables)
     }
 
-    fn mir_body(&mut self, item: stable_mir::DefId) -> stable_mir::mir::Body {
-        let def_id = self[item];
-        self.tcx.instance_mir(ty::InstanceDef::Item(def_id)).stable(self)
+    fn mir_body(&self, item: stable_mir::DefId) -> stable_mir::mir::Body {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[item];
+        tables.tcx.instance_mir(ty::InstanceDef::Item(def_id)).stable(&mut tables)
     }
 
-    fn ty_kind(&mut self, ty: stable_mir::ty::Ty) -> TyKind {
-        self.types[ty.0].clone().stable(self)
+    fn ty_kind(&self, ty: stable_mir::ty::Ty) -> TyKind {
+        let mut tables = self.0.borrow_mut();
+        tables.types[ty.0].clone().stable(&mut *tables)
     }
 
-    fn mk_ty(&mut self, kind: TyKind) -> stable_mir::ty::Ty {
-        let n = self.types.len();
-        self.types.push(MaybeStable::Stable(kind));
+    fn mk_ty(&self, kind: TyKind) -> stable_mir::ty::Ty {
+        let mut tables = self.0.borrow_mut();
+        let n = tables.types.len();
+        tables.types.push(MaybeStable::Stable(kind));
         stable_mir::ty::Ty(n)
     }
 
-    fn generics_of(&mut self, def_id: stable_mir::DefId) -> stable_mir::ty::Generics {
-        let def_id = self[def_id];
-        let generics = self.tcx.generics_of(def_id);
-        generics.stable(self)
+    fn generics_of(&self, def_id: stable_mir::DefId) -> stable_mir::ty::Generics {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[def_id];
+        let generics = tables.tcx.generics_of(def_id);
+        generics.stable(&mut *tables)
     }
 
-    fn predicates_of(&mut self, def_id: stable_mir::DefId) -> stable_mir::ty::GenericPredicates {
-        let def_id = self[def_id];
-        let ty::GenericPredicates { parent, predicates } = self.tcx.predicates_of(def_id);
+    fn predicates_of(&self, def_id: stable_mir::DefId) -> stable_mir::ty::GenericPredicates {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[def_id];
+        let ty::GenericPredicates { parent, predicates } = tables.tcx.predicates_of(def_id);
         stable_mir::ty::GenericPredicates {
-            parent: parent.map(|did| self.trait_def(did)),
+            parent: parent.map(|did| tables.trait_def(did)),
             predicates: predicates
                 .iter()
                 .map(|(clause, span)| {
-                    (clause.as_predicate().kind().skip_binder().stable(self), span.stable(self))
+                    (
+                        clause.as_predicate().kind().skip_binder().stable(&mut *tables),
+                        span.stable(&mut *tables),
+                    )
                 })
                 .collect(),
         }
     }
 
     fn explicit_predicates_of(
-        &mut self,
+        &self,
         def_id: stable_mir::DefId,
     ) -> stable_mir::ty::GenericPredicates {
-        let def_id = self[def_id];
-        let ty::GenericPredicates { parent, predicates } = self.tcx.explicit_predicates_of(def_id);
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[def_id];
+        let ty::GenericPredicates { parent, predicates } =
+            tables.tcx.explicit_predicates_of(def_id);
         stable_mir::ty::GenericPredicates {
-            parent: parent.map(|did| self.trait_def(did)),
+            parent: parent.map(|did| tables.trait_def(did)),
             predicates: predicates
                 .iter()
                 .map(|(clause, span)| {
-                    (clause.as_predicate().kind().skip_binder().stable(self), span.stable(self))
+                    (
+                        clause.as_predicate().kind().skip_binder().stable(&mut *tables),
+                        span.stable(&mut *tables),
+                    )
                 })
                 .collect(),
         }
     }
 
-    fn instance_body(&mut self, def: InstanceDef) -> Body {
-        let instance = self.instances[def];
-        builder::BodyBuilder::new(self.tcx, instance).build(self)
+    fn instance_body(&self, def: InstanceDef) -> Body {
+        let mut tables = self.0.borrow_mut();
+        let instance = tables.instances[def];
+        builder::BodyBuilder::new(tables.tcx, instance).build(&mut *tables)
     }
 
-    fn instance_ty(&mut self, def: InstanceDef) -> stable_mir::ty::Ty {
-        let instance = self.instances[def];
-        let ty = instance.ty(self.tcx, ParamEnv::empty());
-        self.intern_ty(ty)
+    fn instance_ty(&self, def: InstanceDef) -> stable_mir::ty::Ty {
+        let mut tables = self.0.borrow_mut();
+        let instance = tables.instances[def];
+        let ty = instance.ty(tables.tcx, ParamEnv::empty());
+        tables.intern_ty(ty)
     }
 
-    fn instance_def_id(&mut self, def: InstanceDef) -> stable_mir::DefId {
-        let def_id = self.instances[def].def_id();
-        self.create_def_id(def_id)
+    fn instance_def_id(&self, def: InstanceDef) -> stable_mir::DefId {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables.instances[def].def_id();
+        tables.create_def_id(def_id)
     }
 
-    fn mono_instance(&mut self, item: stable_mir::CrateItem) -> stable_mir::mir::mono::Instance {
-        let def_id = self[item.0];
-        Instance::mono(self.tcx, def_id).stable(self)
+    fn mono_instance(&self, item: stable_mir::CrateItem) -> stable_mir::mir::mono::Instance {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[item.0];
+        Instance::mono(tables.tcx, def_id).stable(&mut *tables)
     }
 
     fn requires_monomorphization(&self, def_id: stable_mir::DefId) -> bool {
-        let def_id = self[def_id];
-        let generics = self.tcx.generics_of(def_id);
-        let result = generics.requires_monomorphization(self.tcx);
+        let tables = self.0.borrow();
+        let def_id = tables[def_id];
+        let generics = tables.tcx.generics_of(def_id);
+        let result = generics.requires_monomorphization(tables.tcx);
         result
     }
 
     fn resolve_instance(
-        &mut self,
+        &self,
         def: stable_mir::ty::FnDef,
         args: &stable_mir::ty::GenericArgs,
     ) -> Option<stable_mir::mir::mono::Instance> {
-        let def_id = def.0.internal(self);
-        let args_ref = args.internal(self);
-        match Instance::resolve(self.tcx, ParamEnv::reveal_all(), def_id, args_ref) {
-            Ok(Some(instance)) => Some(instance.stable(self)),
+        let mut tables = self.0.borrow_mut();
+        let def_id = def.0.internal(&mut *tables);
+        let args_ref = args.internal(&mut *tables);
+        match Instance::resolve(tables.tcx, ParamEnv::reveal_all(), def_id, args_ref) {
+            Ok(Some(instance)) => Some(instance.stable(&mut *tables)),
             Ok(None) | Err(_) => None,
         }
     }
@@ -241,13 +279,15 @@ impl<S, R: PartialEq> PartialEq<R> for MaybeStable<S, R> {
     }
 }
 
+pub(crate) struct TablesWrapper<'tcx>(pub(crate) RefCell<Tables<'tcx>>);
+
 pub struct Tables<'tcx> {
-    pub tcx: TyCtxt<'tcx>,
-    pub def_ids: IndexMap<DefId, stable_mir::DefId>,
-    pub alloc_ids: IndexMap<AllocId, stable_mir::AllocId>,
-    pub spans: IndexMap<rustc_span::Span, Span>,
-    pub types: Vec<MaybeStable<TyKind, Ty<'tcx>>>,
-    pub instances: IndexMap<ty::Instance<'tcx>, InstanceDef>,
+    pub(crate) tcx: TyCtxt<'tcx>,
+    pub(crate) def_ids: IndexMap<DefId, stable_mir::DefId>,
+    pub(crate) alloc_ids: IndexMap<AllocId, stable_mir::AllocId>,
+    pub(crate) spans: IndexMap<rustc_span::Span, Span>,
+    pub(crate) types: Vec<MaybeStable<TyKind, Ty<'tcx>>>,
+    pub(crate) instances: IndexMap<ty::Instance<'tcx>, InstanceDef>,
 }
 
 impl<'tcx> Tables<'tcx> {
@@ -270,7 +310,7 @@ fn smir_crate(tcx: TyCtxt<'_>, crate_num: CrateNum) -> stable_mir::Crate {
 }
 
 /// Trait used to convert between an internal MIR type to a Stable MIR type.
-pub(crate) trait Stable<'tcx> {
+pub trait Stable<'tcx> {
     /// The stable representation of the type implementing Stable.
     type T;
     /// Converts an object to the equivalent Stable MIR representation.

compiler/rustc_span/src/symbol.rs

@@ -957,6 +957,7 @@ symbols! {
         log_syntax,
         logf32,
         logf64,
+        loongarch_target_feature,
         loop_break_value,
         lt,
         macro_at_most_once_rep,
@@ -1369,6 +1370,7 @@ symbols! {
         rustc_evaluate_where_clauses,
         rustc_expected_cgu_reuse,
         rustc_has_incoherent_inherent_impls,
+        rustc_hidden_type_of_opaques,
         rustc_host,
         rustc_if_this_changed,
         rustc_inherit_overflow_checks,

compiler/rustc_trait_selection/src/solve/canonicalize.rs

@@ -224,12 +224,20 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> {
         let kind = match *r {
             ty::ReLateBound(..) => return r,
 
-            ty::ReStatic => match self.canonicalize_mode {
+            // We may encounter `ReStatic` in item signatures or the hidden type
+            // of an opaque. `ReErased` should only be encountered in the hidden
+            // type of an opaque for regions that are ignored for the purposes of
+            // captures.
+            //
+            // FIXME: We should investigate the perf implications of not uniquifying
+            // `ReErased`. We may be able to short-circuit registering region
+            // obligations if we encounter a `ReErased` on one side, for example.
+            ty::ReStatic | ty::ReErased => match self.canonicalize_mode {
                 CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
                 CanonicalizeMode::Response { .. } => return r,
             },
 
-            ty::ReErased | ty::ReFree(_) | ty::ReEarlyBound(_) => match self.canonicalize_mode {
+            ty::ReFree(_) | ty::ReEarlyBound(_) => match self.canonicalize_mode {
                 CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
                 CanonicalizeMode::Response { .. } => bug!("unexpected region in response: {r:?}"),
             },

compiler/rustc_trait_selection/src/traits/fulfill.rs

@@ -1,4 +1,5 @@
 use crate::infer::{InferCtxt, TyOrConstInferVar};
+use rustc_data_structures::captures::Captures;
 use rustc_data_structures::obligation_forest::ProcessResult;
 use rustc_data_structures::obligation_forest::{Error, ForestObligation, Outcome};
 use rustc_data_structures::obligation_forest::{ObligationForest, ObligationProcessor};
@@ -68,7 +69,7 @@ pub struct PendingPredicateObligation<'tcx> {
     // should mostly optimize for reading speed, while modifying is not as relevant.
     //
     // For whatever reason using a boxed slice is slower than using a `Vec` here.
-    pub stalled_on: Vec<TyOrConstInferVar<'tcx>>,
+    pub stalled_on: Vec<TyOrConstInferVar>,
 }
 
 // `PendingPredicateObligation` is used a lot. Make sure it doesn't unintentionally get bigger.
@@ -669,7 +670,7 @@ impl<'a, 'tcx> FulfillProcessor<'a, 'tcx> {
         &mut self,
         obligation: &PredicateObligation<'tcx>,
         trait_obligation: PolyTraitObligation<'tcx>,
-        stalled_on: &mut Vec<TyOrConstInferVar<'tcx>>,
+        stalled_on: &mut Vec<TyOrConstInferVar>,
     ) -> ProcessResult<PendingPredicateObligation<'tcx>, FulfillmentErrorCode<'tcx>> {
         let infcx = self.selcx.infcx;
         if obligation.predicate.is_global() && !self.selcx.is_intercrate() {
@@ -722,7 +723,7 @@ impl<'a, 'tcx> FulfillProcessor<'a, 'tcx> {
         &mut self,
         obligation: &PredicateObligation<'tcx>,
         project_obligation: PolyProjectionObligation<'tcx>,
-        stalled_on: &mut Vec<TyOrConstInferVar<'tcx>>,
+        stalled_on: &mut Vec<TyOrConstInferVar>,
     ) -> ProcessResult<PendingPredicateObligation<'tcx>, FulfillmentErrorCode<'tcx>> {
         let tcx = self.selcx.tcx();
 
@@ -775,7 +776,7 @@ impl<'a, 'tcx> FulfillProcessor<'a, 'tcx> {
 fn args_infer_vars<'a, 'tcx>(
     selcx: &SelectionContext<'a, 'tcx>,
     args: ty::Binder<'tcx, GenericArgsRef<'tcx>>,
-) -> impl Iterator<Item = TyOrConstInferVar<'tcx>> {
+) -> impl Iterator<Item = TyOrConstInferVar> + Captures<'tcx> {
     selcx
         .infcx
         .resolve_vars_if_possible(args)

compiler/rustc_type_ir/src/const_kind.rs

@@ -5,8 +5,8 @@ use std::fmt;
 use std::hash;
 
 use crate::{
-    DebruijnIndex, DebugWithInfcx, HashStableContext, InferCtxtLike, Interner, OptWithInfcx,
-    TyDecoder, TyEncoder,
+    DebruijnIndex, DebugWithInfcx, HashStableContext, InferCtxtLike, Interner, TyDecoder,
+    TyEncoder, WithInfcx,
 };
 
 use self::ConstKind::*;
@@ -231,13 +231,13 @@ impl<I: Interner> Clone for ConstKind<I> {
 
 impl<I: Interner> fmt::Debug for ConstKind<I> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        OptWithInfcx::new_no_ctx(self).fmt(f)
+        WithInfcx::with_no_infcx(self).fmt(f)
     }
 }
 
 impl<I: Interner> DebugWithInfcx<I> for ConstKind<I> {
-    fn fmt<InfCtx: InferCtxtLike<I>>(
-        this: OptWithInfcx<'_, I, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = I>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         use ConstKind::*;

compiler/rustc_type_ir/src/debug.rs

@@ -3,38 +3,48 @@ use crate::{Interner, UniverseIndex};
 use core::fmt;
 use std::marker::PhantomData;
 
-pub trait InferCtxtLike<I: Interner> {
-    fn universe_of_ty(&self, ty: I::InferTy) -> Option<UniverseIndex>;
+pub trait InferCtxtLike {
+    type Interner: Interner;
 
-    fn universe_of_lt(&self, lt: I::InferRegion) -> Option<UniverseIndex>;
+    fn universe_of_ty(&self, ty: <Self::Interner as Interner>::InferTy) -> Option<UniverseIndex>;
 
-    fn universe_of_ct(&self, ct: I::InferConst) -> Option<UniverseIndex>;
+    fn universe_of_lt(
+        &self,
+        lt: <Self::Interner as Interner>::InferRegion,
+    ) -> Option<UniverseIndex>;
+
+    fn universe_of_ct(&self, ct: <Self::Interner as Interner>::InferConst)
+    -> Option<UniverseIndex>;
 }
 
-impl<I: Interner> InferCtxtLike<I> for core::convert::Infallible {
+pub struct NoInfcx<I>(PhantomData<I>);
+
+impl<I: Interner> InferCtxtLike for NoInfcx<I> {
+    type Interner = I;
+
     fn universe_of_ty(&self, _ty: <I as Interner>::InferTy) -> Option<UniverseIndex> {
-        match *self {}
+        None
     }
 
     fn universe_of_ct(&self, _ct: <I as Interner>::InferConst) -> Option<UniverseIndex> {
-        match *self {}
+        None
     }
 
     fn universe_of_lt(&self, _lt: <I as Interner>::InferRegion) -> Option<UniverseIndex> {
-        match *self {}
+        None
     }
 }
 
 pub trait DebugWithInfcx<I: Interner>: fmt::Debug {
-    fn fmt<InfCtx: InferCtxtLike<I>>(
-        this: OptWithInfcx<'_, I, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = I>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut fmt::Formatter<'_>,
     ) -> fmt::Result;
 }
 
 impl<I: Interner, T: DebugWithInfcx<I> + ?Sized> DebugWithInfcx<I> for &'_ T {
-    fn fmt<InfCtx: InferCtxtLike<I>>(
-        this: OptWithInfcx<'_, I, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = I>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut fmt::Formatter<'_>,
     ) -> fmt::Result {
         <T as DebugWithInfcx<I>>::fmt(this.map(|&data| data), f)
@@ -42,8 +52,8 @@ impl<I: Interner, T: DebugWithInfcx<I> + ?Sized> DebugWithInfcx<I> for &'_ T {
 }
 
 impl<I: Interner, T: DebugWithInfcx<I>> DebugWithInfcx<I> for [T] {
-    fn fmt<InfCtx: InferCtxtLike<I>>(
-        this: OptWithInfcx<'_, I, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = I>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut fmt::Formatter<'_>,
     ) -> fmt::Result {
         match f.alternate() {
@@ -70,46 +80,45 @@ impl<I: Interner, T: DebugWithInfcx<I>> DebugWithInfcx<I> for [T] {
     }
 }
 
-pub struct OptWithInfcx<'a, I: Interner, InfCtx: InferCtxtLike<I>, T> {
+pub struct WithInfcx<'a, Infcx: InferCtxtLike, T> {
     pub data: T,
-    pub infcx: Option<&'a InfCtx>,
-    _interner: PhantomData<I>,
+    pub infcx: &'a Infcx,
 }
 
-impl<I: Interner, InfCtx: InferCtxtLike<I>, T: Copy> Copy for OptWithInfcx<'_, I, InfCtx, T> {}
+impl<Infcx: InferCtxtLike, T: Copy> Copy for WithInfcx<'_, Infcx, T> {}
 
-impl<I: Interner, InfCtx: InferCtxtLike<I>, T: Clone> Clone for OptWithInfcx<'_, I, InfCtx, T> {
+impl<Infcx: InferCtxtLike, T: Clone> Clone for WithInfcx<'_, Infcx, T> {
     fn clone(&self) -> Self {
-        Self { data: self.data.clone(), infcx: self.infcx, _interner: self._interner }
+        Self { data: self.data.clone(), infcx: self.infcx }
     }
 }
 
-impl<'a, I: Interner, T> OptWithInfcx<'a, I, core::convert::Infallible, T> {
-    pub fn new_no_ctx(data: T) -> Self {
-        Self { data, infcx: None, _interner: PhantomData }
+impl<'a, I: Interner, T> WithInfcx<'a, NoInfcx<I>, T> {
+    pub fn with_no_infcx(data: T) -> Self {
+        Self { data, infcx: &NoInfcx(PhantomData) }
     }
 }
 
-impl<'a, I: Interner, InfCtx: InferCtxtLike<I>, T> OptWithInfcx<'a, I, InfCtx, T> {
-    pub fn new(data: T, infcx: &'a InfCtx) -> Self {
-        Self { data, infcx: Some(infcx), _interner: PhantomData }
+impl<'a, Infcx: InferCtxtLike, T> WithInfcx<'a, Infcx, T> {
+    pub fn new(data: T, infcx: &'a Infcx) -> Self {
+        Self { data, infcx }
     }
 
-    pub fn wrap<U>(self, u: U) -> OptWithInfcx<'a, I, InfCtx, U> {
-        OptWithInfcx { data: u, infcx: self.infcx, _interner: PhantomData }
+    pub fn wrap<U>(self, u: U) -> WithInfcx<'a, Infcx, U> {
+        WithInfcx { data: u, infcx: self.infcx }
     }
 
-    pub fn map<U>(self, f: impl FnOnce(T) -> U) -> OptWithInfcx<'a, I, InfCtx, U> {
-        OptWithInfcx { data: f(self.data), infcx: self.infcx, _interner: PhantomData }
+    pub fn map<U>(self, f: impl FnOnce(T) -> U) -> WithInfcx<'a, Infcx, U> {
+        WithInfcx { data: f(self.data), infcx: self.infcx }
     }
 
-    pub fn as_ref(&self) -> OptWithInfcx<'a, I, InfCtx, &T> {
-        OptWithInfcx { data: &self.data, infcx: self.infcx, _interner: PhantomData }
+    pub fn as_ref(&self) -> WithInfcx<'a, Infcx, &T> {
+        WithInfcx { data: &self.data, infcx: self.infcx }
     }
 }
 
-impl<I: Interner, InfCtx: InferCtxtLike<I>, T: DebugWithInfcx<I>> fmt::Debug
-    for OptWithInfcx<'_, I, InfCtx, T>
+impl<Infcx: InferCtxtLike, T: DebugWithInfcx<Infcx::Interner>> fmt::Debug
+    for WithInfcx<'_, Infcx, T>
 {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         DebugWithInfcx::fmt(self.as_ref(), f)

compiler/rustc_type_ir/src/lib.rs

@@ -34,7 +34,7 @@ mod region_kind;
 
 pub use codec::*;
 pub use const_kind::*;
-pub use debug::{DebugWithInfcx, InferCtxtLike, OptWithInfcx};
+pub use debug::{DebugWithInfcx, InferCtxtLike, WithInfcx};
 pub use flags::*;
 pub use interner::*;
 pub use region_kind::*;

compiler/rustc_type_ir/src/region_kind.rs

@@ -5,8 +5,8 @@ use std::fmt;
 use std::hash;
 
 use crate::{
-    DebruijnIndex, DebugWithInfcx, HashStableContext, InferCtxtLike, Interner, OptWithInfcx,
-    TyDecoder, TyEncoder,
+    DebruijnIndex, DebugWithInfcx, HashStableContext, InferCtxtLike, Interner, TyDecoder,
+    TyEncoder, WithInfcx,
 };
 
 use self::RegionKind::*;
@@ -274,8 +274,8 @@ impl<I: Interner> hash::Hash for RegionKind<I> {
 }
 
 impl<I: Interner> DebugWithInfcx<I> for RegionKind<I> {
-    fn fmt<InfCtx: InferCtxtLike<I>>(
-        this: OptWithInfcx<'_, I, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = I>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         match this.data {
@@ -301,7 +301,7 @@ impl<I: Interner> DebugWithInfcx<I> for RegionKind<I> {
 }
 impl<I: Interner> fmt::Debug for RegionKind<I> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        OptWithInfcx::new_no_ctx(self).fmt(f)
+        WithInfcx::with_no_infcx(self).fmt(f)
     }
 }
 

compiler/rustc_type_ir/src/ty_kind.rs

@@ -11,7 +11,7 @@ use crate::HashStableContext;
 use crate::Interner;
 use crate::TyDecoder;
 use crate::TyEncoder;
-use crate::{DebruijnIndex, DebugWithInfcx, InferCtxtLike, OptWithInfcx};
+use crate::{DebruijnIndex, DebugWithInfcx, InferCtxtLike, WithInfcx};
 
 use self::TyKind::*;
 
@@ -534,8 +534,8 @@ impl<I: Interner> hash::Hash for TyKind<I> {
 }
 
 impl<I: Interner> DebugWithInfcx<I> for TyKind<I> {
-    fn fmt<InfCtx: InferCtxtLike<I>>(
-        this: OptWithInfcx<'_, I, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = I>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> fmt::Result {
         match this.data {
@@ -617,7 +617,7 @@ impl<I: Interner> DebugWithInfcx<I> for TyKind<I> {
 // This is manually implemented because a derive would require `I: Debug`
 impl<I: Interner> fmt::Debug for TyKind<I> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        OptWithInfcx::new_no_ctx(self).fmt(f)
+        WithInfcx::with_no_infcx(self).fmt(f)
     }
 }
 
@@ -1239,12 +1239,12 @@ impl fmt::Debug for InferTy {
 }
 
 impl<I: Interner<InferTy = InferTy>> DebugWithInfcx<I> for InferTy {
-    fn fmt<InfCtx: InferCtxtLike<I>>(
-        this: OptWithInfcx<'_, I, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = I>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut fmt::Formatter<'_>,
     ) -> fmt::Result {
         use InferTy::*;
-        match this.infcx.and_then(|infcx| infcx.universe_of_ty(*this.data)) {
+        match this.infcx.universe_of_ty(*this.data) {
             None => write!(f, "{:?}", this.data),
             Some(universe) => match *this.data {
                 TyVar(ty_vid) => write!(f, "?{}_{}t", ty_vid.index(), universe.index()),

compiler/stable_mir/src/lib.rs

@@ -175,17 +175,17 @@ pub fn trait_impl(trait_impl: &ImplDef) -> ImplTrait {
 }
 
 pub trait Context {
-    fn entry_fn(&mut self) -> Option<CrateItem>;
+    fn entry_fn(&self) -> Option<CrateItem>;
     /// Retrieve all items of the local crate that have a MIR associated with them.
-    fn all_local_items(&mut self) -> CrateItems;
-    fn mir_body(&mut self, item: DefId) -> mir::Body;
-    fn all_trait_decls(&mut self) -> TraitDecls;
-    fn trait_decl(&mut self, trait_def: &TraitDef) -> TraitDecl;
-    fn all_trait_impls(&mut self) -> ImplTraitDecls;
-    fn trait_impl(&mut self, trait_impl: &ImplDef) -> ImplTrait;
-    fn generics_of(&mut self, def_id: DefId) -> Generics;
-    fn predicates_of(&mut self, def_id: DefId) -> GenericPredicates;
-    fn explicit_predicates_of(&mut self, def_id: DefId) -> GenericPredicates;
+    fn all_local_items(&self) -> CrateItems;
+    fn mir_body(&self, item: DefId) -> mir::Body;
+    fn all_trait_decls(&self) -> TraitDecls;
+    fn trait_decl(&self, trait_def: &TraitDef) -> TraitDecl;
+    fn all_trait_impls(&self) -> ImplTraitDecls;
+    fn trait_impl(&self, trait_impl: &ImplDef) -> ImplTrait;
+    fn generics_of(&self, def_id: DefId) -> Generics;
+    fn predicates_of(&self, def_id: DefId) -> GenericPredicates;
+    fn explicit_predicates_of(&self, def_id: DefId) -> GenericPredicates;
     /// Get information about the local crate.
     fn local_crate(&self) -> Crate;
     /// Retrieve a list of all external crates.
@@ -207,61 +207,58 @@ pub trait Context {
     fn get_lines(&self, span: &Span) -> LineInfo;
 
     /// Returns the `kind` of given `DefId`
-    fn def_kind(&mut self, def_id: DefId) -> DefKind;
+    fn def_kind(&self, def_id: DefId) -> DefKind;
 
     /// `Span` of an item
-    fn span_of_an_item(&mut self, def_id: DefId) -> Span;
+    fn span_of_an_item(&self, def_id: DefId) -> Span;
 
     /// Obtain the representation of a type.
-    fn ty_kind(&mut self, ty: Ty) -> TyKind;
+    fn ty_kind(&self, ty: Ty) -> TyKind;
 
     /// Create a new `Ty` from scratch without information from rustc.
-    fn mk_ty(&mut self, kind: TyKind) -> Ty;
+    fn mk_ty(&self, kind: TyKind) -> Ty;
 
     /// Get the body of an Instance.
     /// FIXME: Monomorphize the body.
-    fn instance_body(&mut self, instance: InstanceDef) -> Body;
+    fn instance_body(&self, instance: InstanceDef) -> Body;
 
     /// Get the instance type with generic substitutions applied and lifetimes erased.
-    fn instance_ty(&mut self, instance: InstanceDef) -> Ty;
+    fn instance_ty(&self, instance: InstanceDef) -> Ty;
 
     /// Get the instance.
-    fn instance_def_id(&mut self, instance: InstanceDef) -> DefId;
+    fn instance_def_id(&self, instance: InstanceDef) -> DefId;
 
     /// Convert a non-generic crate item into an instance.
     /// This function will panic if the item is generic.
-    fn mono_instance(&mut self, item: CrateItem) -> Instance;
+    fn mono_instance(&self, item: CrateItem) -> Instance;
 
     /// Item requires monomorphization.
     fn requires_monomorphization(&self, def_id: DefId) -> bool;
 
     /// Resolve an instance from the given function definition and generic arguments.
-    fn resolve_instance(&mut self, def: FnDef, args: &GenericArgs) -> Option<Instance>;
+    fn resolve_instance(&self, def: FnDef, args: &GenericArgs) -> Option<Instance>;
 }
 
 // A thread local variable that stores a pointer to the tables mapping between TyCtxt
 // datastructures and stable MIR datastructures
-scoped_thread_local! (static TLV: Cell<*mut ()>);
+scoped_thread_local! (static TLV: Cell<*const ()>);
 
-pub fn run(mut context: impl Context, f: impl FnOnce()) {
+pub fn run(context: &dyn Context, f: impl FnOnce()) {
     assert!(!TLV.is_set());
-    fn g<'a>(mut context: &mut (dyn Context + 'a), f: impl FnOnce()) {
-        let ptr: *mut () = &mut context as *mut &mut _ as _;
-        TLV.set(&Cell::new(ptr), || {
-            f();
-        });
-    }
-    g(&mut context, f);
+    let ptr: *const () = &context as *const &_ as _;
+    TLV.set(&Cell::new(ptr), || {
+        f();
+    });
 }
 
 /// Loads the current context and calls a function with it.
 /// Do not nest these, as that will ICE.
-pub fn with<R>(f: impl FnOnce(&mut dyn Context) -> R) -> R {
+pub fn with<R>(f: impl FnOnce(&dyn Context) -> R) -> R {
     assert!(TLV.is_set());
     TLV.with(|tlv| {
         let ptr = tlv.get();
         assert!(!ptr.is_null());
-        f(unsafe { *(ptr as *mut &mut dyn Context) })
+        f(unsafe { *(ptr as *const &dyn Context) })
     })
 }
 

library/core/src/ffi/mod.rs

@@ -6,7 +6,7 @@
 //! match those defined by C, so that code that interacts with C will
 //! refer to the correct types.
 
-#![stable(feature = "", since = "1.30.0")]
+#![stable(feature = "core_ffi", since = "1.30.0")]
 #![allow(non_camel_case_types)]
 
 use crate::fmt;

library/core/src/net/socket_addr.rs

@@ -1,6 +1,4 @@
-use crate::cmp::Ordering;
 use crate::fmt::{self, Write};
-use crate::hash;
 use crate::net::{IpAddr, Ipv4Addr, Ipv6Addr};
 
 use super::display_buffer::DisplayBuffer;
@@ -63,7 +61,7 @@ pub enum SocketAddr {
 /// assert_eq!(socket.ip(), &Ipv4Addr::new(127, 0, 0, 1));
 /// assert_eq!(socket.port(), 8080);
 /// ```
-#[derive(Copy, Clone, Eq, PartialEq)]
+#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
 #[stable(feature = "rust1", since = "1.0.0")]
 pub struct SocketAddrV4 {
     ip: Ipv4Addr,
@@ -96,7 +94,7 @@ pub struct SocketAddrV4 {
 /// assert_eq!(socket.ip(), &Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1));
 /// assert_eq!(socket.port(), 8080);
 /// ```
-#[derive(Copy, Clone, Eq, PartialEq)]
+#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
 #[stable(feature = "rust1", since = "1.0.0")]
 pub struct SocketAddrV6 {
     ip: Ipv6Addr,
@@ -644,48 +642,3 @@ impl fmt::Debug for SocketAddrV6 {
         fmt::Display::fmt(self, fmt)
     }
 }
-
-#[stable(feature = "socketaddr_ordering", since = "1.45.0")]
-impl PartialOrd for SocketAddrV4 {
-    #[inline]
-    fn partial_cmp(&self, other: &SocketAddrV4) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-#[stable(feature = "socketaddr_ordering", since = "1.45.0")]
-impl PartialOrd for SocketAddrV6 {
-    #[inline]
-    fn partial_cmp(&self, other: &SocketAddrV6) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-#[stable(feature = "socketaddr_ordering", since = "1.45.0")]
-impl Ord for SocketAddrV4 {
-    #[inline]
-    fn cmp(&self, other: &SocketAddrV4) -> Ordering {
-        self.ip().cmp(other.ip()).then(self.port().cmp(&other.port()))
-    }
-}
-
-#[stable(feature = "socketaddr_ordering", since = "1.45.0")]
-impl Ord for SocketAddrV6 {
-    #[inline]
-    fn cmp(&self, other: &SocketAddrV6) -> Ordering {
-        self.ip().cmp(other.ip()).then(self.port().cmp(&other.port()))
-    }
-}
-
-#[stable(feature = "rust1", since = "1.0.0")]
-impl hash::Hash for SocketAddrV4 {
-    fn hash<H: hash::Hasher>(&self, s: &mut H) {
-        (self.port, self.ip).hash(s)
-    }
-}
-#[stable(feature = "rust1", since = "1.0.0")]
-impl hash::Hash for SocketAddrV6 {
-    fn hash<H: hash::Hasher>(&self, s: &mut H) {
-        (self.port, &self.ip, self.flowinfo, self.scope_id).hash(s)
-    }
-}

library/core/src/num/mod.rs

@@ -114,7 +114,7 @@ macro_rules! midpoint_impl {
                       without modifying the original"]
         #[inline]
         pub const fn midpoint(self, rhs: $SelfT) -> $SelfT {
-            // Use the well known branchless algorthim from Hacker's Delight to compute
+            // Use the well known branchless algorithm from Hacker's Delight to compute
             // `(a + b) / 2` without overflowing: `((a ^ b) >> 1) + (a & b)`.
             ((self ^ rhs) >> 1) + (self & rhs)
         }

library/core/tests/net/socket_addr.rs

@@ -199,6 +199,9 @@ fn compare() {
     let v6_1 = "[2001:db8:f00::1002]:23456".parse::<SocketAddrV6>().unwrap();
     let v6_2 = "[2001:db8:f00::2001]:12345".parse::<SocketAddrV6>().unwrap();
     let v6_3 = "[2001:db8:f00::2001]:23456".parse::<SocketAddrV6>().unwrap();
+    let v6_4 = "[2001:db8:f00::2001%42]:23456".parse::<SocketAddrV6>().unwrap();
+    let mut v6_5 = "[2001:db8:f00::2001]:23456".parse::<SocketAddrV6>().unwrap();
+    v6_5.set_flowinfo(17);
 
     // equality
     assert_eq!(v4_1, v4_1);
@@ -207,6 +210,8 @@ fn compare() {
     assert_eq!(SocketAddr::V6(v6_1), SocketAddr::V6(v6_1));
     assert!(v4_1 != v4_2);
     assert!(v6_1 != v6_2);
+    assert!(v6_3 != v6_4);
+    assert!(v6_3 != v6_5);
 
     // compare different addresses
     assert!(v4_1 < v4_2);
@@ -226,6 +231,12 @@ fn compare() {
     assert!(v4_3 > v4_1);
     assert!(v6_3 > v6_1);
 
+    // compare the same address with different scope_id
+    assert!(v6_3 < v6_4);
+
+    // compare the same address with different flowinfo
+    assert!(v6_3 < v6_5);
+
     // compare with an inferred right-hand side
     assert_eq!(v4_1, "224.120.45.1:23456".parse().unwrap());
     assert_eq!(v6_1, "[2001:db8:f00::1002]:23456".parse().unwrap());

library/std/src/io/mod.rs

@@ -1140,10 +1140,10 @@ pub fn read_to_string<R: Read>(mut reader: R) -> Result<String> {
 #[repr(transparent)]
 pub struct IoSliceMut<'a>(sys::io::IoSliceMut<'a>);
 
-#[stable(feature = "iovec-send-sync", since = "1.44.0")]
+#[stable(feature = "iovec_send_sync", since = "1.44.0")]
 unsafe impl<'a> Send for IoSliceMut<'a> {}
 
-#[stable(feature = "iovec-send-sync", since = "1.44.0")]
+#[stable(feature = "iovec_send_sync", since = "1.44.0")]
 unsafe impl<'a> Sync for IoSliceMut<'a> {}
 
 #[stable(feature = "iovec", since = "1.36.0")]
@@ -1283,10 +1283,10 @@ impl<'a> DerefMut for IoSliceMut<'a> {
 #[repr(transparent)]
 pub struct IoSlice<'a>(sys::io::IoSlice<'a>);
 
-#[stable(feature = "iovec-send-sync", since = "1.44.0")]
+#[stable(feature = "iovec_send_sync", since = "1.44.0")]
 unsafe impl<'a> Send for IoSlice<'a> {}
 
-#[stable(feature = "iovec-send-sync", since = "1.44.0")]
+#[stable(feature = "iovec_send_sync", since = "1.44.0")]
 unsafe impl<'a> Sync for IoSlice<'a> {}
 
 #[stable(feature = "iovec", since = "1.36.0")]

library/std/src/sys/unix/mod.rs

@@ -415,7 +415,6 @@ cfg_if::cfg_if! {
     } else if #[cfg(any(target_os = "ios", target_os = "tvos", target_os = "watchos"))] {
         #[link(name = "System")]
         #[link(name = "objc")]
-        #[link(name = "Security", kind = "framework")]
         #[link(name = "Foundation", kind = "framework")]
         extern "C" {}
     } else if #[cfg(target_os = "fuchsia")] {

library/std/src/sys/unix/rand.rs

@@ -151,40 +151,65 @@ mod imp {
     }
 }
 
-#[cfg(target_os = "macos")]
+#[cfg(target_vendor = "apple")]
 mod imp {
-    use crate::fs::File;
-    use crate::io::Read;
-    use crate::sys::os::errno;
-    use crate::sys::weak::weak;
+    use crate::io;
     use libc::{c_int, c_void, size_t};
 
-    fn getentropy_fill_bytes(v: &mut [u8]) -> bool {
-        weak!(fn getentropy(*mut c_void, size_t) -> c_int);
+    #[inline(always)]
+    fn random_failure() -> ! {
+        panic!("unexpected random generation error: {}", io::Error::last_os_error());
+    }
 
-        getentropy
-            .get()
-            .map(|f| {
-                // getentropy(2) permits a maximum buffer size of 256 bytes
-                for s in v.chunks_mut(256) {
-                    let ret = unsafe { f(s.as_mut_ptr() as *mut c_void, s.len()) };
-                    if ret == -1 {
-                        panic!("unexpected getentropy error: {}", errno());
-                    }
-                }
-                true
-            })
-            .unwrap_or(false)
+    #[cfg(target_os = "macos")]
+    fn getentropy_fill_bytes(v: &mut [u8]) {
+        extern "C" {
+            fn getentropy(bytes: *mut c_void, count: size_t) -> c_int;
+        }
+
+        // getentropy(2) permits a maximum buffer size of 256 bytes
+        for s in v.chunks_mut(256) {
+            let ret = unsafe { getentropy(s.as_mut_ptr().cast(), s.len()) };
+            if ret == -1 {
+                random_failure()
+            }
+        }
+    }
+
+    #[cfg(not(target_os = "macos"))]
+    fn ccrandom_fill_bytes(v: &mut [u8]) {
+        extern "C" {
+            fn CCRandomGenerateBytes(bytes: *mut c_void, count: size_t) -> c_int;
+        }
+
+        let ret = unsafe { CCRandomGenerateBytes(v.as_mut_ptr().cast(), v.len()) };
+        if ret == -1 {
+            random_failure()
+        }
     }
 
     pub fn fill_bytes(v: &mut [u8]) {
-        if getentropy_fill_bytes(v) {
-            return;
-        }
+        // All supported versions of macOS (10.12+) support getentropy.
+        //
+        // `getentropy` is measurably faster (via Divan) then the other alternatives so its preferred
+        // when usable.
+        #[cfg(target_os = "macos")]
+        getentropy_fill_bytes(v);
 
-        // for older macos which doesn't support getentropy
-        let mut file = File::open("/dev/urandom").expect("failed to open /dev/urandom");
-        file.read_exact(v).expect("failed to read /dev/urandom")
+        // On Apple platforms, `CCRandomGenerateBytes` and `SecRandomCopyBytes` simply
+        // call into `CCRandomCopyBytes` with `kCCRandomDefault`. `CCRandomCopyBytes`
+        // manages a CSPRNG which is seeded from the kernel's CSPRNG and which runs on
+        // its own thread accessed via GCD. This seems needlessly heavyweight for our purposes
+        // so we only use it on non-Mac OSes where the better entrypoints are blocked.
+        //
+        // `CCRandomGenerateBytes` is used instead of `SecRandomCopyBytes` because the former is accessible
+        // via `libSystem` (libc) while the other needs to link to `Security.framework`.
+        //
+        // Note that while `getentropy` has a available attribute in the macOS headers, the lack
+        // of a header in the iOS (and others) SDK means that its can cause app store rejections.
+        // Just use `CCRandomGenerateBytes` instead.
+        #[cfg(not(target_os = "macos"))]
+        ccrandom_fill_bytes(v);
     }
 }
 
@@ -203,36 +228,6 @@ mod imp {
     }
 }
 
-// On iOS and MacOS `SecRandomCopyBytes` calls `CCRandomCopyBytes` with
-// `kCCRandomDefault`. `CCRandomCopyBytes` manages a CSPRNG which is seeded
-// from `/dev/random` and which runs on its own thread accessed via GCD.
-// This seems needlessly heavyweight for the purposes of generating two u64s
-// once per thread in `hashmap_random_keys`. Therefore `SecRandomCopyBytes` is
-// only used on iOS where direct access to `/dev/urandom` is blocked by the
-// sandbox.
-#[cfg(any(target_os = "ios", target_os = "tvos", target_os = "watchos"))]
-mod imp {
-    use crate::io;
-    use crate::ptr;
-    use libc::{c_int, size_t};
-
-    enum SecRandom {}
-
-    #[allow(non_upper_case_globals)]
-    const kSecRandomDefault: *const SecRandom = ptr::null();
-
-    extern "C" {
-        fn SecRandomCopyBytes(rnd: *const SecRandom, count: size_t, bytes: *mut u8) -> c_int;
-    }
-
-    pub fn fill_bytes(v: &mut [u8]) {
-        let ret = unsafe { SecRandomCopyBytes(kSecRandomDefault, v.len(), v.as_mut_ptr()) };
-        if ret == -1 {
-            panic!("couldn't generate random bytes: {}", io::Error::last_os_error());
-        }
-    }
-}
-
 // FIXME: once the 10.x release becomes the minimum, this can be dropped for simplification.
 #[cfg(target_os = "netbsd")]
 mod imp {

library/std/src/sys/unix/time.rs

@@ -1,8 +1,6 @@
 use crate::fmt;
 use crate::time::Duration;
 
-pub use self::inner::Instant;
-
 const NSEC_PER_SEC: u64 = 1_000_000_000;
 pub const UNIX_EPOCH: SystemTime = SystemTime { t: Timespec::zero() };
 #[allow(dead_code)] // Used for pthread condvar timeouts
@@ -40,6 +38,10 @@ impl SystemTime {
         SystemTime { t: Timespec::new(tv_sec, tv_nsec) }
     }
 
+    pub fn now() -> SystemTime {
+        SystemTime { t: Timespec::now(libc::CLOCK_REALTIME) }
+    }
+
     pub fn sub_time(&self, other: &SystemTime) -> Result<Duration, Duration> {
         self.t.sub_timespec(&other.t)
     }
@@ -79,6 +81,36 @@ impl Timespec {
         Timespec { tv_sec, tv_nsec: unsafe { Nanoseconds(tv_nsec as u32) } }
     }
 
+    pub fn now(clock: libc::clockid_t) -> Timespec {
+        use crate::mem::MaybeUninit;
+        use crate::sys::cvt;
+
+        // Try to use 64-bit time in preparation for Y2038.
+        #[cfg(all(
+            target_os = "linux",
+            target_env = "gnu",
+            target_pointer_width = "32",
+            not(target_arch = "riscv32")
+        ))]
+        {
+            use crate::sys::weak::weak;
+
+            // __clock_gettime64 was added to 32-bit arches in glibc 2.34,
+            // and it handles both vDSO calls and ENOSYS fallbacks itself.
+            weak!(fn __clock_gettime64(libc::clockid_t, *mut __timespec64) -> libc::c_int);
+
+            if let Some(clock_gettime64) = __clock_gettime64.get() {
+                let mut t = MaybeUninit::uninit();
+                cvt(unsafe { clock_gettime64(clock, t.as_mut_ptr()) }).unwrap();
+                return Timespec::from(unsafe { t.assume_init() });
+            }
+        }
+
+        let mut t = MaybeUninit::uninit();
+        cvt(unsafe { libc::clock_gettime(clock, t.as_mut_ptr()) }).unwrap();
+        Timespec::from(unsafe { t.assume_init() })
+    }
+
     pub fn sub_timespec(&self, other: &Timespec) -> Result<Duration, Duration> {
         if self >= other {
             // NOTE(eddyb) two aspects of this `if`-`else` are required for LLVM
@@ -216,209 +248,59 @@ impl From<__timespec64> for Timespec {
     }
 }
 
-#[cfg(any(
-    all(target_os = "macos", any(not(target_arch = "aarch64"))),
-    target_os = "ios",
-    target_os = "watchos",
-    target_os = "tvos"
-))]
-mod inner {
-    use crate::sync::atomic::{AtomicU64, Ordering};
-    use crate::sys::cvt;
-    use crate::sys_common::mul_div_u64;
-    use crate::time::Duration;
+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Instant {
+    t: Timespec,
+}
 
-    use super::{SystemTime, Timespec, NSEC_PER_SEC};
-
-    #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
-    pub struct Instant {
-        t: u64,
-    }
-
-    #[repr(C)]
-    #[derive(Copy, Clone)]
-    struct mach_timebase_info {
-        numer: u32,
-        denom: u32,
-    }
-    type mach_timebase_info_t = *mut mach_timebase_info;
-    type kern_return_t = libc::c_int;
-
-    impl Instant {
-        pub fn now() -> Instant {
-            extern "C" {
-                fn mach_absolute_time() -> u64;
-            }
-            Instant { t: unsafe { mach_absolute_time() } }
-        }
-
-        pub fn checked_sub_instant(&self, other: &Instant) -> Option<Duration> {
-            let diff = self.t.checked_sub(other.t)?;
-            let info = info();
-            let nanos = mul_div_u64(diff, info.numer as u64, info.denom as u64);
-            Some(Duration::new(nanos / NSEC_PER_SEC, (nanos % NSEC_PER_SEC) as u32))
-        }
-
-        pub fn checked_add_duration(&self, other: &Duration) -> Option<Instant> {
-            Some(Instant { t: self.t.checked_add(checked_dur2intervals(other)?)? })
-        }
-
-        pub fn checked_sub_duration(&self, other: &Duration) -> Option<Instant> {
-            Some(Instant { t: self.t.checked_sub(checked_dur2intervals(other)?)? })
-        }
-    }
-
-    impl SystemTime {
-        pub fn now() -> SystemTime {
-            use crate::ptr;
-
-            let mut s = libc::timeval { tv_sec: 0, tv_usec: 0 };
-            cvt(unsafe { libc::gettimeofday(&mut s, ptr::null_mut()) }).unwrap();
-            return SystemTime::from(s);
-        }
-    }
-
-    impl From<libc::timeval> for Timespec {
-        fn from(t: libc::timeval) -> Timespec {
-            Timespec::new(t.tv_sec as i64, 1000 * t.tv_usec as i64)
-        }
-    }
-
-    impl From<libc::timeval> for SystemTime {
-        fn from(t: libc::timeval) -> SystemTime {
-            SystemTime { t: Timespec::from(t) }
-        }
-    }
-
-    fn checked_dur2intervals(dur: &Duration) -> Option<u64> {
-        let nanos =
-            dur.as_secs().checked_mul(NSEC_PER_SEC)?.checked_add(dur.subsec_nanos() as u64)?;
-        let info = info();
-        Some(mul_div_u64(nanos, info.denom as u64, info.numer as u64))
-    }
-
-    fn info() -> mach_timebase_info {
-        // INFO_BITS conceptually is an `Option<mach_timebase_info>`. We can do
-        // this in 64 bits because we know 0 is never a valid value for the
-        // `denom` field.
+impl Instant {
+    pub fn now() -> Instant {
+        // https://www.manpagez.com/man/3/clock_gettime/
         //
-        // Encoding this as a single `AtomicU64` allows us to use `Relaxed`
-        // operations, as we are only interested in the effects on a single
-        // memory location.
-        static INFO_BITS: AtomicU64 = AtomicU64::new(0);
-
-        // If a previous thread has initialized `INFO_BITS`, use it.
-        let info_bits = INFO_BITS.load(Ordering::Relaxed);
-        if info_bits != 0 {
-            return info_from_bits(info_bits);
-        }
-
-        // ... otherwise learn for ourselves ...
-        extern "C" {
-            fn mach_timebase_info(info: mach_timebase_info_t) -> kern_return_t;
-        }
-
-        let mut info = info_from_bits(0);
-        unsafe {
-            mach_timebase_info(&mut info);
-        }
-        INFO_BITS.store(info_to_bits(info), Ordering::Relaxed);
-        info
+        // CLOCK_UPTIME_RAW   clock that increments monotonically, in the same man-
+        //                    ner as CLOCK_MONOTONIC_RAW, but that does not incre-
+        //                    ment while the system is asleep.  The returned value
+        //                    is identical to the result of mach_absolute_time()
+        //                    after the appropriate mach_timebase conversion is
+        //                    applied.
+        //
+        // Instant on macos was historically implemented using mach_absolute_time;
+        // we preserve this value domain out of an abundance of caution.
+        #[cfg(any(
+            target_os = "macos",
+            target_os = "ios",
+            target_os = "watchos",
+            target_os = "tvos"
+        ))]
+        const clock_id: libc::clockid_t = libc::CLOCK_UPTIME_RAW;
+        #[cfg(not(any(
+            target_os = "macos",
+            target_os = "ios",
+            target_os = "watchos",
+            target_os = "tvos"
+        )))]
+        const clock_id: libc::clockid_t = libc::CLOCK_MONOTONIC;
+        Instant { t: Timespec::now(clock_id) }
     }
 
-    #[inline]
-    fn info_to_bits(info: mach_timebase_info) -> u64 {
-        ((info.denom as u64) << 32) | (info.numer as u64)
+    pub fn checked_sub_instant(&self, other: &Instant) -> Option<Duration> {
+        self.t.sub_timespec(&other.t).ok()
     }
 
-    #[inline]
-    fn info_from_bits(bits: u64) -> mach_timebase_info {
-        mach_timebase_info { numer: bits as u32, denom: (bits >> 32) as u32 }
+    pub fn checked_add_duration(&self, other: &Duration) -> Option<Instant> {
+        Some(Instant { t: self.t.checked_add_duration(other)? })
+    }
+
+    pub fn checked_sub_duration(&self, other: &Duration) -> Option<Instant> {
+        Some(Instant { t: self.t.checked_sub_duration(other)? })
     }
 }
 
-#[cfg(not(any(
-    all(target_os = "macos", any(not(target_arch = "aarch64"))),
-    target_os = "ios",
-    target_os = "watchos",
-    target_os = "tvos"
-)))]
-mod inner {
-    use crate::fmt;
-    use crate::mem::MaybeUninit;
-    use crate::sys::cvt;
-    use crate::time::Duration;
-
-    use super::{SystemTime, Timespec};
-
-    #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-    pub struct Instant {
-        t: Timespec,
-    }
-
-    impl Instant {
-        pub fn now() -> Instant {
-            #[cfg(target_os = "macos")]
-            const clock_id: libc::clockid_t = libc::CLOCK_UPTIME_RAW;
-            #[cfg(not(target_os = "macos"))]
-            const clock_id: libc::clockid_t = libc::CLOCK_MONOTONIC;
-            Instant { t: Timespec::now(clock_id) }
-        }
-
-        pub fn checked_sub_instant(&self, other: &Instant) -> Option<Duration> {
-            self.t.sub_timespec(&other.t).ok()
-        }
-
-        pub fn checked_add_duration(&self, other: &Duration) -> Option<Instant> {
-            Some(Instant { t: self.t.checked_add_duration(other)? })
-        }
-
-        pub fn checked_sub_duration(&self, other: &Duration) -> Option<Instant> {
-            Some(Instant { t: self.t.checked_sub_duration(other)? })
-        }
-    }
-
-    impl fmt::Debug for Instant {
-        fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-            f.debug_struct("Instant")
-                .field("tv_sec", &self.t.tv_sec)
-                .field("tv_nsec", &self.t.tv_nsec.0)
-                .finish()
-        }
-    }
-
-    impl SystemTime {
-        pub fn now() -> SystemTime {
-            SystemTime { t: Timespec::now(libc::CLOCK_REALTIME) }
-        }
-    }
-
-    impl Timespec {
-        pub fn now(clock: libc::clockid_t) -> Timespec {
-            // Try to use 64-bit time in preparation for Y2038.
-            #[cfg(all(
-                target_os = "linux",
-                target_env = "gnu",
-                target_pointer_width = "32",
-                not(target_arch = "riscv32")
-            ))]
-            {
-                use crate::sys::weak::weak;
-
-                // __clock_gettime64 was added to 32-bit arches in glibc 2.34,
-                // and it handles both vDSO calls and ENOSYS fallbacks itself.
-                weak!(fn __clock_gettime64(libc::clockid_t, *mut super::__timespec64) -> libc::c_int);
-
-                if let Some(clock_gettime64) = __clock_gettime64.get() {
-                    let mut t = MaybeUninit::uninit();
-                    cvt(unsafe { clock_gettime64(clock, t.as_mut_ptr()) }).unwrap();
-                    return Timespec::from(unsafe { t.assume_init() });
-                }
-            }
-
-            let mut t = MaybeUninit::uninit();
-            cvt(unsafe { libc::clock_gettime(clock, t.as_mut_ptr()) }).unwrap();
-            Timespec::from(unsafe { t.assume_init() })
-        }
+impl fmt::Debug for Instant {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Instant")
+            .field("tv_sec", &self.t.tv_sec)
+            .field("tv_nsec", &self.t.tv_nsec.0)
+            .finish()
     }
 }

library/std/src/sys/windows/c/windows_sys.lst

@@ -2505,9 +2505,12 @@ Windows.Win32.System.Threading.CREATE_SEPARATE_WOW_VDM
 Windows.Win32.System.Threading.CREATE_SHARED_WOW_VDM
 Windows.Win32.System.Threading.CREATE_SUSPENDED
 Windows.Win32.System.Threading.CREATE_UNICODE_ENVIRONMENT
+Windows.Win32.System.Threading.CREATE_WAITABLE_TIMER_HIGH_RESOLUTION
+Windows.Win32.System.Threading.CREATE_WAITABLE_TIMER_MANUAL_RESET
 Windows.Win32.System.Threading.CreateEventW
 Windows.Win32.System.Threading.CreateProcessW
 Windows.Win32.System.Threading.CreateThread
+Windows.Win32.System.Threading.CreateWaitableTimerExW
 Windows.Win32.System.Threading.DEBUG_ONLY_THIS_PROCESS
 Windows.Win32.System.Threading.DEBUG_PROCESS
 Windows.Win32.System.Threading.DeleteProcThreadAttributeList
@@ -2544,6 +2547,7 @@ Windows.Win32.System.Threading.REALTIME_PRIORITY_CLASS
 Windows.Win32.System.Threading.ReleaseSRWLockExclusive
 Windows.Win32.System.Threading.ReleaseSRWLockShared
 Windows.Win32.System.Threading.SetThreadStackGuarantee
+Windows.Win32.System.Threading.SetWaitableTimer
 Windows.Win32.System.Threading.Sleep
 Windows.Win32.System.Threading.SleepConditionVariableSRW
 Windows.Win32.System.Threading.SleepEx
@@ -2570,6 +2574,8 @@ Windows.Win32.System.Threading.TerminateProcess
 Windows.Win32.System.Threading.THREAD_CREATE_RUN_IMMEDIATELY
 Windows.Win32.System.Threading.THREAD_CREATE_SUSPENDED
 Windows.Win32.System.Threading.THREAD_CREATION_FLAGS
+Windows.Win32.System.Threading.TIMER_ALL_ACCESS
+Windows.Win32.System.Threading.TIMER_MODIFY_STATE
 Windows.Win32.System.Threading.TLS_OUT_OF_INDEXES
 Windows.Win32.System.Threading.TlsAlloc
 Windows.Win32.System.Threading.TlsFree

library/std/src/sys/windows/c/windows_sys.rs

@@ -151,6 +151,15 @@ extern "system" {
     ) -> HANDLE;
 }
 #[link(name = "kernel32")]
+extern "system" {
+    pub fn CreateWaitableTimerExW(
+        lptimerattributes: *const SECURITY_ATTRIBUTES,
+        lptimername: PCWSTR,
+        dwflags: u32,
+        dwdesiredaccess: u32,
+    ) -> HANDLE;
+}
+#[link(name = "kernel32")]
 extern "system" {
     pub fn DeleteFileW(lpfilename: PCWSTR) -> BOOL;
 }
@@ -508,6 +517,17 @@ extern "system" {
     pub fn SetThreadStackGuarantee(stacksizeinbytes: *mut u32) -> BOOL;
 }
 #[link(name = "kernel32")]
+extern "system" {
+    pub fn SetWaitableTimer(
+        htimer: HANDLE,
+        lpduetime: *const i64,
+        lperiod: i32,
+        pfncompletionroutine: PTIMERAPCROUTINE,
+        lpargtocompletionroutine: *const ::core::ffi::c_void,
+        fresume: BOOL,
+    ) -> BOOL;
+}
+#[link(name = "kernel32")]
 extern "system" {
     pub fn Sleep(dwmilliseconds: u32) -> ();
 }
@@ -1165,6 +1185,8 @@ pub const CREATE_SEPARATE_WOW_VDM: PROCESS_CREATION_FLAGS = 2048u32;
 pub const CREATE_SHARED_WOW_VDM: PROCESS_CREATION_FLAGS = 4096u32;
 pub const CREATE_SUSPENDED: PROCESS_CREATION_FLAGS = 4u32;
 pub const CREATE_UNICODE_ENVIRONMENT: PROCESS_CREATION_FLAGS = 1024u32;
+pub const CREATE_WAITABLE_TIMER_HIGH_RESOLUTION: u32 = 2u32;
+pub const CREATE_WAITABLE_TIMER_MANUAL_RESET: u32 = 1u32;
 pub const CSTR_EQUAL: COMPARESTRING_RESULT = 2i32;
 pub const CSTR_GREATER_THAN: COMPARESTRING_RESULT = 3i32;
 pub const CSTR_LESS_THAN: COMPARESTRING_RESULT = 1i32;
@@ -3775,6 +3797,13 @@ pub const PROFILE_SERVER: PROCESS_CREATION_FLAGS = 1073741824u32;
 pub const PROFILE_USER: PROCESS_CREATION_FLAGS = 268435456u32;
 pub const PROGRESS_CONTINUE: u32 = 0u32;
 pub type PSTR = *mut u8;
+pub type PTIMERAPCROUTINE = ::core::option::Option<
+    unsafe extern "system" fn(
+        lpargtocompletionroutine: *const ::core::ffi::c_void,
+        dwtimerlowvalue: u32,
+        dwtimerhighvalue: u32,
+    ) -> (),
+>;
 pub type PWSTR = *mut u16;
 pub const READ_CONTROL: FILE_ACCESS_RIGHTS = 131072u32;
 pub const REALTIME_PRIORITY_CLASS: PROCESS_CREATION_FLAGS = 256u32;
@@ -3922,6 +3951,7 @@ pub type SYMBOLIC_LINK_FLAGS = u32;
 pub const SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE: SYMBOLIC_LINK_FLAGS = 2u32;
 pub const SYMBOLIC_LINK_FLAG_DIRECTORY: SYMBOLIC_LINK_FLAGS = 1u32;
 pub const SYMLINK_FLAG_RELATIVE: u32 = 1u32;
+pub type SYNCHRONIZATION_ACCESS_RIGHTS = u32;
 pub const SYNCHRONIZE: FILE_ACCESS_RIGHTS = 1048576u32;
 #[repr(C)]
 pub struct SYSTEM_INFO {
@@ -3968,6 +3998,8 @@ pub const TCP_NODELAY: i32 = 1i32;
 pub const THREAD_CREATE_RUN_IMMEDIATELY: THREAD_CREATION_FLAGS = 0u32;
 pub const THREAD_CREATE_SUSPENDED: THREAD_CREATION_FLAGS = 4u32;
 pub type THREAD_CREATION_FLAGS = u32;
+pub const TIMER_ALL_ACCESS: SYNCHRONIZATION_ACCESS_RIGHTS = 2031619u32;
+pub const TIMER_MODIFY_STATE: SYNCHRONIZATION_ACCESS_RIGHTS = 2u32;
 #[repr(C)]
 pub struct TIMEVAL {
     pub tv_sec: i32,

library/std/src/sys/windows/thread.rs

@@ -12,6 +12,7 @@ use crate::time::Duration;
 
 use core::ffi::c_void;
 
+use super::time::WaitableTimer;
 use super::to_u16s;
 
 pub const DEFAULT_MIN_STACK_SIZE: usize = 2 * 1024 * 1024;
@@ -87,7 +88,17 @@ impl Thread {
     }
 
     pub fn sleep(dur: Duration) {
-        unsafe { c::Sleep(super::dur2timeout(dur)) }
+        fn high_precision_sleep(dur: Duration) -> Result<(), ()> {
+            let timer = WaitableTimer::high_resolution()?;
+            timer.set(dur)?;
+            timer.wait()
+        }
+        // Attempt to use high-precision sleep (Windows 10, version 1803+).
+        // On error fallback to the standard `Sleep` function.
+        // Also preserves the zero duration behaviour of `Sleep`.
+        if dur.is_zero() || high_precision_sleep(dur).is_err() {
+            unsafe { c::Sleep(super::dur2timeout(dur)) }
+        }
     }
 
     pub fn handle(&self) -> &Handle {

library/std/src/sys/windows/time.rs

@@ -1,11 +1,13 @@
 use crate::cmp::Ordering;
 use crate::fmt;
 use crate::mem;
+use crate::ptr::{null, null_mut};
 use crate::sys::c;
 use crate::sys_common::IntoInner;
 use crate::time::Duration;
 
 use core::hash::{Hash, Hasher};
+use core::ops::Neg;
 
 const NANOS_PER_SEC: u64 = 1_000_000_000;
 const INTERVALS_PER_SEC: u64 = NANOS_PER_SEC / 100;
@@ -222,3 +224,39 @@ mod perf_counter {
         qpc_value
     }
 }
+
+/// A timer you can wait on.
+pub(super) struct WaitableTimer {
+    handle: c::HANDLE,
+}
+impl WaitableTimer {
+    /// Create a high-resolution timer. Will fail before Windows 10, version 1803.
+    pub fn high_resolution() -> Result<Self, ()> {
+        let handle = unsafe {
+            c::CreateWaitableTimerExW(
+                null(),
+                null(),
+                c::CREATE_WAITABLE_TIMER_HIGH_RESOLUTION,
+                c::TIMER_ALL_ACCESS,
+            )
+        };
+        if handle != null_mut() { Ok(Self { handle }) } else { Err(()) }
+    }
+    pub fn set(&self, duration: Duration) -> Result<(), ()> {
+        // Convert the Duration to a format similar to FILETIME.
+        // Negative values are relative times whereas positive values are absolute.
+        // Therefore we negate the relative duration.
+        let time = checked_dur2intervals(&duration).ok_or(())?.neg();
+        let result = unsafe { c::SetWaitableTimer(self.handle, &time, 0, None, null(), c::FALSE) };
+        if result != 0 { Ok(()) } else { Err(()) }
+    }
+    pub fn wait(&self) -> Result<(), ()> {
+        let result = unsafe { c::WaitForSingleObject(self.handle, c::INFINITE) };
+        if result != c::WAIT_FAILED { Ok(()) } else { Err(()) }
+    }
+}
+impl Drop for WaitableTimer {
+    fn drop(&mut self) {
+        unsafe { c::CloseHandle(self.handle) };
+    }
+}

library/std/src/time.rs

@@ -111,7 +111,7 @@ pub use core::time::TryFromFloatSecsError;
 /// |-----------|----------------------------------------------------------------------|
 /// | SGX       | [`insecure_time` usercall]. More information on [timekeeping in SGX] |
 /// | UNIX      | [clock_gettime (Monotonic Clock)]                                    |
-/// | Darwin    | [mach_absolute_time]                                                 |
+/// | Darwin    | [clock_gettime (Monotonic Clock)]                                    |
 /// | VXWorks   | [clock_gettime (Monotonic Clock)]                                    |
 /// | SOLID     | `get_tim`                                                            |
 /// | WASI      | [__wasi_clock_time_get (Monotonic Clock)]                            |
@@ -123,7 +123,6 @@ pub use core::time::TryFromFloatSecsError;
 /// [timekeeping in SGX]: https://edp.fortanix.com/docs/concepts/rust-std/#codestdtimecode
 /// [__wasi_clock_time_get (Monotonic Clock)]: https://github.com/WebAssembly/WASI/blob/main/legacy/preview1/docs.md#clock_time_get
 /// [clock_gettime (Monotonic Clock)]: https://linux.die.net/man/3/clock_gettime
-/// [mach_absolute_time]: https://developer.apple.com/library/archive/documentation/Darwin/Conceptual/KernelProgramming/services/services.html
 ///
 /// **Disclaimer:** These system calls might change over time.
 ///
@@ -224,7 +223,7 @@ pub struct Instant(time::Instant);
 /// |-----------|----------------------------------------------------------------------|
 /// | SGX       | [`insecure_time` usercall]. More information on [timekeeping in SGX] |
 /// | UNIX      | [clock_gettime (Realtime Clock)]                                     |
-/// | Darwin    | [gettimeofday]                                                       |
+/// | Darwin    | [clock_gettime (Realtime Clock)]                                     |
 /// | VXWorks   | [clock_gettime (Realtime Clock)]                                     |
 /// | SOLID     | `SOLID_RTC_ReadTime`                                                 |
 /// | WASI      | [__wasi_clock_time_get (Realtime Clock)]                             |
@@ -233,7 +232,6 @@ pub struct Instant(time::Instant);
 /// [currently]: crate::io#platform-specific-behavior
 /// [`insecure_time` usercall]: https://edp.fortanix.com/docs/api/fortanix_sgx_abi/struct.Usercalls.html#method.insecure_time
 /// [timekeeping in SGX]: https://edp.fortanix.com/docs/concepts/rust-std/#codestdtimecode
-/// [gettimeofday]: https://man7.org/linux/man-pages/man2/gettimeofday.2.html
 /// [clock_gettime (Realtime Clock)]: https://linux.die.net/man/3/clock_gettime
 /// [__wasi_clock_time_get (Realtime Clock)]: https://github.com/WebAssembly/WASI/blob/main/legacy/preview1/docs.md#clock_time_get
 /// [GetSystemTimePreciseAsFileTime]: https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getsystemtimepreciseasfiletime

src/bootstrap/bootstrap_test.py

@@ -103,7 +103,6 @@ class GenerateAndParseConfig(unittest.TestCase):
     """Test that we can serialize and deserialize a config.toml file"""
     def test_no_args(self):
         build = serialize_and_parse([])
-        self.assertEqual(build.get_toml("change-id"), '116998')
         self.assertEqual(build.get_toml("profile"), 'dist')
         self.assertIsNone(build.get_toml("llvm.download-ci-llvm"))
 

src/doc/book

@@ -1 +1 @@
-Subproject commit 72187f5cd0beaaa9c6f584156bcd88f921871e83
+Subproject commit 3dca2fc50b922a8efb94903b9fee8bb42ab48f38

src/doc/embedded-book

@@ -1 +1 @@
-Subproject commit eac173690b8cc99094e1d88bd49dd61127fbd285
+Subproject commit 22bca3d0f6e9b9b556689b54ce96f25b46ecd1b3

src/doc/nomicon

@@ -1 +1 @@
-Subproject commit ddfa4214487686e91b21aa29afb972c08a8f0d5b
+Subproject commit 1842257814919fa62e81bdecd5e8f95be2839dbb

src/doc/reference

@@ -1 +1 @@
-Subproject commit 142b2ed77d33f37a9973772bd95e6144ed9dce43
+Subproject commit 16fd3c06d9e558dae2d52000818274ae70c9e90a

src/doc/rust-by-example

@@ -1 +1 @@
-Subproject commit 8eb3a01ab74c567b7174784892fb807f2c632d6b
+Subproject commit 6709beeb7d0fbc5ffc91ac4893a24434123b9bfa

src/doc/rustc-dev-guide

@@ -1 +1 @@
-Subproject commit b98af7d661e4744baab81fb8dc7a049e44a4a998
+Subproject commit b0ee9ec8fa59a6c7620165e061f4747202377a62

src/doc/unstable-book/src/compiler-flags/check-cfg.md

@@ -123,7 +123,7 @@ rustc --check-cfg 'cfg(is_embedded, has_feathers, values(any()))' \
 fn do_embedded() {}         // and because names exhaustiveness was not disabled
 
 #[cfg(has_feathers)]        // This is expected as "has_feathers" was provided in cfg()
-fn do_features() {}         // and because names exhaustiveness was not disbaled
+fn do_features() {}         // and because names exhaustiveness was not disabled
 
 #[cfg(has_feathers = "zapping")] // This is expected as "has_feathers" was provided in cfg()
                                  // and because no value checking was enable for "has_feathers"

src/tools/clippy/tests/ui/missing_const_for_fn/auxiliary/helper.rs

@@ -1,8 +1,8 @@
 // This file provides a const function that is unstably const forever.
 
 #![feature(staged_api)]
-#![stable(feature = "1", since = "1.0.0")]
+#![stable(feature = "clippytest", since = "1.0.0")]
 
-#[stable(feature = "1", since = "1.0.0")]
+#[stable(feature = "clippytest", since = "1.0.0")]
 #[rustc_const_unstable(feature = "foo", issue = "none")]
 pub const fn unstably_const_fn() {}

src/tools/miri/src/shims/windows/foreign_items.rs

@@ -272,6 +272,18 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
 
                 this.Sleep(timeout)?;
             }
+            "CreateWaitableTimerExW" => {
+                let [attributes, name, flags, access] =
+                    this.check_shim(abi, Abi::System { unwind: false }, link_name, args)?;
+                this.read_pointer(attributes)?;
+                this.read_pointer(name)?;
+                this.read_scalar(flags)?.to_u32()?;
+                this.read_scalar(access)?.to_u32()?;
+                // Unimplemented. Always return failure.
+                let not_supported = this.eval_windows("c", "ERROR_NOT_SUPPORTED");
+                this.set_last_error(not_supported)?;
+                this.write_null(dest)?;
+            }
 
             // Synchronization primitives
             "AcquireSRWLockExclusive" => {

tests/mir-opt/const_debuginfo.main.ConstDebugInfo.diff

@@ -41,7 +41,8 @@
 +                             debug ((f: (bool, bool, u32)).2: u32) => const 123_u32;
                               let _10: std::option::Option<u16>;
                               scope 7 {
-                                  debug o => _10;
+-                                 debug o => _10;
++                                 debug o => const Option::<u16>::Some(99_u16);
                                   let _17: u32;
                                   let _18: u32;
                                   scope 8 {
@@ -81,7 +82,7 @@
           _15 = const false;
           _16 = const 123_u32;
           StorageLive(_10);
-          _10 = Option::<u16>::Some(const 99_u16);
+          _10 = const Option::<u16>::Some(99_u16);
           _17 = const 32_u32;
           _18 = const 32_u32;
           StorageLive(_11);
@@ -97,3 +98,7 @@
       }
   }
   
+  ALLOC0 (size: 4, align: 2) {
+      01 00 63 00                                     │ ..c.
+  }
+