Merge pull request #19212 from lnicola/sync-from-rust

minor: sync from downstream
2025-06-05 19:58:32 +00:00 · 2025-02-24 08:45:06 +00:00 · 2025-02-24 08:45:06 +00:00 · 6d9b9c1363
commit 6d9b9c1363
parent 286595e21e a2bd5a5ed3
1279 changed files with 16123 additions and 10695 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -70,6 +70,7 @@ jobs:
      HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
      DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      SCCACHE_BUCKET: rust-lang-ci-sccache2
      SCCACHE_REGION: us-west-1
      CACHE_DOMAIN: ci-caches.rust-lang.org
    continue-on-error: ${{ matrix.continue_on_error || false }}
    strategy:
@ -173,6 +174,11 @@ jobs:
      - name: ensure the stable version number is correct
        run: src/ci/scripts/verify-stable-version-number.sh
      # Show the environment just before we run the build
      # This makes it easier to diagnose problems with the above install scripts.
      - name: show the current environment
        run: src/ci/scripts/dump-environment.sh
      - name: run the build
        # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
        run: src/ci/scripts/run-build-from-ci.sh 2>&1
--- a/.gitmodules
+++ b/.gitmodules
@ -29,7 +29,7 @@
 [submodule "src/llvm-project"]
 	path = src/llvm-project
 	url = https://github.com/rust-lang/llvm-project.git
-	branch = rustc/19.1-2024-12-03
+	branch = rustc/20.1-2025-02-13
 	shallow = true
 [submodule "src/doc/embedded-book"]
 	path = src/doc/embedded-book
--- a/Cargo.lock
+++ b/Cargo.lock
@ -61,19 +61,6 @@ version = "0.2.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
 [[package]]
 name = "ammonia"
 version = "4.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1ab99eae5ee58501ab236beb6f20f6ca39be615267b014899c89b2f0bc18a459"
 dependencies = [
 "html5ever",
 "maplit",
 "once_cell",
 "tendril",
 "url",
 ]
 [[package]]
 name = "android-tzdata"
 version = "0.1.1"
@ -513,16 +500,6 @@ dependencies = [
 "anstyle",
 "clap_lex",
 "strsim",
 "terminal_size",
 ]
 [[package]]
 name = "clap_complete"
 version = "4.5.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "33a7e468e750fa4b6be660e8b5651ad47372e8fb114030b594c2d75d48c5ffd0"
 dependencies = [
 "clap",
 ]
 [[package]]
@ -1084,18 +1061,6 @@ version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
 [[package]]
 name = "elasticlunr-rs"
 version = "3.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "41e83863a500656dfa214fee6682de9c5b9f03de6860fec531235ed2ae9f6571"
 dependencies = [
 "regex",
 "serde",
 "serde_derive",
 "serde_json",
 ]
 [[package]]
 name = "elsa"
 version = "1.11.0"
@ -1159,13 +1124,6 @@ dependencies = [
 "windows-sys 0.59.0",
 ]
 [[package]]
 name = "error_index_generator"
 version = "0.0.0"
 dependencies = [
 "mdbook",
 ]
 [[package]]
 name = "expect-test"
 version = "1.5.1"
@ -1517,22 +1475,6 @@ dependencies = [
 "serde",
 ]
 [[package]]
 name = "handlebars"
 version = "6.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3d6b224b95c1e668ac0270325ad563b2eef1469fbbb8959bc7c692c844b813d9"
 dependencies = [
 "derive_builder",
 "log",
 "num-order",
 "pest",
 "pest_derive",
 "serde",
 "serde_json",
 "thiserror 2.0.11",
 ]
 [[package]]
 name = "hashbrown"
 version = "0.14.5"
@ -2189,12 +2131,6 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
 [[package]]
 name = "maplit"
 version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
 [[package]]
 name = "markup5ever"
 version = "0.12.1"
@ -2228,34 +2164,6 @@ dependencies = [
 "digest",
 ]
 [[package]]
 name = "mdbook"
 version = "0.4.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fe1f98b8d66e537d2f0ba06e7dec4f44001deec539a2d18bfc102d6a86189148"
 dependencies = [
 "ammonia",
 "anyhow",
 "chrono",
 "clap",
 "clap_complete",
 "elasticlunr-rs",
 "env_logger",
 "handlebars",
 "log",
 "memchr",
 "once_cell",
 "opener",
 "pulldown-cmark 0.10.3",
 "regex",
 "serde",
 "serde_json",
 "shlex",
 "tempfile",
 "toml 0.5.11",
 "topological-sort",
 ]
 [[package]]
 name = "measureme"
 version = "11.0.1"
@ -2303,9 +2211,9 @@ dependencies = [
 [[package]]
 name = "minifier"
-version = "0.3.4"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cf47565b1430f5fe6c81d3afcb4b835271348d7eb35294a4d592e38dd09ea22"
+checksum = "9bfdc64e2f805f3d12965f10522000bae36e88d2cfea44112331f467d4f4bf68"
 [[package]]
 name = "minimal-lexical"
@ -2483,21 +2391,6 @@ dependencies = [
 "num-traits",
 ]
 [[package]]
 name = "num-modular"
 version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "17bb261bf36fa7d83f4c294f834e91256769097b3cb505d44831e0a179ac647f"
 [[package]]
 name = "num-order"
 version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "537b596b97c40fcf8056d153049eb22f481c17ebce72a513ec9286e4986d1bb6"
 dependencies = [
 "num-modular",
 ]
 [[package]]
 name = "num-rational"
 version = "0.4.2"
@ -2718,51 +2611,6 @@ dependencies = [
 "libc",
 ]
 [[package]]
 name = "pest"
 version = "2.7.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc"
 dependencies = [
 "memchr",
 "thiserror 2.0.11",
 "ucd-trie",
 ]
 [[package]]
 name = "pest_derive"
 version = "2.7.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "816518421cfc6887a0d62bf441b6ffb4536fcc926395a69e1a85852d4363f57e"
 dependencies = [
 "pest",
 "pest_generator",
 ]
 [[package]]
 name = "pest_generator"
 version = "2.7.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7d1396fd3a870fc7838768d171b4616d5c91f6cc25e377b673d714567d99377b"
 dependencies = [
 "pest",
 "pest_meta",
 "proc-macro2",
 "quote",
 "syn 2.0.96",
 ]
 [[package]]
 name = "pest_meta"
 version = "2.7.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e1e58089ea25d717bfd31fb534e4f3afcc2cc569c70de3e239778991ea3b7dea"
 dependencies = [
 "once_cell",
 "pest",
 "sha2",
 ]
 [[package]]
 name = "phf"
 version = "0.11.3"
@ -2903,9 +2751,9 @@ dependencies = [
 [[package]]
 name = "psm"
-version = "0.1.24"
+version = "0.1.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "200b9ff220857e53e184257720a14553b2f4aa02577d2ed9842d45d4b9654810"
+checksum = "f58e5423e24c18cc840e1c98370b3993c6649cd1678b4d24318bcf0a083cbe88"
 dependencies = [
 "cc",
 ]
@ -2921,18 +2769,6 @@ dependencies = [
 "unicase",
 ]
 [[package]]
 name = "pulldown-cmark"
 version = "0.10.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "76979bea66e7875e7509c4ec5300112b316af87fa7a252ca91c448b32dfe3993"
 dependencies = [
 "bitflags",
 "memchr",
 "pulldown-cmark-escape 0.10.1",
 "unicase",
 ]
 [[package]]
 name = "pulldown-cmark"
 version = "0.11.3"
@ -2941,16 +2777,10 @@ checksum = "679341d22c78c6c649893cbd6c3278dcbe9fc4faa62fea3a9296ae2b50c14625"
 dependencies = [
 "bitflags",
 "memchr",
- "pulldown-cmark-escape 0.11.0",
+ "pulldown-cmark-escape",
 "unicase",
 ]
 [[package]]
 name = "pulldown-cmark-escape"
 version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bd348ff538bc9caeda7ee8cad2d1d48236a1f443c1fa3913c6a02fe0043b1dd3"
 [[package]]
 name = "pulldown-cmark-escape"
 version = "0.11.0"
@ -3317,6 +3147,7 @@ dependencies = [
 "rand 0.8.5",
 "rand_xoshiro",
 "rustc_data_structures",
 "rustc_hashes",
 "rustc_index",
 "rustc_macros",
 "rustc_serialize",
@ -3544,6 +3375,7 @@ dependencies = [
 "rustc_errors",
 "rustc_fluent_macro",
 "rustc_fs_util",
 "rustc_hashes",
 "rustc_hir",
 "rustc_index",
 "rustc_llvm",
@ -3586,6 +3418,7 @@ dependencies = [
 "rustc_errors",
 "rustc_fluent_macro",
 "rustc_fs_util",
 "rustc_hashes",
 "rustc_hir",
 "rustc_hir_pretty",
 "rustc_incremental",
@ -3658,6 +3491,7 @@ dependencies = [
 "rustc-stable-hash",
 "rustc_arena",
 "rustc_graphviz",
 "rustc_hashes",
 "rustc_index",
 "rustc_macros",
 "rustc_serialize",
@ -3768,6 +3602,7 @@ dependencies = [
 "rustc_error_codes",
 "rustc_error_messages",
 "rustc_fluent_macro",
 "rustc_hashes",
 "rustc_hir",
 "rustc_index",
 "rustc_lexer",
@ -3840,6 +3675,13 @@ version = "0.0.0"
 name = "rustc_graphviz"
 version = "0.0.0"
 [[package]]
 name = "rustc_hashes"
 version = "0.0.0"
 dependencies = [
 "rustc-stable-hash",
 ]
 [[package]]
 name = "rustc_hir"
 version = "0.0.0"
@ -3849,6 +3691,7 @@ dependencies = [
 "rustc_arena",
 "rustc_ast",
 "rustc_data_structures",
 "rustc_hashes",
 "rustc_index",
 "rustc_macros",
 "rustc_serialize",
@ -3905,7 +3748,6 @@ dependencies = [
 "itertools",
 "rustc_abi",
 "rustc_ast",
 "rustc_ast_ir",
 "rustc_attr_parsing",
 "rustc_data_structures",
 "rustc_errors",
@ -3937,6 +3779,7 @@ dependencies = [
 "rustc_fluent_macro",
 "rustc_fs_util",
 "rustc_graphviz",
 "rustc_hashes",
 "rustc_hir",
 "rustc_macros",
 "rustc_middle",
@ -3970,7 +3813,6 @@ dependencies = [
 name = "rustc_infer"
 version = "0.0.0"
 dependencies = [
 "rustc_ast_ir",
 "rustc_data_structures",
 "rustc_errors",
 "rustc_fluent_macro",
@ -4161,7 +4003,6 @@ dependencies = [
 "rustc_apfloat",
 "rustc_arena",
 "rustc_ast",
 "rustc_ast_ir",
 "rustc_attr_parsing",
 "rustc_data_structures",
 "rustc_error_messages",
@ -4169,6 +4010,7 @@ dependencies = [
 "rustc_feature",
 "rustc_fluent_macro",
 "rustc_graphviz",
 "rustc_hashes",
 "rustc_hir",
 "rustc_hir_pretty",
 "rustc_index",
@ -4290,7 +4132,6 @@ name = "rustc_next_trait_solver"
 version = "0.0.0"
 dependencies = [
 "derive-where",
 "rustc_ast_ir",
 "rustc_data_structures",
 "rustc_index",
 "rustc_macros",
@ -4405,6 +4246,7 @@ dependencies = [
 "measureme",
 "rustc_data_structures",
 "rustc_errors",
 "rustc_hashes",
 "rustc_hir",
 "rustc_index",
 "rustc_middle",
@ -4428,6 +4270,7 @@ dependencies = [
 "rustc_errors",
 "rustc_feature",
 "rustc_fluent_macro",
 "rustc_hashes",
 "rustc_hir",
 "rustc_index",
 "rustc_macros",
@ -4488,6 +4331,7 @@ name = "rustc_serialize"
 version = "0.0.0"
 dependencies = [
 "indexmap",
 "rustc_hashes",
 "rustc_macros",
 "smallvec",
 "tempfile",
@ -4508,6 +4352,7 @@ dependencies = [
 "rustc_feature",
 "rustc_fluent_macro",
 "rustc_fs_util",
 "rustc_hashes",
 "rustc_hir",
 "rustc_lint_defs",
 "rustc_macros",
@ -4549,6 +4394,7 @@ dependencies = [
 "md-5",
 "rustc_arena",
 "rustc_data_structures",
 "rustc_hashes",
 "rustc_index",
 "rustc_macros",
 "rustc_serialize",
@ -4568,6 +4414,7 @@ dependencies = [
 "rustc_abi",
 "rustc_data_structures",
 "rustc_errors",
 "rustc_hashes",
 "rustc_hir",
 "rustc_middle",
 "rustc_session",
@ -4604,7 +4451,6 @@ dependencies = [
 "itertools",
 "rustc_abi",
 "rustc_ast",
 "rustc_ast_ir",
 "rustc_attr_parsing",
 "rustc_data_structures",
 "rustc_errors",
@ -4643,7 +4489,6 @@ version = "0.0.0"
 dependencies = [
 "itertools",
 "rustc_abi",
 "rustc_ast_ir",
 "rustc_data_structures",
 "rustc_hir",
 "rustc_infer",
@ -4659,10 +4504,10 @@ version = "0.0.0"
 dependencies = [
 "itertools",
 "rustc_abi",
 "rustc_ast_ir",
 "rustc_data_structures",
 "rustc_errors",
 "rustc_fluent_macro",
 "rustc_hashes",
 "rustc_hir",
 "rustc_index",
 "rustc_infer",
@ -5103,9 +4948,9 @@ dependencies = [
 [[package]]
 name = "stacker"
-version = "0.1.17"
+version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b"
+checksum = "1d08feb8f695b465baed819b03c128dc23f57a694510ab1f06c77f763975685e"
 dependencies = [
 "cc",
 "cfg-if",
@ -5289,16 +5134,6 @@ dependencies = [
 "winapi-util",
 ]
 [[package]]
 name = "terminal_size"
 version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5352447f921fda68cf61b4101566c0bdb5104eff6804d0678e5227580ab6a4e9"
 dependencies = [
 "rustix",
 "windows-sys 0.59.0",
 ]
 [[package]]
 name = "termize"
 version = "0.1.1"
@ -5539,12 +5374,6 @@ dependencies = [
 "winnow",
 ]
 [[package]]
 name = "topological-sort"
 version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ea68304e134ecd095ac6c3574494fc62b909f416c4fca77e440530221e549d3d"
 [[package]]
 name = "tracing"
 version = "0.1.37"
@ -5665,12 +5494,6 @@ dependencies = [
 "regex-lite",
 ]
 [[package]]
 name = "ucd-trie"
 version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
 [[package]]
 name = "ui_test"
 version = "0.26.5"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -13,7 +13,6 @@ members = [
  "src/tools/clippy/clippy_dev",
  "src/tools/compiletest",
  "src/tools/run-make-support",
  "src/tools/error_index_generator",
  "src/tools/linkchecker",
  "src/tools/lint-docs",
  "src/tools/miropt-test-tools",
--- a/RELEASES.md
+++ b/RELEASES.md
@ -34,7 +34,8 @@ Libraries
 ---------
 - [Panics in the standard library now have a leading `library/` in their path](https://github.com/rust-lang/rust/pull/132390)
 - [`std::env::home_dir()` on Windows now ignores the non-standard `$HOME` environment variable](https://github.com/rust-lang/rust/pull/132515)
-   It will be un-deprecated in a subsequent release.
+
  It will be un-deprecated in a subsequent release.
 - [Add `AsyncFn*` to the prelude in all editions.](https://github.com/rust-lang/rust/pull/132611)
 <a id="1.85.0-Stabilized-APIs"></a>
@ -98,15 +99,18 @@ Rustdoc
 Compatibility Notes
 -------------------
 - [`rustc` no longer treats the `test` cfg as a well known check-cfg](https://github.com/rust-lang/rust/pull/131729), instead it is up to the build systems and users of `--check-cfg`[^check-cfg] to set it as a well known cfg using `--check-cfg=cfg(test)`.
  This is done to enable build systems like Cargo to set it conditionally, as not all source files are suitable for unit tests.
  [Cargo (for now) unconditionally sets the `test` cfg as a well known cfg](https://github.com/rust-lang/cargo/pull/14963).
-[^check-cfg]: https://doc.rust-lang.org/nightly/rustc/check-cfg.html
+  [^check-cfg]: https://doc.rust-lang.org/nightly/rustc/check-cfg.html
 - [Disable potentially incorrect type inference if there are trivial and non-trivial where-clauses](https://github.com/rust-lang/rust/pull/132325)
 - `std::env::home_dir()` has been deprecated for years, because it can give surprising results in some Windows configurations if the `HOME` environment variable is set (which is not the normal configuration on Windows). We had previously avoided changing its behavior, out of concern for compatibility with code depending on this non-standard configuration. Given how long this function has been deprecated, we're now fixing its behavior as a bugfix. A subsequent release will remove the deprecation for this function.
 - [Make `core::ffi::c_char` signedness more closely match that of the platform-default `char`](https://github.com/rust-lang/rust/pull/132975)
  This changed `c_char` from an `i8` to `u8` or vice versa on many Tier 2 and 3
  targets (mostly Arm and RISC-V embedded targets). The new definition may
  result in compilation failures but fixes compatibility issues with C.
  The `libc` crate matches this change as of its 0.2.169 release.
 - [When compiling a nested `macro_rules` macro from an external crate, the content of the inner `macro_rules` is now built with the edition of the external crate, not the local crate.](https://github.com/rust-lang/rust/pull/133274)
 - [Increase `sparcv9-sun-solaris` and `x86_64-pc-solaris` Solaris baseline to 11.4.](https://github.com/rust-lang/rust/pull/133293)
--- a/compiler/rustc/Cargo.toml
+++ b/compiler/rustc/Cargo.toml
@ -1,7 +1,7 @@
 [package]
 name = "rustc-main"
 version = "0.0.0"
-edition = "2021"
+edition = "2024"
 [dependencies]
 # tidy-alphabetical-start
--- a/compiler/rustc/src/main.rs
+++ b/compiler/rustc/src/main.rs
@ -65,7 +65,7 @@ fn main() {
        // linking, so we need to explicitly depend on the function.
        #[cfg(target_os = "macos")]
        {
-            extern "C" {
+            unsafe extern "C" {
                fn _rjem_je_zone_register();
            }
--- a/compiler/rustc_abi/Cargo.toml
+++ b/compiler/rustc_abi/Cargo.toml
@ -1,7 +1,7 @@
 [package]
 name = "rustc_abi"
 version = "0.0.0"
-edition = "2021"
+edition = "2024"
 [dependencies]
 # tidy-alphabetical-start
@ -9,6 +9,7 @@ bitflags = "2.4.1"
 rand = { version = "0.8.4", default-features = false, optional = true }
 rand_xoshiro = { version = "0.6.0", optional = true }
 rustc_data_structures = { path = "../rustc_data_structures", optional = true  }
 rustc_hashes = { path = "../rustc_hashes" }
 rustc_index = { path = "../rustc_index", default-features = false }
 rustc_macros = { path = "../rustc_macros", optional = true }
 rustc_serialize = { path = "../rustc_serialize", optional = true  }
--- a/compiler/rustc_abi/src/callconv.rs
+++ b/compiler/rustc_abi/src/callconv.rs
@ -65,8 +65,6 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
        Ty: TyAbiInterface<'a, C> + Copy,
    {
        match self.backend_repr {
            BackendRepr::Uninhabited => Err(Heterogeneous),
            // The primitive for this algorithm.
            BackendRepr::Scalar(scalar) => {
                let kind = match scalar.primitive() {
--- a/compiler/rustc_abi/src/callconv/reg.rs
+++ b/compiler/rustc_abi/src/callconv/reg.rs
@ -57,7 +57,7 @@ impl Reg {
                128 => dl.f128_align.abi,
                _ => panic!("unsupported float: {self:?}"),
            },
-            RegKind::Vector => dl.vector_align(self.size).abi,
+            RegKind::Vector => dl.llvmlike_vector_align(self.size).abi,
        }
    }
 }
--- a/compiler/rustc_abi/src/layout.rs
+++ b/compiler/rustc_abi/src/layout.rs
@ -2,6 +2,7 @@ use std::fmt::{self, Write};
 use std::ops::{Bound, Deref};
 use std::{cmp, iter};
 use rustc_hashes::Hash64;
 use rustc_index::Idx;
 use tracing::debug;
@ -129,11 +130,12 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
            },
            backend_repr: BackendRepr::ScalarPair(a, b),
            largest_niche,
            uninhabited: false,
            align,
            size,
            max_repr_align: None,
            unadjusted_abi_align: align.abi,
-            randomization_seed: combined_seed,
+            randomization_seed: Hash64::new(combined_seed),
        }
    }
@ -220,13 +222,14 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
        LayoutData {
            variants: Variants::Empty,
            fields: FieldsShape::Primitive,
-            backend_repr: BackendRepr::Uninhabited,
+            backend_repr: BackendRepr::Memory { sized: true },
            largest_niche: None,
            uninhabited: true,
            align: dl.i8_align,
            size: Size::ZERO,
            max_repr_align: None,
            unadjusted_abi_align: dl.i8_align.abi,
-            randomization_seed: 0,
+            randomization_seed: Hash64::ZERO,
        }
    }
@ -307,10 +310,10 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
        let mut align = if repr.pack.is_some() { dl.i8_align } else { dl.aggregate_align };
        let mut max_repr_align = repr.align;
-        // If all the non-ZST fields have the same ABI and union ABI optimizations aren't
+        // If all the non-ZST fields have the same repr and union repr optimizations aren't
-        // disabled, we can use that common ABI for the union as a whole.
+        // disabled, we can use that common repr for the union as a whole.
        struct AbiMismatch;
-        let mut common_non_zst_abi_and_align = if repr.inhibits_union_abi_opt() {
+        let mut common_non_zst_repr_and_align = if repr.inhibits_union_abi_opt() {
            // Can't optimize
            Err(AbiMismatch)
        } else {
@ -334,14 +337,14 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                continue;
            }
-            if let Ok(common) = common_non_zst_abi_and_align {
+            if let Ok(common) = common_non_zst_repr_and_align {
                // Discard valid range information and allow undef
                let field_abi = field.backend_repr.to_union();
                if let Some((common_abi, common_align)) = common {
                    if common_abi != field_abi {
                        // Different fields have different ABI: disable opt
-                        common_non_zst_abi_and_align = Err(AbiMismatch);
+                        common_non_zst_repr_and_align = Err(AbiMismatch);
                    } else {
                        // Fields with the same non-Aggregate ABI should also
                        // have the same alignment
@ -354,7 +357,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                    }
                } else {
                    // First non-ZST field: record its ABI and alignment
-                    common_non_zst_abi_and_align = Ok(Some((field_abi, field.align.abi)));
+                    common_non_zst_repr_and_align = Ok(Some((field_abi, field.align.abi)));
                }
            }
        }
@ -373,16 +376,25 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
        // If all non-ZST fields have the same ABI, we may forward that ABI
        // for the union as a whole, unless otherwise inhibited.
-        let abi = match common_non_zst_abi_and_align {
+        let backend_repr = match common_non_zst_repr_and_align {
            Err(AbiMismatch) | Ok(None) => BackendRepr::Memory { sized: true },
-            Ok(Some((abi, _))) => {
+            Ok(Some((repr, _))) => match repr {
-                if abi.inherent_align(dl).map(|a| a.abi) != Some(align.abi) {
+                // Mismatched alignment (e.g. union is #[repr(packed)]): disable opt
-                    // Mismatched alignment (e.g. union is #[repr(packed)]): disable opt
+                BackendRepr::Scalar(_) | BackendRepr::ScalarPair(_, _)
                    if repr.scalar_align(dl).unwrap() != align.abi =>
                {
                    BackendRepr::Memory { sized: true }
                } else {
                    abi
                }
-            }
+                // Vectors require at least element alignment, else disable the opt
                BackendRepr::Vector { element, count: _ } if element.align(dl).abi > align.abi => {
                    BackendRepr::Memory { sized: true }
                }
                // the alignment tests passed and we can use this
                BackendRepr::Scalar(..)
                | BackendRepr::ScalarPair(..)
                | BackendRepr::Vector { .. }
                | BackendRepr::Memory { .. } => repr,
            },
        };
        let Some(union_field_count) = NonZeroUsize::new(only_variant.len()) else {
@ -397,8 +409,9 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
        Ok(LayoutData {
            variants: Variants::Single { index: only_variant_idx },
            fields: FieldsShape::Union(union_field_count),
-            backend_repr: abi,
+            backend_repr,
            largest_niche: None,
            uninhabited: false,
            align,
            size: size.align_to(align.abi),
            max_repr_align,
@ -446,7 +459,6 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                Scalar::Union { .. } => {}
            };
            match &mut st.backend_repr {
                BackendRepr::Uninhabited => {}
                BackendRepr::Scalar(scalar) => hide_niches(scalar),
                BackendRepr::ScalarPair(a, b) => {
                    hide_niches(a);
@ -638,9 +650,8 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
            let same_size = size == variant_layouts[largest_variant_index].size;
            let same_align = align == variant_layouts[largest_variant_index].align;
-            let abi = if variant_layouts.iter().all(|v| v.is_uninhabited()) {
+            let uninhabited = variant_layouts.iter().all(|v| v.is_uninhabited());
-                BackendRepr::Uninhabited
+            let abi = if same_size && same_align && others_zst {
            } else if same_size && same_align && others_zst {
                match variant_layouts[largest_variant_index].backend_repr {
                    // When the total alignment and size match, we can use the
                    // same ABI as the scalar variant with the reserved niche.
@ -682,6 +693,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                },
                backend_repr: abi,
                largest_niche,
                uninhabited,
                size,
                align,
                max_repr_align,
@ -852,9 +864,8 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
        };
        let mut abi = BackendRepr::Memory { sized: true };
-        if layout_variants.iter().all(|v| v.is_uninhabited()) {
+        let uninhabited = layout_variants.iter().all(|v| v.is_uninhabited());
-            abi = BackendRepr::Uninhabited;
+        if tag.size(dl) == size {
        } else if tag.size(dl) == size {
            // Make sure we only use scalar layout when the enum is entirely its
            // own tag (i.e. it has no padding nor any non-ZST variant fields).
            abi = BackendRepr::Scalar(tag);
@ -994,6 +1005,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                memory_index: [0].into(),
            },
            largest_niche,
            uninhabited,
            backend_repr: abi,
            align,
            size,
@ -1058,7 +1070,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
        // unsizable tail fields are excluded so that we use the same seed for the sized and unsized layouts.
        let field_seed = fields_excluding_tail
            .iter()
-            .fold(0u64, |acc, f| acc.wrapping_add(f.randomization_seed));
+            .fold(Hash64::ZERO, |acc, f| acc.wrapping_add(f.randomization_seed));
        if optimize_field_order && fields.len() > 1 {
            // If `-Z randomize-layout` was enabled for the type definition we can shuffle
@ -1072,7 +1084,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                    // `ReprOptions.field_shuffle_seed` is a deterministic seed we can use to randomize field
                    // ordering.
                    let mut rng = rand_xoshiro::Xoshiro128StarStar::seed_from_u64(
-                        field_seed.wrapping_add(repr.field_shuffle_seed),
+                        field_seed.wrapping_add(repr.field_shuffle_seed).as_u64(),
                    );
                    // Shuffle the ordering of the fields.
@ -1354,9 +1366,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                _ => {}
            }
        }
-        if fields.iter().any(|f| f.is_uninhabited()) {
+        let uninhabited = fields.iter().any(|f| f.is_uninhabited());
            abi = BackendRepr::Uninhabited;
        }
        let unadjusted_abi_align = if repr.transparent() {
            match layout_of_single_non_zst_field {
@ -1377,6 +1387,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
            fields: FieldsShape::Arbitrary { offsets, memory_index },
            backend_repr: abi,
            largest_niche,
            uninhabited,
            align,
            size,
            max_repr_align,
--- a/compiler/rustc_abi/src/lib.rs
+++ b/compiler/rustc_abi/src/lib.rs
@ -50,6 +50,7 @@ use std::str::FromStr;
 use bitflags::bitflags;
 #[cfg(feature = "nightly")]
 use rustc_data_structures::stable_hasher::StableOrd;
 use rustc_hashes::Hash64;
 use rustc_index::{Idx, IndexSlice, IndexVec};
 #[cfg(feature = "nightly")]
 use rustc_macros::{Decodable_Generic, Encodable_Generic, HashStable_Generic};
@ -140,7 +141,7 @@ pub struct ReprOptions {
    /// hash without loss, but it does pay the price of being larger.
    /// Everything's a tradeoff, a 64-bit seed should be sufficient for our
    /// purposes (primarily `-Z randomize-layout`)
-    pub field_shuffle_seed: u64,
+    pub field_shuffle_seed: Hash64,
 }
 impl ReprOptions {
@ -328,19 +329,19 @@ impl TargetDataLayout {
                [p] if p.starts_with('P') => {
                    dl.instruction_address_space = parse_address_space(&p[1..], "P")?
                }
-                ["a", ref a @ ..] => dl.aggregate_align = parse_align(a, "a")?,
+                ["a", a @ ..] => dl.aggregate_align = parse_align(a, "a")?,
-                ["f16", ref a @ ..] => dl.f16_align = parse_align(a, "f16")?,
+                ["f16", a @ ..] => dl.f16_align = parse_align(a, "f16")?,
-                ["f32", ref a @ ..] => dl.f32_align = parse_align(a, "f32")?,
+                ["f32", a @ ..] => dl.f32_align = parse_align(a, "f32")?,
-                ["f64", ref a @ ..] => dl.f64_align = parse_align(a, "f64")?,
+                ["f64", a @ ..] => dl.f64_align = parse_align(a, "f64")?,
-                ["f128", ref a @ ..] => dl.f128_align = parse_align(a, "f128")?,
+                ["f128", a @ ..] => dl.f128_align = parse_align(a, "f128")?,
                // FIXME(erikdesjardins): we should be parsing nonzero address spaces
                // this will require replacing TargetDataLayout::{pointer_size,pointer_align}
                // with e.g. `fn pointer_size_in(AddressSpace)`
-                [p @ "p", s, ref a @ ..] | [p @ "p0", s, ref a @ ..] => {
+                [p @ "p", s, a @ ..] | [p @ "p0", s, a @ ..] => {
                    dl.pointer_size = parse_size(s, p)?;
                    dl.pointer_align = parse_align(a, p)?;
                }
-                [s, ref a @ ..] if s.starts_with('i') => {
+                [s, a @ ..] if s.starts_with('i') => {
                    let Ok(bits) = s[1..].parse::<u64>() else {
                        parse_size(&s[1..], "i")?; // For the user error.
                        continue;
@ -361,7 +362,7 @@ impl TargetDataLayout {
                        dl.i128_align = a;
                    }
                }
-                [s, ref a @ ..] if s.starts_with('v') => {
+                [s, a @ ..] if s.starts_with('v') => {
                    let v_size = parse_size(&s[1..], "v")?;
                    let a = parse_align(a, s)?;
                    if let Some(v) = dl.vector_align.iter_mut().find(|v| v.0 == v_size) {
@ -407,16 +408,21 @@ impl TargetDataLayout {
        }
    }
    /// psABI-mandated alignment for a vector type, if any
    #[inline]
-    pub fn vector_align(&self, vec_size: Size) -> AbiAndPrefAlign {
+    fn cabi_vector_align(&self, vec_size: Size) -> Option<AbiAndPrefAlign> {
-        for &(size, align) in &self.vector_align {
+        self.vector_align
-            if size == vec_size {
+            .iter()
-                return align;
+            .find(|(size, _align)| *size == vec_size)
-            }
+            .map(|(_size, align)| *align)
-        }
+    }
-        // Default to natural alignment, which is what LLVM does.
+
-        // That is, use the size, rounded up to a power of 2.
+    /// an alignment resembling the one LLVM would pick for a vector
-        AbiAndPrefAlign::new(Align::from_bytes(vec_size.bytes().next_power_of_two()).unwrap())
+    #[inline]
    pub fn llvmlike_vector_align(&self, vec_size: Size) -> AbiAndPrefAlign {
        self.cabi_vector_align(vec_size).unwrap_or(AbiAndPrefAlign::new(
            Align::from_bytes(vec_size.bytes().next_power_of_two()).unwrap(),
        ))
    }
 }
@ -809,20 +815,19 @@ impl Align {
        self.bits().try_into().unwrap()
    }
-    /// Computes the best alignment possible for the given offset
+    /// Obtain the greatest factor of `size` that is an alignment
-    /// (the largest power of two that the offset is a multiple of).
+    /// (the largest power of two the Size is a multiple of).
    ///
-    /// N.B., for an offset of `0`, this happens to return `2^64`.
+    /// Note that all numbers are factors of 0
    #[inline]
-    pub fn max_for_offset(offset: Size) -> Align {
+    pub fn max_aligned_factor(size: Size) -> Align {
-        Align { pow2: offset.bytes().trailing_zeros() as u8 }
+        Align { pow2: size.bytes().trailing_zeros() as u8 }
    }
-    /// Lower the alignment, if necessary, such that the given offset
+    /// Reduces Align to an aligned factor of `size`.
    /// is aligned to it (the offset is a multiple of the alignment).
    #[inline]
-    pub fn restrict_for_offset(self, offset: Size) -> Align {
+    pub fn restrict_for_offset(self, size: Size) -> Align {
-        self.min(Align::max_for_offset(offset))
+        self.min(Align::max_aligned_factor(size))
    }
 }
@ -1344,7 +1349,7 @@ impl<FieldIdx: Idx> FieldsShape<FieldIdx> {
    /// Gets source indices of the fields by increasing offsets.
    #[inline]
-    pub fn index_by_increasing_offset(&self) -> impl ExactSizeIterator<Item = usize> + '_ {
+    pub fn index_by_increasing_offset(&self) -> impl ExactSizeIterator<Item = usize> {
        let mut inverse_small = [0u8; 64];
        let mut inverse_big = IndexVec::new();
        let use_small = self.count() <= inverse_small.len();
@ -1403,7 +1408,6 @@ impl AddressSpace {
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
 #[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
 pub enum BackendRepr {
    Uninhabited,
    Scalar(Scalar),
    ScalarPair(Scalar, Scalar),
    Vector {
@ -1422,10 +1426,9 @@ impl BackendRepr {
    #[inline]
    pub fn is_unsized(&self) -> bool {
        match *self {
-            BackendRepr::Uninhabited
+            BackendRepr::Scalar(_) | BackendRepr::ScalarPair(..) | BackendRepr::Vector { .. } => {
-            | BackendRepr::Scalar(_)
+                false
-            | BackendRepr::ScalarPair(..)
+            }
            | BackendRepr::Vector { .. } => false,
            BackendRepr::Memory { sized } => !sized,
        }
    }
@ -1444,12 +1447,6 @@ impl BackendRepr {
        }
    }
    /// Returns `true` if this is an uninhabited type
    #[inline]
    pub fn is_uninhabited(&self) -> bool {
        matches!(*self, BackendRepr::Uninhabited)
    }
    /// Returns `true` if this is a scalar type
    #[inline]
    pub fn is_scalar(&self) -> bool {
@ -1462,37 +1459,38 @@ impl BackendRepr {
        matches!(*self, BackendRepr::Scalar(s) if s.is_bool())
    }
-    /// Returns the fixed alignment of this ABI, if any is mandated.
+    /// The psABI alignment for a `Scalar` or `ScalarPair`
-    pub fn inherent_align<C: HasDataLayout>(&self, cx: &C) -> Option<AbiAndPrefAlign> {
+    ///
-        Some(match *self {
+    /// `None` for other variants.
-            BackendRepr::Scalar(s) => s.align(cx),
+    pub fn scalar_align<C: HasDataLayout>(&self, cx: &C) -> Option<Align> {
-            BackendRepr::ScalarPair(s1, s2) => s1.align(cx).max(s2.align(cx)),
+        match *self {
-            BackendRepr::Vector { element, count } => {
+            BackendRepr::Scalar(s) => Some(s.align(cx).abi),
-                cx.data_layout().vector_align(element.size(cx) * count)
+            BackendRepr::ScalarPair(s1, s2) => Some(s1.align(cx).max(s2.align(cx)).abi),
-            }
+            // The align of a Vector can vary in surprising ways
-            BackendRepr::Uninhabited | BackendRepr::Memory { .. } => return None,
+            BackendRepr::Vector { .. } | BackendRepr::Memory { .. } => None,
-        })
+        }
    }
-    /// Returns the fixed size of this ABI, if any is mandated.
+    /// The psABI size for a `Scalar` or `ScalarPair`
-    pub fn inherent_size<C: HasDataLayout>(&self, cx: &C) -> Option<Size> {
+    ///
-        Some(match *self {
+    /// `None` for other variants
-            BackendRepr::Scalar(s) => {
+    pub fn scalar_size<C: HasDataLayout>(&self, cx: &C) -> Option<Size> {
-                // No padding in scalars.
+        match *self {
-                s.size(cx)
+            // No padding in scalars.
-            }
+            BackendRepr::Scalar(s) => Some(s.size(cx)),
            // May have some padding between the pair.
            BackendRepr::ScalarPair(s1, s2) => {
                // May have some padding between the pair.
                let field2_offset = s1.size(cx).align_to(s2.align(cx).abi);
-                (field2_offset + s2.size(cx)).align_to(self.inherent_align(cx)?.abi)
+                let size = (field2_offset + s2.size(cx)).align_to(
                    self.scalar_align(cx)
                        // We absolutely must have an answer here or everything is FUBAR.
                        .unwrap(),
                );
                Some(size)
            }
-            BackendRepr::Vector { element, count } => {
+            // The size of a Vector can vary in surprising ways
-                // No padding in vectors, except possibly for trailing padding
+            BackendRepr::Vector { .. } | BackendRepr::Memory { .. } => None,
-                // to make the size a multiple of align (e.g. for vectors of size 3).
+        }
                (element.size(cx) * count).align_to(self.inherent_align(cx)?.abi)
            }
            BackendRepr::Uninhabited | BackendRepr::Memory { .. } => return None,
        })
    }
    /// Discard validity range information and allow undef.
@ -1505,9 +1503,7 @@ impl BackendRepr {
            BackendRepr::Vector { element, count } => {
                BackendRepr::Vector { element: element.to_union(), count }
            }
-            BackendRepr::Uninhabited | BackendRepr::Memory { .. } => {
+            BackendRepr::Memory { .. } => BackendRepr::Memory { sized: true },
                BackendRepr::Memory { sized: true }
            }
        }
    }
@ -1703,6 +1699,11 @@ pub struct LayoutData<FieldIdx: Idx, VariantIdx: Idx> {
    /// The leaf scalar with the largest number of invalid values
    /// (i.e. outside of its `valid_range`), if it exists.
    pub largest_niche: Option<Niche>,
    /// Is this type known to be uninhabted?
    ///
    /// This is separate from BackendRepr because uninhabited return types can affect ABI,
    /// especially in the case of by-pointer struct returns, which allocate stack even when unused.
    pub uninhabited: bool,
    pub align: AbiAndPrefAlign,
    pub size: Size,
@ -1727,21 +1728,21 @@ pub struct LayoutData<FieldIdx: Idx, VariantIdx: Idx> {
    /// transmuted to `Foo<U>` we aim to create probalistically distinct seeds so that Foo can choose
    /// to reorder its fields based on that information. The current implementation is a conservative
    /// approximation of this goal.
-    pub randomization_seed: u64,
+    pub randomization_seed: Hash64,
 }
 impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
    /// Returns `true` if this is an aggregate type (including a ScalarPair!)
    pub fn is_aggregate(&self) -> bool {
        match self.backend_repr {
-            BackendRepr::Uninhabited | BackendRepr::Scalar(_) | BackendRepr::Vector { .. } => false,
+            BackendRepr::Scalar(_) | BackendRepr::Vector { .. } => false,
            BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => true,
        }
    }
    /// Returns `true` if this is an uninhabited type
    pub fn is_uninhabited(&self) -> bool {
-        self.backend_repr.is_uninhabited()
+        self.uninhabited
    }
    pub fn scalar<C: HasDataLayout>(cx: &C, scalar: Scalar) -> Self {
@ -1777,11 +1778,12 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
            fields: FieldsShape::Primitive,
            backend_repr: BackendRepr::Scalar(scalar),
            largest_niche,
            uninhabited: false,
            size,
            align,
            max_repr_align: None,
            unadjusted_abi_align: align.abi,
-            randomization_seed,
+            randomization_seed: Hash64::new(randomization_seed),
        }
    }
 }
@ -1801,10 +1803,11 @@ where
            backend_repr,
            fields,
            largest_niche,
            uninhabited,
            variants,
            max_repr_align,
            unadjusted_abi_align,
-            ref randomization_seed,
+            randomization_seed,
        } = self;
        f.debug_struct("Layout")
            .field("size", size)
@ -1812,6 +1815,7 @@ where
            .field("abi", backend_repr)
            .field("fields", fields)
            .field("largest_niche", largest_niche)
            .field("uninhabited", uninhabited)
            .field("variants", variants)
            .field("max_repr_align", max_repr_align)
            .field("unadjusted_abi_align", unadjusted_abi_align)
@ -1876,7 +1880,6 @@ impl<FieldIdx: Idx, VariantIdx: Idx> LayoutData<FieldIdx, VariantIdx> {
            BackendRepr::Scalar(_) | BackendRepr::ScalarPair(..) | BackendRepr::Vector { .. } => {
                false
            }
            BackendRepr::Uninhabited => self.size.bytes() == 0,
            BackendRepr::Memory { sized } => sized && self.size.bytes() == 0,
        }
    }
--- a/compiler/rustc_arena/Cargo.toml
+++ b/compiler/rustc_arena/Cargo.toml
@ -1,7 +1,7 @@
 [package]
 name = "rustc_arena"
 version = "0.0.0"
-edition = "2021"
+edition = "2024"
 [dependencies]
 # tidy-alphabetical-start
--- a/compiler/rustc_ast/Cargo.toml
+++ b/compiler/rustc_ast/Cargo.toml
@ -1,7 +1,7 @@
 [package]
 name = "rustc_ast"
 version = "0.0.0"
-edition = "2021"
+edition = "2024"
 [dependencies]
 # tidy-alphabetical-start
--- a/compiler/rustc_ast/src/ast_traits.rs
+++ b/compiler/rustc_ast/src/ast_traits.rs
@ -203,10 +203,8 @@ impl HasTokens for Nonterminal {
            Nonterminal::NtStmt(stmt) => stmt.tokens(),
            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens(),
            Nonterminal::NtPat(pat) => pat.tokens(),
            Nonterminal::NtTy(ty) => ty.tokens(),
            Nonterminal::NtMeta(attr_item) => attr_item.tokens(),
            Nonterminal::NtPath(path) => path.tokens(),
            Nonterminal::NtVis(vis) => vis.tokens(),
            Nonterminal::NtBlock(block) => block.tokens(),
        }
    }
@ -216,10 +214,8 @@ impl HasTokens for Nonterminal {
            Nonterminal::NtStmt(stmt) => stmt.tokens_mut(),
            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => expr.tokens_mut(),
            Nonterminal::NtPat(pat) => pat.tokens_mut(),
            Nonterminal::NtTy(ty) => ty.tokens_mut(),
            Nonterminal::NtMeta(attr_item) => attr_item.tokens_mut(),
            Nonterminal::NtPath(path) => path.tokens_mut(),
            Nonterminal::NtVis(vis) => vis.tokens_mut(),
            Nonterminal::NtBlock(block) => block.tokens_mut(),
        }
    }
--- a/compiler/rustc_ast/src/expand/autodiff_attrs.rs
+++ b/compiler/rustc_ast/src/expand/autodiff_attrs.rs
@ -17,7 +17,6 @@ use crate::{Ty, TyKind};
 /// functions. The proper solution is to recognize and resolve this DAG of autodiff invocations,
 /// as it's already done in the C++ and Julia frontend of Enzyme.
 ///
 /// (FIXME) remove *First variants.
 /// Documentation for using [reverse](https://enzyme.mit.edu/rust/rev.html) and
 /// [forward](https://enzyme.mit.edu/rust/fwd.html) mode is available online.
 #[derive(Clone, Copy, Eq, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
--- a/compiler/rustc_ast/src/mut_visit.rs
+++ b/compiler/rustc_ast/src/mut_visit.rs
@ -907,7 +907,6 @@ fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
        }),
        token::NtPat(pat) => vis.visit_pat(pat),
        token::NtExpr(expr) => vis.visit_expr(expr),
        token::NtTy(ty) => vis.visit_ty(ty),
        token::NtLiteral(expr) => vis.visit_expr(expr),
        token::NtMeta(item) => {
            let AttrItem { unsafety: _, path, args, tokens } = item.deref_mut();
@ -916,7 +915,6 @@ fn visit_nonterminal<T: MutVisitor>(vis: &mut T, nt: &mut token::Nonterminal) {
            visit_lazy_tts(vis, tokens);
        }
        token::NtPath(path) => vis.visit_path(path),
        token::NtVis(visib) => vis.visit_vis(visib),
    }
 }
--- a/compiler/rustc_ast/src/token.rs
+++ b/compiler/rustc_ast/src/token.rs
@ -84,7 +84,9 @@ pub enum MetaVarKind {
        // This field is needed for `Token::can_begin_string_literal`.
        can_begin_string_literal: bool,
    },
-    Ty,
+    Ty {
        is_path: bool,
    },
    Ident,
    Lifetime,
    Literal,
@ -104,7 +106,7 @@ impl fmt::Display for MetaVarKind {
            MetaVarKind::Pat(PatParam { inferred: false }) => sym::pat_param,
            MetaVarKind::Expr { kind: Expr2021 { inferred: true } | Expr, .. } => sym::expr,
            MetaVarKind::Expr { kind: Expr2021 { inferred: false }, .. } => sym::expr_2021,
-            MetaVarKind::Ty => sym::ty,
+            MetaVarKind::Ty { .. } => sym::ty,
            MetaVarKind::Ident => sym::ident,
            MetaVarKind::Lifetime => sym::lifetime,
            MetaVarKind::Literal => sym::literal,
@ -659,7 +661,6 @@ impl Token {
                    | NtMeta(..)
                    | NtPat(..)
                    | NtPath(..)
                    | NtTy(..)
                ),
            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
                MetaVarKind::Expr { .. } |
@ -667,7 +668,7 @@ impl Token {
                MetaVarKind::Meta |
                MetaVarKind::Pat(_) |
                MetaVarKind::Path |
-                MetaVarKind::Ty
+                MetaVarKind::Ty { .. }
            ))) => true,
            _ => false,
        }
@ -688,9 +689,9 @@ impl Token {
            Lifetime(..)                | // lifetime bound in trait object
            Lt | BinOp(Shl)             | // associated path
            PathSep                      => true, // global path
-            Interpolated(ref nt) => matches!(&**nt, NtTy(..) | NtPath(..)),
+            Interpolated(ref nt) => matches!(&**nt, NtPath(..)),
            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
-                MetaVarKind::Ty |
+                MetaVarKind::Ty { .. } |
                MetaVarKind::Path
            ))) => true,
            // For anonymous structs or unions, which only appear in specific positions
@ -969,6 +970,15 @@ impl Token {
        }
    }
    /// Is this an invisible open delimiter at the start of a token sequence
    /// from an expanded metavar?
    pub fn is_metavar_seq(&self) -> Option<MetaVarKind> {
        match self.kind {
            OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => Some(kind),
            _ => None,
        }
    }
    pub fn glue(&self, joint: &Token) -> Option<Token> {
        let kind = match self.kind {
            Eq => match joint.kind {
@ -1067,12 +1077,10 @@ pub enum Nonterminal {
    NtStmt(P<ast::Stmt>),
    NtPat(P<ast::Pat>),
    NtExpr(P<ast::Expr>),
    NtTy(P<ast::Ty>),
    NtLiteral(P<ast::Expr>),
    /// Stuff inside brackets for attributes
    NtMeta(P<ast::AttrItem>),
    NtPath(P<ast::Path>),
    NtVis(P<ast::Visibility>),
 }
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
@ -1166,10 +1174,8 @@ impl Nonterminal {
            NtStmt(stmt) => stmt.span,
            NtPat(pat) => pat.span,
            NtExpr(expr) | NtLiteral(expr) => expr.span,
            NtTy(ty) => ty.span,
            NtMeta(attr_item) => attr_item.span(),
            NtPath(path) => path.span,
            NtVis(vis) => vis.span,
        }
    }
@ -1181,10 +1187,8 @@ impl Nonterminal {
            NtPat(..) => "pattern",
            NtExpr(..) => "expression",
            NtLiteral(..) => "literal",
            NtTy(..) => "type",
            NtMeta(..) => "attribute",
            NtPath(..) => "path",
            NtVis(..) => "visibility",
        }
    }
 }
@ -1207,11 +1211,9 @@ impl fmt::Debug for Nonterminal {
            NtStmt(..) => f.pad("NtStmt(..)"),
            NtPat(..) => f.pad("NtPat(..)"),
            NtExpr(..) => f.pad("NtExpr(..)"),
            NtTy(..) => f.pad("NtTy(..)"),
            NtLiteral(..) => f.pad("NtLiteral(..)"),
            NtMeta(..) => f.pad("NtMeta(..)"),
            NtPath(..) => f.pad("NtPath(..)"),
            NtVis(..) => f.pad("NtVis(..)"),
        }
    }
 }
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@ -469,10 +469,8 @@ impl TokenStream {
            }
            Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt),
            Nonterminal::NtPat(pat) => TokenStream::from_ast(pat),
            Nonterminal::NtTy(ty) => TokenStream::from_ast(ty),
            Nonterminal::NtMeta(attr) => TokenStream::from_ast(attr),
            Nonterminal::NtPath(path) => TokenStream::from_ast(path),
            Nonterminal::NtVis(vis) => TokenStream::from_ast(vis),
            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),
        }
    }
--- a/compiler/rustc_ast/src/visit.rs
+++ b/compiler/rustc_ast/src/visit.rs
@ -597,7 +597,7 @@ pub fn walk_use_tree<'a, V: Visitor<'a>>(
            visit_opt!(visitor, visit_ident, rename);
        }
        UseTreeKind::Glob => {}
-        UseTreeKind::Nested { ref items, span: _ } => {
+        UseTreeKind::Nested { items, span: _ } => {
            for &(ref nested_tree, nested_id) in items {
                try_visit!(visitor.visit_use_tree(nested_tree, nested_id, true));
            }
--- a/compiler/rustc_ast_ir/Cargo.toml
+++ b/compiler/rustc_ast_ir/Cargo.toml
@ -1,7 +1,7 @@
 [package]
 name = "rustc_ast_ir"
 version = "0.0.0"
-edition = "2021"
+edition = "2024"
 [dependencies]
 # tidy-alphabetical-start
--- a/compiler/rustc_ast_ir/src/lib.rs
+++ b/compiler/rustc_ast_ir/src/lib.rs
@ -1,3 +1,10 @@
 //! Common utilities shared by both `rustc_ast` and `rustc_type_ir`.
 //!
 //! Don't depend on this crate directly; both of those crates should re-export
 //! the functionality. Additionally, if you're in scope of `rustc_middle`, then
 //! prefer imports via that too, to avoid needing to directly depend on (e.g.)
 //! `rustc_type_ir` for a single import.
 // tidy-alphabetical-start
 #![cfg_attr(feature = "nightly", allow(internal_features))]
 #![cfg_attr(feature = "nightly", feature(never_type))]
--- a/compiler/rustc_ast_lowering/Cargo.toml
+++ b/compiler/rustc_ast_lowering/Cargo.toml
@ -1,7 +1,7 @@
 [package]
 name = "rustc_ast_lowering"
 version = "0.0.0"
-edition = "2021"
+edition = "2024"
 [lib]
 doctest = false
--- a/compiler/rustc_ast_lowering/src/asm.rs
+++ b/compiler/rustc_ast_lowering/src/asm.rs
@ -1,13 +1,12 @@
 use std::collections::hash_map::Entry;
 use std::fmt::Write;
 use rustc_ast::ptr::P;
 use rustc_ast::*;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_session::parse::feature_err;
-use rustc_span::{Span, kw, sym};
+use rustc_span::{Span, sym};
 use rustc_target::asm;
 use super::LoweringContext;
@ -230,20 +229,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                                tokens: None,
                            };
-                            // Wrap the expression in an AnonConst.
+                            hir::InlineAsmOperand::SymFn { expr: self.lower_expr(&expr) }
                            let parent_def_id = self.current_hir_id_owner.def_id;
                            let node_id = self.next_node_id();
                            self.create_def(
                                parent_def_id,
                                node_id,
                                kw::Empty,
                                DefKind::AnonConst,
                                *op_sp,
                            );
                            let anon_const = AnonConst { id: node_id, value: P(expr) };
                            hir::InlineAsmOperand::SymFn {
                                anon_const: self.lower_anon_const_to_anon_const(&anon_const),
                            }
                        }
                    }
                    InlineAsmOperand::Label { block } => {
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@ -251,7 +251,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
                    .arena
                    .alloc_from_iter(fm.items.iter().map(|x| self.lower_foreign_item_ref(x))),
            },
-            ItemKind::GlobalAsm(asm) => hir::ItemKind::GlobalAsm(self.lower_inline_asm(span, asm)),
+            ItemKind::GlobalAsm(asm) => {
                let asm = self.lower_inline_asm(span, asm);
                let fake_body =
                    self.lower_body(|this| (&[], this.expr(span, hir::ExprKind::InlineAsm(asm))));
                hir::ItemKind::GlobalAsm { asm, fake_body }
            }
            ItemKind::TyAlias(box TyAlias { generics, where_clauses, ty, .. }) => {
                // We lower
                //
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@ -45,7 +45,6 @@ use std::sync::Arc;
 use rustc_ast::node_id::NodeMap;
 use rustc_ast::{self as ast, *};
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::sorted_map::SortedMap;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@ -1821,11 +1820,11 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
        self.new_named_lifetime_with_res(new_id, ident, res)
    }
-    fn lower_generic_params_mut<'s>(
+    fn lower_generic_params_mut(
-        &'s mut self,
+        &mut self,
-        params: &'s [GenericParam],
+        params: &[GenericParam],
        source: hir::GenericParamSource,
-    ) -> impl Iterator<Item = hir::GenericParam<'hir>> + Captures<'a> + Captures<'s> {
+    ) -> impl Iterator<Item = hir::GenericParam<'hir>> {
        params.iter().map(move |param| self.lower_generic_param(param, source))
    }
@ -1986,11 +1985,11 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
        self.arena.alloc_from_iter(self.lower_param_bounds_mut(bounds, itctx))
    }
-    fn lower_param_bounds_mut<'s>(
+    fn lower_param_bounds_mut(
-        &'s mut self,
+        &mut self,
-        bounds: &'s [GenericBound],
+        bounds: &[GenericBound],
        itctx: ImplTraitContext,
-    ) -> impl Iterator<Item = hir::GenericBound<'hir>> + Captures<'s> + Captures<'a> {
+    ) -> impl Iterator<Item = hir::GenericBound<'hir>> {
        bounds.iter().map(move |bound| self.lower_param_bound(bound, itctx))
    }
--- a/compiler/rustc_ast_passes/Cargo.toml
+++ b/compiler/rustc_ast_passes/Cargo.toml
@ -1,7 +1,7 @@
 [package]
 name = "rustc_ast_passes"
 version = "0.0.0"
-edition = "2021"
+edition = "2024"
 [dependencies]
 # tidy-alphabetical-start
--- a/compiler/rustc_ast_passes/src/errors.rs
+++ b/compiler/rustc_ast_passes/src/errors.rs
@ -804,7 +804,14 @@ pub(crate) struct NegativeBoundWithParentheticalNotation {
 pub(crate) struct MatchArmWithNoBody {
    #[primary_span]
    pub span: Span,
-    #[suggestion(code = " => todo!(),", applicability = "has-placeholders")]
+    // We include the braces around `todo!()` so that a comma is optional, and we don't have to have
    // any logic looking at the arm being replaced if there was a comma already or not for the
    // resulting code to be correct.
    #[suggestion(
        code = " => {{ todo!() }}",
        applicability = "has-placeholders",
        style = "verbose"
    )]
    pub suggestion: Span,
 }
--- a/compiler/rustc_ast_pretty/Cargo.toml
+++ b/compiler/rustc_ast_pretty/Cargo.toml
@ -1,7 +1,7 @@
 [package]
 name = "rustc_ast_pretty"
 version = "0.0.0"
-edition = "2021"
+edition = "2024"
 [dependencies]
 # tidy-alphabetical-start
--- a/compiler/rustc_ast_pretty/src/pprust/state.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state.rs
@ -424,20 +424,23 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
        self.ann_post(ident)
    }
-    fn strsep<T, F>(
+    fn strsep<'x, T: 'x, F, I>(
        &mut self,
        sep: &'static str,
        space_before: bool,
        b: Breaks,
-        elts: &[T],
+        elts: I,
        mut op: F,
    ) where
        F: FnMut(&mut Self, &T),
        I: IntoIterator<Item = &'x T>,
    {
        let mut it = elts.into_iter();
        self.rbox(0, b);
-        if let Some((first, rest)) = elts.split_first() {
+        if let Some(first) = it.next() {
            op(self, first);
-            for elt in rest {
+            for elt in it {
                if space_before {
                    self.space();
                }
@ -448,9 +451,10 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
        self.end();
    }
-    fn commasep<T, F>(&mut self, b: Breaks, elts: &[T], op: F)
+    fn commasep<'x, T: 'x, F, I>(&mut self, b: Breaks, elts: I, op: F)
    where
        F: FnMut(&mut Self, &T),
        I: IntoIterator<Item = &'x T>,
    {
        self.strsep(",", false, b, elts, op)
    }
--- a/compiler/rustc_attr_data_structures/Cargo.toml
+++ b/compiler/rustc_attr_data_structures/Cargo.toml
@ -1,7 +1,7 @@
 [package]
 name = "rustc_attr_data_structures"
 version = "0.0.0"
-edition = "2021"
+edition = "2024"
 [dependencies]
 # tidy-alphabetical-start
--- a/compiler/rustc_attr_parsing/Cargo.toml
+++ b/compiler/rustc_attr_parsing/Cargo.toml
@ -1,7 +1,7 @@
 [package]
 name = "rustc_attr_parsing"
 version = "0.0.0"
-edition = "2021"
+edition = "2024"
 [dependencies]
 # tidy-alphabetical-start
--- a/compiler/rustc_attr_parsing/src/attributes/allow_unstable.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/allow_unstable.rs
@ -4,25 +4,25 @@ use rustc_span::{Symbol, sym};
 use crate::session_diagnostics;
-pub fn allow_internal_unstable<'a>(
+pub fn allow_internal_unstable(
-    sess: &'a Session,
+    sess: &Session,
-    attrs: &'a [impl AttributeExt],
+    attrs: &[impl AttributeExt],
-) -> impl Iterator<Item = Symbol> + 'a {
+) -> impl Iterator<Item = Symbol> {
    allow_unstable(sess, attrs, sym::allow_internal_unstable)
 }
-pub fn rustc_allow_const_fn_unstable<'a>(
+pub fn rustc_allow_const_fn_unstable(
-    sess: &'a Session,
+    sess: &Session,
-    attrs: &'a [impl AttributeExt],
+    attrs: &[impl AttributeExt],
-) -> impl Iterator<Item = Symbol> + 'a {
+) -> impl Iterator<Item = Symbol> {
    allow_unstable(sess, attrs, sym::rustc_allow_const_fn_unstable)
 }
-fn allow_unstable<'a>(
+fn allow_unstable(
-    sess: &'a Session,
+    sess: &Session,
-    attrs: &'a [impl AttributeExt],
+    attrs: &[impl AttributeExt],
    symbol: Symbol,
-) -> impl Iterator<Item = Symbol> + 'a {
+) -> impl Iterator<Item = Symbol> {
    let attrs = filter_by_name(attrs, symbol);
    let list = attrs
        .filter_map(move |attr| {
--- a/compiler/rustc_baked_icu_data/Cargo.toml
+++ b/compiler/rustc_baked_icu_data/Cargo.toml
@ -1,7 +1,7 @@
 [package]
 name = "rustc_baked_icu_data"
 version = "0.0.0"
-edition = "2021"
+edition = "2024"
 [dependencies]
 # tidy-alphabetical-start
--- a/compiler/rustc_borrowck/Cargo.toml
+++ b/compiler/rustc_borrowck/Cargo.toml
@ -1,7 +1,7 @@
 [package]
 name = "rustc_borrowck"
 version = "0.0.0"
-edition = "2021"
+edition = "2024"
 [dependencies]
 # tidy-alphabetical-start
--- a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
@ -8,7 +8,6 @@ use std::ops::ControlFlow;
 use either::Either;
 use hir::{ClosureKind, Path};
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_errors::codes::*;
 use rustc_errors::{Applicability, Diag, MultiSpan, struct_span_code_err};
@ -18,12 +17,11 @@ use rustc_hir::intravisit::{Visitor, walk_block, walk_expr};
 use rustc_hir::{CoroutineDesugaring, CoroutineKind, CoroutineSource, LangItem, PatField};
 use rustc_middle::bug;
 use rustc_middle::hir::nested_filter::OnlyBodies;
 use rustc_middle::mir::tcx::PlaceTy;
 use rustc_middle::mir::{
    self, AggregateKind, BindingForm, BorrowKind, ClearCrossCrate, ConstraintCategory,
    FakeBorrowKind, FakeReadCause, LocalDecl, LocalInfo, LocalKind, Location, MutBorrowKind,
-    Operand, Place, PlaceRef, ProjectionElem, Rvalue, Statement, StatementKind, Terminator,
+    Operand, Place, PlaceRef, PlaceTy, ProjectionElem, Rvalue, Statement, StatementKind,
-    TerminatorKind, VarBindingForm, VarDebugInfoContents,
+    Terminator, TerminatorKind, VarBindingForm, VarDebugInfoContents,
 };
 use rustc_middle::ty::print::PrintTraitRefExt as _;
 use rustc_middle::ty::{
@ -386,8 +384,8 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                hir::intravisit::walk_pat(self, p);
            }
        }
-        let hir = self.infcx.tcx.hir();
+        let tcx = self.infcx.tcx;
-        if let Some(body) = hir.maybe_body_owned_by(self.mir_def_id()) {
+        if let Some(body) = tcx.hir_maybe_body_owned_by(self.mir_def_id()) {
            let expr = body.value;
            let place = &self.move_data.move_paths[mpi].place;
            let span = place.as_local().map(|local| self.body.local_decls[local].source_info.span);
@ -396,13 +394,13 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                expr: None,
                pat: None,
                parent_pat: None,
-                tcx: self.infcx.tcx,
+                tcx,
            };
            finder.visit_expr(expr);
            if let Some(span) = span
                && let Some(expr) = finder.expr
            {
-                for (_, expr) in hir.parent_iter(expr.hir_id) {
+                for (_, expr) in tcx.hir_parent_iter(expr.hir_id) {
                    if let hir::Node::Expr(expr) = expr {
                        if expr.span.contains(span) {
                            // If the let binding occurs within the same loop, then that
@ -782,9 +780,9 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
        // We use the statements were the binding was initialized, and inspect the HIR to look
        // for the branching codepaths that aren't covered, to point at them.
-        let map = self.infcx.tcx.hir();
+        let tcx = self.infcx.tcx;
-        let body = map.body_owned_by(self.mir_def_id());
+        let body = tcx.hir_body_owned_by(self.mir_def_id());
-        let mut visitor = ConditionVisitor { tcx: self.infcx.tcx, spans, name, errors: vec![] };
+        let mut visitor = ConditionVisitor { tcx, spans, name, errors: vec![] };
        visitor.visit_body(&body);
        let spans = visitor.spans;
@ -969,7 +967,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
        let mut parent = None;
        // The top-most loop where the moved expression could be moved to a new binding.
        let mut outer_most_loop: Option<&hir::Expr<'_>> = None;
-        for (_, node) in tcx.hir().parent_iter(expr.hir_id) {
+        for (_, node) in tcx.hir_parent_iter(expr.hir_id) {
            let e = match node {
                hir::Node::Expr(e) => e,
                hir::Node::LetStmt(hir::LetStmt { els: Some(els), .. }) => {
@ -1021,8 +1019,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
            }
        }
        let loop_count: usize = tcx
-            .hir()
+            .hir_parent_iter(expr.hir_id)
            .parent_iter(expr.hir_id)
            .map(|(_, node)| match node {
                hir::Node::Expr(hir::Expr { kind: hir::ExprKind::Loop(..), .. }) => 1,
                _ => 0,
@ -1048,8 +1045,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                .collect::<Vec<Span>>();
            // All of the spans for the loops above the expression with the move error.
            let loop_spans: Vec<_> = tcx
-                .hir()
+                .hir_parent_iter(expr.hir_id)
                .parent_iter(expr.hir_id)
                .filter_map(|(_, node)| match node {
                    hir::Node::Expr(hir::Expr { span, kind: hir::ExprKind::Loop(..), .. }) => {
                        Some(*span)
@ -1334,7 +1330,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
    }
    fn in_move_closure(&self, expr: &hir::Expr<'_>) -> bool {
-        for (_, node) in self.infcx.tcx.hir().parent_iter(expr.hir_id) {
+        for (_, node) in self.infcx.tcx.hir_parent_iter(expr.hir_id) {
            if let hir::Node::Expr(hir::Expr { kind: hir::ExprKind::Closure(closure), .. }) = node
                && let hir::CaptureBy::Value { .. } = closure.capture_clause
            {
@ -2118,7 +2114,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
        issued_span: Span,
    ) {
        let tcx = self.infcx.tcx;
        let hir = tcx.hir();
        let has_split_at_mut = |ty: Ty<'tcx>| {
            let ty = ty.peel_refs();
@ -2171,7 +2166,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                return;
            };
-            let Some(object) = hir.parent_id_iter(index1.hir_id).find_map(|id| {
+            let Some(object) = tcx.hir_parent_id_iter(index1.hir_id).find_map(|id| {
                if let hir::Node::Expr(expr) = tcx.hir_node(id)
                    && let hir::ExprKind::Index(obj, ..) = expr.kind
                {
@ -2189,7 +2184,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                return;
            };
-            let Some(swap_call) = hir.parent_id_iter(object.hir_id).find_map(|id| {
+            let Some(swap_call) = tcx.hir_parent_id_iter(object.hir_id).find_map(|id| {
                if let hir::Node::Expr(call) = tcx.hir_node(id)
                    && let hir::ExprKind::Call(callee, ..) = call.kind
                    && let hir::ExprKind::Path(qpath) = callee.kind
@ -2443,7 +2438,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
    ) {
        let &UseSpans::ClosureUse { capture_kind_span, .. } = issued_spans else { return };
        let tcx = self.infcx.tcx;
        let hir = tcx.hir();
        // Get the type of the local that we are trying to borrow
        let local = borrowed_place.local;
@ -2522,7 +2516,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
        // Find the first argument with a matching type, get its name
        let Some((_, this_name)) =
-            params.iter().zip(hir.body_param_names(closure.body)).find(|(param_ty, name)| {
+            params.iter().zip(tcx.hir_body_param_names(closure.body)).find(|(param_ty, name)| {
                // FIXME: also support deref for stuff like `Rc` arguments
                param_ty.peel_refs() == local_ty && name != &Ident::empty()
            })
@ -2622,7 +2616,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                if let hir::Pat { kind: hir::PatKind::Binding(_, hir_id, _ident, _), .. } =
                    local.pat
                    && let Some(init) = local.init
-                    && let hir::Expr {
+                    && let &hir::Expr {
                        kind:
                            hir::ExprKind::Closure(&hir::Closure {
                                kind: hir::ClosureKind::Closure,
@ -3535,10 +3529,10 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
        location: Location,
        mpi: MovePathIndex,
    ) -> (Vec<MoveSite>, Vec<Location>) {
-        fn predecessor_locations<'a, 'tcx>(
+        fn predecessor_locations<'tcx>(
-            body: &'a mir::Body<'tcx>,
+            body: &mir::Body<'tcx>,
            location: Location,
-        ) -> impl Iterator<Item = Location> + Captures<'tcx> + 'a {
+        ) -> impl Iterator<Item = Location> {
            if location.statement_index == 0 {
                let predecessors = body.basic_blocks.predecessors()[location.block].to_vec();
                Either::Left(predecessors.into_iter().map(move |bb| body.terminator_loc(bb)))
@ -4178,7 +4172,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
        debug!("annotate_fn_sig: did={:?} sig={:?}", did, sig);
        let is_closure = self.infcx.tcx.is_closure_like(did.to_def_id());
        let fn_hir_id = self.infcx.tcx.local_def_id_to_hir_id(did);
-        let fn_decl = self.infcx.tcx.hir().fn_decl_by_hir_id(fn_hir_id)?;
+        let fn_decl = self.infcx.tcx.hir_fn_decl_by_hir_id(fn_hir_id)?;
        // We need to work out which arguments to highlight. We do this by looking
        // at the return type, where there are three cases:
--- a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
@ -117,7 +117,7 @@ impl<'tcx> BorrowExplanation<'tcx> {
                let local_decl = &body.local_decls[dropped_local];
                if let &LocalInfo::IfThenRescopeTemp { if_then } = local_decl.local_info()
-                    && let Some((_, hir::Node::Expr(expr))) = tcx.hir().parent_iter(if_then).next()
+                    && let Some((_, hir::Node::Expr(expr))) = tcx.hir_parent_iter(if_then).next()
                    && let hir::ExprKind::If(cond, conseq, alt) = expr.kind
                    && let hir::ExprKind::Let(&hir::LetExpr {
                        span: _,
@ -262,7 +262,7 @@ impl<'tcx> BorrowExplanation<'tcx> {
                            fn visit_expr(&mut self, expr: &'hir hir::Expr<'hir>) {
                                if let hir::ExprKind::If(cond, _conseq, _alt)
                                | hir::ExprKind::Loop(
-                                    hir::Block {
+                                    &hir::Block {
                                        expr:
                                            Some(&hir::Expr {
                                                kind: hir::ExprKind::If(cond, _conseq, _alt),
@ -522,7 +522,7 @@ fn suggest_rewrite_if_let<G: EmissionGuarantee>(
    );
    if expr.span.can_be_used_for_suggestions() && conseq.span.can_be_used_for_suggestions() {
        let needs_block = if let Some(hir::Node::Expr(expr)) =
-            alt.and_then(|alt| tcx.hir().parent_iter(alt.hir_id).next()).map(|(_, node)| node)
+            alt.and_then(|alt| tcx.hir_parent_iter(alt.hir_id).next()).map(|(_, node)| node)
        {
            matches!(expr.kind, hir::ExprKind::If(..))
        } else {
--- a/compiler/rustc_borrowck/src/diagnostics/find_use.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/find_use.rs
@ -1,7 +1,7 @@
 use std::collections::VecDeque;
 use rustc_data_structures::fx::FxIndexSet;
-use rustc_middle::mir::visit::{MirVisitable, PlaceContext, Visitor};
+use rustc_middle::mir::visit::{PlaceContext, Visitor};
 use rustc_middle::mir::{self, Body, Local, Location};
 use rustc_middle::ty::{RegionVid, TyCtxt};
@ -45,7 +45,22 @@ impl<'a, 'tcx> UseFinder<'a, 'tcx> {
            let block_data = &self.body[p.block];
-            match self.def_use(p, block_data.visitable(p.statement_index)) {
+            let mut visitor = DefUseVisitor {
                body: self.body,
                tcx: self.tcx,
                region_vid: self.region_vid,
                def_use_result: None,
            };
            let is_statement = p.statement_index < block_data.statements.len();
            if is_statement {
                visitor.visit_statement(&block_data.statements[p.statement_index], p);
            } else {
                visitor.visit_terminator(block_data.terminator.as_ref().unwrap(), p);
            }
            match visitor.def_use_result {
                Some(DefUseResult::Def) => {}
                Some(DefUseResult::UseLive { local }) => {
@ -57,7 +72,7 @@ impl<'a, 'tcx> UseFinder<'a, 'tcx> {
                }
                None => {
-                    if p.statement_index < block_data.statements.len() {
+                    if is_statement {
                        queue.push_back(p.successor_within_block());
                    } else {
                        queue.extend(
@ -77,19 +92,6 @@ impl<'a, 'tcx> UseFinder<'a, 'tcx> {
        None
    }
    fn def_use(&self, location: Location, thing: &dyn MirVisitable<'tcx>) -> Option<DefUseResult> {
        let mut visitor = DefUseVisitor {
            body: self.body,
            tcx: self.tcx,
            region_vid: self.region_vid,
            def_use_result: None,
        };
        thing.apply(location, &mut visitor);
        visitor.def_use_result
    }
 }
 struct DefUseVisitor<'a, 'tcx> {
--- a/compiler/rustc_borrowck/src/diagnostics/mod.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mod.rs
@ -13,10 +13,9 @@ use rustc_infer::infer::{
 };
 use rustc_infer::traits::SelectionError;
 use rustc_middle::bug;
 use rustc_middle::mir::tcx::PlaceTy;
 use rustc_middle::mir::{
    AggregateKind, CallSource, ConstOperand, ConstraintCategory, FakeReadCause, Local, LocalInfo,
-    LocalKind, Location, Operand, Place, PlaceRef, ProjectionElem, Rvalue, Statement,
+    LocalKind, Location, Operand, Place, PlaceRef, PlaceTy, ProjectionElem, Rvalue, Statement,
    StatementKind, Terminator, TerminatorKind, find_self_call,
 };
 use rustc_middle::ty::print::Print;
@ -1127,7 +1126,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
        let hir_id = self.infcx.tcx.local_def_id_to_hir_id(def_id);
        let expr = &self.infcx.tcx.hir().expect_expr(hir_id).kind;
        debug!("closure_span: hir_id={:?} expr={:?}", hir_id, expr);
-        if let hir::ExprKind::Closure(&hir::Closure { kind, fn_decl_span, .. }) = expr {
+        if let &hir::ExprKind::Closure(&hir::Closure { kind, fn_decl_span, .. }) = expr {
            for (captured_place, place) in
                self.infcx.tcx.closure_captures(def_id).iter().zip(places)
            {
--- a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
@ -388,7 +388,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
        // Search for an appropriate place for the structured `.clone()` suggestion to be applied.
        // If we encounter a statement before the borrow error, we insert a statement there.
-        for (_, node) in tcx.hir().parent_iter(closure_expr.hir_id) {
+        for (_, node) in tcx.hir_parent_iter(closure_expr.hir_id) {
            if let Node::Stmt(stmt) = node {
                let padding = tcx
                    .sess
@ -777,12 +777,12 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
        }
        let Some(pat_span) = pat_span else { return };
-        let hir = self.infcx.tcx.hir();
+        let tcx = self.infcx.tcx;
-        let Some(body) = hir.maybe_body_owned_by(self.mir_def_id()) else { return };
+        let Some(body) = tcx.hir_maybe_body_owned_by(self.mir_def_id()) else { return };
        let typeck_results = self.infcx.tcx.typeck(self.mir_def_id());
        let mut finder = BindingFinder {
            typeck_results,
-            tcx: self.infcx.tcx,
+            tcx,
            pat_span,
            binding_spans,
            found_pat: false,
--- a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
@ -404,7 +404,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                        pat.kind
                {
                    if upvar_ident.name == kw::SelfLower {
-                        for (_, node) in self.infcx.tcx.hir().parent_iter(upvar_hir_id) {
+                        for (_, node) in self.infcx.tcx.hir_parent_iter(upvar_hir_id) {
                            if let Some(fn_decl) = node.fn_decl() {
                                if !matches!(
                                    fn_decl.implicit_self,
@ -648,10 +648,9 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                    }
                }
            }
            let hir_map = self.infcx.tcx.hir();
            let def_id = self.body.source.def_id();
            let Some(local_def_id) = def_id.as_local() else { return };
-            let Some(body) = hir_map.maybe_body_owned_by(local_def_id) else { return };
+            let Some(body) = self.infcx.tcx.hir_maybe_body_owned_by(local_def_id) else { return };
            let mut v = SuggestIndexOperatorAlternativeVisitor {
                assign_span: span,
@ -749,7 +748,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
        // `fn foo(&x: &i32)` -> `fn foo(&(mut x): &i32)`
        let def_id = self.body.source.def_id();
        if let Some(local_def_id) = def_id.as_local()
-            && let Some(body) = self.infcx.tcx.hir().maybe_body_owned_by(local_def_id)
+            && let Some(body) = self.infcx.tcx.hir_maybe_body_owned_by(local_def_id)
            && let Some(hir_id) = (BindingFinder { span: pat_span }).visit_body(&body).break_value()
            && let node = self.infcx.tcx.hir_node(hir_id)
            && let hir::Node::LetStmt(hir::LetStmt {
@ -856,7 +855,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
        use hir::ExprKind::{AddrOf, Block, Call, MethodCall};
        use hir::{BorrowKind, Expr};
-        let hir_map = self.infcx.tcx.hir();
+        let tcx = self.infcx.tcx;
        struct Finder {
            span: Span,
        }
@ -871,7 +870,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                }
            }
        }
-        if let Some(body) = hir_map.maybe_body_owned_by(self.mir_def_id())
+        if let Some(body) = tcx.hir_maybe_body_owned_by(self.mir_def_id())
            && let Block(block, _) = body.value.kind
        {
            // `span` corresponds to the expression being iterated, find the `for`-loop desugared
@ -884,17 +883,15 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                    MethodCall(path_segment, _, _, span) => {
                        // We have `for _ in iter.read_only_iter()`, try to
                        // suggest `for _ in iter.mutable_iter()` instead.
-                        let opt_suggestions = self
+                        let opt_suggestions = tcx
                            .infcx
                            .tcx
                            .typeck(path_segment.hir_id.owner.def_id)
                            .type_dependent_def_id(expr.hir_id)
-                            .and_then(|def_id| self.infcx.tcx.impl_of_method(def_id))
+                            .and_then(|def_id| tcx.impl_of_method(def_id))
-                            .map(|def_id| self.infcx.tcx.associated_items(def_id))
+                            .map(|def_id| tcx.associated_items(def_id))
                            .map(|assoc_items| {
                                assoc_items
                                    .in_definition_order()
-                                    .map(|assoc_item_def| assoc_item_def.ident(self.infcx.tcx))
+                                    .map(|assoc_item_def| assoc_item_def.ident(tcx))
                                    .filter(|&ident| {
                                        let original_method_ident = path_segment.ident;
                                        original_method_ident != ident
@ -937,12 +934,11 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
        err.span_label(sp, format!("cannot {act}"));
        let tcx = self.infcx.tcx;
        let hir = tcx.hir();
        let closure_id = self.mir_hir_id();
        let closure_span = tcx.def_span(self.mir_def_id());
        let fn_call_id = tcx.parent_hir_id(closure_id);
        let node = tcx.hir_node(fn_call_id);
-        let def_id = hir.enclosing_body_owner(fn_call_id);
+        let def_id = tcx.hir_enclosing_body_owner(fn_call_id);
        let mut look_at_return = true;
        // If the HIR node is a function or method call gets the def ID
@ -1020,10 +1016,10 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
            }
        }
-        if look_at_return && hir.get_fn_id_for_return_block(closure_id).is_some() {
+        if look_at_return && tcx.hir_get_fn_id_for_return_block(closure_id).is_some() {
            // ...otherwise we are probably in the tail expression of the function, point at the
            // return type.
-            match tcx.hir_node_by_def_id(hir.get_parent_item(fn_call_id).def_id) {
+            match tcx.hir_node_by_def_id(tcx.hir_get_parent_item(fn_call_id).def_id) {
                hir::Node::Item(hir::Item {
                    ident, kind: hir::ItemKind::Fn { sig, .. }, ..
                })
@ -1275,7 +1271,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
            }) => {
                let def_id = self.body.source.def_id();
                let hir_id = if let Some(local_def_id) = def_id.as_local()
-                    && let Some(body) = self.infcx.tcx.hir().maybe_body_owned_by(local_def_id)
+                    && let Some(body) = self.infcx.tcx.hir_maybe_body_owned_by(local_def_id)
                {
                    BindingFinder { span: err_label_span }.visit_body(&body).break_value()
                } else {
--- a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
@ -1169,8 +1169,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
    #[allow(rustc::diagnostic_outside_of_impl)]
    fn suggest_move_on_borrowing_closure(&self, diag: &mut Diag<'_>) {
-        let map = self.infcx.tcx.hir();
+        let body = self.infcx.tcx.hir_body_owned_by(self.mir_def_id());
        let body = map.body_owned_by(self.mir_def_id());
        let expr = &body.value.peel_blocks();
        let mut closure_span = None::<rustc_span::Span>;
        match expr.kind {
--- a/compiler/rustc_borrowck/src/diagnostics/region_name.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
@ -424,7 +424,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
        &self,
        argument_index: usize,
    ) -> Option<&hir::Ty<'tcx>> {
-        let fn_decl = self.infcx.tcx.hir().fn_decl_by_hir_id(self.mir_hir_id())?;
+        let fn_decl = self.infcx.tcx.hir_fn_decl_by_hir_id(self.mir_hir_id())?;
        let argument_hir_ty: &hir::Ty<'_> = fn_decl.inputs.get(argument_index)?;
        match argument_hir_ty.kind {
            // This indicates a variable with no type annotation, like
@ -671,7 +671,6 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
    #[instrument(level = "trace", skip(self))]
    fn give_name_if_anonymous_region_appears_in_output(&self, fr: RegionVid) -> Option<RegionName> {
        let tcx = self.infcx.tcx;
        let hir = tcx.hir();
        let return_ty = self.regioncx.universal_regions().unnormalized_output_ty;
        debug!("give_name_if_anonymous_region_appears_in_output: return_ty = {:?}", return_ty);
@ -682,7 +681,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
        let mir_hir_id = self.mir_hir_id();
        let (return_span, mir_description, hir_ty) = match tcx.hir_node(mir_hir_id) {
-            hir::Node::Expr(hir::Expr {
+            hir::Node::Expr(&hir::Expr {
                kind: hir::ExprKind::Closure(&hir::Closure { fn_decl, kind, fn_decl_span, .. }),
                ..
            }) => {
@ -711,7 +710,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
                        hir::CoroutineSource::Fn,
                    )) => {
                        let parent_item =
-                            tcx.hir_node_by_def_id(hir.get_parent_item(mir_hir_id).def_id);
+                            tcx.hir_node_by_def_id(tcx.hir_get_parent_item(mir_hir_id).def_id);
                        let output = &parent_item
                            .fn_decl()
                            .expect("coroutine lowered from async fn should be in fn")
@ -741,7 +740,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
                        hir::CoroutineSource::Fn,
                    )) => {
                        let parent_item =
-                            tcx.hir_node_by_def_id(hir.get_parent_item(mir_hir_id).def_id);
+                            tcx.hir_node_by_def_id(tcx.hir_get_parent_item(mir_hir_id).def_id);
                        let output = &parent_item
                            .fn_decl()
                            .expect("coroutine lowered from gen fn should be in fn")
@ -768,7 +767,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
                        hir::CoroutineSource::Fn,
                    )) => {
                        let parent_item =
-                            tcx.hir_node_by_def_id(hir.get_parent_item(mir_hir_id).def_id);
+                            tcx.hir_node_by_def_id(tcx.hir_get_parent_item(mir_hir_id).def_id);
                        let output = &parent_item
                            .fn_decl()
                            .expect("coroutine lowered from async gen fn should be in fn")
@ -874,7 +873,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
            .name;
        let yield_span = match tcx.hir_node(self.mir_hir_id()) {
-            hir::Node::Expr(hir::Expr {
+            hir::Node::Expr(&hir::Expr {
                kind: hir::ExprKind::Closure(&hir::Closure { fn_decl_span, .. }),
                ..
            }) => tcx.sess.source_map().end_point(fn_decl_span),
--- a/compiler/rustc_borrowck/src/lib.rs
+++ b/compiler/rustc_borrowck/src/lib.rs
@ -33,7 +33,6 @@ use rustc_index::{IndexSlice, IndexVec};
 use rustc_infer::infer::{
    InferCtxt, NllRegionVariableOrigin, RegionVariableOrigin, TyCtxtInferExt,
 };
 use rustc_middle::mir::tcx::PlaceTy;
 use rustc_middle::mir::*;
 use rustc_middle::query::Providers;
 use rustc_middle::ty::fold::fold_regions;
@ -188,7 +187,7 @@ fn do_mir_borrowck<'tcx>(
        .iterate_to_fixpoint(tcx, body, Some("borrowck"))
        .into_results_cursor(body);
-    let locals_are_invalidated_at_exit = tcx.hir().body_owner_kind(def).is_fn_or_closure();
+    let locals_are_invalidated_at_exit = tcx.hir_body_owner_kind(def).is_fn_or_closure();
    let borrow_set = BorrowSet::build(tcx, body, locals_are_invalidated_at_exit, &move_data);
    // Compute non-lexical lifetimes.
--- a/compiler/rustc_borrowck/src/member_constraints.rs
+++ b/compiler/rustc_borrowck/src/member_constraints.rs
@ -1,7 +1,6 @@
 use std::hash::Hash;
 use std::ops::Index;
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_index::{IndexSlice, IndexVec};
 use rustc_middle::ty::{self, Ty};
@ -147,9 +146,7 @@ impl<'tcx, R> MemberConstraintSet<'tcx, R>
 where
    R: Copy + Hash + Eq,
 {
-    pub(crate) fn all_indices(
+    pub(crate) fn all_indices(&self) -> impl Iterator<Item = NllMemberConstraintIndex> {
        &self,
    ) -> impl Iterator<Item = NllMemberConstraintIndex> + Captures<'tcx> + '_ {
        self.constraints.indices()
    }
@ -159,7 +156,7 @@ where
    pub(crate) fn indices(
        &self,
        member_region_vid: R,
-    ) -> impl Iterator<Item = NllMemberConstraintIndex> + Captures<'tcx> + '_ {
+    ) -> impl Iterator<Item = NllMemberConstraintIndex> {
        let mut next = self.first_constraints.get(&member_region_vid).cloned();
        std::iter::from_fn(move || -> Option<NllMemberConstraintIndex> {
            if let Some(current) = next {
--- a/compiler/rustc_borrowck/src/polonius/dump.rs
+++ b/compiler/rustc_borrowck/src/polonius/dump.rs
@ -226,7 +226,7 @@ fn emit_polonius_mir<'tcx>(
        regioncx,
        closure_region_requirements,
        borrow_set,
-        pass_where.clone(),
+        pass_where,
        out,
    )?;
--- a/compiler/rustc_borrowck/src/polonius/loan_liveness.rs
+++ b/compiler/rustc_borrowck/src/polonius/loan_liveness.rs
@ -175,7 +175,7 @@ impl LocalizedConstraintGraph {
    }
    /// Returns the outgoing edges of a given node, not its transitive closure.
-    fn outgoing_edges(&self, node: LocalizedNode) -> impl Iterator<Item = LocalizedNode> + use<'_> {
+    fn outgoing_edges(&self, node: LocalizedNode) -> impl Iterator<Item = LocalizedNode> {
        // The outgoing edges are:
        // - the physical edges present at this node,
        // - the materialized logical edges that exist virtually at all points for this node's
--- a/compiler/rustc_borrowck/src/region_infer/mod.rs
+++ b/compiler/rustc_borrowck/src/region_infer/mod.rs
@ -576,9 +576,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
    }
    /// Returns an iterator over all the outlives constraints.
-    pub(crate) fn outlives_constraints(
+    pub(crate) fn outlives_constraints(&self) -> impl Iterator<Item = OutlivesConstraint<'tcx>> {
        &self,
    ) -> impl Iterator<Item = OutlivesConstraint<'tcx>> + '_ {
        self.constraints.outlives().iter().copied()
    }
@ -615,10 +613,10 @@ impl<'tcx> RegionInferenceContext<'tcx> {
        self.scc_values.region_value_str(scc)
    }
-    pub(crate) fn placeholders_contained_in<'a>(
+    pub(crate) fn placeholders_contained_in(
-        &'a self,
+        &self,
        r: RegionVid,
-    ) -> impl Iterator<Item = ty::PlaceholderRegion> + 'a {
+    ) -> impl Iterator<Item = ty::PlaceholderRegion> {
        let scc = self.constraint_sccs.scc(r);
        self.scc_values.placeholders_contained_in(scc)
    }
@ -1267,6 +1265,11 @@ impl<'tcx> RegionInferenceContext<'tcx> {
        let sub_region_scc = self.constraint_sccs.scc(sub_region);
        let sup_region_scc = self.constraint_sccs.scc(sup_region);
        if sub_region_scc == sup_region_scc {
            debug!("{sup_region:?}: {sub_region:?} holds trivially; they are in the same SCC");
            return true;
        }
        // If we are checking that `'sup: 'sub`, and `'sub` contains
        // some placeholder that `'sup` cannot name, then this is only
        // true if `'sup` outlives static.
--- a/compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs
+++ b/compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs
@ -20,10 +20,7 @@ pub(crate) struct ReverseSccGraph {
 impl ReverseSccGraph {
    /// Find all universal regions that are required to outlive the given SCC.
-    pub(super) fn upper_bounds<'a>(
+    pub(super) fn upper_bounds(&self, scc0: ConstraintSccIndex) -> impl Iterator<Item = RegionVid> {
        &'a self,
        scc0: ConstraintSccIndex,
    ) -> impl Iterator<Item = RegionVid> + 'a {
        let mut duplicates = FxIndexSet::default();
        graph::depth_first_search(&self.graph, scc0)
            .flat_map(move |scc1| {
--- a/compiler/rustc_borrowck/src/region_infer/values.rs
+++ b/compiler/rustc_borrowck/src/region_infer/values.rs
@ -88,7 +88,7 @@ impl LivenessValues {
    }
    /// Iterate through each region that has a value in this set.
-    pub(crate) fn regions(&self) -> impl Iterator<Item = RegionVid> + '_ {
+    pub(crate) fn regions(&self) -> impl Iterator<Item = RegionVid> {
        self.points.as_ref().expect("use with_specific_points").rows()
    }
@ -96,7 +96,7 @@ impl LivenessValues {
    // We are passing query instability implications to the caller.
    #[rustc_lint_query_instability]
    #[allow(rustc::potential_query_instability)]
-    pub(crate) fn live_regions_unordered(&self) -> impl Iterator<Item = RegionVid> + '_ {
+    pub(crate) fn live_regions_unordered(&self) -> impl Iterator<Item = RegionVid> {
        self.live_regions.as_ref().unwrap().iter().copied()
    }
@ -143,7 +143,7 @@ impl LivenessValues {
    }
    /// Returns an iterator of all the points where `region` is live.
-    fn live_points(&self, region: RegionVid) -> impl Iterator<Item = PointIndex> + '_ {
+    fn live_points(&self, region: RegionVid) -> impl Iterator<Item = PointIndex> {
        let Some(points) = &self.points else {
            unreachable!(
                "Should be using LivenessValues::with_specific_points to ask whether live at a location"
@ -340,7 +340,7 @@ impl<N: Idx> RegionValues<N> {
    }
    /// Returns the locations contained within a given region `r`.
-    pub(crate) fn locations_outlived_by<'a>(&'a self, r: N) -> impl Iterator<Item = Location> + 'a {
+    pub(crate) fn locations_outlived_by(&self, r: N) -> impl Iterator<Item = Location> {
        self.points.row(r).into_iter().flat_map(move |set| {
            set.iter()
                .take_while(move |&p| self.location_map.point_in_range(p))
@ -349,18 +349,15 @@ impl<N: Idx> RegionValues<N> {
    }
    /// Returns just the universal regions that are contained in a given region's value.
-    pub(crate) fn universal_regions_outlived_by<'a>(
+    pub(crate) fn universal_regions_outlived_by(&self, r: N) -> impl Iterator<Item = RegionVid> {
        &'a self,
        r: N,
    ) -> impl Iterator<Item = RegionVid> + 'a {
        self.free_regions.row(r).into_iter().flat_map(|set| set.iter())
    }
    /// Returns all the elements contained in a given region's value.
-    pub(crate) fn placeholders_contained_in<'a>(
+    pub(crate) fn placeholders_contained_in(
-        &'a self,
+        &self,
        r: N,
-    ) -> impl Iterator<Item = ty::PlaceholderRegion> + 'a {
+    ) -> impl Iterator<Item = ty::PlaceholderRegion> {
        self.placeholders
            .row(r)
            .into_iter()
@ -369,10 +366,7 @@ impl<N: Idx> RegionValues<N> {
    }
    /// Returns all the elements contained in a given region's value.
-    pub(crate) fn elements_contained_in<'a>(
+    pub(crate) fn elements_contained_in(&self, r: N) -> impl Iterator<Item = RegionElement> {
        &'a self,
        r: N,
    ) -> impl Iterator<Item = RegionElement> + 'a {
        let points_iter = self.locations_outlived_by(r).map(RegionElement::Location);
        let free_regions_iter =
--- a/compiler/rustc_borrowck/src/type_check/free_region_relations.rs
+++ b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs
@ -172,7 +172,7 @@ impl UniversalRegionRelations<'_> {
    }
    /// Returns the _non-transitive_ set of known `outlives` constraints between free regions.
-    pub(crate) fn known_outlives(&self) -> impl Iterator<Item = (RegionVid, RegionVid)> + '_ {
+    pub(crate) fn known_outlives(&self) -> impl Iterator<Item = (RegionVid, RegionVid)> {
        self.outlives.base_edges()
    }
 }
--- a/compiler/rustc_borrowck/src/type_check/liveness/local_use_map.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/local_use_map.rs
@ -54,7 +54,7 @@ rustc_index::newtype_index! {
 fn appearances_iter(
    first: Option<AppearanceIndex>,
    appearances: &Appearances,
-) -> impl Iterator<Item = AppearanceIndex> + '_ {
+) -> impl Iterator<Item = AppearanceIndex> {
    AppearancesIter { appearances, current: first }
 }
@ -107,17 +107,17 @@ impl LocalUseMap {
        local_use_map
    }
-    pub(crate) fn defs(&self, local: Local) -> impl Iterator<Item = PointIndex> + '_ {
+    pub(crate) fn defs(&self, local: Local) -> impl Iterator<Item = PointIndex> {
        appearances_iter(self.first_def_at[local], &self.appearances)
            .map(move |aa| self.appearances[aa].point_index)
    }
-    pub(crate) fn uses(&self, local: Local) -> impl Iterator<Item = PointIndex> + '_ {
+    pub(crate) fn uses(&self, local: Local) -> impl Iterator<Item = PointIndex> {
        appearances_iter(self.first_use_at[local], &self.appearances)
            .map(move |aa| self.appearances[aa].point_index)
    }
-    pub(crate) fn drops(&self, local: Local) -> impl Iterator<Item = PointIndex> + '_ {
+    pub(crate) fn drops(&self, local: Local) -> impl Iterator<Item = PointIndex> {
        appearances_iter(self.first_drop_at[local], &self.appearances)
            .map(move |aa| self.appearances[aa].point_index)
    }
--- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
@ -3,7 +3,8 @@ use rustc_index::bit_set::DenseBitSet;
 use rustc_index::interval::IntervalSet;
 use rustc_infer::infer::canonical::QueryRegionConstraints;
 use rustc_infer::infer::outlives::for_liveness;
-use rustc_middle::mir::{BasicBlock, Body, ConstraintCategory, Local, Location};
+use rustc_middle::mir::{BasicBlock, Body, ConstraintCategory, HasLocalDecls, Local, Location};
 use rustc_middle::span_bug;
 use rustc_middle::traits::query::DropckOutlivesResult;
 use rustc_middle::ty::relate::Relate;
 use rustc_middle::ty::{Ty, TyCtxt, TypeVisitable, TypeVisitableExt};
@ -11,7 +12,10 @@ use rustc_mir_dataflow::ResultsCursor;
 use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
 use rustc_mir_dataflow::move_paths::{HasMoveData, MoveData, MovePathIndex};
 use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex};
-use rustc_span::DUMMY_SP;
+use rustc_span::{DUMMY_SP, Span};
 use rustc_trait_selection::error_reporting::InferCtxtErrorExt;
 use rustc_trait_selection::traits::ObligationCtxt;
 use rustc_trait_selection::traits::query::dropck_outlives;
 use rustc_trait_selection::traits::query::type_op::{DropckOutlives, TypeOp, TypeOpOutput};
 use tracing::debug;
@ -162,9 +166,10 @@ impl<'a, 'typeck, 'b, 'tcx> LivenessResults<'a, 'typeck, 'b, 'tcx> {
    fn dropck_boring_locals(&mut self, boring_locals: Vec<Local>) {
        for local in boring_locals {
            let local_ty = self.cx.body.local_decls[local].ty;
            let local_span = self.cx.body.local_decls[local].source_info.span;
            let drop_data = self.cx.drop_data.entry(local_ty).or_insert_with({
                let typeck = &self.cx.typeck;
-                move || LivenessContext::compute_drop_data(typeck, local_ty)
+                move || LivenessContext::compute_drop_data(typeck, local_ty, local_span)
            });
            drop_data.dropck_result.report_overflows(
@ -522,9 +527,10 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
            values::pretty_print_points(self.location_map, live_at.iter()),
        );
        let local_span = self.body.local_decls()[dropped_local].source_info.span;
        let drop_data = self.drop_data.entry(dropped_ty).or_insert_with({
            let typeck = &self.typeck;
-            move || Self::compute_drop_data(typeck, dropped_ty)
+            move || Self::compute_drop_data(typeck, dropped_ty, local_span)
        });
        if let Some(data) = &drop_data.region_constraint_data {
@ -589,19 +595,50 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
        }
    }
-    fn compute_drop_data(typeck: &TypeChecker<'_, 'tcx>, dropped_ty: Ty<'tcx>) -> DropData<'tcx> {
+    fn compute_drop_data(
-        debug!("compute_drop_data(dropped_ty={:?})", dropped_ty,);
+        typeck: &TypeChecker<'_, 'tcx>,
        dropped_ty: Ty<'tcx>,
        span: Span,
    ) -> DropData<'tcx> {
        debug!("compute_drop_data(dropped_ty={:?})", dropped_ty);
-        match typeck
+        let op = typeck.infcx.param_env.and(DropckOutlives { dropped_ty });
-            .infcx
+
-            .param_env
+        match op.fully_perform(typeck.infcx, DUMMY_SP) {
            .and(DropckOutlives { dropped_ty })
            .fully_perform(typeck.infcx, DUMMY_SP)
        {
            Ok(TypeOpOutput { output, constraints, .. }) => {
                DropData { dropck_result: output, region_constraint_data: constraints }
            }
-            Err(_) => DropData { dropck_result: Default::default(), region_constraint_data: None },
+            Err(_) => {
                // We don't run dropck on HIR, and dropck looks inside fields of
                // types, so there's no guarantee that it succeeds. We also
                // can't rely on the the `ErrorGuaranteed` from `fully_perform` here
                // because it comes from delay_span_bug.
                //
                // Do this inside of a probe because we don't particularly care (or want)
                // any region side-effects of this operation in our infcx.
                typeck.infcx.probe(|_| {
                    let ocx = ObligationCtxt::new_with_diagnostics(&typeck.infcx);
                    let errors = match dropck_outlives::compute_dropck_outlives_with_errors(
                        &ocx, op, span,
                    ) {
                        Ok(_) => ocx.select_all_or_error(),
                        Err(e) => {
                            if e.is_empty() {
                                ocx.select_all_or_error()
                            } else {
                                e
                            }
                        }
                    };
                    if !errors.is_empty() {
                        typeck.infcx.err_ctxt().report_fulfillment_errors(errors);
                    } else {
                        span_bug!(span, "Rerunning drop data query produced no error.");
                    }
                });
                DropData { dropck_result: Default::default(), region_constraint_data: None }
            }
        }
    }
 }
--- a/compiler/rustc_borrowck/src/type_check/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/mod.rs
@ -19,7 +19,6 @@ use rustc_infer::infer::{
    BoundRegion, BoundRegionConversionTime, InferCtxt, NllRegionVariableOrigin,
 };
 use rustc_infer::traits::PredicateObligations;
 use rustc_middle::mir::tcx::PlaceTy;
 use rustc_middle::mir::visit::{NonMutatingUseContext, PlaceContext, Visitor};
 use rustc_middle::mir::*;
 use rustc_middle::traits::query::NoSolution;
@ -457,38 +456,38 @@ impl<'a, 'b, 'tcx> Visitor<'tcx> for TypeVerifier<'a, 'b, 'tcx> {
    fn visit_local_decl(&mut self, local: Local, local_decl: &LocalDecl<'tcx>) {
        self.super_local_decl(local, local_decl);
-        if let Some(user_ty) = &local_decl.user_ty {
+        for user_ty in
-            for (user_ty, span) in user_ty.projections_and_spans() {
+            local_decl.user_ty.as_deref().into_iter().flat_map(UserTypeProjections::projections)
-                let ty = if !local_decl.is_nonref_binding() {
+        {
-                    // If we have a binding of the form `let ref x: T = ..`
+            let span = self.typeck.user_type_annotations[user_ty.base].span;
                    // then remove the outermost reference so we can check the
                    // type annotation for the remaining type.
                    if let ty::Ref(_, rty, _) = local_decl.ty.kind() {
                        *rty
                    } else {
                        bug!("{:?} with ref binding has wrong type {}", local, local_decl.ty);
                    }
                } else {
                    local_decl.ty
                };
-                if let Err(terr) = self.typeck.relate_type_and_user_type(
+            let ty = if local_decl.is_nonref_binding() {
-                    ty,
+                local_decl.ty
-                    ty::Invariant,
+            } else if let &ty::Ref(_, rty, _) = local_decl.ty.kind() {
-                    user_ty,
+                // If we have a binding of the form `let ref x: T = ..`
-                    Locations::All(*span),
+                // then remove the outermost reference so we can check the
-                    ConstraintCategory::TypeAnnotation(AnnotationSource::Declaration),
+                // type annotation for the remaining type.
-                ) {
+                rty
-                    span_mirbug!(
+            } else {
-                        self,
+                bug!("{:?} with ref binding has wrong type {}", local, local_decl.ty);
-                        local,
+            };
-                        "bad user type on variable {:?}: {:?} != {:?} ({:?})",
+
-                        local,
+            if let Err(terr) = self.typeck.relate_type_and_user_type(
-                        local_decl.ty,
+                ty,
-                        local_decl.user_ty,
+                ty::Invariant,
-                        terr,
+                user_ty,
-                    );
+                Locations::All(span),
-                }
+                ConstraintCategory::TypeAnnotation(AnnotationSource::Declaration),
            ) {
                span_mirbug!(
                    self,
                    local,
                    "bad user type on variable {:?}: {:?} != {:?} ({:?})",
                    local,
                    local_decl.ty,
                    local_decl.user_ty,
                    terr,
                );
            }
        }
    }
--- a/compiler/rustc_borrowck/src/universal_regions.rs
+++ b/compiler/rustc_borrowck/src/universal_regions.rs
@ -21,6 +21,7 @@ use std::iter;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_errors::Diag;
 use rustc_hir::BodyOwnerKind;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::lang_items::LangItem;
 use rustc_index::IndexVec;
@ -125,6 +126,11 @@ pub(crate) enum DefiningTy<'tcx> {
    /// The MIR represents an inline const. The signature has no inputs and a
    /// single return value found via `InlineConstArgs::ty`.
    InlineConst(DefId, GenericArgsRef<'tcx>),
    // Fake body for a global asm. Not particularly useful or interesting,
    // but we need it so we can properly store the typeck results of the asm
    // operands, which aren't associated with a body otherwise.
    GlobalAsm(DefId),
 }
 impl<'tcx> DefiningTy<'tcx> {
@ -137,9 +143,10 @@ impl<'tcx> DefiningTy<'tcx> {
            DefiningTy::Closure(_, args) => args.as_closure().upvar_tys(),
            DefiningTy::CoroutineClosure(_, args) => args.as_coroutine_closure().upvar_tys(),
            DefiningTy::Coroutine(_, args) => args.as_coroutine().upvar_tys(),
-            DefiningTy::FnDef(..) | DefiningTy::Const(..) | DefiningTy::InlineConst(..) => {
+            DefiningTy::FnDef(..)
-                ty::List::empty()
+            | DefiningTy::Const(..)
-            }
+            | DefiningTy::InlineConst(..)
            | DefiningTy::GlobalAsm(_) => ty::List::empty(),
        }
    }
@ -151,7 +158,10 @@ impl<'tcx> DefiningTy<'tcx> {
            DefiningTy::Closure(..)
            | DefiningTy::CoroutineClosure(..)
            | DefiningTy::Coroutine(..) => 1,
-            DefiningTy::FnDef(..) | DefiningTy::Const(..) | DefiningTy::InlineConst(..) => 0,
+            DefiningTy::FnDef(..)
            | DefiningTy::Const(..)
            | DefiningTy::InlineConst(..)
            | DefiningTy::GlobalAsm(_) => 0,
        }
    }
@ -170,7 +180,8 @@ impl<'tcx> DefiningTy<'tcx> {
            | DefiningTy::Coroutine(def_id, ..)
            | DefiningTy::FnDef(def_id, ..)
            | DefiningTy::Const(def_id, ..)
-            | DefiningTy::InlineConst(def_id, ..) => def_id,
+            | DefiningTy::InlineConst(def_id, ..)
            | DefiningTy::GlobalAsm(def_id) => def_id,
        }
    }
 }
@ -307,7 +318,7 @@ impl<'tcx> UniversalRegions<'tcx> {
    /// Returns an iterator over all the RegionVids corresponding to
    /// universally quantified free regions.
-    pub(crate) fn universal_regions_iter(&self) -> impl Iterator<Item = RegionVid> {
+    pub(crate) fn universal_regions_iter(&self) -> impl Iterator<Item = RegionVid> + 'static {
        (FIRST_GLOBAL_INDEX..self.num_universals).map(RegionVid::from_usize)
    }
@ -331,9 +342,9 @@ impl<'tcx> UniversalRegions<'tcx> {
    }
    /// Gets an iterator over all the early-bound regions that have names.
-    pub(crate) fn named_universal_regions_iter<'s>(
+    pub(crate) fn named_universal_regions_iter(
-        &'s self,
+        &self,
-    ) -> impl Iterator<Item = (ty::Region<'tcx>, ty::RegionVid)> + 's {
+    ) -> impl Iterator<Item = (ty::Region<'tcx>, ty::RegionVid)> {
        self.indices.indices.iter().map(|(&r, &v)| (r, v))
    }
@ -410,6 +421,7 @@ impl<'tcx> UniversalRegions<'tcx> {
                    tcx.def_path_str_with_args(def_id, args),
                ));
            }
            DefiningTy::GlobalAsm(_) => unreachable!(),
        }
    }
@ -576,7 +588,7 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
        let tcx = self.infcx.tcx;
        let typeck_root_def_id = tcx.typeck_root_def_id(self.mir_def.to_def_id());
-        match tcx.hir().body_owner_kind(self.mir_def) {
+        match tcx.hir_body_owner_kind(self.mir_def) {
            BodyOwnerKind::Closure | BodyOwnerKind::Fn => {
                let defining_ty = tcx.type_of(self.mir_def).instantiate_identity();
@ -603,7 +615,10 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
            BodyOwnerKind::Const { .. } | BodyOwnerKind::Static(..) => {
                let identity_args = GenericArgs::identity_for_item(tcx, typeck_root_def_id);
-                if self.mir_def.to_def_id() == typeck_root_def_id {
+                if self.mir_def.to_def_id() == typeck_root_def_id
                    // Do not ICE when checking default_field_values consts with lifetimes (#135649)
                    && DefKind::Field != tcx.def_kind(tcx.parent(typeck_root_def_id))
                {
                    let args =
                        self.infcx.replace_free_regions_with_nll_infer_vars(FR, identity_args);
                    DefiningTy::Const(self.mir_def.to_def_id(), args)
@ -629,6 +644,8 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
                    DefiningTy::InlineConst(self.mir_def.to_def_id(), args)
                }
            }
            BodyOwnerKind::GlobalAsm => DefiningTy::GlobalAsm(self.mir_def.to_def_id()),
        }
    }
@ -662,6 +679,8 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
            }
            DefiningTy::FnDef(_, args) | DefiningTy::Const(_, args) => args,
            DefiningTy::GlobalAsm(_) => ty::List::empty(),
        };
        let global_mapping = iter::once((tcx.lifetimes.re_static, fr_static));
@ -798,6 +817,10 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
                let ty = args.as_inline_const().ty();
                ty::Binder::dummy(tcx.mk_type_list(&[ty]))
            }
            DefiningTy::GlobalAsm(def_id) => {
                ty::Binder::dummy(tcx.mk_type_list(&[tcx.type_of(def_id).instantiate_identity()]))
            }
        };
        // FIXME(#129952): We probably want a more principled approach here.
--- a/compiler/rustc_builtin_macros/Cargo.toml
+++ b/compiler/rustc_builtin_macros/Cargo.toml
@ -1,7 +1,7 @@
 [package]
 name = "rustc_builtin_macros"
 version = "0.0.0"
-edition = "2021"
+edition = "2024"
 [lints.rust]
--- a/compiler/rustc_builtin_macros/src/autodiff.rs
+++ b/compiler/rustc_builtin_macros/src/autodiff.rs
@ -242,6 +242,7 @@ mod llvm_enzyme {
            defaultness: ast::Defaultness::Final,
            sig: d_sig,
            generics: Generics::default(),
            contract: None,
            body: Some(d_body),
        });
        let mut rustc_ad_attr =
--- a/compiler/rustc_builtin_macros/src/standard_library_imports.rs
+++ b/compiler/rustc_builtin_macros/src/standard_library_imports.rs
@ -19,16 +19,12 @@ pub fn inject(
    let edition = sess.psess.edition;
    // the first name in this list is the crate name of the crate with the prelude
-    let names: &[Symbol] = if attr::contains_name(pre_configured_attrs, sym::no_core) {
+    let name: Symbol = if attr::contains_name(pre_configured_attrs, sym::no_core) {
        return 0;
    } else if attr::contains_name(pre_configured_attrs, sym::no_std) {
-        if attr::contains_name(pre_configured_attrs, sym::compiler_builtins) {
+        sym::core
            &[sym::core]
        } else {
            &[sym::core, sym::compiler_builtins]
        }
    } else {
-        &[sym::std]
+        sym::std
    };
    let expn_id = resolver.expansion_for_ast_pass(
@ -43,36 +39,16 @@ pub fn inject(
    let ecfg = ExpansionConfig::default("std_lib_injection".to_string(), features);
    let cx = ExtCtxt::new(sess, ecfg, resolver, None);
-    // .rev() to preserve ordering above in combination with insert(0, ...)
+    let ident_span = if edition >= Edition2018 { span } else { call_site };
    for &name in names.iter().rev() {
        let ident_span = if edition >= Edition2018 { span } else { call_site };
        let item = if name == sym::compiler_builtins {
            // compiler_builtins is a private implementation detail. We only
            // need to insert it into the crate graph for linking and should not
            // expose any of its public API.
            //
            // FIXME(#113634) We should inject this during post-processing like
            // we do for the panic runtime, profiler runtime, etc.
            cx.item(
                span,
                Ident::new(kw::Underscore, ident_span),
                thin_vec![],
                ast::ItemKind::ExternCrate(Some(name)),
            )
        } else {
            cx.item(
                span,
                Ident::new(name, ident_span),
                thin_vec![cx.attr_word(sym::macro_use, span)],
                ast::ItemKind::ExternCrate(None),
            )
        };
        krate.items.insert(0, item);
    }
-    // The crates have been injected, the assumption is that the first one is
+    let item = cx.item(
-    // the one with the prelude.
+        span,
-    let name = names[0];
+        Ident::new(name, ident_span),
        thin_vec![cx.attr_word(sym::macro_use, span)],
        ast::ItemKind::ExternCrate(None),
    );
    krate.items.insert(0, item);
    let root = (edition == Edition2015).then_some(kw::PathRoot);
@ -88,6 +64,7 @@ pub fn inject(
        .map(|&symbol| Ident::new(symbol, span))
        .collect();
    // Inject the relevant crate's prelude.
    let use_item = cx.item(
        span,
        Ident::empty(),
--- a/compiler/rustc_codegen_cranelift/patches/0029-stdlib-Disable-f16-and-f128-in-compiler-builtins.patch
+++ b/compiler/rustc_codegen_cranelift/patches/0029-stdlib-Disable-f16-and-f128-in-compiler-builtins.patch
@ -15,9 +15,9 @@ index 7165c3e48af..968552ad435 100644
 edition = "2021"
 [dependencies]
- core = { path = "../core" }
+ core = { path = "../core", public = true }
-compiler_builtins = { version = "=0.1.146", features = ['rustc-dep-of-std'] }
+-compiler_builtins = { version = "=0.1.147", features = ['rustc-dep-of-std'] }
-+compiler_builtins = { version = "=0.1.146", features = ['rustc-dep-of-std', 'no-f16-f128'] }
+compiler_builtins = { version = "=0.1.147", features = ['rustc-dep-of-std', 'no-f16-f128'] }
 [dev-dependencies]
 rand = { version = "0.8.5", default-features = false, features = ["alloc"] }
--- a/compiler/rustc_codegen_cranelift/src/abi/comments.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/comments.rs
@ -65,7 +65,7 @@ pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
    if fx.clif_comments.enabled() {
        fx.add_global_comment(String::new());
        fx.add_global_comment(
-            "kind  local ty                              size align (abi,pref)".to_string(),
+            "kind  local ty                              size align (abi)".to_string(),
        );
    }
 }
@ -84,14 +84,13 @@ pub(super) fn add_local_place_comments<'tcx>(
    let (kind, extra) = place.debug_comment();
    fx.add_global_comment(format!(
-        "{:<5} {:5} {:30} {:4}b {}, {}{}{}",
+        "{:<5} {:5} {:30} {:4}b {}{}{}",
        kind,
        format!("{:?}", local),
        format!("{:?}", ty),
        size.bytes(),
        align.abi.bytes(),
-        align.pref.bytes(),
+        if extra.is_empty() { "" } else { "                " },
        if extra.is_empty() { "" } else { "              " },
        extra,
    ));
 }
--- a/compiler/rustc_codegen_cranelift/src/constant.rs
+++ b/compiler/rustc_codegen_cranelift/src/constant.rs
@ -272,7 +272,7 @@ fn data_id_for_static(
            .layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(ty))
            .unwrap()
            .align
-            .pref
+            .abi
            .bytes();
        let linkage = if import_linkage == rustc_middle::mir::mono::Linkage::ExternalWeak
--- a/compiler/rustc_codegen_cranelift/src/global_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/global_asm.rs
@ -16,7 +16,7 @@ use crate::prelude::*;
 pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String, item_id: ItemId) {
    let item = tcx.hir_item(item_id);
-    if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind {
+    if let rustc_hir::ItemKind::GlobalAsm { asm, .. } = item.kind {
        let is_x86 =
            matches!(tcx.sess.asm_arch.unwrap(), InlineAsmArch::X86 | InlineAsmArch::X86_64);
@ -55,7 +55,7 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String,
                                }
                            }
                        }
-                        InlineAsmOperand::SymFn { anon_const } => {
+                        InlineAsmOperand::SymFn { expr } => {
                            if cfg!(not(feature = "inline_asm_sym")) {
                                tcx.dcx().span_err(
                                    item.span,
@ -63,7 +63,7 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String,
                                );
                            }
-                            let ty = tcx.typeck_body(anon_const.body).node_type(anon_const.hir_id);
+                            let ty = tcx.typeck(item_id.owner_id).expr_ty(expr);
                            let instance = match ty.kind() {
                                &ty::FnDef(def_id, args) => Instance::new(def_id, args),
                                _ => span_bug!(op_sp, "asm sym is not a function"),
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
@ -340,14 +340,10 @@ fn codegen_float_intrinsic_call<'tcx>(
        sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64, types::F64),
        sym::truncf32 => ("truncf", 1, fx.tcx.types.f32, types::F32),
        sym::truncf64 => ("trunc", 1, fx.tcx.types.f64, types::F64),
-        sym::rintf32 => ("rintf", 1, fx.tcx.types.f32, types::F32),
+        sym::round_ties_even_f32 => ("rintf", 1, fx.tcx.types.f32, types::F32),
-        sym::rintf64 => ("rint", 1, fx.tcx.types.f64, types::F64),
+        sym::round_ties_even_f64 => ("rint", 1, fx.tcx.types.f64, types::F64),
        sym::roundf32 => ("roundf", 1, fx.tcx.types.f32, types::F32),
        sym::roundf64 => ("round", 1, fx.tcx.types.f64, types::F64),
        sym::roundevenf32 => ("roundevenf", 1, fx.tcx.types.f32, types::F32),
        sym::roundevenf64 => ("roundeven", 1, fx.tcx.types.f64, types::F64),
        sym::nearbyintf32 => ("nearbyintf", 1, fx.tcx.types.f32, types::F32),
        sym::nearbyintf64 => ("nearbyint", 1, fx.tcx.types.f64, types::F64),
        sym::sinf32 => ("sinf", 1, fx.tcx.types.f32, types::F32),
        sym::sinf64 => ("sin", 1, fx.tcx.types.f64, types::F64),
        sym::cosf32 => ("cosf", 1, fx.tcx.types.f32, types::F32),
@ -399,8 +395,8 @@ fn codegen_float_intrinsic_call<'tcx>(
        | sym::ceilf64
        | sym::truncf32
        | sym::truncf64
-        | sym::nearbyintf32
+        | sym::round_ties_even_f32
-        | sym::nearbyintf64
+        | sym::round_ties_even_f64
        | sym::sqrtf32
        | sym::sqrtf64 => {
            let val = match intrinsic {
@ -408,7 +404,9 @@ fn codegen_float_intrinsic_call<'tcx>(
                sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(args[0]),
                sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(args[0]),
                sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(args[0]),
-                sym::nearbyintf32 | sym::nearbyintf64 => fx.bcx.ins().nearest(args[0]),
+                sym::round_ties_even_f32 | sym::round_ties_even_f64 => {
                    fx.bcx.ins().nearest(args[0])
                }
                sym::sqrtf32 | sym::sqrtf64 => fx.bcx.ins().sqrt(args[0]),
                _ => unreachable!(),
            };
--- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs
+++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
@ -638,9 +638,7 @@ impl<'tcx> CPlace<'tcx> {
            }
            CPlaceInner::Addr(_, Some(_)) => bug!("Can't write value to unsized place {:?}", self),
            CPlaceInner::Addr(to_ptr, None) => {
-                if dst_layout.size == Size::ZERO
+                if dst_layout.size == Size::ZERO {
                    || dst_layout.backend_repr == BackendRepr::Uninhabited
                {
                    return;
                }
--- a/compiler/rustc_codegen_gcc/src/builder.rs
+++ b/compiler/rustc_codegen_gcc/src/builder.rs
@ -989,10 +989,14 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
            OperandValue::Ref(place.val)
        } else if place.layout.is_gcc_immediate() {
            let load = self.load(place.layout.gcc_type(self), place.val.llval, place.val.align);
-            if let abi::BackendRepr::Scalar(ref scalar) = place.layout.backend_repr {
+            OperandValue::Immediate(
-                scalar_load_metadata(self, load, scalar);
+                if let abi::BackendRepr::Scalar(ref scalar) = place.layout.backend_repr {
-            }
+                    scalar_load_metadata(self, load, scalar);
-            OperandValue::Immediate(self.to_immediate(load, place.layout))
+                    self.to_immediate_scalar(load, *scalar)
                } else {
                    load
                },
            )
        } else if let abi::BackendRepr::ScalarPair(ref a, ref b) = place.layout.backend_repr {
            let b_offset = a.size(self).align_to(b.align(self).abi);
@ -1694,7 +1698,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
    fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
        if scalar.is_bool() {
-            return self.trunc(val, self.cx().type_i1());
+            return self.unchecked_utrunc(val, self.cx().type_i1());
        }
        val
    }
--- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
+++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
@ -9,7 +9,7 @@ use gccjit::FunctionType;
 use gccjit::{ComparisonOp, Function, RValue, ToRValue, Type, UnaryOp};
 #[cfg(feature = "master")]
 use rustc_abi::ExternAbi;
-use rustc_abi::HasDataLayout;
+use rustc_abi::{BackendRepr, HasDataLayout};
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::base::wants_msvc_seh;
 use rustc_codegen_ssa::common::IntPredicate;
@ -84,14 +84,11 @@ fn get_simple_intrinsic<'gcc, 'tcx>(
        sym::ceilf64 => "ceil",
        sym::truncf32 => "truncf",
        sym::truncf64 => "trunc",
-        sym::rintf32 => "rintf",
+        // We match the LLVM backend and lower this to `rint`.
-        sym::rintf64 => "rint",
+        sym::round_ties_even_f32 => "rintf",
-        sym::nearbyintf32 => "nearbyintf",
+        sym::round_ties_even_f64 => "rint",
        sym::nearbyintf64 => "nearbyint",
        sym::roundf32 => "roundf",
        sym::roundf64 => "round",
        sym::roundevenf32 => "roundevenf",
        sym::roundevenf64 => "roundeven",
        sym::abort => "abort",
        _ => return None,
    };
@ -181,14 +178,19 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
            sym::volatile_load | sym::unaligned_volatile_load => {
                let tp_ty = fn_args.type_at(0);
                let ptr = args[0].immediate();
                let layout = self.layout_of(tp_ty);
                let load = if let PassMode::Cast { cast: ref ty, pad_i32: _ } = fn_abi.ret.mode {
                    let gcc_ty = ty.gcc_type(self);
                    self.volatile_load(gcc_ty, ptr)
                } else {
-                    self.volatile_load(self.layout_of(tp_ty).gcc_type(self), ptr)
+                    self.volatile_load(layout.gcc_type(self), ptr)
                };
                // TODO(antoyo): set alignment.
-                self.to_immediate(load, self.layout_of(tp_ty))
+                if let BackendRepr::Scalar(scalar) = layout.backend_repr {
                    self.to_immediate_scalar(load, scalar)
                } else {
                    load
                }
            }
            sym::volatile_store => {
                let dst = args[0].deref(self.cx());
@ -310,7 +312,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
                let layout = self.layout_of(tp_ty).layout;
                let _use_integer_compare = match layout.backend_repr() {
                    Scalar(_) | ScalarPair(_, _) => true,
-                    Uninhabited | Vector { .. } => false,
+                    Vector { .. } => false,
                    Memory { .. } => {
                        // For rusty ABIs, small aggregates are actually passed
                        // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),
--- a/compiler/rustc_codegen_gcc/src/type_of.rs
+++ b/compiler/rustc_codegen_gcc/src/type_of.rs
@ -84,7 +84,7 @@ fn uncached_gcc_type<'gcc, 'tcx>(
                false,
            );
        }
-        BackendRepr::Uninhabited | BackendRepr::Memory { .. } => {}
+        BackendRepr::Memory { .. } => {}
    }
    let name = match *layout.ty.kind() {
@ -179,19 +179,16 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
    fn is_gcc_immediate(&self) -> bool {
        match self.backend_repr {
            BackendRepr::Scalar(_) | BackendRepr::Vector { .. } => true,
-            BackendRepr::ScalarPair(..) | BackendRepr::Uninhabited | BackendRepr::Memory { .. } => {
+            BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => false,
                false
            }
        }
    }
    fn is_gcc_scalar_pair(&self) -> bool {
        match self.backend_repr {
            BackendRepr::ScalarPair(..) => true,
-            BackendRepr::Uninhabited
+            BackendRepr::Scalar(_) | BackendRepr::Vector { .. } | BackendRepr::Memory { .. } => {
-            | BackendRepr::Scalar(_)
+                false
-            | BackendRepr::Vector { .. }
+            }
            | BackendRepr::Memory { .. } => false,
        }
    }
--- a/compiler/rustc_codegen_llvm/Cargo.toml
+++ b/compiler/rustc_codegen_llvm/Cargo.toml
@ -1,7 +1,7 @@
 [package]
 name = "rustc_codegen_llvm"
 version = "0.0.0"
-edition = "2021"
+edition = "2024"
 [lib]
 test = false
@ -25,6 +25,7 @@ rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_fs_util = { path = "../rustc_fs_util" }
 rustc_hashes = { path = "../rustc_hashes" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_llvm = { path = "../rustc_llvm" }
--- a/compiler/rustc_codegen_llvm/messages.ftl
+++ b/compiler/rustc_codegen_llvm/messages.ftl
@ -1,3 +1,4 @@
 codegen_llvm_autodiff_without_enable = using the autodiff feature requires -Z autodiff=Enable
 codegen_llvm_autodiff_without_lto = using the autodiff feature requires using fat-lto
 codegen_llvm_copy_bitcode = failed to copy bitcode to object file: {$err}
--- a/compiler/rustc_codegen_llvm/src/abi.rs
+++ b/compiler/rustc_codegen_llvm/src/abi.rs
@ -2,19 +2,18 @@ use std::borrow::Borrow;
 use std::cmp;
 use libc::c_uint;
-use rustc_abi as abi;
+use rustc_abi::{BackendRepr, HasDataLayout, Primitive, Reg, RegKind, Size};
 pub(crate) use rustc_abi::ExternAbi;
 use rustc_abi::{HasDataLayout, Primitive, Reg, RegKind, Size};
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
 use rustc_codegen_ssa::traits::*;
 use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::LayoutOf;
 pub(crate) use rustc_middle::ty::layout::{WIDE_PTR_ADDR, WIDE_PTR_EXTRA};
 use rustc_middle::{bug, ty};
 use rustc_session::config;
-pub(crate) use rustc_target::callconv::*;
+use rustc_target::callconv::{
    ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, Conv, FnAbi, PassMode,
 };
 use rustc_target::spec::SanitizerSet;
 use smallvec::SmallVec;
@ -458,7 +457,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
        match &self.ret.mode {
            PassMode::Direct(attrs) => {
                attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn);
-                if let abi::BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr {
+                if let BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr {
                    apply_range_attr(llvm::AttributePlace::ReturnValue, scalar);
                }
            }
@ -499,7 +498,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                }
                PassMode::Direct(attrs) => {
                    let i = apply(attrs);
-                    if let abi::BackendRepr::Scalar(scalar) = arg.layout.backend_repr {
+                    if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr {
                        apply_range_attr(llvm::AttributePlace::Argument(i), scalar);
                    }
                }
@ -514,9 +513,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                PassMode::Pair(a, b) => {
                    let i = apply(a);
                    let ii = apply(b);
-                    if let abi::BackendRepr::ScalarPair(scalar_a, scalar_b) =
+                    if let BackendRepr::ScalarPair(scalar_a, scalar_b) = arg.layout.backend_repr {
                        arg.layout.backend_repr
                    {
                        apply_range_attr(llvm::AttributePlace::Argument(i), scalar_a);
                        apply_range_attr(llvm::AttributePlace::Argument(ii), scalar_b);
                    }
@ -576,7 +573,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
        }
        if bx.cx.sess().opts.optimize != config::OptLevel::No
                && llvm_util::get_version() < (19, 0, 0)
-                && let abi::BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr
+                && let BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr
                && matches!(scalar.primitive(), Primitive::Int(..))
                // If the value is a boolean, the range is 0..2 and that ultimately
                // become 0..0 when the type becomes i1, which would be rejected
--- a/compiler/rustc_codegen_llvm/src/back/lto.rs
+++ b/compiler/rustc_codegen_llvm/src/back/lto.rs
@ -362,8 +362,8 @@ fn fat_lto(
                ptr as *const *const libc::c_char,
                symbols_below_threshold.len() as libc::size_t,
            );
            save_temp_bitcode(cgcx, &module, "lto.after-restriction");
        }
        save_temp_bitcode(cgcx, &module, "lto.after-restriction");
    }
    Ok(LtoModuleCodegen::Fat(module))
@ -586,6 +586,42 @@ fn thin_lto(
    }
 }
 fn enable_autodiff_settings(ad: &[config::AutoDiff], module: &mut ModuleCodegen<ModuleLlvm>) {
    for &val in ad {
        match val {
            config::AutoDiff::PrintModBefore => {
                unsafe { llvm::LLVMDumpModule(module.module_llvm.llmod()) };
            }
            config::AutoDiff::PrintPerf => {
                llvm::set_print_perf(true);
            }
            config::AutoDiff::PrintAA => {
                llvm::set_print_activity(true);
            }
            config::AutoDiff::PrintTA => {
                llvm::set_print_type(true);
            }
            config::AutoDiff::Inline => {
                llvm::set_inline(true);
            }
            config::AutoDiff::LooseTypes => {
                llvm::set_loose_types(false);
            }
            config::AutoDiff::PrintSteps => {
                llvm::set_print(true);
            }
            // We handle this below
            config::AutoDiff::PrintModAfter => {}
            // This is required and already checked
            config::AutoDiff::Enable => {}
        }
    }
    // This helps with handling enums for now.
    llvm::set_strict_aliasing(false);
    // FIXME(ZuseZ4): Test this, since it was added a long time ago.
    llvm::set_rust_rules(true);
 }
 pub(crate) fn run_pass_manager(
    cgcx: &CodegenContext<LlvmCodegenBackend>,
    dcx: DiagCtxtHandle<'_>,
@ -604,34 +640,37 @@ pub(crate) fn run_pass_manager(
    let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO };
    let opt_level = config.opt_level.unwrap_or(config::OptLevel::No);
-    // If this rustc version was build with enzyme/autodiff enabled, and if users applied the
+    // The PostAD behavior is the same that we would have if no autodiff was used.
-    // `#[autodiff]` macro at least once, then we will later call llvm_optimize a second time.
+    // It will run the default optimization pipeline. If AD is enabled we select
-    debug!("running llvm pm opt pipeline");
+    // the DuringAD stage, which will disable vectorization and loop unrolling, and
-    unsafe {
+    // schedule two autodiff optimization + differentiation passes.
-        write::llvm_optimize(
+    // We then run the llvm_optimize function a second time, to optimize the code which we generated
-            cgcx,
+    // in the enzyme differentiation pass.
-            dcx,
+    let enable_ad = config.autodiff.contains(&config::AutoDiff::Enable);
-            module,
+    let stage =
-            config,
+        if enable_ad { write::AutodiffStage::DuringAD } else { write::AutodiffStage::PostAD };
-            opt_level,
+
-            opt_stage,
+    if enable_ad {
-            write::AutodiffStage::DuringAD,
+        enable_autodiff_settings(&config.autodiff, module);
        )?;
    }
-    // FIXME(ZuseZ4): Make this more granular
+
-    if cfg!(llvm_enzyme) && !thin {
+    unsafe {
        write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, stage)?;
    }
    if cfg!(llvm_enzyme) && enable_ad {
        let opt_stage = llvm::OptStage::FatLTO;
        let stage = write::AutodiffStage::PostAD;
        unsafe {
-            write::llvm_optimize(
+            write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, stage)?;
-                cgcx,
+        }
-                dcx,
+
-                module,
+        // This is the final IR, so people should be able to inspect the optimized autodiff output.
-                config,
+        if config.autodiff.contains(&config::AutoDiff::PrintModAfter) {
-                opt_level,
+            unsafe { llvm::LLVMDumpModule(module.module_llvm.llmod()) };
                llvm::OptStage::FatLTO,
                write::AutodiffStage::PostAD,
            )?;
        }
    }
    debug!("lto done");
    Ok(())
 }
--- a/compiler/rustc_codegen_llvm/src/back/write.rs
+++ b/compiler/rustc_codegen_llvm/src/back/write.rs
@ -564,19 +564,16 @@ pub(crate) unsafe fn llvm_optimize(
    // FIXME(ZuseZ4): In a future update we could figure out how to only optimize individual functions getting
    // differentiated.
    let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable);
    let run_enzyme = autodiff_stage == AutodiffStage::DuringAD;
    let unroll_loops;
    let vectorize_slp;
    let vectorize_loop;
    let run_enzyme = cfg!(llvm_enzyme) && autodiff_stage == AutodiffStage::DuringAD;
    // When we build rustc with enzyme/autodiff support, we want to postpone size-increasing
    // optimizations until after differentiation. Our pipeline is thus: (opt + enzyme), (full opt).
    // We therefore have two calls to llvm_optimize, if autodiff is used.
-    //
+    if consider_ad && autodiff_stage != AutodiffStage::PostAD {
    // FIXME(ZuseZ4): Before shipping on nightly,
    // we should make this more granular, or at least check that the user has at least one autodiff
    // call in their code, to justify altering the compilation pipeline.
    if cfg!(llvm_enzyme) && autodiff_stage != AutodiffStage::PostAD {
        unroll_loops = false;
        vectorize_slp = false;
        vectorize_loop = false;
@ -706,10 +703,8 @@ pub(crate) unsafe fn optimize(
        // If we know that we will later run AD, then we disable vectorization and loop unrolling.
        // Otherwise we pretend AD is already done and run the normal opt pipeline (=PostAD).
-        // FIXME(ZuseZ4): Make this more granular, only set PreAD if we actually have autodiff
+        let consider_ad = cfg!(llvm_enzyme) && config.autodiff.contains(&config::AutoDiff::Enable);
-        // usages, not just if we build rustc with autodiff support.
+        let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD };
        let autodiff_stage =
            if cfg!(llvm_enzyme) { AutodiffStage::PreAD } else { AutodiffStage::PostAD };
        return unsafe {
            llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, autodiff_stage)
        };
--- a/compiler/rustc_codegen_llvm/src/builder.rs
+++ b/compiler/rustc_codegen_llvm/src/builder.rs
@ -4,7 +4,7 @@ use std::{iter, ptr};
 pub(crate) mod autodiff;
-use libc::{c_char, c_uint};
+use libc::{c_char, c_uint, size_t};
 use rustc_abi as abi;
 use rustc_abi::{Align, Size, WrappingRange};
 use rustc_codegen_ssa::MemFlags;
@ -29,13 +29,15 @@ use smallvec::SmallVec;
 use tracing::{debug, instrument};
 use crate::abi::FnAbiLlvmExt;
 use crate::attributes;
 use crate::common::Funclet;
 use crate::context::{CodegenCx, SimpleCx};
-use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, True};
+use crate::llvm::{
    self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, GEPNoWrapFlags, Metadata, True,
 };
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
 use crate::{attributes, llvm_util};
 #[must_use]
 pub(crate) struct GenericBuilder<'a, 'll, CX: Borrow<SimpleCx<'ll>>> {
@ -333,6 +335,50 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
        }
    }
    fn switch_with_weights(
        &mut self,
        v: Self::Value,
        else_llbb: Self::BasicBlock,
        else_is_cold: bool,
        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
    ) {
        if self.cx.sess().opts.optimize == rustc_session::config::OptLevel::No {
            self.switch(v, else_llbb, cases.map(|(val, dest, _)| (val, dest)));
            return;
        }
        let id_str = "branch_weights";
        let id = unsafe {
            llvm::LLVMMDStringInContext2(self.cx.llcx, id_str.as_ptr().cast(), id_str.len())
        };
        // For switch instructions with 2 targets, the `llvm.expect` intrinsic is used.
        // This function handles switch instructions with more than 2 targets and it needs to
        // emit branch weights metadata instead of using the intrinsic.
        // The values 1 and 2000 are the same as the values used by the `llvm.expect` intrinsic.
        let cold_weight = unsafe { llvm::LLVMValueAsMetadata(self.cx.const_u32(1)) };
        let hot_weight = unsafe { llvm::LLVMValueAsMetadata(self.cx.const_u32(2000)) };
        let weight =
            |is_cold: bool| -> &Metadata { if is_cold { cold_weight } else { hot_weight } };
        let mut md: SmallVec<[&Metadata; 16]> = SmallVec::with_capacity(cases.len() + 2);
        md.push(id);
        md.push(weight(else_is_cold));
        let switch =
            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
        for (on_val, dest, is_cold) in cases {
            let on_val = self.const_uint_big(self.val_ty(v), on_val);
            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
            md.push(weight(is_cold));
        }
        unsafe {
            let md_node = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len() as size_t);
            self.cx.set_metadata(switch, llvm::MD_prof, md_node);
        }
    }
    fn invoke(
        &mut self,
        llty: &'ll Type,
@ -562,7 +608,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
    fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
        if scalar.is_bool() {
-            return self.trunc(val, self.cx().type_i1());
+            return self.unchecked_utrunc(val, self.cx().type_i1());
        }
        val
    }
@ -702,10 +748,12 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
                let load = self.load(llty, place.val.llval, place.val.align);
                if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr {
                    scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO);
                    self.to_immediate_scalar(load, scalar)
                } else {
                    load
                }
                load
            });
-            OperandValue::Immediate(self.to_immediate(llval, place.layout))
+            OperandValue::Immediate(llval)
        } else if let abi::BackendRepr::ScalarPair(a, b) = place.layout.backend_repr {
            let b_offset = a.size(self).align_to(b.align(self).abi);
@ -864,13 +912,14 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
    fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
        unsafe {
-            llvm::LLVMBuildGEP2(
+            llvm::LLVMBuildGEPWithNoWrapFlags(
                self.llbuilder,
                ty,
                ptr,
                indices.as_ptr(),
                indices.len() as c_uint,
                UNNAMED,
                GEPNoWrapFlags::default(),
            )
        }
    }
@ -882,13 +931,33 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
        indices: &[&'ll Value],
    ) -> &'ll Value {
        unsafe {
-            llvm::LLVMBuildInBoundsGEP2(
+            llvm::LLVMBuildGEPWithNoWrapFlags(
                self.llbuilder,
                ty,
                ptr,
                indices.as_ptr(),
                indices.len() as c_uint,
                UNNAMED,
                GEPNoWrapFlags::InBounds,
            )
        }
    }
    fn inbounds_nuw_gep(
        &mut self,
        ty: &'ll Type,
        ptr: &'ll Value,
        indices: &[&'ll Value],
    ) -> &'ll Value {
        unsafe {
            llvm::LLVMBuildGEPWithNoWrapFlags(
                self.llbuilder,
                ty,
                ptr,
                indices.as_ptr(),
                indices.len() as c_uint,
                UNNAMED,
                GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW,
            )
        }
    }
@ -898,6 +967,34 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
        unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
    }
    fn unchecked_utrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
        debug_assert_ne!(self.val_ty(val), dest_ty);
        let trunc = self.trunc(val, dest_ty);
        if llvm_util::get_version() >= (19, 0, 0) {
            unsafe {
                if llvm::LLVMIsAInstruction(trunc).is_some() {
                    llvm::LLVMSetNUW(trunc, True);
                }
            }
        }
        trunc
    }
    fn unchecked_strunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
        debug_assert_ne!(self.val_ty(val), dest_ty);
        let trunc = self.trunc(val, dest_ty);
        if llvm_util::get_version() >= (19, 0, 0) {
            unsafe {
                if llvm::LLVMIsAInstruction(trunc).is_some() {
                    llvm::LLVMSetNSW(trunc, True);
                }
            }
        }
        trunc
    }
    fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
        unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) }
    }
--- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs
+++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs
@ -10,7 +10,7 @@ use crate::back::write::llvm_err;
 use crate::builder::SBuilder;
 use crate::context::SimpleCx;
 use crate::declare::declare_simple_fn;
-use crate::errors::LlvmError;
+use crate::errors::{AutoDiffWithoutEnable, LlvmError};
 use crate::llvm::AttributePlace::Function;
 use crate::llvm::{Metadata, True};
 use crate::value::Value;
@ -46,9 +46,6 @@ fn generate_enzyme_call<'ll>(
    let output = attrs.ret_activity;
    // We have to pick the name depending on whether we want forward or reverse mode autodiff.
    // FIXME(ZuseZ4): The new pass based approach should not need the {Forward/Reverse}First method anymore, since
    // it will handle higher-order derivatives correctly automatically (in theory). Currently
    // higher-order derivatives fail, so we should debug that before adjusting this code.
    let mut ad_name: String = match attrs.mode {
        DiffMode::Forward => "__enzyme_fwddiff",
        DiffMode::Reverse => "__enzyme_autodiff",
@ -291,6 +288,14 @@ pub(crate) fn differentiate<'ll>(
    let diag_handler = cgcx.create_dcx();
    let cx = SimpleCx { llmod: module.module_llvm.llmod(), llcx: module.module_llvm.llcx };
    // First of all, did the user try to use autodiff without using the -Zautodiff=Enable flag?
    if !diff_items.is_empty()
        && !cgcx.opts.unstable_opts.autodiff.contains(&rustc_session::config::AutoDiff::Enable)
    {
        let dcx = cgcx.create_dcx();
        return Err(dcx.handle().emit_almost_fatal(AutoDiffWithoutEnable));
    }
    // Before dumping the module, we want all the TypeTrees to become part of the module.
    for item in diff_items.iter() {
        let name = item.source.clone();
--- a/compiler/rustc_codegen_llvm/src/callee.rs
+++ b/compiler/rustc_codegen_llvm/src/callee.rs
@ -66,9 +66,7 @@ pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'t
            // LLVM will prefix the name with `__imp_`. Ideally, we'd like the
            // existing logic below to set the Storage Class, but it has an
            // exemption for MinGW for backwards compatibility.
-            unsafe {
+            llvm::set_dllimport_storage_class(llfn);
                llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport);
            }
            llfn
        } else {
            cx.declare_fn(sym, fn_abi, Some(instance))
@ -99,65 +97,61 @@ pub(crate) fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'t
        // has been applied to the definition (wherever that definition may be).
        llvm::set_linkage(llfn, llvm::Linkage::ExternalLinkage);
-        unsafe {
+        let is_generic = instance.args.non_erasable_generics().next().is_some();
            let is_generic = instance.args.non_erasable_generics().next().is_some();
-            let is_hidden = if is_generic {
+        let is_hidden = if is_generic {
-                // This is a monomorphization of a generic function.
+            // This is a monomorphization of a generic function.
-                if !(cx.tcx.sess.opts.share_generics()
+            if !(cx.tcx.sess.opts.share_generics()
-                    || tcx.codegen_fn_attrs(instance_def_id).inline
+                || tcx.codegen_fn_attrs(instance_def_id).inline
-                        == rustc_attr_parsing::InlineAttr::Never)
+                    == rustc_attr_parsing::InlineAttr::Never)
                {
                    // When not sharing generics, all instances are in the same
                    // crate and have hidden visibility.
                    true
                } else {
                    if let Some(instance_def_id) = instance_def_id.as_local() {
                        // This is a monomorphization of a generic function
                        // defined in the current crate. It is hidden if:
                        // - the definition is unreachable for downstream
                        //   crates, or
                        // - the current crate does not re-export generics
                        //   (because the crate is a C library or executable)
                        cx.tcx.is_unreachable_local_definition(instance_def_id)
                            || !cx.tcx.local_crate_exports_generics()
                    } else {
                        // This is a monomorphization of a generic function
                        // defined in an upstream crate. It is hidden if:
                        // - it is instantiated in this crate, and
                        // - the current crate does not re-export generics
                        instance.upstream_monomorphization(tcx).is_none()
                            && !cx.tcx.local_crate_exports_generics()
                    }
                }
            } else {
                // This is a non-generic function. It is hidden if:
                // - it is instantiated in the local crate, and
                //   - it is defined an upstream crate (non-local), or
                //   - it is not reachable
                cx.tcx.is_codegened_item(instance_def_id)
                    && (!instance_def_id.is_local()
                        || !cx.tcx.is_reachable_non_generic(instance_def_id))
            };
            if is_hidden {
                llvm::set_visibility(llfn, llvm::Visibility::Hidden);
            }
            // MinGW: For backward compatibility we rely on the linker to decide whether it
            // should use dllimport for functions.
            if cx.use_dll_storage_attrs
                && let Some(library) = tcx.native_library(instance_def_id)
                && library.kind.is_dllimport()
                && !matches!(tcx.sess.target.env.as_ref(), "gnu" | "uclibc")
            {
-                llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport);
+                // When not sharing generics, all instances are in the same
-            }
+                // crate and have hidden visibility.
-
+                true
-            if cx.should_assume_dso_local(llfn, true) {
+            } else {
-                llvm::LLVMRustSetDSOLocal(llfn, true);
+                if let Some(instance_def_id) = instance_def_id.as_local() {
                    // This is a monomorphization of a generic function
                    // defined in the current crate. It is hidden if:
                    // - the definition is unreachable for downstream
                    //   crates, or
                    // - the current crate does not re-export generics
                    //   (because the crate is a C library or executable)
                    cx.tcx.is_unreachable_local_definition(instance_def_id)
                        || !cx.tcx.local_crate_exports_generics()
                } else {
                    // This is a monomorphization of a generic function
                    // defined in an upstream crate. It is hidden if:
                    // - it is instantiated in this crate, and
                    // - the current crate does not re-export generics
                    instance.upstream_monomorphization(tcx).is_none()
                        && !cx.tcx.local_crate_exports_generics()
                }
            }
        } else {
            // This is a non-generic function. It is hidden if:
            // - it is instantiated in the local crate, and
            //   - it is defined an upstream crate (non-local), or
            //   - it is not reachable
            cx.tcx.is_codegened_item(instance_def_id)
                && (!instance_def_id.is_local()
                    || !cx.tcx.is_reachable_non_generic(instance_def_id))
        };
        if is_hidden {
            llvm::set_visibility(llfn, llvm::Visibility::Hidden);
        }
        // MinGW: For backward compatibility we rely on the linker to decide whether it
        // should use dllimport for functions.
        if cx.use_dll_storage_attrs
            && let Some(library) = tcx.native_library(instance_def_id)
            && library.kind.is_dllimport()
            && !matches!(tcx.sess.target.env.as_ref(), "gnu" | "uclibc")
        {
            llvm::set_dllimport_storage_class(llfn);
        }
        cx.assume_dso_local(llfn, true);
        llfn
    };
--- a/compiler/rustc_codegen_llvm/src/common.rs
+++ b/compiler/rustc_codegen_llvm/src/common.rs
@ -7,7 +7,8 @@ use rustc_abi::{AddressSpace, HasDataLayout};
 use rustc_ast::Mutability;
 use rustc_codegen_ssa::common::TypeKind;
 use rustc_codegen_ssa::traits::*;
-use rustc_data_structures::stable_hasher::{Hash128, HashStable, StableHasher};
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_hashes::Hash128;
 use rustc_hir::def_id::DefId;
 use rustc_middle::bug;
 use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar};
--- a/compiler/rustc_codegen_llvm/src/consts.rs
+++ b/compiler/rustc_codegen_llvm/src/consts.rs
@ -336,12 +336,7 @@ impl<'ll> CodegenCx<'ll, '_> {
            llvm::set_thread_local_mode(g, self.tls_model);
        }
-        let dso_local = self.should_assume_dso_local(g, true);
+        let dso_local = self.assume_dso_local(g, true);
        if dso_local {
            unsafe {
                llvm::LLVMRustSetDSOLocal(g, true);
            }
        }
        if !def_id.is_local() {
            let needs_dll_storage_attr = self.use_dll_storage_attrs
@ -375,9 +370,7 @@ impl<'ll> CodegenCx<'ll, '_> {
                // is actually present in the current crate. We can find out via the
                // is_codegened_item query.
                if !self.tcx.is_codegened_item(def_id) {
-                    unsafe {
+                    llvm::set_dllimport_storage_class(g);
                        llvm::LLVMSetDLLStorageClass(g, llvm::DLLStorageClass::DllImport);
                    }
                }
            }
        }
@ -387,9 +380,7 @@ impl<'ll> CodegenCx<'ll, '_> {
            && library.kind.is_dllimport()
        {
            // For foreign (native) libs we know the exact storage type to use.
-            unsafe {
+            llvm::set_dllimport_storage_class(g);
                llvm::LLVMSetDLLStorageClass(g, llvm::DLLStorageClass::DllImport);
            }
        }
        self.instances.borrow_mut().insert(instance, g);
@ -460,9 +451,7 @@ impl<'ll> CodegenCx<'ll, '_> {
            set_global_alignment(self, g, alloc.align);
            llvm::set_initializer(g, v);
-            if self.should_assume_dso_local(g, true) {
+            self.assume_dso_local(g, true);
                llvm::LLVMRustSetDSOLocal(g, true);
            }
            // Forward the allocation's mutability (picked by the const interner) to LLVM.
            if alloc.mutability.is_not() {
--- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs
@ -87,7 +87,7 @@ pub(crate) fn get_or_insert_gdb_debug_scripts_section_global<'ll>(
 pub(crate) fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool {
    let omit_gdb_pretty_printer_section =
-        attr::contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section);
+        attr::contains_name(cx.tcx.hir_krate_attrs(), sym::omit_gdb_pretty_printer_section);
    // To ensure the section `__rustc_debug_gdb_scripts_section__` will not create
    // ODR violations at link time, this section will not be emitted for rlibs since
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
@ -11,7 +11,9 @@ use rustc_codegen_ssa::traits::*;
 use rustc_hir::def::{CtorKind, DefKind};
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
 use rustc_middle::bug;
-use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf, TyAndLayout};
+use rustc_middle::ty::layout::{
    HasTypingEnv, LayoutOf, TyAndLayout, WIDE_PTR_ADDR, WIDE_PTR_EXTRA,
 };
 use rustc_middle::ty::{
    self, AdtKind, CoroutineArgsExt, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility,
 };
@ -34,12 +36,12 @@ use crate::common::{AsCCharPtr, CodegenCx};
 use crate::debuginfo::dwarf_const;
 use crate::debuginfo::metadata::type_map::build_type_with_children;
 use crate::debuginfo::utils::{WidePtrKind, wide_pointer_kind};
 use crate::llvm;
 use crate::llvm::debuginfo::{
    DIDescriptor, DIFile, DIFlags, DILexicalBlock, DIScope, DIType, DebugEmissionKind,
    DebugNameTableKind,
 };
 use crate::value::Value;
 use crate::{abi, llvm};
 impl PartialEq for llvm::Metadata {
    fn eq(&self, other: &Self) -> bool {
@ -211,16 +213,16 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>(
                    };
                    let layout = cx.layout_of(layout_type);
-                    let addr_field = layout.field(cx, abi::WIDE_PTR_ADDR);
+                    let addr_field = layout.field(cx, WIDE_PTR_ADDR);
-                    let extra_field = layout.field(cx, abi::WIDE_PTR_EXTRA);
+                    let extra_field = layout.field(cx, WIDE_PTR_EXTRA);
                    let (addr_field_name, extra_field_name) = match wide_pointer_kind {
                        WidePtrKind::Dyn => ("pointer", "vtable"),
                        WidePtrKind::Slice => ("data_ptr", "length"),
                    };
-                    assert_eq!(abi::WIDE_PTR_ADDR, 0);
+                    assert_eq!(WIDE_PTR_ADDR, 0);
-                    assert_eq!(abi::WIDE_PTR_EXTRA, 1);
+                    assert_eq!(WIDE_PTR_EXTRA, 1);
                    // The data pointer type is a regular, thin pointer, regardless of whether this
                    // is a slice or a trait object.
@ -242,7 +244,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>(
                            owner,
                            addr_field_name,
                            (addr_field.size, addr_field.align.abi),
-                            layout.fields.offset(abi::WIDE_PTR_ADDR),
+                            layout.fields.offset(WIDE_PTR_ADDR),
                            DIFlags::FlagZero,
                            data_ptr_type_di_node,
                            None,
@ -252,7 +254,7 @@ fn build_pointer_or_reference_di_node<'ll, 'tcx>(
                            owner,
                            extra_field_name,
                            (extra_field.size, extra_field.align.abi),
-                            layout.fields.offset(abi::WIDE_PTR_EXTRA),
+                            layout.fields.offset(WIDE_PTR_EXTRA),
                            DIFlags::FlagZero,
                            type_di_node(cx, extra_field.ty),
                            None,
--- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
@ -22,6 +22,7 @@ use rustc_session::config::{self, DebugInfo};
 use rustc_span::{
    BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span, StableSourceFileId, Symbol,
 };
 use rustc_target::callconv::FnAbi;
 use rustc_target::spec::DebuginfoKind;
 use smallvec::SmallVec;
 use tracing::debug;
@ -29,7 +30,6 @@ use tracing::debug;
 use self::metadata::{UNKNOWN_COLUMN_NUMBER, UNKNOWN_LINE_NUMBER, file_metadata, type_di_node};
 use self::namespace::mangled_name_of_instance;
 use self::utils::{DIB, create_DIArray, is_node_local_to_unit};
 use crate::abi::FnAbi;
 use crate::builder::Builder;
 use crate::common::{AsCCharPtr, CodegenCx};
 use crate::llvm;
--- a/compiler/rustc_codegen_llvm/src/declare.rs
+++ b/compiler/rustc_codegen_llvm/src/declare.rs
@ -16,10 +16,11 @@ use rustc_codegen_ssa::traits::TypeMembershipCodegenMethods;
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_middle::ty::{Instance, Ty};
 use rustc_sanitizers::{cfi, kcfi};
 use rustc_target::callconv::FnAbi;
 use smallvec::SmallVec;
 use tracing::debug;
-use crate::abi::{FnAbi, FnAbiLlvmExt};
+use crate::abi::FnAbiLlvmExt;
 use crate::common::AsCCharPtr;
 use crate::context::{CodegenCx, SimpleCx};
 use crate::llvm::AttributePlace::Function;
--- a/compiler/rustc_codegen_llvm/src/errors.rs
+++ b/compiler/rustc_codegen_llvm/src/errors.rs
@ -92,9 +92,12 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for ParseTargetMachineConfig<'_> {
 #[derive(Diagnostic)]
 #[diag(codegen_llvm_autodiff_without_lto)]
 #[note]
 pub(crate) struct AutoDiffWithoutLTO;
 #[derive(Diagnostic)]
 #[diag(codegen_llvm_autodiff_without_enable)]
 pub(crate) struct AutoDiffWithoutEnable;
 #[derive(Diagnostic)]
 #[diag(codegen_llvm_lto_disallowed)]
 pub(crate) struct LtoDisallowed;
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@ -1,7 +1,7 @@
 use std::assert_matches::assert_matches;
 use std::cmp::Ordering;
-use rustc_abi::{self as abi, Align, Float, HasDataLayout, Primitive, Size};
+use rustc_abi::{Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size};
 use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
 use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
 use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization};
@ -14,10 +14,11 @@ use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, LayoutOf};
 use rustc_middle::ty::{self, GenericArgsRef, Ty};
 use rustc_middle::{bug, span_bug};
 use rustc_span::{Span, Symbol, sym};
 use rustc_target::callconv::{FnAbi, PassMode};
 use rustc_target::spec::{HasTargetSpec, PanicStrategy};
 use tracing::debug;
-use crate::abi::{ExternAbi, FnAbi, FnAbiLlvmExt, LlvmType, PassMode};
+use crate::abi::{FnAbiLlvmExt, LlvmType};
 use crate::builder::Builder;
 use crate::context::CodegenCx;
 use crate::llvm::{self, Metadata};
@ -126,15 +127,14 @@ fn get_simple_intrinsic<'ll>(
        sym::truncf64 => "llvm.trunc.f64",
        sym::truncf128 => "llvm.trunc.f128",
-        sym::rintf16 => "llvm.rint.f16",
+        // We could use any of `rint`, `nearbyint`, or `roundeven`
-        sym::rintf32 => "llvm.rint.f32",
+        // for this -- they are all identical in semantics when
-        sym::rintf64 => "llvm.rint.f64",
+        // assuming the default FP environment.
-        sym::rintf128 => "llvm.rint.f128",
+        // `rint` is what we used for $forever.
-
+        sym::round_ties_even_f16 => "llvm.rint.f16",
-        sym::nearbyintf16 => "llvm.nearbyint.f16",
+        sym::round_ties_even_f32 => "llvm.rint.f32",
-        sym::nearbyintf32 => "llvm.nearbyint.f32",
+        sym::round_ties_even_f64 => "llvm.rint.f64",
-        sym::nearbyintf64 => "llvm.nearbyint.f64",
+        sym::round_ties_even_f128 => "llvm.rint.f128",
        sym::nearbyintf128 => "llvm.nearbyint.f128",
        sym::roundf16 => "llvm.round.f16",
        sym::roundf32 => "llvm.round.f32",
@ -143,11 +143,6 @@ fn get_simple_intrinsic<'ll>(
        sym::ptr_mask => "llvm.ptrmask",
        sym::roundevenf16 => "llvm.roundeven.f16",
        sym::roundevenf32 => "llvm.roundeven.f32",
        sym::roundevenf64 => "llvm.roundeven.f64",
        sym::roundevenf128 => "llvm.roundeven.f128",
        _ => return None,
    };
    Some(cx.get_intrinsic(llvm_name))
@ -257,7 +252,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
            }
            sym::va_arg => {
                match fn_abi.ret.layout.backend_repr {
-                    abi::BackendRepr::Scalar(scalar) => {
+                    BackendRepr::Scalar(scalar) => {
                        match scalar.primitive() {
                            Primitive::Int(..) => {
                                if self.cx().size_of(ret_ty).bytes() < 4 {
@ -470,12 +465,12 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
            }
            sym::raw_eq => {
-                use abi::BackendRepr::*;
+                use BackendRepr::*;
                let tp_ty = fn_args.type_at(0);
                let layout = self.layout_of(tp_ty).layout;
                let use_integer_compare = match layout.backend_repr() {
                    Scalar(_) | ScalarPair(_, _) => true,
-                    Uninhabited | Vector { .. } => false,
+                    Vector { .. } => false,
                    Memory { .. } => {
                        // For rusty ABIs, small aggregates are actually passed
                        // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),
@ -582,8 +577,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                }
                let llret_ty = if ret_ty.is_simd()
-                    && let abi::BackendRepr::Memory { .. } =
+                    && let BackendRepr::Memory { .. } = self.layout_of(ret_ty).layout.backend_repr
                        self.layout_of(ret_ty).layout.backend_repr
                {
                    let (size, elem_ty) = ret_ty.simd_size_and_type(self.tcx());
                    let elem_ll_ty = match elem_ty.kind() {
--- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs
@ -7,7 +7,7 @@ use super::ffi::{BasicBlock, Metadata, Module, Type, Value};
 use crate::llvm::Bool;
 #[link(name = "llvm-wrapper", kind = "static")]
-extern "C" {
+unsafe extern "C" {
    // Enzyme
    pub(crate) fn LLVMRustHasMetadata(I: &Value, KindID: c_uint) -> bool;
    pub(crate) fn LLVMRustEraseInstUntilInclusive(BB: &BasicBlock, I: &Value);
@ -18,7 +18,7 @@ extern "C" {
    pub(crate) fn LLVMRustVerifyFunction(V: &Value, action: LLVMRustVerifierFailureAction) -> Bool;
 }
-extern "C" {
+unsafe extern "C" {
    // Enzyme
    pub(crate) fn LLVMDumpModule(M: &Module);
    pub(crate) fn LLVMDumpValue(V: &Value);
@ -35,3 +35,97 @@ pub enum LLVMRustVerifierFailureAction {
    LLVMPrintMessageAction = 1,
    LLVMReturnStatusAction = 2,
 }
 #[cfg(llvm_enzyme)]
 pub use self::Enzyme_AD::*;
 #[cfg(llvm_enzyme)]
 pub mod Enzyme_AD {
    use libc::c_void;
    extern "C" {
        pub fn EnzymeSetCLBool(arg1: *mut ::std::os::raw::c_void, arg2: u8);
    }
    extern "C" {
        static mut EnzymePrintPerf: c_void;
        static mut EnzymePrintActivity: c_void;
        static mut EnzymePrintType: c_void;
        static mut EnzymePrint: c_void;
        static mut EnzymeStrictAliasing: c_void;
        static mut looseTypeAnalysis: c_void;
        static mut EnzymeInline: c_void;
        static mut RustTypeRules: c_void;
    }
    pub fn set_print_perf(print: bool) {
        unsafe {
            EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymePrintPerf), print as u8);
        }
    }
    pub fn set_print_activity(print: bool) {
        unsafe {
            EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymePrintActivity), print as u8);
        }
    }
    pub fn set_print_type(print: bool) {
        unsafe {
            EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymePrintType), print as u8);
        }
    }
    pub fn set_print(print: bool) {
        unsafe {
            EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymePrint), print as u8);
        }
    }
    pub fn set_strict_aliasing(strict: bool) {
        unsafe {
            EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymeStrictAliasing), strict as u8);
        }
    }
    pub fn set_loose_types(loose: bool) {
        unsafe {
            EnzymeSetCLBool(std::ptr::addr_of_mut!(looseTypeAnalysis), loose as u8);
        }
    }
    pub fn set_inline(val: bool) {
        unsafe {
            EnzymeSetCLBool(std::ptr::addr_of_mut!(EnzymeInline), val as u8);
        }
    }
    pub fn set_rust_rules(val: bool) {
        unsafe {
            EnzymeSetCLBool(std::ptr::addr_of_mut!(RustTypeRules), val as u8);
        }
    }
 }
 #[cfg(not(llvm_enzyme))]
 pub use self::Fallback_AD::*;
 #[cfg(not(llvm_enzyme))]
 pub mod Fallback_AD {
    #![allow(unused_variables)]
    pub fn set_inline(val: bool) {
        unimplemented!()
    }
    pub fn set_print_perf(print: bool) {
        unimplemented!()
    }
    pub fn set_print_activity(print: bool) {
        unimplemented!()
    }
    pub fn set_print_type(print: bool) {
        unimplemented!()
    }
    pub fn set_print(print: bool) {
        unimplemented!()
    }
    pub fn set_strict_aliasing(strict: bool) {
        unimplemented!()
    }
    pub fn set_loose_types(loose: bool) {
        unimplemented!()
    }
    pub fn set_rust_rules(val: bool) {
        unimplemented!()
    }
 }
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@ -954,6 +954,17 @@ bitflags! {
    }
 }
 // These values **must** match with LLVMGEPNoWrapFlags
 bitflags! {
    #[repr(transparent)]
    #[derive(Default)]
    pub struct GEPNoWrapFlags : c_uint {
        const InBounds = 1 << 0;
        const NUSW = 1 << 1;
        const NUW = 1 << 2;
    }
 }
 unsafe extern "C" {
    pub type ModuleBuffer;
 }
@ -1454,21 +1465,14 @@ unsafe extern "C" {
    pub(crate) fn LLVMBuildStore<'a>(B: &Builder<'a>, Val: &'a Value, Ptr: &'a Value) -> &'a Value;
-    pub(crate) fn LLVMBuildGEP2<'a>(
+    pub(crate) fn LLVMBuildGEPWithNoWrapFlags<'a>(
        B: &Builder<'a>,
        Ty: &'a Type,
        Pointer: &'a Value,
        Indices: *const &'a Value,
        NumIndices: c_uint,
        Name: *const c_char,
    ) -> &'a Value;
    pub(crate) fn LLVMBuildInBoundsGEP2<'a>(
        B: &Builder<'a>,
        Ty: &'a Type,
        Pointer: &'a Value,
        Indices: *const &'a Value,
        NumIndices: c_uint,
        Name: *const c_char,
        Flags: GEPNoWrapFlags,
    ) -> &'a Value;
    // Casts
--- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs
@ -403,3 +403,15 @@ pub(crate) fn add_module_flag_str(
        );
    }
 }
 pub(crate) fn set_dllimport_storage_class<'ll>(v: &'ll Value) {
    unsafe {
        LLVMSetDLLStorageClass(v, DLLStorageClass::DllImport);
    }
 }
 pub(crate) fn set_dso_local<'ll>(v: &'ll Value) {
    unsafe {
        LLVMRustSetDSOLocal(v, true);
    }
 }
--- a/compiler/rustc_codegen_llvm/src/mono_item.rs
+++ b/compiler/rustc_codegen_llvm/src/mono_item.rs
@ -38,11 +38,7 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
        llvm::set_linkage(g, base::linkage_to_llvm(linkage));
        llvm::set_visibility(g, base::visibility_to_llvm(visibility));
-        unsafe {
+        self.assume_dso_local(g, false);
            if self.should_assume_dso_local(g, false) {
                llvm::LLVMRustSetDSOLocal(g, true);
            }
        }
        self.instances.borrow_mut().insert(instance, g);
    }
@ -79,9 +75,7 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
        debug!("predefine_fn: instance = {:?}", instance);
-        if self.should_assume_dso_local(lldecl, false) {
+        self.assume_dso_local(lldecl, false);
            unsafe { llvm::LLVMRustSetDSOLocal(lldecl, true) };
        }
        self.instances.borrow_mut().insert(instance, lldecl);
    }
@ -90,11 +84,16 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
 impl CodegenCx<'_, '_> {
    /// Whether a definition or declaration can be assumed to be local to a group of
    /// libraries that form a single DSO or executable.
-    pub(crate) fn should_assume_dso_local(
+    /// Marks the local as DSO if so.
-        &self,
+    pub(crate) fn assume_dso_local(&self, llval: &llvm::Value, is_declaration: bool) -> bool {
-        llval: &llvm::Value,
+        let assume = self.should_assume_dso_local(llval, is_declaration);
-        is_declaration: bool,
+        if assume {
-    ) -> bool {
+            llvm::set_dso_local(llval);
        }
        assume
    }
    fn should_assume_dso_local(&self, llval: &llvm::Value, is_declaration: bool) -> bool {
        let linkage = llvm::get_linkage(llval);
        let visibility = llvm::get_visibility(llval);
--- a/compiler/rustc_codegen_llvm/src/type_of.rs
+++ b/compiler/rustc_codegen_llvm/src/type_of.rs
@ -23,7 +23,7 @@ fn uncached_llvm_type<'a, 'tcx>(
            let element = layout.scalar_llvm_type_at(cx, element);
            return cx.type_vector(element, count);
        }
-        BackendRepr::Uninhabited | BackendRepr::Memory { .. } | BackendRepr::ScalarPair(..) => {}
+        BackendRepr::Memory { .. } | BackendRepr::ScalarPair(..) => {}
    }
    let name = match layout.ty.kind() {
@ -172,19 +172,16 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
    fn is_llvm_immediate(&self) -> bool {
        match self.backend_repr {
            BackendRepr::Scalar(_) | BackendRepr::Vector { .. } => true,
-            BackendRepr::ScalarPair(..) | BackendRepr::Uninhabited | BackendRepr::Memory { .. } => {
+            BackendRepr::ScalarPair(..) | BackendRepr::Memory { .. } => false,
                false
            }
        }
    }
    fn is_llvm_scalar_pair(&self) -> bool {
        match self.backend_repr {
            BackendRepr::ScalarPair(..) => true,
-            BackendRepr::Uninhabited
+            BackendRepr::Scalar(_) | BackendRepr::Vector { .. } | BackendRepr::Memory { .. } => {
-            | BackendRepr::Scalar(_)
+                false
-            | BackendRepr::Vector { .. }
+            }
            | BackendRepr::Memory { .. } => false,
        }
    }
--- a/compiler/rustc_codegen_ssa/Cargo.toml
+++ b/compiler/rustc_codegen_ssa/Cargo.toml
@ -1,7 +1,7 @@
 [package]
 name = "rustc_codegen_ssa"
 version = "0.0.0"
-edition = "2021"
+edition = "2024"
 [dependencies]
 # tidy-alphabetical-start
@ -25,6 +25,7 @@ rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_fs_util = { path = "../rustc_fs_util" }
 rustc_hashes = { path = "../rustc_hashes" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_hir_pretty = { path = "../rustc_hir_pretty" }
 rustc_incremental = { path = "../rustc_incremental" }
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@ -244,22 +244,17 @@ pub fn each_linked_rlib(
        fmts
    } else {
-        for combination in info.dependency_formats.iter().combinations(2) {
+        let mut dep_formats = info.dependency_formats.iter();
-            let (ty1, list1) = &combination[0];
+        let (ty1, list1) = dep_formats.next().ok_or(errors::LinkRlibError::MissingFormat)?;
-            let (ty2, list2) = &combination[1];
+        if let Some((ty2, list2)) = dep_formats.find(|(_, list2)| list1 != *list2) {
-            if list1 != list2 {
+            return Err(errors::LinkRlibError::IncompatibleDependencyFormats {
-                return Err(errors::LinkRlibError::IncompatibleDependencyFormats {
+                ty1: format!("{ty1:?}"),
-                    ty1: format!("{ty1:?}"),
+                ty2: format!("{ty2:?}"),
-                    ty2: format!("{ty2:?}"),
+                list1: format!("{list1:?}"),
-                    list1: format!("{list1:?}"),
+                list2: format!("{list2:?}"),
-                    list2: format!("{list2:?}"),
+            });
                });
            }
        }
-        if info.dependency_formats.is_empty() {
+        list1
            return Err(errors::LinkRlibError::MissingFormat);
        }
        info.dependency_formats.first().unwrap().1
    };
    let used_dep_crates = info.used_crates.iter();
@ -626,10 +621,9 @@ fn link_staticlib(
    let mut all_rust_dylibs = vec![];
    for &cnum in crates {
-        match fmts.get(cnum) {
+        let Some(Linkage::Dynamic) = fmts.get(cnum) else {
-            Some(&Linkage::Dynamic) => {}
+            continue;
-            _ => continue,
+        };
        }
        let crate_name = codegen_results.crate_info.crate_name[&cnum];
        let used_crate_source = &codegen_results.crate_info.used_crate_source[&cnum];
        if let Some((path, _)) = &used_crate_source.dylib {
@ -1990,6 +1984,7 @@ fn add_pre_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor)
    if let Some(args) = sess.target.pre_link_args.get(&flavor) {
        cmd.verbatim_args(args.iter().map(Deref::deref));
    }
    cmd.verbatim_args(&sess.opts.unstable_opts.pre_link_args);
 }
@ -2518,6 +2513,12 @@ fn add_order_independent_options(
            "--target-cpu",
            &codegen_results.crate_info.target_cpu,
        ]);
        if codegen_results.crate_info.target_features.len() > 0 {
            cmd.link_arg(&format!(
                "--target-feature={}",
                &codegen_results.crate_info.target_features.join(",")
            ));
        }
    } else if flavor == LinkerFlavor::Ptx {
        cmd.link_args(&["--fallback-arch", &codegen_results.crate_info.target_cpu]);
    } else if flavor == LinkerFlavor::Bpf {
--- a/compiler/rustc_codegen_ssa/src/back/linker.rs
+++ b/compiler/rustc_codegen_ssa/src/back/linker.rs
@ -153,6 +153,7 @@ pub(crate) fn get_linker<'a>(
            hinted_static: None,
            is_ld: cc == Cc::No,
            is_gnu: flavor.is_gnu(),
            uses_lld: flavor.uses_lld(),
        }) as Box<dyn Linker>,
        LinkerFlavor::Msvc(..) => Box::new(MsvcLinker { cmd, sess }) as Box<dyn Linker>,
        LinkerFlavor::EmCc => Box::new(EmLinker { cmd, sess }) as Box<dyn Linker>,
@ -361,6 +362,7 @@ struct GccLinker<'a> {
    // Link as ld
    is_ld: bool,
    is_gnu: bool,
    uses_lld: bool,
 }
 impl<'a> GccLinker<'a> {
@ -552,6 +554,7 @@ impl<'a> Linker for GccLinker<'a> {
                self.link_args(&["--entry", "_initialize"]);
            }
        }
        // VxWorks compiler driver introduced `--static-crt` flag specifically for rustc,
        // it switches linking for libc and similar system libraries to static without using
        // any `#[link]` attributes in the `libc` crate, see #72782 for details.
@ -567,6 +570,15 @@ impl<'a> Linker for GccLinker<'a> {
        {
            self.cc_arg("--static-crt");
        }
        // avr-none doesn't have default ISA, users must specify which specific
        // CPU (well, microcontroller) they are targetting using `-Ctarget-cpu`.
        //
        // Currently this makes sense only when using avr-gcc as a linker, since
        // it brings a couple of hand-written important intrinsics from libgcc.
        if self.sess.target.arch == "avr" && !self.uses_lld {
            self.verbatim_arg(format!("-mmcu={}", self.target_cpu));
        }
    }
    fn link_dylib_by_name(&mut self, name: &str, verbatim: bool, as_needed: bool) {
--- a/compiler/rustc_codegen_ssa/src/back/metadata.rs
+++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs
@ -252,15 +252,7 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static
        // Unsupported architecture.
        _ => return None,
    };
-    let binary_format = if sess.target.is_like_osx {
+    let binary_format = sess.target.binary_format.to_object();
        BinaryFormat::MachO
    } else if sess.target.is_like_windows {
        BinaryFormat::Coff
    } else if sess.target.is_like_aix {
        BinaryFormat::Xcoff
    } else {
        BinaryFormat::Elf
    };
    let mut file = write::Object::new(binary_format, architecture, endianness);
    file.set_sub_architecture(sub_architecture);
--- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
+++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
@ -183,11 +183,11 @@ fn exported_symbols_provider_local(
    });
    let mut symbols: Vec<_> =
-        sorted.iter().map(|(&def_id, &info)| (ExportedSymbol::NonGeneric(def_id), info)).collect();
+        sorted.iter().map(|&(&def_id, &info)| (ExportedSymbol::NonGeneric(def_id), info)).collect();
    // Export TLS shims
    if !tcx.sess.target.dll_tls_export {
-        symbols.extend(sorted.iter().filter_map(|(&def_id, &info)| {
+        symbols.extend(sorted.iter().filter_map(|&(&def_id, &info)| {
            tcx.needs_thread_local_shim(def_id).then(|| {
                (
                    ExportedSymbol::ThreadLocalShim(def_id),
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@ -405,7 +405,8 @@ fn generate_lto_work<B: ExtraBackendMethods>(
            B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise());
        if cgcx.lto == Lto::Fat && !autodiff.is_empty() {
            let config = cgcx.config(ModuleKind::Regular);
-            module = unsafe { module.autodiff(cgcx, autodiff, config).unwrap() };
+            module =
                unsafe { module.autodiff(cgcx, autodiff, config).unwrap_or_else(|e| e.raise()) };
        }
        // We are adding a single work item, so the cost doesn't matter.
        vec![(WorkItem::LTO(module), 0)]
@ -572,10 +573,10 @@ fn produce_final_output_artifacts(
    };
    let copy_if_one_unit = |output_type: OutputType, keep_numbered: bool| {
-        if compiled_modules.modules.len() == 1 {
+        if let [module] = &compiled_modules.modules[..] {
            // 1) Only one codegen unit. In this case it's no difficulty
            //    to copy `foo.0.x` to `foo.x`.
-            let module_name = Some(&compiled_modules.modules[0].name[..]);
+            let module_name = Some(&module.name[..]);
            let path = crate_output.temp_path(output_type, module_name);
            let output = crate_output.path(output_type);
            if !output_type.is_text_output() && output.is_tty() {
@ -707,8 +708,8 @@ fn produce_final_output_artifacts(
    }
    if sess.opts.json_artifact_notifications {
-        if compiled_modules.modules.len() == 1 {
+        if let [module] = &compiled_modules.modules[..] {
-            compiled_modules.modules[0].for_each_output(|_path, ty| {
+            module.for_each_output(|_path, ty| {
                if sess.opts.output_types.contains_key(&ty) {
                    let descr = ty.shorthand();
                    // for single cgu file is renamed to drop cgu specific suffix
@ -864,7 +865,7 @@ pub(crate) fn compute_per_cgu_lto_type(
    // require LTO so the request for LTO is always unconditionally
    // passed down to the backend, but we don't actually want to do
    // anything about it yet until we've got a final product.
-    let is_rlib = sess_crate_types.len() == 1 && sess_crate_types[0] == CrateType::Rlib;
+    let is_rlib = matches!(sess_crate_types, [CrateType::Rlib]);
    match sess_lto {
        Lto::ThinLocal if !linker_does_lto && !is_allocator => ComputedLtoType::Thin,
@ -1537,8 +1538,9 @@ fn start_executing_work<B: ExtraBackendMethods>(
            // Spin up what work we can, only doing this while we've got available
            // parallelism slots and work left to spawn.
            if codegen_state != Aborted {
-                while !work_items.is_empty() && running_with_own_token < tokens.len() {
+                while running_with_own_token < tokens.len()
-                    let (item, _) = work_items.pop().unwrap();
+                    && let Some((item, _)) = work_items.pop()
                {
                    spawn_work(
                        &cgcx,
                        &mut llvm_start_time,
--- a/compiler/rustc_codegen_ssa/src/base.rs
+++ b/compiler/rustc_codegen_ssa/src/base.rs
@ -24,7 +24,7 @@ use rustc_middle::query::Providers;
 use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
 use rustc_session::Session;
-use rustc_session::config::{self, CrateType, EntryFnType, OptLevel, OutputType};
+use rustc_session::config::{self, CrateType, EntryFnType, OutputType};
 use rustc_span::{DUMMY_SP, Symbol, sym};
 use rustc_trait_selection::infer::{BoundRegionConversionTime, TyCtxtInferExt};
 use rustc_trait_selection::traits::{ObligationCause, ObligationCtxt};
@ -364,13 +364,7 @@ pub(crate) fn build_shift_expr_rhs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
    let rhs_sz = bx.cx().int_width(rhs_llty);
    let lhs_sz = bx.cx().int_width(lhs_llty);
    if lhs_sz < rhs_sz {
-        if is_unchecked && bx.sess().opts.optimize != OptLevel::No {
+        if is_unchecked { bx.unchecked_utrunc(rhs, lhs_llty) } else { bx.trunc(rhs, lhs_llty) }
            // FIXME: Use `trunc nuw` once that's available
            let inrange = bx.icmp(IntPredicate::IntULE, rhs, mask);
            bx.assume(inrange);
        }
        bx.trunc(rhs, lhs_llty)
    } else if lhs_sz > rhs_sz {
        // We zero-extend even if the RHS is signed. So e.g. `(x: i32) << -1i8` will zero-extend the
        // RHS to `255i32`. But then we mask the shift amount to be within the size of the LHS
@ -921,6 +915,7 @@ impl CrateInfo {
        let n_crates = crates.len();
        let mut info = CrateInfo {
            target_cpu,
            target_features: tcx.global_backend_features(()).clone(),
            crate_types,
            exported_symbols,
            linked_symbols,
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@ -1,7 +1,6 @@
 use std::str::FromStr;
 use rustc_abi::ExternAbi;
 use rustc_ast::attr::list_contains_name;
 use rustc_ast::expand::autodiff_attrs::{
    AutoDiffAttrs, DiffActivity, DiffMode, valid_input_activity, valid_ret_activity,
 };
@ -377,24 +376,20 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                            let segments =
                                set.path.segments.iter().map(|x| x.ident.name).collect::<Vec<_>>();
                            match segments.as_slice() {
-                                [sym::arm, sym::a32] | [sym::arm, sym::t32] => {
+                                [sym::arm, sym::a32 | sym::t32]
-                                    if !tcx.sess.target.has_thumb_interworking {
+                                    if !tcx.sess.target.has_thumb_interworking =>
-                                        struct_span_code_err!(
+                                {
-                                            tcx.dcx(),
+                                    struct_span_code_err!(
-                                            attr.span,
+                                        tcx.dcx(),
-                                            E0779,
+                                        attr.span,
-                                            "target does not support `#[instruction_set]`"
+                                        E0779,
-                                        )
+                                        "target does not support `#[instruction_set]`"
-                                        .emit();
+                                    )
-                                        None
+                                    .emit();
-                                    } else if segments[1] == sym::a32 {
+                                    None
                                        Some(InstructionSetAttr::ArmA32)
                                    } else if segments[1] == sym::t32 {
                                        Some(InstructionSetAttr::ArmT32)
                                    } else {
                                        unreachable!()
                                    }
                                }
                                [sym::arm, sym::a32] => Some(InstructionSetAttr::ArmA32),
                                [sym::arm, sym::t32] => Some(InstructionSetAttr::ArmT32),
                                _ => {
                                    struct_span_code_err!(
                                        tcx.dcx(),
@ -435,7 +430,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                    && let Some((sym::align, literal)) = item.singleton_lit_list()
                {
                    rustc_attr_parsing::parse_alignment(&literal.kind)
-                        .map_err(|msg| {
+                        .inspect_err(|msg| {
                            struct_span_code_err!(
                                tcx.dcx(),
                                literal.span,
@ -536,25 +531,27 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
        }
        if attr.is_word() {
-            InlineAttr::Hint
+            return InlineAttr::Hint;
-        } else if let Some(ref items) = attr.meta_item_list() {
+        }
-            inline_span = Some(attr.span);
+        let Some(ref items) = attr.meta_item_list() else {
-            if items.len() != 1 {
+            return ia;
-                struct_span_code_err!(tcx.dcx(), attr.span, E0534, "expected one argument").emit();
+        };
                InlineAttr::None
            } else if list_contains_name(items, sym::always) {
                InlineAttr::Always
            } else if list_contains_name(items, sym::never) {
                InlineAttr::Never
            } else {
                struct_span_code_err!(tcx.dcx(), items[0].span(), E0535, "invalid argument")
                    .with_help("valid inline arguments are `always` and `never`")
                    .emit();
-                InlineAttr::None
+        inline_span = Some(attr.span);
-            }
+        let [item] = &items[..] else {
            struct_span_code_err!(tcx.dcx(), attr.span, E0534, "expected one argument").emit();
            return InlineAttr::None;
        };
        if item.has_name(sym::always) {
            InlineAttr::Always
        } else if item.has_name(sym::never) {
            InlineAttr::Never
        } else {
-            ia
+            struct_span_code_err!(tcx.dcx(), item.span(), E0535, "invalid argument")
                .with_help("valid inline arguments are `always` and `never`")
                .emit();
            InlineAttr::None
        }
    });
    codegen_fn_attrs.inline = attrs.iter().fold(codegen_fn_attrs.inline, |ia, attr| {
@ -586,23 +583,25 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
        let err = |sp, s| struct_span_code_err!(tcx.dcx(), sp, E0722, "{}", s).emit();
        if attr.is_word() {
            err(attr.span, "expected one argument");
-            ia
+            return ia;
-        } else if let Some(ref items) = attr.meta_item_list() {
+        }
-            inline_span = Some(attr.span);
+        let Some(ref items) = attr.meta_item_list() else {
-            if items.len() != 1 {
+            return OptimizeAttr::Default;
-                err(attr.span, "expected one argument");
+        };
-                OptimizeAttr::Default
+
-            } else if list_contains_name(items, sym::size) {
+        inline_span = Some(attr.span);
-                OptimizeAttr::Size
+        let [item] = &items[..] else {
-            } else if list_contains_name(items, sym::speed) {
+            err(attr.span, "expected one argument");
-                OptimizeAttr::Speed
+            return OptimizeAttr::Default;
-            } else if list_contains_name(items, sym::none) {
+        };
-                OptimizeAttr::DoNotOptimize
+        if item.has_name(sym::size) {
-            } else {
+            OptimizeAttr::Size
-                err(items[0].span(), "invalid argument");
+        } else if item.has_name(sym::speed) {
-                OptimizeAttr::Default
+            OptimizeAttr::Speed
-            }
+        } else if item.has_name(sym::none) {
            OptimizeAttr::DoNotOptimize
        } else {
            err(item.span(), "invalid argument");
            OptimizeAttr::Default
        }
    });
@ -644,25 +643,20 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
    // llvm/llvm-project#70563).
    if !codegen_fn_attrs.target_features.is_empty()
        && matches!(codegen_fn_attrs.inline, InlineAttr::Always)
        && let Some(span) = inline_span
    {
-        if let Some(span) = inline_span {
+        tcx.dcx().span_err(span, "cannot use `#[inline(always)]` with `#[target_feature]`");
            tcx.dcx().span_err(span, "cannot use `#[inline(always)]` with `#[target_feature]`");
        }
    }
-    if !codegen_fn_attrs.no_sanitize.is_empty() && codegen_fn_attrs.inline.always() {
+    if !codegen_fn_attrs.no_sanitize.is_empty()
-        if let (Some(no_sanitize_span), Some(inline_span)) = (no_sanitize_span, inline_span) {
+        && codegen_fn_attrs.inline.always()
-            let hir_id = tcx.local_def_id_to_hir_id(did);
+        && let (Some(no_sanitize_span), Some(inline_span)) = (no_sanitize_span, inline_span)
-            tcx.node_span_lint(
+    {
-                lint::builtin::INLINE_NO_SANITIZE,
+        let hir_id = tcx.local_def_id_to_hir_id(did);
-                hir_id,
+        tcx.node_span_lint(lint::builtin::INLINE_NO_SANITIZE, hir_id, no_sanitize_span, |lint| {
-                no_sanitize_span,
+            lint.primary_message("`no_sanitize` will have no effect after inlining");
-                |lint| {
+            lint.span_note(inline_span, "inlining requested here");
-                    lint.primary_message("`no_sanitize` will have no effect after inlining");
+        })
                    lint.span_note(inline_span, "inlining requested here");
                },
            )
        }
    }
    // Weak lang items have the same semantics as "std internal" symbols in the
@ -692,10 +686,10 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
    // Any linkage to LLVM intrinsics for now forcibly marks them all as never
    // unwinds since LLVM sometimes can't handle codegen which `invoke`s
    // intrinsic functions.
-    if let Some(name) = &codegen_fn_attrs.link_name {
+    if let Some(name) = &codegen_fn_attrs.link_name
-        if name.as_str().starts_with("llvm.") {
+        && name.as_str().starts_with("llvm.")
-            codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND;
+    {
-        }
+        codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND;
    }
    if let Some(features) = check_tied_features(
@ -756,18 +750,13 @@ fn should_inherit_track_caller(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
 fn check_link_ordinal(tcx: TyCtxt<'_>, attr: &hir::Attribute) -> Option<u16> {
    use rustc_ast::{LitIntType, LitKind, MetaItemLit};
-    let meta_item_list = attr.meta_item_list();
+    let meta_item_list = attr.meta_item_list()?;
-    let meta_item_list = meta_item_list.as_deref();
+    let [sole_meta_list] = &meta_item_list[..] else {
-    let sole_meta_list = match meta_item_list {
+        tcx.dcx().emit_err(errors::InvalidLinkOrdinalNargs { span: attr.span });
-        Some([item]) => item.lit(),
+        return None;
        Some(_) => {
            tcx.dcx().emit_err(errors::InvalidLinkOrdinalNargs { span: attr.span });
            return None;
        }
        _ => None,
    };
    if let Some(MetaItemLit { kind: LitKind::Int(ordinal, LitIntType::Unsuffixed), .. }) =
-        sole_meta_list
+        sole_meta_list.lit()
    {
        // According to the table at
        // https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#import-header, the
@ -894,7 +883,7 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> {
    let [mode, input_activities @ .., ret_activity] = &list[..] else {
        span_bug!(attr.span, "rustc_autodiff attribute must contain mode and activities");
    };
-    let mode = if let MetaItemInner::MetaItem(MetaItem { path: ref p1, .. }) = mode {
+    let mode = if let MetaItemInner::MetaItem(MetaItem { path: p1, .. }) = mode {
        p1.segments.first().unwrap().ident
    } else {
        span_bug!(attr.span, "rustc_autodiff attribute must contain mode");
@ -910,7 +899,7 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> {
    };
    // First read the ret symbol from the attribute
-    let ret_symbol = if let MetaItemInner::MetaItem(MetaItem { path: ref p1, .. }) = ret_activity {
+    let ret_symbol = if let MetaItemInner::MetaItem(MetaItem { path: p1, .. }) = ret_activity {
        p1.segments.first().unwrap().ident
    } else {
        span_bug!(attr.span, "rustc_autodiff attribute must contain the return activity");
@ -924,7 +913,7 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> {
    // Now parse all the intermediate (input) activities
    let mut arg_activities: Vec<DiffActivity> = vec![];
    for arg in input_activities {
-        let arg_symbol = if let MetaItemInner::MetaItem(MetaItem { path: ref p2, .. }) = arg {
+        let arg_symbol = if let MetaItemInner::MetaItem(MetaItem { path: p2, .. }) = arg {
            match p2.segments.first() {
                Some(x) => x.ident,
                None => {
--- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
+++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
@ -15,7 +15,8 @@ use std::fmt::Write;
 use rustc_abi::Integer;
 use rustc_data_structures::fx::FxHashSet;
-use rustc_data_structures::stable_hasher::{Hash64, HashStable, StableHasher};
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_hashes::Hash64;
 use rustc_hir::def_id::DefId;
 use rustc_hir::definitions::{DefPathData, DefPathDataName, DisambiguatedDefPathData};
 use rustc_hir::{CoroutineDesugaring, CoroutineKind, CoroutineSource, Mutability};
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@ -190,6 +190,7 @@ impl From<&cstore::NativeLib> for NativeLib {
 #[derive(Debug, Encodable, Decodable)]
 pub struct CrateInfo {
    pub target_cpu: String,
    pub target_features: Vec<String>,
    pub crate_types: Vec<CrateType>,
    pub exported_symbols: UnordMap<CrateType, Vec<String>>,
    pub linked_symbols: FxIndexMap<CrateType, Vec<(String, SymbolExportKind)>>,
@ -230,6 +231,7 @@ pub fn provide(providers: &mut Providers) {
    crate::base::provide(providers);
    crate::target_features::provide(providers);
    crate::codegen_attrs::provide(providers);
    providers.queries.global_backend_features = |_tcx: TyCtxt<'_>, ()| vec![];
 }
 /// Checks if the given filename ends with the `.rcgu.o` extension that `rustc`
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@ -4,9 +4,7 @@ use rustc_abi::{BackendRepr, ExternAbi, HasDataLayout, Reg, WrappingRange};
 use rustc_ast as ast;
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_hir::lang_items::LangItem;
-use rustc_middle::mir::{
+use rustc_middle::mir::{self, AssertKind, InlineAsmMacro, SwitchTargets, UnwindTerminateReason};
    self, AssertKind, BasicBlock, InlineAsmMacro, SwitchTargets, UnwindTerminateReason,
 };
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement};
 use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 use rustc_middle::ty::{self, Instance, Ty};
@ -429,11 +427,34 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
            let cmp = bx.icmp(IntPredicate::IntEQ, discr_value, llval);
            bx.cond_br(cmp, ll1, ll2);
        } else {
-            bx.switch(
+            let otherwise = targets.otherwise();
-                discr_value,
+            let otherwise_cold = self.cold_blocks[otherwise];
-                helper.llbb_with_cleanup(self, targets.otherwise()),
+            let otherwise_unreachable = self.mir[otherwise].is_empty_unreachable();
-                target_iter.map(|(value, target)| (value, helper.llbb_with_cleanup(self, target))),
+            let cold_count = targets.iter().filter(|(_, target)| self.cold_blocks[*target]).count();
-            );
+            let none_cold = cold_count == 0;
            let all_cold = cold_count == targets.iter().len();
            if (none_cold && (!otherwise_cold || otherwise_unreachable))
                || (all_cold && (otherwise_cold || otherwise_unreachable))
            {
                // All targets have the same weight,
                // or `otherwise` is unreachable and it's the only target with a different weight.
                bx.switch(
                    discr_value,
                    helper.llbb_with_cleanup(self, targets.otherwise()),
                    target_iter
                        .map(|(value, target)| (value, helper.llbb_with_cleanup(self, target))),
                );
            } else {
                // Targets have different weights
                bx.switch_with_weights(
                    discr_value,
                    helper.llbb_with_cleanup(self, targets.otherwise()),
                    otherwise_cold,
                    target_iter.map(|(value, target)| {
                        (value, helper.llbb_with_cleanup(self, target), self.cold_blocks[target])
                    }),
                );
            }
        }
    }
@ -699,14 +720,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
        // Put together the arguments to the panic entry point.
        let (lang_item, args) = match msg {
-            AssertKind::BoundsCheck { ref len, ref index } => {
+            AssertKind::BoundsCheck { len, index } => {
                let len = self.codegen_operand(bx, len).immediate();
                let index = self.codegen_operand(bx, index).immediate();
                // It's `fn panic_bounds_check(index: usize, len: usize)`,
                // and `#[track_caller]` adds an implicit third argument.
                (LangItem::PanicBoundsCheck, vec![index, len, location])
            }
-            AssertKind::MisalignedPointerDereference { ref required, ref found } => {
+            AssertKind::MisalignedPointerDereference { required, found } => {
                let required = self.codegen_operand(bx, required).immediate();
                let found = self.codegen_operand(bx, found).immediate();
                // It's `fn panic_misaligned_pointer_dereference(required: usize, found: usize)`,
@ -919,7 +940,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                    &fn_abi.ret,
                    &mut llargs,
                    Some(intrinsic),
                    target,
                );
                let dest = match ret_dest {
                    _ if fn_abi.ret.is_indirect() => llargs[0],
@ -975,23 +995,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
        };
        let mut llargs = Vec::with_capacity(arg_count);
-        let destination = target.as_ref().map(|&target| {
+
-            (
+        // We still need to call `make_return_dest` even if there's no `target`, since
-                self.make_return_dest(
+        // `fn_abi.ret` could be `PassMode::Indirect`, even if it is uninhabited,
-                    bx,
+        // and `make_return_dest` adds the return-place indirect pointer to `llargs`.
-                    destination,
+        let return_dest = self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs, None);
-                    &fn_abi.ret,
+        let destination = target.map(|target| (return_dest, target));
                    &mut llargs,
                    None,
                    Some(target),
                ),
                target,
            )
        });
        // Split the rust-call tupled arguments off.
-        let (first_args, untuple) = if abi == ExternAbi::RustCall && !args.is_empty() {
+        let (first_args, untuple) = if abi == ExternAbi::RustCall
-            let (tup, args) = args.split_last().unwrap();
+            && let Some((tup, args)) = args.split_last()
        {
            (args, Some(tup))
        } else {
            (args, None)
@ -1017,7 +1031,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                            let (idx, _) = op.layout.non_1zst_field(bx).expect(
                                "not exactly one non-1-ZST field in a `DispatchFromDyn` type",
                            );
-                            op = op.extract_field(bx, idx);
+                            op = op.extract_field(self, bx, idx);
                        }
                        // Now that we have `*dyn Trait` or `&dyn Trait`, split it up into its
@ -1049,7 +1063,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                            let (idx, _) = op.layout.non_1zst_field(bx).expect(
                                "not exactly one non-1-ZST field in a `DispatchFromDyn` type",
                            );
-                            op = op.extract_field(bx, idx);
+                            op = op.extract_field(self, bx, idx);
                        }
                        // Make sure that we've actually unwrapped the rcvr down
@ -1549,9 +1563,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                    if scalar.is_bool() {
                        bx.range_metadata(llval, WrappingRange { start: 0, end: 1 });
                    }
                    // We store bools as `i8` so we need to truncate to `i1`.
                    llval = bx.to_immediate_scalar(llval, scalar);
                }
                // We store bools as `i8` so we need to truncate to `i1`.
                llval = bx.to_immediate(llval, arg.layout);
            }
        }
@ -1581,7 +1595,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
        } else {
            // If the tuple is immediate, the elements are as well.
            for i in 0..tuple.layout.fields.count() {
-                let op = tuple.extract_field(bx, i);
+                let op = tuple.extract_field(self, bx, i);
                self.codegen_argument(bx, op, llargs, &args[i]);
            }
        }
@ -1790,11 +1804,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
        fn_ret: &ArgAbi<'tcx, Ty<'tcx>>,
        llargs: &mut Vec<Bx::Value>,
        intrinsic: Option<ty::IntrinsicDef>,
        target: Option<BasicBlock>,
    ) -> ReturnDest<'tcx, Bx::Value> {
        if target.is_none() {
            return ReturnDest::Nothing;
        }
        // If the return is ignored, we can just return a do-nothing `ReturnDest`.
        if fn_ret.is_ignore() {
            return ReturnDest::Nothing;
--- a/compiler/rustc_codegen_ssa/src/mir/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs
@ -502,14 +502,25 @@ fn find_cold_blocks<'tcx>(
    for (bb, bb_data) in traversal::postorder(mir) {
        let terminator = bb_data.terminator();
-        // If a BB ends with a call to a cold function, mark it as cold.
+        match terminator.kind {
-        if let mir::TerminatorKind::Call { ref func, .. } = terminator.kind
+            // If a BB ends with a call to a cold function, mark it as cold.
-            && let ty::FnDef(def_id, ..) = *func.ty(local_decls, tcx).kind()
+            mir::TerminatorKind::Call { ref func, .. }
-            && let attrs = tcx.codegen_fn_attrs(def_id)
+            | mir::TerminatorKind::TailCall { ref func, .. }
-            && attrs.flags.contains(CodegenFnAttrFlags::COLD)
+                if let ty::FnDef(def_id, ..) = *func.ty(local_decls, tcx).kind()
-        {
+                    && let attrs = tcx.codegen_fn_attrs(def_id)
-            cold_blocks[bb] = true;
+                    && attrs.flags.contains(CodegenFnAttrFlags::COLD) =>
-            continue;
+            {
                cold_blocks[bb] = true;
                continue;
            }
            // If a BB ends with an `unreachable`, also mark it as cold.
            mir::TerminatorKind::Unreachable => {
                cold_blocks[bb] = true;
                continue;
            }
            _ => {}
        }
        // If all successors of a BB are cold and there's at least one of them, mark this BB as cold
--- a/Show More
+++ b/Show More