From 0444056aa3ddd97e02478b7c41d130db74583d16 Mon Sep 17 00:00:00 2001
From: beetrees <b@beetr.ee>
Date: Thu, 25 Jul 2024 22:15:47 +0100
Subject: [PATCH 01/16] Remove macOS 10.10 dynamic linker bug workaround

---
 compiler/rustc_codegen_llvm/src/consts.rs     | 52 -------------------
 library/std/src/fs.rs                         | 13 +++--
 .../issues/issue-44056-macos-tls-align.rs     | 27 ----------
 3 files changed, 6 insertions(+), 86 deletions(-)
 delete mode 100644 tests/codegen/issues/issue-44056-macos-tls-align.rs

diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs
index 80aa2018c81..0e50e32913c 100644
--- a/compiler/rustc_codegen_llvm/src/consts.rs
+++ b/compiler/rustc_codegen_llvm/src/consts.rs
@@ -440,58 +440,6 @@ impl<'ll> CodegenCx<'ll, '_> {
 
             if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) {
                 llvm::set_thread_local_mode(g, self.tls_model);
-
-                // Do not allow LLVM to change the alignment of a TLS on macOS.
-                //
-                // By default a global's alignment can be freely increased.
-                // This allows LLVM to generate more performant instructions
-                // e.g., using load-aligned into a SIMD register.
-                //
-                // However, on macOS 10.10 or below, the dynamic linker does not
-                // respect any alignment given on the TLS (radar 24221680).
-                // This will violate the alignment assumption, and causing segfault at runtime.
-                //
-                // This bug is very easy to trigger. In `println!` and `panic!`,
-                // the `LOCAL_STDOUT`/`LOCAL_STDERR` handles are stored in a TLS,
-                // which the values would be `mem::replace`d on initialization.
-                // The implementation of `mem::replace` will use SIMD
-                // whenever the size is 32 bytes or higher. LLVM notices SIMD is used
-                // and tries to align `LOCAL_STDOUT`/`LOCAL_STDERR` to a 32-byte boundary,
-                // which macOS's dyld disregarded and causing crashes
-                // (see issues #51794, #51758, #50867, #48866 and #44056).
-                //
-                // To workaround the bug, we trick LLVM into not increasing
-                // the global's alignment by explicitly assigning a section to it
-                // (equivalent to automatically generating a `#[link_section]` attribute).
-                // See the comment in the `GlobalValue::canIncreaseAlignment()` function
-                // of `lib/IR/Globals.cpp` for why this works.
-                //
-                // When the alignment is not increased, the optimized `mem::replace`
-                // will use load-unaligned instructions instead, and thus avoiding the crash.
-                //
-                // We could remove this hack whenever we decide to drop macOS 10.10 support.
-                if self.tcx.sess.target.is_like_osx {
-                    // The `inspect` method is okay here because we checked for provenance, and
-                    // because we are doing this access to inspect the final interpreter state
-                    // (not as part of the interpreter execution).
-                    //
-                    // FIXME: This check requires that the (arbitrary) value of undefined bytes
-                    // happens to be zero. Instead, we should only check the value of defined bytes
-                    // and set all undefined bytes to zero if this allocation is headed for the
-                    // BSS.
-                    let all_bytes_are_zero = alloc.provenance().ptrs().is_empty()
-                        && alloc
-                            .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len())
-                            .iter()
-                            .all(|&byte| byte == 0);
-
-                    let sect_name = if all_bytes_are_zero {
-                        c"__DATA,__thread_bss"
-                    } else {
-                        c"__DATA,__thread_data"
-                    };
-                    llvm::LLVMSetSection(g, sect_name.as_ptr());
-                }
             }
 
             // Wasm statics with custom link sections get special treatment as they
diff --git a/library/std/src/fs.rs b/library/std/src/fs.rs
index 536d0d1b356..25540410cc7 100644
--- a/library/std/src/fs.rs
+++ b/library/std/src/fs.rs
@@ -2471,16 +2471,15 @@ pub fn remove_dir<P: AsRef<Path>>(path: P) -> io::Result<()> {
 /// # Platform-specific behavior
 ///
 /// This function currently corresponds to `openat`, `fdopendir`, `unlinkat` and `lstat` functions
-/// on Unix (except for macOS before version 10.10 and REDOX) and the `CreateFileW`,
-/// `GetFileInformationByHandleEx`, `SetFileInformationByHandle`, and `NtCreateFile` functions on
-/// Windows. Note that, this [may change in the future][changes].
+/// on Unix (except for REDOX) and the `CreateFileW`, `GetFileInformationByHandleEx`,
+/// `SetFileInformationByHandle`, and `NtCreateFile` functions on Windows. Note that, this
+/// [may change in the future][changes].
 ///
 /// [changes]: io#platform-specific-behavior
 ///
-/// On macOS before version 10.10 and REDOX, as well as when running in Miri for any target, this
-/// function is not protected against time-of-check to time-of-use (TOCTOU) race conditions, and
-/// should not be used in security-sensitive code on those platforms. All other platforms are
-/// protected.
+/// On REDOX, as well as when running in Miri for any target, this function is not protected against
+/// time-of-check to time-of-use (TOCTOU) race conditions, and should not be used in
+/// security-sensitive code on those platforms. All other platforms are protected.
 ///
 /// # Errors
 ///
diff --git a/tests/codegen/issues/issue-44056-macos-tls-align.rs b/tests/codegen/issues/issue-44056-macos-tls-align.rs
deleted file mode 100644
index 972b8490d18..00000000000
--- a/tests/codegen/issues/issue-44056-macos-tls-align.rs
+++ /dev/null
@@ -1,27 +0,0 @@
-//
-//@ only-apple
-//@ compile-flags: -O
-
-#![crate_type = "rlib"]
-#![feature(thread_local)]
-
-// local_unnamed_addr does not appear when std is built with debug assertions.
-// CHECK: @STATIC_VAR_1 = thread_local {{(local_unnamed_addr )?}}global <{ [32 x i8] }> zeroinitializer, section "__DATA,__thread_bss", align 4
-#[no_mangle]
-#[thread_local]
-static mut STATIC_VAR_1: [u32; 8] = [0; 8];
-
-// CHECK: @STATIC_VAR_2 = thread_local {{(local_unnamed_addr )?}}global <{ [32 x i8] }> <{{[^>]*}}>, section "__DATA,__thread_data", align 4
-#[no_mangle]
-#[thread_local]
-static mut STATIC_VAR_2: [u32; 8] = [4; 8];
-
-#[no_mangle]
-pub unsafe fn f(x: &mut [u32; 8]) {
-    std::mem::swap(x, &mut STATIC_VAR_1)
-}
-
-#[no_mangle]
-pub unsafe fn g(x: &mut [u32; 8]) {
-    std::mem::swap(x, &mut STATIC_VAR_2)
-}

From 7b7992fbcf5b8d9af945e32eac241252baa46fab Mon Sep 17 00:00:00 2001
From: Eric Holk <ericholk@microsoft.com>
Date: Wed, 4 Sep 2024 16:03:47 -0700
Subject: [PATCH 02/16] Begin experimental support for pin reborrowing

This commit adds basic support for reborrowing `Pin` types in argument
position. At the moment it only supports reborrowing `Pin<&mut T>` as
`Pin<&mut T>` by inserting a call to `Pin::as_mut()`, and only in
argument position (not as the receiver in a method call).
---
 compiler/rustc_borrowck/messages.ftl          |  2 +-
 .../rustc_borrowck/src/session_diagnostics.rs |  2 +-
 compiler/rustc_feature/src/unstable.rs        |  2 +
 compiler/rustc_hir/src/lang_items.rs          |  1 +
 compiler/rustc_hir_typeck/src/coercion.rs     | 59 +++++++++++++++++++
 .../rustc_hir_typeck/src/expr_use_visitor.rs  | 15 +++++
 compiler/rustc_middle/src/ty/adjustment.rs    |  3 +
 compiler/rustc_mir_build/src/thir/cx/expr.rs  | 45 ++++++++++++++
 compiler/rustc_span/src/symbol.rs             |  2 +
 library/core/src/pin.rs                       |  1 +
 tests/ui/async-await/pin-reborrow-arg.rs      | 27 +++++++++
 tests/ui/async-await/pin-reborrow-self.rs     | 24 ++++++++
 .../feature-gate-pin_ergonomics.rs            | 15 +++++
 .../feature-gate-pin_ergonomics.stderr        | 21 +++++++
 14 files changed, 217 insertions(+), 2 deletions(-)
 create mode 100644 tests/ui/async-await/pin-reborrow-arg.rs
 create mode 100644 tests/ui/async-await/pin-reborrow-self.rs
 create mode 100644 tests/ui/feature-gates/feature-gate-pin_ergonomics.rs
 create mode 100644 tests/ui/feature-gates/feature-gate-pin_ergonomics.stderr

diff --git a/compiler/rustc_borrowck/messages.ftl b/compiler/rustc_borrowck/messages.ftl
index edb25e12864..ee4b2f95cb1 100644
--- a/compiler/rustc_borrowck/messages.ftl
+++ b/compiler/rustc_borrowck/messages.ftl
@@ -207,7 +207,7 @@ borrowck_simd_intrinsic_arg_const =
         *[other] {$arg}th
     } argument of `{$intrinsic}` is required to be a `const` item
 
-borrowck_suggest_create_freash_reborrow =
+borrowck_suggest_create_fresh_reborrow =
     consider reborrowing the `Pin` instead of moving it
 
 borrowck_suggest_iterate_over_slice =
diff --git a/compiler/rustc_borrowck/src/session_diagnostics.rs b/compiler/rustc_borrowck/src/session_diagnostics.rs
index 4a50b0f0704..b6c6960d4ca 100644
--- a/compiler/rustc_borrowck/src/session_diagnostics.rs
+++ b/compiler/rustc_borrowck/src/session_diagnostics.rs
@@ -415,7 +415,7 @@ pub(crate) enum CaptureReasonSuggest<'tcx> {
         span: Span,
     },
     #[suggestion(
-        borrowck_suggest_create_freash_reborrow,
+        borrowck_suggest_create_fresh_reborrow,
         applicability = "maybe-incorrect",
         code = ".as_mut()",
         style = "verbose"
diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs
index d0c0460ddfe..cf20b79dfa9 100644
--- a/compiler/rustc_feature/src/unstable.rs
+++ b/compiler/rustc_feature/src/unstable.rs
@@ -558,6 +558,8 @@ declare_features! (
     (unstable, optimize_attribute, "1.34.0", Some(54882)),
     /// Allows specifying nop padding on functions for dynamic patching.
     (unstable, patchable_function_entry, "1.81.0", Some(123115)),
+    /// Experimental features that make `Pin` more ergonomic.
+    (incomplete, pin_ergonomics, "CURRENT_RUSTC_VERSION", Some(130494)),
     /// Allows postfix match `expr.match { ... }`
     (unstable, postfix_match, "1.79.0", Some(121618)),
     /// Allows macro attributes on expressions, statements and non-inline modules.
diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs
index e7398fd2226..60e024068e9 100644
--- a/compiler/rustc_hir/src/lang_items.rs
+++ b/compiler/rustc_hir/src/lang_items.rs
@@ -395,6 +395,7 @@ language_item_table! {
     IteratorNext,            sym::next,                next_fn,                    Target::Method(MethodKind::Trait { body: false}), GenericRequirement::None;
 
     PinNewUnchecked,         sym::new_unchecked,       new_unchecked_fn,           Target::Method(MethodKind::Inherent), GenericRequirement::None;
+    PinAsMut,                sym::pin_as_mut,              as_mut_fn,                  Target::Method(MethodKind::Inherent), GenericRequirement::None;
 
     RangeFrom,               sym::RangeFrom,           range_from_struct,          Target::Struct,         GenericRequirement::None;
     RangeFull,               sym::RangeFull,           range_full_struct,          Target::Struct,         GenericRequirement::None;
diff --git a/compiler/rustc_hir_typeck/src/coercion.rs b/compiler/rustc_hir_typeck/src/coercion.rs
index 3bada1de148..3da4b2893d9 100644
--- a/compiler/rustc_hir_typeck/src/coercion.rs
+++ b/compiler/rustc_hir_typeck/src/coercion.rs
@@ -214,6 +214,12 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
             ty::Dynamic(predicates, region, ty::DynStar) if self.tcx.features().dyn_star => {
                 return self.coerce_dyn_star(a, b, predicates, region);
             }
+            ty::Adt(pin, _)
+                if self.tcx.features().pin_ergonomics
+                    && pin.did() == self.tcx.lang_items().pin_type().unwrap() =>
+            {
+                return self.coerce_pin(a, b);
+            }
             _ => {}
         }
 
@@ -774,6 +780,59 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
         })
     }
 
+    /// Applies reborrowing for `Pin`
+    ///
+    /// We currently only support reborrowing `Pin<&mut T>` as `Pin<&mut T>`. This is accomplished
+    /// by inserting a call to `Pin::as_mut` during MIR building.
+    ///
+    /// In the future we might want to support other reborrowing coercions, such as:
+    /// - `Pin<&mut T>` as `Pin<&T>`
+    /// - `Pin<&T>` as `Pin<&T>`
+    /// - `Pin<Box<T>>` as `Pin<&T>`
+    /// - `Pin<Box<T>>` as `Pin<&mut T>`
+    #[instrument(skip(self), level = "trace")]
+    fn coerce_pin(&self, a: Ty<'tcx>, b: Ty<'tcx>) -> CoerceResult<'tcx> {
+        // We need to make sure the two types are compatible for coercion.
+        // Then we will build a ReborrowPin adjustment and return that as an InferOk.
+
+        // Right now we can only reborrow if this is a `Pin<&mut T>`.
+        let can_reborrow = |ty: Ty<'tcx>| {
+            // Get the T out of Pin<T>
+            let ty = match ty.kind() {
+                ty::Adt(pin, args) if pin.did() == self.tcx.lang_items().pin_type().unwrap() => {
+                    args[0].expect_ty()
+                }
+                _ => {
+                    debug!("can't reborrow {:?} as pinned", ty);
+                    return None;
+                }
+            };
+            // Make sure the T is something we understand (just `&mut U` for now)
+            match ty.kind() {
+                ty::Ref(region, ty, ty::Mutability::Mut) => Some((*region, *ty)),
+                _ => {
+                    debug!("can't reborrow pin of inner type {:?}", ty);
+                    None
+                }
+            }
+        };
+
+        let (_, _a_ty) = can_reborrow(a).ok_or(TypeError::Mismatch)?;
+        let (b_region, _b_ty) = can_reborrow(b).ok_or(TypeError::Mismatch)?;
+
+        // To complete the reborrow, we need to make sure we can unify the inner types, and if so we
+        // add the adjustments.
+        self.unify_and(a, b, |_inner_ty| {
+            vec![Adjustment {
+                kind: Adjust::ReborrowPin(AutoBorrow::Ref(
+                    b_region,
+                    AutoBorrowMutability::Mut { allow_two_phase_borrow: AllowTwoPhase::No },
+                )),
+                target: b,
+            }]
+        })
+    }
+
     fn coerce_from_safe_fn<F, G>(
         &self,
         a: Ty<'tcx>,
diff --git a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
index da8c0ad3a30..103a4c72ea8 100644
--- a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
+++ b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
@@ -780,6 +780,20 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx
                 adjustment::Adjust::Borrow(ref autoref) => {
                     self.walk_autoref(expr, &place_with_id, autoref);
                 }
+
+                adjustment::Adjust::ReborrowPin(ref autoref) => {
+                    // Reborrowing a Pin is like a combinations of a deref and a borrow, so we do
+                    // both.
+                    let bk = match autoref {
+                        adjustment::AutoBorrow::Ref(_, m) => {
+                            ty::BorrowKind::from_mutbl((*m).into())
+                        }
+                        adjustment::AutoBorrow::RawPtr(m) => ty::BorrowKind::from_mutbl(*m),
+                    };
+                    self.delegate.borrow_mut().borrow(&place_with_id, place_with_id.hir_id, bk);
+
+                    self.walk_autoref(expr, &place_with_id, autoref);
+                }
             }
             place_with_id = self.cat_expr_adjusted(expr, place_with_id, adjustment)?;
         }
@@ -1284,6 +1298,7 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx
             adjustment::Adjust::NeverToAny
             | adjustment::Adjust::Pointer(_)
             | adjustment::Adjust::Borrow(_)
+            | adjustment::Adjust::ReborrowPin(_)
             | adjustment::Adjust::DynStar => {
                 // Result is an rvalue.
                 Ok(self.cat_rvalue(expr.hir_id, target))
diff --git a/compiler/rustc_middle/src/ty/adjustment.rs b/compiler/rustc_middle/src/ty/adjustment.rs
index 1236c9efb41..de3619f5b62 100644
--- a/compiler/rustc_middle/src/ty/adjustment.rs
+++ b/compiler/rustc_middle/src/ty/adjustment.rs
@@ -104,6 +104,9 @@ pub enum Adjust<'tcx> {
 
     /// Cast into a dyn* object.
     DynStar,
+
+    /// Take a Pin<Ptr> and call either as_mut() or as_ref() to get a Pin<&mut T> or Pin<&T>.
+    ReborrowPin(AutoBorrow<'tcx>),
 }
 
 /// An overloaded autoderef step, representing a `Deref(Mut)::deref(_mut)`
diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs
index aa8ccc8b7dd..8e6e13f94e0 100644
--- a/compiler/rustc_mir_build/src/thir/cx/expr.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs
@@ -74,6 +74,7 @@ impl<'tcx> Cx<'tcx> {
         self.thir.exprs.push(expr)
     }
 
+    #[instrument(level = "trace", skip(self, expr, span))]
     fn apply_adjustment(
         &mut self,
         hir_expr: &'tcx hir::Expr<'tcx>,
@@ -146,6 +147,50 @@ impl<'tcx> Cx<'tcx> {
                 ExprKind::RawBorrow { mutability, arg: self.thir.exprs.push(expr) }
             }
             Adjust::DynStar => ExprKind::Cast { source: self.thir.exprs.push(expr) },
+            Adjust::ReborrowPin(AutoBorrow::Ref(region, m)) => {
+                debug!("apply ReborrowPin adjustment");
+                match m {
+                    AutoBorrowMutability::Mut { .. } => {
+                        // Rewrite `$expr` as `Pin::as_mut(&mut $expr)`
+                        let as_mut_method =
+                            self.tcx().require_lang_item(rustc_hir::LangItem::PinAsMut, Some(span));
+                        let pin_ty_args = match expr.ty.kind() {
+                            ty::Adt(_, args) => args,
+                            _ => bug!("ReborrowPin with non-Pin type"),
+                        };
+                        let as_mut_ty =
+                            Ty::new_fn_def(self.tcx, as_mut_method, pin_ty_args.into_iter());
+
+                        let ty = Ty::new_ref(self.tcx, region, expr.ty, ty::Mutability::Mut);
+                        let arg = ExprKind::Borrow {
+                            borrow_kind: BorrowKind::Mut { kind: mir::MutBorrowKind::Default },
+                            arg: self.thir.exprs.push(expr),
+                        };
+                        debug!(?arg, "borrow arg");
+                        let arg = self.thir.exprs.push(Expr { temp_lifetime, ty, span, kind: arg });
+
+                        let kind = ExprKind::Call {
+                            ty: as_mut_ty,
+                            fun: self.thir.exprs.push(Expr {
+                                temp_lifetime,
+                                ty: as_mut_ty,
+                                span,
+                                kind: ExprKind::ZstLiteral { user_ty: None },
+                            }),
+                            args: Box::new([arg]),
+                            from_hir_call: true,
+                            fn_span: span,
+                        };
+                        debug!(?kind);
+                        kind
+                    }
+                    AutoBorrowMutability::Not => {
+                        // FIXME: We need to call Pin::as_ref on the expression
+                        bug!("ReborrowPin with shared reference is not implemented yet")
+                    }
+                }
+            }
+            Adjust::ReborrowPin(AutoBorrow::RawPtr(_)) => bug!("ReborrowPin with raw pointer"),
         };
 
         Expr { temp_lifetime, ty: adjustment.target, span, kind }
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index cbe2bafef21..979294563d7 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -1418,6 +1418,8 @@ symbols! {
         pic,
         pie,
         pin,
+        pin_as_mut,
+        pin_ergonomics,
         platform_intrinsics,
         plugin,
         plugin_registrar,
diff --git a/library/core/src/pin.rs b/library/core/src/pin.rs
index 9c13662e08e..bb1196e82de 100644
--- a/library/core/src/pin.rs
+++ b/library/core/src/pin.rs
@@ -1408,6 +1408,7 @@ impl<Ptr: DerefMut> Pin<Ptr> {
     ///     }
     /// }
     /// ```
+    #[cfg_attr(not(bootstrap), lang = "pin_as_mut")]
     #[stable(feature = "pin", since = "1.33.0")]
     #[inline(always)]
     pub fn as_mut(&mut self) -> Pin<&mut Ptr::Target> {
diff --git a/tests/ui/async-await/pin-reborrow-arg.rs b/tests/ui/async-await/pin-reborrow-arg.rs
new file mode 100644
index 00000000000..34f23533b65
--- /dev/null
+++ b/tests/ui/async-await/pin-reborrow-arg.rs
@@ -0,0 +1,27 @@
+//@ check-pass
+
+#![feature(pin_ergonomics)]
+#![allow(dead_code, incomplete_features)]
+
+use std::pin::Pin;
+
+struct Foo;
+
+impl Foo {
+    fn baz(self: Pin<&mut Self>) {
+    }
+}
+
+fn foo(_: Pin<&mut Foo>) {
+}
+
+fn bar(mut x: Pin<&mut Foo>) {
+    foo(x);
+    foo(x); // for this to work we need to automatically reborrow,
+            // as if the user had written `foo(x.as_mut())`.
+
+    Foo::baz(x);
+    Foo::baz(x);
+}
+
+fn main() {}
diff --git a/tests/ui/async-await/pin-reborrow-self.rs b/tests/ui/async-await/pin-reborrow-self.rs
new file mode 100644
index 00000000000..b60b6982bb8
--- /dev/null
+++ b/tests/ui/async-await/pin-reborrow-self.rs
@@ -0,0 +1,24 @@
+//@ check-pass
+//@ignore-test
+
+// Currently ignored due to self reborrowing not being implemented for Pin
+
+#![feature(pin_ergonomics)]
+#![allow(incomplete_features)]
+
+use std::pin::Pin;
+
+struct Foo;
+
+impl Foo {
+    fn foo(self: Pin<&mut Self>) {
+    }
+}
+
+fn bar(x: Pin<&mut Foo>) {
+    x.foo();
+    x.foo(); // for this to work we need to automatically reborrow,
+             // as if the user had written `x.as_mut().foo()`.
+}
+
+fn main() {}
diff --git a/tests/ui/feature-gates/feature-gate-pin_ergonomics.rs b/tests/ui/feature-gates/feature-gate-pin_ergonomics.rs
new file mode 100644
index 00000000000..d694531d53a
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-pin_ergonomics.rs
@@ -0,0 +1,15 @@
+#![allow(dead_code, incomplete_features)]
+
+use std::pin::Pin;
+
+struct Foo;
+
+fn foo(_: Pin<&mut Foo>) {
+}
+
+fn bar(mut x: Pin<&mut Foo>) {
+    foo(x);
+    foo(x); //~ ERROR use of moved value: `x`
+}
+
+fn main() {}
diff --git a/tests/ui/feature-gates/feature-gate-pin_ergonomics.stderr b/tests/ui/feature-gates/feature-gate-pin_ergonomics.stderr
new file mode 100644
index 00000000000..6c9029d8176
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-pin_ergonomics.stderr
@@ -0,0 +1,21 @@
+error[E0382]: use of moved value: `x`
+  --> $DIR/feature-gate-pin_ergonomics.rs:12:9
+   |
+LL | fn bar(mut x: Pin<&mut Foo>) {
+   |        ----- move occurs because `x` has type `Pin<&mut Foo>`, which does not implement the `Copy` trait
+LL |     foo(x);
+   |         - value moved here
+LL |     foo(x);
+   |         ^ value used here after move
+   |
+note: consider changing this parameter type in function `foo` to borrow instead if owning the value isn't necessary
+  --> $DIR/feature-gate-pin_ergonomics.rs:7:11
+   |
+LL | fn foo(_: Pin<&mut Foo>) {
+   |    ---    ^^^^^^^^^^^^^ this parameter takes ownership of the value
+   |    |
+   |    in this function
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0382`.

From a73c8b1171be7af6c2e56d6a8c01b4d5b4eb61cb Mon Sep 17 00:00:00 2001
From: Eric Holk <ericholk@microsoft.com>
Date: Wed, 18 Sep 2024 15:37:50 -0700
Subject: [PATCH 03/16] Apply code review suggestions

---
 compiler/rustc_hir_typeck/src/coercion.rs        | 16 ++++++++--------
 .../rustc_hir_typeck/src/expr_use_visitor.rs     |  2 --
 compiler/rustc_middle/src/ty/adjustment.rs       |  3 ++-
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_hir_typeck/src/coercion.rs b/compiler/rustc_hir_typeck/src/coercion.rs
index 3da4b2893d9..d548df98bf9 100644
--- a/compiler/rustc_hir_typeck/src/coercion.rs
+++ b/compiler/rustc_hir_typeck/src/coercion.rs
@@ -216,7 +216,7 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
             }
             ty::Adt(pin, _)
                 if self.tcx.features().pin_ergonomics
-                    && pin.did() == self.tcx.lang_items().pin_type().unwrap() =>
+                    && self.tcx.is_lang_item(pin.did(), hir::LangItem::Pin) =>
             {
                 return self.coerce_pin(a, b);
             }
@@ -796,29 +796,29 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
         // Then we will build a ReborrowPin adjustment and return that as an InferOk.
 
         // Right now we can only reborrow if this is a `Pin<&mut T>`.
-        let can_reborrow = |ty: Ty<'tcx>| {
+        let extract_pin_mut = |ty: Ty<'tcx>| {
             // Get the T out of Pin<T>
             let ty = match ty.kind() {
-                ty::Adt(pin, args) if pin.did() == self.tcx.lang_items().pin_type().unwrap() => {
+                ty::Adt(pin, args) if self.tcx.is_lang_item(pin.did(), hir::LangItem::Pin) => {
                     args[0].expect_ty()
                 }
                 _ => {
                     debug!("can't reborrow {:?} as pinned", ty);
-                    return None;
+                    return Err(TypeError::Mismatch);
                 }
             };
             // Make sure the T is something we understand (just `&mut U` for now)
             match ty.kind() {
-                ty::Ref(region, ty, ty::Mutability::Mut) => Some((*region, *ty)),
+                ty::Ref(region, ty, ty::Mutability::Mut) => Ok((*region, *ty)),
                 _ => {
                     debug!("can't reborrow pin of inner type {:?}", ty);
-                    None
+                    Err(TypeError::Mismatch)
                 }
             }
         };
 
-        let (_, _a_ty) = can_reborrow(a).ok_or(TypeError::Mismatch)?;
-        let (b_region, _b_ty) = can_reborrow(b).ok_or(TypeError::Mismatch)?;
+        let (_, _a_ty) = extract_pin_mut(a)?;
+        let (b_region, _b_ty) = extract_pin_mut(b)?;
 
         // To complete the reborrow, we need to make sure we can unify the inner types, and if so we
         // add the adjustments.
diff --git a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
index 103a4c72ea8..95a3cf7b96b 100644
--- a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
+++ b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
@@ -791,8 +791,6 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx
                         adjustment::AutoBorrow::RawPtr(m) => ty::BorrowKind::from_mutbl(*m),
                     };
                     self.delegate.borrow_mut().borrow(&place_with_id, place_with_id.hir_id, bk);
-
-                    self.walk_autoref(expr, &place_with_id, autoref);
                 }
             }
             place_with_id = self.cat_expr_adjusted(expr, place_with_id, adjustment)?;
diff --git a/compiler/rustc_middle/src/ty/adjustment.rs b/compiler/rustc_middle/src/ty/adjustment.rs
index de3619f5b62..7f56e05901c 100644
--- a/compiler/rustc_middle/src/ty/adjustment.rs
+++ b/compiler/rustc_middle/src/ty/adjustment.rs
@@ -105,7 +105,8 @@ pub enum Adjust<'tcx> {
     /// Cast into a dyn* object.
     DynStar,
 
-    /// Take a Pin<Ptr> and call either as_mut() or as_ref() to get a Pin<&mut T> or Pin<&T>.
+    /// Take a `Pin<Ptr>` and call either `as_mut()` or `as_ref()` to get a `Pin<&mut T>` or
+    /// `Pin<&T>`.
     ReborrowPin(AutoBorrow<'tcx>),
 }
 

From b2b76fb70659f55d5e1842b28c7d6c451ae916d9 Mon Sep 17 00:00:00 2001
From: Eric Holk <ericholk@microsoft.com>
Date: Thu, 19 Sep 2024 11:18:00 -0700
Subject: [PATCH 04/16] Allow shortening reborrows

Generating a call to `as_mut()` let to more restrictive borrows than
what reborrowing usually gives us. Instead, we change the desugaring to
reborrow the pin internals directly which makes things more expressive.
---
 compiler/rustc_hir/src/lang_items.rs          |  1 -
 compiler/rustc_hir_typeck/src/coercion.rs     |  5 +-
 .../rustc_hir_typeck/src/expr_use_visitor.rs  | 12 +--
 compiler/rustc_middle/src/ty/adjustment.rs    |  5 +-
 compiler/rustc_mir_build/src/thir/cx/expr.rs  | 96 +++++++++++--------
 compiler/rustc_span/src/symbol.rs             |  1 -
 library/core/src/pin.rs                       |  1 -
 tests/ui/async-await/pin-reborrow-shorter.rs  | 14 +++
 8 files changed, 77 insertions(+), 58 deletions(-)
 create mode 100644 tests/ui/async-await/pin-reborrow-shorter.rs

diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs
index 60e024068e9..e7398fd2226 100644
--- a/compiler/rustc_hir/src/lang_items.rs
+++ b/compiler/rustc_hir/src/lang_items.rs
@@ -395,7 +395,6 @@ language_item_table! {
     IteratorNext,            sym::next,                next_fn,                    Target::Method(MethodKind::Trait { body: false}), GenericRequirement::None;
 
     PinNewUnchecked,         sym::new_unchecked,       new_unchecked_fn,           Target::Method(MethodKind::Inherent), GenericRequirement::None;
-    PinAsMut,                sym::pin_as_mut,              as_mut_fn,                  Target::Method(MethodKind::Inherent), GenericRequirement::None;
 
     RangeFrom,               sym::RangeFrom,           range_from_struct,          Target::Struct,         GenericRequirement::None;
     RangeFull,               sym::RangeFull,           range_full_struct,          Target::Struct,         GenericRequirement::None;
diff --git a/compiler/rustc_hir_typeck/src/coercion.rs b/compiler/rustc_hir_typeck/src/coercion.rs
index d548df98bf9..d45d7f54ed4 100644
--- a/compiler/rustc_hir_typeck/src/coercion.rs
+++ b/compiler/rustc_hir_typeck/src/coercion.rs
@@ -824,10 +824,7 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
         // add the adjustments.
         self.unify_and(a, b, |_inner_ty| {
             vec![Adjustment {
-                kind: Adjust::ReborrowPin(AutoBorrow::Ref(
-                    b_region,
-                    AutoBorrowMutability::Mut { allow_two_phase_borrow: AllowTwoPhase::No },
-                )),
+                kind: Adjust::ReborrowPin(b_region, hir::Mutability::Mut),
                 target: b,
             }]
         })
diff --git a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
index 95a3cf7b96b..3b2ddf659a1 100644
--- a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
+++ b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
@@ -781,14 +781,12 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx
                     self.walk_autoref(expr, &place_with_id, autoref);
                 }
 
-                adjustment::Adjust::ReborrowPin(ref autoref) => {
+                adjustment::Adjust::ReborrowPin(_, mutbl) => {
                     // Reborrowing a Pin is like a combinations of a deref and a borrow, so we do
                     // both.
-                    let bk = match autoref {
-                        adjustment::AutoBorrow::Ref(_, m) => {
-                            ty::BorrowKind::from_mutbl((*m).into())
-                        }
-                        adjustment::AutoBorrow::RawPtr(m) => ty::BorrowKind::from_mutbl(*m),
+                    let bk = match mutbl {
+                        ty::Mutability::Not => ty::BorrowKind::ImmBorrow,
+                        ty::Mutability::Mut => ty::BorrowKind::MutBorrow,
                     };
                     self.delegate.borrow_mut().borrow(&place_with_id, place_with_id.hir_id, bk);
                 }
@@ -1296,7 +1294,7 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx
             adjustment::Adjust::NeverToAny
             | adjustment::Adjust::Pointer(_)
             | adjustment::Adjust::Borrow(_)
-            | adjustment::Adjust::ReborrowPin(_)
+            | adjustment::Adjust::ReborrowPin(..)
             | adjustment::Adjust::DynStar => {
                 // Result is an rvalue.
                 Ok(self.cat_rvalue(expr.hir_id, target))
diff --git a/compiler/rustc_middle/src/ty/adjustment.rs b/compiler/rustc_middle/src/ty/adjustment.rs
index 7f56e05901c..5a32078760e 100644
--- a/compiler/rustc_middle/src/ty/adjustment.rs
+++ b/compiler/rustc_middle/src/ty/adjustment.rs
@@ -105,9 +105,8 @@ pub enum Adjust<'tcx> {
     /// Cast into a dyn* object.
     DynStar,
 
-    /// Take a `Pin<Ptr>` and call either `as_mut()` or `as_ref()` to get a `Pin<&mut T>` or
-    /// `Pin<&T>`.
-    ReborrowPin(AutoBorrow<'tcx>),
+    /// Take a pinned reference and reborrow as a `Pin<&mut T>` or `Pin<&T>`.
+    ReborrowPin(ty::Region<'tcx>, hir::Mutability),
 }
 
 /// An overloaded autoderef step, representing a `Deref(Mut)::deref(_mut)`
diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs
index 8e6e13f94e0..3bb00e0b028 100644
--- a/compiler/rustc_mir_build/src/thir/cx/expr.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs
@@ -147,50 +147,64 @@ impl<'tcx> Cx<'tcx> {
                 ExprKind::RawBorrow { mutability, arg: self.thir.exprs.push(expr) }
             }
             Adjust::DynStar => ExprKind::Cast { source: self.thir.exprs.push(expr) },
-            Adjust::ReborrowPin(AutoBorrow::Ref(region, m)) => {
+            Adjust::ReborrowPin(region, mutbl) => {
                 debug!("apply ReborrowPin adjustment");
-                match m {
-                    AutoBorrowMutability::Mut { .. } => {
-                        // Rewrite `$expr` as `Pin::as_mut(&mut $expr)`
-                        let as_mut_method =
-                            self.tcx().require_lang_item(rustc_hir::LangItem::PinAsMut, Some(span));
-                        let pin_ty_args = match expr.ty.kind() {
-                            ty::Adt(_, args) => args,
-                            _ => bug!("ReborrowPin with non-Pin type"),
-                        };
-                        let as_mut_ty =
-                            Ty::new_fn_def(self.tcx, as_mut_method, pin_ty_args.into_iter());
+                // Rewrite `$expr` as `Pin { __pointer: &(mut)? *($expr).__pointer }`
 
-                        let ty = Ty::new_ref(self.tcx, region, expr.ty, ty::Mutability::Mut);
-                        let arg = ExprKind::Borrow {
-                            borrow_kind: BorrowKind::Mut { kind: mir::MutBorrowKind::Default },
-                            arg: self.thir.exprs.push(expr),
-                        };
-                        debug!(?arg, "borrow arg");
-                        let arg = self.thir.exprs.push(Expr { temp_lifetime, ty, span, kind: arg });
+                // We'll need these types later on
+                let pin_ty_args = match expr.ty.kind() {
+                    ty::Adt(_, args) => args,
+                    _ => bug!("ReborrowPin with non-Pin type"),
+                };
+                let pin_ty = pin_ty_args.iter().next().unwrap().expect_ty();
+                let ptr_target_ty = match pin_ty.kind() {
+                    ty::Ref(_, ty, _) => *ty,
+                    _ => bug!("ReborrowPin with non-Ref type"),
+                };
 
-                        let kind = ExprKind::Call {
-                            ty: as_mut_ty,
-                            fun: self.thir.exprs.push(Expr {
-                                temp_lifetime,
-                                ty: as_mut_ty,
-                                span,
-                                kind: ExprKind::ZstLiteral { user_ty: None },
-                            }),
-                            args: Box::new([arg]),
-                            from_hir_call: true,
-                            fn_span: span,
-                        };
-                        debug!(?kind);
-                        kind
-                    }
-                    AutoBorrowMutability::Not => {
-                        // FIXME: We need to call Pin::as_ref on the expression
-                        bug!("ReborrowPin with shared reference is not implemented yet")
-                    }
-                }
+                // pointer = ($expr).__pointer
+                let pointer_target = ExprKind::Field {
+                    lhs: self.thir.exprs.push(expr),
+                    variant_index: FIRST_VARIANT,
+                    name: FieldIdx::from(0u32),
+                };
+                let arg = Expr { temp_lifetime, ty: pin_ty, span, kind: pointer_target };
+                let arg = self.thir.exprs.push(arg);
+
+                // arg = *pointer
+                let expr = ExprKind::Deref { arg };
+                let arg = self.thir.exprs.push(Expr {
+                    temp_lifetime,
+                    ty: ptr_target_ty,
+                    span,
+                    kind: expr,
+                });
+
+                // expr = &mut target
+                let expr = self.thir.exprs.push(Expr {
+                    temp_lifetime,
+                    ty: Ty::new_ref(self.tcx, region, ptr_target_ty, mutbl),
+                    span,
+                    kind: ExprKind::Borrow {
+                        borrow_kind: BorrowKind::Mut { kind: mir::MutBorrowKind::Default },
+                        arg,
+                    },
+                });
+
+                // kind = Pin { __pointer: pointer }
+                let pin_did = self.tcx.require_lang_item(rustc_hir::LangItem::Pin, Some(span));
+                let kind = ExprKind::Adt(Box::new(AdtExpr {
+                    adt_def: self.tcx.adt_def(pin_did),
+                    variant_index: FIRST_VARIANT,
+                    args: pin_ty_args,
+                    fields: Box::new([FieldExpr { name: FieldIdx::from(0u32), expr }]),
+                    user_ty: None,
+                    base: None,
+                }));
+
+                debug!(?kind);
+                kind
             }
-            Adjust::ReborrowPin(AutoBorrow::RawPtr(_)) => bug!("ReborrowPin with raw pointer"),
         };
 
         Expr { temp_lifetime, ty: adjustment.target, span, kind }
@@ -1059,7 +1073,7 @@ impl<'tcx> Cx<'tcx> {
 
         // Reconstruct the output assuming it's a reference with the
         // same region and mutability as the receiver. This holds for
-        // `Deref(Mut)::Deref(_mut)` and `Index(Mut)::index(_mut)`.
+        // `Deref(Mut)::deref(_mut)` and `Index(Mut)::index(_mut)`.
         let ty::Ref(region, _, mutbl) = *self.thir[args[0]].ty.kind() else {
             span_bug!(span, "overloaded_place: receiver is not a reference");
         };
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 979294563d7..27ff9afa92c 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -1418,7 +1418,6 @@ symbols! {
         pic,
         pie,
         pin,
-        pin_as_mut,
         pin_ergonomics,
         platform_intrinsics,
         plugin,
diff --git a/library/core/src/pin.rs b/library/core/src/pin.rs
index bb1196e82de..9c13662e08e 100644
--- a/library/core/src/pin.rs
+++ b/library/core/src/pin.rs
@@ -1408,7 +1408,6 @@ impl<Ptr: DerefMut> Pin<Ptr> {
     ///     }
     /// }
     /// ```
-    #[cfg_attr(not(bootstrap), lang = "pin_as_mut")]
     #[stable(feature = "pin", since = "1.33.0")]
     #[inline(always)]
     pub fn as_mut(&mut self) -> Pin<&mut Ptr::Target> {
diff --git a/tests/ui/async-await/pin-reborrow-shorter.rs b/tests/ui/async-await/pin-reborrow-shorter.rs
new file mode 100644
index 00000000000..06c266e0035
--- /dev/null
+++ b/tests/ui/async-await/pin-reborrow-shorter.rs
@@ -0,0 +1,14 @@
+//@check-pass
+
+#![feature(pin_ergonomics)]
+#![allow(dead_code, incomplete_features)]
+
+use std::pin::Pin;
+
+fn shorter<'b, T: 'b>(_: Pin<&'b mut T>) {}
+
+fn test<'a: 'b, 'b, T: 'a>(x: Pin<&'a mut T>) {
+    shorter::<'b>(x);
+}
+
+fn main() {}

From 92a5d21bc4a0b735750a4dff1eac204ce636b513 Mon Sep 17 00:00:00 2001
From: Eric Holk <ericholk@microsoft.com>
Date: Thu, 19 Sep 2024 15:29:01 -0700
Subject: [PATCH 05/16] Add a test case to make sure we don't reborrow twice

---
 tests/ui/async-await/pin-reborrow-arg.rs      |  2 +-
 tests/ui/async-await/pin-reborrow-once.rs     | 13 +++++++++++++
 tests/ui/async-await/pin-reborrow-once.stderr | 12 ++++++++++++
 3 files changed, 26 insertions(+), 1 deletion(-)
 create mode 100644 tests/ui/async-await/pin-reborrow-once.rs
 create mode 100644 tests/ui/async-await/pin-reborrow-once.stderr

diff --git a/tests/ui/async-await/pin-reborrow-arg.rs b/tests/ui/async-await/pin-reborrow-arg.rs
index 34f23533b65..b3e8718d036 100644
--- a/tests/ui/async-await/pin-reborrow-arg.rs
+++ b/tests/ui/async-await/pin-reborrow-arg.rs
@@ -15,7 +15,7 @@ impl Foo {
 fn foo(_: Pin<&mut Foo>) {
 }
 
-fn bar(mut x: Pin<&mut Foo>) {
+fn bar(x: Pin<&mut Foo>) {
     foo(x);
     foo(x); // for this to work we need to automatically reborrow,
             // as if the user had written `foo(x.as_mut())`.
diff --git a/tests/ui/async-await/pin-reborrow-once.rs b/tests/ui/async-await/pin-reborrow-once.rs
new file mode 100644
index 00000000000..241efadef7d
--- /dev/null
+++ b/tests/ui/async-await/pin-reborrow-once.rs
@@ -0,0 +1,13 @@
+#![feature(pin_ergonomics)]
+#![allow(dead_code, incomplete_features)]
+
+// Make sure with pin reborrowing that we can only get one mutable reborrow of a pinned reference.
+
+use std::pin::{pin, Pin};
+
+fn twice(_: Pin<&mut i32>, _: Pin<&mut i32>) {}
+
+fn main() {
+    let x = pin!(42);
+    twice(x, x); //~ ERROR cannot borrow
+}
diff --git a/tests/ui/async-await/pin-reborrow-once.stderr b/tests/ui/async-await/pin-reborrow-once.stderr
new file mode 100644
index 00000000000..b8fde8ffee8
--- /dev/null
+++ b/tests/ui/async-await/pin-reborrow-once.stderr
@@ -0,0 +1,12 @@
+error[E0499]: cannot borrow `*x.__pointer` as mutable more than once at a time
+  --> $DIR/pin-reborrow-once.rs:12:14
+   |
+LL |     twice(x, x);
+   |     ----- -  ^ second mutable borrow occurs here
+   |     |     |
+   |     |     first mutable borrow occurs here
+   |     first borrow later used by call
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0499`.

From a18800f8078f3fb7b986183b53c34a6a1b921d63 Mon Sep 17 00:00:00 2001
From: Eric Holk <ericholk@microsoft.com>
Date: Thu, 19 Sep 2024 16:27:09 -0700
Subject: [PATCH 06/16] pin_ergonomics: allow reborrowing as Pin<&T>

---
 compiler/rustc_hir_typeck/src/coercion.rs     | 24 ++++++++++++-------
 compiler/rustc_mir_build/src/thir/cx/expr.rs  | 15 +++++++-----
 tests/ui/async-await/pin-reborrow-arg.rs      |  9 +++++++
 .../async-await/pin-reborrow-const-as-mut.rs  | 18 ++++++++++++++
 .../pin-reborrow-const-as-mut.stderr          | 19 +++++++++++++++
 5 files changed, 70 insertions(+), 15 deletions(-)
 create mode 100644 tests/ui/async-await/pin-reborrow-const-as-mut.rs
 create mode 100644 tests/ui/async-await/pin-reborrow-const-as-mut.stderr

diff --git a/compiler/rustc_hir_typeck/src/coercion.rs b/compiler/rustc_hir_typeck/src/coercion.rs
index d45d7f54ed4..94b8541075e 100644
--- a/compiler/rustc_hir_typeck/src/coercion.rs
+++ b/compiler/rustc_hir_typeck/src/coercion.rs
@@ -798,9 +798,9 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
         // Right now we can only reborrow if this is a `Pin<&mut T>`.
         let extract_pin_mut = |ty: Ty<'tcx>| {
             // Get the T out of Pin<T>
-            let ty = match ty.kind() {
+            let (pin, ty) = match ty.kind() {
                 ty::Adt(pin, args) if self.tcx.is_lang_item(pin.did(), hir::LangItem::Pin) => {
-                    args[0].expect_ty()
+                    (*pin, args[0].expect_ty())
                 }
                 _ => {
                     debug!("can't reborrow {:?} as pinned", ty);
@@ -809,7 +809,7 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
             };
             // Make sure the T is something we understand (just `&mut U` for now)
             match ty.kind() {
-                ty::Ref(region, ty, ty::Mutability::Mut) => Ok((*region, *ty)),
+                ty::Ref(region, ty, mutbl) => Ok((pin, *region, *ty, *mutbl)),
                 _ => {
                     debug!("can't reborrow pin of inner type {:?}", ty);
                     Err(TypeError::Mismatch)
@@ -817,16 +817,22 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
             }
         };
 
-        let (_, _a_ty) = extract_pin_mut(a)?;
-        let (b_region, _b_ty) = extract_pin_mut(b)?;
+        let (pin, a_region, a_ty, mut_a) = extract_pin_mut(a)?;
+        let (_, b_region, _b_ty, mut_b) = extract_pin_mut(b)?;
+
+        coerce_mutbls(mut_a, mut_b)?;
+
+        // update a with b's mutability since we'll be coercing mutability
+        let a = Ty::new_adt(
+            self.tcx,
+            pin,
+            self.tcx.mk_args(&[Ty::new_ref(self.tcx, a_region, a_ty, mut_b).into()]),
+        );
 
         // To complete the reborrow, we need to make sure we can unify the inner types, and if so we
         // add the adjustments.
         self.unify_and(a, b, |_inner_ty| {
-            vec![Adjustment {
-                kind: Adjust::ReborrowPin(b_region, hir::Mutability::Mut),
-                target: b,
-            }]
+            vec![Adjustment { kind: Adjust::ReborrowPin(b_region, mut_b), target: b }]
         })
     }
 
diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs
index 3bb00e0b028..3f730b5d183 100644
--- a/compiler/rustc_mir_build/src/thir/cx/expr.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs
@@ -181,22 +181,25 @@ impl<'tcx> Cx<'tcx> {
                 });
 
                 // expr = &mut target
+                let borrow_kind = match mutbl {
+                    hir::Mutability::Mut => BorrowKind::Mut { kind: mir::MutBorrowKind::Default },
+                    hir::Mutability::Not => BorrowKind::Shared,
+                };
+                let new_pin_target = Ty::new_ref(self.tcx, region, ptr_target_ty, mutbl);
                 let expr = self.thir.exprs.push(Expr {
                     temp_lifetime,
-                    ty: Ty::new_ref(self.tcx, region, ptr_target_ty, mutbl),
+                    ty: new_pin_target,
                     span,
-                    kind: ExprKind::Borrow {
-                        borrow_kind: BorrowKind::Mut { kind: mir::MutBorrowKind::Default },
-                        arg,
-                    },
+                    kind: ExprKind::Borrow { borrow_kind, arg },
                 });
 
                 // kind = Pin { __pointer: pointer }
                 let pin_did = self.tcx.require_lang_item(rustc_hir::LangItem::Pin, Some(span));
+                let args = self.tcx.mk_args(&[new_pin_target.into()]);
                 let kind = ExprKind::Adt(Box::new(AdtExpr {
                     adt_def: self.tcx.adt_def(pin_did),
                     variant_index: FIRST_VARIANT,
-                    args: pin_ty_args,
+                    args,
                     fields: Box::new([FieldExpr { name: FieldIdx::from(0u32), expr }]),
                     user_ty: None,
                     base: None,
diff --git a/tests/ui/async-await/pin-reborrow-arg.rs b/tests/ui/async-await/pin-reborrow-arg.rs
index b3e8718d036..2008bd1f52d 100644
--- a/tests/ui/async-await/pin-reborrow-arg.rs
+++ b/tests/ui/async-await/pin-reborrow-arg.rs
@@ -15,6 +15,9 @@ impl Foo {
 fn foo(_: Pin<&mut Foo>) {
 }
 
+fn foo_const(_: Pin<&Foo>) {
+}
+
 fn bar(x: Pin<&mut Foo>) {
     foo(x);
     foo(x); // for this to work we need to automatically reborrow,
@@ -22,6 +25,12 @@ fn bar(x: Pin<&mut Foo>) {
 
     Foo::baz(x);
     Foo::baz(x);
+
+    foo_const(x); // make sure we can reborrow &mut as &.
+
+    let x: Pin<&Foo> = Pin::new(&Foo);
+
+    foo_const(x); // make sure reborrowing from & to & works.
 }
 
 fn main() {}
diff --git a/tests/ui/async-await/pin-reborrow-const-as-mut.rs b/tests/ui/async-await/pin-reborrow-const-as-mut.rs
new file mode 100644
index 00000000000..27c70a7b4df
--- /dev/null
+++ b/tests/ui/async-await/pin-reborrow-const-as-mut.rs
@@ -0,0 +1,18 @@
+#![feature(pin_ergonomics)]
+#![allow(dead_code, incomplete_features)]
+
+// make sure we can't accidentally reborrow Pin<&T> as Pin<&mut T>
+
+use std::pin::Pin;
+
+struct Foo;
+
+fn foo(_: Pin<&mut Foo>) {
+}
+
+fn bar(x: Pin<&Foo>) {
+    foo(x); //~ ERROR mismatched types
+            //| ERROR types differ in mutability
+}
+
+fn main() {}
diff --git a/tests/ui/async-await/pin-reborrow-const-as-mut.stderr b/tests/ui/async-await/pin-reborrow-const-as-mut.stderr
new file mode 100644
index 00000000000..2c2d9ec2717
--- /dev/null
+++ b/tests/ui/async-await/pin-reborrow-const-as-mut.stderr
@@ -0,0 +1,19 @@
+error[E0308]: mismatched types
+  --> $DIR/pin-reborrow-const-as-mut.rs:14:9
+   |
+LL |     foo(x);
+   |     --- ^ types differ in mutability
+   |     |
+   |     arguments to this function are incorrect
+   |
+   = note: expected struct `Pin<&mut Foo>`
+              found struct `Pin<&Foo>`
+note: function defined here
+  --> $DIR/pin-reborrow-const-as-mut.rs:10:4
+   |
+LL | fn foo(_: Pin<&mut Foo>) {
+   |    ^^^ ----------------
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0308`.

From bfadadf78fc483aef5d24be97afb51997d5bf156 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gabriel=20Bj=C3=B8rnager=20Jensen?= <gabriel@achernar.io>
Date: Fri, 20 Sep 2024 13:34:09 +0200
Subject: [PATCH 07/16] Address diagnostics regression for
 'const_char_encode_utf8';

---
 library/core/src/char/methods.rs | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/library/core/src/char/methods.rs b/library/core/src/char/methods.rs
index fcaa91184d3..092d427ecea 100644
--- a/library/core/src/char/methods.rs
+++ b/library/core/src/char/methods.rs
@@ -1,6 +1,7 @@
 //! impl char {}
 
 use super::*;
+use crate::intrinsics::const_eval_select;
 use crate::slice;
 use crate::str::from_utf8_unchecked_mut;
 use crate::unicode::printable::is_printable;
@@ -1762,6 +1763,15 @@ const fn len_utf8(code: u32) -> usize {
 #[doc(hidden)]
 #[inline]
 pub const fn encode_utf8_raw(code: u32, dst: &mut [u8]) -> &mut [u8] {
+    const fn panic_at_const(_code: u32, _len: usize, _dst_len: usize) {
+        // Note that we cannot format in constant expressions.
+        panic!("encode_utf8: buffer does not have enough bytes to encode code point");
+    }
+    fn panic_at_rt(code: u32, len: usize, dst_len: usize) {
+        panic!(
+            "encode_utf8: need {len} bytes to encode U+{code:04X} but buffer has just {dst_len}",
+        );
+    }
     let len = len_utf8(code);
     match (len, &mut *dst) {
         (1, [a, ..]) => {
@@ -1782,8 +1792,8 @@ pub const fn encode_utf8_raw(code: u32, dst: &mut [u8]) -> &mut [u8] {
             *c = (code >> 6 & 0x3F) as u8 | TAG_CONT;
             *d = (code & 0x3F) as u8 | TAG_CONT;
         }
-        // Note that we cannot format in constant expressions.
-        _ => panic!("encode_utf8: buffer does not have enough bytes to encode code point"),
+        // FIXME(const-hack): We would prefer to have streamlined panics when formatters become const-friendly.
+        _ => const_eval_select((code, len, dst.len()), panic_at_const, panic_at_rt),
     };
     // SAFETY: `<&mut [u8]>::as_mut_ptr` is guaranteed to return a valid pointer and `len` has been tested to be within bounds.
     unsafe { slice::from_raw_parts_mut(dst.as_mut_ptr(), len) }

From 9104632c4f6a2b5380fc770a29ce368bba563b69 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <guillaume.gomez@huawei.com>
Date: Fri, 20 Sep 2024 16:02:16 +0200
Subject: [PATCH 08/16] Revert additional scraped examples height

---
 src/librustdoc/html/static/css/rustdoc.css | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/librustdoc/html/static/css/rustdoc.css b/src/librustdoc/html/static/css/rustdoc.css
index 38154dee3e2..01506b46af2 100644
--- a/src/librustdoc/html/static/css/rustdoc.css
+++ b/src/librustdoc/html/static/css/rustdoc.css
@@ -801,6 +801,9 @@ both the code example and the line numbers, so we need to remove the radius in t
 	 * and we include additional 10px for padding. */
 	max-height: calc(1.5em * 5 + 10px);
 }
+.more-scraped-examples .scraped-example:not(.expanded) .example-wrap {
+	max-height: calc(1.5em * 10 + 10px);
+}
 
 .rustdoc:not(.src) .scraped-example:not(.expanded) .src-line-numbers,
 .rustdoc:not(.src) .scraped-example:not(.expanded) .src-line-numbers > pre,

From 660210c4af12ebcf9e76f897c2542339fe7ce110 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <guillaume.gomez@huawei.com>
Date: Fri, 20 Sep 2024 15:34:45 +0200
Subject: [PATCH 09/16] Add GUI test to check size of additional scraped
 examples

---
 tests/rustdoc-gui/scrape-examples-layout.goml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/rustdoc-gui/scrape-examples-layout.goml b/tests/rustdoc-gui/scrape-examples-layout.goml
index 6bea352bce4..be14e202b37 100644
--- a/tests/rustdoc-gui/scrape-examples-layout.goml
+++ b/tests/rustdoc-gui/scrape-examples-layout.goml
@@ -37,6 +37,7 @@ assert-property: (
 
 // The "title" should be located at the right bottom corner of the code example.
 store-position: (".scraped-example .example-wrap", {"x": x, "y": y})
+assert-size: (".scraped-example .example-wrap", {"height": 130})
 store-size: (".scraped-example .example-wrap", {"width": width, "height": height})
 store-size: (".scraped-example .scraped-example-title", {
     "width": title_width,
@@ -47,6 +48,13 @@ assert-position: (".scraped-example .scraped-example-title", {
     "y": |y| + |height| - |title_height| - 8,
 })
 
+store-size: (".more-scraped-examples .scraped-example .example-wrap", {"height": more_height})
+assert: |more_height| > |height|
+assert-size: (".more-scraped-examples .scraped-example .example-wrap", {
+    "height": 250,
+    "width": |width|,
+})
+
 // Check that the expand button works and also that line number aligns with code.
 move-cursor-to: ".scraped-example .rust"
 click: ".scraped-example .button-holder .expand"

From 38bf2894feff81eb95c39319297e650b79d12b05 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <guillaume.gomez@huawei.com>
Date: Fri, 20 Sep 2024 15:20:39 +0200
Subject: [PATCH 10/16] Improve paddings for code blocks line numbers

---
 src/librustdoc/html/static/css/rustdoc.css | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/librustdoc/html/static/css/rustdoc.css b/src/librustdoc/html/static/css/rustdoc.css
index 01506b46af2..04b0eba7450 100644
--- a/src/librustdoc/html/static/css/rustdoc.css
+++ b/src/librustdoc/html/static/css/rustdoc.css
@@ -831,10 +831,14 @@ both the code example and the line numbers, so we need to remove the radius in t
 	-webkit-user-select: none;
 	user-select: none;
 	padding: 14px 8px;
+	padding-right: 2px;
 	color: var(--src-line-numbers-span-color);
 }
 
-.rustdoc .scraped-example .src-line-numbers {
+.rustdoc .scraped-example .example-wrap .src-line-numbers {
+	padding: 0;
+}
+.rustdoc .src-line-numbers pre {
 	padding: 14px 0;
 }
 .src-line-numbers a, .src-line-numbers span {
@@ -893,7 +897,7 @@ both the code example and the line numbers, so we need to remove the radius in t
 }
 
 .docblock code, .docblock-short code,
-pre, .rustdoc.src .example-wrap {
+pre, .rustdoc.src .example-wrap, .example-wrap .src-line-numbers {
 	background-color: var(--code-block-background-color);
 }
 

From 55058b6c744a945aea03345bc26611693547a7e5 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <guillaume.gomez@huawei.com>
Date: Fri, 20 Sep 2024 15:59:50 +0200
Subject: [PATCH 11/16] Add GUI tests for line numbers padding

---
 .../docblock-code-block-line-number.goml      | 63 ++++++++++++++++++-
 1 file changed, 62 insertions(+), 1 deletion(-)

diff --git a/tests/rustdoc-gui/docblock-code-block-line-number.goml b/tests/rustdoc-gui/docblock-code-block-line-number.goml
index 03f8f80b10d..fed916ac246 100644
--- a/tests/rustdoc-gui/docblock-code-block-line-number.goml
+++ b/tests/rustdoc-gui/docblock-code-block-line-number.goml
@@ -39,7 +39,10 @@ define-function: (
             {
                 "color": |color|,
                 "margin": "0px",
-                "padding": "14px 8px",
+                "padding-top": "14px",
+                "padding-bottom": "14px",
+                "padding-left": "8px",
+                "padding-right": "2px",
                 "text-align": "right",
                 // There should not be a radius on the right of the line numbers.
                 "border-top-left-radius": "6px",
@@ -141,3 +144,61 @@ assert-css: (
     },
     ALL,
 )
+
+// Checking line numbers on scraped code examples.
+go-to: "file://" + |DOC_PATH| + "/scrape_examples/fn.test_many.html"
+
+define-function: (
+    "check-padding",
+    [path, padding_bottom],
+    block {
+        assert-css: (|path| + " .src-line-numbers", {
+            "padding-top": "0px",
+            "padding-bottom": "0px",
+            "padding-left": "0px",
+            "padding-right": "0px",
+        })
+        assert-css: (|path| + " .src-line-numbers > pre", {
+            "padding-top": "14px",
+            "padding-bottom": |padding_bottom|,
+            "padding-left": "0px",
+            "padding-right": "0px",
+        })
+        assert-css: (|path| + " .src-line-numbers > pre > span", {
+            "padding-top": "0px",
+            "padding-bottom": "0px",
+            "padding-left": "8px",
+            "padding-right": "8px",
+        })
+    },
+)
+
+call-function: ("check-padding", {
+    "path": ".scraped-example .example-wrap",
+    "padding_bottom": "0px",
+})
+
+move-cursor-to: ".scraped-example .example-wrap .rust"
+wait-for: ".scraped-example .example-wrap .button-holder .expand"
+click: ".scraped-example .example-wrap .button-holder .expand"
+wait-for: ".scraped-example.expanded"
+
+call-function: ("check-padding", {
+    "path": ".scraped-example.expanded .example-wrap",
+    "padding_bottom": "14px",
+})
+
+// Now checking the line numbers in the source code page.
+click: ".src"
+assert-css: (".src-line-numbers", {
+    "padding-top": "20px",
+    "padding-bottom": "20px",
+    "padding-left": "4px",
+    "padding-right": "0px",
+})
+assert-css: (".src-line-numbers > a", {
+    "padding-top": "0px",
+    "padding-bottom": "0px",
+    "padding-left": "8px",
+    "padding-right": "8px",
+})

From 5a219cba9563dc869577f7b7a9e318a1565dd4da Mon Sep 17 00:00:00 2001
From: lcnr <rust@lcnr.de>
Date: Fri, 20 Sep 2024 14:17:08 +0000
Subject: [PATCH 12/16] bail if there are too many non-region infer vars

---
 .../src/solve/eval_ctxt/canonical.rs                       | 7 +++++++
 tests/ui/traits/coherence-alias-hang.rs                    | 2 ++
 2 files changed, 9 insertions(+)

diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs
index 5acfec3dee3..3887f9c529c 100644
--- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs
@@ -157,6 +157,13 @@ where
             },
         );
 
+        let num_non_region_vars = canonical.variables.iter().filter(|c| !c.is_region()).count();
+        if num_non_region_vars > self.cx().recursion_limit() {
+            return Ok(self.make_ambiguous_response_no_constraints(MaybeCause::Overflow {
+                suggest_increasing_limit: true,
+            }));
+        }
+
         Ok(canonical)
     }
 
diff --git a/tests/ui/traits/coherence-alias-hang.rs b/tests/ui/traits/coherence-alias-hang.rs
index 37b80739589..c2b4d2e42d2 100644
--- a/tests/ui/traits/coherence-alias-hang.rs
+++ b/tests/ui/traits/coherence-alias-hang.rs
@@ -1,4 +1,6 @@
 //@ check-pass
+//@ revisions: current next
+//[next]@ compile-flags: -Znext-solver
 
 // Regression test for nalgebra hang <https://github.com/rust-lang/rust/issues/130056>.
 

From 9613f487ae0e8c563caffad4ffdede3296002457 Mon Sep 17 00:00:00 2001
From: lcnr <rust@lcnr.de>
Date: Fri, 20 Sep 2024 14:49:03 +0000
Subject: [PATCH 13/16] add comment

---
 .../rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs  | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs
index 3887f9c529c..4d743c05190 100644
--- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs
@@ -157,6 +157,10 @@ where
             },
         );
 
+        // HACK: We bail with overflow if the response would have too many non-region
+        // inference variables. This tends to only happen if we encounter a lot of
+        // ambiguous alias types which get replaced with fresh inference variables
+        // during generalization. This prevents a hang in nalgebra.
         let num_non_region_vars = canonical.variables.iter().filter(|c| !c.is_region()).count();
         if num_non_region_vars > self.cx().recursion_limit() {
             return Ok(self.make_ambiguous_response_no_constraints(MaybeCause::Overflow {

From 340b38ed671fe55582045bd21441aaa59b452e8e Mon Sep 17 00:00:00 2001
From: Artyom Tetyukhin <51746822+arttet@users.noreply.github.com>
Date: Fri, 20 Sep 2024 18:53:09 +0400
Subject: [PATCH 14/16] Add arm64e-apple-tvos target

---
 compiler/rustc_target/src/spec/mod.rs         |  2 +
 .../src/spec/targets/arm64e_apple_tvos.rs     | 24 ++++++++++++
 src/doc/rustc/src/SUMMARY.md                  |  1 +
 src/doc/rustc/src/platform-support.md         |  3 +-
 .../src/platform-support/arm64e-apple-tvos.md | 37 +++++++++++++++++++
 src/tools/build-manifest/src/main.rs          |  5 ++-
 tests/assembly/targets/targets-macho.rs       |  3 ++
 7 files changed, 72 insertions(+), 3 deletions(-)
 create mode 100644 compiler/rustc_target/src/spec/targets/arm64e_apple_tvos.rs
 create mode 100644 src/doc/rustc/src/platform-support/arm64e-apple-tvos.md

diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index f12e3e595ad..1d478f84c43 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -1714,8 +1714,10 @@ supported_targets! {
     ("x86_64-apple-ios-macabi", x86_64_apple_ios_macabi),
     ("aarch64-apple-ios-macabi", aarch64_apple_ios_macabi),
     ("aarch64-apple-ios-sim", aarch64_apple_ios_sim),
+
     ("aarch64-apple-tvos", aarch64_apple_tvos),
     ("aarch64-apple-tvos-sim", aarch64_apple_tvos_sim),
+    ("arm64e-apple-tvos", arm64e_apple_tvos),
     ("x86_64-apple-tvos", x86_64_apple_tvos),
 
     ("armv7k-apple-watchos", armv7k_apple_watchos),
diff --git a/compiler/rustc_target/src/spec/targets/arm64e_apple_tvos.rs b/compiler/rustc_target/src/spec/targets/arm64e_apple_tvos.rs
new file mode 100644
index 00000000000..d4d66c92857
--- /dev/null
+++ b/compiler/rustc_target/src/spec/targets/arm64e_apple_tvos.rs
@@ -0,0 +1,24 @@
+use crate::spec::base::apple::{base, Arch, TargetAbi};
+use crate::spec::{FramePointer, Target, TargetOptions};
+
+pub(crate) fn target() -> Target {
+    let (opts, llvm_target, arch) = base("tvos", Arch::Arm64e, TargetAbi::Normal);
+    Target {
+        llvm_target,
+        metadata: crate::spec::TargetMetadata {
+            description: Some("ARM64e Apple tvOS".into()),
+            tier: Some(3),
+            host_tools: Some(false),
+            std: Some(true),
+        },
+        pointer_width: 64,
+        data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(),
+        arch,
+        options: TargetOptions {
+            features: "+neon,+fp-armv8,+apple-a12,+v8.3a,+pauth".into(),
+            max_atomic_width: Some(128),
+            frame_pointer: FramePointer::NonLeaf,
+            ..opts
+        },
+    }
+}
diff --git a/src/doc/rustc/src/SUMMARY.md b/src/doc/rustc/src/SUMMARY.md
index 3e199539694..b1a5bd604eb 100644
--- a/src/doc/rustc/src/SUMMARY.md
+++ b/src/doc/rustc/src/SUMMARY.md
@@ -25,6 +25,7 @@
         - [\*-apple-ios-macabi](platform-support/apple-ios-macabi.md)
         - [arm64e-apple-ios](platform-support/arm64e-apple-ios.md)
     - [\*-apple-tvos](platform-support/apple-tvos.md)
+        - [arm64e-apple-tvos](platform-support/arm64e-apple-tvos.md)
     - [\*-apple-watchos](platform-support/apple-watchos.md)
     - [\*-apple-visionos](platform-support/apple-visionos.md)
     - [aarch64-nintendo-switch-freestanding](platform-support/aarch64-nintendo-switch-freestanding.md)
diff --git a/src/doc/rustc/src/platform-support.md b/src/doc/rustc/src/platform-support.md
index 9a35b35af71..827a7065f3e 100644
--- a/src/doc/rustc/src/platform-support.md
+++ b/src/doc/rustc/src/platform-support.md
@@ -245,8 +245,9 @@ host tools.
 
 target | std | host | notes
 -------|:---:|:----:|-------
-[`arm64e-apple-ios`](platform-support/arm64e-apple-ios.md) | ✓ | | ARM64e Apple iOS
 [`arm64e-apple-darwin`](platform-support/arm64e-apple-darwin.md)  | ✓ | ✓ | ARM64e Apple Darwin
+[`arm64e-apple-ios`](platform-support/arm64e-apple-ios.md) | ✓ | | ARM64e Apple iOS
+[`arm64e-apple-tvos`](platform-support/arm64e-apple-tvos.md)  | ✓ | | ARM64e Apple tvOS
 [`aarch64-apple-tvos`](platform-support/apple-tvos.md) | ✓ |  | ARM64 tvOS
 [`aarch64-apple-tvos-sim`](platform-support/apple-tvos.md) | ✓ |  | ARM64 tvOS Simulator
 [`aarch64-apple-watchos`](platform-support/apple-watchos.md) | ✓ |  | ARM64 Apple WatchOS
diff --git a/src/doc/rustc/src/platform-support/arm64e-apple-tvos.md b/src/doc/rustc/src/platform-support/arm64e-apple-tvos.md
new file mode 100644
index 00000000000..d49383fb853
--- /dev/null
+++ b/src/doc/rustc/src/platform-support/arm64e-apple-tvos.md
@@ -0,0 +1,37 @@
+# `arm64e-apple-tvos`
+
+**Tier: 3**
+
+ARM64e tvOS (10.0+)
+
+## Target maintainers
+
+- Artyom Tetyukhin ([@arttet](https://github.com/https://github.com/arttet))
+
+## Requirements
+
+This target is cross-compiled and supports `std`.
+To build this target Xcode 12 or higher on macOS is required.
+
+## Building the target
+
+You can build Rust with support for the targets by adding it to the `target` list in `config.toml`:
+
+```toml
+[build]
+target = [ "arm64e-apple-tvos" ]
+```
+
+## Building Rust programs
+
+Rust does not yet ship pre-compiled artifacts for this target.
+To compile for this target, you will need to build Rust with the target enabled (see [Building the target](#building-the-target) above).
+
+## Testing
+
+The target does support running binaries on tvOS platforms with `arm64e` architecture.
+
+## Cross-compilation toolchains and C code
+
+The targets do support `C` code.
+To build compatible `C` code, you have to use XCode with the same compiler and flags.
diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
index 92b21f7dbaa..eb5faeeaf9a 100644
--- a/src/tools/build-manifest/src/main.rs
+++ b/src/tools/build-manifest/src/main.rs
@@ -51,9 +51,7 @@ static HOSTS: &[&str] = &[
 
 static TARGETS: &[&str] = &[
     "aarch64-apple-darwin",
-    "arm64e-apple-darwin",
     "aarch64-apple-ios",
-    "arm64e-apple-ios",
     "aarch64-apple-ios-macabi",
     "aarch64-apple-ios-sim",
     "aarch64-unknown-fuchsia",
@@ -68,6 +66,9 @@ static TARGETS: &[&str] = &[
     "aarch64-unknown-none-softfloat",
     "aarch64-unknown-redox",
     "aarch64-unknown-uefi",
+    "arm64e-apple-darwin",
+    "arm64e-apple-ios",
+    "arm64e-apple-tvos",
     "arm-linux-androideabi",
     "arm-unknown-linux-gnueabi",
     "arm-unknown-linux-gnueabihf",
diff --git a/tests/assembly/targets/targets-macho.rs b/tests/assembly/targets/targets-macho.rs
index 36145a82bda..8095ae9029b 100644
--- a/tests/assembly/targets/targets-macho.rs
+++ b/tests/assembly/targets/targets-macho.rs
@@ -18,6 +18,9 @@
 //@ revisions: aarch64_apple_tvos_sim
 //@ [aarch64_apple_tvos_sim] compile-flags: --target aarch64-apple-tvos-sim
 //@ [aarch64_apple_tvos_sim] needs-llvm-components: aarch64
+//@ revisions: arm64e_apple_tvos
+//@ [arm64e_apple_tvos] compile-flags: --target arm64e-apple-tvos
+//@ [arm64e_apple_tvos] needs-llvm-components: aarch64
 //@ revisions: aarch64_apple_watchos
 //@ [aarch64_apple_watchos] compile-flags: --target aarch64-apple-watchos
 //@ [aarch64_apple_watchos] needs-llvm-components: aarch64

From a6aeba83beac8173bcc1eb819441e9bdc54d0561 Mon Sep 17 00:00:00 2001
From: lcnr <rust@lcnr.de>
Date: Fri, 20 Sep 2024 14:57:58 +0000
Subject: [PATCH 15/16] update tests

---
 .../coinduction/fixpoint-exponential-growth.stderr    |  1 +
 .../overflow/recursion-limit-zero-issue-115351.rs     |  3 ++-
 .../overflow/recursion-limit-zero-issue-115351.stderr | 11 +++++++++++
 3 files changed, 14 insertions(+), 1 deletion(-)
 create mode 100644 tests/ui/traits/next-solver/overflow/recursion-limit-zero-issue-115351.stderr

diff --git a/tests/ui/traits/next-solver/cycles/coinduction/fixpoint-exponential-growth.stderr b/tests/ui/traits/next-solver/cycles/coinduction/fixpoint-exponential-growth.stderr
index 8d7d8cee08a..150100f2c53 100644
--- a/tests/ui/traits/next-solver/cycles/coinduction/fixpoint-exponential-growth.stderr
+++ b/tests/ui/traits/next-solver/cycles/coinduction/fixpoint-exponential-growth.stderr
@@ -4,6 +4,7 @@ error[E0275]: overflow evaluating the requirement `W<_>: Trait`
 LL |     impls::<W<_>>();
    |             ^^^^
    |
+   = help: consider increasing the recursion limit by adding a `#![recursion_limit = "256"]` attribute to your crate (`fixpoint_exponential_growth`)
 note: required by a bound in `impls`
   --> $DIR/fixpoint-exponential-growth.rs:30:13
    |
diff --git a/tests/ui/traits/next-solver/overflow/recursion-limit-zero-issue-115351.rs b/tests/ui/traits/next-solver/overflow/recursion-limit-zero-issue-115351.rs
index 1b80287d9da..86d428cc0f0 100644
--- a/tests/ui/traits/next-solver/overflow/recursion-limit-zero-issue-115351.rs
+++ b/tests/ui/traits/next-solver/overflow/recursion-limit-zero-issue-115351.rs
@@ -1,6 +1,7 @@
+//~ ERROR overflow evaluating the requirement `Self: Trait`
+//~^ ERROR overflow evaluating the requirement `Self well-formed`
 // This is a non-regression test for issue #115351, where a recursion limit of 0 caused an ICE.
 //@ compile-flags: -Znext-solver --crate-type=lib
-//@ check-pass
 
 #![recursion_limit = "0"]
 trait Trait {}
diff --git a/tests/ui/traits/next-solver/overflow/recursion-limit-zero-issue-115351.stderr b/tests/ui/traits/next-solver/overflow/recursion-limit-zero-issue-115351.stderr
new file mode 100644
index 00000000000..2eed7e8f723
--- /dev/null
+++ b/tests/ui/traits/next-solver/overflow/recursion-limit-zero-issue-115351.stderr
@@ -0,0 +1,11 @@
+error[E0275]: overflow evaluating the requirement `Self: Trait`
+   |
+   = help: consider increasing the recursion limit by adding a `#![recursion_limit = "2"]` attribute to your crate (`recursion_limit_zero_issue_115351`)
+
+error[E0275]: overflow evaluating the requirement `Self well-formed`
+   |
+   = help: consider increasing the recursion limit by adding a `#![recursion_limit = "2"]` attribute to your crate (`recursion_limit_zero_issue_115351`)
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0275`.

From ff86269368f942880e8a3bb4049ea3cd43b771ba Mon Sep 17 00:00:00 2001
From: "Tim (Theemathas) Chirananthavat" <theemathas@gmail.com>
Date: Fri, 20 Sep 2024 22:19:53 +0700
Subject: [PATCH 16/16] Add `Vec::as_non_null`

---
 library/alloc/src/vec/mod.rs | 71 +++++++++++++++++++++++++++++++++++-
 1 file changed, 69 insertions(+), 2 deletions(-)

diff --git a/library/alloc/src/vec/mod.rs b/library/alloc/src/vec/mod.rs
index 13b06584223..1984cfeefc1 100644
--- a/library/alloc/src/vec/mod.rs
+++ b/library/alloc/src/vec/mod.rs
@@ -1584,7 +1584,8 @@ impl<T, A: Allocator> Vec<T, A> {
     ///
     /// This method guarantees that for the purpose of the aliasing model, this method
     /// does not materialize a reference to the underlying slice, and thus the returned pointer
-    /// will remain valid when mixed with other calls to [`as_ptr`] and [`as_mut_ptr`].
+    /// will remain valid when mixed with other calls to [`as_ptr`], [`as_mut_ptr`],
+    /// and [`as_non_null`].
     /// Note that calling other methods that materialize mutable references to the slice,
     /// or mutable references to specific elements you are planning on accessing through this pointer,
     /// as well as writing to those elements, may still invalidate this pointer.
@@ -1621,6 +1622,7 @@ impl<T, A: Allocator> Vec<T, A> {
     ///
     /// [`as_mut_ptr`]: Vec::as_mut_ptr
     /// [`as_ptr`]: Vec::as_ptr
+    /// [`as_non_null`]: Vec::as_non_null
     #[stable(feature = "vec_as_ptr", since = "1.37.0")]
     #[rustc_never_returns_null_ptr]
     #[inline]
@@ -1640,7 +1642,8 @@ impl<T, A: Allocator> Vec<T, A> {
     ///
     /// This method guarantees that for the purpose of the aliasing model, this method
     /// does not materialize a reference to the underlying slice, and thus the returned pointer
-    /// will remain valid when mixed with other calls to [`as_ptr`] and [`as_mut_ptr`].
+    /// will remain valid when mixed with other calls to [`as_ptr`], [`as_mut_ptr`],
+    /// and [`as_non_null`].
     /// Note that calling other methods that materialize references to the slice,
     /// or references to specific elements you are planning on accessing through this pointer,
     /// may still invalidate this pointer.
@@ -1680,6 +1683,7 @@ impl<T, A: Allocator> Vec<T, A> {
     ///
     /// [`as_mut_ptr`]: Vec::as_mut_ptr
     /// [`as_ptr`]: Vec::as_ptr
+    /// [`as_non_null`]: Vec::as_non_null
     #[stable(feature = "vec_as_ptr", since = "1.37.0")]
     #[rustc_never_returns_null_ptr]
     #[inline]
@@ -1689,6 +1693,69 @@ impl<T, A: Allocator> Vec<T, A> {
         self.buf.ptr()
     }
 
+    /// Returns a `NonNull` pointer to the vector's buffer, or a dangling
+    /// `NonNull` pointer valid for zero sized reads if the vector didn't allocate.
+    ///
+    /// The caller must ensure that the vector outlives the pointer this
+    /// function returns, or else it will end up dangling.
+    /// Modifying the vector may cause its buffer to be reallocated,
+    /// which would also make any pointers to it invalid.
+    ///
+    /// This method guarantees that for the purpose of the aliasing model, this method
+    /// does not materialize a reference to the underlying slice, and thus the returned pointer
+    /// will remain valid when mixed with other calls to [`as_ptr`], [`as_mut_ptr`],
+    /// and [`as_non_null`].
+    /// Note that calling other methods that materialize references to the slice,
+    /// or references to specific elements you are planning on accessing through this pointer,
+    /// may still invalidate this pointer.
+    /// See the second example below for how this guarantee can be used.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(box_vec_non_null)]
+    ///
+    /// // Allocate vector big enough for 4 elements.
+    /// let size = 4;
+    /// let mut x: Vec<i32> = Vec::with_capacity(size);
+    /// let x_ptr = x.as_non_null();
+    ///
+    /// // Initialize elements via raw pointer writes, then set length.
+    /// unsafe {
+    ///     for i in 0..size {
+    ///         x_ptr.add(i).write(i as i32);
+    ///     }
+    ///     x.set_len(size);
+    /// }
+    /// assert_eq!(&*x, &[0, 1, 2, 3]);
+    /// ```
+    ///
+    /// Due to the aliasing guarantee, the following code is legal:
+    ///
+    /// ```rust
+    /// #![feature(box_vec_non_null)]
+    ///
+    /// unsafe {
+    ///     let mut v = vec![0];
+    ///     let ptr1 = v.as_non_null();
+    ///     ptr1.write(1);
+    ///     let ptr2 = v.as_non_null();
+    ///     ptr2.write(2);
+    ///     // Notably, the write to `ptr2` did *not* invalidate `ptr1`:
+    ///     ptr1.write(3);
+    /// }
+    /// ```
+    ///
+    /// [`as_mut_ptr`]: Vec::as_mut_ptr
+    /// [`as_ptr`]: Vec::as_ptr
+    /// [`as_non_null`]: Vec::as_non_null
+    #[unstable(feature = "box_vec_non_null", reason = "new API", issue = "130364")]
+    #[inline]
+    pub fn as_non_null(&mut self) -> NonNull<T> {
+        // SAFETY: A `Vec` always has a non-null pointer.
+        unsafe { NonNull::new_unchecked(self.as_mut_ptr()) }
+    }
+
     /// Returns a reference to the underlying allocator.
     #[unstable(feature = "allocator_api", issue = "32838")]
     #[inline]