nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2025-06-06 04:08:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rollup merge of #135383 - BoxyUwU:cov_tag_ptr, r=compiler-errors

Browse Source 
De-abstract tagged ptr and make it covariant

In #135272 I needed to use a tagged ptr in `hir::TyKind` in order to not regress hir type sizes. Unfortunately the existing `CopyTaggedPtr` abstraction is insufficient as it makes the `'hir` lifetime invariant.

I spent some time trying to keep existing functionality while making it covariant but in the end I realised that actually we dont use *any* of this code *anywhere* in rustc, so I've just removed everything and replaced it with a much less general abstraction that is suitable for what I need in #135272.

Idk if anyone has a preference for just keeping all the abstractions here in case anyone needs them in the future 🤷‍♀️
This commit is contained in:
Guillaume Gomez 2025-01-12 23:08:58 +01:00 committed by GitHub
commit fad3039124
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 309 additions and 1003 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
compiler/rustc_data_structures/src/marker.rs
View File

@ -72,7 +72,7 @@ impl_dyn_send!(
[Vec<T, A> where T: DynSend, A: std::alloc::Allocator + DynSend] [Vec<T, A> where T: DynSend, A: std::alloc::Allocator + DynSend]
[Box<T, A> where T: ?Sized + DynSend, A: std::alloc::Allocator + DynSend] [Box<T, A> where T: ?Sized + DynSend, A: std::alloc::Allocator + DynSend]
[crate::sync::RwLock<T> where T: DynSend] [crate::sync::RwLock<T> where T: DynSend]
[crate::tagged_ptr::CopyTaggedPtr<P, T, CP> where P: Send + crate::tagged_ptr::Pointer, T: Send + crate::tagged_ptr::Tag, const CP: bool] [crate::tagged_ptr::TaggedRef<'a, P, T> where 'a, P: Sync, T: Send + crate::tagged_ptr::Tag]
[rustc_arena::TypedArena<T> where T: DynSend] [rustc_arena::TypedArena<T> where T: DynSend]
[indexmap::IndexSet<V, S> where V: DynSend, S: DynSend] [indexmap::IndexSet<V, S> where V: DynSend, S: DynSend]
[indexmap::IndexMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend] [indexmap::IndexMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend]
@ -148,7 +148,7 @@ impl_dyn_sync!(
[crate::sync::RwLock<T> where T: DynSend + DynSync] [crate::sync::RwLock<T> where T: DynSend + DynSync]
[crate::sync::WorkerLocal<T> where T: DynSend] [crate::sync::WorkerLocal<T> where T: DynSend]
[crate::intern::Interned<'a, T> where 'a, T: DynSync] [crate::intern::Interned<'a, T> where 'a, T: DynSync]
[crate::tagged_ptr::CopyTaggedPtr<P, T, CP> where P: Sync + crate::tagged_ptr::Pointer, T: Sync + crate::tagged_ptr::Tag, const CP: bool] [crate::tagged_ptr::TaggedRef<'a, P, T> where 'a, P: Sync, T: Sync + crate::tagged_ptr::Tag]
[parking_lot::lock_api::Mutex<R, T> where R: DynSync, T: ?Sized + DynSend] [parking_lot::lock_api::Mutex<R, T> where R: DynSync, T: ?Sized + DynSend]
[parking_lot::lock_api::RwLock<R, T> where R: DynSync, T: ?Sized + DynSend + DynSync] [parking_lot::lock_api::RwLock<R, T> where R: DynSync, T: ?Sized + DynSend + DynSync]
[indexmap::IndexSet<V, S> where V: DynSync, S: DynSync] [indexmap::IndexSet<V, S> where V: DynSync, S: DynSync]

393
compiler/rustc_data_structures/src/tagged_ptr.rs
View File

@ -1,116 +1,26 @@
//! This module implements tagged pointers. //! This module implements tagged pointers. In order to utilize the pointer
//! packing, you must have a tag type implementing the [`Tag`] trait.
//! //!
//! In order to utilize the pointer packing, you must have two types: a pointer, //! We assert that the tag and the reference type is compatible at compile
//! and a tag.
//!
//! The pointer must implement the [`Pointer`] trait, with the primary
//! requirement being convertible to and from a raw pointer. Note that the
//! pointer must be dereferenceable, so raw pointers generally cannot implement
//! the [`Pointer`] trait. This implies that the pointer must also be non-null.
//!
//! Many common pointer types already implement the [`Pointer`] trait.
//!
//! The tag must implement the [`Tag`] trait.
//!
//! We assert that the tag and the [`Pointer`] types are compatible at compile
//! time. //! time.
use std::fmt;
use std::hash::{Hash, Hasher};
use std::marker::PhantomData;
use std::num::NonZero;
use std::ops::Deref; use std::ops::Deref;
use std::ptr::NonNull; use std::ptr::NonNull;
use std::rc::Rc;
use std::sync::Arc;
use crate::aligned::Aligned; use crate::aligned::Aligned;
use crate::stable_hasher::{HashStable, StableHasher};
mod copy; /// This describes tags that the [`TaggedRef`] struct can hold.
mod drop;
mod impl_tag;
pub use copy::CopyTaggedPtr;
pub use drop::TaggedPtr;
/// This describes the pointer type encapsulated by [`TaggedPtr`] and
/// [`CopyTaggedPtr`].
/// ///
/// # Safety /// # Safety
/// ///
/// The pointer returned from [`into_ptr`] must be a [valid], pointer to /// - The [`BITS`] constant must be correct.
/// [`<Self as Deref>::Target`]. /// - No more than [`BITS`] least-significant bits may be set in the returned usize.
/// /// - [`Eq`] and [`Hash`] must be implementable with the returned `usize` from `into_usize`.
/// Note that if `Self` implements [`DerefMut`] the pointer returned from
/// [`into_ptr`] must be valid for writes (and thus calling [`NonNull::as_mut`]
/// on it must be safe).
///
/// The [`BITS`] constant must be correct. [`BITS`] least-significant bits,
/// must be zero on all pointers returned from [`into_ptr`].
///
/// For example, if the alignment of [`Self::Target`] is 2, then `BITS` should be 1.
///
/// [`BITS`]: Pointer::BITS
/// [`into_ptr`]: Pointer::into_ptr
/// [valid]: std::ptr#safety
/// [`<Self as Deref>::Target`]: Deref::Target
/// [`Self::Target`]: Deref::Target
/// [`DerefMut`]: std::ops::DerefMut
pub unsafe trait Pointer: Deref {
/// Number of unused (always zero) **least-significant bits** in this
/// pointer, usually related to the pointees alignment.
///
/// For example if [`BITS`] = `2`, then given `ptr = Self::into_ptr(..)`,
/// `ptr.addr() & 0b11 == 0` must be true.
///
/// Most likely the value you want to use here is the following, unless
/// your [`Self::Target`] type is unsized (e.g., `ty::List<T>` in rustc)
/// or your pointer is over/under aligned, in which case you'll need to
/// manually figure out what the right type to pass to [`bits_for`] is, or
/// what the value to set here.
///
/// ```rust
/// # use std::ops::Deref;
/// # use rustc_data_structures::tagged_ptr::bits_for;
/// # struct T;
/// # impl Deref for T { type Target = u8; fn deref(&self) -> &u8 { &0 } }
/// # impl T {
/// const BITS: u32 = bits_for::<<Self as Deref>::Target>();
/// # }
/// ```
///
/// [`BITS`]: Pointer::BITS
/// [`Self::Target`]: Deref::Target
const BITS: u32;
/// Turns this pointer into a raw, non-null pointer.
///
/// The inverse of this function is [`from_ptr`].
///
/// This function guarantees that the least-significant [`Self::BITS`] bits
/// are zero.
///
/// [`from_ptr`]: Pointer::from_ptr
/// [`Self::BITS`]: Pointer::BITS
fn into_ptr(self) -> NonNull<Self::Target>;
/// Re-creates the original pointer, from a raw pointer returned by [`into_ptr`].
///
/// # Safety
///
/// The passed `ptr` must be returned from [`into_ptr`].
///
/// This acts as [`ptr::read::<Self>()`] semantically, it should not be called more than
/// once on non-[`Copy`] `Pointer`s.
///
/// [`into_ptr`]: Pointer::into_ptr
/// [`ptr::read::<Self>()`]: std::ptr::read
unsafe fn from_ptr(ptr: NonNull<Self::Target>) -> Self;
}
/// This describes tags that the [`TaggedPtr`] struct can hold.
///
/// # Safety
///
/// The [`BITS`] constant must be correct.
///
/// No more than [`BITS`] least-significant bits may be set in the returned usize.
/// ///
/// [`BITS`]: Tag::BITS /// [`BITS`]: Tag::BITS
pub unsafe trait Tag: Copy { pub unsafe trait Tag: Copy {
@ -166,118 +76,217 @@ pub const fn bits_for_tags(mut tags: &[usize]) -> u32 {
bits bits
} }
unsafe impl<T: ?Sized + Aligned> Pointer for Box<T> { /// A covariant [`Copy`] tagged borrow. This is essentially `{ pointer: &'a P, tag: T }` packed
const BITS: u32 = bits_for::<Self::Target>(); /// in a single reference.
pub struct TaggedRef<'a, Pointee: Aligned + ?Sized, T: Tag> {
#[inline] /// This is semantically a pair of `pointer: &'a P` and `tag: T` fields,
fn into_ptr(self) -> NonNull<T> { /// however we pack them in a single pointer, to save space.
// Safety: pointers from `Box::into_raw` are valid & non-null ///
unsafe { NonNull::new_unchecked(Box::into_raw(self)) } /// We pack the tag into the **most**-significant bits of the pointer to
} /// ease retrieval of the value. A left shift is a multiplication and
/// those are embeddable in instruction encoding, for example:
#[inline] ///
unsafe fn from_ptr(ptr: NonNull<T>) -> Self { /// ```asm
// Safety: `ptr` comes from `into_ptr` which calls `Box::into_raw` /// // (<https://godbolt.org/z/jqcYPWEr3>)
unsafe { Box::from_raw(ptr.as_ptr()) } /// example::shift_read3:
} /// mov eax, dword ptr [8*rdi]
/// ret
///
/// example::mask_read3:
/// and rdi, -8
/// mov eax, dword ptr [rdi]
/// ret
/// ```
///
/// This is ASM outputted by rustc for reads of values behind tagged
/// pointers for different approaches of tagging:
/// - `shift_read3` uses `<< 3` (the tag is in the most-significant bits)
/// - `mask_read3` uses `& !0b111` (the tag is in the least-significant bits)
///
/// The shift approach thus produces less instructions and is likely faster
/// (see <https://godbolt.org/z/Y913sMdWb>).
///
/// Encoding diagram:
/// ```text
/// [ packed.addr ]
/// [ tag ] [ pointer.addr >> T::BITS ] <-- usize::BITS - T::BITS bits
/// ^
/// |
/// T::BITS bits
/// ```
///
/// The tag can be retrieved by `packed.addr() >> T::BITS` and the pointer
/// can be retrieved by `packed.map_addr(|addr| addr << T::BITS)`.
packed: NonNull<Pointee>,
tag_pointer_ghost: PhantomData<(&'a Pointee, T)>,
} }
unsafe impl<T: ?Sized + Aligned> Pointer for Rc<T> { impl<'a, P, T> TaggedRef<'a, P, T>
const BITS: u32 = bits_for::<Self::Target>(); where
P: Aligned + ?Sized,
T: Tag,
{
/// Tags `pointer` with `tag`.
///
/// [`TaggedRef`]: crate::tagged_ptr::TaggedRef
#[inline] #[inline]
fn into_ptr(self) -> NonNull<T> { pub fn new(pointer: &'a P, tag: T) -> Self {
// Safety: pointers from `Rc::into_raw` are valid & non-null Self { packed: Self::pack(NonNull::from(pointer), tag), tag_pointer_ghost: PhantomData }
unsafe { NonNull::new_unchecked(Rc::into_raw(self).cast_mut()) }
} }
/// Retrieves the pointer.
#[inline] #[inline]
unsafe fn from_ptr(ptr: NonNull<T>) -> Self { pub fn pointer(self) -> &'a P {
// Safety: `ptr` comes from `into_ptr` which calls `Rc::into_raw` // SAFETY: pointer_raw returns the original pointer
unsafe { Rc::from_raw(ptr.as_ptr()) } unsafe { self.pointer_raw().as_ref() }
}
}
unsafe impl<T: ?Sized + Aligned> Pointer for Arc<T> {
const BITS: u32 = bits_for::<Self::Target>();
#[inline]
fn into_ptr(self) -> NonNull<T> {
// Safety: pointers from `Arc::into_raw` are valid & non-null
unsafe { NonNull::new_unchecked(Arc::into_raw(self).cast_mut()) }
} }
/// Retrieves the tag.
#[inline] #[inline]
unsafe fn from_ptr(ptr: NonNull<T>) -> Self { pub fn tag(&self) -> T {
// Safety: `ptr` comes from `into_ptr` which calls `Arc::into_raw` // Unpack the tag, according to the `self.packed` encoding scheme
unsafe { Arc::from_raw(ptr.as_ptr()) } let tag = self.packed.addr().get() >> Self::TAG_BIT_SHIFT;
}
}
unsafe impl<'a, T: 'a + ?Sized + Aligned> Pointer for &'a T {
const BITS: u32 = bits_for::<Self::Target>();
#[inline]
fn into_ptr(self) -> NonNull<T> {
NonNull::from(self)
}
#[inline]
unsafe fn from_ptr(ptr: NonNull<T>) -> Self {
// Safety: // Safety:
// `ptr` comes from `into_ptr` which gets the pointer from a reference // The shift retrieves the original value from `T::into_usize`,
unsafe { ptr.as_ref() } // satisfying `T::from_usize`'s preconditions.
unsafe { T::from_usize(tag) }
}
/// Sets the tag to a new value.
#[inline]
pub fn set_tag(&mut self, tag: T) {
self.packed = Self::pack(self.pointer_raw(), tag);
}
const TAG_BIT_SHIFT: u32 = usize::BITS - T::BITS;
const ASSERTION: () = { assert!(T::BITS <= bits_for::<P>()) };
/// Pack pointer `ptr` with a `tag`, according to `self.packed` encoding scheme.
#[inline]
fn pack(ptr: NonNull<P>, tag: T) -> NonNull<P> {
// Trigger assert!
let () = Self::ASSERTION;
let packed_tag = tag.into_usize() << Self::TAG_BIT_SHIFT;
ptr.map_addr(|addr| {
// Safety:
// - The pointer is `NonNull` => it's address is `NonZero<usize>`
// - `P::BITS` least significant bits are always zero (`Pointer` contract)
// - `T::BITS <= P::BITS` (from `Self::ASSERTION`)
//
// Thus `addr >> T::BITS` is guaranteed to be non-zero.
//
// `{non_zero} | packed_tag` can't make the value zero.
let packed = (addr.get() >> T::BITS) | packed_tag;
unsafe { NonZero::new_unchecked(packed) }
})
}
/// Retrieves the original raw pointer from `self.packed`.
#[inline]
pub(super) fn pointer_raw(&self) -> NonNull<P> {
self.packed.map_addr(|addr| unsafe { NonZero::new_unchecked(addr.get() << T::BITS) })
} }
} }
unsafe impl<'a, T: 'a + ?Sized + Aligned> Pointer for &'a mut T { impl<P, T> Copy for TaggedRef<'_, P, T>
const BITS: u32 = bits_for::<Self::Target>(); where
P: Aligned + ?Sized,
T: Tag,
{
}
impl<P, T> Clone for TaggedRef<'_, P, T>
where
P: Aligned + ?Sized,
T: Tag,
{
#[inline]
fn clone(&self) -> Self {
*self
}
}
impl<P, T> Deref for TaggedRef<'_, P, T>
where
P: Aligned + ?Sized,
T: Tag,
{
type Target = P;
#[inline] #[inline]
fn into_ptr(self) -> NonNull<T> { fn deref(&self) -> &Self::Target {
NonNull::from(self) self.pointer()
} }
}
impl<P, T> fmt::Debug for TaggedRef<'_, P, T>
where
P: Aligned + fmt::Debug + ?Sized,
T: Tag + fmt::Debug,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("TaggedRef")
.field("pointer", &self.pointer())
.field("tag", &self.tag())
.finish()
}
}
impl<P, T> PartialEq for TaggedRef<'_, P, T>
where
P: Aligned + ?Sized,
T: Tag,
{
#[inline] #[inline]
unsafe fn from_ptr(mut ptr: NonNull<T>) -> Self { #[allow(ambiguous_wide_pointer_comparisons)]
// Safety: fn eq(&self, other: &Self) -> bool {
// `ptr` comes from `into_ptr` which gets the pointer from a reference self.packed == other.packed
unsafe { ptr.as_mut() }
} }
} }
/// A tag type used in [`CopyTaggedPtr`] and [`TaggedPtr`] tests. impl<P, T: Tag> Eq for TaggedRef<'_, P, T> {}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
#[cfg(test)] impl<P, T: Tag> Hash for TaggedRef<'_, P, T> {
enum Tag2 { #[inline]
B00 = 0b00, fn hash<H: Hasher>(&self, state: &mut H) {
B01 = 0b01, self.packed.hash(state);
B10 = 0b10, }
B11 = 0b11, }
impl<'a, P, T, HCX> HashStable<HCX> for TaggedRef<'a, P, T>
where
P: HashStable<HCX> + Aligned + ?Sized,
T: Tag + HashStable<HCX>,
{
fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) {
self.pointer().hash_stable(hcx, hasher);
self.tag().hash_stable(hcx, hasher);
}
}
// Safety:
// `TaggedRef<P, T, ..>` is semantically just `{ ptr: P, tag: T }`, as such
// it's ok to implement `Sync` as long as `P: Sync, T: Sync`
unsafe impl<P, T> Sync for TaggedRef<'_, P, T>
where
P: Sync + Aligned + ?Sized,
T: Sync + Tag,
{
}
// Safety:
// `TaggedRef<P, T, ..>` is semantically just `{ ptr: P, tag: T }`, as such
// it's ok to implement `Send` as long as `P: Send, T: Send`
unsafe impl<P, T> Send for TaggedRef<'_, P, T>
where
P: Sync + Aligned + ?Sized,
T: Send + Tag,
{
} }
#[cfg(test)] #[cfg(test)]
unsafe impl Tag for Tag2 { mod tests;
const BITS: u32 = 2;
fn into_usize(self) -> usize {
self as _
}
unsafe fn from_usize(tag: usize) -> Self {
match tag {
0b00 => Tag2::B00,
0b01 => Tag2::B01,
0b10 => Tag2::B10,
0b11 => Tag2::B11,
_ => unreachable!(),
}
}
}
#[cfg(test)]
impl<HCX> crate::stable_hasher::HashStable<HCX> for Tag2 {
fn hash_stable(&self, hcx: &mut HCX, hasher: &mut crate::stable_hasher::StableHasher) {
(*self as u8).hash_stable(hcx, hasher);
}
}

330
compiler/rustc_data_structures/src/tagged_ptr/copy.rs
View File

@ -1,330 +0,0 @@
use std::fmt;
use std::hash::{Hash, Hasher};
use std::marker::PhantomData;
use std::mem::ManuallyDrop;
use std::num::NonZero;
use std::ops::{Deref, DerefMut};
use std::ptr::NonNull;
use super::{Pointer, Tag};
use crate::stable_hasher::{HashStable, StableHasher};
/// A [`Copy`] tagged pointer.
///
/// This is essentially `{ pointer: P, tag: T }` packed in a single pointer.
///
/// You should use this instead of the [`TaggedPtr`] type in all cases where
/// `P` implements [`Copy`].
///
/// If `COMPARE_PACKED` is true, then the pointers will be compared and hashed without
/// unpacking. Otherwise we don't implement [`PartialEq`], [`Eq`] and [`Hash`];
/// if you want that, wrap the [`CopyTaggedPtr`].
///
/// [`TaggedPtr`]: crate::tagged_ptr::TaggedPtr
pub struct CopyTaggedPtr<P, T, const COMPARE_PACKED: bool>
where
P: Pointer,
T: Tag,
{
/// This is semantically a pair of `pointer: P` and `tag: T` fields,
/// however we pack them in a single pointer, to save space.
///
/// We pack the tag into the **most**-significant bits of the pointer to
/// ease retrieval of the value. A left shift is a multiplication and
/// those are embeddable in instruction encoding, for example:
///
/// ```asm
/// // (<https://godbolt.org/z/jqcYPWEr3>)
/// example::shift_read3:
/// mov eax, dword ptr [8*rdi]
/// ret
///
/// example::mask_read3:
/// and rdi, -8
/// mov eax, dword ptr [rdi]
/// ret
/// ```
///
/// This is ASM outputted by rustc for reads of values behind tagged
/// pointers for different approaches of tagging:
/// - `shift_read3` uses `<< 3` (the tag is in the most-significant bits)
/// - `mask_read3` uses `& !0b111` (the tag is in the least-significant bits)
///
/// The shift approach thus produces less instructions and is likely faster
/// (see <https://godbolt.org/z/Y913sMdWb>).
///
/// Encoding diagram:
/// ```text
/// [ packed.addr ]
/// [ tag ] [ pointer.addr >> T::BITS ] <-- usize::BITS - T::BITS bits
/// ^
/// |
/// T::BITS bits
/// ```
///
/// The tag can be retrieved by `packed.addr() >> T::BITS` and the pointer
/// can be retrieved by `packed.map_addr(|addr| addr << T::BITS)`.
packed: NonNull<P::Target>,
tag_ghost: PhantomData<T>,
}
// Note that even though `CopyTaggedPtr` is only really expected to work with
// `P: Copy`, can't add `P: Copy` bound, because `CopyTaggedPtr` is used in the
// `TaggedPtr`'s implementation.
impl<P, T, const CP: bool> CopyTaggedPtr<P, T, CP>
where
P: Pointer,
T: Tag,
{
/// Tags `pointer` with `tag`.
///
/// Note that this leaks `pointer`: it won't be dropped when
/// `CopyTaggedPtr` is dropped. If you have a pointer with a significant
/// drop, use [`TaggedPtr`] instead.
///
/// [`TaggedPtr`]: crate::tagged_ptr::TaggedPtr
#[inline]
pub fn new(pointer: P, tag: T) -> Self {
Self { packed: Self::pack(P::into_ptr(pointer), tag), tag_ghost: PhantomData }
}
/// Retrieves the pointer.
#[inline]
pub fn pointer(self) -> P
where
P: Copy,
{
// SAFETY: pointer_raw returns the original pointer
//
// Note that this isn't going to double-drop or anything because we have
// P: Copy
unsafe { P::from_ptr(self.pointer_raw()) }
}
/// Retrieves the tag.
#[inline]
pub fn tag(&self) -> T {
// Unpack the tag, according to the `self.packed` encoding scheme
let tag = self.packed.addr().get() >> Self::TAG_BIT_SHIFT;
// Safety:
// The shift retrieves the original value from `T::into_usize`,
// satisfying `T::from_usize`'s preconditions.
unsafe { T::from_usize(tag) }
}
/// Sets the tag to a new value.
#[inline]
pub fn set_tag(&mut self, tag: T) {
self.packed = Self::pack(self.pointer_raw(), tag);
}
const TAG_BIT_SHIFT: u32 = usize::BITS - T::BITS;
const ASSERTION: () = { assert!(T::BITS <= P::BITS) };
/// Pack pointer `ptr` that comes from [`P::into_ptr`] with a `tag`,
/// according to `self.packed` encoding scheme.
///
/// [`P::into_ptr`]: Pointer::into_ptr
#[inline]
fn pack(ptr: NonNull<P::Target>, tag: T) -> NonNull<P::Target> {
// Trigger assert!
let () = Self::ASSERTION;
let packed_tag = tag.into_usize() << Self::TAG_BIT_SHIFT;
ptr.map_addr(|addr| {
// Safety:
// - The pointer is `NonNull` => it's address is `NonZero<usize>`
// - `P::BITS` least significant bits are always zero (`Pointer` contract)
// - `T::BITS <= P::BITS` (from `Self::ASSERTION`)
//
// Thus `addr >> T::BITS` is guaranteed to be non-zero.
//
// `{non_zero} | packed_tag` can't make the value zero.
let packed = (addr.get() >> T::BITS) | packed_tag;
unsafe { NonZero::new_unchecked(packed) }
})
}
/// Retrieves the original raw pointer from `self.packed`.
#[inline]
pub(super) fn pointer_raw(&self) -> NonNull<P::Target> {
self.packed.map_addr(|addr| unsafe { NonZero::new_unchecked(addr.get() << T::BITS) })
}
/// This provides a reference to the `P` pointer itself, rather than the
/// `Deref::Target`. It is used for cases where we want to call methods
/// that may be implement differently for the Pointer than the Pointee
/// (e.g., `Rc::clone` vs cloning the inner value).
pub(super) fn with_pointer_ref<R>(&self, f: impl FnOnce(&P) -> R) -> R {
// Safety:
// - `self.raw.pointer_raw()` is originally returned from `P::into_ptr`
// and as such is valid for `P::from_ptr`.
// - This also allows us to not care whatever `f` panics or not.
// - Even though we create a copy of the pointer, we store it inside
// `ManuallyDrop` and only access it by-ref, so we don't double-drop.
//
// Semantically this is just `f(&self.pointer)` (where `self.pointer`
// is non-packed original pointer).
//
// Note that even though `CopyTaggedPtr` is only really expected to
// work with `P: Copy`, we have to assume `P: ?Copy`, because
// `CopyTaggedPtr` is used in the `TaggedPtr`'s implementation.
let ptr = unsafe { ManuallyDrop::new(P::from_ptr(self.pointer_raw())) };
f(&ptr)
}
}
impl<P, T, const CP: bool> Copy for CopyTaggedPtr<P, T, CP>
where
P: Pointer + Copy,
T: Tag,
{
}
impl<P, T, const CP: bool> Clone for CopyTaggedPtr<P, T, CP>
where
P: Pointer + Copy,
T: Tag,
{
#[inline]
fn clone(&self) -> Self {
*self
}
}
impl<P, T, const CP: bool> Deref for CopyTaggedPtr<P, T, CP>
where
P: Pointer,
T: Tag,
{
type Target = P::Target;
#[inline]
fn deref(&self) -> &Self::Target {
// Safety:
// `pointer_raw` returns the original pointer from `P::into_ptr` which,
// by the `Pointer`'s contract, must be valid.
unsafe { self.pointer_raw().as_ref() }
}
}
impl<P, T, const CP: bool> DerefMut for CopyTaggedPtr<P, T, CP>
where
P: Pointer + DerefMut,
T: Tag,
{
#[inline]
fn deref_mut(&mut self) -> &mut Self::Target {
// Safety:
// `pointer_raw` returns the original pointer from `P::into_ptr` which,
// by the `Pointer`'s contract, must be valid for writes if
// `P: DerefMut`.
unsafe { self.pointer_raw().as_mut() }
}
}
impl<P, T, const CP: bool> fmt::Debug for CopyTaggedPtr<P, T, CP>
where
P: Pointer + fmt::Debug,
T: Tag + fmt::Debug,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.with_pointer_ref(|ptr| {
f.debug_struct("CopyTaggedPtr").field("pointer", ptr).field("tag", &self.tag()).finish()
})
}
}
impl<P, T> PartialEq for CopyTaggedPtr<P, T, true>
where
P: Pointer,
T: Tag,
{
#[inline]
#[allow(ambiguous_wide_pointer_comparisons)]
fn eq(&self, other: &Self) -> bool {
self.packed == other.packed
}
}
impl<P, T> Eq for CopyTaggedPtr<P, T, true>
where
P: Pointer,
T: Tag,
{
}
impl<P, T> Hash for CopyTaggedPtr<P, T, true>
where
P: Pointer,
T: Tag,
{
#[inline]
fn hash<H: Hasher>(&self, state: &mut H) {
self.packed.hash(state);
}
}
impl<P, T, HCX, const CP: bool> HashStable<HCX> for CopyTaggedPtr<P, T, CP>
where
P: Pointer + HashStable<HCX>,
T: Tag + HashStable<HCX>,
{
fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) {
self.with_pointer_ref(|ptr| ptr.hash_stable(hcx, hasher));
self.tag().hash_stable(hcx, hasher);
}
}
// Safety:
// `CopyTaggedPtr<P, T, ..>` is semantically just `{ ptr: P, tag: T }`, as such
// it's ok to implement `Sync` as long as `P: Sync, T: Sync`
unsafe impl<P, T, const CP: bool> Sync for CopyTaggedPtr<P, T, CP>
where
P: Sync + Pointer,
T: Sync + Tag,
{
}
// Safety:
// `CopyTaggedPtr<P, T, ..>` is semantically just `{ ptr: P, tag: T }`, as such
// it's ok to implement `Send` as long as `P: Send, T: Send`
unsafe impl<P, T, const CP: bool> Send for CopyTaggedPtr<P, T, CP>
where
P: Send + Pointer,
T: Send + Tag,
{
}
/// Test that `new` does not compile if there is not enough alignment for the
/// tag in the pointer.
///
/// ```compile_fail,E0080
/// use rustc_data_structures::tagged_ptr::{CopyTaggedPtr, Tag};
///
/// #[derive(Copy, Clone, Debug, PartialEq, Eq)]
/// enum Tag2 { B00 = 0b00, B01 = 0b01, B10 = 0b10, B11 = 0b11 };
///
/// unsafe impl Tag for Tag2 {
/// const BITS: u32 = 2;
///
/// fn into_usize(self) -> usize { todo!() }
/// unsafe fn from_usize(tag: usize) -> Self { todo!() }
/// }
///
/// let value = 12u16;
/// let reference = &value;
/// let tag = Tag2::B01;
///
/// let _ptr = CopyTaggedPtr::<_, _, true>::new(reference, tag);
/// ```
// For some reason miri does not get the compile error
// probably it `check`s instead of `build`ing?
#[cfg(not(miri))]
const _: () = ();
#[cfg(test)]
mod tests;

50
compiler/rustc_data_structures/src/tagged_ptr/copy/tests.rs
View File

@ -1,50 +0,0 @@
use std::ptr;
use crate::hashes::Hash128;
use crate::stable_hasher::{HashStable, StableHasher};
use crate::tagged_ptr::{CopyTaggedPtr, Pointer, Tag, Tag2};
#[test]
fn smoke() {
let value = 12u32;
let reference = &value;
let tag = Tag2::B01;
let ptr = tag_ptr(reference, tag);
assert_eq!(ptr.tag(), tag);
assert_eq!(*ptr, 12);
assert!(ptr::eq(ptr.pointer(), reference));
let copy = ptr;
let mut ptr = ptr;
ptr.set_tag(Tag2::B00);
assert_eq!(ptr.tag(), Tag2::B00);
assert_eq!(copy.tag(), tag);
assert_eq!(*copy, 12);
assert!(ptr::eq(copy.pointer(), reference));
}
#[test]
fn stable_hash_hashes_as_tuple() {
let hash_packed = {
let mut hasher = StableHasher::new();
tag_ptr(&12, Tag2::B11).hash_stable(&mut (), &mut hasher);
hasher.finish::<Hash128>()
};
let hash_tupled = {
let mut hasher = StableHasher::new();
(&12, Tag2::B11).hash_stable(&mut (), &mut hasher);
hasher.finish::<Hash128>()
};
assert_eq!(hash_packed, hash_tupled);
}
/// Helper to create tagged pointers without specifying `COMPARE_PACKED` if it does not matter.
fn tag_ptr<P: Pointer, T: Tag>(ptr: P, tag: T) -> CopyTaggedPtr<P, T, true> {
CopyTaggedPtr::new(ptr, tag)
}

178
compiler/rustc_data_structures/src/tagged_ptr/drop.rs
View File

@ -1,178 +0,0 @@
use std::fmt;
use std::hash::{Hash, Hasher};
use std::ops::{Deref, DerefMut};
use super::{CopyTaggedPtr, Pointer, Tag};
use crate::stable_hasher::{HashStable, StableHasher};
/// A tagged pointer that supports pointers that implement [`Drop`].
///
/// This is essentially `{ pointer: P, tag: T }` packed in a single pointer.
///
/// You should use [`CopyTaggedPtr`] instead of the this type in all cases
/// where `P` implements [`Copy`].
///
/// If `COMPARE_PACKED` is true, then the pointers will be compared and hashed without
/// unpacking. Otherwise we don't implement [`PartialEq`], [`Eq`] and [`Hash`];
/// if you want that, wrap the [`TaggedPtr`].
pub struct TaggedPtr<P, T, const COMPARE_PACKED: bool>
where
P: Pointer,
T: Tag,
{
raw: CopyTaggedPtr<P, T, COMPARE_PACKED>,
}
impl<P, T, const CP: bool> TaggedPtr<P, T, CP>
where
P: Pointer,
T: Tag,
{
/// Tags `pointer` with `tag`.
#[inline]
pub fn new(pointer: P, tag: T) -> Self {
TaggedPtr { raw: CopyTaggedPtr::new(pointer, tag) }
}
/// Retrieves the tag.
#[inline]
pub fn tag(&self) -> T {
self.raw.tag()
}
/// Sets the tag to a new value.
#[inline]
pub fn set_tag(&mut self, tag: T) {
self.raw.set_tag(tag)
}
}
impl<P, T, const CP: bool> Clone for TaggedPtr<P, T, CP>
where
P: Pointer + Clone,
T: Tag,
{
fn clone(&self) -> Self {
let ptr = self.raw.with_pointer_ref(P::clone);
Self::new(ptr, self.tag())
}
}
impl<P, T, const CP: bool> Deref for TaggedPtr<P, T, CP>
where
P: Pointer,
T: Tag,
{
type Target = P::Target;
#[inline]
fn deref(&self) -> &Self::Target {
self.raw.deref()
}
}
impl<P, T, const CP: bool> DerefMut for TaggedPtr<P, T, CP>
where
P: Pointer + DerefMut,
T: Tag,
{
#[inline]
fn deref_mut(&mut self) -> &mut Self::Target {
self.raw.deref_mut()
}
}
impl<P, T, const CP: bool> Drop for TaggedPtr<P, T, CP>
where
P: Pointer,
T: Tag,
{
fn drop(&mut self) {
// No need to drop the tag, as it's Copy
unsafe {
drop(P::from_ptr(self.raw.pointer_raw()));
}
}
}
impl<P, T, const CP: bool> fmt::Debug for TaggedPtr<P, T, CP>
where
P: Pointer + fmt::Debug,
T: Tag + fmt::Debug,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.raw.with_pointer_ref(|ptr| {
f.debug_struct("TaggedPtr").field("pointer", ptr).field("tag", &self.tag()).finish()
})
}
}
impl<P, T> PartialEq for TaggedPtr<P, T, true>
where
P: Pointer,
T: Tag,
{
#[inline]
fn eq(&self, other: &Self) -> bool {
self.raw.eq(&other.raw)
}
}
impl<P, T> Eq for TaggedPtr<P, T, true>
where
P: Pointer,
T: Tag,
{
}
impl<P, T> Hash for TaggedPtr<P, T, true>
where
P: Pointer,
T: Tag,
{
#[inline]
fn hash<H: Hasher>(&self, state: &mut H) {
self.raw.hash(state);
}
}
impl<P, T, HCX, const CP: bool> HashStable<HCX> for TaggedPtr<P, T, CP>
where
P: Pointer + HashStable<HCX>,
T: Tag + HashStable<HCX>,
{
fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) {
self.raw.hash_stable(hcx, hasher);
}
}
/// Test that `new` does not compile if there is not enough alignment for the
/// tag in the pointer.
///
/// ```compile_fail,E0080
/// use rustc_data_structures::tagged_ptr::{TaggedPtr, Tag};
///
/// #[derive(Copy, Clone, Debug, PartialEq, Eq)]
/// enum Tag2 { B00 = 0b00, B01 = 0b01, B10 = 0b10, B11 = 0b11 };
///
/// unsafe impl Tag for Tag2 {
/// const BITS: u32 = 2;
///
/// fn into_usize(self) -> usize { todo!() }
/// unsafe fn from_usize(tag: usize) -> Self { todo!() }
/// }
///
/// let value = 12u16;
/// let reference = &value;
/// let tag = Tag2::B01;
///
/// let _ptr = TaggedPtr::<_, _, true>::new(reference, tag);
/// ```
// For some reason miri does not get the compile error
// probably it `check`s instead of `build`ing?
#[cfg(not(miri))]
const _: () = ();
#[cfg(test)]
mod tests;

72
compiler/rustc_data_structures/src/tagged_ptr/drop/tests.rs
View File

@ -1,72 +0,0 @@
use std::ptr;
use std::sync::Arc;
use crate::tagged_ptr::{Pointer, Tag, Tag2, TaggedPtr};
#[test]
fn smoke() {
let value = 12u32;
let reference = &value;
let tag = Tag2::B01;
let ptr = tag_ptr(reference, tag);
assert_eq!(ptr.tag(), tag);
assert_eq!(*ptr, 12);
let clone = ptr.clone();
assert_eq!(clone.tag(), tag);
assert_eq!(*clone, 12);
let mut ptr = ptr;
ptr.set_tag(Tag2::B00);
assert_eq!(ptr.tag(), Tag2::B00);
assert_eq!(clone.tag(), tag);
assert_eq!(*clone, 12);
assert!(ptr::eq(&*ptr, &*clone))
}
#[test]
fn boxed() {
let value = 12u32;
let boxed = Box::new(value);
let tag = Tag2::B01;
let ptr = tag_ptr(boxed, tag);
assert_eq!(ptr.tag(), tag);
assert_eq!(*ptr, 12);
let clone = ptr.clone();
assert_eq!(clone.tag(), tag);
assert_eq!(*clone, 12);
let mut ptr = ptr;
ptr.set_tag(Tag2::B00);
assert_eq!(ptr.tag(), Tag2::B00);
assert_eq!(clone.tag(), tag);
assert_eq!(*clone, 12);
assert!(!ptr::eq(&*ptr, &*clone))
}
#[test]
fn arclones() {
let value = 12u32;
let arc = Arc::new(value);
let tag = Tag2::B01;
let ptr = tag_ptr(arc, tag);
assert_eq!(ptr.tag(), tag);
assert_eq!(*ptr, 12);
let clone = ptr.clone();
assert!(ptr::eq(&*ptr, &*clone))
}
/// Helper to create tagged pointers without specifying `COMPARE_PACKED` if it does not matter.
fn tag_ptr<P: Pointer, T: Tag>(ptr: P, tag: T) -> TaggedPtr<P, T, true> {
TaggedPtr::new(ptr, tag)
}

144
compiler/rustc_data_structures/src/tagged_ptr/impl_tag.rs
View File

@ -1,144 +0,0 @@
/// Implements [`Tag`] for a given type.
///
/// You can use `impl_tag` on structs and enums.
/// You need to specify the type and all its possible values,
/// which can only be paths with optional fields.
///
/// [`Tag`]: crate::tagged_ptr::Tag
///
/// # Examples
///
/// Basic usage:
///
/// ```
/// #![feature(macro_metavar_expr)]
/// use rustc_data_structures::{impl_tag, tagged_ptr::Tag};
///
/// #[derive(Copy, Clone, PartialEq, Debug)]
/// enum SomeTag {
/// A,
/// B,
/// X { v: bool },
/// Y(bool, bool),
/// }
///
/// impl_tag! {
/// // The type for which the `Tag` will be implemented
/// impl Tag for SomeTag;
/// // You need to specify all possible tag values:
/// SomeTag::A, // 0
/// SomeTag::B, // 1
/// // For variants with fields, you need to specify the fields:
/// SomeTag::X { v: true }, // 2
/// SomeTag::X { v: false }, // 3
/// // For tuple variants use named syntax:
/// SomeTag::Y { 0: true, 1: true }, // 4
/// SomeTag::Y { 0: false, 1: true }, // 5
/// SomeTag::Y { 0: true, 1: false }, // 6
/// SomeTag::Y { 0: false, 1: false }, // 7
/// }
///
/// // Tag values are assigned in order:
/// assert_eq!(SomeTag::A.into_usize(), 0);
/// assert_eq!(SomeTag::X { v: false }.into_usize(), 3);
/// assert_eq!(SomeTag::Y(false, true).into_usize(), 5);
///
/// assert_eq!(unsafe { SomeTag::from_usize(1) }, SomeTag::B);
/// assert_eq!(unsafe { SomeTag::from_usize(2) }, SomeTag::X { v: true });
/// assert_eq!(unsafe { SomeTag::from_usize(7) }, SomeTag::Y(false, false));
/// ```
///
/// Structs are supported:
///
/// ```
/// #![feature(macro_metavar_expr)]
/// # use rustc_data_structures::impl_tag;
/// #[derive(Copy, Clone)]
/// struct Flags { a: bool, b: bool }
///
/// impl_tag! {
/// impl Tag for Flags;
/// Flags { a: true, b: true },
/// Flags { a: false, b: true },
/// Flags { a: true, b: false },
/// Flags { a: false, b: false },
/// }
/// ```
///
/// Not specifying all values results in a compile error:
///
/// ```compile_fail,E0004
/// #![feature(macro_metavar_expr)]
/// # use rustc_data_structures::impl_tag;
/// #[derive(Copy, Clone)]
/// enum E {
/// A,
/// B,
/// }
///
/// impl_tag! {
/// impl Tag for E;
/// E::A,
/// }
/// ```
#[macro_export]
macro_rules! impl_tag {
(
impl Tag for $Self:ty;
$(
$($path:ident)::* $( { $( $fields:tt )* })?,
)*
) => {
// Safety:
// `bits_for_tags` is called on the same `${index()}`-es as
// `into_usize` returns, thus `BITS` constant is correct.
unsafe impl $crate::tagged_ptr::Tag for $Self {
const BITS: u32 = $crate::tagged_ptr::bits_for_tags(&[
$(
${index()},
$( ${ignore($path)} )*
)*
]);
#[inline]
fn into_usize(self) -> usize {
// This forbids use of repeating patterns (`Enum::V`&`Enum::V`, etc)
// (or at least it should, see <https://github.com/rust-lang/rust/issues/110613>)
#[forbid(unreachable_patterns)]
match self {
// `match` is doing heavy lifting here, by requiring exhaustiveness
$(
$($path)::* $( { $( $fields )* } )? => ${index()},
)*
}
}
#[inline]
unsafe fn from_usize(tag: usize) -> Self {
match tag {
$(
${index()} => $($path)::* $( { $( $fields )* } )?,
)*
// Safety:
// `into_usize` only returns `${index()}` of the same
// repetition as we are filtering above, thus if this is
// reached, the safety contract of this function was
// already breached.
_ => unsafe {
debug_assert!(
false,
"invalid tag: {tag}\
(this is a bug in the caller of `from_usize`)"
);
std::hint::unreachable_unchecked()
},
}
}
}
};
}
#[cfg(test)]
mod tests;

34
compiler/rustc_data_structures/src/tagged_ptr/impl_tag/tests.rs
View File

@ -1,34 +0,0 @@
#[test]
fn bits_constant() {
use crate::tagged_ptr::Tag;
#[derive(Copy, Clone)]
struct Unit;
impl_tag! { impl Tag for Unit; Unit, }
assert_eq!(Unit::BITS, 0);
#[derive(Copy, Clone)]
enum Enum3 {
A,
B,
C,
}
impl_tag! { impl Tag for Enum3; Enum3::A, Enum3::B, Enum3::C, }
assert_eq!(Enum3::BITS, 2);
#[derive(Copy, Clone)]
struct Eight(bool, bool, bool);
impl_tag! {
impl Tag for Eight;
Eight { 0: true, 1: true, 2: true },
Eight { 0: true, 1: true, 2: false },
Eight { 0: true, 1: false, 2: true },
Eight { 0: true, 1: false, 2: false },
Eight { 0: false, 1: true, 2: true },
Eight { 0: false, 1: true, 2: false },
Eight { 0: false, 1: false, 2: true },
Eight { 0: false, 1: false, 2: false },
}
assert_eq!(Eight::BITS, 3);
}

105
compiler/rustc_data_structures/src/tagged_ptr/tests.rs Normal file
View File

@ -0,0 +1,105 @@
use std::ptr;
use super::*;
use crate::hashes::Hash128;
use crate::stable_hasher::{HashStable, StableHasher};
/// A tag type used in [`TaggedRef`] tests.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum Tag2 {
B00 = 0b00,
B01 = 0b01,
B10 = 0b10,
B11 = 0b11,
}
unsafe impl Tag for Tag2 {
const BITS: u32 = 2;
fn into_usize(self) -> usize {
self as _
}
unsafe fn from_usize(tag: usize) -> Self {
match tag {
0b00 => Tag2::B00,
0b01 => Tag2::B01,
0b10 => Tag2::B10,
0b11 => Tag2::B11,
_ => unreachable!(),
}
}
}
impl<HCX> crate::stable_hasher::HashStable<HCX> for Tag2 {
fn hash_stable(&self, hcx: &mut HCX, hasher: &mut crate::stable_hasher::StableHasher) {
(*self as u8).hash_stable(hcx, hasher);
}
}
#[test]
fn smoke() {
let value = 12u32;
let reference = &value;
let tag = Tag2::B01;
let ptr = TaggedRef::new(reference, tag);
assert_eq!(ptr.tag(), tag);
assert_eq!(*ptr, 12);
assert!(ptr::eq(ptr.pointer(), reference));
let copy = ptr;
let mut ptr = ptr;
ptr.set_tag(Tag2::B00);
assert_eq!(ptr.tag(), Tag2::B00);
assert_eq!(copy.tag(), tag);
assert_eq!(*copy, 12);
assert!(ptr::eq(copy.pointer(), reference));
}
#[test]
fn stable_hash_hashes_as_tuple() {
let hash_packed = {
let mut hasher = StableHasher::new();
TaggedRef::new(&12, Tag2::B11).hash_stable(&mut (), &mut hasher);
hasher.finish::<Hash128>()
};
let hash_tupled = {
let mut hasher = StableHasher::new();
(&12, Tag2::B11).hash_stable(&mut (), &mut hasher);
hasher.finish::<Hash128>()
};
assert_eq!(hash_packed, hash_tupled);
}
/// Test that `new` does not compile if there is not enough alignment for the
/// tag in the pointer.
///
/// ```compile_fail,E0080
/// use rustc_data_structures::tagged_ptr::{TaggedRef, Tag};
///
/// #[derive(Copy, Clone, Debug, PartialEq, Eq)]
/// enum Tag2 { B00 = 0b00, B01 = 0b01, B10 = 0b10, B11 = 0b11 };
///
/// unsafe impl Tag for Tag2 {
/// const BITS: u32 = 2;
///
/// fn into_usize(self) -> usize { todo!() }
/// unsafe fn from_usize(tag: usize) -> Self { todo!() }
/// }
///
/// let value = 12u16;
/// let reference = &value;
/// let tag = Tag2::B01;
///
/// let _ptr = TaggedRef::<_, _, true>::new(reference, tag);
/// ```
// For some reason miri does not get the compile error
// probably it `check`s instead of `build`ing?
#[cfg(not(miri))]
const _: () = ();

2
compiler/rustc_middle/src/ty/list.rs
View File

@ -21,7 +21,7 @@ use crate::arena::Arena;
/// pointer. /// pointer.
/// - Because of this, you cannot get a `List<T>` that is a sub-list of another /// - Because of this, you cannot get a `List<T>` that is a sub-list of another
/// `List<T>`. You can get a sub-slice `&[T]`, however. /// `List<T>`. You can get a sub-slice `&[T]`, however.
/// - `List<T>` can be used with `CopyTaggedPtr`, which is useful within /// - `List<T>` can be used with `TaggedRef`, which is useful within
/// structs whose size must be minimized. /// structs whose size must be minimized.
/// - Because of the uniqueness assumption, we can use the address of a /// - Because of the uniqueness assumption, we can use the address of a
/// `List<T>` for faster equality comparisons and hashing. /// `List<T>` for faster equality comparisons and hashing.