diff --git a/compiler/rustc_data_structures/src/aligned.rs b/compiler/rustc_data_structures/src/aligned.rs new file mode 100644 index 00000000000..0e5ecfd9bff --- /dev/null +++ b/compiler/rustc_data_structures/src/aligned.rs @@ -0,0 +1,33 @@ +use std::ptr::Alignment; + +/// Returns the ABI-required minimum alignment of a type in bytes. +/// +/// This is equivalent to [`mem::align_of`], but also works for some unsized +/// types (e.g. slices or rustc's `List`s). +/// +/// [`mem::align_of`]: std::mem::align_of +pub const fn align_of() -> Alignment { + T::ALIGN +} + +/// A type with a statically known alignment. +/// +/// # Safety +/// +/// `Self::ALIGN` must be equal to the alignment of `Self`. For sized types it +/// is [`mem::align_of()`], for unsized types it depends on the type, for +/// example `[T]` has alignment of `T`. +/// +/// [`mem::align_of()`]: std::mem::align_of +pub unsafe trait Aligned { + /// Alignment of `Self`. + const ALIGN: Alignment; +} + +unsafe impl Aligned for T { + const ALIGN: Alignment = Alignment::of::(); +} + +unsafe impl Aligned for [T] { + const ALIGN: Alignment = Alignment::of::(); +} diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs index e373bd18402..7768e0fdeb1 100644 --- a/compiler/rustc_data_structures/src/lib.rs +++ b/compiler/rustc_data_structures/src/lib.rs @@ -29,6 +29,8 @@ #![feature(get_mut_unchecked)] #![feature(lint_reasons)] #![feature(unwrap_infallible)] +#![feature(strict_provenance)] +#![feature(ptr_alignment_type)] #![allow(rustc::default_hash_types)] #![allow(rustc::potential_query_instability)] #![deny(rustc::untranslatable_diagnostic)] @@ -82,6 +84,7 @@ pub mod transitive_relation; pub mod vec_linked_list; pub mod work_queue; pub use atomic_ref::AtomicRef; +pub mod aligned; pub mod frozen; pub mod owned_slice; pub mod sso; diff --git a/compiler/rustc_data_structures/src/tagged_ptr.rs b/compiler/rustc_data_structures/src/tagged_ptr.rs index 651bc556c98..c26bffac678 100644 --- a/compiler/rustc_data_structures/src/tagged_ptr.rs +++ b/compiler/rustc_data_structures/src/tagged_ptr.rs @@ -3,166 +3,262 @@ //! In order to utilize the pointer packing, you must have two types: a pointer, //! and a tag. //! -//! The pointer must implement the `Pointer` trait, with the primary requirement -//! being conversion to and from a usize. Note that the pointer must be -//! dereferenceable, so raw pointers generally cannot implement the `Pointer` -//! trait. This implies that the pointer must also be nonzero. +//! The pointer must implement the [`Pointer`] trait, with the primary +//! requirement being convertible to and from a raw pointer. Note that the +//! pointer must be dereferenceable, so raw pointers generally cannot implement +//! the [`Pointer`] trait. This implies that the pointer must also be non-null. //! -//! Many common pointer types already implement the `Pointer` trait. +//! Many common pointer types already implement the [`Pointer`] trait. //! -//! The tag must implement the `Tag` trait. We assert that the tag and `Pointer` -//! are compatible at compile time. +//! The tag must implement the [`Tag`] trait. +//! +//! We assert that the tag and the [`Pointer`] types are compatible at compile +//! time. -use std::mem::ManuallyDrop; use std::ops::Deref; +use std::ptr::NonNull; use std::rc::Rc; use std::sync::Arc; +use crate::aligned::Aligned; + mod copy; mod drop; pub use copy::CopyTaggedPtr; pub use drop::TaggedPtr; -/// This describes the pointer type encapsulated by TaggedPtr. +/// This describes the pointer type encapsulated by [`TaggedPtr`] and +/// [`CopyTaggedPtr`]. /// /// # Safety /// -/// The usize returned from `into_usize` must be a valid, dereferenceable, -/// pointer to `::Target`. Note that pointers to `Pointee` must -/// be thin, even though `Pointee` may not be sized. +/// The pointer returned from [`into_ptr`] must be a [valid], pointer to +/// [`::Target`]. /// -/// Note that the returned pointer from `into_usize` should be castable to `&mut -/// ::Target` if `Pointer: DerefMut`. +/// Note that if `Self` implements [`DerefMut`] the pointer returned from +/// [`into_ptr`] must be valid for writes (and thus calling [`NonNull::as_mut`] +/// on it must be safe). /// -/// The BITS constant must be correct. At least `BITS` bits, least-significant, -/// must be zero on all returned pointers from `into_usize`. +/// The [`BITS`] constant must be correct. [`BITS`] least-significant bits, +/// must be zero on all pointers returned from [`into_ptr`]. /// -/// For example, if the alignment of `Pointee` is 2, then `BITS` should be 1. +/// For example, if the alignment of [`Self::Target`] is 2, then `BITS` should be 1. +/// +/// [`BITS`]: Pointer::BITS +/// [`into_ptr`]: Pointer::into_ptr +/// [valid]: std::ptr#safety +/// [`::Target`]: Deref::Target +/// [`Self::Target`]: Deref::Target +/// [`DerefMut`]: std::ops::DerefMut pub unsafe trait Pointer: Deref { + /// Number of unused (always zero) **least-significant bits** in this + /// pointer, usually related to the pointees alignment. + /// + /// For example if [`BITS`] = `2`, then given `ptr = Self::into_ptr(..)`, + /// `ptr.addr() & 0b11 == 0` must be true. + /// /// Most likely the value you want to use here is the following, unless - /// your Pointee type is unsized (e.g., `ty::List` in rustc) in which - /// case you'll need to manually figure out what the right type to pass to - /// align_of is. + /// your [`Self::Target`] type is unsized (e.g., `ty::List` in rustc) + /// or your pointer is over/under aligned, in which case you'll need to + /// manually figure out what the right type to pass to [`bits_for`] is, or + /// what the value to set here. /// - /// ```ignore UNSOLVED (what to do about the Self) + /// ```rust /// # use std::ops::Deref; - /// std::mem::align_of::<::Target>().trailing_zeros() as usize; + /// # use rustc_data_structures::tagged_ptr::bits_for; + /// # struct T; + /// # impl Deref for T { type Target = u8; fn deref(&self) -> &u8 { &0 } } + /// # impl T { + /// const BITS: u32 = bits_for::<::Target>(); + /// # } /// ``` - const BITS: usize; - fn into_usize(self) -> usize; - - /// # Safety /// - /// The passed `ptr` must be returned from `into_usize`. - /// - /// This acts as `ptr::read` semantically, it should not be called more than - /// once on non-`Copy` `Pointer`s. - unsafe fn from_usize(ptr: usize) -> Self; + /// [`BITS`]: Pointer::BITS + /// [`Self::Target`]: Deref::Target + const BITS: u32; - /// This provides a reference to the `Pointer` itself, rather than the - /// `Deref::Target`. It is used for cases where we want to call methods that - /// may be implement differently for the Pointer than the Pointee (e.g., - /// `Rc::clone` vs cloning the inner value). + /// Turns this pointer into a raw, non-null pointer. + /// + /// The inverse of this function is [`from_ptr`]. + /// + /// This function guarantees that the least-significant [`Self::BITS`] bits + /// are zero. + /// + /// [`from_ptr`]: Pointer::from_ptr + /// [`Self::BITS`]: Pointer::BITS + fn into_ptr(self) -> NonNull; + + /// Re-creates the original pointer, from a raw pointer returned by [`into_ptr`]. /// /// # Safety /// - /// The passed `ptr` must be returned from `into_usize`. - unsafe fn with_ref R>(ptr: usize, f: F) -> R; + /// The passed `ptr` must be returned from [`into_ptr`]. + /// + /// This acts as [`ptr::read::()`] semantically, it should not be called more than + /// once on non-[`Copy`] `Pointer`s. + /// + /// [`into_ptr`]: Pointer::into_ptr + /// [`ptr::read::()`]: std::ptr::read + unsafe fn from_ptr(ptr: NonNull) -> Self; } -/// This describes tags that the `TaggedPtr` struct can hold. +/// This describes tags that the [`TaggedPtr`] struct can hold. /// /// # Safety /// -/// The BITS constant must be correct. +/// The [`BITS`] constant must be correct. /// -/// No more than `BITS` least significant bits may be set in the returned usize. +/// No more than [`BITS`] least-significant bits may be set in the returned usize. +/// +/// [`BITS`]: Tag::BITS pub unsafe trait Tag: Copy { - const BITS: usize; + /// Number of least-significant bits in the return value of [`into_usize`] + /// which may be non-zero. In other words this is the bit width of the + /// value. + /// + /// [`into_usize`]: Tag::into_usize + const BITS: u32; + /// Turns this tag into an integer. + /// + /// The inverse of this function is [`from_usize`]. + /// + /// This function guarantees that only the least-significant [`Self::BITS`] + /// bits can be non-zero. + /// + /// [`from_usize`]: Tag::from_usize + /// [`Self::BITS`]: Tag::BITS fn into_usize(self) -> usize; + /// Re-creates the tag from the integer returned by [`into_usize`]. + /// /// # Safety /// - /// The passed `tag` must be returned from `into_usize`. + /// The passed `tag` must be returned from [`into_usize`]. + /// + /// [`into_usize`]: Tag::into_usize unsafe fn from_usize(tag: usize) -> Self; } -unsafe impl Pointer for Box { - const BITS: usize = std::mem::align_of::().trailing_zeros() as usize; +unsafe impl Pointer for Box { + const BITS: u32 = bits_for::(); + #[inline] - fn into_usize(self) -> usize { - Box::into_raw(self) as usize + fn into_ptr(self) -> NonNull { + // Safety: pointers from `Box::into_raw` are valid & non-null + unsafe { NonNull::new_unchecked(Box::into_raw(self)) } } + #[inline] - unsafe fn from_usize(ptr: usize) -> Self { - Box::from_raw(ptr as *mut T) - } - unsafe fn with_ref R>(ptr: usize, f: F) -> R { - let raw = ManuallyDrop::new(Self::from_usize(ptr)); - f(&raw) + unsafe fn from_ptr(ptr: NonNull) -> Self { + // Safety: `ptr` comes from `into_ptr` which calls `Box::into_raw` + Box::from_raw(ptr.as_ptr()) } } -unsafe impl Pointer for Rc { - const BITS: usize = std::mem::align_of::().trailing_zeros() as usize; +unsafe impl Pointer for Rc { + const BITS: u32 = bits_for::(); + #[inline] - fn into_usize(self) -> usize { - Rc::into_raw(self) as usize + fn into_ptr(self) -> NonNull { + // Safety: pointers from `Rc::into_raw` are valid & non-null + unsafe { NonNull::new_unchecked(Rc::into_raw(self).cast_mut()) } } + #[inline] - unsafe fn from_usize(ptr: usize) -> Self { - Rc::from_raw(ptr as *const T) - } - unsafe fn with_ref R>(ptr: usize, f: F) -> R { - let raw = ManuallyDrop::new(Self::from_usize(ptr)); - f(&raw) + unsafe fn from_ptr(ptr: NonNull) -> Self { + // Safety: `ptr` comes from `into_ptr` which calls `Rc::into_raw` + Rc::from_raw(ptr.as_ptr()) } } -unsafe impl Pointer for Arc { - const BITS: usize = std::mem::align_of::().trailing_zeros() as usize; +unsafe impl Pointer for Arc { + const BITS: u32 = bits_for::(); + #[inline] - fn into_usize(self) -> usize { - Arc::into_raw(self) as usize + fn into_ptr(self) -> NonNull { + // Safety: pointers from `Arc::into_raw` are valid & non-null + unsafe { NonNull::new_unchecked(Arc::into_raw(self).cast_mut()) } } + #[inline] - unsafe fn from_usize(ptr: usize) -> Self { - Arc::from_raw(ptr as *const T) - } - unsafe fn with_ref R>(ptr: usize, f: F) -> R { - let raw = ManuallyDrop::new(Self::from_usize(ptr)); - f(&raw) + unsafe fn from_ptr(ptr: NonNull) -> Self { + // Safety: `ptr` comes from `into_ptr` which calls `Arc::into_raw` + Arc::from_raw(ptr.as_ptr()) } } -unsafe impl<'a, T: 'a> Pointer for &'a T { - const BITS: usize = std::mem::align_of::().trailing_zeros() as usize; +unsafe impl<'a, T: 'a + ?Sized + Aligned> Pointer for &'a T { + const BITS: u32 = bits_for::(); + #[inline] - fn into_usize(self) -> usize { - self as *const T as usize + fn into_ptr(self) -> NonNull { + NonNull::from(self) } + #[inline] - unsafe fn from_usize(ptr: usize) -> Self { - &*(ptr as *const T) - } - unsafe fn with_ref R>(ptr: usize, f: F) -> R { - f(&*(&ptr as *const usize as *const Self)) + unsafe fn from_ptr(ptr: NonNull) -> Self { + // Safety: + // `ptr` comes from `into_ptr` which gets the pointer from a reference + ptr.as_ref() } } -unsafe impl<'a, T: 'a> Pointer for &'a mut T { - const BITS: usize = std::mem::align_of::().trailing_zeros() as usize; +unsafe impl<'a, T: 'a + ?Sized + Aligned> Pointer for &'a mut T { + const BITS: u32 = bits_for::(); + #[inline] - fn into_usize(self) -> usize { - self as *mut T as usize + fn into_ptr(self) -> NonNull { + NonNull::from(self) } + #[inline] - unsafe fn from_usize(ptr: usize) -> Self { - &mut *(ptr as *mut T) - } - unsafe fn with_ref R>(ptr: usize, f: F) -> R { - f(&*(&ptr as *const usize as *const Self)) + unsafe fn from_ptr(mut ptr: NonNull) -> Self { + // Safety: + // `ptr` comes from `into_ptr` which gets the pointer from a reference + ptr.as_mut() + } +} + +/// Returns the number of bits available for use for tags in a pointer to `T` +/// (this is based on `T`'s alignment). +pub const fn bits_for() -> u32 { + crate::aligned::align_of::().as_nonzero().trailing_zeros() +} + +/// A tag type used in [`CopyTaggedPtr`] and [`TaggedPtr`] tests. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[cfg(test)] +enum Tag2 { + B00 = 0b00, + B01 = 0b01, + B10 = 0b10, + B11 = 0b11, +} + +#[cfg(test)] +unsafe impl Tag for Tag2 { + const BITS: u32 = 2; + + fn into_usize(self) -> usize { + self as _ + } + + unsafe fn from_usize(tag: usize) -> Self { + match tag { + 0b00 => Tag2::B00, + 0b01 => Tag2::B01, + 0b10 => Tag2::B10, + 0b11 => Tag2::B11, + _ => unreachable!(), + } + } +} + +#[cfg(test)] +impl crate::stable_hasher::HashStable for Tag2 { + fn hash_stable(&self, hcx: &mut HCX, hasher: &mut crate::stable_hasher::StableHasher) { + (*self as u8).hash_stable(hcx, hasher); } } diff --git a/compiler/rustc_data_structures/src/tagged_ptr/copy.rs b/compiler/rustc_data_structures/src/tagged_ptr/copy.rs index e1d3e0bd35a..691e92f196a 100644 --- a/compiler/rustc_data_structures/src/tagged_ptr/copy.rs +++ b/compiler/rustc_data_structures/src/tagged_ptr/copy.rs @@ -1,78 +1,92 @@ use super::{Pointer, Tag}; use crate::stable_hasher::{HashStable, StableHasher}; use std::fmt; +use std::hash::{Hash, Hasher}; use std::marker::PhantomData; +use std::mem::ManuallyDrop; use std::num::NonZeroUsize; +use std::ops::{Deref, DerefMut}; +use std::ptr::NonNull; -/// A `Copy` TaggedPtr. +/// A [`Copy`] tagged pointer. /// -/// You should use this instead of the `TaggedPtr` type in all cases where -/// `P: Copy`. +/// This is essentially `{ pointer: P, tag: T }` packed in a single pointer. +/// +/// You should use this instead of the [`TaggedPtr`] type in all cases where +/// `P` implements [`Copy`]. /// /// If `COMPARE_PACKED` is true, then the pointers will be compared and hashed without -/// unpacking. Otherwise we don't implement PartialEq/Eq/Hash; if you want that, -/// wrap the TaggedPtr. +/// unpacking. Otherwise we don't implement [`PartialEq`], [`Eq`] and [`Hash`]; +/// if you want that, wrap the [`CopyTaggedPtr`]. +/// +/// [`TaggedPtr`]: crate::tagged_ptr::TaggedPtr pub struct CopyTaggedPtr where P: Pointer, T: Tag, { - packed: NonZeroUsize, - data: PhantomData<(P, T)>, + /// This is semantically a pair of `pointer: P` and `tag: T` fields, + /// however we pack them in a single pointer, to save space. + /// + /// We pack the tag into the **most**-significant bits of the pointer to + /// ease retrieval of the value. A left shift is a multiplication and + /// those are embeddable in instruction encoding, for example: + /// + /// ```asm + /// // () + /// example::shift_read3: + /// mov eax, dword ptr [8*rdi] + /// ret + /// + /// example::mask_read3: + /// and rdi, -8 + /// mov eax, dword ptr [rdi] + /// ret + /// ``` + /// + /// This is ASM outputted by rustc for reads of values behind tagged + /// pointers for different approaches of tagging: + /// - `shift_read3` uses `<< 3` (the tag is in the most-significant bits) + /// - `mask_read3` uses `& !0b111` (the tag is in the least-significant bits) + /// + /// The shift approach thus produces less instructions and is likely faster + /// (see ). + /// + /// Encoding diagram: + /// ```text + /// [ packed.addr ] + /// [ tag ] [ pointer.addr >> T::BITS ] <-- usize::BITS - T::BITS bits + /// ^ + /// | + /// T::BITS bits + /// ``` + /// + /// The tag can be retrieved by `packed.addr() >> T::BITS` and the pointer + /// can be retrieved by `packed.map_addr(|addr| addr << T::BITS)`. + packed: NonNull, + tag_ghost: PhantomData, } -impl Copy for CopyTaggedPtr -where - P: Pointer, - T: Tag, - P: Copy, -{ -} - -impl Clone for CopyTaggedPtr -where - P: Pointer, - T: Tag, - P: Copy, -{ - fn clone(&self) -> Self { - *self - } -} - -// We pack the tag into the *upper* bits of the pointer to ease retrieval of the -// value; a left shift is a multiplication and those are embeddable in -// instruction encoding. -impl CopyTaggedPtr +// Note that even though `CopyTaggedPtr` is only really expected to work with +// `P: Copy`, can't add `P: Copy` bound, because `CopyTaggedPtr` is used in the +// `TaggedPtr`'s implementation. +impl CopyTaggedPtr where P: Pointer, T: Tag, { - const TAG_BIT_SHIFT: usize = usize::BITS as usize - T::BITS; - const ASSERTION: () = { - assert!(T::BITS <= P::BITS); - // Used for the transmute_copy's below - assert!(std::mem::size_of::<&P::Target>() == std::mem::size_of::()); - }; - + /// Tags `pointer` with `tag`. + /// + /// Note that this leaks `pointer`: it won't be dropped when + /// `CopyTaggedPtr` is dropped. If you have a pointer with a significant + /// drop, use [`TaggedPtr`] instead. + /// + /// [`TaggedPtr`]: crate::tagged_ptr::TaggedPtr pub fn new(pointer: P, tag: T) -> Self { - // Trigger assert! - let () = Self::ASSERTION; - let packed_tag = tag.into_usize() << Self::TAG_BIT_SHIFT; - - Self { - // SAFETY: We know that the pointer is non-null, as it must be - // dereferenceable per `Pointer` safety contract. - packed: unsafe { - NonZeroUsize::new_unchecked((P::into_usize(pointer) >> T::BITS) | packed_tag) - }, - data: PhantomData, - } + Self { packed: Self::pack(P::into_ptr(pointer), tag), tag_ghost: PhantomData } } - pub(super) fn pointer_raw(&self) -> usize { - self.packed.get() << T::BITS - } + /// Retrieves the pointer. pub fn pointer(self) -> P where P: Copy, @@ -81,66 +95,138 @@ where // // Note that this isn't going to double-drop or anything because we have // P: Copy - unsafe { P::from_usize(self.pointer_raw()) } - } - pub fn pointer_ref(&self) -> &P::Target { - // SAFETY: pointer_raw returns the original pointer - unsafe { std::mem::transmute_copy(&self.pointer_raw()) } - } - pub fn pointer_mut(&mut self) -> &mut P::Target - where - P: std::ops::DerefMut, - { - // SAFETY: pointer_raw returns the original pointer - unsafe { std::mem::transmute_copy(&self.pointer_raw()) } + unsafe { P::from_ptr(self.pointer_raw()) } } + + /// Retrieves the tag. #[inline] pub fn tag(&self) -> T { - unsafe { T::from_usize(self.packed.get() >> Self::TAG_BIT_SHIFT) } + // Unpack the tag, according to the `self.packed` encoding scheme + let tag = self.packed.addr().get() >> Self::TAG_BIT_SHIFT; + + // Safety: + // The shift retrieves the original value from `T::into_usize`, + // satisfying `T::from_usize`'s preconditions. + unsafe { T::from_usize(tag) } } + + /// Sets the tag to a new value. #[inline] pub fn set_tag(&mut self, tag: T) { - let mut packed = self.packed.get(); - let new_tag = T::into_usize(tag) << Self::TAG_BIT_SHIFT; - let tag_mask = (1 << T::BITS) - 1; - packed &= !(tag_mask << Self::TAG_BIT_SHIFT); - packed |= new_tag; - self.packed = unsafe { NonZeroUsize::new_unchecked(packed) }; + self.packed = Self::pack(self.pointer_raw(), tag); + } + + const TAG_BIT_SHIFT: u32 = usize::BITS - T::BITS; + const ASSERTION: () = { assert!(T::BITS <= P::BITS) }; + + /// Pack pointer `ptr` that comes from [`P::into_ptr`] with a `tag`, + /// according to `self.packed` encoding scheme. + /// + /// [`P::into_ptr`]: Pointer::into_ptr + fn pack(ptr: NonNull, tag: T) -> NonNull { + // Trigger assert! + let () = Self::ASSERTION; + + let packed_tag = tag.into_usize() << Self::TAG_BIT_SHIFT; + + ptr.map_addr(|addr| { + // Safety: + // - The pointer is `NonNull` => it's address is `NonZeroUsize` + // - `P::BITS` least significant bits are always zero (`Pointer` contract) + // - `T::BITS <= P::BITS` (from `Self::ASSERTION`) + // + // Thus `addr >> T::BITS` is guaranteed to be non-zero. + // + // `{non_zero} | packed_tag` can't make the value zero. + + let packed = (addr.get() >> T::BITS) | packed_tag; + unsafe { NonZeroUsize::new_unchecked(packed) } + }) + } + + /// Retrieves the original raw pointer from `self.packed`. + pub(super) fn pointer_raw(&self) -> NonNull { + self.packed.map_addr(|addr| unsafe { NonZeroUsize::new_unchecked(addr.get() << T::BITS) }) + } + + /// This provides a reference to the `P` pointer itself, rather than the + /// `Deref::Target`. It is used for cases where we want to call methods + /// that may be implement differently for the Pointer than the Pointee + /// (e.g., `Rc::clone` vs cloning the inner value). + pub(super) fn with_pointer_ref(&self, f: impl FnOnce(&P) -> R) -> R { + // Safety: + // - `self.raw.pointer_raw()` is originally returned from `P::into_ptr` + // and as such is valid for `P::from_ptr`. + // - This also allows us to not care whatever `f` panics or not. + // - Even though we create a copy of the pointer, we store it inside + // `ManuallyDrop` and only access it by-ref, so we don't double-drop. + // + // Semantically this is just `f(&self.pointer)` (where `self.pointer` + // is non-packed original pointer). + // + // Note that even though `CopyTaggedPtr` is only really expected to + // work with `P: Copy`, we have to assume `P: ?Copy`, because + // `CopyTaggedPtr` is used in the `TaggedPtr`'s implementation. + let ptr = unsafe { ManuallyDrop::new(P::from_ptr(self.pointer_raw())) }; + f(&ptr) } } -impl std::ops::Deref for CopyTaggedPtr +impl Copy for CopyTaggedPtr +where + P: Pointer + Copy, + T: Tag, +{ +} + +impl Clone for CopyTaggedPtr +where + P: Pointer + Copy, + T: Tag, +{ + fn clone(&self) -> Self { + *self + } +} + +impl Deref for CopyTaggedPtr where P: Pointer, T: Tag, { type Target = P::Target; + fn deref(&self) -> &Self::Target { - self.pointer_ref() + // Safety: + // `pointer_raw` returns the original pointer from `P::into_ptr` which, + // by the `Pointer`'s contract, must be valid. + unsafe { self.pointer_raw().as_ref() } } } -impl std::ops::DerefMut for CopyTaggedPtr +impl DerefMut for CopyTaggedPtr where - P: Pointer + std::ops::DerefMut, + P: Pointer + DerefMut, T: Tag, { fn deref_mut(&mut self) -> &mut Self::Target { - self.pointer_mut() + // Safety: + // `pointer_raw` returns the original pointer from `P::into_ptr` which, + // by the `Pointer`'s contract, must be valid for writes if + // `P: DerefMut`. + unsafe { self.pointer_raw().as_mut() } } } -impl fmt::Debug for CopyTaggedPtr +impl fmt::Debug for CopyTaggedPtr where - P: Pointer, - P::Target: fmt::Debug, + P: Pointer + fmt::Debug, T: Tag + fmt::Debug, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("CopyTaggedPtr") - .field("pointer", &self.pointer_ref()) - .field("tag", &self.tag()) - .finish() + self.with_pointer_ref(|ptr| { + f.debug_struct("CopyTaggedPtr").field("pointer", ptr).field("tag", &self.tag()).finish() + }) } } @@ -161,25 +247,73 @@ where { } -impl std::hash::Hash for CopyTaggedPtr +impl Hash for CopyTaggedPtr where P: Pointer, T: Tag, { - fn hash(&self, state: &mut H) { + fn hash(&self, state: &mut H) { self.packed.hash(state); } } -impl HashStable for CopyTaggedPtr +impl HashStable for CopyTaggedPtr where P: Pointer + HashStable, T: Tag + HashStable, { fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) { - unsafe { - Pointer::with_ref(self.pointer_raw(), |p: &P| p.hash_stable(hcx, hasher)); - } + self.with_pointer_ref(|ptr| ptr.hash_stable(hcx, hasher)); self.tag().hash_stable(hcx, hasher); } } + +// Safety: +// `CopyTaggedPtr` is semantically just `{ ptr: P, tag: T }`, as such +// it's ok to implement `Sync` as long as `P: Sync, T: Sync` +unsafe impl Sync for CopyTaggedPtr +where + P: Sync + Pointer, + T: Sync + Tag, +{ +} + +// Safety: +// `CopyTaggedPtr` is semantically just `{ ptr: P, tag: T }`, as such +// it's ok to implement `Send` as long as `P: Send, T: Send` +unsafe impl Send for CopyTaggedPtr +where + P: Send + Pointer, + T: Send + Tag, +{ +} + +/// Test that `new` does not compile if there is not enough alignment for the +/// tag in the pointer. +/// +/// ```compile_fail,E0080 +/// use rustc_data_structures::tagged_ptr::{CopyTaggedPtr, Tag}; +/// +/// #[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// enum Tag2 { B00 = 0b00, B01 = 0b01, B10 = 0b10, B11 = 0b11 }; +/// +/// unsafe impl Tag for Tag2 { +/// const BITS: u32 = 2; +/// +/// fn into_usize(self) -> usize { todo!() } +/// unsafe fn from_usize(tag: usize) -> Self { todo!() } +/// } +/// +/// let value = 12u16; +/// let reference = &value; +/// let tag = Tag2::B01; +/// +/// let _ptr = CopyTaggedPtr::<_, _, true>::new(reference, tag); +/// ``` +// For some reason miri does not get the compile error +// probably it `check`s instead of `build`ing? +#[cfg(not(miri))] +const _: () = (); + +#[cfg(test)] +mod tests; diff --git a/compiler/rustc_data_structures/src/tagged_ptr/copy/tests.rs b/compiler/rustc_data_structures/src/tagged_ptr/copy/tests.rs new file mode 100644 index 00000000000..bfcc2e603de --- /dev/null +++ b/compiler/rustc_data_structures/src/tagged_ptr/copy/tests.rs @@ -0,0 +1,50 @@ +use std::ptr; + +use crate::stable_hasher::{HashStable, StableHasher}; +use crate::tagged_ptr::{CopyTaggedPtr, Pointer, Tag, Tag2}; + +#[test] +fn smoke() { + let value = 12u32; + let reference = &value; + let tag = Tag2::B01; + + let ptr = tag_ptr(reference, tag); + + assert_eq!(ptr.tag(), tag); + assert_eq!(*ptr, 12); + assert!(ptr::eq(ptr.pointer(), reference)); + + let copy = ptr; + + let mut ptr = ptr; + ptr.set_tag(Tag2::B00); + assert_eq!(ptr.tag(), Tag2::B00); + + assert_eq!(copy.tag(), tag); + assert_eq!(*copy, 12); + assert!(ptr::eq(copy.pointer(), reference)); +} + +#[test] +fn stable_hash_hashes_as_tuple() { + let hash_packed = { + let mut hasher = StableHasher::new(); + tag_ptr(&12, Tag2::B11).hash_stable(&mut (), &mut hasher); + + hasher.finalize() + }; + + let hash_tupled = { + let mut hasher = StableHasher::new(); + (&12, Tag2::B11).hash_stable(&mut (), &mut hasher); + hasher.finalize() + }; + + assert_eq!(hash_packed, hash_tupled); +} + +/// Helper to create tagged pointers without specifying `COMPARE_PACKED` if it does not matter. +fn tag_ptr(ptr: P, tag: T) -> CopyTaggedPtr { + CopyTaggedPtr::new(ptr, tag) +} diff --git a/compiler/rustc_data_structures/src/tagged_ptr/drop.rs b/compiler/rustc_data_structures/src/tagged_ptr/drop.rs index b0315c93d93..d418c06b7eb 100644 --- a/compiler/rustc_data_structures/src/tagged_ptr/drop.rs +++ b/compiler/rustc_data_structures/src/tagged_ptr/drop.rs @@ -1,14 +1,21 @@ -use super::{Pointer, Tag}; -use crate::stable_hasher::{HashStable, StableHasher}; use std::fmt; +use std::hash::{Hash, Hasher}; +use std::ops::{Deref, DerefMut}; use super::CopyTaggedPtr; +use super::{Pointer, Tag}; +use crate::stable_hasher::{HashStable, StableHasher}; -/// A TaggedPtr implementing `Drop`. +/// A tagged pointer that supports pointers that implement [`Drop`]. +/// +/// This is essentially `{ pointer: P, tag: T }` packed in a single pointer. +/// +/// You should use [`CopyTaggedPtr`] instead of the this type in all cases +/// where `P` implements [`Copy`]. /// /// If `COMPARE_PACKED` is true, then the pointers will be compared and hashed without -/// unpacking. Otherwise we don't implement PartialEq/Eq/Hash; if you want that, -/// wrap the TaggedPtr. +/// unpacking. Otherwise we don't implement [`PartialEq`], [`Eq`] and [`Hash`]; +/// if you want that, wrap the [`TaggedPtr`]. pub struct TaggedPtr where P: Pointer, @@ -17,58 +24,61 @@ where raw: CopyTaggedPtr, } -impl Clone for TaggedPtr +impl TaggedPtr +where + P: Pointer, + T: Tag, +{ + /// Tags `pointer` with `tag`. + pub fn new(pointer: P, tag: T) -> Self { + TaggedPtr { raw: CopyTaggedPtr::new(pointer, tag) } + } + + /// Retrieves the tag. + pub fn tag(&self) -> T { + self.raw.tag() + } + + /// Sets the tag to a new value. + pub fn set_tag(&mut self, tag: T) { + self.raw.set_tag(tag) + } +} + +impl Clone for TaggedPtr where P: Pointer + Clone, T: Tag, { fn clone(&self) -> Self { - unsafe { Self::new(P::with_ref(self.raw.pointer_raw(), |p| p.clone()), self.raw.tag()) } + let ptr = self.raw.with_pointer_ref(P::clone); + + Self::new(ptr, self.tag()) } } -// We pack the tag into the *upper* bits of the pointer to ease retrieval of the -// value; a right shift is a multiplication and those are embeddable in -// instruction encoding. -impl TaggedPtr -where - P: Pointer, - T: Tag, -{ - pub fn new(pointer: P, tag: T) -> Self { - TaggedPtr { raw: CopyTaggedPtr::new(pointer, tag) } - } - - pub fn pointer_ref(&self) -> &P::Target { - self.raw.pointer_ref() - } - pub fn tag(&self) -> T { - self.raw.tag() - } -} - -impl std::ops::Deref for TaggedPtr +impl Deref for TaggedPtr where P: Pointer, T: Tag, { type Target = P::Target; fn deref(&self) -> &Self::Target { - self.raw.pointer_ref() + self.raw.deref() } } -impl std::ops::DerefMut for TaggedPtr +impl DerefMut for TaggedPtr where - P: Pointer + std::ops::DerefMut, + P: Pointer + DerefMut, T: Tag, { fn deref_mut(&mut self) -> &mut Self::Target { - self.raw.pointer_mut() + self.raw.deref_mut() } } -impl Drop for TaggedPtr +impl Drop for TaggedPtr where P: Pointer, T: Tag, @@ -76,22 +86,20 @@ where fn drop(&mut self) { // No need to drop the tag, as it's Copy unsafe { - drop(P::from_usize(self.raw.pointer_raw())); + drop(P::from_ptr(self.raw.pointer_raw())); } } } -impl fmt::Debug for TaggedPtr +impl fmt::Debug for TaggedPtr where - P: Pointer, - P::Target: fmt::Debug, + P: Pointer + fmt::Debug, T: Tag + fmt::Debug, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("TaggedPtr") - .field("pointer", &self.pointer_ref()) - .field("tag", &self.tag()) - .finish() + self.raw.with_pointer_ref(|ptr| { + f.debug_struct("TaggedPtr").field("pointer", ptr).field("tag", &self.tag()).finish() + }) } } @@ -112,17 +120,17 @@ where { } -impl std::hash::Hash for TaggedPtr +impl Hash for TaggedPtr where P: Pointer, T: Tag, { - fn hash(&self, state: &mut H) { + fn hash(&self, state: &mut H) { self.raw.hash(state); } } -impl HashStable for TaggedPtr +impl HashStable for TaggedPtr where P: Pointer + HashStable, T: Tag + HashStable, @@ -131,3 +139,33 @@ where self.raw.hash_stable(hcx, hasher); } } + +/// Test that `new` does not compile if there is not enough alignment for the +/// tag in the pointer. +/// +/// ```compile_fail,E0080 +/// use rustc_data_structures::tagged_ptr::{TaggedPtr, Tag}; +/// +/// #[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// enum Tag2 { B00 = 0b00, B01 = 0b01, B10 = 0b10, B11 = 0b11 }; +/// +/// unsafe impl Tag for Tag2 { +/// const BITS: u32 = 2; +/// +/// fn into_usize(self) -> usize { todo!() } +/// unsafe fn from_usize(tag: usize) -> Self { todo!() } +/// } +/// +/// let value = 12u16; +/// let reference = &value; +/// let tag = Tag2::B01; +/// +/// let _ptr = TaggedPtr::<_, _, true>::new(reference, tag); +/// ``` +// For some reason miri does not get the compile error +// probably it `check`s instead of `build`ing? +#[cfg(not(miri))] +const _: () = (); + +#[cfg(test)] +mod tests; diff --git a/compiler/rustc_data_structures/src/tagged_ptr/drop/tests.rs b/compiler/rustc_data_structures/src/tagged_ptr/drop/tests.rs new file mode 100644 index 00000000000..2c17d678d3a --- /dev/null +++ b/compiler/rustc_data_structures/src/tagged_ptr/drop/tests.rs @@ -0,0 +1,71 @@ +use std::{ptr, sync::Arc}; + +use crate::tagged_ptr::{Pointer, Tag, Tag2, TaggedPtr}; + +#[test] +fn smoke() { + let value = 12u32; + let reference = &value; + let tag = Tag2::B01; + + let ptr = tag_ptr(reference, tag); + + assert_eq!(ptr.tag(), tag); + assert_eq!(*ptr, 12); + + let clone = ptr.clone(); + assert_eq!(clone.tag(), tag); + assert_eq!(*clone, 12); + + let mut ptr = ptr; + ptr.set_tag(Tag2::B00); + assert_eq!(ptr.tag(), Tag2::B00); + + assert_eq!(clone.tag(), tag); + assert_eq!(*clone, 12); + assert!(ptr::eq(&*ptr, &*clone)) +} + +#[test] +fn boxed() { + let value = 12u32; + let boxed = Box::new(value); + let tag = Tag2::B01; + + let ptr = tag_ptr(boxed, tag); + + assert_eq!(ptr.tag(), tag); + assert_eq!(*ptr, 12); + + let clone = ptr.clone(); + assert_eq!(clone.tag(), tag); + assert_eq!(*clone, 12); + + let mut ptr = ptr; + ptr.set_tag(Tag2::B00); + assert_eq!(ptr.tag(), Tag2::B00); + + assert_eq!(clone.tag(), tag); + assert_eq!(*clone, 12); + assert!(!ptr::eq(&*ptr, &*clone)) +} + +#[test] +fn arclones() { + let value = 12u32; + let arc = Arc::new(value); + let tag = Tag2::B01; + + let ptr = tag_ptr(arc, tag); + + assert_eq!(ptr.tag(), tag); + assert_eq!(*ptr, 12); + + let clone = ptr.clone(); + assert!(ptr::eq(&*ptr, &*clone)) +} + +/// Helper to create tagged pointers without specifying `COMPARE_PACKED` if it does not matter. +fn tag_ptr(ptr: P, tag: T) -> TaggedPtr { + TaggedPtr::new(ptr, tag) +} diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs index 1ddc08eac7d..2fa769d3abb 100644 --- a/compiler/rustc_middle/src/lib.rs +++ b/compiler/rustc_middle/src/lib.rs @@ -59,6 +59,7 @@ #![feature(result_option_inspect)] #![feature(const_option)] #![feature(trait_alias)] +#![feature(ptr_alignment_type)] #![recursion_limit = "512"] #![allow(rustc::potential_query_instability)] diff --git a/compiler/rustc_middle/src/ty/list.rs b/compiler/rustc_middle/src/ty/list.rs index 79365ef281b..30f036e471c 100644 --- a/compiler/rustc_middle/src/ty/list.rs +++ b/compiler/rustc_middle/src/ty/list.rs @@ -1,4 +1,5 @@ use crate::arena::Arena; +use rustc_data_structures::aligned::{align_of, Aligned}; use rustc_serialize::{Encodable, Encoder}; use std::alloc::Layout; use std::cmp::Ordering; @@ -198,22 +199,17 @@ impl<'a, T: Copy> IntoIterator for &'a List { unsafe impl Sync for List {} -unsafe impl<'a, T: 'a> rustc_data_structures::tagged_ptr::Pointer for &'a List { - const BITS: usize = std::mem::align_of::().trailing_zeros() as usize; +// Safety: +// Layouts of `Equivalent` and `List` are the same, modulo opaque tail, +// thus aligns of `Equivalent` and `List` must be the same. +unsafe impl Aligned for List { + const ALIGN: ptr::Alignment = { + #[repr(C)] + struct Equivalent { + _len: usize, + _data: [T; 0], + } - #[inline] - fn into_usize(self) -> usize { - self as *const List as usize - } - - #[inline] - unsafe fn from_usize(ptr: usize) -> &'a List { - &*(ptr as *const List) - } - - unsafe fn with_ref R>(ptr: usize, f: F) -> R { - // `Self` is `&'a List` which impls `Copy`, so this is fine. - let ptr = Self::from_usize(ptr); - f(&ptr) - } + align_of::>() + }; } diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 5501763d66f..1061c320793 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -1627,7 +1627,8 @@ struct ParamTag { } unsafe impl rustc_data_structures::tagged_ptr::Tag for ParamTag { - const BITS: usize = 2; + const BITS: u32 = 2; + #[inline] fn into_usize(self) -> usize { match self { @@ -1637,6 +1638,7 @@ unsafe impl rustc_data_structures::tagged_ptr::Tag for ParamTag { Self { reveal: traits::Reveal::All, constness: hir::Constness::Const } => 3, } } + #[inline] unsafe fn from_usize(ptr: usize) -> Self { match ptr {