nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-25 08:13:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Auto merge of #110243 - WaffleLapkin:bless_tagged_pointers🙏, r=Nilstrieb

Browse Source 
Tagged pointers, now with strict provenance!

This is a big refactor of tagged pointers in rustc, with three main goals:
1. Porting the code to the strict provenance
2. Cleanup the code
3. Document the code (and safety invariants) better

This PR has grown quite a bit (almost a complete rewrite at this point...), so I'm not sure what's the best way to review this, but reviewing commit-by-commit should be fine.

r? `@Nilstrieb`
This commit is contained in:
bors 2023-04-17 21:50:13 +00:00
commit 7908a1d654
10 changed files with 665 additions and 241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
compiler/rustc_data_structures/src/aligned.rs Normal file
View File

@ -0,0 +1,33 @@
use std::ptr::Alignment;
/// Returns the ABI-required minimum alignment of a type in bytes.
///
/// This is equivalent to [`mem::align_of`], but also works for some unsized
/// types (e.g. slices or rustc's `List`s).
///
/// [`mem::align_of`]: std::mem::align_of
pub const fn align_of<T: ?Sized + Aligned>() -> Alignment {
T::ALIGN
}
/// A type with a statically known alignment.
///
/// # Safety
///
/// `Self::ALIGN` must be equal to the alignment of `Self`. For sized types it
/// is [`mem::align_of<Self>()`], for unsized types it depends on the type, for
/// example `[T]` has alignment of `T`.
///
/// [`mem::align_of<Self>()`]: std::mem::align_of
pub unsafe trait Aligned {
/// Alignment of `Self`.
const ALIGN: Alignment;
}
unsafe impl<T> Aligned for T {
const ALIGN: Alignment = Alignment::of::<Self>();
}
unsafe impl<T> Aligned for [T] {
const ALIGN: Alignment = Alignment::of::<T>();
}

3
compiler/rustc_data_structures/src/lib.rs
View File

@ -29,6 +29,8 @@
#![feature(get_mut_unchecked)]
#![feature(lint_reasons)]
#![feature(unwrap_infallible)]
#![feature(strict_provenance)]
#![feature(ptr_alignment_type)]
#![allow(rustc::default_hash_types)]
#![allow(rustc::potential_query_instability)]
#![deny(rustc::untranslatable_diagnostic)]
@ -82,6 +84,7 @@ pub mod transitive_relation;
pub mod vec_linked_list;
pub mod work_queue;
pub use atomic_ref::AtomicRef;
pub mod aligned;
pub mod frozen;
pub mod owned_slice;
pub mod sso;

276
compiler/rustc_data_structures/src/tagged_ptr.rs
View File

@ -3,166 +3,262 @@
//! In order to utilize the pointer packing, you must have two types: a pointer,
//! and a tag.
//!
//! The pointer must implement the `Pointer` trait, with the primary requirement
//! being conversion to and from a usize. Note that the pointer must be
//! dereferenceable, so raw pointers generally cannot implement the `Pointer`
//! trait. This implies that the pointer must also be nonzero.
//! The pointer must implement the [`Pointer`] trait, with the primary
//! requirement being convertible to and from a raw pointer. Note that the
//! pointer must be dereferenceable, so raw pointers generally cannot implement
//! the [`Pointer`] trait. This implies that the pointer must also be non-null.
//!
//! Many common pointer types already implement the `Pointer` trait.
//! Many common pointer types already implement the [`Pointer`] trait.
//!
//! The tag must implement the `Tag` trait. We assert that the tag and `Pointer`
//! are compatible at compile time.
//! The tag must implement the [`Tag`] trait.
//!
//! We assert that the tag and the [`Pointer`] types are compatible at compile
//! time.
use std::mem::ManuallyDrop;
use std::ops::Deref;
use std::ptr::NonNull;
use std::rc::Rc;
use std::sync::Arc;
use crate::aligned::Aligned;
mod copy;
mod drop;
pub use copy::CopyTaggedPtr;
pub use drop::TaggedPtr;
/// This describes the pointer type encapsulated by TaggedPtr.
/// This describes the pointer type encapsulated by [`TaggedPtr`] and
/// [`CopyTaggedPtr`].
///
/// # Safety
///
/// The usize returned from `into_usize` must be a valid, dereferenceable,
/// pointer to `<Self as Deref>::Target`. Note that pointers to `Pointee` must
/// be thin, even though `Pointee` may not be sized.
/// The pointer returned from [`into_ptr`] must be a [valid], pointer to
/// [`<Self as Deref>::Target`].
///
/// Note that the returned pointer from `into_usize` should be castable to `&mut
/// <Self as Deref>::Target` if `Pointer: DerefMut`.
/// Note that if `Self` implements [`DerefMut`] the pointer returned from
/// [`into_ptr`] must be valid for writes (and thus calling [`NonNull::as_mut`]
/// on it must be safe).
///
/// The BITS constant must be correct. At least `BITS` bits, least-significant,
/// must be zero on all returned pointers from `into_usize`.
/// The [`BITS`] constant must be correct. [`BITS`] least-significant bits,
/// must be zero on all pointers returned from [`into_ptr`].
///
/// For example, if the alignment of `Pointee` is 2, then `BITS` should be 1.
/// For example, if the alignment of [`Self::Target`] is 2, then `BITS` should be 1.
///
/// [`BITS`]: Pointer::BITS
/// [`into_ptr`]: Pointer::into_ptr
/// [valid]: std::ptr#safety
/// [`<Self as Deref>::Target`]: Deref::Target
/// [`Self::Target`]: Deref::Target
/// [`DerefMut`]: std::ops::DerefMut
pub unsafe trait Pointer: Deref {
/// Number of unused (always zero) **least-significant bits** in this
/// pointer, usually related to the pointees alignment.
///
/// For example if [`BITS`] = `2`, then given `ptr = Self::into_ptr(..)`,
/// `ptr.addr() & 0b11 == 0` must be true.
///
/// Most likely the value you want to use here is the following, unless
/// your Pointee type is unsized (e.g., `ty::List<T>` in rustc) in which
/// case you'll need to manually figure out what the right type to pass to
/// align_of is.
/// your [`Self::Target`] type is unsized (e.g., `ty::List<T>` in rustc)
/// or your pointer is over/under aligned, in which case you'll need to
/// manually figure out what the right type to pass to [`bits_for`] is, or
/// what the value to set here.
///
/// ```ignore UNSOLVED (what to do about the Self)
/// ```rust
/// # use std::ops::Deref;
/// std::mem::align_of::<<Self as Deref>::Target>().trailing_zeros() as usize;
/// # use rustc_data_structures::tagged_ptr::bits_for;
/// # struct T;
/// # impl Deref for T { type Target = u8; fn deref(&self) -> &u8 { &0 } }
/// # impl T {
/// const BITS: u32 = bits_for::<<Self as Deref>::Target>();
/// # }
/// ```
const BITS: usize;
fn into_usize(self) -> usize;
/// # Safety
///
/// The passed `ptr` must be returned from `into_usize`.
///
/// This acts as `ptr::read` semantically, it should not be called more than
/// once on non-`Copy` `Pointer`s.
unsafe fn from_usize(ptr: usize) -> Self;
/// [`BITS`]: Pointer::BITS
/// [`Self::Target`]: Deref::Target
const BITS: u32;
/// This provides a reference to the `Pointer` itself, rather than the
/// `Deref::Target`. It is used for cases where we want to call methods that
/// may be implement differently for the Pointer than the Pointee (e.g.,
/// `Rc::clone` vs cloning the inner value).
/// Turns this pointer into a raw, non-null pointer.
///
/// The inverse of this function is [`from_ptr`].
///
/// This function guarantees that the least-significant [`Self::BITS`] bits
/// are zero.
///
/// [`from_ptr`]: Pointer::from_ptr
/// [`Self::BITS`]: Pointer::BITS
fn into_ptr(self) -> NonNull<Self::Target>;
/// Re-creates the original pointer, from a raw pointer returned by [`into_ptr`].
///
/// # Safety
///
/// The passed `ptr` must be returned from `into_usize`.
unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R;
/// The passed `ptr` must be returned from [`into_ptr`].
///
/// This acts as [`ptr::read::<Self>()`] semantically, it should not be called more than
/// once on non-[`Copy`] `Pointer`s.
///
/// [`into_ptr`]: Pointer::into_ptr
/// [`ptr::read::<Self>()`]: std::ptr::read
unsafe fn from_ptr(ptr: NonNull<Self::Target>) -> Self;
}
/// This describes tags that the `TaggedPtr` struct can hold.
/// This describes tags that the [`TaggedPtr`] struct can hold.
///
/// # Safety
///
/// The BITS constant must be correct.
/// The [`BITS`] constant must be correct.
///
/// No more than `BITS` least significant bits may be set in the returned usize.
/// No more than [`BITS`] least-significant bits may be set in the returned usize.
///
/// [`BITS`]: Tag::BITS
pub unsafe trait Tag: Copy {
const BITS: usize;
/// Number of least-significant bits in the return value of [`into_usize`]
/// which may be non-zero. In other words this is the bit width of the
/// value.
///
/// [`into_usize`]: Tag::into_usize
const BITS: u32;
/// Turns this tag into an integer.
///
/// The inverse of this function is [`from_usize`].
///
/// This function guarantees that only the least-significant [`Self::BITS`]
/// bits can be non-zero.
///
/// [`from_usize`]: Tag::from_usize
/// [`Self::BITS`]: Tag::BITS
fn into_usize(self) -> usize;
/// Re-creates the tag from the integer returned by [`into_usize`].
///
/// # Safety
///
/// The passed `tag` must be returned from `into_usize`.
/// The passed `tag` must be returned from [`into_usize`].
///
/// [`into_usize`]: Tag::into_usize
unsafe fn from_usize(tag: usize) -> Self;
}
unsafe impl<T> Pointer for Box<T> {
const BITS: usize = std::mem::align_of::<T>().trailing_zeros() as usize;
unsafe impl<T: ?Sized + Aligned> Pointer for Box<T> {
const BITS: u32 = bits_for::<Self::Target>();
#[inline]
fn into_usize(self) -> usize {
Box::into_raw(self) as usize
fn into_ptr(self) -> NonNull<T> {
// Safety: pointers from `Box::into_raw` are valid & non-null
unsafe { NonNull::new_unchecked(Box::into_raw(self)) }
}
#[inline]
unsafe fn from_usize(ptr: usize) -> Self {
Box::from_raw(ptr as *mut T)
}
unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R {
let raw = ManuallyDrop::new(Self::from_usize(ptr));
f(&raw)
unsafe fn from_ptr(ptr: NonNull<T>) -> Self {
// Safety: `ptr` comes from `into_ptr` which calls `Box::into_raw`
Box::from_raw(ptr.as_ptr())
}
}
unsafe impl<T> Pointer for Rc<T> {
const BITS: usize = std::mem::align_of::<T>().trailing_zeros() as usize;
unsafe impl<T: ?Sized + Aligned> Pointer for Rc<T> {
const BITS: u32 = bits_for::<Self::Target>();
#[inline]
fn into_usize(self) -> usize {
Rc::into_raw(self) as usize
fn into_ptr(self) -> NonNull<T> {
// Safety: pointers from `Rc::into_raw` are valid & non-null
unsafe { NonNull::new_unchecked(Rc::into_raw(self).cast_mut()) }
}
#[inline]
unsafe fn from_usize(ptr: usize) -> Self {
Rc::from_raw(ptr as *const T)
}
unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R {
let raw = ManuallyDrop::new(Self::from_usize(ptr));
f(&raw)
unsafe fn from_ptr(ptr: NonNull<T>) -> Self {
// Safety: `ptr` comes from `into_ptr` which calls `Rc::into_raw`
Rc::from_raw(ptr.as_ptr())
}
}
unsafe impl<T> Pointer for Arc<T> {
const BITS: usize = std::mem::align_of::<T>().trailing_zeros() as usize;
unsafe impl<T: ?Sized + Aligned> Pointer for Arc<T> {
const BITS: u32 = bits_for::<Self::Target>();
#[inline]
fn into_usize(self) -> usize {
Arc::into_raw(self) as usize
fn into_ptr(self) -> NonNull<T> {
// Safety: pointers from `Arc::into_raw` are valid & non-null
unsafe { NonNull::new_unchecked(Arc::into_raw(self).cast_mut()) }
}
#[inline]
unsafe fn from_usize(ptr: usize) -> Self {
Arc::from_raw(ptr as *const T)
}
unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R {
let raw = ManuallyDrop::new(Self::from_usize(ptr));
f(&raw)
unsafe fn from_ptr(ptr: NonNull<T>) -> Self {
// Safety: `ptr` comes from `into_ptr` which calls `Arc::into_raw`
Arc::from_raw(ptr.as_ptr())
}
}
unsafe impl<'a, T: 'a> Pointer for &'a T {
const BITS: usize = std::mem::align_of::<T>().trailing_zeros() as usize;
unsafe impl<'a, T: 'a + ?Sized + Aligned> Pointer for &'a T {
const BITS: u32 = bits_for::<Self::Target>();
#[inline]
fn into_usize(self) -> usize {
self as *const T as usize
fn into_ptr(self) -> NonNull<T> {
NonNull::from(self)
}
#[inline]
unsafe fn from_usize(ptr: usize) -> Self {
&*(ptr as *const T)
}
unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R {
f(&*(&ptr as *const usize as *const Self))
unsafe fn from_ptr(ptr: NonNull<T>) -> Self {
// Safety:
// `ptr` comes from `into_ptr` which gets the pointer from a reference
ptr.as_ref()
}
}
unsafe impl<'a, T: 'a> Pointer for &'a mut T {
const BITS: usize = std::mem::align_of::<T>().trailing_zeros() as usize;
unsafe impl<'a, T: 'a + ?Sized + Aligned> Pointer for &'a mut T {
const BITS: u32 = bits_for::<Self::Target>();
#[inline]
fn into_ptr(self) -> NonNull<T> {
NonNull::from(self)
}
#[inline]
unsafe fn from_ptr(mut ptr: NonNull<T>) -> Self {
// Safety:
// `ptr` comes from `into_ptr` which gets the pointer from a reference
ptr.as_mut()
}
}
/// Returns the number of bits available for use for tags in a pointer to `T`
/// (this is based on `T`'s alignment).
pub const fn bits_for<T: ?Sized + Aligned>() -> u32 {
crate::aligned::align_of::<T>().as_nonzero().trailing_zeros()
}
/// A tag type used in [`CopyTaggedPtr`] and [`TaggedPtr`] tests.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
#[cfg(test)]
enum Tag2 {
B00 = 0b00,
B01 = 0b01,
B10 = 0b10,
B11 = 0b11,
}
#[cfg(test)]
unsafe impl Tag for Tag2 {
const BITS: u32 = 2;
fn into_usize(self) -> usize {
self as *mut T as usize
self as _
}
#[inline]
unsafe fn from_usize(ptr: usize) -> Self {
&mut *(ptr as *mut T)
}
unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R {
f(&*(&ptr as *const usize as *const Self))
unsafe fn from_usize(tag: usize) -> Self {
match tag {
0b00 => Tag2::B00,
0b01 => Tag2::B01,
0b10 => Tag2::B10,
0b11 => Tag2::B11,
_ => unreachable!(),
}
}
}
#[cfg(test)]
impl<HCX> crate::stable_hasher::HashStable<HCX> for Tag2 {
fn hash_stable(&self, hcx: &mut HCX, hasher: &mut crate::stable_hasher::StableHasher) {
(*self as u8).hash_stable(hcx, hasher);
}
}

312
compiler/rustc_data_structures/src/tagged_ptr/copy.rs
View File

@ -1,78 +1,92 @@
use super::{Pointer, Tag};
use crate::stable_hasher::{HashStable, StableHasher};
use std::fmt;
use std::hash::{Hash, Hasher};
use std::marker::PhantomData;
use std::mem::ManuallyDrop;
use std::num::NonZeroUsize;
use std::ops::{Deref, DerefMut};
use std::ptr::NonNull;
/// A `Copy` TaggedPtr.
/// A [`Copy`] tagged pointer.
///
/// You should use this instead of the `TaggedPtr` type in all cases where
/// `P: Copy`.
/// This is essentially `{ pointer: P, tag: T }` packed in a single pointer.
///
/// You should use this instead of the [`TaggedPtr`] type in all cases where
/// `P` implements [`Copy`].
///
/// If `COMPARE_PACKED` is true, then the pointers will be compared and hashed without
/// unpacking. Otherwise we don't implement PartialEq/Eq/Hash; if you want that,
/// wrap the TaggedPtr.
/// unpacking. Otherwise we don't implement [`PartialEq`], [`Eq`] and [`Hash`];
/// if you want that, wrap the [`CopyTaggedPtr`].
///
/// [`TaggedPtr`]: crate::tagged_ptr::TaggedPtr
pub struct CopyTaggedPtr<P, T, const COMPARE_PACKED: bool>
where
P: Pointer,
T: Tag,
{
packed: NonZeroUsize,
data: PhantomData<(P, T)>,
/// This is semantically a pair of `pointer: P` and `tag: T` fields,
/// however we pack them in a single pointer, to save space.
///
/// We pack the tag into the **most**-significant bits of the pointer to
/// ease retrieval of the value. A left shift is a multiplication and
/// those are embeddable in instruction encoding, for example:
///
/// ```asm
/// // (<https://godbolt.org/z/jqcYPWEr3>)
/// example::shift_read3:
/// mov eax, dword ptr [8*rdi]
/// ret
///
/// example::mask_read3:
/// and rdi, -8
/// mov eax, dword ptr [rdi]
/// ret
/// ```
///
/// This is ASM outputted by rustc for reads of values behind tagged
/// pointers for different approaches of tagging:
/// - `shift_read3` uses `<< 3` (the tag is in the most-significant bits)
/// - `mask_read3` uses `& !0b111` (the tag is in the least-significant bits)
///
/// The shift approach thus produces less instructions and is likely faster
/// (see <https://godbolt.org/z/Y913sMdWb>).
///
/// Encoding diagram:
/// ```text
/// [ packed.addr ]
/// [ tag ] [ pointer.addr >> T::BITS ] <-- usize::BITS - T::BITS bits
/// ^
/// |
/// T::BITS bits
/// ```
///
/// The tag can be retrieved by `packed.addr() >> T::BITS` and the pointer
/// can be retrieved by `packed.map_addr(|addr| addr << T::BITS)`.
packed: NonNull<P::Target>,
tag_ghost: PhantomData<T>,
}
impl<P, T, const COMPARE_PACKED: bool> Copy for CopyTaggedPtr<P, T, COMPARE_PACKED>
where
P: Pointer,
T: Tag,
P: Copy,
{
}
impl<P, T, const COMPARE_PACKED: bool> Clone for CopyTaggedPtr<P, T, COMPARE_PACKED>
where
P: Pointer,
T: Tag,
P: Copy,
{
fn clone(&self) -> Self {
*self
}
}
// We pack the tag into the *upper* bits of the pointer to ease retrieval of the
// value; a left shift is a multiplication and those are embeddable in
// instruction encoding.
impl<P, T, const COMPARE_PACKED: bool> CopyTaggedPtr<P, T, COMPARE_PACKED>
// Note that even though `CopyTaggedPtr` is only really expected to work with
// `P: Copy`, can't add `P: Copy` bound, because `CopyTaggedPtr` is used in the
// `TaggedPtr`'s implementation.
impl<P, T, const CP: bool> CopyTaggedPtr<P, T, CP>
where
P: Pointer,
T: Tag,
{
const TAG_BIT_SHIFT: usize = usize::BITS as usize - T::BITS;
const ASSERTION: () = {
assert!(T::BITS <= P::BITS);
// Used for the transmute_copy's below
assert!(std::mem::size_of::<&P::Target>() == std::mem::size_of::<usize>());
};
/// Tags `pointer` with `tag`.
///
/// Note that this leaks `pointer`: it won't be dropped when
/// `CopyTaggedPtr` is dropped. If you have a pointer with a significant
/// drop, use [`TaggedPtr`] instead.
///
/// [`TaggedPtr`]: crate::tagged_ptr::TaggedPtr
pub fn new(pointer: P, tag: T) -> Self {
// Trigger assert!
let () = Self::ASSERTION;
let packed_tag = tag.into_usize() << Self::TAG_BIT_SHIFT;
Self {
// SAFETY: We know that the pointer is non-null, as it must be
// dereferenceable per `Pointer` safety contract.
packed: unsafe {
NonZeroUsize::new_unchecked((P::into_usize(pointer) >> T::BITS) | packed_tag)
},
data: PhantomData,
}
Self { packed: Self::pack(P::into_ptr(pointer), tag), tag_ghost: PhantomData }
}
pub(super) fn pointer_raw(&self) -> usize {
self.packed.get() << T::BITS
}
/// Retrieves the pointer.
pub fn pointer(self) -> P
where
P: Copy,
@ -81,66 +95,138 @@ where
//
// Note that this isn't going to double-drop or anything because we have
// P: Copy
unsafe { P::from_usize(self.pointer_raw()) }
}
pub fn pointer_ref(&self) -> &P::Target {
// SAFETY: pointer_raw returns the original pointer
unsafe { std::mem::transmute_copy(&self.pointer_raw()) }
}
pub fn pointer_mut(&mut self) -> &mut P::Target
where
P: std::ops::DerefMut,
{
// SAFETY: pointer_raw returns the original pointer
unsafe { std::mem::transmute_copy(&self.pointer_raw()) }
unsafe { P::from_ptr(self.pointer_raw()) }
}
/// Retrieves the tag.
#[inline]
pub fn tag(&self) -> T {
unsafe { T::from_usize(self.packed.get() >> Self::TAG_BIT_SHIFT) }
// Unpack the tag, according to the `self.packed` encoding scheme
let tag = self.packed.addr().get() >> Self::TAG_BIT_SHIFT;
// Safety:
// The shift retrieves the original value from `T::into_usize`,
// satisfying `T::from_usize`'s preconditions.
unsafe { T::from_usize(tag) }
}
/// Sets the tag to a new value.
#[inline]
pub fn set_tag(&mut self, tag: T) {
let mut packed = self.packed.get();
let new_tag = T::into_usize(tag) << Self::TAG_BIT_SHIFT;
let tag_mask = (1 << T::BITS) - 1;
packed &= !(tag_mask << Self::TAG_BIT_SHIFT);
packed |= new_tag;
self.packed = unsafe { NonZeroUsize::new_unchecked(packed) };
self.packed = Self::pack(self.pointer_raw(), tag);
}
const TAG_BIT_SHIFT: u32 = usize::BITS - T::BITS;
const ASSERTION: () = { assert!(T::BITS <= P::BITS) };
/// Pack pointer `ptr` that comes from [`P::into_ptr`] with a `tag`,
/// according to `self.packed` encoding scheme.
///
/// [`P::into_ptr`]: Pointer::into_ptr
fn pack(ptr: NonNull<P::Target>, tag: T) -> NonNull<P::Target> {
// Trigger assert!
let () = Self::ASSERTION;
let packed_tag = tag.into_usize() << Self::TAG_BIT_SHIFT;
ptr.map_addr(|addr| {
// Safety:
// - The pointer is `NonNull` => it's address is `NonZeroUsize`
// - `P::BITS` least significant bits are always zero (`Pointer` contract)
// - `T::BITS <= P::BITS` (from `Self::ASSERTION`)
//
// Thus `addr >> T::BITS` is guaranteed to be non-zero.
//
// `{non_zero} | packed_tag` can't make the value zero.
let packed = (addr.get() >> T::BITS) | packed_tag;
unsafe { NonZeroUsize::new_unchecked(packed) }
})
}
/// Retrieves the original raw pointer from `self.packed`.
pub(super) fn pointer_raw(&self) -> NonNull<P::Target> {
self.packed.map_addr(|addr| unsafe { NonZeroUsize::new_unchecked(addr.get() << T::BITS) })
}
/// This provides a reference to the `P` pointer itself, rather than the
/// `Deref::Target`. It is used for cases where we want to call methods
/// that may be implement differently for the Pointer than the Pointee
/// (e.g., `Rc::clone` vs cloning the inner value).
pub(super) fn with_pointer_ref<R>(&self, f: impl FnOnce(&P) -> R) -> R {
// Safety:
// - `self.raw.pointer_raw()` is originally returned from `P::into_ptr`
// and as such is valid for `P::from_ptr`.
// - This also allows us to not care whatever `f` panics or not.
// - Even though we create a copy of the pointer, we store it inside
// `ManuallyDrop` and only access it by-ref, so we don't double-drop.
//
// Semantically this is just `f(&self.pointer)` (where `self.pointer`
// is non-packed original pointer).
//
// Note that even though `CopyTaggedPtr` is only really expected to
// work with `P: Copy`, we have to assume `P: ?Copy`, because
// `CopyTaggedPtr` is used in the `TaggedPtr`'s implementation.
let ptr = unsafe { ManuallyDrop::new(P::from_ptr(self.pointer_raw())) };
f(&ptr)
}
}
impl<P, T, const COMPARE_PACKED: bool> std::ops::Deref for CopyTaggedPtr<P, T, COMPARE_PACKED>
impl<P, T, const CP: bool> Copy for CopyTaggedPtr<P, T, CP>
where
P: Pointer + Copy,
T: Tag,
{
}
impl<P, T, const CP: bool> Clone for CopyTaggedPtr<P, T, CP>
where
P: Pointer + Copy,
T: Tag,
{
fn clone(&self) -> Self {
*self
}
}
impl<P, T, const CP: bool> Deref for CopyTaggedPtr<P, T, CP>
where
P: Pointer,
T: Tag,
{
type Target = P::Target;
fn deref(&self) -> &Self::Target {
self.pointer_ref()
// Safety:
// `pointer_raw` returns the original pointer from `P::into_ptr` which,
// by the `Pointer`'s contract, must be valid.
unsafe { self.pointer_raw().as_ref() }
}
}
impl<P, T, const COMPARE_PACKED: bool> std::ops::DerefMut for CopyTaggedPtr<P, T, COMPARE_PACKED>
impl<P, T, const CP: bool> DerefMut for CopyTaggedPtr<P, T, CP>
where
P: Pointer + std::ops::DerefMut,
P: Pointer + DerefMut,
T: Tag,
{
fn deref_mut(&mut self) -> &mut Self::Target {
self.pointer_mut()
// Safety:
// `pointer_raw` returns the original pointer from `P::into_ptr` which,
// by the `Pointer`'s contract, must be valid for writes if
// `P: DerefMut`.
unsafe { self.pointer_raw().as_mut() }
}
}
impl<P, T, const COMPARE_PACKED: bool> fmt::Debug for CopyTaggedPtr<P, T, COMPARE_PACKED>
impl<P, T, const CP: bool> fmt::Debug for CopyTaggedPtr<P, T, CP>
where
P: Pointer,
P::Target: fmt::Debug,
P: Pointer + fmt::Debug,
T: Tag + fmt::Debug,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("CopyTaggedPtr")
.field("pointer", &self.pointer_ref())
.field("tag", &self.tag())
.finish()
self.with_pointer_ref(|ptr| {
f.debug_struct("CopyTaggedPtr").field("pointer", ptr).field("tag", &self.tag()).finish()
})
}
}
@ -161,25 +247,73 @@ where
{
}
impl<P, T> std::hash::Hash for CopyTaggedPtr<P, T, true>
impl<P, T> Hash for CopyTaggedPtr<P, T, true>
where
P: Pointer,
T: Tag,
{
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
fn hash<H: Hasher>(&self, state: &mut H) {
self.packed.hash(state);
}
}
impl<P, T, HCX, const COMPARE_PACKED: bool> HashStable<HCX> for CopyTaggedPtr<P, T, COMPARE_PACKED>
impl<P, T, HCX, const CP: bool> HashStable<HCX> for CopyTaggedPtr<P, T, CP>
where
P: Pointer + HashStable<HCX>,
T: Tag + HashStable<HCX>,
{
fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) {
unsafe {
Pointer::with_ref(self.pointer_raw(), |p: &P| p.hash_stable(hcx, hasher));
}
self.with_pointer_ref(|ptr| ptr.hash_stable(hcx, hasher));
self.tag().hash_stable(hcx, hasher);
}
}
// Safety:
// `CopyTaggedPtr<P, T, ..>` is semantically just `{ ptr: P, tag: T }`, as such
// it's ok to implement `Sync` as long as `P: Sync, T: Sync`
unsafe impl<P, T, const CP: bool> Sync for CopyTaggedPtr<P, T, CP>
where
P: Sync + Pointer,
T: Sync + Tag,
{
}
// Safety:
// `CopyTaggedPtr<P, T, ..>` is semantically just `{ ptr: P, tag: T }`, as such
// it's ok to implement `Send` as long as `P: Send, T: Send`
unsafe impl<P, T, const CP: bool> Send for CopyTaggedPtr<P, T, CP>
where
P: Send + Pointer,
T: Send + Tag,
{
}
/// Test that `new` does not compile if there is not enough alignment for the
/// tag in the pointer.
///
/// ```compile_fail,E0080
/// use rustc_data_structures::tagged_ptr::{CopyTaggedPtr, Tag};
///
/// #[derive(Copy, Clone, Debug, PartialEq, Eq)]
/// enum Tag2 { B00 = 0b00, B01 = 0b01, B10 = 0b10, B11 = 0b11 };
///
/// unsafe impl Tag for Tag2 {
/// const BITS: u32 = 2;
///
/// fn into_usize(self) -> usize { todo!() }
/// unsafe fn from_usize(tag: usize) -> Self { todo!() }
/// }
///
/// let value = 12u16;
/// let reference = &value;
/// let tag = Tag2::B01;
///
/// let _ptr = CopyTaggedPtr::<_, _, true>::new(reference, tag);
/// ```
// For some reason miri does not get the compile error
// probably it `check`s instead of `build`ing?
#[cfg(not(miri))]
const _: () = ();
#[cfg(test)]
mod tests;

50
compiler/rustc_data_structures/src/tagged_ptr/copy/tests.rs Normal file
View File

@ -0,0 +1,50 @@
use std::ptr;
use crate::stable_hasher::{HashStable, StableHasher};
use crate::tagged_ptr::{CopyTaggedPtr, Pointer, Tag, Tag2};
#[test]
fn smoke() {
let value = 12u32;
let reference = &value;
let tag = Tag2::B01;
let ptr = tag_ptr(reference, tag);
assert_eq!(ptr.tag(), tag);
assert_eq!(*ptr, 12);
assert!(ptr::eq(ptr.pointer(), reference));
let copy = ptr;
let mut ptr = ptr;
ptr.set_tag(Tag2::B00);
assert_eq!(ptr.tag(), Tag2::B00);
assert_eq!(copy.tag(), tag);
assert_eq!(*copy, 12);
assert!(ptr::eq(copy.pointer(), reference));
}
#[test]
fn stable_hash_hashes_as_tuple() {
let hash_packed = {
let mut hasher = StableHasher::new();
tag_ptr(&12, Tag2::B11).hash_stable(&mut (), &mut hasher);
hasher.finalize()
};
let hash_tupled = {
let mut hasher = StableHasher::new();
(&12, Tag2::B11).hash_stable(&mut (), &mut hasher);
hasher.finalize()
};
assert_eq!(hash_packed, hash_tupled);
}
/// Helper to create tagged pointers without specifying `COMPARE_PACKED` if it does not matter.
fn tag_ptr<P: Pointer, T: Tag>(ptr: P, tag: T) -> CopyTaggedPtr<P, T, true> {
CopyTaggedPtr::new(ptr, tag)
}

126
compiler/rustc_data_structures/src/tagged_ptr/drop.rs
View File

@ -1,14 +1,21 @@
use super::{Pointer, Tag};
use crate::stable_hasher::{HashStable, StableHasher};
use std::fmt;
use std::hash::{Hash, Hasher};
use std::ops::{Deref, DerefMut};
use super::CopyTaggedPtr;
use super::{Pointer, Tag};
use crate::stable_hasher::{HashStable, StableHasher};
/// A TaggedPtr implementing `Drop`.
/// A tagged pointer that supports pointers that implement [`Drop`].
///
/// This is essentially `{ pointer: P, tag: T }` packed in a single pointer.
///
/// You should use [`CopyTaggedPtr`] instead of the this type in all cases
/// where `P` implements [`Copy`].
///
/// If `COMPARE_PACKED` is true, then the pointers will be compared and hashed without
/// unpacking. Otherwise we don't implement PartialEq/Eq/Hash; if you want that,
/// wrap the TaggedPtr.
/// unpacking. Otherwise we don't implement [`PartialEq`], [`Eq`] and [`Hash`];
/// if you want that, wrap the [`TaggedPtr`].
pub struct TaggedPtr<P, T, const COMPARE_PACKED: bool>
where
P: Pointer,
@ -17,58 +24,61 @@ where
raw: CopyTaggedPtr<P, T, COMPARE_PACKED>,
}
impl<P, T, const COMPARE_PACKED: bool> Clone for TaggedPtr<P, T, COMPARE_PACKED>
impl<P, T, const CP: bool> TaggedPtr<P, T, CP>
where
P: Pointer,
T: Tag,
{
/// Tags `pointer` with `tag`.
pub fn new(pointer: P, tag: T) -> Self {
TaggedPtr { raw: CopyTaggedPtr::new(pointer, tag) }
}
/// Retrieves the tag.
pub fn tag(&self) -> T {
self.raw.tag()
}
/// Sets the tag to a new value.
pub fn set_tag(&mut self, tag: T) {
self.raw.set_tag(tag)
}
}
impl<P, T, const CP: bool> Clone for TaggedPtr<P, T, CP>
where
P: Pointer + Clone,
T: Tag,
{
fn clone(&self) -> Self {
unsafe { Self::new(P::with_ref(self.raw.pointer_raw(), |p| p.clone()), self.raw.tag()) }
let ptr = self.raw.with_pointer_ref(P::clone);
Self::new(ptr, self.tag())
}
}
// We pack the tag into the *upper* bits of the pointer to ease retrieval of the
// value; a right shift is a multiplication and those are embeddable in
// instruction encoding.
impl<P, T, const COMPARE_PACKED: bool> TaggedPtr<P, T, COMPARE_PACKED>
where
P: Pointer,
T: Tag,
{
pub fn new(pointer: P, tag: T) -> Self {
TaggedPtr { raw: CopyTaggedPtr::new(pointer, tag) }
}
pub fn pointer_ref(&self) -> &P::Target {
self.raw.pointer_ref()
}
pub fn tag(&self) -> T {
self.raw.tag()
}
}
impl<P, T, const COMPARE_PACKED: bool> std::ops::Deref for TaggedPtr<P, T, COMPARE_PACKED>
impl<P, T, const CP: bool> Deref for TaggedPtr<P, T, CP>
where
P: Pointer,
T: Tag,
{
type Target = P::Target;
fn deref(&self) -> &Self::Target {
self.raw.pointer_ref()
self.raw.deref()
}
}
impl<P, T, const COMPARE_PACKED: bool> std::ops::DerefMut for TaggedPtr<P, T, COMPARE_PACKED>
impl<P, T, const CP: bool> DerefMut for TaggedPtr<P, T, CP>
where
P: Pointer + std::ops::DerefMut,
P: Pointer + DerefMut,
T: Tag,
{
fn deref_mut(&mut self) -> &mut Self::Target {
self.raw.pointer_mut()
self.raw.deref_mut()
}
}
impl<P, T, const COMPARE_PACKED: bool> Drop for TaggedPtr<P, T, COMPARE_PACKED>
impl<P, T, const CP: bool> Drop for TaggedPtr<P, T, CP>
where
P: Pointer,
T: Tag,
@ -76,22 +86,20 @@ where
fn drop(&mut self) {
// No need to drop the tag, as it's Copy
unsafe {
drop(P::from_usize(self.raw.pointer_raw()));
drop(P::from_ptr(self.raw.pointer_raw()));
}
}
}
impl<P, T, const COMPARE_PACKED: bool> fmt::Debug for TaggedPtr<P, T, COMPARE_PACKED>
impl<P, T, const CP: bool> fmt::Debug for TaggedPtr<P, T, CP>
where
P: Pointer,
P::Target: fmt::Debug,
P: Pointer + fmt::Debug,
T: Tag + fmt::Debug,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("TaggedPtr")
.field("pointer", &self.pointer_ref())
.field("tag", &self.tag())
.finish()
self.raw.with_pointer_ref(|ptr| {
f.debug_struct("TaggedPtr").field("pointer", ptr).field("tag", &self.tag()).finish()
})
}
}
@ -112,17 +120,17 @@ where
{
}
impl<P, T> std::hash::Hash for TaggedPtr<P, T, true>
impl<P, T> Hash for TaggedPtr<P, T, true>
where
P: Pointer,
T: Tag,
{
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
fn hash<H: Hasher>(&self, state: &mut H) {
self.raw.hash(state);
}
}
impl<P, T, HCX, const COMPARE_PACKED: bool> HashStable<HCX> for TaggedPtr<P, T, COMPARE_PACKED>
impl<P, T, HCX, const CP: bool> HashStable<HCX> for TaggedPtr<P, T, CP>
where
P: Pointer + HashStable<HCX>,
T: Tag + HashStable<HCX>,
@ -131,3 +139,33 @@ where
self.raw.hash_stable(hcx, hasher);
}
}
/// Test that `new` does not compile if there is not enough alignment for the
/// tag in the pointer.
///
/// ```compile_fail,E0080
/// use rustc_data_structures::tagged_ptr::{TaggedPtr, Tag};
///
/// #[derive(Copy, Clone, Debug, PartialEq, Eq)]
/// enum Tag2 { B00 = 0b00, B01 = 0b01, B10 = 0b10, B11 = 0b11 };
///
/// unsafe impl Tag for Tag2 {
/// const BITS: u32 = 2;
///
/// fn into_usize(self) -> usize { todo!() }
/// unsafe fn from_usize(tag: usize) -> Self { todo!() }
/// }
///
/// let value = 12u16;
/// let reference = &value;
/// let tag = Tag2::B01;
///
/// let _ptr = TaggedPtr::<_, _, true>::new(reference, tag);
/// ```
// For some reason miri does not get the compile error
// probably it `check`s instead of `build`ing?
#[cfg(not(miri))]
const _: () = ();
#[cfg(test)]
mod tests;

71
compiler/rustc_data_structures/src/tagged_ptr/drop/tests.rs Normal file
View File

@ -0,0 +1,71 @@
use std::{ptr, sync::Arc};
use crate::tagged_ptr::{Pointer, Tag, Tag2, TaggedPtr};
#[test]
fn smoke() {
let value = 12u32;
let reference = &value;
let tag = Tag2::B01;
let ptr = tag_ptr(reference, tag);
assert_eq!(ptr.tag(), tag);
assert_eq!(*ptr, 12);
let clone = ptr.clone();
assert_eq!(clone.tag(), tag);
assert_eq!(*clone, 12);
let mut ptr = ptr;
ptr.set_tag(Tag2::B00);
assert_eq!(ptr.tag(), Tag2::B00);
assert_eq!(clone.tag(), tag);
assert_eq!(*clone, 12);
assert!(ptr::eq(&*ptr, &*clone))
}
#[test]
fn boxed() {
let value = 12u32;
let boxed = Box::new(value);
let tag = Tag2::B01;
let ptr = tag_ptr(boxed, tag);
assert_eq!(ptr.tag(), tag);
assert_eq!(*ptr, 12);
let clone = ptr.clone();
assert_eq!(clone.tag(), tag);
assert_eq!(*clone, 12);
let mut ptr = ptr;
ptr.set_tag(Tag2::B00);
assert_eq!(ptr.tag(), Tag2::B00);
assert_eq!(clone.tag(), tag);
assert_eq!(*clone, 12);
assert!(!ptr::eq(&*ptr, &*clone))
}
#[test]
fn arclones() {
let value = 12u32;
let arc = Arc::new(value);
let tag = Tag2::B01;
let ptr = tag_ptr(arc, tag);
assert_eq!(ptr.tag(), tag);
assert_eq!(*ptr, 12);
let clone = ptr.clone();
assert!(ptr::eq(&*ptr, &*clone))
}
/// Helper to create tagged pointers without specifying `COMPARE_PACKED` if it does not matter.
fn tag_ptr<P: Pointer, T: Tag>(ptr: P, tag: T) -> TaggedPtr<P, T, true> {
TaggedPtr::new(ptr, tag)
}

1
compiler/rustc_middle/src/lib.rs
View File

@ -59,6 +59,7 @@
#![feature(result_option_inspect)]
#![feature(const_option)]
#![feature(trait_alias)]
#![feature(ptr_alignment_type)]
#![recursion_limit = "512"]
#![allow(rustc::potential_query_instability)]

28
compiler/rustc_middle/src/ty/list.rs
View File

@ -1,4 +1,5 @@
use crate::arena::Arena;
use rustc_data_structures::aligned::{align_of, Aligned};
use rustc_serialize::{Encodable, Encoder};
use std::alloc::Layout;
use std::cmp::Ordering;
@ -198,22 +199,17 @@ impl<'a, T: Copy> IntoIterator for &'a List<T> {
unsafe impl<T: Sync> Sync for List<T> {}
unsafe impl<'a, T: 'a> rustc_data_structures::tagged_ptr::Pointer for &'a List<T> {
const BITS: usize = std::mem::align_of::<usize>().trailing_zeros() as usize;
#[inline]
fn into_usize(self) -> usize {
self as *const List<T> as usize
// Safety:
// Layouts of `Equivalent<T>` and `List<T>` are the same, modulo opaque tail,
// thus aligns of `Equivalent<T>` and `List<T>` must be the same.
unsafe impl<T> Aligned for List<T> {
const ALIGN: ptr::Alignment = {
#[repr(C)]
struct Equivalent<T> {
_len: usize,
_data: [T; 0],
}
#[inline]
unsafe fn from_usize(ptr: usize) -> &'a List<T> {
&*(ptr as *const List<T>)
}
unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R {
// `Self` is `&'a List<T>` which impls `Copy`, so this is fine.
let ptr = Self::from_usize(ptr);
f(&ptr)
}
align_of::<Equivalent<T>>()
};
}

4
compiler/rustc_middle/src/ty/mod.rs
View File

@ -1627,7 +1627,8 @@ struct ParamTag {
}
unsafe impl rustc_data_structures::tagged_ptr::Tag for ParamTag {
const BITS: usize = 2;
const BITS: u32 = 2;
#[inline]
fn into_usize(self) -> usize {
match self {
@ -1637,6 +1638,7 @@ unsafe impl rustc_data_structures::tagged_ptr::Tag for ParamTag {
Self { reveal: traits::Reveal::All, constness: hir::Constness::Const } => 3,
}
}
#[inline]
unsafe fn from_usize(ptr: usize) -> Self {
match ptr {