nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-25 08:13:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove const-support for align_offset

Browse Source 
Operations like is_aligned would return actively wrong results at compile-time,
i.e. calling it on the same pointer at compiletime and runtime could yield
different results. That's no good.

Instead of having hacks to make align_offset kind-of work in const-eval, just
use const_eval_select in the few places where it makes sense, which also ensures
those places are all aware they need to make sure the fallback behavior is
consistent.
This commit is contained in:
Ralf Jung 2024-10-31 21:30:23 +01:00
parent 67395551d0
commit 19e287060d
16 changed files with 185 additions and 1022 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
compiler/rustc_const_eval/messages.ftl
View File

@ -1,9 +1,6 @@
const_eval_address_space_full =
there are no more free addresses in the address space
const_eval_align_offset_invalid_align =
`align_offset` called with non-power-of-two align: {$target_align}
const_eval_alignment_check_failed =
{$msg ->
[AccessedPtr] accessing memory

88
compiler/rustc_const_eval/src/const_eval/machine.rs
View File

@ -1,7 +1,6 @@
use std::borrow::{Borrow, Cow};
use std::fmt;
use std::hash::Hash;
use std::ops::ControlFlow;
use rustc_ast::Mutability;
use rustc_data_structures::fx::{FxHashMap, FxIndexMap, IndexEntry};
@ -9,7 +8,7 @@ use rustc_hir::def_id::{DefId, LocalDefId};
use rustc_hir::{self as hir, CRATE_HIR_ID, LangItem};
use rustc_middle::mir::AssertMessage;
use rustc_middle::query::TyCtxtAt;
use rustc_middle::ty::layout::{FnAbiOf, TyAndLayout};
use rustc_middle::ty::layout::TyAndLayout;
use rustc_middle::ty::{self, Ty, TyCtxt};
use rustc_middle::{bug, mir};
use rustc_span::Span;
@ -23,9 +22,9 @@ use crate::errors::{LongRunning, LongRunningWarn};
use crate::fluent_generated as fluent;
use crate::interpret::{
self, AllocId, AllocRange, ConstAllocation, CtfeProvenance, FnArg, Frame, GlobalAlloc, ImmTy,
InterpCx, InterpResult, MPlaceTy, OpTy, Pointer, PointerArithmetic, RangeSet, Scalar,
StackPopCleanup, compile_time_machine, interp_ok, throw_exhaust, throw_inval, throw_ub,
throw_ub_custom, throw_unsup, throw_unsup_format,
InterpCx, InterpResult, MPlaceTy, OpTy, Pointer, RangeSet, Scalar, compile_time_machine,
interp_ok, throw_exhaust, throw_inval, throw_ub, throw_ub_custom, throw_unsup,
throw_unsup_format,
};
/// When hitting this many interpreted terminators we emit a deny by default lint
@ -227,8 +226,8 @@ impl<'tcx> CompileTimeInterpCx<'tcx> {
&mut self,
instance: ty::Instance<'tcx>,
args: &[FnArg<'tcx>],
dest: &MPlaceTy<'tcx>,
ret: Option<mir::BasicBlock>,
_dest: &MPlaceTy<'tcx>,
_ret: Option<mir::BasicBlock>,
) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
let def_id = instance.def_id();
@ -260,85 +259,10 @@ impl<'tcx> CompileTimeInterpCx<'tcx> {
);
return interp_ok(Some(new_instance));
} else if self.tcx.is_lang_item(def_id, LangItem::AlignOffset) {
let args = self.copy_fn_args(args);
// For align_offset, we replace the function call if the pointer has no address.
match self.align_offset(instance, &args, dest, ret)? {
ControlFlow::Continue(()) => return interp_ok(Some(instance)),
ControlFlow::Break(()) => return interp_ok(None),
}
}
interp_ok(Some(instance))
}
/// `align_offset(ptr, target_align)` needs special handling in const eval, because the pointer
/// may not have an address.
///
/// If `ptr` does have a known address, then we return `Continue(())` and the function call should
/// proceed as normal.
///
/// If `ptr` doesn't have an address, but its underlying allocation's alignment is at most
/// `target_align`, then we call the function again with an dummy address relative to the
/// allocation.
///
/// If `ptr` doesn't have an address and `target_align` is stricter than the underlying
/// allocation's alignment, then we return `usize::MAX` immediately.
fn align_offset(
&mut self,
instance: ty::Instance<'tcx>,
args: &[OpTy<'tcx>],
dest: &MPlaceTy<'tcx>,
ret: Option<mir::BasicBlock>,
) -> InterpResult<'tcx, ControlFlow<()>> {
assert_eq!(args.len(), 2);
let ptr = self.read_pointer(&args[0])?;
let target_align = self.read_scalar(&args[1])?.to_target_usize(self)?;
if !target_align.is_power_of_two() {
throw_ub_custom!(
fluent::const_eval_align_offset_invalid_align,
target_align = target_align,
);
}
match self.ptr_try_get_alloc_id(ptr, 0) {
Ok((alloc_id, offset, _extra)) => {
let (_size, alloc_align, _kind) = self.get_alloc_info(alloc_id);
if target_align <= alloc_align.bytes() {
// Extract the address relative to the allocation base that is definitely
// sufficiently aligned and call `align_offset` again.
let addr = ImmTy::from_uint(offset.bytes(), args[0].layout).into();
let align = ImmTy::from_uint(target_align, args[1].layout).into();
let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty())?;
// Push the stack frame with our own adjusted arguments.
self.init_stack_frame(
instance,
self.load_mir(instance.def, None)?,
fn_abi,
&[FnArg::Copy(addr), FnArg::Copy(align)],
/* with_caller_location = */ false,
dest,
StackPopCleanup::Goto { ret, unwind: mir::UnwindAction::Unreachable },
)?;
interp_ok(ControlFlow::Break(()))
} else {
// Not alignable in const, return `usize::MAX`.
let usize_max = Scalar::from_target_usize(self.target_usize_max(), self);
self.write_scalar(usize_max, dest)?;
self.return_to_block(ret)?;
interp_ok(ControlFlow::Break(()))
}
}
Err(_addr) => {
// The pointer has an address, continue with function call.
interp_ok(ControlFlow::Continue(()))
}
}
}
/// See documentation on the `ptr_guaranteed_cmp` intrinsic.
fn guaranteed_cmp(&mut self, a: Scalar, b: Scalar) -> InterpResult<'tcx, u8> {
interp_ok(match (a, b) {

3
compiler/rustc_hir/src/lang_items.rs
View File

@ -348,9 +348,6 @@ language_item_table! {
MaybeUninit, sym::maybe_uninit, maybe_uninit, Target::Union, GenericRequirement::None;
/// Align offset for stride != 1; must not panic.
AlignOffset, sym::align_offset, align_offset_fn, Target::Fn, GenericRequirement::None;
Termination, sym::termination, termination, Target::Trait, GenericRequirement::None;
Try, sym::Try, try_trait, Target::Trait, GenericRequirement::None;

1
compiler/rustc_span/src/symbol.rs
View File

@ -378,7 +378,6 @@ symbols! {
aggregate_raw_ptr,
alias,
align,
align_offset,
alignment,
all,
alloc,

2
library/core/src/lib.rs
View File

@ -112,7 +112,6 @@
#![feature(asm_experimental_arch)]
#![feature(const_align_of_val)]
#![feature(const_align_of_val_raw)]
#![feature(const_align_offset)]
#![feature(const_alloc_layout)]
#![feature(const_arguments_as_str)]
#![feature(const_black_box)]
@ -125,7 +124,6 @@
#![feature(const_nonnull_new)]
#![feature(const_option_ext)]
#![feature(const_pin_2)]
#![feature(const_pointer_is_aligned)]
#![feature(const_ptr_is_null)]
#![feature(const_ptr_sub_ptr)]
#![feature(const_raw_ptr_comparison)]

192
library/core/src/ptr/const_ptr.rs
View File

@ -1358,15 +1358,6 @@ impl<T: ?Sized> *const T {
/// beyond the allocation that the pointer points into. It is up to the caller to ensure that
/// the returned offset is correct in all terms other than alignment.
///
/// When this is called during compile-time evaluation (which is unstable), the implementation
/// may return `usize::MAX` in cases where that can never happen at runtime. This is because the
/// actual alignment of pointers is not known yet during compile-time, so an offset with
/// guaranteed alignment can sometimes not be computed. For example, a buffer declared as `[u8;
/// N]` might be allocated at an odd or an even address, but at compile-time this is not yet
/// known, so the execution has to be correct for either choice. It is therefore impossible to
/// find an offset that is guaranteed to be 2-aligned. (This behavior is subject to change, as usual
/// for unstable APIs.)
///
/// # Panics
///
/// The function panics if `align` is not a power-of-two.
@ -1395,8 +1386,7 @@ impl<T: ?Sized> *const T {
#[must_use]
#[inline]
#[stable(feature = "align_offset", since = "1.36.0")]
#[rustc_const_unstable(feature = "const_align_offset", issue = "90962")]
pub const fn align_offset(self, align: usize) -> usize
pub fn align_offset(self, align: usize) -> usize
where
T: Sized,
{
@ -1431,94 +1421,10 @@ impl<T: ?Sized> *const T {
/// assert!(ptr.is_aligned());
/// assert!(!ptr.wrapping_byte_add(1).is_aligned());
/// ```
///
/// # At compiletime
/// **Note: Alignment at compiletime is experimental and subject to change. See the
/// [tracking issue] for details.**
///
/// At compiletime, the compiler may not know where a value will end up in memory.
/// Calling this function on a pointer created from a reference at compiletime will only
/// return `true` if the pointer is guaranteed to be aligned. This means that the pointer
/// is never aligned if cast to a type with a stricter alignment than the reference's
/// underlying allocation.
///
/// ```
/// #![feature(const_pointer_is_aligned)]
///
/// // On some platforms, the alignment of primitives is less than their size.
/// #[repr(align(4))]
/// struct AlignedI32(i32);
/// #[repr(align(8))]
/// struct AlignedI64(i64);
///
/// const _: () = {
/// let data = AlignedI32(42);
/// let ptr = &data as *const AlignedI32;
/// assert!(ptr.is_aligned());
///
/// // At runtime either `ptr1` or `ptr2` would be aligned, but at compiletime neither is aligned.
/// let ptr1 = ptr.cast::<AlignedI64>();
/// let ptr2 = ptr.wrapping_add(1).cast::<AlignedI64>();
/// assert!(!ptr1.is_aligned());
/// assert!(!ptr2.is_aligned());
/// };
/// ```
///
/// Due to this behavior, it is possible that a runtime pointer derived from a compiletime
/// pointer is aligned, even if the compiletime pointer wasn't aligned.
///
/// ```
/// #![feature(const_pointer_is_aligned)]
///
/// // On some platforms, the alignment of primitives is less than their size.
/// #[repr(align(4))]
/// struct AlignedI32(i32);
/// #[repr(align(8))]
/// struct AlignedI64(i64);
///
/// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned.
/// const COMPTIME_PTR: *const AlignedI32 = &AlignedI32(42);
/// const _: () = assert!(!COMPTIME_PTR.cast::<AlignedI64>().is_aligned());
/// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).cast::<AlignedI64>().is_aligned());
///
/// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned.
/// let runtime_ptr = COMPTIME_PTR;
/// assert_ne!(
/// runtime_ptr.cast::<AlignedI64>().is_aligned(),
/// runtime_ptr.wrapping_add(1).cast::<AlignedI64>().is_aligned(),
/// );
/// ```
///
/// If a pointer is created from a fixed address, this function behaves the same during
/// runtime and compiletime.
///
/// ```
/// #![feature(const_pointer_is_aligned)]
///
/// // On some platforms, the alignment of primitives is less than their size.
/// #[repr(align(4))]
/// struct AlignedI32(i32);
/// #[repr(align(8))]
/// struct AlignedI64(i64);
///
/// const _: () = {
/// let ptr = 40 as *const AlignedI32;
/// assert!(ptr.is_aligned());
///
/// // For pointers with a known address, runtime and compiletime behavior are identical.
/// let ptr1 = ptr.cast::<AlignedI64>();
/// let ptr2 = ptr.wrapping_add(1).cast::<AlignedI64>();
/// assert!(ptr1.is_aligned());
/// assert!(!ptr2.is_aligned());
/// };
/// ```
///
/// [tracking issue]: https://github.com/rust-lang/rust/issues/104203
#[must_use]
#[inline]
#[stable(feature = "pointer_is_aligned", since = "1.79.0")]
#[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")]
pub const fn is_aligned(self) -> bool
pub fn is_aligned(self) -> bool
where
T: Sized,
{
@ -1555,105 +1461,15 @@ impl<T: ?Sized> *const T {
///
/// assert_ne!(ptr.is_aligned_to(8), ptr.wrapping_add(1).is_aligned_to(8));
/// ```
///
/// # At compiletime
/// **Note: Alignment at compiletime is experimental and subject to change. See the
/// [tracking issue] for details.**
///
/// At compiletime, the compiler may not know where a value will end up in memory.
/// Calling this function on a pointer created from a reference at compiletime will only
/// return `true` if the pointer is guaranteed to be aligned. This means that the pointer
/// cannot be stricter aligned than the reference's underlying allocation.
///
/// ```
/// #![feature(pointer_is_aligned_to)]
/// #![feature(const_pointer_is_aligned)]
///
/// // On some platforms, the alignment of i32 is less than 4.
/// #[repr(align(4))]
/// struct AlignedI32(i32);
///
/// const _: () = {
/// let data = AlignedI32(42);
/// let ptr = &data as *const AlignedI32;
///
/// assert!(ptr.is_aligned_to(1));
/// assert!(ptr.is_aligned_to(2));
/// assert!(ptr.is_aligned_to(4));
///
/// // At compiletime, we know for sure that the pointer isn't aligned to 8.
/// assert!(!ptr.is_aligned_to(8));
/// assert!(!ptr.wrapping_add(1).is_aligned_to(8));
/// };
/// ```
///
/// Due to this behavior, it is possible that a runtime pointer derived from a compiletime
/// pointer is aligned, even if the compiletime pointer wasn't aligned.
///
/// ```
/// #![feature(pointer_is_aligned_to)]
/// #![feature(const_pointer_is_aligned)]
///
/// // On some platforms, the alignment of i32 is less than 4.
/// #[repr(align(4))]
/// struct AlignedI32(i32);
///
/// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned.
/// const COMPTIME_PTR: *const AlignedI32 = &AlignedI32(42);
/// const _: () = assert!(!COMPTIME_PTR.is_aligned_to(8));
/// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).is_aligned_to(8));
///
/// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned.
/// let runtime_ptr = COMPTIME_PTR;
/// assert_ne!(
/// runtime_ptr.is_aligned_to(8),
/// runtime_ptr.wrapping_add(1).is_aligned_to(8),
/// );
/// ```
///
/// If a pointer is created from a fixed address, this function behaves the same during
/// runtime and compiletime.
///
/// ```
/// #![feature(pointer_is_aligned_to)]
/// #![feature(const_pointer_is_aligned)]
///
/// const _: () = {
/// let ptr = 40 as *const u8;
/// assert!(ptr.is_aligned_to(1));
/// assert!(ptr.is_aligned_to(2));
/// assert!(ptr.is_aligned_to(4));
/// assert!(ptr.is_aligned_to(8));
/// assert!(!ptr.is_aligned_to(16));
/// };
/// ```
///
/// [tracking issue]: https://github.com/rust-lang/rust/issues/104203
#[must_use]
#[inline]
#[unstable(feature = "pointer_is_aligned_to", issue = "96284")]
#[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")]
pub const fn is_aligned_to(self, align: usize) -> bool {
pub fn is_aligned_to(self, align: usize) -> bool {
if !align.is_power_of_two() {
panic!("is_aligned_to: align is not a power-of-two");
}
#[inline]
fn runtime_impl(ptr: *const (), align: usize) -> bool {
ptr.addr() & (align - 1) == 0
}
#[inline]
#[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")]
const fn const_impl(ptr: *const (), align: usize) -> bool {
// We can't use the address of `self` in a `const fn`, so we use `align_offset` instead.
ptr.align_offset(align) == 0
}
// The cast to `()` is used to
// 1. deal with fat pointers; and
// 2. ensure that `align_offset` (in `const_impl`) doesn't actually try to compute an offset.
const_eval_select((self.cast::<()>(), align), const_impl, runtime_impl)
self.addr() & (align - 1) == 0
}
}

10
library/core/src/ptr/mod.rs
View File

@ -1852,9 +1852,7 @@ pub unsafe fn write_volatile<T>(dst: *mut T, src: T) {
///
/// Any questions go to @nagisa.
#[allow(ptr_to_integer_transmute_in_consts)]
#[lang = "align_offset"]
#[rustc_const_unstable(feature = "const_align_offset", issue = "90962")]
pub(crate) const unsafe fn align_offset<T: Sized>(p: *const T, a: usize) -> usize {
pub(crate) unsafe fn align_offset<T: Sized>(p: *const T, a: usize) -> usize {
// FIXME(#75598): Direct use of these intrinsics improves codegen significantly at opt-level <=
// 1, where the method versions of these operations are not inlined.
use intrinsics::{
@ -1915,11 +1913,7 @@ pub(crate) const unsafe fn align_offset<T: Sized>(p: *const T, a: usize) -> usiz
let stride = mem::size_of::<T>();
// SAFETY: This is just an inlined `p.addr()` (which is not
// a `const fn` so we cannot call it).
// During const eval, we hook this function to ensure that the pointer never
// has provenance, making this sound.
let addr: usize = unsafe { mem::transmute(p) };
let addr: usize = p.addr();
// SAFETY: `a` is a power-of-two, therefore non-zero.
let a_minus_one = unsafe { unchecked_sub(a, 1) };

186
library/core/src/ptr/mut_ptr.rs
View File

@ -1,6 +1,5 @@
use super::*;
use crate::cmp::Ordering::{Equal, Greater, Less};
use crate::intrinsics::const_eval_select;
use crate::mem::SizedTypeProperties;
use crate::slice::{self, SliceIndex};
@ -1636,8 +1635,7 @@ impl<T: ?Sized> *mut T {
#[must_use]
#[inline]
#[stable(feature = "align_offset", since = "1.36.0")]
#[rustc_const_unstable(feature = "const_align_offset", issue = "90962")]
pub const fn align_offset(self, align: usize) -> usize
pub fn align_offset(self, align: usize) -> usize
where
T: Sized,
{
@ -1675,95 +1673,10 @@ impl<T: ?Sized> *mut T {
/// assert!(ptr.is_aligned());
/// assert!(!ptr.wrapping_byte_add(1).is_aligned());
/// ```
///
/// # At compiletime
/// **Note: Alignment at compiletime is experimental and subject to change. See the
/// [tracking issue] for details.**
///
/// At compiletime, the compiler may not know where a value will end up in memory.
/// Calling this function on a pointer created from a reference at compiletime will only
/// return `true` if the pointer is guaranteed to be aligned. This means that the pointer
/// is never aligned if cast to a type with a stricter alignment than the reference's
/// underlying allocation.
///
/// ```
/// #![feature(const_pointer_is_aligned)]
///
/// // On some platforms, the alignment of primitives is less than their size.
/// #[repr(align(4))]
/// struct AlignedI32(i32);
/// #[repr(align(8))]
/// struct AlignedI64(i64);
///
/// const _: () = {
/// let mut data = AlignedI32(42);
/// let ptr = &mut data as *mut AlignedI32;
/// assert!(ptr.is_aligned());
///
/// // At runtime either `ptr1` or `ptr2` would be aligned, but at compiletime neither is aligned.
/// let ptr1 = ptr.cast::<AlignedI64>();
/// let ptr2 = ptr.wrapping_add(1).cast::<AlignedI64>();
/// assert!(!ptr1.is_aligned());
/// assert!(!ptr2.is_aligned());
/// };
/// ```
///
/// Due to this behavior, it is possible that a runtime pointer derived from a compiletime
/// pointer is aligned, even if the compiletime pointer wasn't aligned.
///
/// ```
/// #![feature(const_pointer_is_aligned)]
///
/// // On some platforms, the alignment of primitives is less than their size.
/// #[repr(align(4))]
/// struct AlignedI32(i32);
/// #[repr(align(8))]
/// struct AlignedI64(i64);
///
/// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned.
/// // Also, note that mutable references are not allowed in the final value of constants.
/// const COMPTIME_PTR: *mut AlignedI32 = (&AlignedI32(42) as *const AlignedI32).cast_mut();
/// const _: () = assert!(!COMPTIME_PTR.cast::<AlignedI64>().is_aligned());
/// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).cast::<AlignedI64>().is_aligned());
///
/// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned.
/// let runtime_ptr = COMPTIME_PTR;
/// assert_ne!(
/// runtime_ptr.cast::<AlignedI64>().is_aligned(),
/// runtime_ptr.wrapping_add(1).cast::<AlignedI64>().is_aligned(),
/// );
/// ```
///
/// If a pointer is created from a fixed address, this function behaves the same during
/// runtime and compiletime.
///
/// ```
/// #![feature(const_pointer_is_aligned)]
///
/// // On some platforms, the alignment of primitives is less than their size.
/// #[repr(align(4))]
/// struct AlignedI32(i32);
/// #[repr(align(8))]
/// struct AlignedI64(i64);
///
/// const _: () = {
/// let ptr = 40 as *mut AlignedI32;
/// assert!(ptr.is_aligned());
///
/// // For pointers with a known address, runtime and compiletime behavior are identical.
/// let ptr1 = ptr.cast::<AlignedI64>();
/// let ptr2 = ptr.wrapping_add(1).cast::<AlignedI64>();
/// assert!(ptr1.is_aligned());
/// assert!(!ptr2.is_aligned());
/// };
/// ```
///
/// [tracking issue]: https://github.com/rust-lang/rust/issues/104203
#[must_use]
#[inline]
#[stable(feature = "pointer_is_aligned", since = "1.79.0")]
#[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")]
pub const fn is_aligned(self) -> bool
pub fn is_aligned(self) -> bool
where
T: Sized,
{
@ -1800,106 +1713,15 @@ impl<T: ?Sized> *mut T {
///
/// assert_ne!(ptr.is_aligned_to(8), ptr.wrapping_add(1).is_aligned_to(8));
/// ```
///
/// # At compiletime
/// **Note: Alignment at compiletime is experimental and subject to change. See the
/// [tracking issue] for details.**
///
/// At compiletime, the compiler may not know where a value will end up in memory.
/// Calling this function on a pointer created from a reference at compiletime will only
/// return `true` if the pointer is guaranteed to be aligned. This means that the pointer
/// cannot be stricter aligned than the reference's underlying allocation.
///
/// ```
/// #![feature(pointer_is_aligned_to)]
/// #![feature(const_pointer_is_aligned)]
///
/// // On some platforms, the alignment of i32 is less than 4.
/// #[repr(align(4))]
/// struct AlignedI32(i32);
///
/// const _: () = {
/// let mut data = AlignedI32(42);
/// let ptr = &mut data as *mut AlignedI32;
///
/// assert!(ptr.is_aligned_to(1));
/// assert!(ptr.is_aligned_to(2));
/// assert!(ptr.is_aligned_to(4));
///
/// // At compiletime, we know for sure that the pointer isn't aligned to 8.
/// assert!(!ptr.is_aligned_to(8));
/// assert!(!ptr.wrapping_add(1).is_aligned_to(8));
/// };
/// ```
///
/// Due to this behavior, it is possible that a runtime pointer derived from a compiletime
/// pointer is aligned, even if the compiletime pointer wasn't aligned.
///
/// ```
/// #![feature(pointer_is_aligned_to)]
/// #![feature(const_pointer_is_aligned)]
///
/// // On some platforms, the alignment of i32 is less than 4.
/// #[repr(align(4))]
/// struct AlignedI32(i32);
///
/// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned.
/// // Also, note that mutable references are not allowed in the final value of constants.
/// const COMPTIME_PTR: *mut AlignedI32 = (&AlignedI32(42) as *const AlignedI32).cast_mut();
/// const _: () = assert!(!COMPTIME_PTR.is_aligned_to(8));
/// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).is_aligned_to(8));
///
/// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned.
/// let runtime_ptr = COMPTIME_PTR;
/// assert_ne!(
/// runtime_ptr.is_aligned_to(8),
/// runtime_ptr.wrapping_add(1).is_aligned_to(8),
/// );
/// ```
///
/// If a pointer is created from a fixed address, this function behaves the same during
/// runtime and compiletime.
///
/// ```
/// #![feature(pointer_is_aligned_to)]
/// #![feature(const_pointer_is_aligned)]
///
/// const _: () = {
/// let ptr = 40 as *mut u8;
/// assert!(ptr.is_aligned_to(1));
/// assert!(ptr.is_aligned_to(2));
/// assert!(ptr.is_aligned_to(4));
/// assert!(ptr.is_aligned_to(8));
/// assert!(!ptr.is_aligned_to(16));
/// };
/// ```
///
/// [tracking issue]: https://github.com/rust-lang/rust/issues/104203
#[must_use]
#[inline]
#[unstable(feature = "pointer_is_aligned_to", issue = "96284")]
#[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")]
pub const fn is_aligned_to(self, align: usize) -> bool {
pub fn is_aligned_to(self, align: usize) -> bool {
if !align.is_power_of_two() {
panic!("is_aligned_to: align is not a power-of-two");
}
#[inline]
fn runtime_impl(ptr: *mut (), align: usize) -> bool {
ptr.addr() & (align - 1) == 0
}
#[inline]
#[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")]
const fn const_impl(ptr: *mut (), align: usize) -> bool {
// We can't use the address of `self` in a `const fn`, so we use `align_offset` instead.
ptr.align_offset(align) == 0
}
// The cast to `()` is used to
// 1. deal with fat pointers; and
// 2. ensure that `align_offset` (in `const_impl`) doesn't actually try to compute an offset.
const_eval_select((self.cast::<()>(), align), const_impl, runtime_impl)
self.addr() & (align - 1) == 0
}
}

170
library/core/src/ptr/non_null.rs
View File

@ -1192,8 +1192,7 @@ impl<T: ?Sized> NonNull<T> {
#[inline]
#[must_use]
#[stable(feature = "non_null_convenience", since = "1.80.0")]
#[rustc_const_unstable(feature = "const_align_offset", issue = "90962")]
pub const fn align_offset(self, align: usize) -> usize
pub fn align_offset(self, align: usize) -> usize
where
T: Sized,
{
@ -1224,98 +1223,10 @@ impl<T: ?Sized> NonNull<T> {
/// assert!(ptr.is_aligned());
/// assert!(!NonNull::new(ptr.as_ptr().wrapping_byte_add(1)).unwrap().is_aligned());
/// ```
///
/// # At compiletime
/// **Note: Alignment at compiletime is experimental and subject to change. See the
/// [tracking issue] for details.**
///
/// At compiletime, the compiler may not know where a value will end up in memory.
/// Calling this function on a pointer created from a reference at compiletime will only
/// return `true` if the pointer is guaranteed to be aligned. This means that the pointer
/// is never aligned if cast to a type with a stricter alignment than the reference's
/// underlying allocation.
///
/// ```
/// #![feature(const_nonnull_new)]
/// #![feature(const_pointer_is_aligned)]
/// use std::ptr::NonNull;
///
/// // On some platforms, the alignment of primitives is less than their size.
/// #[repr(align(4))]
/// struct AlignedI32(i32);
/// #[repr(align(8))]
/// struct AlignedI64(i64);
///
/// const _: () = {
/// let data = [AlignedI32(42), AlignedI32(42)];
/// let ptr = NonNull::<AlignedI32>::new(&data[0] as *const _ as *mut _).unwrap();
/// assert!(ptr.is_aligned());
///
/// // At runtime either `ptr1` or `ptr2` would be aligned, but at compiletime neither is aligned.
/// let ptr1 = ptr.cast::<AlignedI64>();
/// let ptr2 = unsafe { ptr.add(1).cast::<AlignedI64>() };
/// assert!(!ptr1.is_aligned());
/// assert!(!ptr2.is_aligned());
/// };
/// ```
///
/// Due to this behavior, it is possible that a runtime pointer derived from a compiletime
/// pointer is aligned, even if the compiletime pointer wasn't aligned.
///
/// ```
/// #![feature(const_pointer_is_aligned)]
///
/// // On some platforms, the alignment of primitives is less than their size.
/// #[repr(align(4))]
/// struct AlignedI32(i32);
/// #[repr(align(8))]
/// struct AlignedI64(i64);
///
/// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned.
/// const COMPTIME_PTR: *const AlignedI32 = &AlignedI32(42);
/// const _: () = assert!(!COMPTIME_PTR.cast::<AlignedI64>().is_aligned());
/// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).cast::<AlignedI64>().is_aligned());
///
/// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned.
/// let runtime_ptr = COMPTIME_PTR;
/// assert_ne!(
/// runtime_ptr.cast::<AlignedI64>().is_aligned(),
/// runtime_ptr.wrapping_add(1).cast::<AlignedI64>().is_aligned(),
/// );
/// ```
///
/// If a pointer is created from a fixed address, this function behaves the same during
/// runtime and compiletime.
///
/// ```
/// #![feature(const_pointer_is_aligned)]
/// #![feature(const_nonnull_new)]
/// use std::ptr::NonNull;
///
/// // On some platforms, the alignment of primitives is less than their size.
/// #[repr(align(4))]
/// struct AlignedI32(i32);
/// #[repr(align(8))]
/// struct AlignedI64(i64);
///
/// const _: () = {
/// let ptr = NonNull::new(40 as *mut AlignedI32).unwrap();
/// assert!(ptr.is_aligned());
///
/// // For pointers with a known address, runtime and compiletime behavior are identical.
/// let ptr1 = ptr.cast::<AlignedI64>();
/// let ptr2 = NonNull::new(ptr.as_ptr().wrapping_add(1)).unwrap().cast::<AlignedI64>();
/// assert!(ptr1.is_aligned());
/// assert!(!ptr2.is_aligned());
/// };
/// ```
///
/// [tracking issue]: https://github.com/rust-lang/rust/issues/104203
#[inline]
#[must_use]
#[stable(feature = "pointer_is_aligned", since = "1.79.0")]
#[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")]
pub const fn is_aligned(self) -> bool
pub fn is_aligned(self) -> bool
where
T: Sized,
{
@ -1352,85 +1263,10 @@ impl<T: ?Sized> NonNull<T> {
///
/// assert_ne!(ptr.is_aligned_to(8), ptr.wrapping_add(1).is_aligned_to(8));
/// ```
///
/// # At compiletime
/// **Note: Alignment at compiletime is experimental and subject to change. See the
/// [tracking issue] for details.**
///
/// At compiletime, the compiler may not know where a value will end up in memory.
/// Calling this function on a pointer created from a reference at compiletime will only
/// return `true` if the pointer is guaranteed to be aligned. This means that the pointer
/// cannot be stricter aligned than the reference's underlying allocation.
///
/// ```
/// #![feature(pointer_is_aligned_to)]
/// #![feature(const_pointer_is_aligned)]
///
/// // On some platforms, the alignment of i32 is less than 4.
/// #[repr(align(4))]
/// struct AlignedI32(i32);
///
/// const _: () = {
/// let data = AlignedI32(42);
/// let ptr = &data as *const AlignedI32;
///
/// assert!(ptr.is_aligned_to(1));
/// assert!(ptr.is_aligned_to(2));
/// assert!(ptr.is_aligned_to(4));
///
/// // At compiletime, we know for sure that the pointer isn't aligned to 8.
/// assert!(!ptr.is_aligned_to(8));
/// assert!(!ptr.wrapping_add(1).is_aligned_to(8));
/// };
/// ```
///
/// Due to this behavior, it is possible that a runtime pointer derived from a compiletime
/// pointer is aligned, even if the compiletime pointer wasn't aligned.
///
/// ```
/// #![feature(pointer_is_aligned_to)]
/// #![feature(const_pointer_is_aligned)]
///
/// // On some platforms, the alignment of i32 is less than 4.
/// #[repr(align(4))]
/// struct AlignedI32(i32);
///
/// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned.
/// const COMPTIME_PTR: *const AlignedI32 = &AlignedI32(42);
/// const _: () = assert!(!COMPTIME_PTR.is_aligned_to(8));
/// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).is_aligned_to(8));
///
/// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned.
/// let runtime_ptr = COMPTIME_PTR;
/// assert_ne!(
/// runtime_ptr.is_aligned_to(8),
/// runtime_ptr.wrapping_add(1).is_aligned_to(8),
/// );
/// ```
///
/// If a pointer is created from a fixed address, this function behaves the same during
/// runtime and compiletime.
///
/// ```
/// #![feature(pointer_is_aligned_to)]
/// #![feature(const_pointer_is_aligned)]
///
/// const _: () = {
/// let ptr = 40 as *const u8;
/// assert!(ptr.is_aligned_to(1));
/// assert!(ptr.is_aligned_to(2));
/// assert!(ptr.is_aligned_to(4));
/// assert!(ptr.is_aligned_to(8));
/// assert!(!ptr.is_aligned_to(16));
/// };
/// ```
///
/// [tracking issue]: https://github.com/rust-lang/rust/issues/104203
#[inline]
#[must_use]
#[unstable(feature = "pointer_is_aligned_to", issue = "96284")]
#[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")]
pub const fn is_aligned_to(self, align: usize) -> bool {
pub fn is_aligned_to(self, align: usize) -> bool {
self.pointer.is_aligned_to(align)
}
}

157
library/core/src/slice/ascii.rs
View File

@ -3,6 +3,7 @@
use core::ascii::EscapeDefault;
use crate::fmt::{self, Write};
use crate::intrinsics::const_eval_select;
use crate::{ascii, iter, mem, ops};
#[cfg(not(test))]
@ -346,89 +347,93 @@ pub const fn is_ascii_simple(mut bytes: &[u8]) -> bool {
/// If any of these loads produces something for which `contains_nonascii`
/// (above) returns true, then we know the answer is false.
#[inline]
#[rustc_allow_const_fn_unstable(const_raw_ptr_comparison, const_pointer_is_aligned)] // only in a debug assertion
#[rustc_allow_const_fn_unstable(const_align_offset)] // behavior does not change when `align_offset` fails
#[rustc_allow_const_fn_unstable(const_eval_select)] // fallback impl has same behavior
const fn is_ascii(s: &[u8]) -> bool {
const USIZE_SIZE: usize = mem::size_of::<usize>();
// The runtime version behaves the same as the compiletime version, it's
// just more optimized.
return const_eval_select((s,), compiletime, runtime);
let len = s.len();
let align_offset = s.as_ptr().align_offset(USIZE_SIZE);
// If we wouldn't gain anything from the word-at-a-time implementation, fall
// back to a scalar loop.
//
// We also do this for architectures where `size_of::<usize>()` isn't
// sufficient alignment for `usize`, because it's a weird edge case.
if len < USIZE_SIZE || len < align_offset || USIZE_SIZE < mem::align_of::<usize>() {
return is_ascii_simple(s);
const fn compiletime(s: &[u8]) -> bool {
is_ascii_simple(s)
}
// We always read the first word unaligned, which means `align_offset` is
// 0, we'd read the same value again for the aligned read.
let offset_to_aligned = if align_offset == 0 { USIZE_SIZE } else { align_offset };
#[inline]
fn runtime(s: &[u8]) -> bool {
const USIZE_SIZE: usize = mem::size_of::<usize>();
let start = s.as_ptr();
// SAFETY: We verify `len < USIZE_SIZE` above.
let first_word = unsafe { (start as *const usize).read_unaligned() };
let len = s.len();
let align_offset = s.as_ptr().align_offset(USIZE_SIZE);
if contains_nonascii(first_word) {
return false;
}
// We checked this above, somewhat implicitly. Note that `offset_to_aligned`
// is either `align_offset` or `USIZE_SIZE`, both of are explicitly checked
// above.
debug_assert!(offset_to_aligned <= len);
// SAFETY: word_ptr is the (properly aligned) usize ptr we use to read the
// middle chunk of the slice.
let mut word_ptr = unsafe { start.add(offset_to_aligned) as *const usize };
// `byte_pos` is the byte index of `word_ptr`, used for loop end checks.
let mut byte_pos = offset_to_aligned;
// Paranoia check about alignment, since we're about to do a bunch of
// unaligned loads. In practice this should be impossible barring a bug in
// `align_offset` though.
// While this method is allowed to spuriously fail in CTFE, if it doesn't
// have alignment information it should have given a `usize::MAX` for
// `align_offset` earlier, sending things through the scalar path instead of
// this one, so this check should pass if it's reachable.
debug_assert!(word_ptr.is_aligned_to(mem::align_of::<usize>()));
// Read subsequent words until the last aligned word, excluding the last
// aligned word by itself to be done in tail check later, to ensure that
// tail is always one `usize` at most to extra branch `byte_pos == len`.
while byte_pos < len - USIZE_SIZE {
// Sanity check that the read is in bounds
debug_assert!(byte_pos + USIZE_SIZE <= len);
// And that our assumptions about `byte_pos` hold.
debug_assert!(matches!(
word_ptr.cast::<u8>().guaranteed_eq(start.wrapping_add(byte_pos)),
// These are from the same allocation, so will hopefully always be
// known to match even in CTFE, but if it refuses to compare them
// that's ok since it's just a debug check anyway.
None | Some(true),
));
// SAFETY: We know `word_ptr` is properly aligned (because of
// `align_offset`), and we know that we have enough bytes between `word_ptr` and the end
let word = unsafe { word_ptr.read() };
if contains_nonascii(word) {
return false;
// If we wouldn't gain anything from the word-at-a-time implementation, fall
// back to a scalar loop.
//
// We also do this for architectures where `size_of::<usize>()` isn't
// sufficient alignment for `usize`, because it's a weird edge case.
if len < USIZE_SIZE || len < align_offset || USIZE_SIZE < mem::align_of::<usize>() {
return is_ascii_simple(s);
}
byte_pos += USIZE_SIZE;
// SAFETY: We know that `byte_pos <= len - USIZE_SIZE`, which means that
// after this `add`, `word_ptr` will be at most one-past-the-end.
word_ptr = unsafe { word_ptr.add(1) };
// We always read the first word unaligned, which means `align_offset` is
// 0, we'd read the same value again for the aligned read.
let offset_to_aligned = if align_offset == 0 { USIZE_SIZE } else { align_offset };
let start = s.as_ptr();
// SAFETY: We verify `len < USIZE_SIZE` above.
let first_word = unsafe { (start as *const usize).read_unaligned() };
if contains_nonascii(first_word) {
return false;
}
// We checked this above, somewhat implicitly. Note that `offset_to_aligned`
// is either `align_offset` or `USIZE_SIZE`, both of are explicitly checked
// above.
debug_assert!(offset_to_aligned <= len);
// SAFETY: word_ptr is the (properly aligned) usize ptr we use to read the
// middle chunk of the slice.
let mut word_ptr = unsafe { start.add(offset_to_aligned) as *const usize };
// `byte_pos` is the byte index of `word_ptr`, used for loop end checks.
let mut byte_pos = offset_to_aligned;
// Paranoia check about alignment, since we're about to do a bunch of
// unaligned loads. In practice this should be impossible barring a bug in
// `align_offset` though.
// While this method is allowed to spuriously fail in CTFE, if it doesn't
// have alignment information it should have given a `usize::MAX` for
// `align_offset` earlier, sending things through the scalar path instead of
// this one, so this check should pass if it's reachable.
debug_assert!(word_ptr.is_aligned_to(mem::align_of::<usize>()));
// Read subsequent words until the last aligned word, excluding the last
// aligned word by itself to be done in tail check later, to ensure that
// tail is always one `usize` at most to extra branch `byte_pos == len`.
while byte_pos < len - USIZE_SIZE {
// Sanity check that the read is in bounds
debug_assert!(byte_pos + USIZE_SIZE <= len);
// And that our assumptions about `byte_pos` hold.
debug_assert!(word_ptr.cast::<u8>() == start.wrapping_add(byte_pos));
// SAFETY: We know `word_ptr` is properly aligned (because of
// `align_offset`), and we know that we have enough bytes between `word_ptr` and the end
let word = unsafe { word_ptr.read() };
if contains_nonascii(word) {
return false;
}
byte_pos += USIZE_SIZE;
// SAFETY: We know that `byte_pos <= len - USIZE_SIZE`, which means that
// after this `add`, `word_ptr` will be at most one-past-the-end.
word_ptr = unsafe { word_ptr.add(1) };
}
// Sanity check to ensure there really is only one `usize` left. This should
// be guaranteed by our loop condition.
debug_assert!(byte_pos <= len && len - byte_pos <= USIZE_SIZE);
// SAFETY: This relies on `len >= USIZE_SIZE`, which we check at the start.
let last_word = unsafe { (start.add(len - USIZE_SIZE) as *const usize).read_unaligned() };
!contains_nonascii(last_word)
}
// Sanity check to ensure there really is only one `usize` left. This should
// be guaranteed by our loop condition.
debug_assert!(byte_pos <= len && len - byte_pos <= USIZE_SIZE);
// SAFETY: This relies on `len >= USIZE_SIZE`, which we check at the start.
let last_word = unsafe { (start.add(len - USIZE_SIZE) as *const usize).read_unaligned() };
!contains_nonascii(last_word)
}

95
library/core/src/slice/memchr.rs
View File

@ -1,6 +1,7 @@
// Original implementation taken from rust-memchr.
// Copyright 2015 Andrew Gallant, bluss and Nicolas Koch
use crate::intrinsics::const_eval_select;
use crate::mem;
const LO_USIZE: usize = usize::repeat_u8(0x01);
@ -50,58 +51,66 @@ const fn memchr_naive(x: u8, text: &[u8]) -> Option<usize> {
None
}
#[rustc_allow_const_fn_unstable(const_cmp)]
#[rustc_allow_const_fn_unstable(const_align_offset)]
#[rustc_allow_const_fn_unstable(const_eval_select)] // fallback impl has same behavior
#[cfg_attr(bootstrap, rustc_const_stable(feature = "const_memchr", since = "1.65.0"))]
const fn memchr_aligned(x: u8, text: &[u8]) -> Option<usize> {
// Scan for a single byte value by reading two `usize` words at a time.
//
// Split `text` in three parts
// - unaligned initial part, before the first word aligned address in text
// - body, scan by 2 words at a time
// - the last remaining part, < 2 word size
// The runtime version behaves the same as the compiletime version, it's
// just more optimized.
return const_eval_select((x, text), compiletime, runtime);
// search up to an aligned boundary
let len = text.len();
let ptr = text.as_ptr();
let mut offset = ptr.align_offset(USIZE_BYTES);
if offset > 0 {
// FIXME(const-hack, fee1-dead): replace with min
offset = if offset < len { offset } else { len };
// FIXME(const-hack, fee1-dead): replace with range slicing
// SAFETY: offset is within bounds
let slice = unsafe { super::from_raw_parts(text.as_ptr(), offset) };
if let Some(index) = memchr_naive(x, slice) {
return Some(index);
}
const fn compiletime(x: u8, text: &[u8]) -> Option<usize> {
memchr_naive(x, text)
}
// search the body of the text
let repeated_x = usize::repeat_u8(x);
while offset <= len - 2 * USIZE_BYTES {
// SAFETY: the while's predicate guarantees a distance of at least 2 * usize_bytes
// between the offset and the end of the slice.
unsafe {
let u = *(ptr.add(offset) as *const usize);
let v = *(ptr.add(offset + USIZE_BYTES) as *const usize);
#[inline]
fn runtime(x: u8, text: &[u8]) -> Option<usize> {
// Scan for a single byte value by reading two `usize` words at a time.
//
// Split `text` in three parts
// - unaligned initial part, before the first word aligned address in text
// - body, scan by 2 words at a time
// - the last remaining part, < 2 word size
// break if there is a matching byte
let zu = contains_zero_byte(u ^ repeated_x);
let zv = contains_zero_byte(v ^ repeated_x);
if zu || zv {
break;
// search up to an aligned boundary
let len = text.len();
let ptr = text.as_ptr();
let mut offset = ptr.align_offset(USIZE_BYTES);
if offset > 0 {
offset = offset.min(len);
let slice = &text[..offset];
if let Some(index) = memchr_naive(x, slice) {
return Some(index);
}
}
offset += USIZE_BYTES * 2;
}
// Find the byte after the point the body loop stopped.
// FIXME(const-hack): Use `?` instead.
// FIXME(const-hack, fee1-dead): use range slicing
// SAFETY: offset is within bounds
let slice = unsafe { super::from_raw_parts(text.as_ptr().add(offset), text.len() - offset) };
if let Some(i) = memchr_naive(x, slice) { Some(offset + i) } else { None }
// search the body of the text
let repeated_x = usize::repeat_u8(x);
while offset <= len - 2 * USIZE_BYTES {
// SAFETY: the while's predicate guarantees a distance of at least 2 * usize_bytes
// between the offset and the end of the slice.
unsafe {
let u = *(ptr.add(offset) as *const usize);
let v = *(ptr.add(offset + USIZE_BYTES) as *const usize);
// break if there is a matching byte
let zu = contains_zero_byte(u ^ repeated_x);
let zv = contains_zero_byte(v ^ repeated_x);
if zu || zv {
break;
}
}
offset += USIZE_BYTES * 2;
}
// Find the byte after the point the body loop stopped.
// FIXME(const-hack): Use `?` instead.
// FIXME(const-hack, fee1-dead): use range slicing
let slice =
// SAFETY: offset is within bounds
unsafe { super::from_raw_parts(text.as_ptr().add(offset), text.len() - offset) };
if let Some(i) = memchr_naive(x, slice) { Some(offset + i) } else { None }
}
}
/// Returns the last index matching the byte `x` in `text`.

1
library/core/src/str/converts.rs
View File

@ -82,7 +82,6 @@ use crate::{mem, ptr};
/// ```
#[stable(feature = "rust1", since = "1.0.0")]
#[rustc_const_stable(feature = "const_str_from_utf8_shared", since = "1.63.0")]
#[rustc_allow_const_fn_unstable(str_internals)]
#[rustc_diagnostic_item = "str_from_utf8"]
pub const fn from_utf8(v: &[u8]) -> Result<&str, Utf8Error> {
// FIXME(const-hack): This should use `?` again, once it's `const`

26
library/core/src/str/validations.rs
View File

@ -1,6 +1,7 @@
//! Operations related to UTF-8 validation.
use super::Utf8Error;
use crate::intrinsics::const_eval_select;
use crate::mem;
/// Returns the initial codepoint accumulator for the first byte.
@ -122,15 +123,28 @@ const fn contains_nonascii(x: usize) -> bool {
/// Walks through `v` checking that it's a valid UTF-8 sequence,
/// returning `Ok(())` in that case, or, if it is invalid, `Err(err)`.
#[inline(always)]
#[rustc_const_unstable(feature = "str_internals", issue = "none")]
#[rustc_allow_const_fn_unstable(const_eval_select)] // fallback impl has same behavior
pub(super) const fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> {
let mut index = 0;
let len = v.len();
let usize_bytes = mem::size_of::<usize>();
let ascii_block_size = 2 * usize_bytes;
const USIZE_BYTES: usize = mem::size_of::<usize>();
let ascii_block_size = 2 * USIZE_BYTES;
let blocks_end = if len >= ascii_block_size { len - ascii_block_size + 1 } else { 0 };
let align = v.as_ptr().align_offset(usize_bytes);
let align = {
const fn compiletime(_v: &[u8]) -> usize {
usize::MAX
}
fn runtime(v: &[u8]) -> usize {
v.as_ptr().align_offset(USIZE_BYTES)
}
// Below, we safely fall back to a slower codepath if the offset is `usize::MAX`,
// so the end-to-end behavior is the same at compiletime and runtime.
const_eval_select((v,), compiletime, runtime)
};
while index < len {
let old_offset = index;
@ -209,11 +223,11 @@ pub(super) const fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> {
// Ascii case, try to skip forward quickly.
// When the pointer is aligned, read 2 words of data per iteration
// until we find a word containing a non-ascii byte.
if align != usize::MAX && align.wrapping_sub(index) % usize_bytes == 0 {
if align != usize::MAX && align.wrapping_sub(index) % USIZE_BYTES == 0 {
let ptr = v.as_ptr();
while index < blocks_end {
// SAFETY: since `align - index` and `ascii_block_size` are
// multiples of `usize_bytes`, `block = ptr.add(index)` is
// multiples of `USIZE_BYTES`, `block = ptr.add(index)` is
// always aligned with a `usize` so it's safe to dereference
// both `block` and `block.add(1)`.
unsafe {

14
library/core/src/ub_checks.rs
View File

@ -120,7 +120,19 @@ pub(crate) const fn check_language_ub() -> bool {
#[inline]
#[rustc_const_unstable(feature = "const_ub_checks", issue = "none")]
pub(crate) const fn is_aligned_and_not_null(ptr: *const (), align: usize, is_zst: bool) -> bool {
ptr.is_aligned_to(align) && (is_zst || !ptr.is_null())
#[inline]
fn runtime(ptr: *const (), align: usize, is_zst: bool) -> bool {
ptr.is_aligned_to(align) && (is_zst || !ptr.is_null())
}
#[inline]
#[rustc_const_unstable(feature = "const_ub_checks", issue = "none")]
const fn comptime(ptr: *const (), _align: usize, is_zst: bool) -> bool {
is_zst || !ptr.is_null()
}
// This is just for safety checks so we can const_eval_select.
const_eval_select((ptr, align, is_zst), comptime, runtime)
}
#[inline]

2
library/core/tests/lib.rs
View File

@ -16,7 +16,6 @@
#![feature(cell_update)]
#![feature(clone_to_uninit)]
#![feature(const_align_of_val_raw)]
#![feature(const_align_offset)]
#![feature(const_black_box)]
#![feature(const_eval_select)]
#![feature(const_hash)]
@ -24,7 +23,6 @@
#![feature(const_nonnull_new)]
#![feature(const_option_ext)]
#![feature(const_pin_2)]
#![feature(const_pointer_is_aligned)]
#![feature(const_three_way_compare)]
#![feature(const_trait_impl)]
#![feature(core_intrinsics)]

257
library/core/tests/ptr.rs
View File

@ -359,22 +359,6 @@ fn align_offset_zst() {
}
}
#[test]
fn align_offset_zst_const() {
const {
// For pointers of stride = 0, the pointer is already aligned or it cannot be aligned at
// all, because no amount of elements will align the pointer.
let mut p = 1;
while p < 1024 {
assert!(ptr::without_provenance::<()>(p).align_offset(p) == 0);
if p != 1 {
assert!(ptr::without_provenance::<()>(p + 1).align_offset(p) == !0);
}
p = (p + 1).next_power_of_two();
}
}
}
#[test]
fn align_offset_stride_one() {
// For pointers of stride = 1, the pointer can always be aligned. The offset is equal to
@ -396,25 +380,6 @@ fn align_offset_stride_one() {
}
}
#[test]
fn align_offset_stride_one_const() {
const {
// For pointers of stride = 1, the pointer can always be aligned. The offset is equal to
// number of bytes.
let mut align = 1;
while align < 1024 {
let mut ptr = 1;
while ptr < 2 * align {
let expected = ptr % align;
let offset = if expected == 0 { 0 } else { align - expected };
assert!(ptr::without_provenance::<u8>(ptr).align_offset(align) == offset);
ptr += 1;
}
align = (align + 1).next_power_of_two();
}
}
}
#[test]
fn align_offset_various_strides() {
unsafe fn test_stride<T>(ptr: *const T, align: usize) -> bool {
@ -495,192 +460,6 @@ fn align_offset_various_strides() {
assert!(!x);
}
#[test]
fn align_offset_various_strides_const() {
const unsafe fn test_stride<T>(ptr: *const T, numptr: usize, align: usize) {
let mut expected = usize::MAX;
// Naive but definitely correct way to find the *first* aligned element of stride::<T>.
let mut el = 0;
while el < align {
if (numptr + el * ::std::mem::size_of::<T>()) % align == 0 {
expected = el;
break;
}
el += 1;
}
let got = ptr.align_offset(align);
assert!(got == expected);
}
const {
// For pointers of stride != 1, we verify the algorithm against the naivest possible
// implementation
let mut align = 1;
let limit = 32;
while align < limit {
let mut ptr = 1;
while ptr < 4 * align {
unsafe {
#[repr(packed)]
struct A3(#[allow(dead_code)] u16, #[allow(dead_code)] u8);
test_stride::<A3>(ptr::without_provenance::<A3>(ptr), ptr, align);
struct A4(#[allow(dead_code)] u32);
test_stride::<A4>(ptr::without_provenance::<A4>(ptr), ptr, align);
#[repr(packed)]
struct A5(#[allow(dead_code)] u32, #[allow(dead_code)] u8);
test_stride::<A5>(ptr::without_provenance::<A5>(ptr), ptr, align);
#[repr(packed)]
struct A6(#[allow(dead_code)] u32, #[allow(dead_code)] u16);
test_stride::<A6>(ptr::without_provenance::<A6>(ptr), ptr, align);
#[repr(packed)]
struct A7(
#[allow(dead_code)] u32,
#[allow(dead_code)] u16,
#[allow(dead_code)] u8,
);
test_stride::<A7>(ptr::without_provenance::<A7>(ptr), ptr, align);
#[repr(packed)]
struct A8(#[allow(dead_code)] u32, #[allow(dead_code)] u32);
test_stride::<A8>(ptr::without_provenance::<A8>(ptr), ptr, align);
#[repr(packed)]
struct A9(
#[allow(dead_code)] u32,
#[allow(dead_code)] u32,
#[allow(dead_code)] u8,
);
test_stride::<A9>(ptr::without_provenance::<A9>(ptr), ptr, align);
#[repr(packed)]
struct A10(
#[allow(dead_code)] u32,
#[allow(dead_code)] u32,
#[allow(dead_code)] u16,
);
test_stride::<A10>(ptr::without_provenance::<A10>(ptr), ptr, align);
test_stride::<u32>(ptr::without_provenance::<u32>(ptr), ptr, align);
test_stride::<u128>(ptr::without_provenance::<u128>(ptr), ptr, align);
}
ptr += 1;
}
align = (align + 1).next_power_of_two();
}
}
}
#[test]
fn align_offset_with_provenance_const() {
const {
// On some platforms (e.g. msp430-none-elf), the alignment of `i32` is less than 4.
#[repr(align(4))]
struct AlignedI32(i32);
let data = AlignedI32(42);
// `stride % align == 0` (usual case)
let ptr: *const i32 = &data.0;
assert!(ptr.align_offset(1) == 0);
assert!(ptr.align_offset(2) == 0);
assert!(ptr.align_offset(4) == 0);
assert!(ptr.align_offset(8) == usize::MAX);
assert!(ptr.wrapping_byte_add(1).align_offset(1) == 0);
assert!(ptr.wrapping_byte_add(1).align_offset(2) == usize::MAX);
assert!(ptr.wrapping_byte_add(2).align_offset(1) == 0);
assert!(ptr.wrapping_byte_add(2).align_offset(2) == 0);
assert!(ptr.wrapping_byte_add(2).align_offset(4) == usize::MAX);
assert!(ptr.wrapping_byte_add(3).align_offset(1) == 0);
assert!(ptr.wrapping_byte_add(3).align_offset(2) == usize::MAX);
assert!(ptr.wrapping_add(42).align_offset(4) == 0);
assert!(ptr.wrapping_add(42).align_offset(8) == usize::MAX);
let ptr1: *const i8 = ptr.cast();
assert!(ptr1.align_offset(1) == 0);
assert!(ptr1.align_offset(2) == 0);
assert!(ptr1.align_offset(4) == 0);
assert!(ptr1.align_offset(8) == usize::MAX);
assert!(ptr1.wrapping_byte_add(1).align_offset(1) == 0);
assert!(ptr1.wrapping_byte_add(1).align_offset(2) == 1);
assert!(ptr1.wrapping_byte_add(1).align_offset(4) == 3);
assert!(ptr1.wrapping_byte_add(1).align_offset(8) == usize::MAX);
assert!(ptr1.wrapping_byte_add(2).align_offset(1) == 0);
assert!(ptr1.wrapping_byte_add(2).align_offset(2) == 0);
assert!(ptr1.wrapping_byte_add(2).align_offset(4) == 2);
assert!(ptr1.wrapping_byte_add(2).align_offset(8) == usize::MAX);
assert!(ptr1.wrapping_byte_add(3).align_offset(1) == 0);
assert!(ptr1.wrapping_byte_add(3).align_offset(2) == 1);
assert!(ptr1.wrapping_byte_add(3).align_offset(4) == 1);
assert!(ptr1.wrapping_byte_add(3).align_offset(8) == usize::MAX);
let ptr2: *const i16 = ptr.cast();
assert!(ptr2.align_offset(1) == 0);
assert!(ptr2.align_offset(2) == 0);
assert!(ptr2.align_offset(4) == 0);
assert!(ptr2.align_offset(8) == usize::MAX);
assert!(ptr2.wrapping_byte_add(1).align_offset(1) == 0);
assert!(ptr2.wrapping_byte_add(1).align_offset(2) == usize::MAX);
assert!(ptr2.wrapping_byte_add(2).align_offset(1) == 0);
assert!(ptr2.wrapping_byte_add(2).align_offset(2) == 0);
assert!(ptr2.wrapping_byte_add(2).align_offset(4) == 1);
assert!(ptr2.wrapping_byte_add(2).align_offset(8) == usize::MAX);
assert!(ptr2.wrapping_byte_add(3).align_offset(1) == 0);
assert!(ptr2.wrapping_byte_add(3).align_offset(2) == usize::MAX);
let ptr3: *const i64 = ptr.cast();
assert!(ptr3.align_offset(1) == 0);
assert!(ptr3.align_offset(2) == 0);
assert!(ptr3.align_offset(4) == 0);
assert!(ptr3.align_offset(8) == usize::MAX);
assert!(ptr3.wrapping_byte_add(1).align_offset(1) == 0);
assert!(ptr3.wrapping_byte_add(1).align_offset(2) == usize::MAX);
// `stride % align != 0` (edge case)
let ptr4: *const [u8; 3] = ptr.cast();
assert!(ptr4.align_offset(1) == 0);
assert!(ptr4.align_offset(2) == 0);
assert!(ptr4.align_offset(4) == 0);
assert!(ptr4.align_offset(8) == usize::MAX);
assert!(ptr4.wrapping_byte_add(1).align_offset(1) == 0);
assert!(ptr4.wrapping_byte_add(1).align_offset(2) == 1);
assert!(ptr4.wrapping_byte_add(1).align_offset(4) == 1);
assert!(ptr4.wrapping_byte_add(1).align_offset(8) == usize::MAX);
assert!(ptr4.wrapping_byte_add(2).align_offset(1) == 0);
assert!(ptr4.wrapping_byte_add(2).align_offset(2) == 0);
assert!(ptr4.wrapping_byte_add(2).align_offset(4) == 2);
assert!(ptr4.wrapping_byte_add(2).align_offset(8) == usize::MAX);
assert!(ptr4.wrapping_byte_add(3).align_offset(1) == 0);
assert!(ptr4.wrapping_byte_add(3).align_offset(2) == 1);
assert!(ptr4.wrapping_byte_add(3).align_offset(4) == 3);
assert!(ptr4.wrapping_byte_add(3).align_offset(8) == usize::MAX);
let ptr5: *const [u8; 5] = ptr.cast();
assert!(ptr5.align_offset(1) == 0);
assert!(ptr5.align_offset(2) == 0);
assert!(ptr5.align_offset(4) == 0);
assert!(ptr5.align_offset(8) == usize::MAX);
assert!(ptr5.wrapping_byte_add(1).align_offset(1) == 0);
assert!(ptr5.wrapping_byte_add(1).align_offset(2) == 1);
assert!(ptr5.wrapping_byte_add(1).align_offset(4) == 3);
assert!(ptr5.wrapping_byte_add(1).align_offset(8) == usize::MAX);
assert!(ptr5.wrapping_byte_add(2).align_offset(1) == 0);
assert!(ptr5.wrapping_byte_add(2).align_offset(2) == 0);
assert!(ptr5.wrapping_byte_add(2).align_offset(4) == 2);
assert!(ptr5.wrapping_byte_add(2).align_offset(8) == usize::MAX);
assert!(ptr5.wrapping_byte_add(3).align_offset(1) == 0);
assert!(ptr5.wrapping_byte_add(3).align_offset(2) == 1);
assert!(ptr5.wrapping_byte_add(3).align_offset(4) == 1);
assert!(ptr5.wrapping_byte_add(3).align_offset(8) == usize::MAX);
}
}
#[test]
fn align_offset_issue_103361() {
#[cfg(target_pointer_width = "64")]
@ -693,23 +472,6 @@ fn align_offset_issue_103361() {
let _ = ptr::without_provenance::<HugeSize>(SIZE).align_offset(SIZE);
}
#[test]
fn align_offset_issue_103361_const() {
#[cfg(target_pointer_width = "64")]
const SIZE: usize = 1 << 47;
#[cfg(target_pointer_width = "32")]
const SIZE: usize = 1 << 30;
#[cfg(target_pointer_width = "16")]
const SIZE: usize = 1 << 13;
struct HugeSize(#[allow(dead_code)] [u8; SIZE - 1]);
const {
assert!(ptr::without_provenance::<HugeSize>(SIZE - 1).align_offset(SIZE) == SIZE - 1);
assert!(ptr::without_provenance::<HugeSize>(SIZE).align_offset(SIZE) == 0);
assert!(ptr::without_provenance::<HugeSize>(SIZE + 1).align_offset(SIZE) == 1);
}
}
#[test]
fn is_aligned() {
let data = 42;
@ -726,25 +488,6 @@ fn is_aligned() {
assert_ne!(ptr.is_aligned_to(8), ptr.wrapping_add(1).is_aligned_to(8));
}
#[test]
fn is_aligned_const() {
const {
let data = 42;
let ptr: *const i32 = &data;
assert!(ptr.is_aligned());
assert!(ptr.is_aligned_to(1));
assert!(ptr.is_aligned_to(2));
assert!(ptr.is_aligned_to(4));
assert!(ptr.wrapping_byte_add(2).is_aligned_to(1));
assert!(ptr.wrapping_byte_add(2).is_aligned_to(2));
assert!(!ptr.wrapping_byte_add(2).is_aligned_to(4));
// At comptime neither `ptr` nor `ptr+1` is aligned to 8.
assert!(!ptr.is_aligned_to(8));
assert!(!ptr.wrapping_add(1).is_aligned_to(8));
}
}
#[test]
fn offset_from() {
let mut a = [0; 5];