From 19e287060de9a641d7000a5575bb943a9bfba318 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Thu, 31 Oct 2024 21:30:23 +0100 Subject: [PATCH] remove const-support for align_offset Operations like is_aligned would return actively wrong results at compile-time, i.e. calling it on the same pointer at compiletime and runtime could yield different results. That's no good. Instead of having hacks to make align_offset kind-of work in const-eval, just use const_eval_select in the few places where it makes sense, which also ensures those places are all aware they need to make sure the fallback behavior is consistent. --- compiler/rustc_const_eval/messages.ftl | 3 - .../src/const_eval/machine.rs | 88 +----- compiler/rustc_hir/src/lang_items.rs | 3 - compiler/rustc_span/src/symbol.rs | 1 - library/core/src/lib.rs | 2 - library/core/src/ptr/const_ptr.rs | 192 +------------ library/core/src/ptr/mod.rs | 10 +- library/core/src/ptr/mut_ptr.rs | 186 +------------ library/core/src/ptr/non_null.rs | 170 +----------- library/core/src/slice/ascii.rs | 157 +++++------ library/core/src/slice/memchr.rs | 95 ++++--- library/core/src/str/converts.rs | 1 - library/core/src/str/validations.rs | 26 +- library/core/src/ub_checks.rs | 14 +- library/core/tests/lib.rs | 2 - library/core/tests/ptr.rs | 257 ------------------ 16 files changed, 185 insertions(+), 1022 deletions(-) diff --git a/compiler/rustc_const_eval/messages.ftl b/compiler/rustc_const_eval/messages.ftl index 3e4f83c8242..2bc5adb2dce 100644 --- a/compiler/rustc_const_eval/messages.ftl +++ b/compiler/rustc_const_eval/messages.ftl @@ -1,9 +1,6 @@ const_eval_address_space_full = there are no more free addresses in the address space -const_eval_align_offset_invalid_align = - `align_offset` called with non-power-of-two align: {$target_align} - const_eval_alignment_check_failed = {$msg -> [AccessedPtr] accessing memory diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index 5f0bc8539ee..5938e836a8b 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -1,7 +1,6 @@ use std::borrow::{Borrow, Cow}; use std::fmt; use std::hash::Hash; -use std::ops::ControlFlow; use rustc_ast::Mutability; use rustc_data_structures::fx::{FxHashMap, FxIndexMap, IndexEntry}; @@ -9,7 +8,7 @@ use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::{self as hir, CRATE_HIR_ID, LangItem}; use rustc_middle::mir::AssertMessage; use rustc_middle::query::TyCtxtAt; -use rustc_middle::ty::layout::{FnAbiOf, TyAndLayout}; +use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_middle::{bug, mir}; use rustc_span::Span; @@ -23,9 +22,9 @@ use crate::errors::{LongRunning, LongRunningWarn}; use crate::fluent_generated as fluent; use crate::interpret::{ self, AllocId, AllocRange, ConstAllocation, CtfeProvenance, FnArg, Frame, GlobalAlloc, ImmTy, - InterpCx, InterpResult, MPlaceTy, OpTy, Pointer, PointerArithmetic, RangeSet, Scalar, - StackPopCleanup, compile_time_machine, interp_ok, throw_exhaust, throw_inval, throw_ub, - throw_ub_custom, throw_unsup, throw_unsup_format, + InterpCx, InterpResult, MPlaceTy, OpTy, Pointer, RangeSet, Scalar, compile_time_machine, + interp_ok, throw_exhaust, throw_inval, throw_ub, throw_ub_custom, throw_unsup, + throw_unsup_format, }; /// When hitting this many interpreted terminators we emit a deny by default lint @@ -227,8 +226,8 @@ impl<'tcx> CompileTimeInterpCx<'tcx> { &mut self, instance: ty::Instance<'tcx>, args: &[FnArg<'tcx>], - dest: &MPlaceTy<'tcx>, - ret: Option, + _dest: &MPlaceTy<'tcx>, + _ret: Option, ) -> InterpResult<'tcx, Option>> { let def_id = instance.def_id(); @@ -260,85 +259,10 @@ impl<'tcx> CompileTimeInterpCx<'tcx> { ); return interp_ok(Some(new_instance)); - } else if self.tcx.is_lang_item(def_id, LangItem::AlignOffset) { - let args = self.copy_fn_args(args); - // For align_offset, we replace the function call if the pointer has no address. - match self.align_offset(instance, &args, dest, ret)? { - ControlFlow::Continue(()) => return interp_ok(Some(instance)), - ControlFlow::Break(()) => return interp_ok(None), - } } interp_ok(Some(instance)) } - /// `align_offset(ptr, target_align)` needs special handling in const eval, because the pointer - /// may not have an address. - /// - /// If `ptr` does have a known address, then we return `Continue(())` and the function call should - /// proceed as normal. - /// - /// If `ptr` doesn't have an address, but its underlying allocation's alignment is at most - /// `target_align`, then we call the function again with an dummy address relative to the - /// allocation. - /// - /// If `ptr` doesn't have an address and `target_align` is stricter than the underlying - /// allocation's alignment, then we return `usize::MAX` immediately. - fn align_offset( - &mut self, - instance: ty::Instance<'tcx>, - args: &[OpTy<'tcx>], - dest: &MPlaceTy<'tcx>, - ret: Option, - ) -> InterpResult<'tcx, ControlFlow<()>> { - assert_eq!(args.len(), 2); - - let ptr = self.read_pointer(&args[0])?; - let target_align = self.read_scalar(&args[1])?.to_target_usize(self)?; - - if !target_align.is_power_of_two() { - throw_ub_custom!( - fluent::const_eval_align_offset_invalid_align, - target_align = target_align, - ); - } - - match self.ptr_try_get_alloc_id(ptr, 0) { - Ok((alloc_id, offset, _extra)) => { - let (_size, alloc_align, _kind) = self.get_alloc_info(alloc_id); - - if target_align <= alloc_align.bytes() { - // Extract the address relative to the allocation base that is definitely - // sufficiently aligned and call `align_offset` again. - let addr = ImmTy::from_uint(offset.bytes(), args[0].layout).into(); - let align = ImmTy::from_uint(target_align, args[1].layout).into(); - let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty())?; - - // Push the stack frame with our own adjusted arguments. - self.init_stack_frame( - instance, - self.load_mir(instance.def, None)?, - fn_abi, - &[FnArg::Copy(addr), FnArg::Copy(align)], - /* with_caller_location = */ false, - dest, - StackPopCleanup::Goto { ret, unwind: mir::UnwindAction::Unreachable }, - )?; - interp_ok(ControlFlow::Break(())) - } else { - // Not alignable in const, return `usize::MAX`. - let usize_max = Scalar::from_target_usize(self.target_usize_max(), self); - self.write_scalar(usize_max, dest)?; - self.return_to_block(ret)?; - interp_ok(ControlFlow::Break(())) - } - } - Err(_addr) => { - // The pointer has an address, continue with function call. - interp_ok(ControlFlow::Continue(())) - } - } - } - /// See documentation on the `ptr_guaranteed_cmp` intrinsic. fn guaranteed_cmp(&mut self, a: Scalar, b: Scalar) -> InterpResult<'tcx, u8> { interp_ok(match (a, b) { diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs index 7d81f977c22..c952d5f6d77 100644 --- a/compiler/rustc_hir/src/lang_items.rs +++ b/compiler/rustc_hir/src/lang_items.rs @@ -348,9 +348,6 @@ language_item_table! { MaybeUninit, sym::maybe_uninit, maybe_uninit, Target::Union, GenericRequirement::None; - /// Align offset for stride != 1; must not panic. - AlignOffset, sym::align_offset, align_offset_fn, Target::Fn, GenericRequirement::None; - Termination, sym::termination, termination, Target::Trait, GenericRequirement::None; Try, sym::Try, try_trait, Target::Trait, GenericRequirement::None; diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 890c4fdafef..fac2180d63b 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -378,7 +378,6 @@ symbols! { aggregate_raw_ptr, alias, align, - align_offset, alignment, all, alloc, diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index bbfe32027e8..686378fc5cf 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -112,7 +112,6 @@ #![feature(asm_experimental_arch)] #![feature(const_align_of_val)] #![feature(const_align_of_val_raw)] -#![feature(const_align_offset)] #![feature(const_alloc_layout)] #![feature(const_arguments_as_str)] #![feature(const_black_box)] @@ -125,7 +124,6 @@ #![feature(const_nonnull_new)] #![feature(const_option_ext)] #![feature(const_pin_2)] -#![feature(const_pointer_is_aligned)] #![feature(const_ptr_is_null)] #![feature(const_ptr_sub_ptr)] #![feature(const_raw_ptr_comparison)] diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs index f00984da527..a4e8e373e04 100644 --- a/library/core/src/ptr/const_ptr.rs +++ b/library/core/src/ptr/const_ptr.rs @@ -1358,15 +1358,6 @@ impl *const T { /// beyond the allocation that the pointer points into. It is up to the caller to ensure that /// the returned offset is correct in all terms other than alignment. /// - /// When this is called during compile-time evaluation (which is unstable), the implementation - /// may return `usize::MAX` in cases where that can never happen at runtime. This is because the - /// actual alignment of pointers is not known yet during compile-time, so an offset with - /// guaranteed alignment can sometimes not be computed. For example, a buffer declared as `[u8; - /// N]` might be allocated at an odd or an even address, but at compile-time this is not yet - /// known, so the execution has to be correct for either choice. It is therefore impossible to - /// find an offset that is guaranteed to be 2-aligned. (This behavior is subject to change, as usual - /// for unstable APIs.) - /// /// # Panics /// /// The function panics if `align` is not a power-of-two. @@ -1395,8 +1386,7 @@ impl *const T { #[must_use] #[inline] #[stable(feature = "align_offset", since = "1.36.0")] - #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] - pub const fn align_offset(self, align: usize) -> usize + pub fn align_offset(self, align: usize) -> usize where T: Sized, { @@ -1431,94 +1421,10 @@ impl *const T { /// assert!(ptr.is_aligned()); /// assert!(!ptr.wrapping_byte_add(1).is_aligned()); /// ``` - /// - /// # At compiletime - /// **Note: Alignment at compiletime is experimental and subject to change. See the - /// [tracking issue] for details.** - /// - /// At compiletime, the compiler may not know where a value will end up in memory. - /// Calling this function on a pointer created from a reference at compiletime will only - /// return `true` if the pointer is guaranteed to be aligned. This means that the pointer - /// is never aligned if cast to a type with a stricter alignment than the reference's - /// underlying allocation. - /// - /// ``` - /// #![feature(const_pointer_is_aligned)] - /// - /// // On some platforms, the alignment of primitives is less than their size. - /// #[repr(align(4))] - /// struct AlignedI32(i32); - /// #[repr(align(8))] - /// struct AlignedI64(i64); - /// - /// const _: () = { - /// let data = AlignedI32(42); - /// let ptr = &data as *const AlignedI32; - /// assert!(ptr.is_aligned()); - /// - /// // At runtime either `ptr1` or `ptr2` would be aligned, but at compiletime neither is aligned. - /// let ptr1 = ptr.cast::(); - /// let ptr2 = ptr.wrapping_add(1).cast::(); - /// assert!(!ptr1.is_aligned()); - /// assert!(!ptr2.is_aligned()); - /// }; - /// ``` - /// - /// Due to this behavior, it is possible that a runtime pointer derived from a compiletime - /// pointer is aligned, even if the compiletime pointer wasn't aligned. - /// - /// ``` - /// #![feature(const_pointer_is_aligned)] - /// - /// // On some platforms, the alignment of primitives is less than their size. - /// #[repr(align(4))] - /// struct AlignedI32(i32); - /// #[repr(align(8))] - /// struct AlignedI64(i64); - /// - /// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned. - /// const COMPTIME_PTR: *const AlignedI32 = &AlignedI32(42); - /// const _: () = assert!(!COMPTIME_PTR.cast::().is_aligned()); - /// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).cast::().is_aligned()); - /// - /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. - /// let runtime_ptr = COMPTIME_PTR; - /// assert_ne!( - /// runtime_ptr.cast::().is_aligned(), - /// runtime_ptr.wrapping_add(1).cast::().is_aligned(), - /// ); - /// ``` - /// - /// If a pointer is created from a fixed address, this function behaves the same during - /// runtime and compiletime. - /// - /// ``` - /// #![feature(const_pointer_is_aligned)] - /// - /// // On some platforms, the alignment of primitives is less than their size. - /// #[repr(align(4))] - /// struct AlignedI32(i32); - /// #[repr(align(8))] - /// struct AlignedI64(i64); - /// - /// const _: () = { - /// let ptr = 40 as *const AlignedI32; - /// assert!(ptr.is_aligned()); - /// - /// // For pointers with a known address, runtime and compiletime behavior are identical. - /// let ptr1 = ptr.cast::(); - /// let ptr2 = ptr.wrapping_add(1).cast::(); - /// assert!(ptr1.is_aligned()); - /// assert!(!ptr2.is_aligned()); - /// }; - /// ``` - /// - /// [tracking issue]: https://github.com/rust-lang/rust/issues/104203 #[must_use] #[inline] #[stable(feature = "pointer_is_aligned", since = "1.79.0")] - #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")] - pub const fn is_aligned(self) -> bool + pub fn is_aligned(self) -> bool where T: Sized, { @@ -1555,105 +1461,15 @@ impl *const T { /// /// assert_ne!(ptr.is_aligned_to(8), ptr.wrapping_add(1).is_aligned_to(8)); /// ``` - /// - /// # At compiletime - /// **Note: Alignment at compiletime is experimental and subject to change. See the - /// [tracking issue] for details.** - /// - /// At compiletime, the compiler may not know where a value will end up in memory. - /// Calling this function on a pointer created from a reference at compiletime will only - /// return `true` if the pointer is guaranteed to be aligned. This means that the pointer - /// cannot be stricter aligned than the reference's underlying allocation. - /// - /// ``` - /// #![feature(pointer_is_aligned_to)] - /// #![feature(const_pointer_is_aligned)] - /// - /// // On some platforms, the alignment of i32 is less than 4. - /// #[repr(align(4))] - /// struct AlignedI32(i32); - /// - /// const _: () = { - /// let data = AlignedI32(42); - /// let ptr = &data as *const AlignedI32; - /// - /// assert!(ptr.is_aligned_to(1)); - /// assert!(ptr.is_aligned_to(2)); - /// assert!(ptr.is_aligned_to(4)); - /// - /// // At compiletime, we know for sure that the pointer isn't aligned to 8. - /// assert!(!ptr.is_aligned_to(8)); - /// assert!(!ptr.wrapping_add(1).is_aligned_to(8)); - /// }; - /// ``` - /// - /// Due to this behavior, it is possible that a runtime pointer derived from a compiletime - /// pointer is aligned, even if the compiletime pointer wasn't aligned. - /// - /// ``` - /// #![feature(pointer_is_aligned_to)] - /// #![feature(const_pointer_is_aligned)] - /// - /// // On some platforms, the alignment of i32 is less than 4. - /// #[repr(align(4))] - /// struct AlignedI32(i32); - /// - /// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned. - /// const COMPTIME_PTR: *const AlignedI32 = &AlignedI32(42); - /// const _: () = assert!(!COMPTIME_PTR.is_aligned_to(8)); - /// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).is_aligned_to(8)); - /// - /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. - /// let runtime_ptr = COMPTIME_PTR; - /// assert_ne!( - /// runtime_ptr.is_aligned_to(8), - /// runtime_ptr.wrapping_add(1).is_aligned_to(8), - /// ); - /// ``` - /// - /// If a pointer is created from a fixed address, this function behaves the same during - /// runtime and compiletime. - /// - /// ``` - /// #![feature(pointer_is_aligned_to)] - /// #![feature(const_pointer_is_aligned)] - /// - /// const _: () = { - /// let ptr = 40 as *const u8; - /// assert!(ptr.is_aligned_to(1)); - /// assert!(ptr.is_aligned_to(2)); - /// assert!(ptr.is_aligned_to(4)); - /// assert!(ptr.is_aligned_to(8)); - /// assert!(!ptr.is_aligned_to(16)); - /// }; - /// ``` - /// - /// [tracking issue]: https://github.com/rust-lang/rust/issues/104203 #[must_use] #[inline] #[unstable(feature = "pointer_is_aligned_to", issue = "96284")] - #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")] - pub const fn is_aligned_to(self, align: usize) -> bool { + pub fn is_aligned_to(self, align: usize) -> bool { if !align.is_power_of_two() { panic!("is_aligned_to: align is not a power-of-two"); } - #[inline] - fn runtime_impl(ptr: *const (), align: usize) -> bool { - ptr.addr() & (align - 1) == 0 - } - - #[inline] - #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")] - const fn const_impl(ptr: *const (), align: usize) -> bool { - // We can't use the address of `self` in a `const fn`, so we use `align_offset` instead. - ptr.align_offset(align) == 0 - } - - // The cast to `()` is used to - // 1. deal with fat pointers; and - // 2. ensure that `align_offset` (in `const_impl`) doesn't actually try to compute an offset. - const_eval_select((self.cast::<()>(), align), const_impl, runtime_impl) + self.addr() & (align - 1) == 0 } } diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs index 7c2205fdcd1..b98e2b5eac8 100644 --- a/library/core/src/ptr/mod.rs +++ b/library/core/src/ptr/mod.rs @@ -1852,9 +1852,7 @@ pub unsafe fn write_volatile(dst: *mut T, src: T) { /// /// Any questions go to @nagisa. #[allow(ptr_to_integer_transmute_in_consts)] -#[lang = "align_offset"] -#[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] -pub(crate) const unsafe fn align_offset(p: *const T, a: usize) -> usize { +pub(crate) unsafe fn align_offset(p: *const T, a: usize) -> usize { // FIXME(#75598): Direct use of these intrinsics improves codegen significantly at opt-level <= // 1, where the method versions of these operations are not inlined. use intrinsics::{ @@ -1915,11 +1913,7 @@ pub(crate) const unsafe fn align_offset(p: *const T, a: usize) -> usiz let stride = mem::size_of::(); - // SAFETY: This is just an inlined `p.addr()` (which is not - // a `const fn` so we cannot call it). - // During const eval, we hook this function to ensure that the pointer never - // has provenance, making this sound. - let addr: usize = unsafe { mem::transmute(p) }; + let addr: usize = p.addr(); // SAFETY: `a` is a power-of-two, therefore non-zero. let a_minus_one = unsafe { unchecked_sub(a, 1) }; diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs index 60a2dcd1ee7..0d94a7f491c 100644 --- a/library/core/src/ptr/mut_ptr.rs +++ b/library/core/src/ptr/mut_ptr.rs @@ -1,6 +1,5 @@ use super::*; use crate::cmp::Ordering::{Equal, Greater, Less}; -use crate::intrinsics::const_eval_select; use crate::mem::SizedTypeProperties; use crate::slice::{self, SliceIndex}; @@ -1636,8 +1635,7 @@ impl *mut T { #[must_use] #[inline] #[stable(feature = "align_offset", since = "1.36.0")] - #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] - pub const fn align_offset(self, align: usize) -> usize + pub fn align_offset(self, align: usize) -> usize where T: Sized, { @@ -1675,95 +1673,10 @@ impl *mut T { /// assert!(ptr.is_aligned()); /// assert!(!ptr.wrapping_byte_add(1).is_aligned()); /// ``` - /// - /// # At compiletime - /// **Note: Alignment at compiletime is experimental and subject to change. See the - /// [tracking issue] for details.** - /// - /// At compiletime, the compiler may not know where a value will end up in memory. - /// Calling this function on a pointer created from a reference at compiletime will only - /// return `true` if the pointer is guaranteed to be aligned. This means that the pointer - /// is never aligned if cast to a type with a stricter alignment than the reference's - /// underlying allocation. - /// - /// ``` - /// #![feature(const_pointer_is_aligned)] - /// - /// // On some platforms, the alignment of primitives is less than their size. - /// #[repr(align(4))] - /// struct AlignedI32(i32); - /// #[repr(align(8))] - /// struct AlignedI64(i64); - /// - /// const _: () = { - /// let mut data = AlignedI32(42); - /// let ptr = &mut data as *mut AlignedI32; - /// assert!(ptr.is_aligned()); - /// - /// // At runtime either `ptr1` or `ptr2` would be aligned, but at compiletime neither is aligned. - /// let ptr1 = ptr.cast::(); - /// let ptr2 = ptr.wrapping_add(1).cast::(); - /// assert!(!ptr1.is_aligned()); - /// assert!(!ptr2.is_aligned()); - /// }; - /// ``` - /// - /// Due to this behavior, it is possible that a runtime pointer derived from a compiletime - /// pointer is aligned, even if the compiletime pointer wasn't aligned. - /// - /// ``` - /// #![feature(const_pointer_is_aligned)] - /// - /// // On some platforms, the alignment of primitives is less than their size. - /// #[repr(align(4))] - /// struct AlignedI32(i32); - /// #[repr(align(8))] - /// struct AlignedI64(i64); - /// - /// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned. - /// // Also, note that mutable references are not allowed in the final value of constants. - /// const COMPTIME_PTR: *mut AlignedI32 = (&AlignedI32(42) as *const AlignedI32).cast_mut(); - /// const _: () = assert!(!COMPTIME_PTR.cast::().is_aligned()); - /// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).cast::().is_aligned()); - /// - /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. - /// let runtime_ptr = COMPTIME_PTR; - /// assert_ne!( - /// runtime_ptr.cast::().is_aligned(), - /// runtime_ptr.wrapping_add(1).cast::().is_aligned(), - /// ); - /// ``` - /// - /// If a pointer is created from a fixed address, this function behaves the same during - /// runtime and compiletime. - /// - /// ``` - /// #![feature(const_pointer_is_aligned)] - /// - /// // On some platforms, the alignment of primitives is less than their size. - /// #[repr(align(4))] - /// struct AlignedI32(i32); - /// #[repr(align(8))] - /// struct AlignedI64(i64); - /// - /// const _: () = { - /// let ptr = 40 as *mut AlignedI32; - /// assert!(ptr.is_aligned()); - /// - /// // For pointers with a known address, runtime and compiletime behavior are identical. - /// let ptr1 = ptr.cast::(); - /// let ptr2 = ptr.wrapping_add(1).cast::(); - /// assert!(ptr1.is_aligned()); - /// assert!(!ptr2.is_aligned()); - /// }; - /// ``` - /// - /// [tracking issue]: https://github.com/rust-lang/rust/issues/104203 #[must_use] #[inline] #[stable(feature = "pointer_is_aligned", since = "1.79.0")] - #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")] - pub const fn is_aligned(self) -> bool + pub fn is_aligned(self) -> bool where T: Sized, { @@ -1800,106 +1713,15 @@ impl *mut T { /// /// assert_ne!(ptr.is_aligned_to(8), ptr.wrapping_add(1).is_aligned_to(8)); /// ``` - /// - /// # At compiletime - /// **Note: Alignment at compiletime is experimental and subject to change. See the - /// [tracking issue] for details.** - /// - /// At compiletime, the compiler may not know where a value will end up in memory. - /// Calling this function on a pointer created from a reference at compiletime will only - /// return `true` if the pointer is guaranteed to be aligned. This means that the pointer - /// cannot be stricter aligned than the reference's underlying allocation. - /// - /// ``` - /// #![feature(pointer_is_aligned_to)] - /// #![feature(const_pointer_is_aligned)] - /// - /// // On some platforms, the alignment of i32 is less than 4. - /// #[repr(align(4))] - /// struct AlignedI32(i32); - /// - /// const _: () = { - /// let mut data = AlignedI32(42); - /// let ptr = &mut data as *mut AlignedI32; - /// - /// assert!(ptr.is_aligned_to(1)); - /// assert!(ptr.is_aligned_to(2)); - /// assert!(ptr.is_aligned_to(4)); - /// - /// // At compiletime, we know for sure that the pointer isn't aligned to 8. - /// assert!(!ptr.is_aligned_to(8)); - /// assert!(!ptr.wrapping_add(1).is_aligned_to(8)); - /// }; - /// ``` - /// - /// Due to this behavior, it is possible that a runtime pointer derived from a compiletime - /// pointer is aligned, even if the compiletime pointer wasn't aligned. - /// - /// ``` - /// #![feature(pointer_is_aligned_to)] - /// #![feature(const_pointer_is_aligned)] - /// - /// // On some platforms, the alignment of i32 is less than 4. - /// #[repr(align(4))] - /// struct AlignedI32(i32); - /// - /// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned. - /// // Also, note that mutable references are not allowed in the final value of constants. - /// const COMPTIME_PTR: *mut AlignedI32 = (&AlignedI32(42) as *const AlignedI32).cast_mut(); - /// const _: () = assert!(!COMPTIME_PTR.is_aligned_to(8)); - /// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).is_aligned_to(8)); - /// - /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. - /// let runtime_ptr = COMPTIME_PTR; - /// assert_ne!( - /// runtime_ptr.is_aligned_to(8), - /// runtime_ptr.wrapping_add(1).is_aligned_to(8), - /// ); - /// ``` - /// - /// If a pointer is created from a fixed address, this function behaves the same during - /// runtime and compiletime. - /// - /// ``` - /// #![feature(pointer_is_aligned_to)] - /// #![feature(const_pointer_is_aligned)] - /// - /// const _: () = { - /// let ptr = 40 as *mut u8; - /// assert!(ptr.is_aligned_to(1)); - /// assert!(ptr.is_aligned_to(2)); - /// assert!(ptr.is_aligned_to(4)); - /// assert!(ptr.is_aligned_to(8)); - /// assert!(!ptr.is_aligned_to(16)); - /// }; - /// ``` - /// - /// [tracking issue]: https://github.com/rust-lang/rust/issues/104203 #[must_use] #[inline] #[unstable(feature = "pointer_is_aligned_to", issue = "96284")] - #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")] - pub const fn is_aligned_to(self, align: usize) -> bool { + pub fn is_aligned_to(self, align: usize) -> bool { if !align.is_power_of_two() { panic!("is_aligned_to: align is not a power-of-two"); } - #[inline] - fn runtime_impl(ptr: *mut (), align: usize) -> bool { - ptr.addr() & (align - 1) == 0 - } - - #[inline] - #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")] - const fn const_impl(ptr: *mut (), align: usize) -> bool { - // We can't use the address of `self` in a `const fn`, so we use `align_offset` instead. - ptr.align_offset(align) == 0 - } - - // The cast to `()` is used to - // 1. deal with fat pointers; and - // 2. ensure that `align_offset` (in `const_impl`) doesn't actually try to compute an offset. - const_eval_select((self.cast::<()>(), align), const_impl, runtime_impl) + self.addr() & (align - 1) == 0 } } diff --git a/library/core/src/ptr/non_null.rs b/library/core/src/ptr/non_null.rs index 44eeb052661..ad24e50aa15 100644 --- a/library/core/src/ptr/non_null.rs +++ b/library/core/src/ptr/non_null.rs @@ -1192,8 +1192,7 @@ impl NonNull { #[inline] #[must_use] #[stable(feature = "non_null_convenience", since = "1.80.0")] - #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] - pub const fn align_offset(self, align: usize) -> usize + pub fn align_offset(self, align: usize) -> usize where T: Sized, { @@ -1224,98 +1223,10 @@ impl NonNull { /// assert!(ptr.is_aligned()); /// assert!(!NonNull::new(ptr.as_ptr().wrapping_byte_add(1)).unwrap().is_aligned()); /// ``` - /// - /// # At compiletime - /// **Note: Alignment at compiletime is experimental and subject to change. See the - /// [tracking issue] for details.** - /// - /// At compiletime, the compiler may not know where a value will end up in memory. - /// Calling this function on a pointer created from a reference at compiletime will only - /// return `true` if the pointer is guaranteed to be aligned. This means that the pointer - /// is never aligned if cast to a type with a stricter alignment than the reference's - /// underlying allocation. - /// - /// ``` - /// #![feature(const_nonnull_new)] - /// #![feature(const_pointer_is_aligned)] - /// use std::ptr::NonNull; - /// - /// // On some platforms, the alignment of primitives is less than their size. - /// #[repr(align(4))] - /// struct AlignedI32(i32); - /// #[repr(align(8))] - /// struct AlignedI64(i64); - /// - /// const _: () = { - /// let data = [AlignedI32(42), AlignedI32(42)]; - /// let ptr = NonNull::::new(&data[0] as *const _ as *mut _).unwrap(); - /// assert!(ptr.is_aligned()); - /// - /// // At runtime either `ptr1` or `ptr2` would be aligned, but at compiletime neither is aligned. - /// let ptr1 = ptr.cast::(); - /// let ptr2 = unsafe { ptr.add(1).cast::() }; - /// assert!(!ptr1.is_aligned()); - /// assert!(!ptr2.is_aligned()); - /// }; - /// ``` - /// - /// Due to this behavior, it is possible that a runtime pointer derived from a compiletime - /// pointer is aligned, even if the compiletime pointer wasn't aligned. - /// - /// ``` - /// #![feature(const_pointer_is_aligned)] - /// - /// // On some platforms, the alignment of primitives is less than their size. - /// #[repr(align(4))] - /// struct AlignedI32(i32); - /// #[repr(align(8))] - /// struct AlignedI64(i64); - /// - /// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned. - /// const COMPTIME_PTR: *const AlignedI32 = &AlignedI32(42); - /// const _: () = assert!(!COMPTIME_PTR.cast::().is_aligned()); - /// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).cast::().is_aligned()); - /// - /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. - /// let runtime_ptr = COMPTIME_PTR; - /// assert_ne!( - /// runtime_ptr.cast::().is_aligned(), - /// runtime_ptr.wrapping_add(1).cast::().is_aligned(), - /// ); - /// ``` - /// - /// If a pointer is created from a fixed address, this function behaves the same during - /// runtime and compiletime. - /// - /// ``` - /// #![feature(const_pointer_is_aligned)] - /// #![feature(const_nonnull_new)] - /// use std::ptr::NonNull; - /// - /// // On some platforms, the alignment of primitives is less than their size. - /// #[repr(align(4))] - /// struct AlignedI32(i32); - /// #[repr(align(8))] - /// struct AlignedI64(i64); - /// - /// const _: () = { - /// let ptr = NonNull::new(40 as *mut AlignedI32).unwrap(); - /// assert!(ptr.is_aligned()); - /// - /// // For pointers with a known address, runtime and compiletime behavior are identical. - /// let ptr1 = ptr.cast::(); - /// let ptr2 = NonNull::new(ptr.as_ptr().wrapping_add(1)).unwrap().cast::(); - /// assert!(ptr1.is_aligned()); - /// assert!(!ptr2.is_aligned()); - /// }; - /// ``` - /// - /// [tracking issue]: https://github.com/rust-lang/rust/issues/104203 #[inline] #[must_use] #[stable(feature = "pointer_is_aligned", since = "1.79.0")] - #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")] - pub const fn is_aligned(self) -> bool + pub fn is_aligned(self) -> bool where T: Sized, { @@ -1352,85 +1263,10 @@ impl NonNull { /// /// assert_ne!(ptr.is_aligned_to(8), ptr.wrapping_add(1).is_aligned_to(8)); /// ``` - /// - /// # At compiletime - /// **Note: Alignment at compiletime is experimental and subject to change. See the - /// [tracking issue] for details.** - /// - /// At compiletime, the compiler may not know where a value will end up in memory. - /// Calling this function on a pointer created from a reference at compiletime will only - /// return `true` if the pointer is guaranteed to be aligned. This means that the pointer - /// cannot be stricter aligned than the reference's underlying allocation. - /// - /// ``` - /// #![feature(pointer_is_aligned_to)] - /// #![feature(const_pointer_is_aligned)] - /// - /// // On some platforms, the alignment of i32 is less than 4. - /// #[repr(align(4))] - /// struct AlignedI32(i32); - /// - /// const _: () = { - /// let data = AlignedI32(42); - /// let ptr = &data as *const AlignedI32; - /// - /// assert!(ptr.is_aligned_to(1)); - /// assert!(ptr.is_aligned_to(2)); - /// assert!(ptr.is_aligned_to(4)); - /// - /// // At compiletime, we know for sure that the pointer isn't aligned to 8. - /// assert!(!ptr.is_aligned_to(8)); - /// assert!(!ptr.wrapping_add(1).is_aligned_to(8)); - /// }; - /// ``` - /// - /// Due to this behavior, it is possible that a runtime pointer derived from a compiletime - /// pointer is aligned, even if the compiletime pointer wasn't aligned. - /// - /// ``` - /// #![feature(pointer_is_aligned_to)] - /// #![feature(const_pointer_is_aligned)] - /// - /// // On some platforms, the alignment of i32 is less than 4. - /// #[repr(align(4))] - /// struct AlignedI32(i32); - /// - /// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned. - /// const COMPTIME_PTR: *const AlignedI32 = &AlignedI32(42); - /// const _: () = assert!(!COMPTIME_PTR.is_aligned_to(8)); - /// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).is_aligned_to(8)); - /// - /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. - /// let runtime_ptr = COMPTIME_PTR; - /// assert_ne!( - /// runtime_ptr.is_aligned_to(8), - /// runtime_ptr.wrapping_add(1).is_aligned_to(8), - /// ); - /// ``` - /// - /// If a pointer is created from a fixed address, this function behaves the same during - /// runtime and compiletime. - /// - /// ``` - /// #![feature(pointer_is_aligned_to)] - /// #![feature(const_pointer_is_aligned)] - /// - /// const _: () = { - /// let ptr = 40 as *const u8; - /// assert!(ptr.is_aligned_to(1)); - /// assert!(ptr.is_aligned_to(2)); - /// assert!(ptr.is_aligned_to(4)); - /// assert!(ptr.is_aligned_to(8)); - /// assert!(!ptr.is_aligned_to(16)); - /// }; - /// ``` - /// - /// [tracking issue]: https://github.com/rust-lang/rust/issues/104203 #[inline] #[must_use] #[unstable(feature = "pointer_is_aligned_to", issue = "96284")] - #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")] - pub const fn is_aligned_to(self, align: usize) -> bool { + pub fn is_aligned_to(self, align: usize) -> bool { self.pointer.is_aligned_to(align) } } diff --git a/library/core/src/slice/ascii.rs b/library/core/src/slice/ascii.rs index 21e0460072f..8dcd34929e1 100644 --- a/library/core/src/slice/ascii.rs +++ b/library/core/src/slice/ascii.rs @@ -3,6 +3,7 @@ use core::ascii::EscapeDefault; use crate::fmt::{self, Write}; +use crate::intrinsics::const_eval_select; use crate::{ascii, iter, mem, ops}; #[cfg(not(test))] @@ -346,89 +347,93 @@ pub const fn is_ascii_simple(mut bytes: &[u8]) -> bool { /// If any of these loads produces something for which `contains_nonascii` /// (above) returns true, then we know the answer is false. #[inline] -#[rustc_allow_const_fn_unstable(const_raw_ptr_comparison, const_pointer_is_aligned)] // only in a debug assertion -#[rustc_allow_const_fn_unstable(const_align_offset)] // behavior does not change when `align_offset` fails +#[rustc_allow_const_fn_unstable(const_eval_select)] // fallback impl has same behavior const fn is_ascii(s: &[u8]) -> bool { - const USIZE_SIZE: usize = mem::size_of::(); + // The runtime version behaves the same as the compiletime version, it's + // just more optimized. + return const_eval_select((s,), compiletime, runtime); - let len = s.len(); - let align_offset = s.as_ptr().align_offset(USIZE_SIZE); - - // If we wouldn't gain anything from the word-at-a-time implementation, fall - // back to a scalar loop. - // - // We also do this for architectures where `size_of::()` isn't - // sufficient alignment for `usize`, because it's a weird edge case. - if len < USIZE_SIZE || len < align_offset || USIZE_SIZE < mem::align_of::() { - return is_ascii_simple(s); + const fn compiletime(s: &[u8]) -> bool { + is_ascii_simple(s) } - // We always read the first word unaligned, which means `align_offset` is - // 0, we'd read the same value again for the aligned read. - let offset_to_aligned = if align_offset == 0 { USIZE_SIZE } else { align_offset }; + #[inline] + fn runtime(s: &[u8]) -> bool { + const USIZE_SIZE: usize = mem::size_of::(); - let start = s.as_ptr(); - // SAFETY: We verify `len < USIZE_SIZE` above. - let first_word = unsafe { (start as *const usize).read_unaligned() }; + let len = s.len(); + let align_offset = s.as_ptr().align_offset(USIZE_SIZE); - if contains_nonascii(first_word) { - return false; - } - // We checked this above, somewhat implicitly. Note that `offset_to_aligned` - // is either `align_offset` or `USIZE_SIZE`, both of are explicitly checked - // above. - debug_assert!(offset_to_aligned <= len); - - // SAFETY: word_ptr is the (properly aligned) usize ptr we use to read the - // middle chunk of the slice. - let mut word_ptr = unsafe { start.add(offset_to_aligned) as *const usize }; - - // `byte_pos` is the byte index of `word_ptr`, used for loop end checks. - let mut byte_pos = offset_to_aligned; - - // Paranoia check about alignment, since we're about to do a bunch of - // unaligned loads. In practice this should be impossible barring a bug in - // `align_offset` though. - // While this method is allowed to spuriously fail in CTFE, if it doesn't - // have alignment information it should have given a `usize::MAX` for - // `align_offset` earlier, sending things through the scalar path instead of - // this one, so this check should pass if it's reachable. - debug_assert!(word_ptr.is_aligned_to(mem::align_of::())); - - // Read subsequent words until the last aligned word, excluding the last - // aligned word by itself to be done in tail check later, to ensure that - // tail is always one `usize` at most to extra branch `byte_pos == len`. - while byte_pos < len - USIZE_SIZE { - // Sanity check that the read is in bounds - debug_assert!(byte_pos + USIZE_SIZE <= len); - // And that our assumptions about `byte_pos` hold. - debug_assert!(matches!( - word_ptr.cast::().guaranteed_eq(start.wrapping_add(byte_pos)), - // These are from the same allocation, so will hopefully always be - // known to match even in CTFE, but if it refuses to compare them - // that's ok since it's just a debug check anyway. - None | Some(true), - )); - - // SAFETY: We know `word_ptr` is properly aligned (because of - // `align_offset`), and we know that we have enough bytes between `word_ptr` and the end - let word = unsafe { word_ptr.read() }; - if contains_nonascii(word) { - return false; + // If we wouldn't gain anything from the word-at-a-time implementation, fall + // back to a scalar loop. + // + // We also do this for architectures where `size_of::()` isn't + // sufficient alignment for `usize`, because it's a weird edge case. + if len < USIZE_SIZE || len < align_offset || USIZE_SIZE < mem::align_of::() { + return is_ascii_simple(s); } - byte_pos += USIZE_SIZE; - // SAFETY: We know that `byte_pos <= len - USIZE_SIZE`, which means that - // after this `add`, `word_ptr` will be at most one-past-the-end. - word_ptr = unsafe { word_ptr.add(1) }; + // We always read the first word unaligned, which means `align_offset` is + // 0, we'd read the same value again for the aligned read. + let offset_to_aligned = if align_offset == 0 { USIZE_SIZE } else { align_offset }; + + let start = s.as_ptr(); + // SAFETY: We verify `len < USIZE_SIZE` above. + let first_word = unsafe { (start as *const usize).read_unaligned() }; + + if contains_nonascii(first_word) { + return false; + } + // We checked this above, somewhat implicitly. Note that `offset_to_aligned` + // is either `align_offset` or `USIZE_SIZE`, both of are explicitly checked + // above. + debug_assert!(offset_to_aligned <= len); + + // SAFETY: word_ptr is the (properly aligned) usize ptr we use to read the + // middle chunk of the slice. + let mut word_ptr = unsafe { start.add(offset_to_aligned) as *const usize }; + + // `byte_pos` is the byte index of `word_ptr`, used for loop end checks. + let mut byte_pos = offset_to_aligned; + + // Paranoia check about alignment, since we're about to do a bunch of + // unaligned loads. In practice this should be impossible barring a bug in + // `align_offset` though. + // While this method is allowed to spuriously fail in CTFE, if it doesn't + // have alignment information it should have given a `usize::MAX` for + // `align_offset` earlier, sending things through the scalar path instead of + // this one, so this check should pass if it's reachable. + debug_assert!(word_ptr.is_aligned_to(mem::align_of::())); + + // Read subsequent words until the last aligned word, excluding the last + // aligned word by itself to be done in tail check later, to ensure that + // tail is always one `usize` at most to extra branch `byte_pos == len`. + while byte_pos < len - USIZE_SIZE { + // Sanity check that the read is in bounds + debug_assert!(byte_pos + USIZE_SIZE <= len); + // And that our assumptions about `byte_pos` hold. + debug_assert!(word_ptr.cast::() == start.wrapping_add(byte_pos)); + + // SAFETY: We know `word_ptr` is properly aligned (because of + // `align_offset`), and we know that we have enough bytes between `word_ptr` and the end + let word = unsafe { word_ptr.read() }; + if contains_nonascii(word) { + return false; + } + + byte_pos += USIZE_SIZE; + // SAFETY: We know that `byte_pos <= len - USIZE_SIZE`, which means that + // after this `add`, `word_ptr` will be at most one-past-the-end. + word_ptr = unsafe { word_ptr.add(1) }; + } + + // Sanity check to ensure there really is only one `usize` left. This should + // be guaranteed by our loop condition. + debug_assert!(byte_pos <= len && len - byte_pos <= USIZE_SIZE); + + // SAFETY: This relies on `len >= USIZE_SIZE`, which we check at the start. + let last_word = unsafe { (start.add(len - USIZE_SIZE) as *const usize).read_unaligned() }; + + !contains_nonascii(last_word) } - - // Sanity check to ensure there really is only one `usize` left. This should - // be guaranteed by our loop condition. - debug_assert!(byte_pos <= len && len - byte_pos <= USIZE_SIZE); - - // SAFETY: This relies on `len >= USIZE_SIZE`, which we check at the start. - let last_word = unsafe { (start.add(len - USIZE_SIZE) as *const usize).read_unaligned() }; - - !contains_nonascii(last_word) } diff --git a/library/core/src/slice/memchr.rs b/library/core/src/slice/memchr.rs index 57604623262..b7c4a1f6f08 100644 --- a/library/core/src/slice/memchr.rs +++ b/library/core/src/slice/memchr.rs @@ -1,6 +1,7 @@ // Original implementation taken from rust-memchr. // Copyright 2015 Andrew Gallant, bluss and Nicolas Koch +use crate::intrinsics::const_eval_select; use crate::mem; const LO_USIZE: usize = usize::repeat_u8(0x01); @@ -50,58 +51,66 @@ const fn memchr_naive(x: u8, text: &[u8]) -> Option { None } -#[rustc_allow_const_fn_unstable(const_cmp)] -#[rustc_allow_const_fn_unstable(const_align_offset)] +#[rustc_allow_const_fn_unstable(const_eval_select)] // fallback impl has same behavior #[cfg_attr(bootstrap, rustc_const_stable(feature = "const_memchr", since = "1.65.0"))] const fn memchr_aligned(x: u8, text: &[u8]) -> Option { - // Scan for a single byte value by reading two `usize` words at a time. - // - // Split `text` in three parts - // - unaligned initial part, before the first word aligned address in text - // - body, scan by 2 words at a time - // - the last remaining part, < 2 word size + // The runtime version behaves the same as the compiletime version, it's + // just more optimized. + return const_eval_select((x, text), compiletime, runtime); - // search up to an aligned boundary - let len = text.len(); - let ptr = text.as_ptr(); - let mut offset = ptr.align_offset(USIZE_BYTES); - - if offset > 0 { - // FIXME(const-hack, fee1-dead): replace with min - offset = if offset < len { offset } else { len }; - // FIXME(const-hack, fee1-dead): replace with range slicing - // SAFETY: offset is within bounds - let slice = unsafe { super::from_raw_parts(text.as_ptr(), offset) }; - if let Some(index) = memchr_naive(x, slice) { - return Some(index); - } + const fn compiletime(x: u8, text: &[u8]) -> Option { + memchr_naive(x, text) } - // search the body of the text - let repeated_x = usize::repeat_u8(x); - while offset <= len - 2 * USIZE_BYTES { - // SAFETY: the while's predicate guarantees a distance of at least 2 * usize_bytes - // between the offset and the end of the slice. - unsafe { - let u = *(ptr.add(offset) as *const usize); - let v = *(ptr.add(offset + USIZE_BYTES) as *const usize); + #[inline] + fn runtime(x: u8, text: &[u8]) -> Option { + // Scan for a single byte value by reading two `usize` words at a time. + // + // Split `text` in three parts + // - unaligned initial part, before the first word aligned address in text + // - body, scan by 2 words at a time + // - the last remaining part, < 2 word size - // break if there is a matching byte - let zu = contains_zero_byte(u ^ repeated_x); - let zv = contains_zero_byte(v ^ repeated_x); - if zu || zv { - break; + // search up to an aligned boundary + let len = text.len(); + let ptr = text.as_ptr(); + let mut offset = ptr.align_offset(USIZE_BYTES); + + if offset > 0 { + offset = offset.min(len); + let slice = &text[..offset]; + if let Some(index) = memchr_naive(x, slice) { + return Some(index); } } - offset += USIZE_BYTES * 2; - } - // Find the byte after the point the body loop stopped. - // FIXME(const-hack): Use `?` instead. - // FIXME(const-hack, fee1-dead): use range slicing - // SAFETY: offset is within bounds - let slice = unsafe { super::from_raw_parts(text.as_ptr().add(offset), text.len() - offset) }; - if let Some(i) = memchr_naive(x, slice) { Some(offset + i) } else { None } + // search the body of the text + let repeated_x = usize::repeat_u8(x); + while offset <= len - 2 * USIZE_BYTES { + // SAFETY: the while's predicate guarantees a distance of at least 2 * usize_bytes + // between the offset and the end of the slice. + unsafe { + let u = *(ptr.add(offset) as *const usize); + let v = *(ptr.add(offset + USIZE_BYTES) as *const usize); + + // break if there is a matching byte + let zu = contains_zero_byte(u ^ repeated_x); + let zv = contains_zero_byte(v ^ repeated_x); + if zu || zv { + break; + } + } + offset += USIZE_BYTES * 2; + } + + // Find the byte after the point the body loop stopped. + // FIXME(const-hack): Use `?` instead. + // FIXME(const-hack, fee1-dead): use range slicing + let slice = + // SAFETY: offset is within bounds + unsafe { super::from_raw_parts(text.as_ptr().add(offset), text.len() - offset) }; + if let Some(i) = memchr_naive(x, slice) { Some(offset + i) } else { None } + } } /// Returns the last index matching the byte `x` in `text`. diff --git a/library/core/src/str/converts.rs b/library/core/src/str/converts.rs index 2bc1f1e4973..c7bae42765f 100644 --- a/library/core/src/str/converts.rs +++ b/library/core/src/str/converts.rs @@ -82,7 +82,6 @@ use crate::{mem, ptr}; /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[rustc_const_stable(feature = "const_str_from_utf8_shared", since = "1.63.0")] -#[rustc_allow_const_fn_unstable(str_internals)] #[rustc_diagnostic_item = "str_from_utf8"] pub const fn from_utf8(v: &[u8]) -> Result<&str, Utf8Error> { // FIXME(const-hack): This should use `?` again, once it's `const` diff --git a/library/core/src/str/validations.rs b/library/core/src/str/validations.rs index cca8ff74dda..6095b589e18 100644 --- a/library/core/src/str/validations.rs +++ b/library/core/src/str/validations.rs @@ -1,6 +1,7 @@ //! Operations related to UTF-8 validation. use super::Utf8Error; +use crate::intrinsics::const_eval_select; use crate::mem; /// Returns the initial codepoint accumulator for the first byte. @@ -122,15 +123,28 @@ const fn contains_nonascii(x: usize) -> bool { /// Walks through `v` checking that it's a valid UTF-8 sequence, /// returning `Ok(())` in that case, or, if it is invalid, `Err(err)`. #[inline(always)] -#[rustc_const_unstable(feature = "str_internals", issue = "none")] +#[rustc_allow_const_fn_unstable(const_eval_select)] // fallback impl has same behavior pub(super) const fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> { let mut index = 0; let len = v.len(); - let usize_bytes = mem::size_of::(); - let ascii_block_size = 2 * usize_bytes; + const USIZE_BYTES: usize = mem::size_of::(); + + let ascii_block_size = 2 * USIZE_BYTES; let blocks_end = if len >= ascii_block_size { len - ascii_block_size + 1 } else { 0 }; - let align = v.as_ptr().align_offset(usize_bytes); + let align = { + const fn compiletime(_v: &[u8]) -> usize { + usize::MAX + } + + fn runtime(v: &[u8]) -> usize { + v.as_ptr().align_offset(USIZE_BYTES) + } + + // Below, we safely fall back to a slower codepath if the offset is `usize::MAX`, + // so the end-to-end behavior is the same at compiletime and runtime. + const_eval_select((v,), compiletime, runtime) + }; while index < len { let old_offset = index; @@ -209,11 +223,11 @@ pub(super) const fn run_utf8_validation(v: &[u8]) -> Result<(), Utf8Error> { // Ascii case, try to skip forward quickly. // When the pointer is aligned, read 2 words of data per iteration // until we find a word containing a non-ascii byte. - if align != usize::MAX && align.wrapping_sub(index) % usize_bytes == 0 { + if align != usize::MAX && align.wrapping_sub(index) % USIZE_BYTES == 0 { let ptr = v.as_ptr(); while index < blocks_end { // SAFETY: since `align - index` and `ascii_block_size` are - // multiples of `usize_bytes`, `block = ptr.add(index)` is + // multiples of `USIZE_BYTES`, `block = ptr.add(index)` is // always aligned with a `usize` so it's safe to dereference // both `block` and `block.add(1)`. unsafe { diff --git a/library/core/src/ub_checks.rs b/library/core/src/ub_checks.rs index 91566439ade..dd1454f401e 100644 --- a/library/core/src/ub_checks.rs +++ b/library/core/src/ub_checks.rs @@ -120,7 +120,19 @@ pub(crate) const fn check_language_ub() -> bool { #[inline] #[rustc_const_unstable(feature = "const_ub_checks", issue = "none")] pub(crate) const fn is_aligned_and_not_null(ptr: *const (), align: usize, is_zst: bool) -> bool { - ptr.is_aligned_to(align) && (is_zst || !ptr.is_null()) + #[inline] + fn runtime(ptr: *const (), align: usize, is_zst: bool) -> bool { + ptr.is_aligned_to(align) && (is_zst || !ptr.is_null()) + } + + #[inline] + #[rustc_const_unstable(feature = "const_ub_checks", issue = "none")] + const fn comptime(ptr: *const (), _align: usize, is_zst: bool) -> bool { + is_zst || !ptr.is_null() + } + + // This is just for safety checks so we can const_eval_select. + const_eval_select((ptr, align, is_zst), comptime, runtime) } #[inline] diff --git a/library/core/tests/lib.rs b/library/core/tests/lib.rs index 7346d9f2319..5f14baf96ae 100644 --- a/library/core/tests/lib.rs +++ b/library/core/tests/lib.rs @@ -16,7 +16,6 @@ #![feature(cell_update)] #![feature(clone_to_uninit)] #![feature(const_align_of_val_raw)] -#![feature(const_align_offset)] #![feature(const_black_box)] #![feature(const_eval_select)] #![feature(const_hash)] @@ -24,7 +23,6 @@ #![feature(const_nonnull_new)] #![feature(const_option_ext)] #![feature(const_pin_2)] -#![feature(const_pointer_is_aligned)] #![feature(const_three_way_compare)] #![feature(const_trait_impl)] #![feature(core_intrinsics)] diff --git a/library/core/tests/ptr.rs b/library/core/tests/ptr.rs index 78d1b137e63..91f8c977d08 100644 --- a/library/core/tests/ptr.rs +++ b/library/core/tests/ptr.rs @@ -359,22 +359,6 @@ fn align_offset_zst() { } } -#[test] -fn align_offset_zst_const() { - const { - // For pointers of stride = 0, the pointer is already aligned or it cannot be aligned at - // all, because no amount of elements will align the pointer. - let mut p = 1; - while p < 1024 { - assert!(ptr::without_provenance::<()>(p).align_offset(p) == 0); - if p != 1 { - assert!(ptr::without_provenance::<()>(p + 1).align_offset(p) == !0); - } - p = (p + 1).next_power_of_two(); - } - } -} - #[test] fn align_offset_stride_one() { // For pointers of stride = 1, the pointer can always be aligned. The offset is equal to @@ -396,25 +380,6 @@ fn align_offset_stride_one() { } } -#[test] -fn align_offset_stride_one_const() { - const { - // For pointers of stride = 1, the pointer can always be aligned. The offset is equal to - // number of bytes. - let mut align = 1; - while align < 1024 { - let mut ptr = 1; - while ptr < 2 * align { - let expected = ptr % align; - let offset = if expected == 0 { 0 } else { align - expected }; - assert!(ptr::without_provenance::(ptr).align_offset(align) == offset); - ptr += 1; - } - align = (align + 1).next_power_of_two(); - } - } -} - #[test] fn align_offset_various_strides() { unsafe fn test_stride(ptr: *const T, align: usize) -> bool { @@ -495,192 +460,6 @@ fn align_offset_various_strides() { assert!(!x); } -#[test] -fn align_offset_various_strides_const() { - const unsafe fn test_stride(ptr: *const T, numptr: usize, align: usize) { - let mut expected = usize::MAX; - // Naive but definitely correct way to find the *first* aligned element of stride::. - let mut el = 0; - while el < align { - if (numptr + el * ::std::mem::size_of::()) % align == 0 { - expected = el; - break; - } - el += 1; - } - let got = ptr.align_offset(align); - assert!(got == expected); - } - - const { - // For pointers of stride != 1, we verify the algorithm against the naivest possible - // implementation - let mut align = 1; - let limit = 32; - while align < limit { - let mut ptr = 1; - while ptr < 4 * align { - unsafe { - #[repr(packed)] - struct A3(#[allow(dead_code)] u16, #[allow(dead_code)] u8); - test_stride::(ptr::without_provenance::(ptr), ptr, align); - - struct A4(#[allow(dead_code)] u32); - test_stride::(ptr::without_provenance::(ptr), ptr, align); - - #[repr(packed)] - struct A5(#[allow(dead_code)] u32, #[allow(dead_code)] u8); - test_stride::(ptr::without_provenance::(ptr), ptr, align); - - #[repr(packed)] - struct A6(#[allow(dead_code)] u32, #[allow(dead_code)] u16); - test_stride::(ptr::without_provenance::(ptr), ptr, align); - - #[repr(packed)] - struct A7( - #[allow(dead_code)] u32, - #[allow(dead_code)] u16, - #[allow(dead_code)] u8, - ); - test_stride::(ptr::without_provenance::(ptr), ptr, align); - - #[repr(packed)] - struct A8(#[allow(dead_code)] u32, #[allow(dead_code)] u32); - test_stride::(ptr::without_provenance::(ptr), ptr, align); - - #[repr(packed)] - struct A9( - #[allow(dead_code)] u32, - #[allow(dead_code)] u32, - #[allow(dead_code)] u8, - ); - test_stride::(ptr::without_provenance::(ptr), ptr, align); - - #[repr(packed)] - struct A10( - #[allow(dead_code)] u32, - #[allow(dead_code)] u32, - #[allow(dead_code)] u16, - ); - test_stride::(ptr::without_provenance::(ptr), ptr, align); - - test_stride::(ptr::without_provenance::(ptr), ptr, align); - test_stride::(ptr::without_provenance::(ptr), ptr, align); - } - ptr += 1; - } - align = (align + 1).next_power_of_two(); - } - } -} - -#[test] -fn align_offset_with_provenance_const() { - const { - // On some platforms (e.g. msp430-none-elf), the alignment of `i32` is less than 4. - #[repr(align(4))] - struct AlignedI32(i32); - - let data = AlignedI32(42); - - // `stride % align == 0` (usual case) - - let ptr: *const i32 = &data.0; - assert!(ptr.align_offset(1) == 0); - assert!(ptr.align_offset(2) == 0); - assert!(ptr.align_offset(4) == 0); - assert!(ptr.align_offset(8) == usize::MAX); - assert!(ptr.wrapping_byte_add(1).align_offset(1) == 0); - assert!(ptr.wrapping_byte_add(1).align_offset(2) == usize::MAX); - assert!(ptr.wrapping_byte_add(2).align_offset(1) == 0); - assert!(ptr.wrapping_byte_add(2).align_offset(2) == 0); - assert!(ptr.wrapping_byte_add(2).align_offset(4) == usize::MAX); - assert!(ptr.wrapping_byte_add(3).align_offset(1) == 0); - assert!(ptr.wrapping_byte_add(3).align_offset(2) == usize::MAX); - - assert!(ptr.wrapping_add(42).align_offset(4) == 0); - assert!(ptr.wrapping_add(42).align_offset(8) == usize::MAX); - - let ptr1: *const i8 = ptr.cast(); - assert!(ptr1.align_offset(1) == 0); - assert!(ptr1.align_offset(2) == 0); - assert!(ptr1.align_offset(4) == 0); - assert!(ptr1.align_offset(8) == usize::MAX); - assert!(ptr1.wrapping_byte_add(1).align_offset(1) == 0); - assert!(ptr1.wrapping_byte_add(1).align_offset(2) == 1); - assert!(ptr1.wrapping_byte_add(1).align_offset(4) == 3); - assert!(ptr1.wrapping_byte_add(1).align_offset(8) == usize::MAX); - assert!(ptr1.wrapping_byte_add(2).align_offset(1) == 0); - assert!(ptr1.wrapping_byte_add(2).align_offset(2) == 0); - assert!(ptr1.wrapping_byte_add(2).align_offset(4) == 2); - assert!(ptr1.wrapping_byte_add(2).align_offset(8) == usize::MAX); - assert!(ptr1.wrapping_byte_add(3).align_offset(1) == 0); - assert!(ptr1.wrapping_byte_add(3).align_offset(2) == 1); - assert!(ptr1.wrapping_byte_add(3).align_offset(4) == 1); - assert!(ptr1.wrapping_byte_add(3).align_offset(8) == usize::MAX); - - let ptr2: *const i16 = ptr.cast(); - assert!(ptr2.align_offset(1) == 0); - assert!(ptr2.align_offset(2) == 0); - assert!(ptr2.align_offset(4) == 0); - assert!(ptr2.align_offset(8) == usize::MAX); - assert!(ptr2.wrapping_byte_add(1).align_offset(1) == 0); - assert!(ptr2.wrapping_byte_add(1).align_offset(2) == usize::MAX); - assert!(ptr2.wrapping_byte_add(2).align_offset(1) == 0); - assert!(ptr2.wrapping_byte_add(2).align_offset(2) == 0); - assert!(ptr2.wrapping_byte_add(2).align_offset(4) == 1); - assert!(ptr2.wrapping_byte_add(2).align_offset(8) == usize::MAX); - assert!(ptr2.wrapping_byte_add(3).align_offset(1) == 0); - assert!(ptr2.wrapping_byte_add(3).align_offset(2) == usize::MAX); - - let ptr3: *const i64 = ptr.cast(); - assert!(ptr3.align_offset(1) == 0); - assert!(ptr3.align_offset(2) == 0); - assert!(ptr3.align_offset(4) == 0); - assert!(ptr3.align_offset(8) == usize::MAX); - assert!(ptr3.wrapping_byte_add(1).align_offset(1) == 0); - assert!(ptr3.wrapping_byte_add(1).align_offset(2) == usize::MAX); - - // `stride % align != 0` (edge case) - - let ptr4: *const [u8; 3] = ptr.cast(); - assert!(ptr4.align_offset(1) == 0); - assert!(ptr4.align_offset(2) == 0); - assert!(ptr4.align_offset(4) == 0); - assert!(ptr4.align_offset(8) == usize::MAX); - assert!(ptr4.wrapping_byte_add(1).align_offset(1) == 0); - assert!(ptr4.wrapping_byte_add(1).align_offset(2) == 1); - assert!(ptr4.wrapping_byte_add(1).align_offset(4) == 1); - assert!(ptr4.wrapping_byte_add(1).align_offset(8) == usize::MAX); - assert!(ptr4.wrapping_byte_add(2).align_offset(1) == 0); - assert!(ptr4.wrapping_byte_add(2).align_offset(2) == 0); - assert!(ptr4.wrapping_byte_add(2).align_offset(4) == 2); - assert!(ptr4.wrapping_byte_add(2).align_offset(8) == usize::MAX); - assert!(ptr4.wrapping_byte_add(3).align_offset(1) == 0); - assert!(ptr4.wrapping_byte_add(3).align_offset(2) == 1); - assert!(ptr4.wrapping_byte_add(3).align_offset(4) == 3); - assert!(ptr4.wrapping_byte_add(3).align_offset(8) == usize::MAX); - - let ptr5: *const [u8; 5] = ptr.cast(); - assert!(ptr5.align_offset(1) == 0); - assert!(ptr5.align_offset(2) == 0); - assert!(ptr5.align_offset(4) == 0); - assert!(ptr5.align_offset(8) == usize::MAX); - assert!(ptr5.wrapping_byte_add(1).align_offset(1) == 0); - assert!(ptr5.wrapping_byte_add(1).align_offset(2) == 1); - assert!(ptr5.wrapping_byte_add(1).align_offset(4) == 3); - assert!(ptr5.wrapping_byte_add(1).align_offset(8) == usize::MAX); - assert!(ptr5.wrapping_byte_add(2).align_offset(1) == 0); - assert!(ptr5.wrapping_byte_add(2).align_offset(2) == 0); - assert!(ptr5.wrapping_byte_add(2).align_offset(4) == 2); - assert!(ptr5.wrapping_byte_add(2).align_offset(8) == usize::MAX); - assert!(ptr5.wrapping_byte_add(3).align_offset(1) == 0); - assert!(ptr5.wrapping_byte_add(3).align_offset(2) == 1); - assert!(ptr5.wrapping_byte_add(3).align_offset(4) == 1); - assert!(ptr5.wrapping_byte_add(3).align_offset(8) == usize::MAX); - } -} - #[test] fn align_offset_issue_103361() { #[cfg(target_pointer_width = "64")] @@ -693,23 +472,6 @@ fn align_offset_issue_103361() { let _ = ptr::without_provenance::(SIZE).align_offset(SIZE); } -#[test] -fn align_offset_issue_103361_const() { - #[cfg(target_pointer_width = "64")] - const SIZE: usize = 1 << 47; - #[cfg(target_pointer_width = "32")] - const SIZE: usize = 1 << 30; - #[cfg(target_pointer_width = "16")] - const SIZE: usize = 1 << 13; - struct HugeSize(#[allow(dead_code)] [u8; SIZE - 1]); - - const { - assert!(ptr::without_provenance::(SIZE - 1).align_offset(SIZE) == SIZE - 1); - assert!(ptr::without_provenance::(SIZE).align_offset(SIZE) == 0); - assert!(ptr::without_provenance::(SIZE + 1).align_offset(SIZE) == 1); - } -} - #[test] fn is_aligned() { let data = 42; @@ -726,25 +488,6 @@ fn is_aligned() { assert_ne!(ptr.is_aligned_to(8), ptr.wrapping_add(1).is_aligned_to(8)); } -#[test] -fn is_aligned_const() { - const { - let data = 42; - let ptr: *const i32 = &data; - assert!(ptr.is_aligned()); - assert!(ptr.is_aligned_to(1)); - assert!(ptr.is_aligned_to(2)); - assert!(ptr.is_aligned_to(4)); - assert!(ptr.wrapping_byte_add(2).is_aligned_to(1)); - assert!(ptr.wrapping_byte_add(2).is_aligned_to(2)); - assert!(!ptr.wrapping_byte_add(2).is_aligned_to(4)); - - // At comptime neither `ptr` nor `ptr+1` is aligned to 8. - assert!(!ptr.is_aligned_to(8)); - assert!(!ptr.wrapping_add(1).is_aligned_to(8)); - } -} - #[test] fn offset_from() { let mut a = [0; 5];