diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 3b7fefee80a..4c5b183cfe9 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -435,18 +435,22 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { scalar: abi::Scalar, backend_ty: Bx::Type, ) { - if matches!(self.cx.sess().opts.optimize, OptLevel::No) - // For now, the critical niches are all over `Int`eger values. - // Should floating-point values or pointers ever get more complex - // niches, then this code will probably want to handle them too. - || !matches!(scalar.primitive(), abi::Primitive::Int(..)) - || scalar.is_always_valid(self.cx) - { + if matches!(self.cx.sess().opts.optimize, OptLevel::No) || scalar.is_always_valid(self.cx) { return; } - let range = scalar.valid_range(self.cx); - bx.assume_integer_range(imm, backend_ty, range); + match scalar.primitive() { + abi::Primitive::Int(..) => { + let range = scalar.valid_range(self.cx); + bx.assume_integer_range(imm, backend_ty, range); + } + abi::Primitive::Pointer(abi::AddressSpace::DATA) + if !scalar.valid_range(self.cx).contains(0) => + { + bx.assume_nonnull(imm); + } + abi::Primitive::Pointer(..) | abi::Primitive::Float(..) => {} + } } pub(crate) fn codegen_rvalue_unsized( diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index b7dcf16fa2b..d9bb5ae773e 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -243,6 +243,19 @@ pub trait BuilderMethods<'a, 'tcx>: self.assume(cmp); } + /// Emits an `assume` that the `val` of pointer type is non-null. + /// + /// You may want to check the optimization level before bothering calling this. + fn assume_nonnull(&mut self, val: Self::Value) { + // Arguably in LLVM it'd be better to emit an assume operand bundle instead + // + // but this works fine for all backends. + + let null = self.const_null(self.type_ptr()); + let is_null = self.icmp(IntPredicate::IntNE, val, null); + self.assume(is_null); + } + fn range_metadata(&mut self, load: Self::Value, range: WrappingRange); fn nonnull_metadata(&mut self, load: Self::Value); diff --git a/library/core/src/slice/iter/macros.rs b/library/core/src/slice/iter/macros.rs index 830debe02ea..45e320e66bc 100644 --- a/library/core/src/slice/iter/macros.rs +++ b/library/core/src/slice/iter/macros.rs @@ -30,7 +30,7 @@ macro_rules! if_zst { $zst_body } else { // SAFETY: for non-ZSTs, the type invariant ensures it cannot be null - let $end = unsafe { *(&raw const $this.end_or_len).cast::>() }; + let $end = unsafe { mem::transmute::<*const T, NonNull>($this.end_or_len) }; $other_body } }}; diff --git a/tests/codegen/intrinsics/transmute-niched.rs b/tests/codegen/intrinsics/transmute-niched.rs index 88119ccb8b2..5ded0e192fa 100644 --- a/tests/codegen/intrinsics/transmute-niched.rs +++ b/tests/codegen/intrinsics/transmute-niched.rs @@ -6,6 +6,7 @@ use std::mem::transmute; use std::num::NonZero; +use std::ptr::NonNull; #[repr(u8)] pub enum SmallEnum { @@ -192,3 +193,31 @@ pub unsafe fn check_bool_to_ordering(x: bool) -> std::cmp::Ordering { transmute(x) } + +// CHECK-LABEL: @check_nonnull_to_ptr( +#[no_mangle] +pub unsafe fn check_nonnull_to_ptr(x: NonNull) -> *const u8 { + // CHECK-NOT: icmp + // CHECK-NOT: assume + // OPT: %0 = icmp ne ptr %x, null + // OPT: call void @llvm.assume(i1 %0) + // CHECK-NOT: icmp + // CHECK-NOT: assume + // CHECK: ret ptr %x + + transmute(x) +} + +// CHECK-LABEL: @check_ptr_to_nonnull( +#[no_mangle] +pub unsafe fn check_ptr_to_nonnull(x: *const u8) -> NonNull { + // CHECK-NOT: icmp + // CHECK-NOT: assume + // OPT: %0 = icmp ne ptr %x, null + // OPT: call void @llvm.assume(i1 %0) + // CHECK-NOT: icmp + // CHECK-NOT: assume + // CHECK: ret ptr %x + + transmute(x) +} diff --git a/tests/codegen/intrinsics/transmute.rs b/tests/codegen/intrinsics/transmute.rs index 541333a52b0..5b40a6a12c0 100644 --- a/tests/codegen/intrinsics/transmute.rs +++ b/tests/codegen/intrinsics/transmute.rs @@ -370,9 +370,11 @@ pub unsafe fn check_issue_110005(x: (usize, bool)) -> Option> { #[no_mangle] pub unsafe fn check_pair_to_dst_ref<'a>(x: (usize, usize)) -> &'a [u8] { // CHECK: %_0.0 = getelementptr i8, ptr null, i64 %x.0 - // CHECK: %0 = insertvalue { ptr, i64 } poison, ptr %_0.0, 0 - // CHECK: %1 = insertvalue { ptr, i64 } %0, i64 %x.1, 1 - // CHECK: ret { ptr, i64 } %1 + // CHECK: %0 = icmp ne ptr %_0.0, null + // CHECK: call void @llvm.assume(i1 %0) + // CHECK: %1 = insertvalue { ptr, i64 } poison, ptr %_0.0, 0 + // CHECK: %2 = insertvalue { ptr, i64 } %1, i64 %x.1, 1 + // CHECK: ret { ptr, i64 } %2 transmute(x) } diff --git a/tests/codegen/slice-iter-len-eq-zero.rs b/tests/codegen/slice-iter-len-eq-zero.rs index c85861d47f8..6998d98e498 100644 --- a/tests/codegen/slice-iter-len-eq-zero.rs +++ b/tests/codegen/slice-iter-len-eq-zero.rs @@ -1,4 +1,5 @@ //@ compile-flags: -Copt-level=3 +//@ needs-deterministic-layouts (opposite scalar pair orders breaks it) #![crate_type = "lib"] type Demo = [u8; 3]; @@ -7,7 +8,40 @@ type Demo = [u8; 3]; #[no_mangle] pub fn slice_iter_len_eq_zero(y: std::slice::Iter<'_, Demo>) -> bool { // CHECK-NOT: sub - // CHECK: %[[RET:.+]] = icmp eq ptr {{%1|%0}}, {{%1|%0}} + // CHECK: %[[RET:.+]] = icmp eq ptr {{%y.0, %y.1|%y.1, %y.0}} + // CHECK: ret i1 %[[RET]] + y.len() == 0 +} + +// CHECK-LABEL: @slice_iter_len_eq_zero_ref +#[no_mangle] +pub fn slice_iter_len_eq_zero_ref(y: &mut std::slice::Iter<'_, Demo>) -> bool { + // CHECK-NOT: sub + // CHECK: %[[A:.+]] = load ptr + // CHECK-SAME: !nonnull + // CHECK: %[[B:.+]] = load ptr + // CHECK-SAME: !nonnull + // CHECK: %[[RET:.+]] = icmp eq ptr %[[A]], %[[B]] + // CHECK: ret i1 %[[RET]] + y.len() == 0 +} + +struct MyZST; + +// CHECK-LABEL: @slice_zst_iter_len_eq_zero +#[no_mangle] +pub fn slice_zst_iter_len_eq_zero(y: std::slice::Iter<'_, MyZST>) -> bool { + // CHECK: %[[RET:.+]] = icmp eq ptr %y.1, null + // CHECK: ret i1 %[[RET]] + y.len() == 0 +} + +// CHECK-LABEL: @slice_zst_iter_len_eq_zero_ref +#[no_mangle] +pub fn slice_zst_iter_len_eq_zero_ref(y: &mut std::slice::Iter<'_, MyZST>) -> bool { + // CHECK: %[[LEN:.+]] = load ptr + // CHECK-NOT: !nonnull + // CHECK: %[[RET:.+]] = icmp eq ptr %[[LEN]], null // CHECK: ret i1 %[[RET]] y.len() == 0 } diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-abort.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-abort.mir index 22be48c47b2..38d00cfbabd 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-abort.mir @@ -4,22 +4,21 @@ fn slice_iter_generic_is_empty(_1: &std::slice::Iter<'_, T>) -> bool { debug it => _1; let mut _0: bool; scope 1 (inlined as ExactSizeIterator>::is_empty) { - let mut _2: *const *const T; - let mut _3: *const std::ptr::NonNull; - let mut _8: *const T; + let mut _2: *const T; + let mut _7: *const T; scope 2 { - let _4: std::ptr::NonNull; - let _9: usize; + let _3: std::ptr::NonNull; + let _8: usize; scope 3 { } scope 4 { - scope 8 (inlined as PartialEq>::eq) { - let mut _5: std::ptr::NonNull; + scope 7 (inlined as PartialEq>::eq) { + let mut _4: std::ptr::NonNull; + let mut _5: *mut T; let mut _6: *mut T; - let mut _7: *mut T; - scope 9 (inlined NonNull::::as_ptr) { + scope 8 (inlined NonNull::::as_ptr) { } - scope 10 (inlined NonNull::::as_ptr) { + scope 9 (inlined NonNull::::as_ptr) { } } } @@ -27,50 +26,45 @@ fn slice_iter_generic_is_empty(_1: &std::slice::Iter<'_, T>) -> bool { scope 6 (inlined std::ptr::const_ptr::::cast::<()>) { } } - scope 7 (inlined std::ptr::const_ptr::::cast::>) { - } } } bb0: { - StorageLive(_9); StorageLive(_8); - StorageLive(_4); + StorageLive(_7); + StorageLive(_3); switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2]; } bb1: { - StorageLive(_3); StorageLive(_2); - _2 = &raw const ((*_1).1: *const T); - _3 = copy _2 as *const std::ptr::NonNull (PtrToPtr); + _2 = copy ((*_1).1: *const T); + _3 = move _2 as std::ptr::NonNull (Transmute); StorageDead(_2); - _4 = copy (*_3); - StorageDead(_3); - StorageLive(_6); StorageLive(_5); - _5 = copy ((*_1).0: std::ptr::NonNull); - _6 = copy _5 as *mut T (Transmute); - StorageDead(_5); - StorageLive(_7); - _7 = copy _4 as *mut T (Transmute); - _0 = Eq(move _6, move _7); - StorageDead(_7); + StorageLive(_4); + _4 = copy ((*_1).0: std::ptr::NonNull); + _5 = copy _4 as *mut T (Transmute); + StorageDead(_4); + StorageLive(_6); + _6 = copy _3 as *mut T (Transmute); + _0 = Eq(move _5, move _6); StorageDead(_6); + StorageDead(_5); goto -> bb3; } bb2: { - _8 = copy ((*_1).1: *const T); - _9 = copy _8 as usize (Transmute); - _0 = Eq(copy _9, const 0_usize); + _7 = copy ((*_1).1: *const T); + _8 = copy _7 as usize (Transmute); + _0 = Eq(copy _8, const 0_usize); goto -> bb3; } bb3: { - StorageDead(_4); + StorageDead(_3); + StorageDead(_7); StorageDead(_8); - StorageDead(_9); return; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-unwind.mir index 22be48c47b2..38d00cfbabd 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_generic_is_empty.PreCodegen.after.panic-unwind.mir @@ -4,22 +4,21 @@ fn slice_iter_generic_is_empty(_1: &std::slice::Iter<'_, T>) -> bool { debug it => _1; let mut _0: bool; scope 1 (inlined as ExactSizeIterator>::is_empty) { - let mut _2: *const *const T; - let mut _3: *const std::ptr::NonNull; - let mut _8: *const T; + let mut _2: *const T; + let mut _7: *const T; scope 2 { - let _4: std::ptr::NonNull; - let _9: usize; + let _3: std::ptr::NonNull; + let _8: usize; scope 3 { } scope 4 { - scope 8 (inlined as PartialEq>::eq) { - let mut _5: std::ptr::NonNull; + scope 7 (inlined as PartialEq>::eq) { + let mut _4: std::ptr::NonNull; + let mut _5: *mut T; let mut _6: *mut T; - let mut _7: *mut T; - scope 9 (inlined NonNull::::as_ptr) { + scope 8 (inlined NonNull::::as_ptr) { } - scope 10 (inlined NonNull::::as_ptr) { + scope 9 (inlined NonNull::::as_ptr) { } } } @@ -27,50 +26,45 @@ fn slice_iter_generic_is_empty(_1: &std::slice::Iter<'_, T>) -> bool { scope 6 (inlined std::ptr::const_ptr::::cast::<()>) { } } - scope 7 (inlined std::ptr::const_ptr::::cast::>) { - } } } bb0: { - StorageLive(_9); StorageLive(_8); - StorageLive(_4); + StorageLive(_7); + StorageLive(_3); switchInt(const ::IS_ZST) -> [0: bb1, otherwise: bb2]; } bb1: { - StorageLive(_3); StorageLive(_2); - _2 = &raw const ((*_1).1: *const T); - _3 = copy _2 as *const std::ptr::NonNull (PtrToPtr); + _2 = copy ((*_1).1: *const T); + _3 = move _2 as std::ptr::NonNull (Transmute); StorageDead(_2); - _4 = copy (*_3); - StorageDead(_3); - StorageLive(_6); StorageLive(_5); - _5 = copy ((*_1).0: std::ptr::NonNull); - _6 = copy _5 as *mut T (Transmute); - StorageDead(_5); - StorageLive(_7); - _7 = copy _4 as *mut T (Transmute); - _0 = Eq(move _6, move _7); - StorageDead(_7); + StorageLive(_4); + _4 = copy ((*_1).0: std::ptr::NonNull); + _5 = copy _4 as *mut T (Transmute); + StorageDead(_4); + StorageLive(_6); + _6 = copy _3 as *mut T (Transmute); + _0 = Eq(move _5, move _6); StorageDead(_6); + StorageDead(_5); goto -> bb3; } bb2: { - _8 = copy ((*_1).1: *const T); - _9 = copy _8 as usize (Transmute); - _0 = Eq(copy _9, const 0_usize); + _7 = copy ((*_1).1: *const T); + _8 = copy _7 as usize (Transmute); + _0 = Eq(copy _8, const 0_usize); goto -> bb3; } bb3: { - StorageDead(_4); + StorageDead(_3); + StorageDead(_7); StorageDead(_8); - StorageDead(_9); return; } }