Enable GVN for AggregateKind::RawPtr & UnOp::PtrMetadata

This commit is contained in:
Scott McMurray 2024-05-12 00:22:42 -07:00
parent 003a902792
commit 021ccf6c4e
14 changed files with 519 additions and 26 deletions

View File

@ -83,8 +83,8 @@
//! that contain `AllocId`s.
use rustc_const_eval::const_eval::DummyMachine;
use rustc_const_eval::interpret::{intern_const_alloc_for_constprop, MemoryKind};
use rustc_const_eval::interpret::{ImmTy, InterpCx, OpTy, Projectable, Scalar};
use rustc_const_eval::interpret::{intern_const_alloc_for_constprop, MemPlaceMeta, MemoryKind};
use rustc_const_eval::interpret::{ImmTy, Immediate, InterpCx, OpTy, Projectable, Scalar};
use rustc_data_structures::fx::FxIndexSet;
use rustc_data_structures::graph::dominators::Dominators;
use rustc_hir::def::DefKind;
@ -99,7 +99,7 @@ use rustc_middle::ty::layout::LayoutOf;
use rustc_middle::ty::{self, Ty, TyCtxt};
use rustc_span::def_id::DefId;
use rustc_span::DUMMY_SP;
use rustc_target::abi::{self, Abi, Size, VariantIdx, FIRST_VARIANT};
use rustc_target::abi::{self, Abi, FieldIdx, Size, VariantIdx, FIRST_VARIANT};
use smallvec::SmallVec;
use std::borrow::Cow;
@ -177,6 +177,12 @@ enum AggregateTy<'tcx> {
Array,
Tuple,
Def(DefId, ty::GenericArgsRef<'tcx>),
RawPtr {
/// Needed for cast propagation.
data_pointer_ty: Ty<'tcx>,
/// The data pointer can be anything thin, so doesn't determine the output.
output_pointer_ty: Ty<'tcx>,
},
}
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
@ -385,11 +391,22 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
AggregateTy::Def(def_id, args) => {
self.tcx.type_of(def_id).instantiate(self.tcx, args)
}
AggregateTy::RawPtr { output_pointer_ty, .. } => output_pointer_ty,
};
let variant = if ty.is_enum() { Some(variant) } else { None };
let ty = self.ecx.layout_of(ty).ok()?;
if ty.is_zst() {
ImmTy::uninit(ty).into()
} else if matches!(kind, AggregateTy::RawPtr { .. }) {
// Pointers don't have fields, so don't `project_field` them.
let data = self.ecx.read_pointer(fields[0]).ok()?;
let meta = if fields[1].layout.is_zst() {
MemPlaceMeta::None
} else {
MemPlaceMeta::Meta(self.ecx.read_scalar(fields[1]).ok()?)
};
let ptr_imm = Immediate::new_pointer_with_meta(data, meta, &self.ecx);
ImmTy::from_immediate(ptr_imm, ty).into()
} else if matches!(ty.abi, Abi::Scalar(..) | Abi::ScalarPair(..)) {
let dest = self.ecx.allocate(ty, MemoryKind::Stack).ok()?;
let variant_dest = if let Some(variant) = variant {
@ -862,10 +879,10 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
rvalue: &mut Rvalue<'tcx>,
location: Location,
) -> Option<VnIndex> {
let Rvalue::Aggregate(box ref kind, ref mut fields) = *rvalue else { bug!() };
let Rvalue::Aggregate(box ref kind, ref mut field_ops) = *rvalue else { bug!() };
let tcx = self.tcx;
if fields.is_empty() {
if field_ops.is_empty() {
let is_zst = match *kind {
AggregateKind::Array(..)
| AggregateKind::Tuple
@ -884,13 +901,13 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
}
}
let (ty, variant_index) = match *kind {
let (mut ty, variant_index) = match *kind {
AggregateKind::Array(..) => {
assert!(!fields.is_empty());
assert!(!field_ops.is_empty());
(AggregateTy::Array, FIRST_VARIANT)
}
AggregateKind::Tuple => {
assert!(!fields.is_empty());
assert!(!field_ops.is_empty());
(AggregateTy::Tuple, FIRST_VARIANT)
}
AggregateKind::Closure(did, args)
@ -901,15 +918,49 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
}
// Do not track unions.
AggregateKind::Adt(_, _, _, _, Some(_)) => return None,
// FIXME: Do the extra work to GVN `from_raw_parts`
AggregateKind::RawPtr(..) => return None,
AggregateKind::RawPtr(pointee_ty, mtbl) => {
assert_eq!(field_ops.len(), 2);
let data_pointer_ty = field_ops[FieldIdx::ZERO].ty(self.local_decls, self.tcx);
let output_pointer_ty = Ty::new_ptr(self.tcx, pointee_ty, mtbl);
(AggregateTy::RawPtr { data_pointer_ty, output_pointer_ty }, FIRST_VARIANT)
}
};
let fields: Option<Vec<_>> = fields
let fields: Option<Vec<_>> = field_ops
.iter_mut()
.map(|op| self.simplify_operand(op, location).or_else(|| self.new_opaque()))
.collect();
let fields = fields?;
let mut fields = fields?;
if let AggregateTy::RawPtr { data_pointer_ty, output_pointer_ty } = &mut ty {
let mut was_updated = false;
// Any thin pointer of matching mutability is fine as the data pointer.
while let Value::Cast {
kind: CastKind::PtrToPtr,
value: cast_value,
from: cast_from,
to: _,
} = self.get(fields[0])
&& let ty::RawPtr(from_pointee_ty, from_mtbl) = cast_from.kind()
&& let ty::RawPtr(_, output_mtbl) = output_pointer_ty.kind()
&& from_mtbl == output_mtbl
&& from_pointee_ty.is_sized(self.tcx, self.param_env)
{
fields[0] = *cast_value;
*data_pointer_ty = *cast_from;
was_updated = true;
}
if was_updated {
if let Some(const_) = self.try_as_constant(fields[0]) {
field_ops[FieldIdx::ZERO] = Operand::Constant(Box::new(const_));
} else if let Some(local) = self.try_as_local(fields[0], location) {
field_ops[FieldIdx::ZERO] = Operand::Copy(Place::from(local));
self.reused_locals.insert(local);
}
}
}
if let AggregateTy::Array = ty
&& fields.len() > 4
@ -941,6 +992,9 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
(UnOp::Not, Value::BinaryOp(BinOp::Ne, lhs, rhs)) => {
Value::BinaryOp(BinOp::Eq, *lhs, *rhs)
}
(UnOp::PtrMetadata, Value::Aggregate(AggregateTy::RawPtr { .. }, _, fields)) => {
return Some(fields[1]);
}
_ => return None,
};
@ -1092,6 +1146,23 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
return self.new_opaque();
}
let mut was_updated = false;
// If that cast just casts away the metadata again,
if let PtrToPtr = kind
&& let Value::Aggregate(AggregateTy::RawPtr { data_pointer_ty, .. }, _, fields) =
self.get(value)
&& let ty::RawPtr(to_pointee, _) = to.kind()
&& to_pointee.is_sized(self.tcx, self.param_env)
{
from = *data_pointer_ty;
value = fields[0];
was_updated = true;
if *data_pointer_ty == to {
return Some(fields[0]);
}
}
if let PtrToPtr | PointerCoercion(MutToConstPointer) = kind
&& let Value::Cast { kind: inner_kind, value: inner_value, from: inner_from, to: _ } =
*self.get(value)
@ -1100,9 +1171,13 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
from = inner_from;
value = inner_value;
*kind = PtrToPtr;
was_updated = true;
if inner_from == to {
return Some(inner_value);
}
}
if was_updated {
if let Some(const_) = self.try_as_constant(value) {
*operand = Operand::Constant(Box::new(const_));
} else if let Some(local) = self.try_as_local(value, location) {

View File

@ -0,0 +1,60 @@
- // MIR for `casts_before_aggregate_raw_ptr` before GVN
+ // MIR for `casts_before_aggregate_raw_ptr` after GVN
fn casts_before_aggregate_raw_ptr(_1: *const u32) -> *const [u8] {
debug x => _1;
let mut _0: *const [u8];
let _2: *const [u8; 4];
let mut _3: *const u32;
let mut _5: *const [u8; 4];
let mut _7: *const u8;
let mut _8: *const ();
scope 1 {
debug x => _2;
let _4: *const u8;
scope 2 {
debug x => _4;
let _6: *const ();
scope 3 {
debug x => _6;
}
}
}
bb0: {
- StorageLive(_2);
+ nop;
StorageLive(_3);
_3 = _1;
- _2 = move _3 as *const [u8; 4] (PtrToPtr);
+ _2 = _1 as *const [u8; 4] (PtrToPtr);
StorageDead(_3);
- StorageLive(_4);
+ nop;
StorageLive(_5);
_5 = _2;
- _4 = move _5 as *const u8 (PtrToPtr);
+ _4 = _1 as *const u8 (PtrToPtr);
StorageDead(_5);
- StorageLive(_6);
+ nop;
StorageLive(_7);
_7 = _4;
- _6 = move _7 as *const () (PtrToPtr);
+ _6 = _1 as *const () (PtrToPtr);
StorageDead(_7);
StorageLive(_8);
_8 = _6;
- _0 = *const [u8] from (move _8, const 4_usize);
+ _0 = *const [u8] from (_1, const 4_usize);
StorageDead(_8);
- StorageDead(_6);
- StorageDead(_4);
- StorageDead(_2);
+ nop;
+ nop;
+ nop;
return;
}
}

View File

@ -0,0 +1,60 @@
- // MIR for `casts_before_aggregate_raw_ptr` before GVN
+ // MIR for `casts_before_aggregate_raw_ptr` after GVN
fn casts_before_aggregate_raw_ptr(_1: *const u32) -> *const [u8] {
debug x => _1;
let mut _0: *const [u8];
let _2: *const [u8; 4];
let mut _3: *const u32;
let mut _5: *const [u8; 4];
let mut _7: *const u8;
let mut _8: *const ();
scope 1 {
debug x => _2;
let _4: *const u8;
scope 2 {
debug x => _4;
let _6: *const ();
scope 3 {
debug x => _6;
}
}
}
bb0: {
- StorageLive(_2);
+ nop;
StorageLive(_3);
_3 = _1;
- _2 = move _3 as *const [u8; 4] (PtrToPtr);
+ _2 = _1 as *const [u8; 4] (PtrToPtr);
StorageDead(_3);
- StorageLive(_4);
+ nop;
StorageLive(_5);
_5 = _2;
- _4 = move _5 as *const u8 (PtrToPtr);
+ _4 = _1 as *const u8 (PtrToPtr);
StorageDead(_5);
- StorageLive(_6);
+ nop;
StorageLive(_7);
_7 = _4;
- _6 = move _7 as *const () (PtrToPtr);
+ _6 = _1 as *const () (PtrToPtr);
StorageDead(_7);
StorageLive(_8);
_8 = _6;
- _0 = *const [u8] from (move _8, const 4_usize);
+ _0 = *const [u8] from (_1, const 4_usize);
StorageDead(_8);
- StorageDead(_6);
- StorageDead(_4);
- StorageDead(_2);
+ nop;
+ nop;
+ nop;
return;
}
}

View File

@ -0,0 +1,32 @@
- // MIR for `meta_of_ref_to_slice` before GVN
+ // MIR for `meta_of_ref_to_slice` after GVN
fn meta_of_ref_to_slice(_1: *const i32) -> usize {
debug x => _1;
let mut _0: usize;
let _2: *const [i32];
let mut _3: *const i32;
let mut _4: *const [i32];
scope 1 {
debug ptr => _2;
}
bb0: {
- StorageLive(_2);
+ nop;
StorageLive(_3);
_3 = _1;
- _2 = *const [i32] from (move _3, const 1_usize);
+ _2 = *const [i32] from (_1, const 1_usize);
StorageDead(_3);
StorageLive(_4);
_4 = _2;
- _0 = PtrMetadata(move _4);
+ _0 = const 1_usize;
StorageDead(_4);
- StorageDead(_2);
+ nop;
return;
}
}

View File

@ -0,0 +1,32 @@
- // MIR for `meta_of_ref_to_slice` before GVN
+ // MIR for `meta_of_ref_to_slice` after GVN
fn meta_of_ref_to_slice(_1: *const i32) -> usize {
debug x => _1;
let mut _0: usize;
let _2: *const [i32];
let mut _3: *const i32;
let mut _4: *const [i32];
scope 1 {
debug ptr => _2;
}
bb0: {
- StorageLive(_2);
+ nop;
StorageLive(_3);
_3 = _1;
- _2 = *const [i32] from (move _3, const 1_usize);
+ _2 = *const [i32] from (_1, const 1_usize);
StorageDead(_3);
StorageLive(_4);
_4 = _2;
- _0 = PtrMetadata(move _4);
+ _0 = const 1_usize;
StorageDead(_4);
- StorageDead(_2);
+ nop;
return;
}
}

View File

@ -783,6 +783,39 @@ fn non_freeze<T: Copy>(x: T) {
}
}
// Check that we can const-prop into `from_raw_parts`
fn slice_const_length(x: &[i32]) -> *const [i32] {
// CHECK-LABEL: fn slice_const_length(
// CHECK: _0 = *const [i32] from ({{_[0-9]+}}, const 123_usize);
let ptr = x.as_ptr();
let len = 123;
std::intrinsics::aggregate_raw_ptr(ptr, len)
}
fn meta_of_ref_to_slice(x: *const i32) -> usize {
// CHECK-LABEL: fn meta_of_ref_to_slice
// CHECK: _0 = const 1_usize
let ptr: *const [i32] = std::intrinsics::aggregate_raw_ptr(x, 1);
std::intrinsics::ptr_metadata(ptr)
}
fn slice_from_raw_parts_as_ptr(x: *const u16, n: usize) -> (*const u16, *const f32) {
// CHECK-LABEL: fn slice_from_raw_parts_as_ptr
// CHECK: _8 = _1 as *const f32 (PtrToPtr);
// CHECK: _0 = (_1, move _8);
let ptr: *const [u16] = std::intrinsics::aggregate_raw_ptr(x, n);
(ptr as *const u16, ptr as *const f32)
}
fn casts_before_aggregate_raw_ptr(x: *const u32) -> *const [u8] {
// CHECK-LABEL: fn casts_before_aggregate_raw_ptr
// CHECK: _0 = *const [u8] from (_1, const 4_usize);
let x = x as *const [u8; 4];
let x = x as *const u8;
let x = x as *const ();
std::intrinsics::aggregate_raw_ptr(x, 4)
}
fn main() {
subexpression_elimination(2, 4, 5);
wrap_unwrap(5);
@ -807,6 +840,9 @@ fn main() {
wide_ptr_integer();
borrowed(5);
non_freeze(5);
slice_const_length(&[1]);
meta_of_ref_to_slice(&42);
slice_from_raw_parts_as_ptr(&123, 456);
}
#[inline(never)]
@ -840,3 +876,7 @@ fn identity<T>(x: T) -> T {
// EMIT_MIR gvn.wide_ptr_integer.GVN.diff
// EMIT_MIR gvn.borrowed.GVN.diff
// EMIT_MIR gvn.non_freeze.GVN.diff
// EMIT_MIR gvn.slice_const_length.GVN.diff
// EMIT_MIR gvn.meta_of_ref_to_slice.GVN.diff
// EMIT_MIR gvn.slice_from_raw_parts_as_ptr.GVN.diff
// EMIT_MIR gvn.casts_before_aggregate_raw_ptr.GVN.diff

View File

@ -0,0 +1,48 @@
- // MIR for `slice_const_length` before GVN
+ // MIR for `slice_const_length` after GVN
fn slice_const_length(_1: &[i32]) -> *const [i32] {
debug x => _1;
let mut _0: *const [i32];
let _2: *const i32;
let mut _3: &[i32];
let mut _5: *const i32;
let mut _6: usize;
scope 1 {
debug ptr => _2;
let _4: usize;
scope 2 {
debug len => _4;
}
}
bb0: {
- StorageLive(_2);
+ nop;
StorageLive(_3);
_3 = &(*_1);
_2 = core::slice::<impl [i32]>::as_ptr(move _3) -> [return: bb1, unwind unreachable];
}
bb1: {
StorageDead(_3);
- StorageLive(_4);
+ nop;
_4 = const 123_usize;
StorageLive(_5);
_5 = _2;
StorageLive(_6);
- _6 = _4;
- _0 = *const [i32] from (move _5, move _6);
+ _6 = const 123_usize;
+ _0 = *const [i32] from (_2, const 123_usize);
StorageDead(_6);
StorageDead(_5);
- StorageDead(_4);
- StorageDead(_2);
+ nop;
+ nop;
return;
}
}

View File

@ -0,0 +1,48 @@
- // MIR for `slice_const_length` before GVN
+ // MIR for `slice_const_length` after GVN
fn slice_const_length(_1: &[i32]) -> *const [i32] {
debug x => _1;
let mut _0: *const [i32];
let _2: *const i32;
let mut _3: &[i32];
let mut _5: *const i32;
let mut _6: usize;
scope 1 {
debug ptr => _2;
let _4: usize;
scope 2 {
debug len => _4;
}
}
bb0: {
- StorageLive(_2);
+ nop;
StorageLive(_3);
_3 = &(*_1);
_2 = core::slice::<impl [i32]>::as_ptr(move _3) -> [return: bb1, unwind continue];
}
bb1: {
StorageDead(_3);
- StorageLive(_4);
+ nop;
_4 = const 123_usize;
StorageLive(_5);
_5 = _2;
StorageLive(_6);
- _6 = _4;
- _0 = *const [i32] from (move _5, move _6);
+ _6 = const 123_usize;
+ _0 = *const [i32] from (_2, const 123_usize);
StorageDead(_6);
StorageDead(_5);
- StorageDead(_4);
- StorageDead(_2);
+ nop;
+ nop;
return;
}
}

View File

@ -0,0 +1,51 @@
- // MIR for `slice_from_raw_parts_as_ptr` before GVN
+ // MIR for `slice_from_raw_parts_as_ptr` after GVN
fn slice_from_raw_parts_as_ptr(_1: *const u16, _2: usize) -> (*const u16, *const f32) {
debug x => _1;
debug n => _2;
let mut _0: (*const u16, *const f32);
let _3: *const [u16];
let mut _4: *const u16;
let mut _5: usize;
let mut _6: *const u16;
let mut _7: *const [u16];
let mut _8: *const f32;
let mut _9: *const [u16];
scope 1 {
debug ptr => _3;
}
bb0: {
- StorageLive(_3);
+ nop;
StorageLive(_4);
_4 = _1;
StorageLive(_5);
_5 = _2;
- _3 = *const [u16] from (move _4, move _5);
+ _3 = *const [u16] from (_1, _2);
StorageDead(_5);
StorageDead(_4);
StorageLive(_6);
StorageLive(_7);
_7 = _3;
- _6 = move _7 as *const u16 (PtrToPtr);
+ _6 = _1;
StorageDead(_7);
StorageLive(_8);
StorageLive(_9);
_9 = _3;
- _8 = move _9 as *const f32 (PtrToPtr);
+ _8 = _1 as *const f32 (PtrToPtr);
StorageDead(_9);
- _0 = (move _6, move _8);
+ _0 = (_1, move _8);
StorageDead(_8);
StorageDead(_6);
- StorageDead(_3);
+ nop;
return;
}
}

View File

@ -0,0 +1,51 @@
- // MIR for `slice_from_raw_parts_as_ptr` before GVN
+ // MIR for `slice_from_raw_parts_as_ptr` after GVN
fn slice_from_raw_parts_as_ptr(_1: *const u16, _2: usize) -> (*const u16, *const f32) {
debug x => _1;
debug n => _2;
let mut _0: (*const u16, *const f32);
let _3: *const [u16];
let mut _4: *const u16;
let mut _5: usize;
let mut _6: *const u16;
let mut _7: *const [u16];
let mut _8: *const f32;
let mut _9: *const [u16];
scope 1 {
debug ptr => _3;
}
bb0: {
- StorageLive(_3);
+ nop;
StorageLive(_4);
_4 = _1;
StorageLive(_5);
_5 = _2;
- _3 = *const [u16] from (move _4, move _5);
+ _3 = *const [u16] from (_1, _2);
StorageDead(_5);
StorageDead(_4);
StorageLive(_6);
StorageLive(_7);
_7 = _3;
- _6 = move _7 as *const u16 (PtrToPtr);
+ _6 = _1;
StorageDead(_7);
StorageLive(_8);
StorageLive(_9);
_9 = _3;
- _8 = move _9 as *const f32 (PtrToPtr);
+ _8 = _1 as *const f32 (PtrToPtr);
StorageDead(_9);
- _0 = (move _6, move _8);
+ _0 = (_1, move _8);
StorageDead(_8);
StorageDead(_6);
- StorageDead(_3);
+ nop;
return;
}
}

View File

@ -12,8 +12,7 @@ fn demo_byte_add_fat(_1: *const [u32], _2: usize) -> *const [u32] {
scope 3 (inlined std::ptr::const_ptr::<impl *const u8>::add) {
}
scope 4 (inlined std::ptr::const_ptr::<impl *const u8>::with_metadata_of::<[u32]>) {
let mut _5: *const ();
let mut _6: usize;
let mut _5: usize;
scope 5 (inlined std::ptr::metadata::<[u32]>) {
}
scope 6 (inlined std::ptr::from_raw_parts::<[u32], ()>) {
@ -28,11 +27,8 @@ fn demo_byte_add_fat(_1: *const [u32], _2: usize) -> *const [u32] {
_4 = Offset(_3, _2);
StorageDead(_3);
StorageLive(_5);
_5 = _4 as *const () (PtrToPtr);
StorageLive(_6);
_6 = PtrMetadata(_1);
_0 = *const [u32] from (_5, _6);
StorageDead(_6);
_5 = PtrMetadata(_1);
_0 = *const [u32] from (_4, _5);
StorageDead(_5);
StorageDead(_4);
return;

View File

@ -12,8 +12,7 @@ fn demo_byte_add_fat(_1: *const [u32], _2: usize) -> *const [u32] {
scope 3 (inlined std::ptr::const_ptr::<impl *const u8>::add) {
}
scope 4 (inlined std::ptr::const_ptr::<impl *const u8>::with_metadata_of::<[u32]>) {
let mut _5: *const ();
let mut _6: usize;
let mut _5: usize;
scope 5 (inlined std::ptr::metadata::<[u32]>) {
}
scope 6 (inlined std::ptr::from_raw_parts::<[u32], ()>) {
@ -28,11 +27,8 @@ fn demo_byte_add_fat(_1: *const [u32], _2: usize) -> *const [u32] {
_4 = Offset(_3, _2);
StorageDead(_3);
StorageLive(_5);
_5 = _4 as *const () (PtrToPtr);
StorageLive(_6);
_6 = PtrMetadata(_1);
_0 = *const [u32] from (_5, _6);
StorageDead(_6);
_5 = PtrMetadata(_1);
_0 = *const [u32] from (_4, _5);
StorageDead(_5);
StorageDead(_4);
return;

View File

@ -20,11 +20,13 @@ fn demo_byte_add_thin(_1: *const u32, _2: usize) -> *const u32 {
}
bb0: {
StorageLive(_4);
StorageLive(_3);
_3 = _1 as *const u8 (PtrToPtr);
_4 = Offset(_3, _2);
StorageDead(_3);
_0 = _4 as *const u32 (PtrToPtr);
StorageDead(_4);
return;
}
}

View File

@ -20,11 +20,13 @@ fn demo_byte_add_thin(_1: *const u32, _2: usize) -> *const u32 {
}
bb0: {
StorageLive(_4);
StorageLive(_3);
_3 = _1 as *const u8 (PtrToPtr);
_4 = Offset(_3, _2);
StorageDead(_3);
_0 = _4 as *const u32 (PtrToPtr);
StorageDead(_4);
return;
}
}