Omit non-needs_drop drop_in_place in vtables

This replaces the drop_in_place reference with null in vtables. On
librustc_driver.so, this drops about ~17k dynamic relocations from the
output, since many vtables can now be placed in read-only memory, rather
than having a relocated pointer included.

This makes a tradeoff by adding a null check at vtable call sites.
That's hard to avoid without changing the vtable format (e.g., to use a
pc-relative relocation instead of an absolute address, and avoid the
dynamic relocation that way). But it seems likely that the check is
cheap at runtime.
This commit is contained in:
Mark Rousskov 2024-03-17 17:42:37 -04:00
parent b0f8618938
commit 4c002fce9d
7 changed files with 203 additions and 116 deletions

View File

@ -593,6 +593,7 @@ pub(crate) fn codegen_drop<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>, fx: &mut FunctionCx<'_, '_, 'tcx>,
source_info: mir::SourceInfo, source_info: mir::SourceInfo,
drop_place: CPlace<'tcx>, drop_place: CPlace<'tcx>,
target: BasicBlock,
) { ) {
let ty = drop_place.layout().ty; let ty = drop_place.layout().ty;
let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx); let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx);
@ -620,6 +621,12 @@ pub(crate) fn codegen_drop<'tcx>(
let ptr = ptr.get_addr(fx); let ptr = ptr.get_addr(fx);
let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable); let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable);
let is_null = fx.bcx.ins().icmp_imm(IntCC::Equal, drop_fn, 0);
let target_block = fx.get_block(target);
let continued = fx.bcx.create_block();
fx.bcx.ins().brif(is_null, target_block, &[], continued, &[]);
fx.bcx.switch_to_block(continued);
// FIXME(eddyb) perhaps move some of this logic into // FIXME(eddyb) perhaps move some of this logic into
// `Instance::resolve_drop_in_place`? // `Instance::resolve_drop_in_place`?
let virtual_drop = Instance { let virtual_drop = Instance {
@ -659,6 +666,12 @@ pub(crate) fn codegen_drop<'tcx>(
let (data, vtable) = drop_place.to_cvalue(fx).dyn_star_force_data_on_stack(fx); let (data, vtable) = drop_place.to_cvalue(fx).dyn_star_force_data_on_stack(fx);
let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable); let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable);
let is_null = fx.bcx.ins().icmp_imm(IntCC::Equal, drop_fn, 0);
let target_block = fx.get_block(target);
let continued = fx.bcx.create_block();
fx.bcx.ins().brif(is_null, target_block, &[], continued, &[]);
fx.bcx.switch_to_block(continued);
let virtual_drop = Instance { let virtual_drop = Instance {
def: ty::InstanceDef::Virtual(drop_instance.def_id(), 0), def: ty::InstanceDef::Virtual(drop_instance.def_id(), 0),
args: drop_instance.args, args: drop_instance.args,
@ -697,4 +710,7 @@ pub(crate) fn codegen_drop<'tcx>(
} }
} }
} }
let target_block = fx.get_block(target);
fx.bcx.ins().jump(target_block, &[]);
} }

View File

@ -548,10 +548,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
} }
TerminatorKind::Drop { place, target, unwind: _, replace: _ } => { TerminatorKind::Drop { place, target, unwind: _, replace: _ } => {
let drop_place = codegen_place(fx, *place); let drop_place = codegen_place(fx, *place);
crate::abi::codegen_drop(fx, source_info, drop_place); crate::abi::codegen_drop(fx, source_info, drop_place, *target);
let target_block = fx.get_block(*target);
fx.bcx.ins().jump(target_block, &[]);
} }
}; };
} }

View File

@ -15,12 +15,13 @@ impl<'a, 'tcx> VirtualIndex {
VirtualIndex(index as u64) VirtualIndex(index as u64)
} }
pub fn get_fn<Bx: BuilderMethods<'a, 'tcx>>( fn get_fn_inner<Bx: BuilderMethods<'a, 'tcx>>(
self, self,
bx: &mut Bx, bx: &mut Bx,
llvtable: Bx::Value, llvtable: Bx::Value,
ty: Ty<'tcx>, ty: Ty<'tcx>,
fn_abi: &FnAbi<'tcx, Ty<'tcx>>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
nonnull: bool,
) -> Bx::Value { ) -> Bx::Value {
// Load the function pointer from the object. // Load the function pointer from the object.
debug!("get_fn({llvtable:?}, {ty:?}, {self:?})"); debug!("get_fn({llvtable:?}, {ty:?}, {self:?})");
@ -41,13 +42,35 @@ impl<'a, 'tcx> VirtualIndex {
} else { } else {
let gep = bx.inbounds_ptradd(llvtable, bx.const_usize(vtable_byte_offset)); let gep = bx.inbounds_ptradd(llvtable, bx.const_usize(vtable_byte_offset));
let ptr = bx.load(llty, gep, ptr_align); let ptr = bx.load(llty, gep, ptr_align);
bx.nonnull_metadata(ptr);
// VTable loads are invariant. // VTable loads are invariant.
bx.set_invariant_load(ptr); bx.set_invariant_load(ptr);
if nonnull {
bx.nonnull_metadata(ptr);
}
ptr ptr
} }
} }
pub fn get_optional_fn<Bx: BuilderMethods<'a, 'tcx>>(
self,
bx: &mut Bx,
llvtable: Bx::Value,
ty: Ty<'tcx>,
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
) -> Bx::Value {
self.get_fn_inner(bx, llvtable, ty, fn_abi, false)
}
pub fn get_fn<Bx: BuilderMethods<'a, 'tcx>>(
self,
bx: &mut Bx,
llvtable: Bx::Value,
ty: Ty<'tcx>,
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
) -> Bx::Value {
self.get_fn_inner(bx, llvtable, ty, fn_abi, true)
}
pub fn get_usize<Bx: BuilderMethods<'a, 'tcx>>( pub fn get_usize<Bx: BuilderMethods<'a, 'tcx>>(
self, self,
bx: &mut Bx, bx: &mut Bx,

View File

@ -500,6 +500,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
&mut self, &mut self,
helper: TerminatorCodegenHelper<'tcx>, helper: TerminatorCodegenHelper<'tcx>,
bx: &mut Bx, bx: &mut Bx,
source_info: &mir::SourceInfo,
location: mir::Place<'tcx>, location: mir::Place<'tcx>,
target: mir::BasicBlock, target: mir::BasicBlock,
unwind: mir::UnwindAction, unwind: mir::UnwindAction,
@ -523,90 +524,106 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
args1 = [place.val.llval]; args1 = [place.val.llval];
&args1[..] &args1[..]
}; };
let (drop_fn, fn_abi, drop_instance) = let (maybe_null, drop_fn, fn_abi, drop_instance) = match ty.kind() {
match ty.kind() { // FIXME(eddyb) perhaps move some of this logic into
// FIXME(eddyb) perhaps move some of this logic into // `Instance::resolve_drop_in_place`?
// `Instance::resolve_drop_in_place`? ty::Dynamic(_, _, ty::Dyn) => {
ty::Dynamic(_, _, ty::Dyn) => { // IN THIS ARM, WE HAVE:
// IN THIS ARM, WE HAVE: // ty = *mut (dyn Trait)
// ty = *mut (dyn Trait) // which is: exists<T> ( *mut T, Vtable<T: Trait> )
// which is: exists<T> ( *mut T, Vtable<T: Trait> ) // args[0] args[1]
// args[0] args[1] //
// // args = ( Data, Vtable )
// args = ( Data, Vtable ) // |
// | // v
// v // /-------\
// /-------\ // | ... |
// | ... | // \-------/
// \-------/ //
// let virtual_drop = Instance {
let virtual_drop = Instance { def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), // idx 0: the drop function
def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), // idx 0: the drop function args: drop_fn.args,
args: drop_fn.args, };
}; debug!("ty = {:?}", ty);
debug!("ty = {:?}", ty); debug!("drop_fn = {:?}", drop_fn);
debug!("drop_fn = {:?}", drop_fn); debug!("args = {:?}", args);
debug!("args = {:?}", args); let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty()); let vtable = args[1];
let vtable = args[1]; // Truncate vtable off of args list
// Truncate vtable off of args list args = &args[..1];
args = &args[..1]; (
( true,
meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE) meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
.get_fn(bx, vtable, ty, fn_abi), .get_optional_fn(bx, vtable, ty, fn_abi),
fn_abi, fn_abi,
virtual_drop, virtual_drop,
) )
} }
ty::Dynamic(_, _, ty::DynStar) => { ty::Dynamic(_, _, ty::DynStar) => {
// IN THIS ARM, WE HAVE: // IN THIS ARM, WE HAVE:
// ty = *mut (dyn* Trait) // ty = *mut (dyn* Trait)
// which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>) // which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>)
// //
// args = [ * ] // args = [ * ]
// | // |
// v // v
// ( Data, Vtable ) // ( Data, Vtable )
// | // |
// v // v
// /-------\ // /-------\
// | ... | // | ... |
// \-------/ // \-------/
// //
// //
// WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING // WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING
// //
// data = &(*args[0]).0 // gives a pointer to Data above (really the same pointer) // data = &(*args[0]).0 // gives a pointer to Data above (really the same pointer)
// vtable = (*args[0]).1 // loads the vtable out // vtable = (*args[0]).1 // loads the vtable out
// (data, vtable) // an equivalent Rust `*mut dyn Trait` // (data, vtable) // an equivalent Rust `*mut dyn Trait`
// //
// SO THEN WE CAN USE THE ABOVE CODE. // SO THEN WE CAN USE THE ABOVE CODE.
let virtual_drop = Instance { let virtual_drop = Instance {
def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), // idx 0: the drop function def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), // idx 0: the drop function
args: drop_fn.args, args: drop_fn.args,
}; };
debug!("ty = {:?}", ty); debug!("ty = {:?}", ty);
debug!("drop_fn = {:?}", drop_fn); debug!("drop_fn = {:?}", drop_fn);
debug!("args = {:?}", args); debug!("args = {:?}", args);
let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty()); let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
let meta_ptr = place.project_field(bx, 1); let meta_ptr = place.project_field(bx, 1);
let meta = bx.load_operand(meta_ptr); let meta = bx.load_operand(meta_ptr);
// Truncate vtable off of args list // Truncate vtable off of args list
args = &args[..1]; args = &args[..1];
debug!("args' = {:?}", args); debug!("args' = {:?}", args);
( (
meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE) true,
.get_fn(bx, meta.immediate(), ty, fn_abi), meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
fn_abi, .get_optional_fn(bx, meta.immediate(), ty, fn_abi),
virtual_drop, fn_abi,
) virtual_drop,
} )
_ => ( }
bx.get_fn_addr(drop_fn), _ => (
bx.fn_abi_of_instance(drop_fn, ty::List::empty()), false,
drop_fn, bx.get_fn_addr(drop_fn),
), bx.fn_abi_of_instance(drop_fn, ty::List::empty()),
}; drop_fn,
),
};
// We generate a null check for the drop_fn. This saves a bunch of relocations being
// generated for no-op drops.
if maybe_null {
let is_not_null = bx.append_sibling_block("is_not_null");
let llty = bx.fn_ptr_backend_type(fn_abi);
let null = bx.const_null(llty);
let non_null =
bx.icmp(base::bin_op_to_icmp_predicate(mir::BinOp::Ne, false), drop_fn, null);
bx.cond_br(non_null, is_not_null, helper.llbb_with_cleanup(self, target));
bx.switch_to_block(is_not_null);
self.set_debug_loc(bx, *source_info);
}
helper.do_call( helper.do_call(
self, self,
bx, bx,
@ -617,7 +634,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
unwind, unwind,
&[], &[],
Some(drop_instance), Some(drop_instance),
mergeable_succ, !maybe_null && mergeable_succ,
) )
} }
@ -1346,9 +1363,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
MergingSucc::False MergingSucc::False
} }
mir::TerminatorKind::Drop { place, target, unwind, replace: _ } => { mir::TerminatorKind::Drop { place, target, unwind, replace: _ } => self
self.codegen_drop_terminator(helper, bx, place, target, unwind, mergeable_succ()) .codegen_drop_terminator(
} helper,
bx,
&terminator.source_info,
place,
target,
unwind,
mergeable_succ(),
),
mir::TerminatorKind::Assert { ref cond, expected, ref msg, target, unwind } => self mir::TerminatorKind::Assert { ref cond, expected, ref msg, target, unwind } => self
.codegen_assert_terminator( .codegen_assert_terminator(

View File

@ -84,10 +84,14 @@ pub(super) fn vtable_allocation_provider<'tcx>(
let idx: u64 = u64::try_from(idx).unwrap(); let idx: u64 = u64::try_from(idx).unwrap();
let scalar = match entry { let scalar = match entry {
VtblEntry::MetadataDropInPlace => { VtblEntry::MetadataDropInPlace => {
let instance = ty::Instance::resolve_drop_in_place(tcx, ty); if ty.needs_drop(tcx, ty::ParamEnv::reveal_all()) {
let fn_alloc_id = tcx.reserve_and_set_fn_alloc(instance); let instance = ty::Instance::resolve_drop_in_place(tcx, ty);
let fn_ptr = Pointer::from(fn_alloc_id); let fn_alloc_id = tcx.reserve_and_set_fn_alloc(instance);
Scalar::from_pointer(fn_ptr, &tcx) let fn_ptr = Pointer::from(fn_alloc_id);
Scalar::from_pointer(fn_ptr, &tcx)
} else {
Scalar::from_maybe_pointer(Pointer::null(), &tcx)
}
} }
VtblEntry::MetadataSize => Scalar::from_uint(size, ptr_size), VtblEntry::MetadataSize => Scalar::from_uint(size, ptr_size),
VtblEntry::MetadataAlign => Scalar::from_uint(align, ptr_size), VtblEntry::MetadataAlign => Scalar::from_uint(align, ptr_size),

View File

@ -9,19 +9,25 @@
#![feature(start)] #![feature(start)]
mod mod1 { mod mod1 {
struct NeedsDrop;
impl Drop for NeedsDrop {
fn drop(&mut self) {}
}
pub trait Trait1 { pub trait Trait1 {
fn do_something(&self) {} fn do_something(&self) {}
fn do_something_else(&self) {} fn do_something_else(&self) {}
} }
impl Trait1 for u32 {} impl Trait1 for NeedsDrop {}
pub trait Trait1Gen<T> { pub trait Trait1Gen<T> {
fn do_something(&self, x: T) -> T; fn do_something(&self, x: T) -> T;
fn do_something_else(&self, x: T) -> T; fn do_something_else(&self, x: T) -> T;
} }
impl<T> Trait1Gen<T> for u32 { impl<T> Trait1Gen<T> for NeedsDrop {
fn do_something(&self, x: T) -> T { x } fn do_something(&self, x: T) -> T { x }
fn do_something_else(&self, x: T) -> T { x } fn do_something_else(&self, x: T) -> T { x }
} }
@ -30,8 +36,8 @@ mod mod1 {
fn id<T>(x: T) -> T { x } fn id<T>(x: T) -> T { x }
// These are referenced, so they produce mono-items (see start()) // These are referenced, so they produce mono-items (see start())
pub const TRAIT1_REF: &'static Trait1 = &0u32 as &Trait1; pub const TRAIT1_REF: &'static Trait1 = &NeedsDrop as &Trait1;
pub const TRAIT1_GEN_REF: &'static Trait1Gen<u8> = &0u32 as &Trait1Gen<u8>; pub const TRAIT1_GEN_REF: &'static Trait1Gen<u8> = &NeedsDrop as &Trait1Gen<u8>;
pub const ID_CHAR: fn(char) -> char = id::<char>; pub const ID_CHAR: fn(char) -> char = id::<char>;
@ -41,47 +47,48 @@ mod mod1 {
fn do_something_else(&self) {} fn do_something_else(&self) {}
} }
//~ MONO_ITEM fn <u32 as mod1::Trait2>::do_something @@ vtable_through_const-mod1.volatile[Internal] //~ MONO_ITEM fn <mod1::NeedsDrop as mod1::Trait2>::do_something @@ vtable_through_const-mod1.volatile[Internal]
//~ MONO_ITEM fn <u32 as mod1::Trait2>::do_something_else @@ vtable_through_const-mod1.volatile[Internal] //~ MONO_ITEM fn <mod1::NeedsDrop as mod1::Trait2>::do_something_else @@ vtable_through_const-mod1.volatile[Internal]
impl Trait2 for u32 {} impl Trait2 for NeedsDrop {}
pub trait Trait2Gen<T> { pub trait Trait2Gen<T> {
fn do_something(&self, x: T) -> T; fn do_something(&self, x: T) -> T;
fn do_something_else(&self, x: T) -> T; fn do_something_else(&self, x: T) -> T;
} }
impl<T> Trait2Gen<T> for u32 { impl<T> Trait2Gen<T> for NeedsDrop {
fn do_something(&self, x: T) -> T { x } fn do_something(&self, x: T) -> T { x }
fn do_something_else(&self, x: T) -> T { x } fn do_something_else(&self, x: T) -> T { x }
} }
// These are not referenced, so they do not produce mono-items // These are not referenced, so they do not produce mono-items
pub const TRAIT2_REF: &'static Trait2 = &0u32 as &Trait2; pub const TRAIT2_REF: &'static Trait2 = &NeedsDrop as &Trait2;
pub const TRAIT2_GEN_REF: &'static Trait2Gen<u8> = &0u32 as &Trait2Gen<u8>; pub const TRAIT2_GEN_REF: &'static Trait2Gen<u8> = &NeedsDrop as &Trait2Gen<u8>;
pub const ID_I64: fn(i64) -> i64 = id::<i64>; pub const ID_I64: fn(i64) -> i64 = id::<i64>;
} }
//~ MONO_ITEM fn start //~ MONO_ITEM fn start
#[start] #[start]
fn start(_: isize, _: *const *const u8) -> isize { fn start(_: isize, _: *const *const u8) -> isize {
//~ MONO_ITEM fn std::ptr::drop_in_place::<u32> - shim(None) @@ vtable_through_const[Internal] //~ MONO_ITEM fn <mod1::NeedsDrop as std::ops::Drop>::drop @@ vtable_through_const-fallback.cgu[Internal]
//~ MONO_ITEM fn std::ptr::drop_in_place::<mod1::NeedsDrop> - shim(Some(mod1::NeedsDrop)) @@ vtable_through_const-fallback.cgu[External]
// Since Trait1::do_something() is instantiated via its default implementation, // Since Trait1::do_something() is instantiated via its default implementation,
// it is considered a generic and is instantiated here only because it is // it is considered a generic and is instantiated here only because it is
// referenced in this module. // referenced in this module.
//~ MONO_ITEM fn <u32 as mod1::Trait1>::do_something_else @@ vtable_through_const-mod1.volatile[External] //~ MONO_ITEM fn <mod1::NeedsDrop as mod1::Trait1>::do_something_else @@ vtable_through_const-mod1.volatile[External]
// Although it is never used, Trait1::do_something_else() has to be // Although it is never used, Trait1::do_something_else() has to be
// instantiated locally here too, otherwise the <&u32 as &Trait1> vtable // instantiated locally here too, otherwise the <&NeedsDrop as &Trait1> vtable
// could not be fully constructed. // could not be fully constructed.
//~ MONO_ITEM fn <u32 as mod1::Trait1>::do_something @@ vtable_through_const-mod1.volatile[External] //~ MONO_ITEM fn <mod1::NeedsDrop as mod1::Trait1>::do_something @@ vtable_through_const-mod1.volatile[External]
mod1::TRAIT1_REF.do_something(); mod1::TRAIT1_REF.do_something();
// Same as above // Same as above
//~ MONO_ITEM fn <u32 as mod1::Trait1Gen<u8>>::do_something @@ vtable_through_const-mod1.volatile[External] //~ MONO_ITEM fn <mod1::NeedsDrop as mod1::Trait1Gen<u8>>::do_something @@ vtable_through_const-mod1.volatile[External]
//~ MONO_ITEM fn <u32 as mod1::Trait1Gen<u8>>::do_something_else @@ vtable_through_const-mod1.volatile[External] //~ MONO_ITEM fn <mod1::NeedsDrop as mod1::Trait1Gen<u8>>::do_something_else @@ vtable_through_const-mod1.volatile[External]
//~ MONO_ITEM fn <u32 as mod1::Trait2Gen<u8>>::do_something @@ vtable_through_const-mod1.volatile[Internal] //~ MONO_ITEM fn <mod1::NeedsDrop as mod1::Trait2Gen<u8>>::do_something @@ vtable_through_const-mod1.volatile[Internal]
//~ MONO_ITEM fn <u32 as mod1::Trait2Gen<u8>>::do_something_else @@ vtable_through_const-mod1.volatile[Internal] //~ MONO_ITEM fn <mod1::NeedsDrop as mod1::Trait2Gen<u8>>::do_something_else @@ vtable_through_const-mod1.volatile[Internal]
mod1::TRAIT1_GEN_REF.do_something(0u8); mod1::TRAIT1_GEN_REF.do_something(0u8);
//~ MONO_ITEM fn mod1::id::<char> @@ vtable_through_const-mod1.volatile[External] //~ MONO_ITEM fn mod1::id::<char> @@ vtable_through_const-mod1.volatile[External]

View File

@ -0,0 +1,16 @@
//@ compile-flags: -O
#![crate_type = "lib"]
// CHECK-LABEL: @loop_skips_vtable_load
#[no_mangle]
pub fn loop_skips_vtable_load(x: &dyn Fn()) {
// CHECK: load ptr, ptr %0{{.*}}, !invariant.load
// CHECK-NEXT: tail call void %1
// CHECK-NOT: load ptr
x();
for _ in 0..100 {
// CHECK: tail call void %1
x();
}
}