From e96e6e2c89265613e136369ccf314c408e1eb002 Mon Sep 17 00:00:00 2001 From: flip1995 Date: Thu, 21 Apr 2022 14:02:54 +0100 Subject: [PATCH] Add metadata generation for vtables when using VFE This adds the typeid and `vcall_visibility` metadata to vtables when the -Cvirtual-function-elimination flag is set. The typeid is generated in the same way as for the `llvm.type.checked.load` intrinsic from the trait_ref. The offset that is added to the typeid is always 0. This is because LLVM assumes that vtables are constructed according to the definition in the Itanium ABI. This includes an "address point" of the vtable. In C++ this is the offset in the vtable where information for RTTI is placed. Since there is no RTTI information in Rust's vtables, this "address point" is always 0. This "address point" in combination with the offset passed to the `llvm.type.checked.load` intrinsic determines the final function that should be loaded from the vtable in the `WholeProgramDevirtualization` pass in LLVM. That's why the `llvm.type.checked.load` intrinsics are generated with the typeid of the trait, rather than with that of the function that is called. This matches what `clang` does for C++. The vcall_visibility metadata depends on three factors: 1. LTO level: Currently this is always fat LTO, because LLVM only supports this optimization with fat LTO. 2. Visibility of the trait: If the trait is publicly visible, VFE can only act on its vtables after linking. 3. Number of CGUs: if there is more than one CGU, also vtables with restricted visibility could be seen outside of the CGU, so VFE can only act on them after linking. To reflect this, there are three visibility levels: Public, LinkageUnit, and TranslationUnit. --- Cargo.lock | 1 + compiler/rustc_codegen_llvm/Cargo.toml | 1 + .../src/debuginfo/metadata.rs | 99 +++++++++++++++++-- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 7 ++ .../rustc_llvm/llvm-wrapper/RustWrapper.cpp | 5 + 5 files changed, 106 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5716ee2fc14..f586e9facc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3695,6 +3695,7 @@ dependencies = [ "rustc_serialize", "rustc_session", "rustc_span", + "rustc_symbol_mangling", "rustc_target", "smallvec", "tracing", diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml index 67183ff5887..9bded25c09e 100644 --- a/compiler/rustc_codegen_llvm/Cargo.toml +++ b/compiler/rustc_codegen_llvm/Cargo.toml @@ -19,6 +19,7 @@ rustc-demangle = "0.1.21" rustc_arena = { path = "../rustc_arena" } rustc_attr = { path = "../rustc_attr" } rustc_codegen_ssa = { path = "../rustc_codegen_ssa" } +rustc_symbol_mangling = { path = "../rustc_symbol_mangling" } rustc_data_structures = { path = "../rustc_data_structures" } rustc_errors = { path = "../rustc_errors" } rustc_fs_util = { path = "../rustc_fs_util" } diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index f5cbbc7ca91..d5f39a45670 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -30,20 +30,21 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_index::vec::{Idx, IndexVec}; use rustc_middle::bug; use rustc_middle::mir::{self, GeneratorLayout}; -use rustc_middle::ty::layout::LayoutOf; -use rustc_middle::ty::layout::TyAndLayout; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::subst::GenericArgKind; -use rustc_middle::ty::{self, AdtKind, Instance, ParamEnv, Ty, TyCtxt}; -use rustc_session::config::{self, DebugInfo}; +use rustc_middle::ty::{ + self, AdtKind, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt, Visibility, +}; +use rustc_session::config::{self, DebugInfo, Lto}; use rustc_span::symbol::Symbol; use rustc_span::FileName; -use rustc_span::FileNameDisplayPreference; -use rustc_span::{self, SourceFile}; +use rustc_span::{self, FileNameDisplayPreference, SourceFile}; +use rustc_symbol_mangling::typeid_for_trait_ref; use rustc_target::abi::{Align, Size}; use smallvec::smallvec; use tracing::debug; -use libc::{c_longlong, c_uint}; +use libc::{c_char, c_longlong, c_uint}; use std::borrow::Cow; use std::fmt::{self, Write}; use std::hash::{Hash, Hasher}; @@ -1468,6 +1469,84 @@ fn build_vtable_type_di_node<'ll, 'tcx>( .di_node } +fn vcall_visibility_metadata<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, + trait_ref: Option>, + vtable: &'ll Value, +) { + enum VCallVisibility { + Public = 0, + LinkageUnit = 1, + TranslationUnit = 2, + } + + let Some(trait_ref) = trait_ref else { return }; + + let trait_ref_self = trait_ref.with_self_ty(cx.tcx, ty); + let trait_ref_self = cx.tcx.erase_regions(trait_ref_self); + let trait_def_id = trait_ref_self.def_id(); + let trait_vis = cx.tcx.visibility(trait_def_id); + + let cgus = cx.sess().codegen_units(); + let single_cgu = cgus == 1; + + let lto = cx.sess().lto(); + + // Since LLVM requires full LTO for the virtual function elimination optimization to apply, + // only the `Lto::Fat` cases are relevant currently. + let vcall_visibility = match (lto, trait_vis, single_cgu) { + // If there is not LTO and the visibility in public, we have to assume that the vtable can + // be seen from anywhere. With multiple CGUs, the vtable is quasi-public. + (Lto::No | Lto::ThinLocal, Visibility::Public, _) + | (Lto::No, Visibility::Restricted(_) | Visibility::Invisible, false) => { + VCallVisibility::Public + } + // With LTO and a quasi-public visibility, the usages of the functions of the vtable are + // all known by the `LinkageUnit`. + // FIXME: LLVM only supports this optimization for `Lto::Fat` currently. Once it also + // supports `Lto::Thin` the `VCallVisibility` may have to be adjusted for those. + (Lto::Fat | Lto::Thin, Visibility::Public, _) + | ( + Lto::ThinLocal | Lto::Thin | Lto::Fat, + Visibility::Restricted(_) | Visibility::Invisible, + false, + ) => VCallVisibility::LinkageUnit, + // If there is only one CGU, private vtables can only be seen by that CGU/translation unit + // and therefore we know of all usages of functions in the vtable. + (_, Visibility::Restricted(_) | Visibility::Invisible, true) => { + VCallVisibility::TranslationUnit + } + }; + + let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); + + unsafe { + let typeid = llvm::LLVMMDStringInContext( + cx.llcx, + trait_ref_typeid.as_ptr() as *const c_char, + trait_ref_typeid.as_bytes().len() as c_uint, + ); + let v = [cx.const_usize(0), typeid]; + llvm::LLVMRustGlobalAddMetadata( + vtable, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + cx.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ); + let vcall_visibility = llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64)); + let vcall_visibility_metadata = llvm::LLVMMDNodeInContext2(cx.llcx, &vcall_visibility, 1); + llvm::LLVMGlobalSetMetadata( + vtable, + llvm::MetadataType::MD_vcall_visibility as c_uint, + vcall_visibility_metadata, + ); + } +} + /// Creates debug information for the given vtable, which is for the /// given type. /// @@ -1478,6 +1557,12 @@ pub fn create_vtable_di_node<'ll, 'tcx>( poly_trait_ref: Option>, vtable: &'ll Value, ) { + // FIXME(flip1995): The virtual function elimination optimization only works with full LTO in + // LLVM at the moment. + if cx.sess().opts.debugging_opts.virtual_function_elimination && cx.sess().lto() == Lto::Fat { + vcall_visibility_metadata(cx, ty, poly_trait_ref, vtable); + } + if cx.dbg_cx.is_none() { return; } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 4f7b082dac4..b831423994f 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -442,6 +442,7 @@ pub enum MetadataType { MD_nonnull = 11, MD_align = 17, MD_type = 19, + MD_vcall_visibility = 28, MD_noundef = 29, } @@ -1067,6 +1068,7 @@ extern "C" { pub fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value); pub fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Node: &'a Value); pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); + pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; // Operations on constants of any type @@ -1080,6 +1082,11 @@ extern "C" { Vals: *const &'a Value, Count: c_uint, ) -> &'a Value; + pub fn LLVMMDNodeInContext2<'a>( + C: &'a Context, + Vals: *const &'a Metadata, + Count: size_t, + ) -> &'a Metadata; pub fn LLVMAddNamedMetadataOperand<'a>(M: &'a Module, Name: *const c_char, Val: &'a Value); // Operations on scalar constants diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index a2dd0a7bcdc..a52d5340242 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -681,6 +681,11 @@ extern "C" LLVMValueRef LLVMRustMetadataAsValue(LLVMContextRef C, LLVMMetadataRe return wrap(MetadataAsValue::get(*unwrap(C), unwrap(MD))); } +extern "C" void LLVMRustGlobalAddMetadata( + LLVMValueRef Global, unsigned Kind, LLVMMetadataRef MD) { + unwrap(Global)->addMetadata(Kind, *unwrap(MD)); +} + extern "C" LLVMRustDIBuilderRef LLVMRustDIBuilderCreate(LLVMModuleRef M) { return new DIBuilder(*unwrap(M)); }