From 439dbfa1ecb1b45db0acc728094d6e8ed3dc3080 Mon Sep 17 00:00:00 2001 From: zhuyunxing Date: Fri, 19 Apr 2024 10:58:16 +0800 Subject: [PATCH] coverage. Lowering MC/DC statements to llvm-ir --- compiler/rustc_codegen_llvm/src/builder.rs | 126 +++++++++++++++++- .../src/coverageinfo/mod.rs | 70 +++++++++- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 4 + .../rustc_llvm/llvm-wrapper/RustWrapper.cpp | 27 ++++ compiler/rustc_middle/src/mir/query.rs | 4 + .../rustc_mir_transform/src/coverage/query.rs | 12 +- 6 files changed, 240 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 160f361b9b5..4c2bdb2f5ec 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -17,7 +17,7 @@ use rustc_data_structures::small_c_str::SmallCStr; use rustc_hir::def_id::DefId; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; use rustc_middle::ty::layout::{ - FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout, + FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, LayoutError, LayoutOfHelpers, TyAndLayout, }; use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; use rustc_sanitizers::{cfi, kcfi}; @@ -1702,4 +1702,128 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { }; kcfi_bundle } + + pub(crate) fn mcdc_parameters( + &mut self, + fn_name: &'ll Value, + hash: &'ll Value, + bitmap_bytes: &'ll Value, + ) -> &'ll Value { + debug!("mcdc_parameters() with args ({:?}, {:?}, {:?})", fn_name, hash, bitmap_bytes); + + assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later"); + + let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCParametersIntrinsic(self.cx().llmod) }; + let llty = self.cx.type_func( + &[self.cx.type_ptr(), self.cx.type_i64(), self.cx.type_i32()], + self.cx.type_void(), + ); + let args = &[fn_name, hash, bitmap_bytes]; + let args = self.check_call("call", llty, llfn, args); + + unsafe { + let _ = llvm::LLVMRustBuildCall( + self.llbuilder, + llty, + llfn, + args.as_ptr() as *const &llvm::Value, + args.len() as c_uint, + [].as_ptr(), + 0 as c_uint, + ); + // Create condition bitmap named `mcdc.addr`. + let mut bx = Builder::with_cx(self.cx); + bx.position_at_start(llvm::LLVMGetFirstBasicBlock(self.llfn())); + let cond_bitmap = { + let alloca = + llvm::LLVMBuildAlloca(bx.llbuilder, bx.cx.type_i32(), c"mcdc.addr".as_ptr()); + llvm::LLVMSetAlignment(alloca, 4); + alloca + }; + bx.store(self.const_i32(0), cond_bitmap, self.tcx().data_layout.i32_align.abi); + cond_bitmap + } + } + + pub(crate) fn mcdc_tvbitmap_update( + &mut self, + fn_name: &'ll Value, + hash: &'ll Value, + bitmap_bytes: &'ll Value, + bitmap_index: &'ll Value, + mcdc_temp: &'ll Value, + ) { + debug!( + "mcdc_tvbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})", + fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp + ); + assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later"); + + let llfn = + unsafe { llvm::LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(self.cx().llmod) }; + let llty = self.cx.type_func( + &[ + self.cx.type_ptr(), + self.cx.type_i64(), + self.cx.type_i32(), + self.cx.type_i32(), + self.cx.type_ptr(), + ], + self.cx.type_void(), + ); + let args = &[fn_name, hash, bitmap_bytes, bitmap_index, mcdc_temp]; + let args = self.check_call("call", llty, llfn, args); + unsafe { + let _ = llvm::LLVMRustBuildCall( + self.llbuilder, + llty, + llfn, + args.as_ptr() as *const &llvm::Value, + args.len() as c_uint, + [].as_ptr(), + 0 as c_uint, + ); + } + let i32_align = self.tcx().data_layout.i32_align.abi; + self.store(self.const_i32(0), mcdc_temp, i32_align); + } + + pub(crate) fn mcdc_condbitmap_update( + &mut self, + fn_name: &'ll Value, + hash: &'ll Value, + cond_loc: &'ll Value, + mcdc_temp: &'ll Value, + bool_value: &'ll Value, + ) { + debug!( + "mcdc_condbitmap_update() with args ({:?}, {:?}, {:?}, {:?}, {:?})", + fn_name, hash, cond_loc, mcdc_temp, bool_value + ); + assert!(llvm_util::get_version() >= (18, 0, 0), "MCDC intrinsics require LLVM 18 or later"); + let llfn = unsafe { llvm::LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(self.cx().llmod) }; + let llty = self.cx.type_func( + &[ + self.cx.type_ptr(), + self.cx.type_i64(), + self.cx.type_i32(), + self.cx.type_ptr(), + self.cx.type_i1(), + ], + self.cx.type_void(), + ); + let args = &[fn_name, hash, cond_loc, mcdc_temp, bool_value]; + self.check_call("call", llty, llfn, args); + unsafe { + let _ = llvm::LLVMRustBuildCall( + self.llbuilder, + llty, + llfn, + args.as_ptr() as *const &llvm::Value, + args.len() as c_uint, + [].as_ptr(), + 0 as c_uint, + ); + } + } } diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs index 140566e8da9..085ce15d81f 100644 --- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -13,7 +13,7 @@ use rustc_codegen_ssa::traits::{ use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_llvm::RustString; use rustc_middle::bug; -use rustc_middle::mir::coverage::CoverageKind; +use rustc_middle::mir::coverage::{CoverageKind, FunctionCoverageInfo}; use rustc_middle::ty::layout::HasTyCtxt; use rustc_middle::ty::Instance; use rustc_target::abi::Align; @@ -30,6 +30,7 @@ pub struct CrateCoverageContext<'ll, 'tcx> { pub(crate) function_coverage_map: RefCell, FunctionCoverageCollector<'tcx>>>, pub(crate) pgo_func_name_var_map: RefCell, &'ll llvm::Value>>, + pub(crate) mcdc_condition_bitmap_map: RefCell, &'ll llvm::Value>>, } impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { @@ -37,6 +38,7 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { Self { function_coverage_map: Default::default(), pgo_func_name_var_map: Default::default(), + mcdc_condition_bitmap_map: Default::default(), } } @@ -45,6 +47,12 @@ impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { ) -> FxIndexMap, FunctionCoverageCollector<'tcx>> { self.function_coverage_map.replace(FxIndexMap::default()) } + + /// LLVM use a temp value to record evaluated mcdc test vector of each decision, which is called condition bitmap. + /// This value is named `mcdc.addr` (same as clang) and is a 32-bit integer. + fn try_get_mcdc_condition_bitmap(&self, instance: &Instance<'tcx>) -> Option<&'ll llvm::Value> { + self.mcdc_condition_bitmap_map.borrow().get(instance).copied() + } } // These methods used to be part of trait `CoverageInfoMethods`, which no longer @@ -90,6 +98,10 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { return; }; + if function_coverage_info.mcdc_bitmap_bytes > 0 { + ensure_mcdc_parameters(bx, instance, function_coverage_info); + } + let Some(coverage_context) = bx.coverage_context() else { return }; let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); let func_coverage = coverage_map @@ -131,10 +143,66 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { CoverageKind::ExpressionUsed { id } => { func_coverage.mark_expression_id_seen(id); } + CoverageKind::CondBitmapUpdate { id, value, .. } => { + drop(coverage_map); + assert_ne!( + id.as_u32(), + 0, + "ConditionId of evaluated conditions should never be zero" + ); + let cond_bitmap = coverage_context + .try_get_mcdc_condition_bitmap(&instance) + .expect("mcdc cond bitmap should have been allocated for updating"); + let cond_loc = bx.const_i32(id.as_u32() as i32 - 1); + let bool_value = bx.const_bool(value); + let fn_name = bx.get_pgo_func_name_var(instance); + let hash = bx.const_u64(function_coverage_info.function_source_hash); + bx.mcdc_condbitmap_update(fn_name, hash, cond_loc, cond_bitmap, bool_value); + } + CoverageKind::TestVectorBitmapUpdate { bitmap_idx } => { + drop(coverage_map); + let cond_bitmap = coverage_context + .try_get_mcdc_condition_bitmap(&instance) + .expect("mcdc cond bitmap should have been allocated for merging into the global bitmap"); + let bitmap_bytes = bx.tcx().coverage_ids_info(instance.def).mcdc_bitmap_bytes; + assert!(bitmap_idx < bitmap_bytes, "bitmap index of the decision out of range"); + assert!( + bitmap_bytes <= function_coverage_info.mcdc_bitmap_bytes, + "bitmap length disagreement: query says {bitmap_bytes} but function info only has {}", + function_coverage_info.mcdc_bitmap_bytes + ); + + let fn_name = bx.get_pgo_func_name_var(instance); + let hash = bx.const_u64(function_coverage_info.function_source_hash); + let bitmap_bytes = bx.const_u32(bitmap_bytes); + let bitmap_index = bx.const_u32(bitmap_idx); + bx.mcdc_tvbitmap_update(fn_name, hash, bitmap_bytes, bitmap_index, cond_bitmap); + } } } } +fn ensure_mcdc_parameters<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + instance: Instance<'tcx>, + function_coverage_info: &FunctionCoverageInfo, +) { + let Some(cx) = bx.coverage_context() else { return }; + if cx.mcdc_condition_bitmap_map.borrow().contains_key(&instance) { + return; + } + + let fn_name = bx.get_pgo_func_name_var(instance); + let hash = bx.const_u64(function_coverage_info.function_source_hash); + let bitmap_bytes = bx.const_u32(function_coverage_info.mcdc_bitmap_bytes); + let cond_bitmap = bx.mcdc_parameters(fn_name, hash, bitmap_bytes); + bx.coverage_context() + .expect("already checked above") + .mcdc_condition_bitmap_map + .borrow_mut() + .insert(instance, cond_bitmap); +} + /// Calls llvm::createPGOFuncNameVar() with the given function instance's /// mangled function name. The LLVM API returns an llvm::GlobalVariable /// containing the function name, with the specific variable name and linkage diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 83158f6f1d5..a10dc61967e 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1631,6 +1631,10 @@ extern "C" { // Miscellaneous instructions pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value; + pub fn LLVMRustGetInstrProfMCDCParametersIntrinsic(M: &Module) -> &Value; + pub fn LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(M: &Module) -> &Value; + pub fn LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(M: &Module) -> &Value; + pub fn LLVMRustBuildCall<'a>( B: &Builder<'a>, Ty: &'a Type, diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 6e11fd629e4..2a9de499622 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -1546,6 +1546,33 @@ extern "C" LLVMValueRef LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) unwrap(M), llvm::Intrinsic::instrprof_increment)); } +extern "C" LLVMValueRef LLVMRustGetInstrProfMCDCParametersIntrinsic(LLVMModuleRef M) { +#if LLVM_VERSION_GE(18, 0) + return wrap(llvm::Intrinsic::getDeclaration( + unwrap(M), llvm::Intrinsic::instrprof_mcdc_parameters)); +#else + report_fatal_error("LLVM 18.0 is required for mcdc intrinsic functions"); +#endif +} + +extern "C" LLVMValueRef LLVMRustGetInstrProfMCDCTVBitmapUpdateIntrinsic(LLVMModuleRef M) { +#if LLVM_VERSION_GE(18, 0) + return wrap(llvm::Intrinsic::getDeclaration( + unwrap(M), llvm::Intrinsic::instrprof_mcdc_tvbitmap_update)); +#else + report_fatal_error("LLVM 18.0 is required for mcdc intrinsic functions"); +#endif +} + +extern "C" LLVMValueRef LLVMRustGetInstrProfMCDCCondBitmapIntrinsic(LLVMModuleRef M) { +#if LLVM_VERSION_GE(18, 0) + return wrap(llvm::Intrinsic::getDeclaration( + unwrap(M), llvm::Intrinsic::instrprof_mcdc_condbitmap_update)); +#else + report_fatal_error("LLVM 18.0 is required for mcdc intrinsic functions"); +#endif +} + extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B, LLVMValueRef Dst, unsigned DstAlign, LLVMValueRef Src, unsigned SrcAlign, diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs index 105f30f1db8..e3f58729fbd 100644 --- a/compiler/rustc_middle/src/mir/query.rs +++ b/compiler/rustc_middle/src/mir/query.rs @@ -361,4 +361,8 @@ pub struct CoverageIdsInfo { /// InstrumentCoverage MIR pass, if the highest-numbered counter increments /// were removed by MIR optimizations. pub max_counter_id: mir::coverage::CounterId, + + /// Coverage codegen for mcdc needs to know the size of the global bitmap so that it can + /// set the `bytemap-bytes` argument of the `llvm.instrprof.mcdc.tvbitmap.update` intrinsic. + pub mcdc_bitmap_bytes: u32, } diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs index 65715253647..f77ee63d02c 100644 --- a/compiler/rustc_mir_transform/src/coverage/query.rs +++ b/compiler/rustc_mir_transform/src/coverage/query.rs @@ -61,7 +61,17 @@ fn coverage_ids_info<'tcx>( .max() .unwrap_or(CounterId::ZERO); - CoverageIdsInfo { max_counter_id } + let mcdc_bitmap_bytes = mir_body + .coverage_branch_info + .as_deref() + .map(|info| { + info.mcdc_decision_spans + .iter() + .fold(0, |acc, decision| acc + (1_u32 << decision.conditions_num).div_ceil(8)) + }) + .unwrap_or_default(); + + CoverageIdsInfo { max_counter_id, mcdc_bitmap_bytes } } fn all_coverage_in_mir_body<'a, 'tcx>(