2021-03-11 23:35:13 +00:00
|
|
|
use std::borrow::Borrow;
|
2020-03-11 11:49:08 +00:00
|
|
|
use std::cell::{Cell, RefCell};
|
2024-09-23 15:45:13 +00:00
|
|
|
use std::ffi::{CStr, c_uint};
|
2019-02-17 18:58:58 +00:00
|
|
|
use std::str;
|
2017-09-20 20:07:47 +00:00
|
|
|
|
2024-11-03 02:32:52 +00:00
|
|
|
use rustc_abi::{HasDataLayout, TargetDataLayout, VariantIdx};
|
2024-11-01 16:07:18 +00:00
|
|
|
use rustc_codegen_ssa::back::versioned_llvm_target;
|
2023-05-01 19:24:21 +00:00
|
|
|
use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh};
|
2023-08-28 19:40:39 +00:00
|
|
|
use rustc_codegen_ssa::errors as ssa_errors;
|
2020-03-11 11:49:08 +00:00
|
|
|
use rustc_codegen_ssa::traits::*;
|
2024-04-03 18:09:21 +00:00
|
|
|
use rustc_data_structures::base_n::{ALPHANUMERIC_ONLY, ToBaseN};
|
2019-12-24 04:02:53 +00:00
|
|
|
use rustc_data_structures::fx::FxHashMap;
|
2018-08-07 14:04:34 +00:00
|
|
|
use rustc_data_structures::small_c_str::SmallCStr;
|
2022-03-01 00:53:25 +00:00
|
|
|
use rustc_hir::def_id::DefId;
|
2024-08-16 20:33:58 +00:00
|
|
|
use rustc_middle::middle::codegen_fn_attrs::PatchableFunctionEntry;
|
2020-03-29 14:41:09 +00:00
|
|
|
use rustc_middle::mir::mono::CodegenUnit;
|
2021-09-01 21:09:34 +00:00
|
|
|
use rustc_middle::ty::layout::{
|
|
|
|
FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, LayoutError, LayoutOfHelpers,
|
|
|
|
};
|
2020-03-29 14:41:09 +00:00
|
|
|
use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
|
2021-08-30 15:01:58 +00:00
|
|
|
use rustc_middle::{bug, span_bug};
|
2020-03-11 11:49:08 +00:00
|
|
|
use rustc_session::Session;
|
2022-01-28 17:48:59 +00:00
|
|
|
use rustc_session::config::{
|
2024-09-25 08:23:30 +00:00
|
|
|
BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, FunctionReturn, PAuthKey, PacRet,
|
2022-01-28 17:48:59 +00:00
|
|
|
};
|
2022-11-04 19:08:01 +00:00
|
|
|
use rustc_span::source_map::Spanned;
|
2024-07-01 20:32:32 +00:00
|
|
|
use rustc_span::{DUMMY_SP, Span};
|
2024-09-10 19:19:16 +00:00
|
|
|
use rustc_target::spec::{HasTargetSpec, RelocModel, SmallDataThresholdSupport, Target, TlsModel};
|
2021-08-09 12:25:33 +00:00
|
|
|
use smallvec::SmallVec;
|
2020-03-11 11:49:08 +00:00
|
|
|
|
2021-03-11 23:35:13 +00:00
|
|
|
use crate::back::write::to_llvm_code_model;
|
2022-05-28 23:10:44 +00:00
|
|
|
use crate::callee::get_fn;
|
2024-10-28 07:52:39 +00:00
|
|
|
use crate::common::AsCCharPtr;
|
2022-05-28 23:10:44 +00:00
|
|
|
use crate::debuginfo::metadata::apply_vcall_visibility_metadata;
|
2024-09-19 10:56:02 +00:00
|
|
|
use crate::llvm::{Metadata, MetadataType};
|
2024-01-17 14:26:26 +00:00
|
|
|
use crate::type_::Type;
|
2013-12-19 02:35:33 +00:00
|
|
|
use crate::value::Value;
|
|
|
|
use crate::{attributes, coverageinfo, debuginfo, llvm, llvm_util};
|
2013-06-13 02:49:01 +00:00
|
|
|
|
2024-09-18 02:45:28 +00:00
|
|
|
/// There is one `CodegenCx` per codegen unit. Each one has its own LLVM
|
|
|
|
/// `llvm::Context` so that several codegen units may be processed in parallel.
|
2018-06-27 14:57:25 +00:00
|
|
|
/// All other LLVM data structures in the `CodegenCx` are tied to that `llvm::Context`.
|
2024-07-06 12:26:42 +00:00
|
|
|
pub(crate) struct CodegenCx<'ll, 'tcx> {
|
2019-06-13 21:48:52 +00:00
|
|
|
pub tcx: TyCtxt<'tcx>,
|
2018-01-05 04:58:34 +00:00
|
|
|
pub use_dll_storage_attrs: bool,
|
|
|
|
pub tls_model: llvm::ThreadLocalMode,
|
2014-07-16 18:27:57 +00:00
|
|
|
|
2018-09-26 14:01:43 +00:00
|
|
|
pub llmod: &'ll llvm::Module,
|
|
|
|
pub llcx: &'ll llvm::Context,
|
2020-03-14 11:41:32 +00:00
|
|
|
pub codegen_unit: &'tcx CodegenUnit<'tcx>,
|
2017-07-12 15:37:58 +00:00
|
|
|
|
2016-02-23 20:04:27 +00:00
|
|
|
/// Cache instances of monomorphic and polymorphic items
|
2018-11-16 11:48:26 +00:00
|
|
|
pub instances: RefCell<FxHashMap<Instance<'tcx>, &'ll Value>>,
|
2014-04-01 11:11:23 +00:00
|
|
|
/// Cache generated vtables
|
2019-06-11 21:11:55 +00:00
|
|
|
pub vtables:
|
|
|
|
RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), &'ll Value>>,
|
2014-04-01 11:11:23 +00:00
|
|
|
/// Cache of constant strings,
|
2022-06-28 17:34:24 +00:00
|
|
|
pub const_str_cache: RefCell<FxHashMap<String, &'ll Value>>,
|
2013-06-13 02:02:33 +00:00
|
|
|
|
2015-01-29 12:03:34 +00:00
|
|
|
/// Cache of emitted const globals (value -> global)
|
2018-11-16 11:48:26 +00:00
|
|
|
pub const_globals: RefCell<FxHashMap<&'ll Value, &'ll Value>>,
|
2013-06-13 02:02:33 +00:00
|
|
|
|
2015-06-28 17:36:46 +00:00
|
|
|
/// List of globals for static variables which need to be passed to the
|
2018-05-08 13:10:16 +00:00
|
|
|
/// LLVM function ReplaceAllUsesWith (RAUW) when codegen is complete.
|
2018-07-10 10:28:39 +00:00
|
|
|
/// (We have to make sure we don't invalidate any Values referring
|
2015-06-28 17:36:46 +00:00
|
|
|
/// to constants.)
|
2018-11-16 11:48:26 +00:00
|
|
|
pub statics_to_rauw: RefCell<Vec<(&'ll Value, &'ll Value)>>,
|
2015-06-28 17:36:46 +00:00
|
|
|
|
2021-08-20 19:13:18 +00:00
|
|
|
/// Statics that will be placed in the llvm.used variable
|
|
|
|
/// See <https://llvm.org/docs/LangRef.html#the-llvm-used-global-variable> for details
|
|
|
|
pub used_statics: RefCell<Vec<&'ll Value>>,
|
|
|
|
|
2021-08-07 15:04:32 +00:00
|
|
|
/// Statics that will be placed in the llvm.compiler.used variable
|
|
|
|
/// See <https://llvm.org/docs/LangRef.html#the-llvm-compiler-used-global-variable> for details
|
2021-08-20 19:13:18 +00:00
|
|
|
pub compiler_used_statics: RefCell<Vec<&'ll Value>>,
|
2017-02-20 19:42:47 +00:00
|
|
|
|
2024-03-08 01:40:24 +00:00
|
|
|
/// Mapping of non-scalar types to llvm types.
|
|
|
|
pub type_lowering: RefCell<FxHashMap<(Ty<'tcx>, Option<VariantIdx>), &'ll Type>>,
|
2021-08-05 17:14:55 +00:00
|
|
|
|
|
|
|
/// Mapping of scalar types to llvm types.
|
2018-09-26 14:01:43 +00:00
|
|
|
pub scalar_lltypes: RefCell<FxHashMap<Ty<'tcx>, &'ll Type>>,
|
2021-08-05 17:14:55 +00:00
|
|
|
|
2018-09-26 14:01:43 +00:00
|
|
|
pub isize_ty: &'ll Type,
|
2014-05-29 05:26:56 +00:00
|
|
|
|
2024-10-24 11:23:13 +00:00
|
|
|
/// Extra codegen state needed when coverage instrumentation is enabled.
|
coverage bug fixes and optimization support
Adjusted LLVM codegen for code compiled with `-Zinstrument-coverage` to
address multiple, somewhat related issues.
Fixed a significant flaw in prior coverage solution: Every counter
generated a new counter variable, but there should have only been one
counter variable per function. This appears to have bloated .profraw
files significantly. (For a small program, it increased the size by
about 40%. I have not tested large programs, but there is anecdotal
evidence that profraw files were way too large. This is a good fix,
regardless, but hopefully it also addresses related issues.
Fixes: #82144
Invalid LLVM coverage data produced when compiled with -C opt-level=1
Existing tests now work up to at least `opt-level=3`. This required a
detailed analysis of the LLVM IR, comparisons with Clang C++ LLVM IR
when compiled with coverage, and a lot of trial and error with codegen
adjustments.
The biggest hurdle was figuring out how to continue to support coverage
results for unused functions and generics. Rust's coverage results have
three advantages over Clang's coverage results:
1. Rust's coverage map does not include any overlapping code regions,
making coverage counting unambiguous.
2. Rust generates coverage results (showing zero counts) for all unused
functions, including generics. (Clang does not generate coverage for
uninstantiated template functions.)
3. Rust's unused functions produce minimal stubbed functions in LLVM IR,
sufficient for including in the coverage results; while Clang must
generate the complete LLVM IR for each unused function, even though
it will never be called.
This PR removes the previous hack of attempting to inject coverage into
some other existing function instance, and generates dedicated instances
for each unused function. This change, and a few other adjustments
(similar to what is required for `-C link-dead-code`, but with lower
impact), makes it possible to support LLVM optimizations.
Fixes: #79651
Coverage report: "Unexecuted instantiation:..." for a generic function
from multiple crates
Fixed by removing the aforementioned hack. Some "Unexecuted
instantiation" notices are unavoidable, as explained in the
`used_crate.rs` test, but `-Zinstrument-coverage` has new options to
back off support for either unused generics, or all unused functions,
which avoids the notice, at the cost of less coverage of unused
functions.
Fixes: #82875
Invalid LLVM coverage data produced with crate brotli_decompressor
Fixed by disabling the LLVM function attribute that forces inlining, if
`-Z instrument-coverage` is enabled. This attribute is applied to
Rust functions with `#[inline(always)], and in some cases, the forced
inlining breaks coverage instrumentation and reports.
2021-03-15 23:32:45 +00:00
|
|
|
pub coverage_cx: Option<coverageinfo::CrateCoverageContext<'ll, 'tcx>>,
|
2022-03-03 10:15:25 +00:00
|
|
|
pub dbg_cx: Option<debuginfo::CodegenUnitDebugContext<'ll, 'tcx>>,
|
2014-04-09 23:56:31 +00:00
|
|
|
|
2018-11-16 11:48:26 +00:00
|
|
|
eh_personality: Cell<Option<&'ll Value>>,
|
2020-03-21 07:50:38 +00:00
|
|
|
eh_catch_typeinfo: Cell<Option<&'ll Value>>,
|
2021-08-03 22:09:57 +00:00
|
|
|
pub rust_try_fn: Cell<Option<(&'ll Type, &'ll Value)>>,
|
2014-05-19 16:30:09 +00:00
|
|
|
|
2021-08-03 22:09:57 +00:00
|
|
|
intrinsics: RefCell<FxHashMap<&'static str, (&'ll Type, &'ll Value)>>,
|
2014-07-21 23:42:34 +00:00
|
|
|
|
2016-10-21 16:41:20 +00:00
|
|
|
/// A counter that is used for generating local symbol names
|
|
|
|
local_gen_sym_counter: Cell<usize>,
|
2022-03-01 00:53:25 +00:00
|
|
|
|
|
|
|
/// `codegen_static` will sometimes create a second global variable with a
|
|
|
|
/// different type and clear the symbol name of the original global.
|
|
|
|
/// `global_asm!` needs to be able to find this new global so that it can
|
|
|
|
/// compute the correct mangled symbol name to insert into the asm.
|
|
|
|
pub renamed_statics: RefCell<FxHashMap<DefId, &'ll Value>>,
|
run optimization and codegen on worker threads
Refactor the code in `llvm::back` that invokes LLVM optimization and codegen
passes so that it can be called from worker threads. (Previously, it used
`&Session` extensively, and `Session` is not `Share`.) The new code can handle
multiple compilation units, by compiling each unit to `crate.0.o`, `crate.1.o`,
etc., and linking together all the `crate.N.o` files into a single `crate.o`
using `ld -r`. The later linking steps can then be run unchanged.
The new code preserves the behavior of `--emit`/`-o` when building a single
compilation unit. With multiple compilation units, the `--emit=asm/ir/bc`
options produce multiple files, so combinations like `--emit=ir -o foo.ll` will
not actually produce `foo.ll` (they instead produce several `foo.N.ll` files).
The new code supports `-Z lto` only when using a single compilation unit.
Compiling with multiple compilation units and `-Z lto` will produce an error.
(I can't think of any good reason to do such a thing.) Linking with `-Z lto`
against a library that was built as multiple compilation units will also fail,
because the rlib does not contain a `crate.bytecode.deflate` file. This could
be supported in the future by linking together the `crate.N.bc` files produced
when compiling the library into a single `crate.bc`, or by making the LTO code
support multiple `crate.N.bytecode.deflate` files.
2014-07-17 17:52:52 +00:00
|
|
|
}
|
|
|
|
|
2020-04-25 18:45:21 +00:00
|
|
|
fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode {
|
|
|
|
match tls_model {
|
|
|
|
TlsModel::GeneralDynamic => llvm::ThreadLocalMode::GeneralDynamic,
|
|
|
|
TlsModel::LocalDynamic => llvm::ThreadLocalMode::LocalDynamic,
|
|
|
|
TlsModel::InitialExec => llvm::ThreadLocalMode::InitialExec,
|
|
|
|
TlsModel::LocalExec => llvm::ThreadLocalMode::LocalExec,
|
2023-11-13 12:48:23 +00:00
|
|
|
TlsModel::Emulated => llvm::ThreadLocalMode::GeneralDynamic,
|
2017-10-31 18:24:04 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-07-06 12:26:42 +00:00
|
|
|
pub(crate) unsafe fn create_module<'ll>(
|
2019-06-13 21:48:52 +00:00
|
|
|
tcx: TyCtxt<'_>,
|
2018-07-17 15:26:58 +00:00
|
|
|
llcx: &'ll llvm::Context,
|
|
|
|
mod_name: &str,
|
|
|
|
) -> &'ll llvm::Module {
|
2018-10-27 12:29:06 +00:00
|
|
|
let sess = tcx.sess;
|
2018-08-07 14:04:34 +00:00
|
|
|
let mod_name = SmallCStr::new(mod_name);
|
2024-07-14 18:27:57 +00:00
|
|
|
let llmod = unsafe { llvm::LLVMModuleCreateWithNameInContext(mod_name.as_ptr(), llcx) };
|
2014-11-25 21:28:35 +00:00
|
|
|
|
2022-03-22 10:43:05 +00:00
|
|
|
let mut target_data_layout = sess.target.data_layout.to_string();
|
2022-02-02 09:01:02 +00:00
|
|
|
let llvm_version = llvm_util::get_version();
|
2019-07-07 16:41:28 +00:00
|
|
|
|
2024-05-06 16:44:43 +00:00
|
|
|
if llvm_version < (19, 0, 0) {
|
|
|
|
if sess.target.arch == "aarch64" || sess.target.arch.starts_with("arm64") {
|
|
|
|
// LLVM 19 sets -Fn32 in its data layout string for 64-bit ARM
|
|
|
|
// Earlier LLVMs leave this default, so remove it.
|
|
|
|
// See https://github.com/llvm/llvm-project/pull/90702
|
|
|
|
target_data_layout = target_data_layout.replace("-Fn32", "");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-06-12 10:39:09 +00:00
|
|
|
if llvm_version < (19, 0, 0) {
|
|
|
|
if sess.target.arch == "loongarch64" {
|
|
|
|
// LLVM 19 updates the LoongArch64 data layout.
|
|
|
|
// See https://github.com/llvm/llvm-project/pull/93814
|
|
|
|
target_data_layout = target_data_layout.replace("-n32:64", "-n64");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-10-15 20:56:20 +00:00
|
|
|
if llvm_version < (20, 0, 0) {
|
|
|
|
if sess.target.arch == "aarch64" || sess.target.arch.starts_with("arm64") {
|
|
|
|
// LLVM 20 defines three additional address spaces for alternate
|
|
|
|
// pointer kinds used in Windows.
|
|
|
|
// See https://github.com/llvm/llvm-project/pull/111879
|
|
|
|
target_data_layout =
|
|
|
|
target_data_layout.replace("-p270:32:32-p271:32:32-p272:64:64", "");
|
|
|
|
}
|
2024-10-31 20:35:24 +00:00
|
|
|
if sess.target.arch.starts_with("sparc") {
|
|
|
|
// LLVM 20 updates the sparc layout to correctly align 128 bit integers to 128 bit.
|
|
|
|
// See https://github.com/llvm/llvm-project/pull/106951
|
|
|
|
target_data_layout = target_data_layout.replace("-i128:128", "");
|
|
|
|
}
|
2024-11-07 19:59:50 +00:00
|
|
|
if sess.target.arch.starts_with("mips64") {
|
|
|
|
// LLVM 20 updates the mips64 layout to correctly align 128 bit integers to 128 bit.
|
|
|
|
// See https://github.com/llvm/llvm-project/pull/112084
|
|
|
|
target_data_layout = target_data_layout.replace("-i128:128", "");
|
|
|
|
}
|
2024-10-15 20:56:20 +00:00
|
|
|
}
|
|
|
|
|
2016-04-18 12:38:45 +00:00
|
|
|
// Ensure the data-layout values hardcoded remain the defaults.
|
2024-01-17 14:26:26 +00:00
|
|
|
{
|
2024-08-04 03:51:37 +00:00
|
|
|
let tm = crate::back::write::create_informational_target_machine(tcx.sess, false);
|
2024-07-14 18:27:57 +00:00
|
|
|
unsafe {
|
|
|
|
llvm::LLVMRustSetDataLayoutFromTargetMachine(llmod, &tm);
|
|
|
|
}
|
2016-04-18 12:38:45 +00:00
|
|
|
|
2024-07-14 18:27:57 +00:00
|
|
|
let llvm_data_layout = unsafe { llvm::LLVMGetDataLayoutStr(llmod) };
|
|
|
|
let llvm_data_layout =
|
|
|
|
str::from_utf8(unsafe { CStr::from_ptr(llvm_data_layout) }.to_bytes())
|
|
|
|
.expect("got a non-UTF8 data-layout from LLVM");
|
2016-04-18 12:38:45 +00:00
|
|
|
|
2024-01-17 14:26:26 +00:00
|
|
|
if target_data_layout != llvm_data_layout {
|
|
|
|
tcx.dcx().emit_err(crate::errors::MismatchedDataLayout {
|
|
|
|
rustc_target: sess.opts.target_triple.to_string().as_str(),
|
|
|
|
rustc_layout: target_data_layout.as_str(),
|
|
|
|
llvm_target: sess.target.llvm_target.borrow(),
|
|
|
|
llvm_layout: llvm_data_layout,
|
|
|
|
});
|
2016-04-18 12:38:45 +00:00
|
|
|
}
|
2015-07-16 22:48:16 +00:00
|
|
|
}
|
2014-11-25 21:28:35 +00:00
|
|
|
|
2019-07-07 16:41:28 +00:00
|
|
|
let data_layout = SmallCStr::new(&target_data_layout);
|
2024-07-14 18:27:57 +00:00
|
|
|
unsafe {
|
|
|
|
llvm::LLVMSetDataLayout(llmod, data_layout.as_ptr());
|
|
|
|
}
|
2016-04-18 12:38:45 +00:00
|
|
|
|
2024-11-01 16:07:18 +00:00
|
|
|
let llvm_target = SmallCStr::new(&versioned_llvm_target(sess));
|
2024-07-14 18:27:57 +00:00
|
|
|
unsafe {
|
|
|
|
llvm::LLVMRustSetNormalizedTarget(llmod, llvm_target.as_ptr());
|
|
|
|
}
|
2016-07-14 21:40:14 +00:00
|
|
|
|
2021-09-10 13:11:56 +00:00
|
|
|
let reloc_model = sess.relocation_model();
|
|
|
|
if matches!(reloc_model, RelocModel::Pic | RelocModel::Pie) {
|
2024-07-14 18:27:57 +00:00
|
|
|
unsafe {
|
|
|
|
llvm::LLVMRustSetModulePICLevel(llmod);
|
|
|
|
}
|
2020-05-21 17:53:41 +00:00
|
|
|
// PIE is potentially more effective than PIC, but can only be used in executables.
|
|
|
|
// If all our outputs are executables, then we can relax PIC to PIE.
|
2021-09-10 13:11:56 +00:00
|
|
|
if reloc_model == RelocModel::Pie
|
2023-08-08 10:28:20 +00:00
|
|
|
|| tcx.crate_types().iter().all(|ty| *ty == CrateType::Executable)
|
2021-09-10 13:11:56 +00:00
|
|
|
{
|
2024-07-14 18:27:57 +00:00
|
|
|
unsafe {
|
|
|
|
llvm::LLVMRustSetModulePIELevel(llmod);
|
|
|
|
}
|
2020-05-21 17:53:41 +00:00
|
|
|
}
|
2016-07-14 21:40:14 +00:00
|
|
|
}
|
|
|
|
|
2021-03-11 23:35:13 +00:00
|
|
|
// Linking object files with different code models is undefined behavior
|
|
|
|
// because the compiler would have to generate additional code (to span
|
|
|
|
// longer jumps) if a larger code model is used with a smaller one.
|
|
|
|
//
|
|
|
|
// See https://reviews.llvm.org/D52322 and https://reviews.llvm.org/D52323.
|
2024-07-14 18:27:57 +00:00
|
|
|
unsafe {
|
|
|
|
llvm::LLVMRustSetModuleCodeModel(llmod, to_llvm_code_model(sess.code_model()));
|
|
|
|
}
|
2021-03-11 23:35:13 +00:00
|
|
|
|
2018-09-26 16:19:55 +00:00
|
|
|
// If skipping the PLT is enabled, we need to add some module metadata
|
|
|
|
// to ensure intrinsic calls don't use it.
|
|
|
|
if !sess.needs_plt() {
|
2024-10-29 02:38:17 +00:00
|
|
|
llvm::add_module_flag_u32(llmod, llvm::ModuleFlagMergeBehavior::Warning, "RtLibUseGOT", 1);
|
2018-09-26 16:19:55 +00:00
|
|
|
}
|
|
|
|
|
2022-12-13 06:42:44 +00:00
|
|
|
// Enable canonical jump tables if CFI is enabled. (See https://reviews.llvm.org/D65629.)
|
|
|
|
if sess.is_sanitizer_cfi_canonical_jump_tables_enabled() && sess.is_sanitizer_cfi_enabled() {
|
2024-10-29 02:38:17 +00:00
|
|
|
llvm::add_module_flag_u32(
|
|
|
|
llmod,
|
|
|
|
llvm::ModuleFlagMergeBehavior::Override,
|
|
|
|
"CFI Canonical Jump Tables",
|
|
|
|
1,
|
|
|
|
);
|
2021-10-07 22:33:13 +00:00
|
|
|
}
|
|
|
|
|
2024-08-16 20:33:58 +00:00
|
|
|
// If we're normalizing integers with CFI, ensure LLVM generated functions do the same.
|
|
|
|
// See https://github.com/llvm/llvm-project/pull/104826
|
|
|
|
if sess.is_sanitizer_cfi_normalize_integers_enabled() {
|
2024-10-29 02:38:17 +00:00
|
|
|
llvm::add_module_flag_u32(
|
|
|
|
llmod,
|
|
|
|
llvm::ModuleFlagMergeBehavior::Override,
|
|
|
|
"cfi-normalize-integers",
|
|
|
|
1,
|
|
|
|
);
|
2024-08-16 20:33:58 +00:00
|
|
|
}
|
|
|
|
|
2024-09-18 03:34:49 +00:00
|
|
|
// Enable LTO unit splitting if specified or if CFI is enabled. (See
|
|
|
|
// https://reviews.llvm.org/D53891.)
|
2022-12-13 06:42:44 +00:00
|
|
|
if sess.is_split_lto_unit_enabled() || sess.is_sanitizer_cfi_enabled() {
|
2024-10-29 02:38:17 +00:00
|
|
|
llvm::add_module_flag_u32(
|
|
|
|
llmod,
|
|
|
|
llvm::ModuleFlagMergeBehavior::Override,
|
|
|
|
"EnableSplitLTOUnit",
|
|
|
|
1,
|
|
|
|
);
|
2022-12-13 06:42:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Add "kcfi" module flag if KCFI is enabled. (See https://reviews.llvm.org/D119296.)
|
2022-11-22 05:29:00 +00:00
|
|
|
if sess.is_sanitizer_kcfi_enabled() {
|
2024-10-29 02:38:17 +00:00
|
|
|
llvm::add_module_flag_u32(llmod, llvm::ModuleFlagMergeBehavior::Override, "kcfi", 1);
|
2024-08-16 20:33:58 +00:00
|
|
|
|
|
|
|
// Add "kcfi-offset" module flag with -Z patchable-function-entry (See
|
|
|
|
// https://reviews.llvm.org/D141172).
|
|
|
|
let pfe =
|
|
|
|
PatchableFunctionEntry::from_config(sess.opts.unstable_opts.patchable_function_entry);
|
|
|
|
if pfe.prefix() > 0 {
|
2024-10-29 02:38:17 +00:00
|
|
|
llvm::add_module_flag_u32(
|
|
|
|
llmod,
|
|
|
|
llvm::ModuleFlagMergeBehavior::Override,
|
|
|
|
"kcfi-offset",
|
|
|
|
pfe.prefix().into(),
|
|
|
|
);
|
2024-08-16 20:33:58 +00:00
|
|
|
}
|
2022-11-22 05:29:00 +00:00
|
|
|
}
|
|
|
|
|
2020-07-06 15:10:42 +00:00
|
|
|
// Control Flow Guard is currently only supported by the MSVC linker on Windows.
|
2020-11-08 11:27:51 +00:00
|
|
|
if sess.target.is_like_msvc {
|
2024-10-29 02:38:17 +00:00
|
|
|
match sess.opts.cg.control_flow_guard {
|
|
|
|
CFGuard::Disabled => {}
|
|
|
|
CFGuard::NoChecks => {
|
|
|
|
// Set `cfguard=1` module flag to emit metadata only.
|
|
|
|
llvm::add_module_flag_u32(
|
|
|
|
llmod,
|
|
|
|
llvm::ModuleFlagMergeBehavior::Warning,
|
|
|
|
"cfguard",
|
|
|
|
1,
|
|
|
|
);
|
|
|
|
}
|
|
|
|
CFGuard::Checks => {
|
|
|
|
// Set `cfguard=2` module flag to emit metadata and checks.
|
|
|
|
llvm::add_module_flag_u32(
|
|
|
|
llmod,
|
|
|
|
llvm::ModuleFlagMergeBehavior::Warning,
|
|
|
|
"cfguard",
|
|
|
|
2,
|
|
|
|
);
|
2020-07-06 15:10:42 +00:00
|
|
|
}
|
2020-01-13 13:25:39 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-07-06 12:44:47 +00:00
|
|
|
if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection {
|
2022-12-13 17:04:02 +00:00
|
|
|
if sess.target.arch == "aarch64" {
|
2024-10-29 02:38:17 +00:00
|
|
|
llvm::add_module_flag_u32(
|
|
|
|
llmod,
|
|
|
|
llvm::ModuleFlagMergeBehavior::Min,
|
|
|
|
"branch-target-enforcement",
|
|
|
|
bti.into(),
|
|
|
|
);
|
|
|
|
llvm::add_module_flag_u32(
|
|
|
|
llmod,
|
|
|
|
llvm::ModuleFlagMergeBehavior::Min,
|
|
|
|
"sign-return-address",
|
|
|
|
pac_ret.is_some().into(),
|
|
|
|
);
|
2024-10-16 14:39:58 +00:00
|
|
|
let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, pc: false, key: PAuthKey::A });
|
|
|
|
llvm::add_module_flag_u32(
|
|
|
|
llmod,
|
|
|
|
llvm::ModuleFlagMergeBehavior::Min,
|
|
|
|
"branch-protection-pauth-lr",
|
|
|
|
pac_opts.pc.into(),
|
|
|
|
);
|
2024-10-29 02:38:17 +00:00
|
|
|
llvm::add_module_flag_u32(
|
|
|
|
llmod,
|
|
|
|
llvm::ModuleFlagMergeBehavior::Min,
|
|
|
|
"sign-return-address-all",
|
|
|
|
pac_opts.leaf.into(),
|
|
|
|
);
|
|
|
|
llvm::add_module_flag_u32(
|
|
|
|
llmod,
|
|
|
|
llvm::ModuleFlagMergeBehavior::Min,
|
|
|
|
"sign-return-address-with-bkey",
|
|
|
|
u32::from(pac_opts.key == PAuthKey::B),
|
|
|
|
);
|
2022-12-13 17:04:02 +00:00
|
|
|
} else {
|
|
|
|
bug!(
|
|
|
|
"branch-protection used on non-AArch64 target; \
|
|
|
|
this should be checked in rustc_session."
|
|
|
|
);
|
2022-01-31 19:47:07 +00:00
|
|
|
}
|
2021-07-13 11:14:26 +00:00
|
|
|
}
|
|
|
|
|
2022-01-28 17:48:59 +00:00
|
|
|
// Pass on the control-flow protection flags to LLVM (equivalent to `-fcf-protection` in Clang).
|
2022-07-06 12:44:47 +00:00
|
|
|
if let CFProtection::Branch | CFProtection::Full = sess.opts.unstable_opts.cf_protection {
|
2024-10-29 02:38:17 +00:00
|
|
|
llvm::add_module_flag_u32(
|
|
|
|
llmod,
|
|
|
|
llvm::ModuleFlagMergeBehavior::Override,
|
|
|
|
"cf-protection-branch",
|
|
|
|
1,
|
|
|
|
);
|
2022-01-28 17:48:59 +00:00
|
|
|
}
|
2022-07-06 12:44:47 +00:00
|
|
|
if let CFProtection::Return | CFProtection::Full = sess.opts.unstable_opts.cf_protection {
|
2024-10-29 02:38:17 +00:00
|
|
|
llvm::add_module_flag_u32(
|
|
|
|
llmod,
|
|
|
|
llvm::ModuleFlagMergeBehavior::Override,
|
|
|
|
"cf-protection-return",
|
|
|
|
1,
|
|
|
|
);
|
2022-01-28 17:48:59 +00:00
|
|
|
}
|
|
|
|
|
2022-07-06 12:44:47 +00:00
|
|
|
if sess.opts.unstable_opts.virtual_function_elimination {
|
2024-10-29 02:38:17 +00:00
|
|
|
llvm::add_module_flag_u32(
|
|
|
|
llmod,
|
|
|
|
llvm::ModuleFlagMergeBehavior::Error,
|
|
|
|
"Virtual Function Elim",
|
|
|
|
1,
|
|
|
|
);
|
2022-04-21 12:35:40 +00:00
|
|
|
}
|
2023-11-21 22:24:23 +00:00
|
|
|
|
2023-11-17 18:05:38 +00:00
|
|
|
// Set module flag to enable Windows EHCont Guard (/guard:ehcont).
|
2023-11-21 22:24:23 +00:00
|
|
|
if sess.opts.unstable_opts.ehcont_guard {
|
2024-10-29 02:38:17 +00:00
|
|
|
llvm::add_module_flag_u32(llmod, llvm::ModuleFlagMergeBehavior::Warning, "ehcontguard", 1);
|
2023-11-17 18:05:38 +00:00
|
|
|
}
|
2022-04-21 12:35:40 +00:00
|
|
|
|
2024-09-25 08:23:30 +00:00
|
|
|
match sess.opts.unstable_opts.function_return {
|
|
|
|
FunctionReturn::Keep => {}
|
2024-10-29 02:38:17 +00:00
|
|
|
FunctionReturn::ThunkExtern => {
|
|
|
|
llvm::add_module_flag_u32(
|
2024-09-25 08:23:30 +00:00
|
|
|
llmod,
|
2024-10-29 02:38:17 +00:00
|
|
|
llvm::ModuleFlagMergeBehavior::Override,
|
|
|
|
"function_return_thunk_extern",
|
2024-09-25 08:23:30 +00:00
|
|
|
1,
|
2024-10-29 02:38:17 +00:00
|
|
|
);
|
|
|
|
}
|
2024-09-25 08:23:30 +00:00
|
|
|
}
|
|
|
|
|
2024-09-10 19:19:16 +00:00
|
|
|
match (sess.opts.unstable_opts.small_data_threshold, sess.target.small_data_threshold_support())
|
|
|
|
{
|
|
|
|
// Set up the small-data optimization limit for architectures that use
|
|
|
|
// an LLVM module flag to control this.
|
|
|
|
(Some(threshold), SmallDataThresholdSupport::LlvmModuleFlag(flag)) => {
|
2024-10-29 02:38:17 +00:00
|
|
|
llvm::add_module_flag_u32(
|
|
|
|
llmod,
|
|
|
|
llvm::ModuleFlagMergeBehavior::Error,
|
|
|
|
&flag,
|
|
|
|
threshold as u32,
|
|
|
|
);
|
2024-09-10 19:19:16 +00:00
|
|
|
}
|
|
|
|
_ => (),
|
|
|
|
};
|
|
|
|
|
2022-05-28 23:10:44 +00:00
|
|
|
// Insert `llvm.ident` metadata.
|
|
|
|
//
|
|
|
|
// On the wasm targets it will get hooked up to the "producer" sections
|
|
|
|
// `processed-by` information.
|
2024-02-24 22:11:09 +00:00
|
|
|
#[allow(clippy::option_env_unwrap)]
|
2022-05-28 23:10:44 +00:00
|
|
|
let rustc_producer =
|
|
|
|
format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION"));
|
2024-07-14 18:27:57 +00:00
|
|
|
let name_metadata = unsafe {
|
2024-09-19 01:39:28 +00:00
|
|
|
llvm::LLVMMDStringInContext2(
|
2024-07-14 18:27:57 +00:00
|
|
|
llcx,
|
2024-10-28 07:52:39 +00:00
|
|
|
rustc_producer.as_c_char_ptr(),
|
2024-09-19 01:39:28 +00:00
|
|
|
rustc_producer.as_bytes().len(),
|
2024-07-14 18:27:57 +00:00
|
|
|
)
|
|
|
|
};
|
|
|
|
unsafe {
|
|
|
|
llvm::LLVMAddNamedMetadataOperand(
|
|
|
|
llmod,
|
|
|
|
c"llvm.ident".as_ptr(),
|
2024-09-19 01:39:28 +00:00
|
|
|
&llvm::LLVMMetadataAsValue(llcx, llvm::LLVMMDNodeInContext2(llcx, &name_metadata, 1)),
|
2024-07-14 18:27:57 +00:00
|
|
|
);
|
|
|
|
}
|
2022-05-28 23:10:44 +00:00
|
|
|
|
2024-01-28 10:38:41 +00:00
|
|
|
// Emit RISC-V specific target-abi metadata
|
|
|
|
// to workaround lld as the LTO plugin not
|
|
|
|
// correctly setting target-abi for the LTO object
|
|
|
|
// FIXME: https://github.com/llvm/llvm-project/issues/50591
|
|
|
|
// If llvm_abiname is empty, emit nothing.
|
2024-04-09 06:47:06 +00:00
|
|
|
let llvm_abiname = &sess.target.options.llvm_abiname;
|
|
|
|
if matches!(sess.target.arch.as_ref(), "riscv32" | "riscv64") && !llvm_abiname.is_empty() {
|
2024-10-29 02:38:17 +00:00
|
|
|
llvm::add_module_flag_str(
|
|
|
|
llmod,
|
|
|
|
llvm::ModuleFlagMergeBehavior::Error,
|
|
|
|
"target-abi",
|
|
|
|
llvm_abiname,
|
|
|
|
);
|
2024-01-28 10:38:41 +00:00
|
|
|
}
|
|
|
|
|
2023-10-09 09:22:14 +00:00
|
|
|
// Add module flags specified via -Z llvm_module_flag
|
2024-10-29 02:38:17 +00:00
|
|
|
for (key, value, merge_behavior) in &sess.opts.unstable_opts.llvm_module_flag {
|
|
|
|
let merge_behavior = match merge_behavior.as_str() {
|
|
|
|
"error" => llvm::ModuleFlagMergeBehavior::Error,
|
|
|
|
"warning" => llvm::ModuleFlagMergeBehavior::Warning,
|
|
|
|
"require" => llvm::ModuleFlagMergeBehavior::Require,
|
|
|
|
"override" => llvm::ModuleFlagMergeBehavior::Override,
|
|
|
|
"append" => llvm::ModuleFlagMergeBehavior::Append,
|
|
|
|
"appendunique" => llvm::ModuleFlagMergeBehavior::AppendUnique,
|
|
|
|
"max" => llvm::ModuleFlagMergeBehavior::Max,
|
|
|
|
"min" => llvm::ModuleFlagMergeBehavior::Min,
|
2023-10-09 09:22:14 +00:00
|
|
|
// We already checked this during option parsing
|
|
|
|
_ => unreachable!(),
|
|
|
|
};
|
2024-10-29 02:38:17 +00:00
|
|
|
llvm::add_module_flag_u32(llmod, merge_behavior, key, *value);
|
2023-10-09 09:22:14 +00:00
|
|
|
}
|
|
|
|
|
2018-06-27 14:57:25 +00:00
|
|
|
llmod
|
2014-07-16 18:27:57 +00:00
|
|
|
}
|
|
|
|
|
2018-09-26 14:01:43 +00:00
|
|
|
impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
|
2022-05-20 23:51:09 +00:00
|
|
|
pub(crate) fn new(
|
2019-06-13 21:48:52 +00:00
|
|
|
tcx: TyCtxt<'tcx>,
|
2020-03-14 11:41:32 +00:00
|
|
|
codegen_unit: &'tcx CodegenUnit<'tcx>,
|
2019-06-11 21:11:55 +00:00
|
|
|
llvm_module: &'ll crate::ModuleLlvm,
|
|
|
|
) -> Self {
|
2015-05-12 21:46:19 +00:00
|
|
|
// An interesting part of Windows which MSVC forces our hand on (and
|
|
|
|
// apparently MinGW didn't) is the usage of `dllimport` and `dllexport`
|
|
|
|
// attributes in LLVM IR as well as native dependencies (in C these
|
|
|
|
// correspond to `__declspec(dllimport)`).
|
|
|
|
//
|
2020-05-07 09:52:21 +00:00
|
|
|
// LD (BFD) in MinGW mode can often correctly guess `dllexport` but
|
|
|
|
// relying on that can result in issues like #50176.
|
|
|
|
// LLD won't support that and expects symbols with proper attributes.
|
|
|
|
// Because of that we make MinGW target emit dllexport just like MSVC.
|
|
|
|
// When it comes to dllimport we use it for constants but for functions
|
|
|
|
// rely on the linker to do the right thing. Opposed to dllexport this
|
|
|
|
// task is easy for them (both LD and LLD) and allows us to easily use
|
|
|
|
// symbols from static libraries in shared libraries.
|
|
|
|
//
|
|
|
|
// Whenever a dynamic library is built on Windows it must have its public
|
2015-05-12 21:46:19 +00:00
|
|
|
// interface specified by functions tagged with `dllexport` or otherwise
|
|
|
|
// they're not available to be linked against. This poses a few problems
|
|
|
|
// for the compiler, some of which are somewhat fundamental, but we use
|
|
|
|
// the `use_dll_storage_attrs` variable below to attach the `dllexport`
|
2018-11-27 02:59:49 +00:00
|
|
|
// attribute to all LLVM functions that are exported e.g., they're
|
2015-05-12 21:46:19 +00:00
|
|
|
// already tagged with external linkage). This is suboptimal for a few
|
|
|
|
// reasons:
|
|
|
|
//
|
|
|
|
// * If an object file will never be included in a dynamic library,
|
|
|
|
// there's no need to attach the dllexport attribute. Most object
|
|
|
|
// files in Rust are not destined to become part of a dll as binaries
|
|
|
|
// are statically linked by default.
|
|
|
|
// * If the compiler is emitting both an rlib and a dylib, the same
|
|
|
|
// source object file is currently used but with MSVC this may be less
|
|
|
|
// feasible. The compiler may be able to get around this, but it may
|
|
|
|
// involve some invasive changes to deal with this.
|
|
|
|
//
|
2022-03-30 05:39:38 +00:00
|
|
|
// The flip side of this situation is that whenever you link to a dll and
|
2015-05-12 21:46:19 +00:00
|
|
|
// you import a function from it, the import should be tagged with
|
|
|
|
// `dllimport`. At this time, however, the compiler does not emit
|
|
|
|
// `dllimport` for any declarations other than constants (where it is
|
|
|
|
// required), which is again suboptimal for even more reasons!
|
|
|
|
//
|
|
|
|
// * Calling a function imported from another dll without using
|
|
|
|
// `dllimport` causes the linker/compiler to have extra overhead (one
|
|
|
|
// `jmp` instruction on x86) when calling the function.
|
|
|
|
// * The same object file may be used in different circumstances, so a
|
|
|
|
// function may be imported from a dll if the object is linked into a
|
|
|
|
// dll, but it may be just linked against if linked into an rlib.
|
|
|
|
// * The compiler has no knowledge about whether native functions should
|
|
|
|
// be tagged dllimport or not.
|
|
|
|
//
|
|
|
|
// For now the compiler takes the perf hit (I do not have any numbers to
|
|
|
|
// this effect) by marking very little as `dllimport` and praying the
|
|
|
|
// linker will take care of everything. Fixing this problem will likely
|
|
|
|
// require adding a few attributes to Rust itself (feature gated at the
|
2020-05-07 09:52:21 +00:00
|
|
|
// start) and then strongly recommending static linkage on Windows!
|
2020-11-08 11:27:51 +00:00
|
|
|
let use_dll_storage_attrs = tcx.sess.target.is_like_windows;
|
2015-05-12 21:46:19 +00:00
|
|
|
|
2020-04-25 18:45:21 +00:00
|
|
|
let tls_model = to_llvm_tls_model(tcx.sess.tls_model());
|
2017-11-03 00:23:56 +00:00
|
|
|
|
2018-06-27 14:57:25 +00:00
|
|
|
let (llcx, llmod) = (&*llvm_module.llcx, llvm_module.llmod());
|
|
|
|
|
2023-02-15 11:43:41 +00:00
|
|
|
let coverage_cx =
|
|
|
|
tcx.sess.instrument_coverage().then(coverageinfo::CrateCoverageContext::new);
|
2020-06-22 06:29:08 +00:00
|
|
|
|
2018-07-26 17:41:10 +00:00
|
|
|
let dbg_cx = if tcx.sess.opts.debuginfo != DebugInfo::None {
|
2022-03-03 10:15:25 +00:00
|
|
|
let dctx = debuginfo::CodegenUnitDebugContext::new(llmod);
|
|
|
|
debuginfo::metadata::build_compile_unit_di_node(
|
|
|
|
tcx,
|
|
|
|
codegen_unit.name().as_str(),
|
|
|
|
&dctx,
|
|
|
|
);
|
2018-06-27 14:57:25 +00:00
|
|
|
Some(dctx)
|
|
|
|
} else {
|
|
|
|
None
|
|
|
|
};
|
|
|
|
|
2018-08-22 15:48:32 +00:00
|
|
|
let isize_ty = Type::ix_llcx(llcx, tcx.data_layout.pointer_size.bits());
|
2018-06-27 14:57:25 +00:00
|
|
|
|
|
|
|
CodegenCx {
|
|
|
|
tcx,
|
|
|
|
use_dll_storage_attrs,
|
|
|
|
tls_model,
|
|
|
|
llmod,
|
|
|
|
llcx,
|
|
|
|
codegen_unit,
|
2018-10-16 14:57:53 +00:00
|
|
|
instances: Default::default(),
|
|
|
|
vtables: Default::default(),
|
2022-03-05 04:14:38 +00:00
|
|
|
const_str_cache: Default::default(),
|
2018-10-16 14:57:53 +00:00
|
|
|
const_globals: Default::default(),
|
2018-06-27 14:57:25 +00:00
|
|
|
statics_to_rauw: RefCell::new(Vec::new()),
|
|
|
|
used_statics: RefCell::new(Vec::new()),
|
2021-08-20 19:13:18 +00:00
|
|
|
compiler_used_statics: RefCell::new(Vec::new()),
|
2021-08-05 17:14:55 +00:00
|
|
|
type_lowering: Default::default(),
|
2018-10-16 14:57:53 +00:00
|
|
|
scalar_lltypes: Default::default(),
|
2018-06-27 14:57:25 +00:00
|
|
|
isize_ty,
|
2020-06-22 06:29:08 +00:00
|
|
|
coverage_cx,
|
2018-06-27 14:57:25 +00:00
|
|
|
dbg_cx,
|
|
|
|
eh_personality: Cell::new(None),
|
2020-03-21 07:50:38 +00:00
|
|
|
eh_catch_typeinfo: Cell::new(None),
|
2018-06-27 14:57:25 +00:00
|
|
|
rust_try_fn: Cell::new(None),
|
2018-10-16 14:57:53 +00:00
|
|
|
intrinsics: Default::default(),
|
2018-06-27 14:57:25 +00:00
|
|
|
local_gen_sym_counter: Cell::new(0),
|
2022-03-01 00:53:25 +00:00
|
|
|
renamed_statics: Default::default(),
|
2014-07-16 18:27:57 +00:00
|
|
|
}
|
2014-03-15 20:29:34 +00:00
|
|
|
}
|
2018-11-24 13:18:13 +00:00
|
|
|
|
2022-05-20 23:51:09 +00:00
|
|
|
pub(crate) fn statics_to_rauw(&self) -> &RefCell<Vec<(&'ll Value, &'ll Value)>> {
|
2018-11-24 13:18:13 +00:00
|
|
|
&self.statics_to_rauw
|
|
|
|
}
|
2020-06-22 06:29:08 +00:00
|
|
|
|
2024-10-24 11:23:13 +00:00
|
|
|
/// Extra state that is only available when coverage instrumentation is enabled.
|
2020-06-22 06:29:08 +00:00
|
|
|
#[inline]
|
2024-10-31 10:12:15 +00:00
|
|
|
#[track_caller]
|
2024-10-24 11:23:13 +00:00
|
|
|
pub(crate) fn coverage_cx(&self) -> &coverageinfo::CrateCoverageContext<'ll, 'tcx> {
|
|
|
|
self.coverage_cx.as_ref().expect("only called when coverage instrumentation is enabled")
|
2020-06-22 06:29:08 +00:00
|
|
|
}
|
2021-08-20 19:13:18 +00:00
|
|
|
|
2022-10-01 16:45:07 +00:00
|
|
|
pub(crate) fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) {
|
2022-12-06 05:07:28 +00:00
|
|
|
let array = self.const_array(self.type_ptr(), values);
|
2021-08-20 19:13:18 +00:00
|
|
|
|
|
|
|
unsafe {
|
|
|
|
let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr());
|
|
|
|
llvm::LLVMSetInitializer(g, array);
|
2024-10-25 10:40:38 +00:00
|
|
|
llvm::set_linkage(g, llvm::Linkage::AppendingLinkage);
|
2024-10-29 23:02:46 +00:00
|
|
|
llvm::set_section(g, c"llvm.metadata");
|
2021-08-20 19:13:18 +00:00
|
|
|
}
|
|
|
|
}
|
2014-07-16 18:27:57 +00:00
|
|
|
}
|
2014-03-15 20:29:34 +00:00
|
|
|
|
2024-09-17 00:15:26 +00:00
|
|
|
impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
|
2018-09-13 12:58:19 +00:00
|
|
|
fn vtables(
|
|
|
|
&self,
|
2018-12-04 11:28:06 +00:00
|
|
|
) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), &'ll Value>>
|
2018-09-13 12:58:19 +00:00
|
|
|
{
|
|
|
|
&self.vtables
|
|
|
|
}
|
2018-09-20 13:47:22 +00:00
|
|
|
|
2024-03-30 12:01:57 +00:00
|
|
|
fn apply_vcall_visibility_metadata(
|
|
|
|
&self,
|
|
|
|
ty: Ty<'tcx>,
|
|
|
|
poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
|
|
|
|
vtable: &'ll Value,
|
|
|
|
) {
|
|
|
|
apply_vcall_visibility_metadata(self, ty, poly_trait_ref, vtable);
|
|
|
|
}
|
|
|
|
|
2019-10-13 10:05:40 +00:00
|
|
|
fn get_fn(&self, instance: Instance<'tcx>) -> &'ll Value {
|
|
|
|
get_fn(self, instance)
|
2018-09-20 13:47:22 +00:00
|
|
|
}
|
|
|
|
|
2019-10-13 10:05:40 +00:00
|
|
|
fn get_fn_addr(&self, instance: Instance<'tcx>) -> &'ll Value {
|
2018-11-24 16:03:53 +00:00
|
|
|
get_fn(self, instance)
|
2018-09-13 12:58:19 +00:00
|
|
|
}
|
2018-09-20 13:47:22 +00:00
|
|
|
|
|
|
|
fn eh_personality(&self) -> &'ll Value {
|
|
|
|
// The exception handling personality function.
|
|
|
|
//
|
|
|
|
// If our compilation unit has the `eh_personality` lang item somewhere
|
|
|
|
// within it, then we just need to codegen that. Otherwise, we're
|
|
|
|
// building an rlib which will depend on some upstream implementation of
|
|
|
|
// this function, so we just codegen a generic reference to it. We don't
|
|
|
|
// specify any of the types for the function, we just make it a symbol
|
|
|
|
// that LLVM can later use.
|
|
|
|
//
|
|
|
|
// Note that MSVC is a little special here in that we don't use the
|
|
|
|
// `eh_personality` lang item at all. Currently LLVM has support for
|
|
|
|
// both Dwarf and SEH unwind mechanisms for MSVC targets and uses the
|
|
|
|
// *name of the personality function* to decide what kind of unwind side
|
|
|
|
// tables/landing pads to emit. It looks like Dwarf is used by default,
|
|
|
|
// injecting a dependency on the `_Unwind_Resume` symbol for resuming
|
|
|
|
// an "exception", but for MSVC we want to force SEH. This means that we
|
|
|
|
// can't actually have the personality function be our standard
|
|
|
|
// `rust_eh_personality` function, but rather we wired it up to the
|
|
|
|
// CRT's custom personality function, which forces LLVM to consider
|
|
|
|
// landing pads as "landing pads for SEH".
|
|
|
|
if let Some(llpersonality) = self.eh_personality.get() {
|
|
|
|
return llpersonality;
|
|
|
|
}
|
2023-05-01 19:24:21 +00:00
|
|
|
|
|
|
|
let name = if wants_msvc_seh(self.sess()) {
|
|
|
|
Some("__CxxFrameHandler3")
|
|
|
|
} else if wants_wasm_eh(self.sess()) {
|
2023-05-21 22:28:44 +00:00
|
|
|
// LLVM specifically tests for the name of the personality function
|
|
|
|
// There is no need for this function to exist anywhere, it will
|
|
|
|
// not be called. However, its name has to be "__gxx_wasm_personality_v0"
|
|
|
|
// for native wasm exceptions.
|
2023-05-01 19:24:21 +00:00
|
|
|
Some("__gxx_wasm_personality_v0")
|
|
|
|
} else {
|
|
|
|
None
|
|
|
|
};
|
|
|
|
|
2018-09-20 13:47:22 +00:00
|
|
|
let tcx = self.tcx;
|
|
|
|
let llfn = match tcx.lang_items().eh_personality() {
|
2024-03-10 10:49:27 +00:00
|
|
|
Some(def_id) if name.is_none() => self.get_fn_addr(ty::Instance::expect_resolve(
|
|
|
|
tcx,
|
|
|
|
ty::ParamEnv::reveal_all(),
|
|
|
|
def_id,
|
|
|
|
ty::List::empty(),
|
2024-07-01 20:32:32 +00:00
|
|
|
DUMMY_SP,
|
2024-03-10 10:49:27 +00:00
|
|
|
)),
|
2018-09-20 13:47:22 +00:00
|
|
|
_ => {
|
2023-05-01 19:24:21 +00:00
|
|
|
let name = name.unwrap_or("rust_eh_personality");
|
2021-07-07 00:00:00 +00:00
|
|
|
if let Some(llfn) = self.get_declared_value(name) {
|
|
|
|
llfn
|
|
|
|
} else {
|
|
|
|
let fty = self.type_variadic_func(&[], self.type_i32());
|
|
|
|
let llfn = self.declare_cfn(name, llvm::UnnamedAddr::Global, fty);
|
2022-02-21 16:19:16 +00:00
|
|
|
let target_cpu = attributes::target_cpu_attr(self);
|
|
|
|
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[target_cpu]);
|
2021-07-07 00:00:00 +00:00
|
|
|
llfn
|
|
|
|
}
|
2018-09-20 13:47:22 +00:00
|
|
|
}
|
|
|
|
};
|
|
|
|
self.eh_personality.set(Some(llfn));
|
|
|
|
llfn
|
|
|
|
}
|
|
|
|
|
|
|
|
fn sess(&self) -> &Session {
|
2021-09-30 17:38:50 +00:00
|
|
|
self.tcx.sess
|
2018-09-20 13:47:22 +00:00
|
|
|
}
|
|
|
|
|
2020-03-14 11:41:32 +00:00
|
|
|
fn codegen_unit(&self) -> &'tcx CodegenUnit<'tcx> {
|
|
|
|
self.codegen_unit
|
2018-09-21 14:13:15 +00:00
|
|
|
}
|
2018-09-24 13:26:39 +00:00
|
|
|
|
2021-06-26 20:53:35 +00:00
|
|
|
fn set_frame_pointer_type(&self, llfn: &'ll Value) {
|
2022-02-21 16:19:16 +00:00
|
|
|
if let Some(attr) = attributes::frame_pointer_type_attr(self) {
|
|
|
|
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]);
|
|
|
|
}
|
2018-09-24 13:26:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
fn apply_target_cpu_attr(&self, llfn: &'ll Value) {
|
2022-02-21 16:19:16 +00:00
|
|
|
let mut attrs = SmallVec::<[_; 2]>::new();
|
|
|
|
attrs.push(attributes::target_cpu_attr(self));
|
|
|
|
attrs.extend(attributes::tune_cpu_attr(self));
|
|
|
|
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs);
|
2018-09-24 13:26:39 +00:00
|
|
|
}
|
2018-09-26 15:00:01 +00:00
|
|
|
|
2020-09-18 11:06:53 +00:00
|
|
|
fn declare_c_main(&self, fn_type: Self::Type) -> Option<Self::Function> {
|
2022-11-05 09:06:38 +00:00
|
|
|
let entry_name = self.sess().target.entry_name.as_ref();
|
|
|
|
if self.get_declared_value(entry_name).is_none() {
|
|
|
|
Some(self.declare_entry_fn(
|
|
|
|
entry_name,
|
|
|
|
self.sess().target.entry_abi.into(),
|
|
|
|
llvm::UnnamedAddr::Global,
|
|
|
|
fn_type,
|
|
|
|
))
|
2020-09-18 11:06:53 +00:00
|
|
|
} else {
|
|
|
|
// If the symbol already exists, it is an error: for example, the user wrote
|
|
|
|
// #[no_mangle] extern "C" fn main(..) {..}
|
|
|
|
// instead of #[start]
|
|
|
|
None
|
|
|
|
}
|
|
|
|
}
|
2018-09-13 12:58:19 +00:00
|
|
|
}
|
|
|
|
|
2021-12-14 18:49:49 +00:00
|
|
|
impl<'ll> CodegenCx<'ll, '_> {
|
2022-05-20 23:51:09 +00:00
|
|
|
pub(crate) fn get_intrinsic(&self, key: &str) -> (&'ll Type, &'ll Value) {
|
2018-01-05 04:58:34 +00:00
|
|
|
if let Some(v) = self.intrinsics.borrow().get(key).cloned() {
|
2014-11-29 21:41:21 +00:00
|
|
|
return v;
|
2014-04-09 23:56:31 +00:00
|
|
|
}
|
2018-10-08 14:58:26 +00:00
|
|
|
|
2018-09-07 22:39:39 +00:00
|
|
|
self.declare_intrinsic(key).unwrap_or_else(|| bug!("unknown intrinsic '{}'", key))
|
|
|
|
}
|
|
|
|
|
2019-02-08 15:41:37 +00:00
|
|
|
fn insert_intrinsic(
|
|
|
|
&self,
|
|
|
|
name: &'static str,
|
2021-12-14 18:49:49 +00:00
|
|
|
args: Option<&[&'ll llvm::Type]>,
|
|
|
|
ret: &'ll llvm::Type,
|
|
|
|
) -> (&'ll llvm::Type, &'ll llvm::Value) {
|
2019-02-08 15:41:37 +00:00
|
|
|
let fn_ty = if let Some(args) = args {
|
|
|
|
self.type_func(args, ret)
|
|
|
|
} else {
|
|
|
|
self.type_variadic_func(&[], ret)
|
|
|
|
};
|
2021-01-23 22:19:49 +00:00
|
|
|
let f = self.declare_cfn(name, llvm::UnnamedAddr::No, fn_ty);
|
2021-08-03 22:09:57 +00:00
|
|
|
self.intrinsics.borrow_mut().insert(name, (fn_ty, f));
|
|
|
|
(fn_ty, f)
|
2019-02-08 15:41:37 +00:00
|
|
|
}
|
|
|
|
|
2021-12-14 18:49:49 +00:00
|
|
|
fn declare_intrinsic(&self, key: &str) -> Option<(&'ll Type, &'ll Value)> {
|
2018-09-07 22:39:39 +00:00
|
|
|
macro_rules! ifn {
|
|
|
|
($name:expr, fn() -> $ret:expr) => (
|
|
|
|
if key == $name {
|
2019-02-08 15:41:37 +00:00
|
|
|
return Some(self.insert_intrinsic($name, Some(&[]), $ret));
|
2018-09-07 22:39:39 +00:00
|
|
|
}
|
|
|
|
);
|
|
|
|
($name:expr, fn(...) -> $ret:expr) => (
|
|
|
|
if key == $name {
|
2019-02-08 15:41:37 +00:00
|
|
|
return Some(self.insert_intrinsic($name, None, $ret));
|
2018-09-07 22:39:39 +00:00
|
|
|
}
|
|
|
|
);
|
|
|
|
($name:expr, fn($($arg:expr),*) -> $ret:expr) => (
|
|
|
|
if key == $name {
|
2019-02-08 15:41:37 +00:00
|
|
|
return Some(self.insert_intrinsic($name, Some(&[$($arg),*]), $ret));
|
2018-09-07 22:39:39 +00:00
|
|
|
}
|
|
|
|
);
|
|
|
|
}
|
|
|
|
macro_rules! mk_struct {
|
2018-11-07 10:08:41 +00:00
|
|
|
($($field_ty:expr),*) => (self.type_struct( &[$($field_ty),*], false))
|
2018-09-07 22:39:39 +00:00
|
|
|
}
|
|
|
|
|
2022-12-06 05:07:28 +00:00
|
|
|
let ptr = self.type_ptr();
|
2018-11-07 10:08:41 +00:00
|
|
|
let void = self.type_void();
|
|
|
|
let i1 = self.type_i1();
|
|
|
|
let t_i8 = self.type_i8();
|
|
|
|
let t_i16 = self.type_i16();
|
|
|
|
let t_i32 = self.type_i32();
|
|
|
|
let t_i64 = self.type_i64();
|
|
|
|
let t_i128 = self.type_i128();
|
2021-05-30 17:25:41 +00:00
|
|
|
let t_isize = self.type_isize();
|
2024-03-01 08:53:26 +00:00
|
|
|
let t_f16 = self.type_f16();
|
2018-11-07 10:08:41 +00:00
|
|
|
let t_f32 = self.type_f32();
|
|
|
|
let t_f64 = self.type_f64();
|
2024-03-01 08:53:26 +00:00
|
|
|
let t_f128 = self.type_f128();
|
2022-04-21 12:58:25 +00:00
|
|
|
let t_metadata = self.type_metadata();
|
2023-05-01 19:22:54 +00:00
|
|
|
let t_token = self.type_token();
|
|
|
|
|
2022-12-06 05:07:28 +00:00
|
|
|
ifn!("llvm.wasm.get.exception", fn(t_token) -> ptr);
|
2023-05-01 19:22:54 +00:00
|
|
|
ifn!("llvm.wasm.get.ehselector", fn(t_token) -> t_i32);
|
2018-11-07 10:08:41 +00:00
|
|
|
|
2020-07-22 21:51:12 +00:00
|
|
|
ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32);
|
|
|
|
ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32);
|
|
|
|
ifn!("llvm.wasm.trunc.unsigned.i64.f32", fn(t_f32) -> t_i64);
|
|
|
|
ifn!("llvm.wasm.trunc.unsigned.i64.f64", fn(t_f64) -> t_i64);
|
|
|
|
ifn!("llvm.wasm.trunc.signed.i32.f32", fn(t_f32) -> t_i32);
|
|
|
|
ifn!("llvm.wasm.trunc.signed.i32.f64", fn(t_f64) -> t_i32);
|
|
|
|
ifn!("llvm.wasm.trunc.signed.i64.f32", fn(t_f32) -> t_i64);
|
|
|
|
ifn!("llvm.wasm.trunc.signed.i64.f64", fn(t_f64) -> t_i64);
|
2020-06-25 15:05:12 +00:00
|
|
|
|
2021-04-19 17:55:32 +00:00
|
|
|
ifn!("llvm.fptosi.sat.i8.f32", fn(t_f32) -> t_i8);
|
|
|
|
ifn!("llvm.fptosi.sat.i16.f32", fn(t_f32) -> t_i16);
|
|
|
|
ifn!("llvm.fptosi.sat.i32.f32", fn(t_f32) -> t_i32);
|
|
|
|
ifn!("llvm.fptosi.sat.i64.f32", fn(t_f32) -> t_i64);
|
|
|
|
ifn!("llvm.fptosi.sat.i128.f32", fn(t_f32) -> t_i128);
|
|
|
|
ifn!("llvm.fptosi.sat.i8.f64", fn(t_f64) -> t_i8);
|
|
|
|
ifn!("llvm.fptosi.sat.i16.f64", fn(t_f64) -> t_i16);
|
|
|
|
ifn!("llvm.fptosi.sat.i32.f64", fn(t_f64) -> t_i32);
|
|
|
|
ifn!("llvm.fptosi.sat.i64.f64", fn(t_f64) -> t_i64);
|
|
|
|
ifn!("llvm.fptosi.sat.i128.f64", fn(t_f64) -> t_i128);
|
|
|
|
|
|
|
|
ifn!("llvm.fptoui.sat.i8.f32", fn(t_f32) -> t_i8);
|
|
|
|
ifn!("llvm.fptoui.sat.i16.f32", fn(t_f32) -> t_i16);
|
|
|
|
ifn!("llvm.fptoui.sat.i32.f32", fn(t_f32) -> t_i32);
|
|
|
|
ifn!("llvm.fptoui.sat.i64.f32", fn(t_f32) -> t_i64);
|
|
|
|
ifn!("llvm.fptoui.sat.i128.f32", fn(t_f32) -> t_i128);
|
|
|
|
ifn!("llvm.fptoui.sat.i8.f64", fn(t_f64) -> t_i8);
|
|
|
|
ifn!("llvm.fptoui.sat.i16.f64", fn(t_f64) -> t_i16);
|
|
|
|
ifn!("llvm.fptoui.sat.i32.f64", fn(t_f64) -> t_i32);
|
|
|
|
ifn!("llvm.fptoui.sat.i64.f64", fn(t_f64) -> t_i64);
|
|
|
|
ifn!("llvm.fptoui.sat.i128.f64", fn(t_f64) -> t_i128);
|
|
|
|
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.trap", fn() -> void);
|
|
|
|
ifn!("llvm.debugtrap", fn() -> void);
|
2022-12-06 05:07:28 +00:00
|
|
|
ifn!("llvm.frameaddress", fn(t_i32) -> ptr);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2024-07-12 07:54:54 +00:00
|
|
|
ifn!("llvm.powi.f16.i32", fn(t_f16, t_i32) -> t_f16);
|
|
|
|
ifn!("llvm.powi.f32.i32", fn(t_f32, t_i32) -> t_f32);
|
|
|
|
ifn!("llvm.powi.f64.i32", fn(t_f64, t_i32) -> t_f64);
|
|
|
|
ifn!("llvm.powi.f128.i32", fn(t_f128, t_i32) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.pow.f16", fn(t_f16, t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.pow.f32", fn(t_f32, t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.pow.f64", fn(t_f64, t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.pow.f128", fn(t_f128, t_f128) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.sqrt.f16", fn(t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.sqrt.f32", fn(t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.sqrt.f64", fn(t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.sqrt.f128", fn(t_f128) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.sin.f16", fn(t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.sin.f32", fn(t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.sin.f64", fn(t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.sin.f128", fn(t_f128) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.cos.f16", fn(t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.cos.f32", fn(t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.cos.f64", fn(t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.cos.f128", fn(t_f128) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.exp.f16", fn(t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.exp.f32", fn(t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.exp.f64", fn(t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.exp.f128", fn(t_f128) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.exp2.f16", fn(t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.exp2.f32", fn(t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.exp2.f64", fn(t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.exp2.f128", fn(t_f128) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.log.f16", fn(t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.log.f32", fn(t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.log.f64", fn(t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.log.f128", fn(t_f128) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.log10.f16", fn(t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.log10.f32", fn(t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.log10.f64", fn(t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.log10.f128", fn(t_f128) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.log2.f16", fn(t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.log2.f32", fn(t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.log2.f64", fn(t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.log2.f128", fn(t_f128) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.fma.f16", fn(t_f16, t_f16, t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.fma.f32", fn(t_f32, t_f32, t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.fma.f64", fn(t_f64, t_f64, t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.fma.f128", fn(t_f128, t_f128, t_f128) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
intrinsics.fmuladdf{16,32,64,128}: expose llvm.fmuladd.* semantics
Add intrinsics `fmuladd{f16,f32,f64,f128}`. This computes `(a * b) +
c`, to be fused if the code generator determines that (i) the target
instruction set has support for a fused operation, and (ii) that the
fused operation is more efficient than the equivalent, separate pair
of `mul` and `add` instructions.
https://llvm.org/docs/LangRef.html#llvm-fmuladd-intrinsic
MIRI support is included for f32 and f64.
The codegen_cranelift uses the `fma` function from libc, which is a
correct implementation, but without the desired performance semantic. I
think this requires an update to cranelift to expose a suitable
instruction in its IR.
I have not tested with codegen_gcc, but it should behave the same
way (using `fma` from libc).
2024-01-06 04:04:41 +00:00
|
|
|
ifn!("llvm.fmuladd.f16", fn(t_f16, t_f16, t_f16) -> t_f16);
|
|
|
|
ifn!("llvm.fmuladd.f32", fn(t_f32, t_f32, t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.fmuladd.f64", fn(t_f64, t_f64, t_f64) -> t_f64);
|
|
|
|
ifn!("llvm.fmuladd.f128", fn(t_f128, t_f128, t_f128) -> t_f128);
|
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.fabs.f16", fn(t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.fabs.f32", fn(t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.fabs.f64", fn(t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.fabs.f128", fn(t_f128) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.minnum.f16", fn(t_f16, t_f16) -> t_f16);
|
2019-06-06 20:27:23 +00:00
|
|
|
ifn!("llvm.minnum.f32", fn(t_f32, t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.minnum.f64", fn(t_f64, t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.minnum.f128", fn(t_f128, t_f128) -> t_f128);
|
|
|
|
|
|
|
|
ifn!("llvm.maxnum.f16", fn(t_f16, t_f16) -> t_f16);
|
2019-06-06 20:27:23 +00:00
|
|
|
ifn!("llvm.maxnum.f32", fn(t_f32, t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.maxnum.f64", fn(t_f64, t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.maxnum.f128", fn(t_f128, t_f128) -> t_f128);
|
2019-06-06 20:27:23 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.floor.f16", fn(t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.floor.f32", fn(t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.floor.f64", fn(t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.floor.f128", fn(t_f128) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.ceil.f16", fn(t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.ceil.f32", fn(t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.ceil.f64", fn(t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.ceil.f128", fn(t_f128) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.trunc.f16", fn(t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.trunc.f32", fn(t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.trunc.f64", fn(t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.trunc.f128", fn(t_f128) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.copysign.f16", fn(t_f16, t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.copysign.f32", fn(t_f32, t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.copysign.f64", fn(t_f64, t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.copysign.f128", fn(t_f128, t_f128) -> t_f128);
|
2022-03-25 06:44:16 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.round.f16", fn(t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.round.f32", fn(t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.round.f64", fn(t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.round.f128", fn(t_f128) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.roundeven.f16", fn(t_f16) -> t_f16);
|
2022-03-25 06:44:16 +00:00
|
|
|
ifn!("llvm.roundeven.f32", fn(t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.roundeven.f64", fn(t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.roundeven.f128", fn(t_f128) -> t_f128);
|
2022-03-25 06:44:16 +00:00
|
|
|
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.rint.f16", fn(t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.rint.f32", fn(t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.rint.f64", fn(t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.rint.f128", fn(t_f128) -> t_f128);
|
|
|
|
|
|
|
|
ifn!("llvm.nearbyint.f16", fn(t_f16) -> t_f16);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.nearbyint.f32", fn(t_f32) -> t_f32);
|
|
|
|
ifn!("llvm.nearbyint.f64", fn(t_f64) -> t_f64);
|
2024-03-01 08:53:26 +00:00
|
|
|
ifn!("llvm.nearbyint.f128", fn(t_f128) -> t_f128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
|
|
|
ifn!("llvm.ctpop.i8", fn(t_i8) -> t_i8);
|
|
|
|
ifn!("llvm.ctpop.i16", fn(t_i16) -> t_i16);
|
|
|
|
ifn!("llvm.ctpop.i32", fn(t_i32) -> t_i32);
|
|
|
|
ifn!("llvm.ctpop.i64", fn(t_i64) -> t_i64);
|
|
|
|
ifn!("llvm.ctpop.i128", fn(t_i128) -> t_i128);
|
|
|
|
|
|
|
|
ifn!("llvm.ctlz.i8", fn(t_i8, i1) -> t_i8);
|
|
|
|
ifn!("llvm.ctlz.i16", fn(t_i16, i1) -> t_i16);
|
|
|
|
ifn!("llvm.ctlz.i32", fn(t_i32, i1) -> t_i32);
|
|
|
|
ifn!("llvm.ctlz.i64", fn(t_i64, i1) -> t_i64);
|
|
|
|
ifn!("llvm.ctlz.i128", fn(t_i128, i1) -> t_i128);
|
|
|
|
|
|
|
|
ifn!("llvm.cttz.i8", fn(t_i8, i1) -> t_i8);
|
|
|
|
ifn!("llvm.cttz.i16", fn(t_i16, i1) -> t_i16);
|
|
|
|
ifn!("llvm.cttz.i32", fn(t_i32, i1) -> t_i32);
|
|
|
|
ifn!("llvm.cttz.i64", fn(t_i64, i1) -> t_i64);
|
|
|
|
ifn!("llvm.cttz.i128", fn(t_i128, i1) -> t_i128);
|
|
|
|
|
|
|
|
ifn!("llvm.bswap.i16", fn(t_i16) -> t_i16);
|
|
|
|
ifn!("llvm.bswap.i32", fn(t_i32) -> t_i32);
|
|
|
|
ifn!("llvm.bswap.i64", fn(t_i64) -> t_i64);
|
|
|
|
ifn!("llvm.bswap.i128", fn(t_i128) -> t_i128);
|
|
|
|
|
|
|
|
ifn!("llvm.bitreverse.i8", fn(t_i8) -> t_i8);
|
|
|
|
ifn!("llvm.bitreverse.i16", fn(t_i16) -> t_i16);
|
|
|
|
ifn!("llvm.bitreverse.i32", fn(t_i32) -> t_i32);
|
|
|
|
ifn!("llvm.bitreverse.i64", fn(t_i64) -> t_i64);
|
|
|
|
ifn!("llvm.bitreverse.i128", fn(t_i128) -> t_i128);
|
|
|
|
|
2018-10-23 23:13:33 +00:00
|
|
|
ifn!("llvm.fshl.i8", fn(t_i8, t_i8, t_i8) -> t_i8);
|
|
|
|
ifn!("llvm.fshl.i16", fn(t_i16, t_i16, t_i16) -> t_i16);
|
|
|
|
ifn!("llvm.fshl.i32", fn(t_i32, t_i32, t_i32) -> t_i32);
|
|
|
|
ifn!("llvm.fshl.i64", fn(t_i64, t_i64, t_i64) -> t_i64);
|
|
|
|
ifn!("llvm.fshl.i128", fn(t_i128, t_i128, t_i128) -> t_i128);
|
|
|
|
|
|
|
|
ifn!("llvm.fshr.i8", fn(t_i8, t_i8, t_i8) -> t_i8);
|
|
|
|
ifn!("llvm.fshr.i16", fn(t_i16, t_i16, t_i16) -> t_i16);
|
|
|
|
ifn!("llvm.fshr.i32", fn(t_i32, t_i32, t_i32) -> t_i32);
|
|
|
|
ifn!("llvm.fshr.i64", fn(t_i64, t_i64, t_i64) -> t_i64);
|
|
|
|
ifn!("llvm.fshr.i128", fn(t_i128, t_i128, t_i128) -> t_i128);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
|
|
|
ifn!("llvm.sadd.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
|
|
|
|
ifn!("llvm.sadd.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
|
|
|
|
ifn!("llvm.sadd.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
|
|
|
|
ifn!("llvm.sadd.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
|
|
|
|
ifn!("llvm.sadd.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
|
2019-12-22 22:42:04 +00:00
|
|
|
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.uadd.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
|
|
|
|
ifn!("llvm.uadd.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
|
|
|
|
ifn!("llvm.uadd.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
|
|
|
|
ifn!("llvm.uadd.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
|
|
|
|
ifn!("llvm.uadd.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
|
2019-12-22 22:42:04 +00:00
|
|
|
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.ssub.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
|
|
|
|
ifn!("llvm.ssub.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
|
|
|
|
ifn!("llvm.ssub.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
|
|
|
|
ifn!("llvm.ssub.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
|
|
|
|
ifn!("llvm.ssub.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
|
2019-12-22 22:42:04 +00:00
|
|
|
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.usub.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
|
|
|
|
ifn!("llvm.usub.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
|
|
|
|
ifn!("llvm.usub.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
|
|
|
|
ifn!("llvm.usub.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
|
|
|
|
ifn!("llvm.usub.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
|
2019-12-22 22:42:04 +00:00
|
|
|
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.smul.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
|
|
|
|
ifn!("llvm.smul.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
|
|
|
|
ifn!("llvm.smul.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
|
|
|
|
ifn!("llvm.smul.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
|
|
|
|
ifn!("llvm.smul.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
|
2019-12-22 22:42:04 +00:00
|
|
|
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.umul.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1});
|
|
|
|
ifn!("llvm.umul.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1});
|
|
|
|
ifn!("llvm.umul.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1});
|
|
|
|
ifn!("llvm.umul.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1});
|
|
|
|
ifn!("llvm.umul.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1});
|
|
|
|
|
2019-01-29 21:16:43 +00:00
|
|
|
ifn!("llvm.sadd.sat.i8", fn(t_i8, t_i8) -> t_i8);
|
|
|
|
ifn!("llvm.sadd.sat.i16", fn(t_i16, t_i16) -> t_i16);
|
|
|
|
ifn!("llvm.sadd.sat.i32", fn(t_i32, t_i32) -> t_i32);
|
|
|
|
ifn!("llvm.sadd.sat.i64", fn(t_i64, t_i64) -> t_i64);
|
|
|
|
ifn!("llvm.sadd.sat.i128", fn(t_i128, t_i128) -> t_i128);
|
|
|
|
|
|
|
|
ifn!("llvm.uadd.sat.i8", fn(t_i8, t_i8) -> t_i8);
|
|
|
|
ifn!("llvm.uadd.sat.i16", fn(t_i16, t_i16) -> t_i16);
|
|
|
|
ifn!("llvm.uadd.sat.i32", fn(t_i32, t_i32) -> t_i32);
|
|
|
|
ifn!("llvm.uadd.sat.i64", fn(t_i64, t_i64) -> t_i64);
|
|
|
|
ifn!("llvm.uadd.sat.i128", fn(t_i128, t_i128) -> t_i128);
|
|
|
|
|
|
|
|
ifn!("llvm.ssub.sat.i8", fn(t_i8, t_i8) -> t_i8);
|
|
|
|
ifn!("llvm.ssub.sat.i16", fn(t_i16, t_i16) -> t_i16);
|
|
|
|
ifn!("llvm.ssub.sat.i32", fn(t_i32, t_i32) -> t_i32);
|
|
|
|
ifn!("llvm.ssub.sat.i64", fn(t_i64, t_i64) -> t_i64);
|
|
|
|
ifn!("llvm.ssub.sat.i128", fn(t_i128, t_i128) -> t_i128);
|
|
|
|
|
|
|
|
ifn!("llvm.usub.sat.i8", fn(t_i8, t_i8) -> t_i8);
|
|
|
|
ifn!("llvm.usub.sat.i16", fn(t_i16, t_i16) -> t_i16);
|
|
|
|
ifn!("llvm.usub.sat.i32", fn(t_i32, t_i32) -> t_i32);
|
|
|
|
ifn!("llvm.usub.sat.i64", fn(t_i64, t_i64) -> t_i64);
|
|
|
|
ifn!("llvm.usub.sat.i128", fn(t_i128, t_i128) -> t_i128);
|
|
|
|
|
2022-12-06 05:07:28 +00:00
|
|
|
ifn!("llvm.lifetime.start.p0i8", fn(t_i64, ptr) -> void);
|
|
|
|
ifn!("llvm.lifetime.end.p0i8", fn(t_i64, ptr) -> void);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2023-08-01 15:35:12 +00:00
|
|
|
// FIXME: This is an infinitesimally small portion of the types you can
|
|
|
|
// pass to this intrinsic, if we can ever lazily register intrinsics we
|
|
|
|
// should register these when they're used, that way any type can be
|
|
|
|
// passed.
|
|
|
|
ifn!("llvm.is.constant.i1", fn(i1) -> i1);
|
|
|
|
ifn!("llvm.is.constant.i8", fn(t_i8) -> i1);
|
|
|
|
ifn!("llvm.is.constant.i16", fn(t_i16) -> i1);
|
|
|
|
ifn!("llvm.is.constant.i32", fn(t_i32) -> i1);
|
|
|
|
ifn!("llvm.is.constant.i64", fn(t_i64) -> i1);
|
|
|
|
ifn!("llvm.is.constant.i128", fn(t_i128) -> i1);
|
|
|
|
ifn!("llvm.is.constant.isize", fn(t_isize) -> i1);
|
2024-08-16 22:42:10 +00:00
|
|
|
ifn!("llvm.is.constant.f16", fn(t_f16) -> i1);
|
2023-08-01 15:35:12 +00:00
|
|
|
ifn!("llvm.is.constant.f32", fn(t_f32) -> i1);
|
|
|
|
ifn!("llvm.is.constant.f64", fn(t_f64) -> i1);
|
2024-08-16 22:42:10 +00:00
|
|
|
ifn!("llvm.is.constant.f128", fn(t_f128) -> i1);
|
2024-01-30 02:01:15 +00:00
|
|
|
ifn!("llvm.is.constant.ptr", fn(ptr) -> i1);
|
2023-08-01 15:35:12 +00:00
|
|
|
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.expect.i1", fn(i1, i1) -> i1);
|
2022-12-06 05:07:28 +00:00
|
|
|
ifn!("llvm.eh.typeid.for", fn(ptr) -> t_i32);
|
2018-09-07 22:39:39 +00:00
|
|
|
ifn!("llvm.localescape", fn(...) -> void);
|
2022-12-06 05:07:28 +00:00
|
|
|
ifn!("llvm.localrecover", fn(ptr, ptr, t_i32) -> ptr);
|
|
|
|
ifn!("llvm.x86.seh.recoverfp", fn(ptr, ptr) -> ptr);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
|
|
|
ifn!("llvm.assume", fn(i1) -> void);
|
2022-12-06 05:07:28 +00:00
|
|
|
ifn!("llvm.prefetch", fn(ptr, t_i32, t_i32, t_i32) -> void);
|
2018-09-07 22:39:39 +00:00
|
|
|
|
2021-05-30 17:25:41 +00:00
|
|
|
// This isn't an "LLVM intrinsic", but LLVM's optimization passes
|
2023-08-02 19:45:52 +00:00
|
|
|
// recognize it like one (including turning it into `bcmp` sometimes)
|
|
|
|
// and we use it to implement intrinsics like `raw_eq` and `compare_bytes`
|
2022-03-22 10:43:05 +00:00
|
|
|
match self.sess().target.arch.as_ref() {
|
2022-12-06 05:07:28 +00:00
|
|
|
"avr" | "msp430" => ifn!("memcmp", fn(ptr, ptr, t_isize) -> t_i16),
|
|
|
|
_ => ifn!("memcmp", fn(ptr, ptr, t_isize) -> t_i32),
|
2021-11-11 04:14:23 +00:00
|
|
|
}
|
2021-05-30 17:25:41 +00:00
|
|
|
|
2018-10-23 23:13:33 +00:00
|
|
|
// variadic intrinsics
|
2022-12-06 05:07:28 +00:00
|
|
|
ifn!("llvm.va_start", fn(ptr) -> void);
|
|
|
|
ifn!("llvm.va_end", fn(ptr) -> void);
|
|
|
|
ifn!("llvm.va_copy", fn(ptr, ptr) -> void);
|
2018-10-23 23:13:33 +00:00
|
|
|
|
coverage bug fixes and optimization support
Adjusted LLVM codegen for code compiled with `-Zinstrument-coverage` to
address multiple, somewhat related issues.
Fixed a significant flaw in prior coverage solution: Every counter
generated a new counter variable, but there should have only been one
counter variable per function. This appears to have bloated .profraw
files significantly. (For a small program, it increased the size by
about 40%. I have not tested large programs, but there is anecdotal
evidence that profraw files were way too large. This is a good fix,
regardless, but hopefully it also addresses related issues.
Fixes: #82144
Invalid LLVM coverage data produced when compiled with -C opt-level=1
Existing tests now work up to at least `opt-level=3`. This required a
detailed analysis of the LLVM IR, comparisons with Clang C++ LLVM IR
when compiled with coverage, and a lot of trial and error with codegen
adjustments.
The biggest hurdle was figuring out how to continue to support coverage
results for unused functions and generics. Rust's coverage results have
three advantages over Clang's coverage results:
1. Rust's coverage map does not include any overlapping code regions,
making coverage counting unambiguous.
2. Rust generates coverage results (showing zero counts) for all unused
functions, including generics. (Clang does not generate coverage for
uninstantiated template functions.)
3. Rust's unused functions produce minimal stubbed functions in LLVM IR,
sufficient for including in the coverage results; while Clang must
generate the complete LLVM IR for each unused function, even though
it will never be called.
This PR removes the previous hack of attempting to inject coverage into
some other existing function instance, and generates dedicated instances
for each unused function. This change, and a few other adjustments
(similar to what is required for `-C link-dead-code`, but with lower
impact), makes it possible to support LLVM optimizations.
Fixes: #79651
Coverage report: "Unexecuted instantiation:..." for a generic function
from multiple crates
Fixed by removing the aforementioned hack. Some "Unexecuted
instantiation" notices are unavoidable, as explained in the
`used_crate.rs` test, but `-Zinstrument-coverage` has new options to
back off support for either unused generics, or all unused functions,
which avoids the notice, at the cost of less coverage of unused
functions.
Fixes: #82875
Invalid LLVM coverage data produced with crate brotli_decompressor
Fixed by disabling the LLVM function attribute that forces inlining, if
`-Z instrument-coverage` is enabled. This attribute is applied to
Rust functions with `#[inline(always)], and in some cases, the forced
inlining breaks coverage instrumentation and reports.
2021-03-15 23:32:45 +00:00
|
|
|
if self.sess().instrument_coverage() {
|
2022-12-06 05:07:28 +00:00
|
|
|
ifn!("llvm.instrprof.increment", fn(ptr, t_i64, t_i32, t_i32) -> void);
|
2024-10-25 02:25:19 +00:00
|
|
|
if crate::llvm_util::get_version() >= (19, 0, 0) {
|
|
|
|
ifn!("llvm.instrprof.mcdc.parameters", fn(ptr, t_i64, t_i32) -> void);
|
|
|
|
ifn!("llvm.instrprof.mcdc.tvbitmap.update", fn(ptr, t_i64, t_i32, ptr) -> void);
|
|
|
|
}
|
2020-06-22 06:29:08 +00:00
|
|
|
}
|
|
|
|
|
2022-12-06 05:07:28 +00:00
|
|
|
ifn!("llvm.type.test", fn(ptr, t_metadata) -> i1);
|
|
|
|
ifn!("llvm.type.checked.load", fn(ptr, t_i32, t_metadata) -> mk_struct! {ptr, i1});
|
2021-10-07 22:33:13 +00:00
|
|
|
|
2018-09-07 22:39:39 +00:00
|
|
|
if self.sess().opts.debuginfo != DebugInfo::None {
|
2022-04-21 12:58:25 +00:00
|
|
|
ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void);
|
|
|
|
ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void);
|
2018-09-07 22:39:39 +00:00
|
|
|
}
|
2022-05-11 13:52:00 +00:00
|
|
|
|
2022-12-06 05:07:28 +00:00
|
|
|
ifn!("llvm.ptrmask", fn(ptr, t_isize) -> ptr);
|
2022-05-11 13:52:00 +00:00
|
|
|
|
2020-03-20 14:03:11 +00:00
|
|
|
None
|
2014-04-09 23:56:31 +00:00
|
|
|
}
|
2020-03-21 07:50:38 +00:00
|
|
|
|
2022-05-20 23:51:09 +00:00
|
|
|
pub(crate) fn eh_catch_typeinfo(&self) -> &'ll Value {
|
2020-03-21 07:50:38 +00:00
|
|
|
if let Some(eh_catch_typeinfo) = self.eh_catch_typeinfo.get() {
|
|
|
|
return eh_catch_typeinfo;
|
|
|
|
}
|
|
|
|
let tcx = self.tcx;
|
2022-06-17 22:01:19 +00:00
|
|
|
assert!(self.sess().target.os == "emscripten");
|
2020-03-21 07:50:38 +00:00
|
|
|
let eh_catch_typeinfo = match tcx.lang_items().eh_catch_typeinfo() {
|
|
|
|
Some(def_id) => self.get_static(def_id),
|
|
|
|
_ => {
|
2022-12-06 05:07:28 +00:00
|
|
|
let ty = self.type_struct(&[self.type_ptr(), self.type_ptr()], false);
|
2020-03-21 07:50:38 +00:00
|
|
|
self.declare_global("rust_eh_catch_typeinfo", ty)
|
|
|
|
}
|
|
|
|
};
|
|
|
|
self.eh_catch_typeinfo.set(Some(eh_catch_typeinfo));
|
|
|
|
eh_catch_typeinfo
|
|
|
|
}
|
2018-08-07 15:14:40 +00:00
|
|
|
}
|
2014-05-12 17:03:34 +00:00
|
|
|
|
2021-12-14 18:49:49 +00:00
|
|
|
impl CodegenCx<'_, '_> {
|
2019-02-08 13:53:55 +00:00
|
|
|
/// Generates a new symbol name with the given prefix. This symbol name must
|
2016-10-21 16:41:20 +00:00
|
|
|
/// only be used for definitions with `internal` or `private` linkage.
|
2024-07-06 12:26:42 +00:00
|
|
|
pub(crate) fn generate_local_symbol_name(&self, prefix: &str) -> String {
|
2018-01-05 04:14:44 +00:00
|
|
|
let idx = self.local_gen_sym_counter.get();
|
|
|
|
self.local_gen_sym_counter.set(idx + 1);
|
2016-10-21 16:41:20 +00:00
|
|
|
// Include a '.' character, so there can be no accidental conflicts with
|
|
|
|
// user defined names
|
2016-11-04 21:37:42 +00:00
|
|
|
let mut name = String::with_capacity(prefix.len() + 6);
|
|
|
|
name.push_str(prefix);
|
2020-10-27 01:02:48 +00:00
|
|
|
name.push('.');
|
2024-04-03 18:09:21 +00:00
|
|
|
name.push_str(&(idx as u64).to_base(ALPHANUMERIC_ONLY));
|
2016-11-04 21:37:42 +00:00
|
|
|
name
|
2016-10-21 16:41:20 +00:00
|
|
|
}
|
2024-09-19 10:52:09 +00:00
|
|
|
|
|
|
|
/// A wrapper for [`llvm::LLVMSetMetadata`], but it takes `Metadata` as a parameter instead of `Value`.
|
2024-09-19 10:56:02 +00:00
|
|
|
pub(crate) fn set_metadata<'a>(&self, val: &'a Value, kind_id: MetadataType, md: &'a Metadata) {
|
2024-09-19 10:52:09 +00:00
|
|
|
unsafe {
|
|
|
|
let node = llvm::LLVMMetadataAsValue(&self.llcx, md);
|
2024-09-19 10:56:02 +00:00
|
|
|
llvm::LLVMSetMetadata(val, kind_id as c_uint, node);
|
2024-09-19 10:52:09 +00:00
|
|
|
}
|
|
|
|
}
|
2017-09-12 21:33:56 +00:00
|
|
|
}
|
|
|
|
|
2021-12-14 18:49:49 +00:00
|
|
|
impl HasDataLayout for CodegenCx<'_, '_> {
|
2021-06-01 00:00:00 +00:00
|
|
|
#[inline]
|
2020-03-31 16:16:47 +00:00
|
|
|
fn data_layout(&self) -> &TargetDataLayout {
|
2018-01-05 04:14:44 +00:00
|
|
|
&self.tcx.data_layout
|
2017-03-10 04:25:51 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-12-14 18:49:49 +00:00
|
|
|
impl HasTargetSpec for CodegenCx<'_, '_> {
|
2021-06-01 00:00:00 +00:00
|
|
|
#[inline]
|
2018-04-18 13:01:26 +00:00
|
|
|
fn target_spec(&self) -> &Target {
|
2020-10-15 09:44:00 +00:00
|
|
|
&self.tcx.sess.target
|
2018-04-18 13:01:26 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-12-14 18:49:49 +00:00
|
|
|
impl<'tcx> ty::layout::HasTyCtxt<'tcx> for CodegenCx<'_, 'tcx> {
|
2021-06-01 00:00:00 +00:00
|
|
|
#[inline]
|
2019-06-13 21:48:52 +00:00
|
|
|
fn tcx(&self) -> TyCtxt<'tcx> {
|
2018-01-05 04:14:44 +00:00
|
|
|
self.tcx
|
2017-05-19 21:27:25 +00:00
|
|
|
}
|
2017-09-12 21:33:56 +00:00
|
|
|
}
|
2017-04-20 00:14:39 +00:00
|
|
|
|
2021-09-01 11:19:49 +00:00
|
|
|
impl<'tcx, 'll> HasParamEnv<'tcx> for CodegenCx<'ll, 'tcx> {
|
|
|
|
fn param_env(&self) -> ty::ParamEnv<'tcx> {
|
|
|
|
ty::ParamEnv::reveal_all()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-12-14 18:49:49 +00:00
|
|
|
impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> {
|
2021-08-30 15:01:58 +00:00
|
|
|
#[inline]
|
|
|
|
fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
|
2023-08-04 05:28:04 +00:00
|
|
|
if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err {
|
2023-12-18 11:21:37 +00:00
|
|
|
self.tcx.dcx().emit_fatal(Spanned { span, node: err.into_diagnostic() })
|
2021-08-30 15:01:58 +00:00
|
|
|
} else {
|
2023-12-18 11:21:37 +00:00
|
|
|
self.tcx.dcx().emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err })
|
2021-08-30 15:01:58 +00:00
|
|
|
}
|
2017-03-10 04:25:51 +00:00
|
|
|
}
|
|
|
|
}
|
2021-09-01 21:09:34 +00:00
|
|
|
|
2021-12-14 18:49:49 +00:00
|
|
|
impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> {
|
2021-09-01 21:09:34 +00:00
|
|
|
#[inline]
|
|
|
|
fn handle_fn_abi_err(
|
|
|
|
&self,
|
|
|
|
err: FnAbiError<'tcx>,
|
|
|
|
span: Span,
|
2021-09-01 21:29:15 +00:00
|
|
|
fn_abi_request: FnAbiRequest<'tcx>,
|
2021-09-01 21:09:34 +00:00
|
|
|
) -> ! {
|
2024-10-08 20:46:00 +00:00
|
|
|
match err {
|
|
|
|
FnAbiError::Layout(LayoutError::SizeOverflow(_) | LayoutError::Cycle(_)) => {
|
|
|
|
self.tcx.dcx().emit_fatal(Spanned { span, node: err });
|
|
|
|
}
|
|
|
|
_ => match fn_abi_request {
|
2021-09-01 21:09:34 +00:00
|
|
|
FnAbiRequest::OfFnPtr { sig, extra_args } => {
|
2023-05-17 10:30:14 +00:00
|
|
|
span_bug!(span, "`fn_abi_of_fn_ptr({sig}, {extra_args:?})` failed: {err:?}",);
|
2021-09-01 21:09:34 +00:00
|
|
|
}
|
|
|
|
FnAbiRequest::OfInstance { instance, extra_args } => {
|
|
|
|
span_bug!(
|
|
|
|
span,
|
2023-05-17 10:30:14 +00:00
|
|
|
"`fn_abi_of_instance({instance}, {extra_args:?})` failed: {err:?}",
|
2021-09-01 21:09:34 +00:00
|
|
|
);
|
|
|
|
}
|
2024-10-08 20:46:00 +00:00
|
|
|
},
|
2021-09-01 21:09:34 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|