nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-25 16:24:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Auto merge of #125074 - bjorn3:sync_cg_clif-2024-05-13, r=bjorn3

Browse Source 
Subtree sync for rustc_codegen_cranelift

A variety of bug fixes, added support for naked functions, a couple more vendor intrinsics implemented.

r? `@ghost`

`@rustbot` label +A-codegen +A-cranelift +T-compiler
This commit is contained in:
bors 2024-05-13 15:30:18 +00:00
commit 030a12ce2b
31 changed files with 481 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
compiler/rustc_codegen_cranelift/.github/workflows/abi-cafe.yml vendored
View File

@ -51,6 +51,14 @@ jobs:
if: matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
run: rustup set default-host x86_64-pc-windows-gnu
- name: Use x86_64 compiler on macOS
if: matrix.os == 'macos-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-apple-darwin'
run: rustup set default-host x86_64-apple-darwin
- name: Select XCode version
if: matrix.os == 'macos-latest'
run: sudo xcode-select -s /Applications/Xcode_14.3.1.app
- name: Prepare dependencies
run: ./y.sh prepare

16
compiler/rustc_codegen_cranelift/.github/workflows/main.yml vendored
View File

@ -98,12 +98,20 @@ jobs:
if: matrix.os == 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
run: rustup set default-host x86_64-pc-windows-gnu
- name: Use x86_64 compiler on macOS
if: matrix.os == 'macos-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-apple-darwin'
run: rustup set default-host x86_64-apple-darwin
- name: Install toolchain and emulator
if: matrix.apt_deps != null
run: |
sudo apt-get update
sudo apt-get install -y ${{ matrix.apt_deps }}
- name: Select XCode version
if: matrix.os == 'macos-latest'
run: sudo xcode-select -s /Applications/Xcode_14.3.1.app
- name: Prepare dependencies
run: ./y.sh prepare
@ -230,12 +238,20 @@ jobs:
if: matrix.os == 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
run: rustup set default-host x86_64-pc-windows-gnu
- name: Use x86_64 compiler on macOS
if: matrix.os == 'macos-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-apple-darwin'
run: rustup set default-host x86_64-apple-darwin
- name: Install MinGW toolchain
if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu'
run: |
sudo apt-get update
sudo apt-get install -y gcc-mingw-w64-x86-64
- name: Select XCode version
if: matrix.os == 'macos-latest'
run: sudo xcode-select -s /Applications/Xcode_14.3.1.app
- name: Prepare dependencies
run: ./y.sh prepare

4
compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml vendored
View File

@ -20,7 +20,7 @@ jobs:
uses: actions/cache@v4
with:
path: build/cg_clif
key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
key: ${{ runner.os }}-rustc-test-cargo-build-target-${{ hashFiles('rust-toolchain', 'Cargo.lock') }}
- name: Prepare dependencies
run: ./y.sh prepare
@ -43,7 +43,7 @@ jobs:
uses: actions/cache@v4
with:
path: build/cg_clif
key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }}
key: ${{ runner.os }}-rustc-test-cargo-build-target-${{ hashFiles('rust-toolchain', 'Cargo.lock') }}
- name: Install ripgrep
run: |

4
compiler/rustc_codegen_cranelift/.gitignore vendored
View File

@ -1,8 +1,4 @@
# Build artifacts during normal use
/y.bin
/y.bin.dSYM
/y.exe
/y.pdb
/download
/build
/dist

3
compiler/rustc_codegen_cranelift/build_system/Cargo.toml
View File

@ -11,3 +11,6 @@ path = "main.rs"
unstable-features = [] # for rust-analyzer
# Do not add any dependencies
[profile.dev]
debug = 1

37
compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
View File

@ -267,12 +267,16 @@ fn build_clif_sysroot_for_triple(
prefix.to_str().unwrap()
));
}
rustflags.push("-Zunstable-options".to_owned());
for (name, values) in EXTRA_CHECK_CFGS {
rustflags.push(check_cfg_arg(name, *values));
}
compiler.rustflags.extend(rustflags);
let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs);
if channel == "release" {
build_cmd.arg("--release");
}
build_cmd.arg("--features").arg("compiler-builtins-no-asm backtrace panic-unwind");
build_cmd.arg("--features").arg("backtrace panic-unwind");
build_cmd.env("CARGO_PROFILE_RELEASE_DEBUG", "true");
build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
if compiler.triple.contains("apple") {
@ -326,3 +330,34 @@ fn build_rtstartup(dirs: &Dirs, compiler: &Compiler) -> Option<SysrootTarget> {
Some(target_libs)
}
// Copied from https://github.com/rust-lang/rust/blob/4fd98a4b1b100f5329c6efae18031791f64372d2/src/bootstrap/src/utils/helpers.rs#L569-L585
/// Create a `--check-cfg` argument invocation for a given name
/// and it's values.
fn check_cfg_arg(name: &str, values: Option<&[&str]>) -> String {
// Creating a string of the values by concatenating each value:
// ',values("tvos","watchos")' or '' (nothing) when there are no values.
let next = match values {
Some(values) => {
let mut tmp = values.iter().flat_map(|val| [",", "\"", val, "\""]).collect::<String>();
tmp.insert_str(1, "values(");
tmp.push(')');
tmp
}
None => "".to_string(),
};
format!("--check-cfg=cfg({name}{next})")
}
const EXTRA_CHECK_CFGS: &[(&str, Option<&[&str]>)] = &[
("bootstrap", None),
("stdarch_intel_sde", None),
("no_fp_fmt_parse", None),
("no_global_oom_handling", None),
("no_rc", None),
("no_sync", None),
("netbsd10", None),
("backtrace_in_libstd", None),
("target_arch", Some(&["xtensa"])),
];

8
compiler/rustc_codegen_cranelift/build_system/main.rs
View File

@ -147,9 +147,11 @@ fn main() {
let rustup_toolchain_name = match (env::var("CARGO"), env::var("RUSTC"), env::var("RUSTDOC")) {
(Ok(_), Ok(_), Ok(_)) => None,
(Err(_), Err(_), Err(_)) => Some(rustc_info::get_toolchain_name()),
_ => {
eprintln!("All of CARGO, RUSTC and RUSTDOC need to be set or none must be set");
(_, Err(_), Err(_)) => Some(rustc_info::get_toolchain_name()),
vars => {
eprintln!(
"If RUSTC or RUSTDOC is set, both need to be set and in addition CARGO needs to be set: {vars:?}"
);
process::exit(1);
}
};

2
compiler/rustc_codegen_cranelift/build_system/tests.rs
View File

@ -77,7 +77,7 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[
),
TestCase::build_lib("build.alloc_system", "example/alloc_system.rs", "lib"),
TestCase::build_bin_and_run("aot.alloc_example", "example/alloc_example.rs", &[]),
TestCase::jit_bin("jit.std_example", "example/std_example.rs", ""),
TestCase::jit_bin("jit.std_example", "example/std_example.rs", "arg"),
TestCase::build_bin_and_run("aot.std_example", "example/std_example.rs", &["arg"]),
TestCase::build_bin_and_run("aot.dst_field_align", "example/dst-field-align.rs", &[]),
TestCase::build_bin_and_run(

10
compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs
View File

@ -4,6 +4,7 @@
never_type,
linkage,
extern_types,
naked_functions,
thread_local,
repr_simd,
raw_ref_op
@ -340,6 +341,7 @@ fn main() {
))]
unsafe {
global_asm_test();
naked_test();
}
// Both statics have a reference that points to the same anonymous allocation.
@ -395,6 +397,14 @@ global_asm! {
"
}
#[cfg(all(not(jit), not(no_unstable_features), target_arch = "x86_64"))]
#[naked]
extern "C" fn naked_test() {
unsafe {
asm!("ret", options(noreturn));
}
}
#[repr(C)]
enum c_void {
_1,

29
compiler/rustc_codegen_cranelift/example/std_example.rs
View File

@ -210,6 +210,21 @@ struct I64X2(i64, i64);
#[allow(improper_ctypes_definitions)]
extern "C" fn foo(_a: I64X2) {}
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "sse4.2")]
#[cfg(not(jit))]
unsafe fn test_crc32() {
assert!(is_x86_feature_detected!("sse4.2"));
let a = 42u32;
let b = 0xdeadbeefu64;
assert_eq!(_mm_crc32_u8(a, b as u8), 4135334616);
assert_eq!(_mm_crc32_u16(a, b as u16), 1200687288);
assert_eq!(_mm_crc32_u32(a, b as u32), 2543798776);
assert_eq!(_mm_crc32_u64(a as u64, b as u64), 241952147);
}
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "sse2")]
unsafe fn test_simd() {
@ -244,10 +259,14 @@ unsafe fn test_simd() {
test_mm256_shuffle_epi8();
test_mm256_permute2x128_si256();
test_mm256_permutevar8x32_epi32();
#[rustfmt::skip]
let mask1 = _mm_movemask_epi8(dbg!(_mm_setr_epi8(255u8 as i8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)));
assert_eq!(mask1, 1);
#[cfg(not(jit))]
test_crc32();
}
#[cfg(target_arch = "x86_64")]
@ -447,6 +466,16 @@ unsafe fn test_mm256_permute2x128_si256() {
assert_eq_m256i(r, e);
}
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx2")]
unsafe fn test_mm256_permutevar8x32_epi32() {
let a = _mm256_setr_epi32(100, 200, 300, 400, 500, 600, 700, 800);
let idx = _mm256_setr_epi32(7, 6, 5, 4, 3, 2, 1, 0);
let r = _mm256_setr_epi32(800, 700, 600, 500, 400, 300, 200, 100);
let e = _mm256_permutevar8x32_epi32(a, idx);
assert_eq_m256i(r, e);
}
fn test_checked_mul() {
let u: Option<u8> = u8::from_str_radix("1000", 10).ok();
assert_eq!(u, None);

4
compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml
View File

@ -42,9 +42,9 @@ checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"
[[package]]
name = "cc"
version = "1.0.90"
version = "1.0.97"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5"
checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4"
[[package]]
name = "cfg-if"

2
compiler/rustc_codegen_cranelift/rust-toolchain
View File

@ -1,3 +1,3 @@
[toolchain]
channel = "nightly-2024-04-23"
channel = "nightly-2024-05-13"
components = ["rust-src", "rustc-dev", "llvm-tools"]

4
compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
View File

@ -44,6 +44,7 @@ rm tests/incremental/hashes/statics.rs # same
rm tests/ui/abi/mir/mir_codegen_calls_variadic.rs # requires float varargs
rm tests/ui/abi/variadic-ffi.rs # requires callee side vararg support
rm -r tests/run-make/c-link-to-rust-va-list-fn # requires callee side vararg support
rm tests/ui/delegation/fn-header.rs
# unsized locals
rm -r tests/run-pass-valgrind/unsized-locals
@ -87,6 +88,7 @@ rm -r tests/run-make/no-builtins-attribute # same
rm tests/ui/abi/stack-protector.rs # requires stack protector support
rm -r tests/run-make/emit-stack-sizes # requires support for -Z emit-stack-sizes
rm -r tests/run-make/optimization-remarks-dir # remarks are LLVM specific
rm -r tests/run-make/print-to-output # requires --print relocation-models
# requires asm, llvm-ir and/or llvm-bc emit support
# =============================================
@ -151,7 +153,7 @@ index 9607ff02f96..b7d97caf9a2 100644
let mut cmd = setup_common();
- let target_rpath_dir = env::var_os("TARGET_RPATH_DIR").unwrap();
- cmd.arg(format!("-L{}", target_rpath_dir.to_string_lossy()));
Self { cmd }
Self { cmd, stdin: None }
}
EOF

6
compiler/rustc_codegen_cranelift/src/abi/mod.rs
View File

@ -412,7 +412,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
Err(instance) => Some(instance),
}
}
InstanceDef::DropGlue(_, None) => {
InstanceDef::DropGlue(_, None) | ty::InstanceDef::AsyncDropGlueCtorShim(_, None) => {
// empty drop glue - a nop.
let dest = target.expect("Non terminating drop_in_place_real???");
let ret_block = fx.get_block(dest);
@ -597,7 +597,9 @@ pub(crate) fn codegen_drop<'tcx>(
let ty = drop_place.layout().ty;
let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx);
if let ty::InstanceDef::DropGlue(_, None) = drop_instance.def {
if let ty::InstanceDef::DropGlue(_, None) | ty::InstanceDef::AsyncDropGlueCtorShim(_, None) =
drop_instance.def
{
// we don't actually need to drop anything
} else {
match ty.kind() {

37
compiler/rustc_codegen_cranelift/src/base.rs
View File

@ -6,6 +6,7 @@ use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
use cranelift_module::ModuleError;
use rustc_ast::InlineAsmOptions;
use rustc_index::IndexVec;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::ty::adjustment::PointerCoercion;
use rustc_middle::ty::layout::FnAbiOf;
use rustc_middle::ty::print::with_no_trimmed_paths;
@ -14,6 +15,7 @@ use rustc_monomorphize::is_call_from_compiler_builtins_to_upstream_monomorphizat
use crate::constant::ConstantCx;
use crate::debuginfo::{FunctionDebugContext, TypeDebugContext};
use crate::inline_asm::codegen_naked_asm;
use crate::prelude::*;
use crate::pretty_clif::CommentWriter;
@ -32,7 +34,7 @@ pub(crate) fn codegen_fn<'tcx>(
cached_func: Function,
module: &mut dyn Module,
instance: Instance<'tcx>,
) -> CodegenedFunction {
) -> Option<CodegenedFunction> {
debug_assert!(!instance.args.has_infer());
let symbol_name = tcx.symbol_name(instance).name.to_string();
@ -48,6 +50,37 @@ pub(crate) fn codegen_fn<'tcx>(
String::from_utf8_lossy(&buf).into_owned()
});
if tcx.codegen_fn_attrs(instance.def_id()).flags.contains(CodegenFnAttrFlags::NAKED) {
assert_eq!(mir.basic_blocks.len(), 1);
assert!(mir.basic_blocks[START_BLOCK].statements.is_empty());
match &mir.basic_blocks[START_BLOCK].terminator().kind {
TerminatorKind::InlineAsm {
template,
operands,
options,
line_spans: _,
targets: _,
unwind: _,
} => {
codegen_naked_asm(
tcx,
cx,
module,
instance,
mir.basic_blocks[START_BLOCK].terminator().source_info.span,
&symbol_name,
template,
operands,
*options,
);
}
_ => unreachable!(),
}
return None;
}
// Declare function
let sig = get_function_sig(tcx, module.target_config().default_call_conv, instance);
let func_id = module.declare_function(&symbol_name, Linkage::Local, &sig).unwrap();
@ -128,7 +161,7 @@ pub(crate) fn codegen_fn<'tcx>(
// Verify function
verify_func(tcx, &clif_comments, &func);
CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx }
Some(CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx })
}
pub(crate) fn compile_fn(

21
compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs
View File

@ -6,7 +6,7 @@ use rustc_session::Session;
// FIXME don't panic when a worker thread panics
pub(super) struct ConcurrencyLimiter {
helper_thread: Option<HelperThread>,
helper_thread: Option<Mutex<HelperThread>>,
state: Arc<Mutex<state::ConcurrencyLimiterState>>,
available_token_condvar: Arc<Condvar>,
finished: bool,
@ -39,14 +39,14 @@ impl ConcurrencyLimiter {
})
.unwrap();
ConcurrencyLimiter {
helper_thread: Some(helper_thread),
helper_thread: Some(Mutex::new(helper_thread)),
state,
available_token_condvar,
finished: false,
}
}
pub(super) fn acquire(&mut self, dcx: &rustc_errors::DiagCtxt) -> ConcurrencyLimiterToken {
pub(super) fn acquire(&self, dcx: &rustc_errors::DiagCtxt) -> ConcurrencyLimiterToken {
let mut state = self.state.lock().unwrap();
loop {
state.assert_invariants();
@ -73,16 +73,11 @@ impl ConcurrencyLimiter {
}
}
self.helper_thread.as_mut().unwrap().request_token();
self.helper_thread.as_ref().unwrap().lock().unwrap().request_token();
state = self.available_token_condvar.wait(state).unwrap();
}
}
pub(super) fn job_already_done(&mut self) {
let mut state = self.state.lock().unwrap();
state.job_already_done();
}
pub(crate) fn finished(mut self) {
self.helper_thread.take();
@ -190,14 +185,6 @@ mod state {
self.assert_invariants();
}
pub(super) fn job_already_done(&mut self) {
self.assert_invariants();
self.pending_jobs -= 1;
self.assert_invariants();
self.drop_excess_capacity();
self.assert_invariants();
}
pub(super) fn poison(&mut self, error: String) {
self.poisoned = true;
self.stored_error = Some(error);

9
compiler/rustc_codegen_cranelift/src/config.rs
View File

@ -64,8 +64,13 @@ impl Default for BackendConfig {
BackendConfig {
codegen_mode: CodegenMode::Aot,
jit_args: {
let args = std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new());
args.split(' ').map(|arg| arg.to_string()).collect()
match std::env::var("CG_CLIF_JIT_ARGS") {
Ok(args) => args.split(' ').map(|arg| arg.to_string()).collect(),
Err(std::env::VarError::NotPresent) => vec![],
Err(std::env::VarError::NotUnicode(s)) => {
panic!("CG_CLIF_JIT_ARGS not unicode: {:?}", s);
}
}
},
enable_verifier: cfg!(debug_assertions) || bool_env_var("CG_CLIF_ENABLE_VERIFIER"),
disable_incr_cache: bool_env_var("CG_CLIF_DISABLE_INCR_CACHE"),

2
compiler/rustc_codegen_cranelift/src/constant.rs
View File

@ -258,7 +258,7 @@ fn data_id_for_static(
) -> DataId {
let attrs = tcx.codegen_fn_attrs(def_id);
let instance = Instance::mono(tcx, def_id).polymorphize(tcx);
let instance = Instance::mono(tcx, def_id);
let symbol_name = tcx.symbol_name(instance).name;
if let Some(import_linkage) = attrs.import_linkage {

45
compiler/rustc_codegen_cranelift/src/discriminant.rs
View File

@ -28,16 +28,20 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
} => {
let ptr = place.place_field(fx, FieldIdx::new(tag_field));
let to = layout.ty.discriminant_for_variant(fx.tcx, variant_index).unwrap().val;
let to = if ptr.layout().abi.is_signed() {
ty::ScalarInt::try_from_int(
ptr.layout().size.sign_extend(to) as i128,
ptr.layout().size,
)
.unwrap()
} else {
ty::ScalarInt::try_from_uint(to, ptr.layout().size).unwrap()
let to = match ptr.layout().ty.kind() {
ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
let lsb = fx.bcx.ins().iconst(types::I64, to as u64 as i64);
let msb = fx.bcx.ins().iconst(types::I64, (to >> 64) as u64 as i64);
fx.bcx.ins().iconcat(lsb, msb)
}
ty::Uint(_) | ty::Int(_) => {
let clif_ty = fx.clif_type(ptr.layout().ty).unwrap();
let raw_val = ptr.layout().size.truncate(to);
fx.bcx.ins().iconst(clif_ty, raw_val as i64)
}
_ => unreachable!(),
};
let discr = CValue::const_val(fx, ptr.layout(), to);
let discr = CValue::by_val(to, ptr.layout());
ptr.write_cvalue(fx, discr);
}
Variants::Multiple {
@ -85,16 +89,21 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
.ty
.discriminant_for_variant(fx.tcx, *index)
.map_or(u128::from(index.as_u32()), |discr| discr.val);
let discr_val = if dest_layout.abi.is_signed() {
ty::ScalarInt::try_from_int(
dest_layout.size.sign_extend(discr_val) as i128,
dest_layout.size,
)
.unwrap()
} else {
ty::ScalarInt::try_from_uint(discr_val, dest_layout.size).unwrap()
let val = match dest_layout.ty.kind() {
ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
let lsb = fx.bcx.ins().iconst(types::I64, discr_val as u64 as i64);
let msb = fx.bcx.ins().iconst(types::I64, (discr_val >> 64) as u64 as i64);
fx.bcx.ins().iconcat(lsb, msb)
}
ty::Uint(_) | ty::Int(_) => {
let clif_ty = fx.clif_type(dest_layout.ty).unwrap();
let raw_val = dest_layout.size.truncate(discr_val);
fx.bcx.ins().iconst(clif_ty, raw_val as i64)
}
_ => unreachable!(),
};
let res = CValue::const_val(fx, dest_layout, discr_val);
let res = CValue::by_val(val, dest_layout);
dest.write_cvalue(fx, res);
return;
}

72
compiler/rustc_codegen_cranelift/src/driver/aot.rs
View File

@ -15,6 +15,7 @@ use rustc_codegen_ssa::errors as ssa_errors;
use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind};
use rustc_data_structures::profiling::SelfProfilerRef;
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_data_structures::sync::{par_map, IntoDynSyncSend};
use rustc_metadata::fs::copy_to_stdout;
use rustc_metadata::EncodedMetadata;
use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
@ -481,15 +482,16 @@ fn module_codegen(
for (mono_item, _) in mono_items {
match mono_item {
MonoItem::Fn(inst) => {
let codegened_function = crate::base::codegen_fn(
if let Some(codegened_function) = crate::base::codegen_fn(
tcx,
&mut cx,
&mut type_dbg,
Function::new(),
&mut module,
inst,
);
codegened_functions.push(codegened_function);
) {
codegened_functions.push(codegened_function);
}
}
MonoItem::Static(def_id) => {
let data_id = crate::constant::codegen_static(tcx, &mut module, def_id);
@ -604,39 +606,39 @@ pub(crate) fn run_aot(
let global_asm_config = Arc::new(crate::global_asm::GlobalAsmConfig::new(tcx));
let mut concurrency_limiter = ConcurrencyLimiter::new(tcx.sess, cgus.len());
let (todo_cgus, done_cgus) =
cgus.into_iter().enumerate().partition::<Vec<_>, _>(|&(i, _)| match cgu_reuse[i] {
_ if backend_config.disable_incr_cache => true,
CguReuse::No => true,
CguReuse::PreLto | CguReuse::PostLto => false,
});
let concurrency_limiter = IntoDynSyncSend(ConcurrencyLimiter::new(tcx.sess, todo_cgus.len()));
let modules = tcx.sess.time("codegen mono items", || {
cgus.iter()
.enumerate()
.map(|(i, cgu)| {
let cgu_reuse =
if backend_config.disable_incr_cache { CguReuse::No } else { cgu_reuse[i] };
match cgu_reuse {
CguReuse::No => {
let dep_node = cgu.codegen_dep_node(tcx);
tcx.dep_graph
.with_task(
dep_node,
tcx,
(
backend_config.clone(),
global_asm_config.clone(),
cgu.name(),
concurrency_limiter.acquire(tcx.dcx()),
),
module_codegen,
Some(rustc_middle::dep_graph::hash_result),
)
.0
}
CguReuse::PreLto | CguReuse::PostLto => {
concurrency_limiter.job_already_done();
OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, cgu))
}
}
})
.collect::<Vec<_>>()
let mut modules: Vec<_> = par_map(todo_cgus, |(_, cgu)| {
let dep_node = cgu.codegen_dep_node(tcx);
tcx.dep_graph
.with_task(
dep_node,
tcx,
(
backend_config.clone(),
global_asm_config.clone(),
cgu.name(),
concurrency_limiter.acquire(tcx.dcx()),
),
module_codegen,
Some(rustc_middle::dep_graph::hash_result),
)
.0
});
modules.extend(
done_cgus
.into_iter()
.map(|(_, cgu)| OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, cgu))),
);
modules
});
let mut allocator_module = make_module(tcx.sess, &backend_config, "allocator_shim".to_string());
@ -705,6 +707,6 @@ pub(crate) fn run_aot(
metadata_module,
metadata,
crate_info: CrateInfo::new(tcx, target_cpu),
concurrency_limiter,
concurrency_limiter: concurrency_limiter.0,
})
}

23
compiler/rustc_codegen_cranelift/src/driver/jit.rs
View File

@ -83,13 +83,6 @@ fn create_jit_module(
);
crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context);
crate::main_shim::maybe_create_entry_wrapper(
tcx,
&mut jit_module,
&mut cx.unwind_context,
true,
true,
);
(jit_module, cx)
}
@ -153,6 +146,14 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
tcx.dcx().fatal("Inline asm is not supported in JIT mode");
}
crate::main_shim::maybe_create_entry_wrapper(
tcx,
&mut jit_module,
&mut cx.unwind_context,
true,
true,
);
tcx.dcx().abort_if_errors();
jit_module.finalize_definitions().unwrap();
@ -231,16 +232,16 @@ pub(crate) fn codegen_and_compile_fn<'tcx>(
crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name));
let cached_func = std::mem::replace(&mut cached_context.func, Function::new());
let codegened_func = crate::base::codegen_fn(
if let Some(codegened_func) = crate::base::codegen_fn(
tcx,
cx,
&mut TypeDebugContext::default(),
cached_func,
module,
instance,
);
crate::base::compile_fn(cx, cached_context, module, codegened_func);
) {
crate::base::compile_fn(cx, cached_context, module, codegened_func);
}
});
}

16
compiler/rustc_codegen_cranelift/src/driver/mod.rs
View File

@ -5,6 +5,7 @@
//! [`codegen_static`]: crate::constant::codegen_static
use rustc_data_structures::profiling::SelfProfilerRef;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::mir::mono::{MonoItem, MonoItemData};
use crate::prelude::*;
@ -33,7 +34,20 @@ fn predefine_mono_items<'tcx>(
data.visibility,
is_compiler_builtins,
);
module.declare_function(name, linkage, &sig).unwrap();
let is_naked = tcx
.codegen_fn_attrs(instance.def_id())
.flags
.contains(CodegenFnAttrFlags::NAKED);
module
.declare_function(
name,
// Naked functions are defined in a separate object
// file from the codegen unit rustc expects them to
// be defined in.
if is_naked { Linkage::Import } else { linkage },
&sig,
)
.unwrap();
}
MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {}
}

2
compiler/rustc_codegen_cranelift/src/global_asm.rs
View File

@ -81,7 +81,7 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String,
);
}
let instance = Instance::mono(tcx, def_id).polymorphize(tcx);
let instance = Instance::mono(tcx, def_id);
let symbol = tcx.symbol_name(instance);
global_asm.push_str(symbol.name);
}

201
compiler/rustc_codegen_cranelift/src/inline_asm.rs
View File

@ -127,7 +127,7 @@ pub(crate) fn codegen_inline_asm_terminator<'tcx>(
}
InlineAsmOperand::SymStatic { def_id } => {
assert!(fx.tcx.is_static(def_id));
let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
let instance = Instance::mono(fx.tcx, def_id);
CInlineAsmOperand::Symbol { symbol: fx.tcx.symbol_name(instance).name.to_owned() }
}
InlineAsmOperand::Label { .. } => {
@ -169,6 +169,7 @@ pub(crate) fn codegen_inline_asm_inner<'tcx>(
stack_slots_input: Vec::new(),
stack_slots_output: Vec::new(),
stack_slot_size: Size::from_bytes(0),
is_naked: false,
};
asm_gen.allocate_registers();
asm_gen.allocate_stack_slots();
@ -209,6 +210,121 @@ pub(crate) fn codegen_inline_asm_inner<'tcx>(
call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
}
pub(crate) fn codegen_naked_asm<'tcx>(
tcx: TyCtxt<'tcx>,
cx: &mut crate::CodegenCx,
module: &mut dyn Module,
instance: Instance<'tcx>,
span: Span,
symbol_name: &str,
template: &[InlineAsmTemplatePiece],
operands: &[InlineAsmOperand<'tcx>],
options: InlineAsmOptions,
) {
// FIXME add .eh_frame unwind info directives
let operands = operands
.iter()
.map(|operand| match *operand {
InlineAsmOperand::In { .. }
| InlineAsmOperand::Out { .. }
| InlineAsmOperand::InOut { .. } => {
span_bug!(span, "invalid operand type for naked asm")
}
InlineAsmOperand::Const { ref value } => {
let cv = instance.instantiate_mir_and_normalize_erasing_regions(
tcx,
ty::ParamEnv::reveal_all(),
ty::EarlyBinder::bind(value.const_),
);
let const_value = cv
.eval(tcx, ty::ParamEnv::reveal_all(), value.span)
.expect("erroneous constant missed by mono item collection");
let value = rustc_codegen_ssa::common::asm_const_to_str(
tcx,
span,
const_value,
RevealAllLayoutCx(tcx).layout_of(cv.ty()),
);
CInlineAsmOperand::Const { value }
}
InlineAsmOperand::SymFn { ref value } => {
if cfg!(not(feature = "inline_asm_sym")) {
tcx.dcx()
.span_err(span, "asm! and global_asm! sym operands are not yet supported");
}
let const_ = instance.instantiate_mir_and_normalize_erasing_regions(
tcx,
ty::ParamEnv::reveal_all(),
ty::EarlyBinder::bind(value.const_),
);
if let ty::FnDef(def_id, args) = *const_.ty().kind() {
let instance = ty::Instance::resolve_for_fn_ptr(
tcx,
ty::ParamEnv::reveal_all(),
def_id,
args,
)
.unwrap();
let symbol = tcx.symbol_name(instance);
// Pass a wrapper rather than the function itself as the function itself may not
// be exported from the main codegen unit and may thus be unreachable from the
// object file created by an external assembler.
let inline_asm_index = cx.inline_asm_index.get();
cx.inline_asm_index.set(inline_asm_index + 1);
let wrapper_name = format!(
"__inline_asm_{}_wrapper_n{}",
cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
inline_asm_index
);
let sig =
get_function_sig(tcx, module.target_config().default_call_conv, instance);
create_wrapper_function(
module,
&mut cx.unwind_context,
sig,
&wrapper_name,
symbol.name,
);
CInlineAsmOperand::Symbol { symbol: wrapper_name }
} else {
span_bug!(span, "invalid type for asm sym (fn)");
}
}
InlineAsmOperand::SymStatic { def_id } => {
assert!(tcx.is_static(def_id));
let instance = Instance::mono(tcx, def_id);
CInlineAsmOperand::Symbol { symbol: tcx.symbol_name(instance).name.to_owned() }
}
InlineAsmOperand::Label { .. } => {
span_bug!(span, "asm! label operands are not yet supported");
}
})
.collect::<Vec<_>>();
let asm_gen = InlineAssemblyGenerator {
tcx,
arch: tcx.sess.asm_arch.unwrap(),
enclosing_def_id: instance.def_id(),
template,
operands: &operands,
options,
registers: Vec::new(),
stack_slots_clobber: Vec::new(),
stack_slots_input: Vec::new(),
stack_slots_output: Vec::new(),
stack_slot_size: Size::from_bytes(0),
is_naked: true,
};
let generated_asm = asm_gen.generate_asm_wrapper(symbol_name);
cx.global_asm.push_str(&generated_asm);
}
struct InlineAssemblyGenerator<'a, 'tcx> {
tcx: TyCtxt<'tcx>,
arch: InlineAsmArch,
@ -221,10 +337,13 @@ struct InlineAssemblyGenerator<'a, 'tcx> {
stack_slots_input: Vec<Option<Size>>,
stack_slots_output: Vec<Option<Size>>,
stack_slot_size: Size,
is_naked: bool,
}
impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
fn allocate_registers(&mut self) {
assert!(!self.is_naked);
let sess = self.tcx.sess;
let map = allocatable_registers(
self.arch,
@ -348,6 +467,8 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
}
fn allocate_stack_slots(&mut self) {
assert!(!self.is_naked);
let mut slot_size = Size::from_bytes(0);
let mut slots_clobber = vec![None; self.operands.len()];
let mut slots_input = vec![None; self.operands.len()];
@ -468,30 +589,32 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
if is_x86 {
generated_asm.push_str(".intel_syntax noprefix\n");
}
Self::prologue(&mut generated_asm, self.arch);
if !self.is_naked {
Self::prologue(&mut generated_asm, self.arch);
// Save clobbered registers
if !self.options.contains(InlineAsmOptions::NORETURN) {
// Save clobbered registers
if !self.options.contains(InlineAsmOptions::NORETURN) {
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_clobber.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::save_register(&mut generated_asm, self.arch, reg, slot);
}
}
// Write input registers
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_clobber.iter().copied())
.zip(self.stack_slots_input.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::save_register(&mut generated_asm, self.arch, reg, slot);
Self::restore_register(&mut generated_asm, self.arch, reg, slot);
}
}
// Write input registers
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_input.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::restore_register(&mut generated_asm, self.arch, reg, slot);
}
if is_x86 && self.options.contains(InlineAsmOptions::ATT_SYNTAX) {
generated_asm.push_str(".att_syntax\n");
}
@ -553,30 +676,32 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
generated_asm.push_str(".intel_syntax noprefix\n");
}
if !self.options.contains(InlineAsmOptions::NORETURN) {
// Read output registers
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_output.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::save_register(&mut generated_asm, self.arch, reg, slot);
}
if !self.is_naked {
if !self.options.contains(InlineAsmOptions::NORETURN) {
// Read output registers
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_output.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::save_register(&mut generated_asm, self.arch, reg, slot);
}
// Restore clobbered registers
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_clobber.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::restore_register(&mut generated_asm, self.arch, reg, slot);
}
// Restore clobbered registers
for (reg, slot) in self
.registers
.iter()
.zip(self.stack_slots_clobber.iter().copied())
.filter_map(|(r, s)| r.zip(s))
{
Self::restore_register(&mut generated_asm, self.arch, reg, slot);
}
Self::epilogue(&mut generated_asm, self.arch);
} else {
Self::epilogue_noreturn(&mut generated_asm, self.arch);
Self::epilogue(&mut generated_asm, self.arch);
} else {
Self::epilogue_noreturn(&mut generated_asm, self.arch);
}
}
if is_x86 {

52
compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
View File

@ -374,6 +374,21 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
}
}
}
"llvm.x86.avx2.permd" => {
// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_permutevar8x32_epi32
intrinsic_args!(fx, args => (a, idx); intrinsic);
for j in 0..=7 {
let index = idx.value_typed_lane(fx, fx.tcx.types.u32, j).load_scalar(fx);
let index = fx.bcx.ins().uextend(fx.pointer_type, index);
let value = a.value_lane_dyn(fx, index).load_scalar(fx);
ret.place_typed_lane(fx, fx.tcx.types.u32, j).to_ptr().store(
fx,
value,
MemFlags::trusted(),
);
}
}
"llvm.x86.avx2.vperm2i128"
| "llvm.x86.avx.vperm2f128.ps.256"
| "llvm.x86.avx.vperm2f128.pd.256" => {
@ -832,6 +847,43 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
}
}
"llvm.x86.sse42.crc32.32.8"
| "llvm.x86.sse42.crc32.32.16"
| "llvm.x86.sse42.crc32.32.32"
| "llvm.x86.sse42.crc32.64.64" => {
// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#ig_expand=1419&text=_mm_crc32_u32
intrinsic_args!(fx, args => (crc, v); intrinsic);
let crc = crc.load_scalar(fx);
let v = v.load_scalar(fx);
let asm = match intrinsic {
"llvm.x86.sse42.crc32.32.8" => "crc32 eax, dl",
"llvm.x86.sse42.crc32.32.16" => "crc32 eax, dx",
"llvm.x86.sse42.crc32.32.32" => "crc32 eax, edx",
"llvm.x86.sse42.crc32.64.64" => "crc32 rax, rdx",
_ => unreachable!(),
};
codegen_inline_asm_inner(
fx,
&[InlineAsmTemplatePiece::String(asm.to_string())],
&[
CInlineAsmOperand::InOut {
reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
_late: true,
in_value: crc,
out_place: Some(ret),
},
CInlineAsmOperand::In {
reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
value: v,
},
],
InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
);
}
"llvm.x86.sse42.pcmpestri128" => {
// https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpestri&ig_expand=939
intrinsic_args!(fx, args => (a, la, b, lb, _imm8); intrinsic);

6
compiler/rustc_codegen_cranelift/src/lib.rs
View File

@ -331,9 +331,9 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn TargetIs
sess.dcx().fatal(format!("can't compile for {}: {}", target_triple, err));
});
if target_triple.architecture == target_lexicon::Architecture::X86_64 {
// Don't use "haswell" as the default, as it implies `has_lzcnt`.
// macOS CI is still at Ivy Bridge EP, so `lzcnt` is interpreted as `bsr`.
builder.enable("nehalem").unwrap();
// Only set the target cpu on x86_64 as Cranelift is missing
// the target cpu list for most other targets.
builder.enable(sess.target.cpu.as_ref()).unwrap();
}
builder
}

2
compiler/rustc_codegen_cranelift/src/main_shim.rs
View File

@ -29,7 +29,7 @@ pub(crate) fn maybe_create_entry_wrapper(
if main_def_id.is_local() {
let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx);
if !is_jit && module.get_name(tcx.symbol_name(instance).name).is_none() {
if module.get_name(tcx.symbol_name(instance).name).is_none() {
return;
}
} else if !is_primary_cgu {

8
compiler/rustc_codegen_cranelift/src/value_and_place.rs
View File

@ -317,14 +317,6 @@ impl<'tcx> CValue<'tcx> {
let clif_ty = fx.clif_type(layout.ty).unwrap();
if let ty::Bool = layout.ty.kind() {
assert!(
const_val == ty::ScalarInt::FALSE || const_val == ty::ScalarInt::TRUE,
"Invalid bool 0x{:032X}",
const_val
);
}
let val = match layout.ty.kind() {
ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
let const_val = const_val.assert_bits(layout.size);

4
compiler/rustc_codegen_cranelift/y.cmd
View File

@ -1,8 +1,6 @@
@echo off
echo [BUILD] build system >&2
mkdir build 2>nul
rustc build_system/main.rs -o build\y.exe -Cdebuginfo=1 --edition 2021 || goto :error
build\y.exe %* || goto :error
cargo run --manifest-path build_system/Cargo.toml -- %* || goto :error
goto :EOF
:error

7
compiler/rustc_codegen_cranelift/y.ps1 Normal file → Executable file
View File

@ -1,12 +1,7 @@
$ErrorActionPreference = "Stop"
$host.ui.WriteErrorLine("[BUILD] build system")
New-Item -ItemType Directory -Force -Path build | Out-Null
& rustc build_system/main.rs -o build\y.exe -Cdebuginfo=1 --edition 2021
if ($LASTEXITCODE -ne 0) {
exit $LASTEXITCODE
}
& build\y.exe $args
& cargo run --manifest-path build_system/Cargo.toml -- $args
if ($LASTEXITCODE -ne 0) {
exit $LASTEXITCODE
}

3
compiler/rustc_codegen_cranelift/y.sh
View File

@ -2,5 +2,4 @@
set -e
echo "[BUILD] build system" 1>&2
rustc build_system/main.rs -o y.bin -Cdebuginfo=1 --edition 2021
exec ./y.bin "$@"
exec cargo run --manifest-path build_system/Cargo.toml -- "$@"