Likely unlikely fix

This commit is contained in:
Jiri Bobek 2024-07-26 15:51:46 +02:00
parent f2a35426b6
commit 777003ae9f
22 changed files with 256 additions and 73 deletions

View File

@ -453,11 +453,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
fx.bcx.ins().trap(TrapCode::user(2).unwrap()); fx.bcx.ins().trap(TrapCode::user(2).unwrap());
return Ok(()); return Ok(());
} }
sym::likely | sym::unlikely => {
intrinsic_args!(fx, args => (a); intrinsic);
ret.write_cvalue(fx, a);
}
sym::breakpoint => { sym::breakpoint => {
intrinsic_args!(fx, args => (); intrinsic); intrinsic_args!(fx, args => (); intrinsic);
@ -1267,6 +1262,10 @@ fn codegen_regular_intrinsic_call<'tcx>(
); );
} }
sym::cold_path => {
// This is a no-op. The intrinsic is just a hint to the optimizer.
}
// Unimplemented intrinsics must have a fallback body. The fallback body is obtained // Unimplemented intrinsics must have a fallback body. The fallback body is obtained
// by converting the `InstanceKind::Intrinsic` to an `InstanceKind::Item`. // by converting the `InstanceKind::Intrinsic` to an `InstanceKind::Item`.
_ => { _ => {

View File

@ -139,8 +139,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
&args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
) )
} }
sym::likely => self.expect(args[0].immediate(), true),
sym::unlikely => self.expect(args[0].immediate(), false),
sym::is_val_statically_known => { sym::is_val_statically_known => {
let a = args[0].immediate(); let a = args[0].immediate();
let builtin = self.context.get_builtin_function("__builtin_constant_p"); let builtin = self.context.get_builtin_function("__builtin_constant_p");

View File

@ -192,7 +192,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
Some(instance), Some(instance),
) )
} }
sym::likely => self.expect(args[0].immediate(), true),
sym::is_val_statically_known => { sym::is_val_statically_known => {
let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx); let intrinsic_type = args[0].layout.immediate_llvm_type(self.cx);
let kind = self.type_kind(intrinsic_type); let kind = self.type_kind(intrinsic_type);
@ -213,7 +212,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
self.const_bool(false) self.const_bool(false)
} }
} }
sym::unlikely => self.expect(args[0].immediate(), false),
sym::select_unpredictable => { sym::select_unpredictable => {
let cond = args[0].immediate(); let cond = args[0].immediate();
assert_eq!(args[1].layout, args[2].layout); assert_eq!(args[1].layout, args[2].layout);

View File

@ -377,20 +377,32 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
// If there are two targets (one conditional, one fallback), emit `br` instead of // If there are two targets (one conditional, one fallback), emit `br` instead of
// `switch`. // `switch`.
let (test_value, target) = target_iter.next().unwrap(); let (test_value, target) = target_iter.next().unwrap();
let lltrue = helper.llbb_with_cleanup(self, target); let otherwise = targets.otherwise();
let llfalse = helper.llbb_with_cleanup(self, targets.otherwise()); let lltarget = helper.llbb_with_cleanup(self, target);
let llotherwise = helper.llbb_with_cleanup(self, otherwise);
let target_cold = self.cold_blocks[target];
let otherwise_cold = self.cold_blocks[otherwise];
// If `target_cold == otherwise_cold`, the branches have the same weight
// so there is no expectation. If they differ, the `target` branch is expected
// when the `otherwise` branch is cold.
let expect = if target_cold == otherwise_cold { None } else { Some(otherwise_cold) };
if switch_ty == bx.tcx().types.bool { if switch_ty == bx.tcx().types.bool {
// Don't generate trivial icmps when switching on bool. // Don't generate trivial icmps when switching on bool.
match test_value { match test_value {
0 => bx.cond_br(discr_value, llfalse, lltrue), 0 => {
1 => bx.cond_br(discr_value, lltrue, llfalse), let expect = expect.map(|e| !e);
bx.cond_br_with_expect(discr_value, llotherwise, lltarget, expect);
}
1 => {
bx.cond_br_with_expect(discr_value, lltarget, llotherwise, expect);
}
_ => bug!(), _ => bug!(),
} }
} else { } else {
let switch_llty = bx.immediate_backend_type(bx.layout_of(switch_ty)); let switch_llty = bx.immediate_backend_type(bx.layout_of(switch_ty));
let llval = bx.const_uint_big(switch_llty, test_value); let llval = bx.const_uint_big(switch_llty, test_value);
let cmp = bx.icmp(IntPredicate::IntEQ, discr_value, llval); let cmp = bx.icmp(IntPredicate::IntEQ, discr_value, llval);
bx.cond_br(cmp, lltrue, llfalse); bx.cond_br_with_expect(cmp, lltarget, llotherwise, expect);
} }
} else if self.cx.sess().opts.optimize == OptLevel::No } else if self.cx.sess().opts.optimize == OptLevel::No
&& target_iter.len() == 2 && target_iter.len() == 2

View File

@ -498,6 +498,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
} }
} }
sym::cold_path => {
// This is a no-op. The intrinsic is just a hint to the optimizer.
return Ok(());
}
_ => { _ => {
// Need to use backend-specific things in the implementation. // Need to use backend-specific things in the implementation.
return bx.codegen_intrinsic_call(instance, fn_abi, args, llresult, span); return bx.codegen_intrinsic_call(instance, fn_abi, args, llresult, span);

View File

@ -91,6 +91,10 @@ pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
/// Cached terminate upon unwinding block and its reason /// Cached terminate upon unwinding block and its reason
terminate_block: Option<(Bx::BasicBlock, UnwindTerminateReason)>, terminate_block: Option<(Bx::BasicBlock, UnwindTerminateReason)>,
/// A bool flag for each basic block indicating whether it is a cold block.
/// A cold block is a block that is unlikely to be executed at runtime.
cold_blocks: IndexVec<mir::BasicBlock, bool>,
/// The location where each MIR arg/var/tmp/ret is stored. This is /// The location where each MIR arg/var/tmp/ret is stored. This is
/// usually an `PlaceRef` representing an alloca, but not always: /// usually an `PlaceRef` representing an alloca, but not always:
/// sometimes we can skip the alloca and just store the value /// sometimes we can skip the alloca and just store the value
@ -207,6 +211,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
cleanup_kinds, cleanup_kinds,
landing_pads: IndexVec::from_elem(None, &mir.basic_blocks), landing_pads: IndexVec::from_elem(None, &mir.basic_blocks),
funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks.len()), funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks.len()),
cold_blocks: find_cold_blocks(cx.tcx(), mir),
locals: locals::Locals::empty(), locals: locals::Locals::empty(),
debug_context, debug_context,
per_local_var_debug_info: None, per_local_var_debug_info: None,
@ -477,3 +482,39 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
args args
} }
fn find_cold_blocks<'tcx>(
tcx: TyCtxt<'tcx>,
mir: &mir::Body<'tcx>,
) -> IndexVec<mir::BasicBlock, bool> {
let local_decls = &mir.local_decls;
let mut cold_blocks: IndexVec<mir::BasicBlock, bool> =
IndexVec::from_elem(false, &mir.basic_blocks);
// Traverse all basic blocks from end of the function to the start.
for (bb, bb_data) in traversal::postorder(mir) {
let terminator = bb_data.terminator();
// If a BB ends with a call to a cold function, mark it as cold.
if let mir::TerminatorKind::Call { ref func, .. } = terminator.kind
&& let ty::FnDef(def_id, ..) = *func.ty(local_decls, tcx).kind()
&& let attrs = tcx.codegen_fn_attrs(def_id)
&& attrs.flags.contains(CodegenFnAttrFlags::COLD)
{
cold_blocks[bb] = true;
continue;
}
// If all successors of a BB are cold and there's at least one of them, mark this BB as cold
let mut succ = terminator.successors();
if let Some(first) = succ.next()
&& cold_blocks[first]
&& succ.all(|s| cold_blocks[s])
{
cold_blocks[bb] = true;
}
}
cold_blocks
}

View File

@ -84,6 +84,26 @@ pub trait BuilderMethods<'a, 'tcx>:
then_llbb: Self::BasicBlock, then_llbb: Self::BasicBlock,
else_llbb: Self::BasicBlock, else_llbb: Self::BasicBlock,
); );
// Conditional with expectation.
//
// This function is opt-in for back ends.
//
// The default implementation calls `self.expect()` before emiting the branch
// by calling `self.cond_br()`
fn cond_br_with_expect(
&mut self,
mut cond: Self::Value,
then_llbb: Self::BasicBlock,
else_llbb: Self::BasicBlock,
expect: Option<bool>,
) {
if let Some(expect) = expect {
cond = self.expect(cond, expect);
}
self.cond_br(cond, then_llbb, else_llbb)
}
fn switch( fn switch(
&mut self, &mut self,
v: Self::Value, v: Self::Value,

View File

@ -417,6 +417,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// These just return their argument // These just return their argument
self.copy_op(&args[0], dest)?; self.copy_op(&args[0], dest)?;
} }
sym::cold_path => {
// This is a no-op. The intrinsic is just a hint to the optimizer.
}
sym::raw_eq => { sym::raw_eq => {
let result = self.raw_eq_intrinsic(&args[0], &args[1])?; let result = self.raw_eq_intrinsic(&args[0], &args[1])?;
self.write_scalar(result, dest)?; self.write_scalar(result, dest)?;

View File

@ -109,9 +109,8 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -
| sym::three_way_compare | sym::three_way_compare
| sym::discriminant_value | sym::discriminant_value
| sym::type_id | sym::type_id
| sym::likely
| sym::unlikely
| sym::select_unpredictable | sym::select_unpredictable
| sym::cold_path
| sym::ptr_guaranteed_cmp | sym::ptr_guaranteed_cmp
| sym::minnumf16 | sym::minnumf16
| sym::minnumf32 | sym::minnumf32
@ -489,9 +488,8 @@ pub fn check_intrinsic_type(
sym::float_to_int_unchecked => (2, 0, vec![param(0)], param(1)), sym::float_to_int_unchecked => (2, 0, vec![param(0)], param(1)),
sym::assume => (0, 0, vec![tcx.types.bool], tcx.types.unit), sym::assume => (0, 0, vec![tcx.types.bool], tcx.types.unit),
sym::likely => (0, 0, vec![tcx.types.bool], tcx.types.bool),
sym::unlikely => (0, 0, vec![tcx.types.bool], tcx.types.bool),
sym::select_unpredictable => (1, 0, vec![tcx.types.bool, param(0), param(0)], param(0)), sym::select_unpredictable => (1, 0, vec![tcx.types.bool, param(0), param(0)], param(0)),
sym::cold_path => (0, 0, vec![], tcx.types.unit),
sym::read_via_copy => (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)), sym::read_via_copy => (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
sym::write_via_move => { sym::write_via_move => {

View File

@ -589,6 +589,7 @@ symbols! {
cmse_nonsecure_entry, cmse_nonsecure_entry,
coerce_unsized, coerce_unsized,
cold, cold,
cold_path,
collapse_debuginfo, collapse_debuginfo,
column, column,
compare_bytes, compare_bytes,

View File

@ -1465,6 +1465,22 @@ pub const unsafe fn assume(b: bool) {
} }
} }
/// Hints to the compiler that current code path is cold.
///
/// Note that, unlike most intrinsics, this is safe to call;
/// it does not require an `unsafe` block.
/// Therefore, implementations must not require the user to uphold
/// any safety invariants.
///
/// This intrinsic does not have a stable counterpart.
#[unstable(feature = "core_intrinsics", issue = "none")]
#[cfg_attr(not(bootstrap), rustc_intrinsic)]
#[cfg(not(bootstrap))]
#[rustc_nounwind]
#[miri::intrinsic_fallback_is_spec]
#[cold]
pub const fn cold_path() {}
/// Hints to the compiler that branch condition is likely to be true. /// Hints to the compiler that branch condition is likely to be true.
/// Returns the value passed to it. /// Returns the value passed to it.
/// ///
@ -1480,13 +1496,21 @@ pub const unsafe fn assume(b: bool) {
bootstrap, bootstrap,
rustc_const_stable(feature = "const_likely", since = "CURRENT_RUSTC_VERSION") rustc_const_stable(feature = "const_likely", since = "CURRENT_RUSTC_VERSION")
)] )]
#[cfg_attr(not(bootstrap), rustc_const_stable_intrinsic)]
#[unstable(feature = "core_intrinsics", issue = "none")] #[unstable(feature = "core_intrinsics", issue = "none")]
#[rustc_intrinsic]
#[rustc_nounwind] #[rustc_nounwind]
#[miri::intrinsic_fallback_is_spec] #[inline(always)]
pub const fn likely(b: bool) -> bool { pub const fn likely(b: bool) -> bool {
b #[cfg(bootstrap)]
{
b
}
#[cfg(not(bootstrap))]
if b {
true
} else {
cold_path();
false
}
} }
/// Hints to the compiler that branch condition is likely to be false. /// Hints to the compiler that branch condition is likely to be false.
@ -1504,13 +1528,21 @@ pub const fn likely(b: bool) -> bool {
bootstrap, bootstrap,
rustc_const_stable(feature = "const_likely", since = "CURRENT_RUSTC_VERSION") rustc_const_stable(feature = "const_likely", since = "CURRENT_RUSTC_VERSION")
)] )]
#[cfg_attr(not(bootstrap), rustc_const_stable_intrinsic)]
#[unstable(feature = "core_intrinsics", issue = "none")] #[unstable(feature = "core_intrinsics", issue = "none")]
#[rustc_intrinsic]
#[rustc_nounwind] #[rustc_nounwind]
#[miri::intrinsic_fallback_is_spec] #[inline(always)]
pub const fn unlikely(b: bool) -> bool { pub const fn unlikely(b: bool) -> bool {
b #[cfg(bootstrap)]
{
b
}
#[cfg(not(bootstrap))]
if b {
cold_path();
true
} else {
false
}
} }
/// Returns either `true_val` or `false_val` depending on condition `b` with a /// Returns either `true_val` or `false_val` depending on condition `b` with a

View File

@ -1,2 +1,2 @@
The loop took around 1250ms The loop took around 1350ms
(It's fine for this number to change when you `--bless` this test.) (It's fine for this number to change when you `--bless` this test.)

View File

@ -90,7 +90,7 @@ pub fn checked_shr_signed(a: i32, b: u32) -> Option<i32> {
#[no_mangle] #[no_mangle]
pub fn checked_add_one_unwrap_unsigned(x: u32) -> u32 { pub fn checked_add_one_unwrap_unsigned(x: u32) -> u32 {
// CHECK: %[[IS_MAX:.+]] = icmp eq i32 %x, -1 // CHECK: %[[IS_MAX:.+]] = icmp eq i32 %x, -1
// CHECK: br i1 %[[IS_MAX]], label %[[NONE_BB:.+]], label %[[SOME_BB:.+]] // CHECK: br i1 %[[IS_MAX]], label %[[NONE_BB:.+]], label %[[SOME_BB:.+]],
// CHECK: [[SOME_BB]]: // CHECK: [[SOME_BB]]:
// CHECK: %[[R:.+]] = add nuw i32 %x, 1 // CHECK: %[[R:.+]] = add nuw i32 %x, 1
// CHECK: ret i32 %[[R]] // CHECK: ret i32 %[[R]]

View File

@ -0,0 +1,13 @@
//@ compile-flags: -O
#![crate_type = "lib"]
#![feature(core_intrinsics)]
use std::intrinsics::cold_path;
#[no_mangle]
pub fn test_cold_path(x: bool) {
cold_path();
}
// CHECK-LABEL: @test_cold_path(
// CHECK-NOT: cold_path

View File

@ -1,22 +1,35 @@
//@ compile-flags: -C no-prepopulate-passes -Copt-level=1 //@ compile-flags: -O
#![crate_type = "lib"] #![crate_type = "lib"]
#![feature(core_intrinsics)] #![feature(core_intrinsics)]
use std::intrinsics::{likely, unlikely}; use std::intrinsics::likely;
#[inline(never)]
#[no_mangle] #[no_mangle]
pub fn check_likely(x: i32, y: i32) -> Option<i32> { pub fn path_a() {
unsafe { println!("path a");
// CHECK: call i1 @llvm.expect.i1(i1 %{{.*}}, i1 true) }
if likely(x == y) { None } else { Some(x + y) }
} #[inline(never)]
#[no_mangle]
pub fn path_b() {
println!("path b");
} }
#[no_mangle] #[no_mangle]
pub fn check_unlikely(x: i32, y: i32) -> Option<i32> { pub fn test_likely(x: bool) {
unsafe { if likely(x) {
// CHECK: call i1 @llvm.expect.i1(i1 %{{.*}}, i1 false) path_a();
if unlikely(x == y) { None } else { Some(x + y) } } else {
path_b();
} }
} }
// CHECK-LABEL: @test_likely(
// CHECK: br i1 %x, label %bb2, label %bb3, !prof ![[NUM:[0-9]+]]
// CHECK: bb3:
// CHECK-NOT: cold_path
// CHECK: path_b
// CHECK: bb2:
// CHECK: path_a
// CHECK: ![[NUM]] = !{!"branch_weights", {{(!"expected", )?}}i32 2000, i32 1}

View File

@ -0,0 +1,17 @@
//@ compile-flags: -O
#![crate_type = "lib"]
#[no_mangle]
pub fn test_assert(x: bool) {
assert!(x);
}
// check that assert! emits branch weights
// CHECK-LABEL: @test_assert(
// CHECK: br i1 %x, label %bb2, label %bb1, !prof ![[NUM:[0-9]+]]
// CHECK: bb1:
// CHECK: panic
// CHECK: bb2:
// CHECK: ret void
// CHECK: ![[NUM]] = !{!"branch_weights", {{(!"expected", )?}}i32 2000, i32 1}

View File

@ -0,0 +1,35 @@
//@ compile-flags: -O
#![crate_type = "lib"]
#![feature(core_intrinsics)]
use std::intrinsics::unlikely;
#[inline(never)]
#[no_mangle]
pub fn path_a() {
println!("path a");
}
#[inline(never)]
#[no_mangle]
pub fn path_b() {
println!("path b");
}
#[no_mangle]
pub fn test_unlikely(x: bool) {
if unlikely(x) {
path_a();
} else {
path_b();
}
}
// CHECK-LABEL: @test_unlikely(
// CHECK: br i1 %x, label %bb2, label %bb4, !prof ![[NUM:[0-9]+]]
// CHECK: bb4:
// CHECK: path_b
// CHECK: bb2:
// CHECK-NOT: cold_path
// CHECK: path_a
// CHECK: ![[NUM]] = !{!"branch_weights", {{(!"expected", )?}}i32 1, i32 2000}

View File

@ -13,7 +13,9 @@ fn step_forward(_1: u16, _2: usize) -> u16 {
scope 6 (inlined core::num::<impl u16>::checked_add) { scope 6 (inlined core::num::<impl u16>::checked_add) {
let mut _5: (u16, bool); let mut _5: (u16, bool);
let mut _6: bool; let mut _6: bool;
let mut _7: bool; scope 7 (inlined unlikely) {
let _7: ();
}
} }
} }
scope 5 (inlined convert::num::ptr_try_from_impls::<impl TryFrom<usize> for u16>::try_from) { scope 5 (inlined convert::num::ptr_try_from_impls::<impl TryFrom<usize> for u16>::try_from) {
@ -21,11 +23,11 @@ fn step_forward(_1: u16, _2: usize) -> u16 {
let mut _4: u16; let mut _4: u16;
} }
} }
scope 7 (inlined Option::<u16>::is_none) { scope 8 (inlined Option::<u16>::is_none) {
scope 8 (inlined Option::<u16>::is_some) { scope 9 (inlined Option::<u16>::is_some) {
} }
} }
scope 9 (inlined core::num::<impl u16>::wrapping_add) { scope 10 (inlined core::num::<impl u16>::wrapping_add) {
} }
} }
@ -39,29 +41,26 @@ fn step_forward(_1: u16, _2: usize) -> u16 {
bb1: { bb1: {
_4 = copy _2 as u16 (IntToInt); _4 = copy _2 as u16 (IntToInt);
StorageDead(_3); StorageDead(_3);
StorageLive(_7);
StorageLive(_6); StorageLive(_6);
StorageLive(_5); StorageLive(_5);
_5 = AddWithOverflow(copy _1, copy _4); _5 = AddWithOverflow(copy _1, copy _4);
_6 = copy (_5.1: bool); _6 = copy (_5.1: bool);
_7 = unlikely(move _6) -> [return: bb2, unwind unreachable]; switchInt(copy _6) -> [0: bb2, otherwise: bb3];
} }
bb2: { bb2: {
switchInt(move _7) -> [0: bb3, otherwise: bb4]; StorageDead(_5);
StorageDead(_6);
goto -> bb7;
} }
bb3: { bb3: {
StorageDead(_5); _7 = cold_path() -> [return: bb4, unwind unreachable];
StorageDead(_6);
StorageDead(_7);
goto -> bb7;
} }
bb4: { bb4: {
StorageDead(_5); StorageDead(_5);
StorageDead(_6); StorageDead(_6);
StorageDead(_7);
goto -> bb6; goto -> bb6;
} }

View File

@ -13,7 +13,9 @@ fn step_forward(_1: u16, _2: usize) -> u16 {
scope 6 (inlined core::num::<impl u16>::checked_add) { scope 6 (inlined core::num::<impl u16>::checked_add) {
let mut _5: (u16, bool); let mut _5: (u16, bool);
let mut _6: bool; let mut _6: bool;
let mut _7: bool; scope 7 (inlined unlikely) {
let _7: ();
}
} }
} }
scope 5 (inlined convert::num::ptr_try_from_impls::<impl TryFrom<usize> for u16>::try_from) { scope 5 (inlined convert::num::ptr_try_from_impls::<impl TryFrom<usize> for u16>::try_from) {
@ -21,11 +23,11 @@ fn step_forward(_1: u16, _2: usize) -> u16 {
let mut _4: u16; let mut _4: u16;
} }
} }
scope 7 (inlined Option::<u16>::is_none) { scope 8 (inlined Option::<u16>::is_none) {
scope 8 (inlined Option::<u16>::is_some) { scope 9 (inlined Option::<u16>::is_some) {
} }
} }
scope 9 (inlined core::num::<impl u16>::wrapping_add) { scope 10 (inlined core::num::<impl u16>::wrapping_add) {
} }
} }
@ -39,29 +41,26 @@ fn step_forward(_1: u16, _2: usize) -> u16 {
bb1: { bb1: {
_4 = copy _2 as u16 (IntToInt); _4 = copy _2 as u16 (IntToInt);
StorageDead(_3); StorageDead(_3);
StorageLive(_7);
StorageLive(_6); StorageLive(_6);
StorageLive(_5); StorageLive(_5);
_5 = AddWithOverflow(copy _1, copy _4); _5 = AddWithOverflow(copy _1, copy _4);
_6 = copy (_5.1: bool); _6 = copy (_5.1: bool);
_7 = unlikely(move _6) -> [return: bb2, unwind unreachable]; switchInt(copy _6) -> [0: bb2, otherwise: bb3];
} }
bb2: { bb2: {
switchInt(move _7) -> [0: bb3, otherwise: bb4]; StorageDead(_5);
StorageDead(_6);
goto -> bb7;
} }
bb3: { bb3: {
StorageDead(_5); _7 = cold_path() -> [return: bb4, unwind unreachable];
StorageDead(_6);
StorageDead(_7);
goto -> bb7;
} }
bb4: { bb4: {
StorageDead(_5); StorageDead(_5);
StorageDead(_6); StorageDead(_6);
StorageDead(_7);
goto -> bb6; goto -> bb6;
} }

View File

@ -64,7 +64,7 @@ fn check_instance(instance: &Instance) {
if instance.has_body() { if instance.has_body() {
let Some(body) = instance.body() else { unreachable!("Expected a body") }; let Some(body) = instance.body() else { unreachable!("Expected a body") };
assert!(!body.blocks.is_empty()); assert!(!body.blocks.is_empty());
assert_eq!(&name, "likely"); assert_eq!(&name, "select_unpredictable");
} else { } else {
assert!(instance.body().is_none()); assert!(instance.body().is_none());
assert_matches!(name.as_str(), "size_of_val" | "vtable_size"); assert_matches!(name.as_str(), "size_of_val" | "vtable_size");
@ -78,7 +78,7 @@ fn check_def(fn_def: FnDef) {
let name = intrinsic.fn_name(); let name = intrinsic.fn_name();
match name.as_str() { match name.as_str() {
"likely" => { "select_unpredictable" => {
assert!(!intrinsic.must_be_overridden()); assert!(!intrinsic.must_be_overridden());
assert!(fn_def.has_body()); assert!(fn_def.has_body());
} }
@ -132,7 +132,7 @@ fn generate_input(path: &str) -> std::io::Result<()> {
pub fn use_intrinsics(init: bool) -> bool {{ pub fn use_intrinsics(init: bool) -> bool {{
let vtable_sz = unsafe {{ vtable_size(0 as *const ()) }}; let vtable_sz = unsafe {{ vtable_size(0 as *const ()) }};
let sz = unsafe {{ size_of_val("hi") }}; let sz = unsafe {{ size_of_val("hi") }};
likely(init && sz == 2) select_unpredictable(init && sz == 2, false, true)
}} }}
"# "#
)?; )?;

View File

@ -13,9 +13,9 @@ fn b() {
} }
fn c() { fn c() {
let _: [unsafe extern "rust-intrinsic" fn(bool) -> bool; 2] = [ let _: [unsafe extern "rust-intrinsic" fn(f32) -> f32; 2] = [
std::intrinsics::likely, //~ ERROR cannot coerce std::intrinsics::floorf32, //~ ERROR cannot coerce
std::intrinsics::unlikely, std::intrinsics::log2f32,
]; ];
} }

View File

@ -18,11 +18,11 @@ LL | let _ = std::mem::transmute as unsafe extern "rust-intrinsic" fn(isize)
error[E0308]: cannot coerce intrinsics to function pointers error[E0308]: cannot coerce intrinsics to function pointers
--> $DIR/reify-intrinsic.rs:17:9 --> $DIR/reify-intrinsic.rs:17:9
| |
LL | std::intrinsics::likely, LL | std::intrinsics::floorf32,
| ^^^^^^^^^^^^^^^^^^^^^^^ cannot coerce intrinsics to function pointers | ^^^^^^^^^^^^^^^^^^^^^^^^^ cannot coerce intrinsics to function pointers
| |
= note: expected fn pointer `unsafe extern "rust-intrinsic" fn(_) -> _` = note: expected fn pointer `unsafe extern "rust-intrinsic" fn(_) -> _`
found fn item `fn(_) -> _ {likely}` found fn item `unsafe extern "rust-intrinsic" fn(_) -> _ {floorf32}`
error: aborting due to 3 previous errors error: aborting due to 3 previous errors