2019-07-30 12:37:20 +00:00
|
|
|
use crate::intrinsics::*;
|
2019-08-31 17:28:09 +00:00
|
|
|
use crate::prelude::*;
|
2019-07-28 07:54:57 +00:00
|
|
|
|
2020-03-31 11:20:19 +00:00
|
|
|
use rustc_middle::ty::subst::SubstsRef;
|
2019-07-28 07:54:57 +00:00
|
|
|
|
2020-03-27 11:14:45 +00:00
|
|
|
pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
|
2019-08-18 14:52:07 +00:00
|
|
|
fx: &mut FunctionCx<'_, 'tcx, impl Backend>,
|
2019-07-28 07:54:57 +00:00
|
|
|
intrinsic: &str,
|
|
|
|
substs: SubstsRef<'tcx>,
|
2019-07-28 09:24:33 +00:00
|
|
|
args: &[mir::Operand<'tcx>],
|
2019-07-28 07:54:57 +00:00
|
|
|
destination: Option<(CPlace<'tcx>, BasicBlock)>,
|
|
|
|
) {
|
2019-07-29 10:43:24 +00:00
|
|
|
let ret = match destination {
|
|
|
|
Some((place, _)) => place,
|
|
|
|
None => {
|
|
|
|
// Insert non returning intrinsics here
|
|
|
|
match intrinsic {
|
|
|
|
"abort" => {
|
|
|
|
trap_panic(fx, "Called intrinsic::abort.");
|
|
|
|
}
|
|
|
|
"unreachable" => {
|
|
|
|
trap_unreachable(fx, "[corruption] Called intrinsic::unreachable.");
|
|
|
|
}
|
|
|
|
_ => unimplemented!("unsupported instrinsic {}", intrinsic),
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2019-07-30 12:37:20 +00:00
|
|
|
intrinsic_match! {
|
2019-07-29 10:43:24 +00:00
|
|
|
fx, intrinsic, substs, args,
|
|
|
|
_ => {
|
|
|
|
fx.tcx.sess.warn(&format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic));
|
|
|
|
crate::trap::trap_unimplemented(fx, intrinsic);
|
|
|
|
};
|
|
|
|
|
2019-07-29 10:50:20 +00:00
|
|
|
// Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8`
|
2019-07-30 12:37:20 +00:00
|
|
|
llvm.x86.sse2.pmovmskb.128 | llvm.x86.avx2.pmovmskb | llvm.x86.sse2.movmsk.pd, (c a) {
|
2019-11-18 20:15:43 +00:00
|
|
|
let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, a.layout());
|
2019-07-30 12:37:20 +00:00
|
|
|
let lane_ty = fx.clif_type(lane_layout.ty).unwrap();
|
|
|
|
assert!(lane_count <= 32);
|
2019-07-29 10:43:24 +00:00
|
|
|
|
|
|
|
let mut res = fx.bcx.ins().iconst(types::I32, 0);
|
|
|
|
|
2019-07-29 16:59:17 +00:00
|
|
|
for lane in (0..lane_count).rev() {
|
2019-07-29 10:43:24 +00:00
|
|
|
let a_lane = a.value_field(fx, mir::Field::new(lane.try_into().unwrap())).load_scalar(fx);
|
2019-07-30 12:37:20 +00:00
|
|
|
|
|
|
|
// cast float to int
|
|
|
|
let a_lane = match lane_ty {
|
|
|
|
types::F32 => fx.bcx.ins().bitcast(types::I32, a_lane),
|
|
|
|
types::F64 => fx.bcx.ins().bitcast(types::I64, a_lane),
|
|
|
|
_ => a_lane,
|
|
|
|
};
|
|
|
|
|
|
|
|
// extract sign bit of an int
|
|
|
|
let a_lane_sign = fx.bcx.ins().ushr_imm(a_lane, i64::from(lane_ty.bits() - 1));
|
|
|
|
|
|
|
|
// shift sign bit into result
|
|
|
|
let a_lane_sign = clif_intcast(fx, a_lane_sign, types::I32, false);
|
2019-07-29 10:43:24 +00:00
|
|
|
res = fx.bcx.ins().ishl_imm(res, 1);
|
|
|
|
res = fx.bcx.ins().bor(res, a_lane_sign);
|
|
|
|
}
|
|
|
|
|
|
|
|
let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32));
|
|
|
|
ret.write_cvalue(fx, res);
|
|
|
|
};
|
2019-07-30 12:37:20 +00:00
|
|
|
llvm.x86.sse2.cmp.ps | llvm.x86.sse2.cmp.pd, (c x, c y, o kind) {
|
|
|
|
let kind_const = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const");
|
|
|
|
let flt_cc = match kind_const.val.try_to_bits(Size::from_bytes(1)).expect(&format!("kind not scalar: {:?}", kind_const)) {
|
|
|
|
0 => FloatCC::Equal,
|
|
|
|
1 => FloatCC::LessThan,
|
|
|
|
2 => FloatCC::LessThanOrEqual,
|
|
|
|
7 => {
|
|
|
|
unimplemented!("Compares corresponding elements in `a` and `b` to see if neither is `NaN`.");
|
|
|
|
}
|
|
|
|
3 => {
|
|
|
|
unimplemented!("Compares corresponding elements in `a` and `b` to see if either is `NaN`.");
|
|
|
|
}
|
|
|
|
4 => FloatCC::NotEqual,
|
|
|
|
5 => {
|
|
|
|
unimplemented!("not less than");
|
|
|
|
}
|
|
|
|
6 => {
|
|
|
|
unimplemented!("not less than or equal");
|
|
|
|
}
|
|
|
|
kind => unreachable!("kind {:?}", kind),
|
|
|
|
};
|
|
|
|
|
2019-12-31 14:53:18 +00:00
|
|
|
simd_pair_for_each_lane(fx, x, y, ret, |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
|
2019-09-28 09:13:40 +00:00
|
|
|
let res_lane = match lane_layout.ty.kind {
|
2019-07-30 12:37:20 +00:00
|
|
|
ty::Float(_) => fx.bcx.ins().fcmp(flt_cc, x_lane, y_lane),
|
|
|
|
_ => unreachable!("{:?}", lane_layout.ty),
|
|
|
|
};
|
|
|
|
bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
|
|
|
|
});
|
|
|
|
};
|
2019-07-29 10:43:24 +00:00
|
|
|
}
|
2019-07-28 07:54:57 +00:00
|
|
|
|
|
|
|
if let Some((_, dest)) = destination {
|
2020-02-14 17:23:29 +00:00
|
|
|
let ret_block = fx.get_block(dest);
|
|
|
|
fx.bcx.ins().jump(ret_block, &[]);
|
2019-07-28 07:54:57 +00:00
|
|
|
} else {
|
|
|
|
trap_unreachable(fx, "[corruption] Diverging intrinsic returned.");
|
|
|
|
}
|
|
|
|
}
|
2019-07-28 08:24:57 +00:00
|
|
|
|
|
|
|
// llvm.x86.avx2.vperm2i128
|
|
|
|
// llvm.x86.ssse3.pshuf.b.128
|
|
|
|
// llvm.x86.avx2.pshuf.b
|
|
|
|
// llvm.x86.avx2.psrli.w
|
|
|
|
// llvm.x86.sse2.psrli.w
|