rust/compiler/rustc_codegen_cranelift/src/codegen_f16_f128.rs

use crate::prelude::*;

pub(crate) fn f16_to_f32(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
    let (value, arg_ty) =
        if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" {
            (
                fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value),
                lib_call_arg_param(fx.tcx, types::I16, false),
            )
        } else {
            (value, AbiParam::new(types::F16))
        };
    fx.lib_call("__extendhfsf2", vec![arg_ty], vec![AbiParam::new(types::F32)], &[value])[0]
}

fn f16_to_f64(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
    let ret = f16_to_f32(fx, value);
    fx.bcx.ins().fpromote(types::F64, ret)
}

pub(crate) fn f32_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
    let ret_ty = if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" {
        types::I16
    } else {
        types::F16
    };
    let ret = fx.lib_call(
        "__truncsfhf2",
        vec![AbiParam::new(types::F32)],
        vec![AbiParam::new(ret_ty)],
        &[value],
    )[0];
    if ret_ty == types::I16 { fx.bcx.ins().bitcast(types::F16, MemFlags::new(), ret) } else { ret }
}

fn f64_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
    let ret_ty = if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" {
        types::I16
    } else {
        types::F16
    };
    let ret = fx.lib_call(
        "__truncdfhf2",
        vec![AbiParam::new(types::F64)],
        vec![AbiParam::new(ret_ty)],
        &[value],
    )[0];
    if ret_ty == types::I16 { fx.bcx.ins().bitcast(types::F16, MemFlags::new(), ret) } else { ret }
}

pub(crate) fn fcmp(fx: &mut FunctionCx<'_, '_, '_>, cc: FloatCC, lhs: Value, rhs: Value) -> Value {
    let ty = fx.bcx.func.dfg.value_type(lhs);
    match ty {
        types::F32 | types::F64 => fx.bcx.ins().fcmp(cc, lhs, rhs),
        types::F16 => {
            let lhs = f16_to_f32(fx, lhs);
            let rhs = f16_to_f32(fx, rhs);
            fx.bcx.ins().fcmp(cc, lhs, rhs)
        }
        types::F128 => {
            let (name, int_cc) = match cc {
                FloatCC::Equal => ("__eqtf2", IntCC::Equal),
                FloatCC::NotEqual => ("__netf2", IntCC::NotEqual),
                FloatCC::LessThan => ("__lttf2", IntCC::SignedLessThan),
                FloatCC::LessThanOrEqual => ("__letf2", IntCC::SignedLessThanOrEqual),
                FloatCC::GreaterThan => ("__gttf2", IntCC::SignedGreaterThan),
                FloatCC::GreaterThanOrEqual => ("__getf2", IntCC::SignedGreaterThanOrEqual),
                _ => unreachable!("not currently used in rustc_codegen_cranelift: {cc:?}"),
            };
            let res = fx.lib_call(
                name,
                vec![AbiParam::new(types::F128), AbiParam::new(types::F128)],
                // FIXME(rust-lang/compiler-builtins#919): This should be `I64` on non-AArch64
                // architectures, but switching it before compiler-builtins is fixed causes test
                // failures.
                vec![AbiParam::new(types::I32)],
                &[lhs, rhs],
            )[0];
            let zero = fx.bcx.ins().iconst(types::I32, 0);
            let res = fx.bcx.ins().icmp(int_cc, res, zero);
            res
        }
        _ => unreachable!("{ty:?}"),
    }
}

pub(crate) fn codegen_f128_binop(
    fx: &mut FunctionCx<'_, '_, '_>,
    bin_op: BinOp,
    lhs: Value,
    rhs: Value,
) -> Value {
    let name = match bin_op {
        BinOp::Add => "__addtf3",
        BinOp::Sub => "__subtf3",
        BinOp::Mul => "__multf3",
        BinOp::Div => "__divtf3",
        _ => unreachable!("handled in `codegen_float_binop`"),
    };
    fx.lib_call(
        name,
        vec![AbiParam::new(types::F128), AbiParam::new(types::F128)],
        vec![AbiParam::new(types::F128)],
        &[lhs, rhs],
    )[0]
}

pub(crate) fn neg_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
    let bits = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value);
    let bits = fx.bcx.ins().bxor_imm(bits, 0x8000);
    fx.bcx.ins().bitcast(types::F16, MemFlags::new(), bits)
}

pub(crate) fn neg_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
    let bits = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), value);
    let (low, high) = fx.bcx.ins().isplit(bits);
    let high = fx.bcx.ins().bxor_imm(high, 0x8000_0000_0000_0000_u64 as i64);
    let bits = fx.bcx.ins().iconcat(low, high);
    fx.bcx.ins().bitcast(types::F128, MemFlags::new(), bits)
}

pub(crate) fn abs_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
    let bits = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value);
    let bits = fx.bcx.ins().band_imm(bits, 0x7fff);
    fx.bcx.ins().bitcast(types::F16, MemFlags::new(), bits)
}

pub(crate) fn abs_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
    let bits = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), value);
    let (low, high) = fx.bcx.ins().isplit(bits);
    let high = fx.bcx.ins().band_imm(high, 0x7fff_ffff_ffff_ffff_u64 as i64);
    let bits = fx.bcx.ins().iconcat(low, high);
    fx.bcx.ins().bitcast(types::F128, MemFlags::new(), bits)
}

pub(crate) fn copysign_f16(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value {
    let lhs = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), lhs);
    let rhs = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), rhs);
    let res = fx.bcx.ins().band_imm(lhs, 0x7fff);
    let sign = fx.bcx.ins().band_imm(rhs, 0x8000);
    let res = fx.bcx.ins().bor(res, sign);
    fx.bcx.ins().bitcast(types::F16, MemFlags::new(), res)
}

pub(crate) fn copysign_f128(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value {
    let lhs = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), lhs);
    let rhs = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), rhs);
    let (low, lhs_high) = fx.bcx.ins().isplit(lhs);
    let (_, rhs_high) = fx.bcx.ins().isplit(rhs);
    let high = fx.bcx.ins().band_imm(lhs_high, 0x7fff_ffff_ffff_ffff_u64 as i64);
    let sign = fx.bcx.ins().band_imm(rhs_high, 0x8000_0000_0000_0000_u64 as i64);
    let high = fx.bcx.ins().bor(high, sign);
    let res = fx.bcx.ins().iconcat(low, high);
    fx.bcx.ins().bitcast(types::F128, MemFlags::new(), res)
}

pub(crate) fn codegen_cast(
    fx: &mut FunctionCx<'_, '_, '_>,
    from: Value,
    from_signed: bool,
    to_ty: Type,
    to_signed: bool,
) -> Value {
    let from_ty = fx.bcx.func.dfg.value_type(from);
    if from_ty.is_float() && to_ty.is_float() {
        let name = match (from_ty, to_ty) {
            (types::F16, types::F32) => return f16_to_f32(fx, from),
            (types::F16, types::F64) => return f16_to_f64(fx, from),
            (types::F16, types::F128) => "__extendhftf2",
            (types::F32, types::F128) => "__extendsftf2",
            (types::F64, types::F128) => "__extenddftf2",
            (types::F128, types::F64) => "__trunctfdf2",
            (types::F128, types::F32) => "__trunctfsf2",
            (types::F128, types::F16) => "__trunctfhf2",
            (types::F64, types::F16) => return f64_to_f16(fx, from),
            (types::F32, types::F16) => return f32_to_f16(fx, from),
            _ => unreachable!("{from_ty:?} -> {to_ty:?}"),
        };
        fx.lib_call(name, vec![AbiParam::new(from_ty)], vec![AbiParam::new(to_ty)], &[from])[0]
    } else if from_ty.is_int() && to_ty == types::F16 {
        let res = clif_int_or_float_cast(fx, from, from_signed, types::F32, false);
        f32_to_f16(fx, res)
    } else if from_ty == types::F16 && to_ty.is_int() {
        let from = f16_to_f32(fx, from);
        clif_int_or_float_cast(fx, from, false, to_ty, to_signed)
    } else if from_ty.is_int() && to_ty == types::F128 {
        let (from, from_ty) = if from_ty.bits() < 32 {
            (clif_int_or_float_cast(fx, from, from_signed, types::I32, from_signed), types::I32)
        } else {
            (from, from_ty)
        };
        let name = format!(
            "__float{sign}{size}itf",
            sign = if from_signed { "" } else { "un" },
            size = match from_ty {
                types::I32 => 's',
                types::I64 => 'd',
                types::I128 => 't',
                _ => unreachable!("{from_ty:?}"),
            },
        );
        fx.lib_call(
            &name,
            vec![lib_call_arg_param(fx.tcx, from_ty, from_signed)],
            vec![AbiParam::new(to_ty)],
            &[from],
        )[0]
    } else if from_ty == types::F128 && to_ty.is_int() {
        let ret_ty = if to_ty.bits() < 32 { types::I32 } else { to_ty };
        let name = format!(
            "__fix{sign}tf{size}i",
            sign = if from_signed { "" } else { "un" },
            size = match ret_ty {
                types::I32 => 's',
                types::I64 => 'd',
                types::I128 => 't',
                _ => unreachable!("{from_ty:?}"),
            },
        );
        let ret =
            fx.lib_call(&name, vec![AbiParam::new(from_ty)], vec![AbiParam::new(to_ty)], &[from])
                [0];
        let val = if ret_ty == to_ty {
            ret
        } else {
            let (min, max) = match (to_ty, to_signed) {
                (types::I8, false) => (0, i64::from(u8::MAX)),
                (types::I16, false) => (0, i64::from(u16::MAX)),
                (types::I8, true) => (i64::from(i8::MIN as u32), i64::from(i8::MAX as u32)),
                (types::I16, true) => (i64::from(i16::MIN as u32), i64::from(i16::MAX as u32)),
                _ => unreachable!("{to_ty:?}"),
            };
            let min_val = fx.bcx.ins().iconst(types::I32, min);
            let max_val = fx.bcx.ins().iconst(types::I32, max);

            let val = if to_signed {
                let has_underflow = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, ret, min);
                let has_overflow = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThan, ret, max);
                let bottom_capped = fx.bcx.ins().select(has_underflow, min_val, ret);
                fx.bcx.ins().select(has_overflow, max_val, bottom_capped)
            } else {
                let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, ret, max);
                fx.bcx.ins().select(has_overflow, max_val, ret)
            };
            fx.bcx.ins().ireduce(to_ty, val)
        };

        if let Some(false) = fx.tcx.sess.opts.unstable_opts.saturating_float_casts {
            return val;
        }

        let is_not_nan = fcmp(fx, FloatCC::Equal, from, from);
        let zero = type_zero_value(&mut fx.bcx, to_ty);
        fx.bcx.ins().select(is_not_nan, val, zero)
    } else {
        unreachable!("{from_ty:?} -> {to_ty:?}");
    }
}

pub(crate) fn fma_f16(fx: &mut FunctionCx<'_, '_, '_>, x: Value, y: Value, z: Value) -> Value {
    let x = f16_to_f64(fx, x);
    let y = f16_to_f64(fx, y);
    let z = f16_to_f64(fx, z);
    let res = fx.bcx.ins().fma(x, y, z);
    f64_to_f16(fx, res)
}

pub(crate) fn fmin_f128(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value {
    fx.lib_call(
        "fminimumf128",
        vec![AbiParam::new(types::F128), AbiParam::new(types::F128)],
        vec![AbiParam::new(types::F128)],
        &[a, b],
    )[0]
}

pub(crate) fn fmax_f128(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value {
    fx.lib_call(
        "fmaximumf128",
        vec![AbiParam::new(types::F128), AbiParam::new(types::F128)],
        vec![AbiParam::new(types::F128)],
        &[a, b],
    )[0]
}