mirror of
https://github.com/rust-lang/rust.git
synced 2025-05-01 12:37:37 +00:00
4560efe46c
This makes load generation compatible with opaque pointers. The generation of nontemporal copies still accesses the pointer element type, as fixing this requires more movement.
211 lines
8.1 KiB
Rust
211 lines
8.1 KiB
Rust
use crate::builder::Builder;
|
|
use crate::type_::Type;
|
|
use crate::type_of::LayoutLlvmExt;
|
|
use crate::value::Value;
|
|
use rustc_codegen_ssa::mir::operand::OperandRef;
|
|
use rustc_codegen_ssa::{
|
|
common::IntPredicate,
|
|
traits::{BaseTypeMethods, BuilderMethods, ConstMethods, DerivedTypeMethods},
|
|
};
|
|
use rustc_middle::ty::layout::HasTyCtxt;
|
|
use rustc_middle::ty::Ty;
|
|
use rustc_target::abi::{Align, Endian, HasDataLayout, LayoutOf, Size};
|
|
|
|
fn round_pointer_up_to_alignment(
|
|
bx: &mut Builder<'a, 'll, 'tcx>,
|
|
addr: &'ll Value,
|
|
align: Align,
|
|
ptr_ty: &'ll Type,
|
|
) -> &'ll Value {
|
|
let mut ptr_as_int = bx.ptrtoint(addr, bx.cx().type_isize());
|
|
ptr_as_int = bx.add(ptr_as_int, bx.cx().const_i32(align.bytes() as i32 - 1));
|
|
ptr_as_int = bx.and(ptr_as_int, bx.cx().const_i32(-(align.bytes() as i32)));
|
|
bx.inttoptr(ptr_as_int, ptr_ty)
|
|
}
|
|
|
|
fn emit_direct_ptr_va_arg(
|
|
bx: &mut Builder<'a, 'll, 'tcx>,
|
|
list: OperandRef<'tcx, &'ll Value>,
|
|
llty: &'ll Type,
|
|
size: Size,
|
|
align: Align,
|
|
slot_size: Align,
|
|
allow_higher_align: bool,
|
|
) -> (&'ll Value, Align) {
|
|
let va_list_ty = bx.type_i8p();
|
|
let va_list_ptr_ty = bx.type_ptr_to(va_list_ty);
|
|
let va_list_addr = if list.layout.llvm_type(bx.cx) != va_list_ptr_ty {
|
|
bx.bitcast(list.immediate(), va_list_ptr_ty)
|
|
} else {
|
|
list.immediate()
|
|
};
|
|
|
|
let ptr = bx.load(va_list_ty, va_list_addr, bx.tcx().data_layout.pointer_align.abi);
|
|
|
|
let (addr, addr_align) = if allow_higher_align && align > slot_size {
|
|
(round_pointer_up_to_alignment(bx, ptr, align, bx.cx().type_i8p()), align)
|
|
} else {
|
|
(ptr, slot_size)
|
|
};
|
|
|
|
let aligned_size = size.align_to(slot_size).bytes() as i32;
|
|
let full_direct_size = bx.cx().const_i32(aligned_size);
|
|
let next = bx.inbounds_gep(addr, &[full_direct_size]);
|
|
bx.store(next, va_list_addr, bx.tcx().data_layout.pointer_align.abi);
|
|
|
|
if size.bytes() < slot_size.bytes() && bx.tcx().sess.target.endian == Endian::Big {
|
|
let adjusted_size = bx.cx().const_i32((slot_size.bytes() - size.bytes()) as i32);
|
|
let adjusted = bx.inbounds_gep(addr, &[adjusted_size]);
|
|
(bx.bitcast(adjusted, bx.cx().type_ptr_to(llty)), addr_align)
|
|
} else {
|
|
(bx.bitcast(addr, bx.cx().type_ptr_to(llty)), addr_align)
|
|
}
|
|
}
|
|
|
|
fn emit_ptr_va_arg(
|
|
bx: &mut Builder<'a, 'll, 'tcx>,
|
|
list: OperandRef<'tcx, &'ll Value>,
|
|
target_ty: Ty<'tcx>,
|
|
indirect: bool,
|
|
slot_size: Align,
|
|
allow_higher_align: bool,
|
|
) -> &'ll Value {
|
|
let layout = bx.cx.layout_of(target_ty);
|
|
let (llty, size, align) = if indirect {
|
|
(
|
|
bx.cx.layout_of(bx.cx.tcx.mk_imm_ptr(target_ty)).llvm_type(bx.cx),
|
|
bx.cx.data_layout().pointer_size,
|
|
bx.cx.data_layout().pointer_align,
|
|
)
|
|
} else {
|
|
(layout.llvm_type(bx.cx), layout.size, layout.align)
|
|
};
|
|
let (addr, addr_align) =
|
|
emit_direct_ptr_va_arg(bx, list, llty, size, align.abi, slot_size, allow_higher_align);
|
|
if indirect {
|
|
let tmp_ret = bx.load(llty, addr, addr_align);
|
|
bx.load(bx.cx.layout_of(target_ty).llvm_type(bx.cx), tmp_ret, align.abi)
|
|
} else {
|
|
bx.load(llty, addr, addr_align)
|
|
}
|
|
}
|
|
|
|
fn emit_aapcs_va_arg(
|
|
bx: &mut Builder<'a, 'll, 'tcx>,
|
|
list: OperandRef<'tcx, &'ll Value>,
|
|
target_ty: Ty<'tcx>,
|
|
) -> &'ll Value {
|
|
// Implementation of the AAPCS64 calling convention for va_args see
|
|
// https://github.com/ARM-software/abi-aa/blob/master/aapcs64/aapcs64.rst
|
|
let va_list_addr = list.immediate();
|
|
let layout = bx.cx.layout_of(target_ty);
|
|
|
|
let mut maybe_reg = bx.build_sibling_block("va_arg.maybe_reg");
|
|
let mut in_reg = bx.build_sibling_block("va_arg.in_reg");
|
|
let mut on_stack = bx.build_sibling_block("va_arg.on_stack");
|
|
let mut end = bx.build_sibling_block("va_arg.end");
|
|
let zero = bx.const_i32(0);
|
|
let offset_align = Align::from_bytes(4).unwrap();
|
|
|
|
let gr_type = target_ty.is_any_ptr() || target_ty.is_integral();
|
|
let (reg_off, reg_top_index, slot_size) = if gr_type {
|
|
let gr_offs = bx.struct_gep(va_list_addr, 7);
|
|
let nreg = (layout.size.bytes() + 7) / 8;
|
|
(gr_offs, 3, nreg * 8)
|
|
} else {
|
|
let vr_off = bx.struct_gep(va_list_addr, 9);
|
|
let nreg = (layout.size.bytes() + 15) / 16;
|
|
(vr_off, 5, nreg * 16)
|
|
};
|
|
|
|
// if the offset >= 0 then the value will be on the stack
|
|
let mut reg_off_v = bx.load(bx.type_i32(), reg_off, offset_align);
|
|
let use_stack = bx.icmp(IntPredicate::IntSGE, reg_off_v, zero);
|
|
bx.cond_br(use_stack, &on_stack.llbb(), &maybe_reg.llbb());
|
|
|
|
// The value at this point might be in a register, but there is a chance that
|
|
// it could be on the stack so we have to update the offset and then check
|
|
// the offset again.
|
|
|
|
if gr_type && layout.align.abi.bytes() > 8 {
|
|
reg_off_v = maybe_reg.add(reg_off_v, bx.const_i32(15));
|
|
reg_off_v = maybe_reg.and(reg_off_v, bx.const_i32(-16));
|
|
}
|
|
let new_reg_off_v = maybe_reg.add(reg_off_v, bx.const_i32(slot_size as i32));
|
|
|
|
maybe_reg.store(new_reg_off_v, reg_off, offset_align);
|
|
|
|
// Check to see if we have overflowed the registers as a result of this.
|
|
// If we have then we need to use the stack for this value
|
|
let use_stack = maybe_reg.icmp(IntPredicate::IntSGT, new_reg_off_v, zero);
|
|
maybe_reg.cond_br(use_stack, &on_stack.llbb(), &in_reg.llbb());
|
|
|
|
let top_type = bx.type_i8p();
|
|
let top = in_reg.struct_gep(va_list_addr, reg_top_index);
|
|
let top = in_reg.load(top_type, top, bx.tcx().data_layout.pointer_align.abi);
|
|
|
|
// reg_value = *(@top + reg_off_v);
|
|
let mut reg_addr = in_reg.gep(top, &[reg_off_v]);
|
|
if bx.tcx().sess.target.endian == Endian::Big && layout.size.bytes() != slot_size {
|
|
// On big-endian systems the value is right-aligned in its slot.
|
|
let offset = bx.const_i32((slot_size - layout.size.bytes()) as i32);
|
|
reg_addr = in_reg.gep(reg_addr, &[offset]);
|
|
}
|
|
let reg_type = layout.llvm_type(bx);
|
|
let reg_addr = in_reg.bitcast(reg_addr, bx.cx.type_ptr_to(reg_type));
|
|
let reg_value = in_reg.load(reg_type, reg_addr, layout.align.abi);
|
|
in_reg.br(&end.llbb());
|
|
|
|
// On Stack block
|
|
let stack_value =
|
|
emit_ptr_va_arg(&mut on_stack, list, target_ty, false, Align::from_bytes(8).unwrap(), true);
|
|
on_stack.br(&end.llbb());
|
|
|
|
let val = end.phi(
|
|
layout.immediate_llvm_type(bx),
|
|
&[reg_value, stack_value],
|
|
&[&in_reg.llbb(), &on_stack.llbb()],
|
|
);
|
|
|
|
*bx = end;
|
|
val
|
|
}
|
|
|
|
pub(super) fn emit_va_arg(
|
|
bx: &mut Builder<'a, 'll, 'tcx>,
|
|
addr: OperandRef<'tcx, &'ll Value>,
|
|
target_ty: Ty<'tcx>,
|
|
) -> &'ll Value {
|
|
// Determine the va_arg implementation to use. The LLVM va_arg instruction
|
|
// is lacking in some instances, so we should only use it as a fallback.
|
|
let target = &bx.cx.tcx.sess.target;
|
|
let arch = &bx.cx.tcx.sess.target.arch;
|
|
match &**arch {
|
|
// Windows x86
|
|
"x86" if target.is_like_windows => {
|
|
emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), false)
|
|
}
|
|
// Generic x86
|
|
"x86" => emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), true),
|
|
// Windows AArch64
|
|
"aarch64" if target.is_like_windows => {
|
|
emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), false)
|
|
}
|
|
// macOS / iOS AArch64
|
|
"aarch64" if target.is_like_osx => {
|
|
emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), true)
|
|
}
|
|
"aarch64" => emit_aapcs_va_arg(bx, addr, target_ty),
|
|
// Windows x86_64
|
|
"x86_64" if target.is_like_windows => {
|
|
let target_ty_size = bx.cx.size_of(target_ty).bytes();
|
|
let indirect: bool = target_ty_size > 8 || !target_ty_size.is_power_of_two();
|
|
emit_ptr_va_arg(bx, addr, target_ty, indirect, Align::from_bytes(8).unwrap(), false)
|
|
}
|
|
// For all other architecture/OS combinations fall back to using
|
|
// the LLVM va_arg instruction.
|
|
// https://llvm.org/docs/LangRef.html#va-arg-instruction
|
|
_ => bx.va_arg(addr.immediate(), bx.cx.layout_of(target_ty).llvm_type(bx.cx)),
|
|
}
|
|
}
|