From 967709f31a025e402a68a6ab2744d0d5a51432ad Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 20 Dec 2019 16:02:47 +0100 Subject: [PATCH] [OPT] Use load and store offsets instead of iadd_imm --- src/abi/comments.rs | 6 +- src/abi/mod.rs | 6 +- src/abi/pass_mode.rs | 2 +- src/abi/returning.rs | 2 +- src/base.rs | 18 ++--- src/constant.rs | 6 +- src/intrinsics.rs | 14 ++-- src/lib.rs | 2 + src/pointer.rs | 101 +++++++++++++++++++++++++ src/trap.rs | 6 +- src/value_and_place.rs | 162 ++++++++++++++++++++++------------------- test.sh | 4 +- 12 files changed, 218 insertions(+), 111 deletions(-) create mode 100644 src/pointer.rs diff --git a/src/abi/comments.rs b/src/abi/comments.rs index 656731eb97c..3fc5b2f0bc8 100644 --- a/src/abi/comments.rs +++ b/src/abi/comments.rs @@ -94,14 +94,14 @@ pub fn add_local_place_comments<'tcx>( align.abi.bytes(), align.pref.bytes(), )), - CPlaceInner::Addr(addr, None) => fx.add_global_comment(format!( - "reuse {:5} {:20} {:4}b {}, {} storage={}", + CPlaceInner::Addr(ptr, None) => fx.add_global_comment(format!( + "reuse {:5} {:20} {:4}b {}, {} storage={:?}", format!("{:?}", local), format!("{:?}", ty), size.bytes(), align.abi.bytes(), align.pref.bytes(), - addr, + ptr, )), CPlaceInner::Addr(_, Some(_)) => unreachable!(), } diff --git a/src/abi/mod.rs b/src/abi/mod.rs index c33bb249f25..f11e343581c 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -250,9 +250,7 @@ impl<'tcx, B: Backend + 'static> FunctionCx<'_, 'tcx, B> { let ret_vals = self.lib_call(name, input_tys, return_tys, &args); match *ret_vals { [] => CValue::by_ref( - self.bcx - .ins() - .iconst(self.pointer_type, self.pointer_type.bytes() as i64), + Pointer::const_addr(self, self.pointer_type.bytes() as i64), return_layout, ), [val] => CValue::by_val(val, return_layout), @@ -352,7 +350,7 @@ pub fn codegen_fn_prelude(fx: &mut FunctionCx<'_, '_, impl Backend>, start_ebb: // We wont mutate this argument, so it is fine to borrow the backing storage // of this argument, to prevent a copy. - let place = CPlace::for_addr(addr, val.layout()); + let place = CPlace::for_ptr(Pointer::new(addr), val.layout()); #[cfg(debug_assertions)] self::comments::add_local_place_comments(fx, place, local); diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 64d53566dcc..9c43f2c80af 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -158,6 +158,6 @@ pub fn cvalue_for_param<'tcx>( let (a, b) = ebb_params.assert_pair(); Some(CValue::by_val_pair(a, b, layout)) } - PassMode::ByRef => Some(CValue::by_ref(ebb_params.assert_single(), layout)), + PassMode::ByRef => Some(CValue::by_ref(Pointer::new(ebb_params.assert_single()), layout)), } } diff --git a/src/abi/returning.rs b/src/abi/returning.rs index f80a5d3cb84..489f52b306f 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -28,7 +28,7 @@ pub fn codegen_return_param( PassMode::ByRef => { let ret_param = fx.bcx.append_ebb_param(start_ebb, fx.pointer_type); fx.local_map - .insert(RETURN_PLACE, CPlace::for_addr(ret_param, ret_layout)); + .insert(RETURN_PLACE, CPlace::for_ptr(Pointer::new(ret_param), ret_layout)); Single(ret_param) } diff --git a/src/base.rs b/src/base.rs index 68f9bc342b0..db7ebd07535 100644 --- a/src/base.rs +++ b/src/base.rs @@ -601,7 +601,7 @@ fn codegen_array_len<'tcx>( fx.bcx.ins().iconst(fx.pointer_type, len) } ty::Slice(_elem_ty) => place - .to_addr_maybe_unsized(fx) + .to_ptr_maybe_unsized(fx) .1 .expect("Length metadata for slice place"), _ => bug!("Rvalue::Len({:?})", place), @@ -659,25 +659,21 @@ pub fn trans_place<'tcx>( match cplace.layout().ty.kind { ty::Array(elem_ty, len) => { let elem_layout = fx.layout_of(elem_ty); - let ptr = cplace.to_addr(fx); + let ptr = cplace.to_ptr(fx); let len = crate::constant::force_eval_const(fx, len) .eval_usize(fx.tcx, ParamEnv::reveal_all()); - cplace = CPlace::for_addr( - fx.bcx - .ins() - .iadd_imm(ptr, elem_layout.size.bytes() as i64 * from as i64), + cplace = CPlace::for_ptr( + ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * from as i64), fx.layout_of(fx.tcx.mk_array(elem_ty, len - from as u64 - to as u64)), ); } ty::Slice(elem_ty) => { assert!(from_end, "slice subslices should be `from_end`"); let elem_layout = fx.layout_of(elem_ty); - let (ptr, len) = cplace.to_addr_maybe_unsized(fx); + let (ptr, len) = cplace.to_ptr_maybe_unsized(fx); let len = len.unwrap(); - cplace = CPlace::for_addr_with_extra( - fx.bcx - .ins() - .iadd_imm(ptr, elem_layout.size.bytes() as i64 * from as i64), + cplace = CPlace::for_ptr_with_extra( + ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * from as i64), fx.bcx.ins().iadd_imm(len, -(from as i64 + to as i64)), cplace.layout(), ); diff --git a/src/constant.rs b/src/constant.rs index dade64b348a..53e0941d7e9 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -132,9 +132,7 @@ pub fn trans_const_value<'tcx>( CValue::by_val(val, layout) } ty::FnDef(_def_id, _substs) => CValue::by_ref( - fx.bcx - .ins() - .iconst(fx.pointer_type, fx.pointer_type.bytes() as i64), + crate::pointer::Pointer::const_addr(fx, fx.pointer_type.bytes() as i64), layout, ), _ => trans_const_place(fx, const_).to_cvalue(fx), @@ -265,7 +263,7 @@ fn cplace_for_dataid<'tcx>( let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id); let layout = fx.layout_of(fx.monomorphize(&ty)); assert!(!layout.is_unsized(), "unsized statics aren't supported"); - CPlace::for_addr(global_ptr, layout) + CPlace::for_ptr(crate::pointer::Pointer::new(global_ptr), layout) } fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut Module, cx: &mut ConstantCx) { diff --git a/src/intrinsics.rs b/src/intrinsics.rs index e05d32ffd79..af0f8b0be6f 100644 --- a/src/intrinsics.rs +++ b/src/intrinsics.rs @@ -446,7 +446,7 @@ pub fn codegen_intrinsic_call<'tcx>( }; discriminant_value, (c ptr) { let pointee_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); - let val = CValue::by_ref(ptr.load_scalar(fx), pointee_layout); + let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), pointee_layout); let discr = crate::discriminant::codegen_get_discriminant(fx, val, ret.layout()); ret.write_cvalue(fx, discr); }; @@ -598,7 +598,7 @@ pub fn codegen_intrinsic_call<'tcx>( transmute, (c from) { assert_eq!(from.layout().ty, src_ty); - let addr = from.force_stack(fx); + let addr = Pointer::new(from.force_stack(fx)); let dst_layout = fx.layout_of(dst_ty); ret.write_cvalue(fx, CValue::by_ref(addr, dst_layout)) }; @@ -815,12 +815,12 @@ pub fn codegen_intrinsic_call<'tcx>( // Cranelift treats loads as volatile by default let inner_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); - let val = CValue::by_ref(ptr.load_scalar(fx), inner_layout); + let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout); ret.write_cvalue(fx, val); }; volatile_store, (v ptr, c val) { // Cranelift treats stores as volatile by default - let dest = CPlace::for_addr(ptr, val.layout()); + let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout()); dest.write_cvalue(fx, val); }; @@ -854,11 +854,11 @@ pub fn codegen_intrinsic_call<'tcx>( _ if intrinsic.starts_with("atomic_load"), (c ptr) { let inner_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); - let val = CValue::by_ref(ptr.load_scalar(fx), inner_layout); + let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout); ret.write_cvalue(fx, val); }; _ if intrinsic.starts_with("atomic_store"), (v ptr, c val) { - let dest = CPlace::for_addr(ptr, val.layout()); + let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout()); dest.write_cvalue(fx, val); }; _ if intrinsic.starts_with("atomic_xchg"), (v ptr, c src) { @@ -868,7 +868,7 @@ pub fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T))); // Write new - let dest = CPlace::for_addr(ptr, src.layout()); + let dest = CPlace::for_ptr(Pointer::new(ptr), src.layout()); dest.write_cvalue(fx, src); }; _ if intrinsic.starts_with("atomic_cxchg"), (v ptr, v test_old, v new) { // both atomic_cxchg_* and atomic_cxchgweak_* diff --git a/src/lib.rs b/src/lib.rs index 5b1b9239687..dedad3b79c1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -49,6 +49,7 @@ mod llvm_intrinsics; mod main_shim; mod metadata; mod num; +mod pointer; mod pretty_clif; mod target_features_whitelist; mod trap; @@ -106,6 +107,7 @@ mod prelude { pub use crate::cast::*; pub use crate::common::*; pub use crate::debuginfo::{DebugContext, FunctionDebugContext}; + pub use crate::pointer::Pointer; pub use crate::trap::*; pub use crate::unimpl::unimpl; pub use crate::value_and_place::{CPlace, CPlaceInner, CValue}; diff --git a/src/pointer.rs b/src/pointer.rs new file mode 100644 index 00000000000..cadf57255cc --- /dev/null +++ b/src/pointer.rs @@ -0,0 +1,101 @@ +use crate::prelude::*; + +use cranelift::codegen::ir::immediates::Offset32; + +#[derive(Copy, Clone, Debug)] +pub struct Pointer { + base_addr: Value, + offset: Offset32, +} + +impl Pointer { + pub fn new(addr: Value) -> Self { + Pointer { + base_addr: addr, + offset: Offset32::new(0), + } + } + + pub fn const_addr<'a, 'tcx>(fx: &mut FunctionCx<'a, 'tcx, impl Backend>, addr: i64) -> Self { + let addr = fx.bcx.ins().iconst(fx.pointer_type, addr); + Pointer { + base_addr: addr, + offset: Offset32::new(0), + } + } + + pub fn get_addr<'a, 'tcx>(self, fx: &mut FunctionCx<'a, 'tcx, impl Backend>) -> Value { + let offset: i64 = self.offset.into(); + if offset == 0 { + self.base_addr + } else { + fx.bcx.ins().iadd_imm(self.base_addr, offset) + } + } + + pub fn get_addr_and_offset(self) -> (Value, Offset32) { + (self.base_addr, self.offset) + } + + pub fn offset<'a, 'tcx>( + self, + fx: &mut FunctionCx<'a, 'tcx, impl Backend>, + extra_offset: Offset32, + ) -> Self { + self.offset_i64(fx, extra_offset.into()) + } + + pub fn offset_i64<'a, 'tcx>( + self, + fx: &mut FunctionCx<'a, 'tcx, impl Backend>, + extra_offset: i64, + ) -> Self { + if let Some(new_offset) = self.offset.try_add_i64(extra_offset) { + Pointer { + base_addr: self.base_addr, + offset: new_offset, + } + } else { + let base_offset: i64 = self.offset.into(); + if let Some(new_offset) = base_offset.checked_add(extra_offset){ + let addr = fx.bcx.ins().iadd_imm(self.base_addr, new_offset); + Pointer { + base_addr: addr, + offset: Offset32::new(0), + } + } else { + panic!("self.offset ({}) + extra_offset ({}) not representable in i64", base_offset, extra_offset); + } + } + } + + pub fn offset_value<'a, 'tcx>( + self, + fx: &mut FunctionCx<'a, 'tcx, impl Backend>, + extra_offset: Value, + ) -> Self { + let base_addr = fx.bcx.ins().iadd(self.base_addr, extra_offset); + Pointer { + base_addr, + offset: self.offset, + } + } + + pub fn load<'a, 'tcx>( + self, + fx: &mut FunctionCx<'a, 'tcx, impl Backend>, + ty: Type, + flags: MemFlags, + ) -> Value { + fx.bcx.ins().load(ty, flags, self.base_addr, self.offset) + } + + pub fn store<'a, 'tcx>( + self, + fx: &mut FunctionCx<'a, 'tcx, impl Backend>, + value: Value, + flags: MemFlags, + ) { + fx.bcx.ins().store(flags, value, self.base_addr, self.offset); + } +} diff --git a/src/trap.rs b/src/trap.rs index 64d88bc48ce..e297b27debb 100644 --- a/src/trap.rs +++ b/src/trap.rs @@ -91,8 +91,7 @@ pub fn trap_unreachable_ret_value<'tcx>( msg: impl AsRef, ) -> CValue<'tcx> { trap_unimplemented(fx, msg); - let zero = fx.bcx.ins().iconst(fx.pointer_type, 0); - CValue::by_ref(zero, dest_layout) + CValue::by_ref(Pointer::const_addr(fx, 0), dest_layout) } /// Like `trap_unreachable` but returns a fake place for the specified type. @@ -104,6 +103,5 @@ pub fn trap_unreachable_ret_place<'tcx>( msg: impl AsRef, ) -> CPlace<'tcx> { trap_unimplemented(fx, msg); - let zero = fx.bcx.ins().iconst(fx.pointer_type, 0); - CPlace::for_addr(zero, dest_layout) + CPlace::for_ptr(Pointer::const_addr(fx, 0), dest_layout) } diff --git a/src/value_and_place.rs b/src/value_and_place.rs index 64fd30def06..2d3248b94fa 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -1,24 +1,22 @@ use crate::prelude::*; +use cranelift::codegen::ir::immediates::Offset32; + fn codegen_field<'tcx>( fx: &mut FunctionCx<'_, 'tcx, impl Backend>, - base: Value, + base: Pointer, extra: Option, layout: TyLayout<'tcx>, field: mir::Field, -) -> (Value, TyLayout<'tcx>) { +) -> (Pointer, TyLayout<'tcx>) { let field_offset = layout.fields.offset(field.index()); let field_layout = layout.field(&*fx, field.index()); let simple = |fx: &mut FunctionCx<_>| { - if field_offset.bytes() > 0 { - ( - fx.bcx.ins().iadd_imm(base, field_offset.bytes() as i64), - field_layout, - ) - } else { - (base, field_layout) - } + ( + base.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()), + field_layout, + ) }; if let Some(extra) = extra { @@ -44,7 +42,7 @@ fn codegen_field<'tcx>( let offset = fx.bcx.ins().band(and_lhs, and_rhs); ( - fx.bcx.ins().iadd(base, offset), + base.offset_value(fx, offset), field_layout, ) } @@ -54,12 +52,12 @@ fn codegen_field<'tcx>( } } -fn scalar_pair_calculate_b_offset(tcx: TyCtxt<'_>, a_scalar: &Scalar, b_scalar: &Scalar) -> i32 { +fn scalar_pair_calculate_b_offset(tcx: TyCtxt<'_>, a_scalar: &Scalar, b_scalar: &Scalar) -> Offset32 { let b_offset = a_scalar .value .size(&tcx) .align_to(b_scalar.value.align(&tcx).abi); - b_offset.bytes().try_into().unwrap() + Offset32::new(b_offset.bytes().try_into().unwrap()) } /// A read-only value @@ -68,14 +66,14 @@ pub struct CValue<'tcx>(CValueInner, TyLayout<'tcx>); #[derive(Debug, Copy, Clone)] enum CValueInner { - ByRef(Value), + ByRef(Pointer), ByVal(Value), ByValPair(Value, Value), } impl<'tcx> CValue<'tcx> { - pub fn by_ref(value: Value, layout: TyLayout<'tcx>) -> CValue<'tcx> { - CValue(CValueInner::ByRef(value), layout) + pub fn by_ref(ptr: Pointer, layout: TyLayout<'tcx>) -> CValue<'tcx> { + CValue(CValueInner::ByRef(ptr), layout) } pub fn by_val(value: Value, layout: TyLayout<'tcx>) -> CValue<'tcx> { @@ -93,7 +91,7 @@ impl<'tcx> CValue<'tcx> { pub fn force_stack<'a>(self, fx: &mut FunctionCx<'_, 'tcx, impl Backend>) -> Value { let layout = self.1; match self.0 { - CValueInner::ByRef(value) => value, + CValueInner::ByRef(ptr) => ptr.get_addr(fx), CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => { let cplace = CPlace::new_stack_slot(fx, layout.ty); cplace.write_cvalue(fx, self); @@ -104,7 +102,14 @@ impl<'tcx> CValue<'tcx> { pub fn try_to_addr(self) -> Option { match self.0 { - CValueInner::ByRef(addr) => Some(addr), + CValueInner::ByRef(ptr) => { + let (base_addr, offset) = ptr.get_addr_and_offset(); + if offset == Offset32::new(0) { + Some(base_addr) + } else { + None + } + } CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => None, } } @@ -113,13 +118,13 @@ impl<'tcx> CValue<'tcx> { pub fn load_scalar<'a>(self, fx: &mut FunctionCx<'_, 'tcx, impl Backend>) -> Value { let layout = self.1; match self.0 { - CValueInner::ByRef(addr) => { + CValueInner::ByRef(ptr) => { let scalar = match layout.abi { layout::Abi::Scalar(ref scalar) => scalar.clone(), _ => unreachable!(), }; let clif_ty = scalar_to_clif_type(fx.tcx, scalar); - fx.bcx.ins().load(clif_ty, MemFlags::new(), addr, 0) + ptr.load(fx, clif_ty, MemFlags::new()) } CValueInner::ByVal(value) => value, CValueInner::ByValPair(_, _) => bug!("Please use load_scalar_pair for ByValPair"), @@ -133,7 +138,7 @@ impl<'tcx> CValue<'tcx> { ) -> (Value, Value) { let layout = self.1; match self.0 { - CValueInner::ByRef(addr) => { + CValueInner::ByRef(ptr) => { let (a_scalar, b_scalar) = match &layout.abi { layout::Abi::ScalarPair(a, b) => (a, b), _ => unreachable!("load_scalar_pair({:?})", self), @@ -141,8 +146,8 @@ impl<'tcx> CValue<'tcx> { let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); let clif_ty1 = scalar_to_clif_type(fx.tcx, a_scalar.clone()); let clif_ty2 = scalar_to_clif_type(fx.tcx, b_scalar.clone()); - let val1 = fx.bcx.ins().load(clif_ty1, MemFlags::new(), addr, 0); - let val2 = fx.bcx.ins().load(clif_ty2, MemFlags::new(), addr, b_offset); + let val1 = ptr.load(fx, clif_ty1, MemFlags::new()); + let val2 = ptr.offset(fx, b_offset).load(fx, clif_ty2, MemFlags::new()); (val1, val2) } CValueInner::ByVal(_) => bug!("Please use load_scalar for ByVal"), @@ -156,12 +161,12 @@ impl<'tcx> CValue<'tcx> { field: mir::Field, ) -> CValue<'tcx> { let layout = self.1; - let base = match self.0 { - CValueInner::ByRef(addr) => addr, + let ptr = match self.0 { + CValueInner::ByRef(ptr) => ptr, _ => bug!("place_field for {:?}", self), }; - let (field_ptr, field_layout) = codegen_field(fx, base, None, layout, field); + let (field_ptr, field_layout) = codegen_field(fx, ptr, None, layout, field); CValue::by_ref(field_ptr, field_layout) } @@ -224,7 +229,7 @@ pub struct CPlace<'tcx> { #[derive(Debug, Copy, Clone)] pub enum CPlaceInner { Var(Local), - Addr(Value, Option), + Addr(Pointer, Option), Stack(StackSlot), NoPlace, } @@ -282,16 +287,16 @@ impl<'tcx> CPlace<'tcx> { } } - pub fn for_addr(addr: Value, layout: TyLayout<'tcx>) -> CPlace<'tcx> { + pub fn for_ptr(ptr: Pointer, layout: TyLayout<'tcx>) -> CPlace<'tcx> { CPlace { - inner: CPlaceInner::Addr(addr, None), + inner: CPlaceInner::Addr(ptr, None), layout, } } - pub fn for_addr_with_extra(addr: Value, extra: Value, layout: TyLayout<'tcx>) -> CPlace<'tcx> { + pub fn for_ptr_with_extra(ptr: Pointer, extra: Value, layout: TyLayout<'tcx>) -> CPlace<'tcx> { CPlace { - inner: CPlaceInner::Addr(addr, Some(extra)), + inner: CPlaceInner::Addr(ptr, Some(extra)), layout, } } @@ -304,51 +309,58 @@ impl<'tcx> CPlace<'tcx> { fx.bcx.set_val_label(val, cranelift::codegen::ir::ValueLabel::from_u32(var.as_u32())); CValue::by_val(val, layout) } - CPlaceInner::Addr(addr, extra) => { + CPlaceInner::Addr(ptr, extra) => { assert!(extra.is_none(), "unsized values are not yet supported"); - CValue::by_ref(addr, layout) + CValue::by_ref(ptr, layout) } CPlaceInner::Stack(stack_slot) => CValue::by_ref( - fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0), + Pointer::new(fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0)), layout, ), CPlaceInner::NoPlace => CValue::by_ref( - fx.bcx - .ins() - .iconst(fx.pointer_type, fx.pointer_type.bytes() as i64), + Pointer::const_addr(fx, i64::try_from(self.layout.align.pref.bytes()).unwrap()), layout, ), } } - pub fn to_addr(self, fx: &mut FunctionCx<'_, 'tcx, impl Backend>) -> Value { - match self.to_addr_maybe_unsized(fx) { - (addr, None) => addr, + pub fn to_ptr(self, fx: &mut FunctionCx<'_, 'tcx, impl Backend>) -> Pointer { + match self.to_ptr_maybe_unsized(fx) { + (ptr, None) => ptr, (_, Some(_)) => bug!("Expected sized cplace, found {:?}", self), } } + pub fn to_addr(self, fx: &mut FunctionCx<'_, 'tcx, impl Backend>) -> Value { + self.to_ptr(fx).get_addr(fx) + } + + pub fn to_ptr_maybe_unsized( + self, + fx: &mut FunctionCx<'_, 'tcx, impl Backend>, + ) -> (Pointer, Option) { + match self.inner { + CPlaceInner::Addr(ptr, extra) => (ptr, extra), + CPlaceInner::Stack(stack_slot) => ( + Pointer::new(fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0)), + None, + ), + CPlaceInner::NoPlace => { + ( + Pointer::const_addr(fx, i64::try_from(self.layout.align.pref.bytes()).unwrap()), + None, + ) + } + CPlaceInner::Var(_) => bug!("Expected CPlace::Addr, found CPlace::Var"), + } + } + pub fn to_addr_maybe_unsized( self, fx: &mut FunctionCx<'_, 'tcx, impl Backend>, ) -> (Value, Option) { - match self.inner { - CPlaceInner::Addr(addr, extra) => (addr, extra), - CPlaceInner::Stack(stack_slot) => ( - fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0), - None, - ), - CPlaceInner::NoPlace => { - ( - fx.bcx.ins().iconst( - fx.pointer_type, - i64::try_from(self.layout.align.pref.bytes()).unwrap(), - ), - None - ) - } - CPlaceInner::Var(_) => bug!("Expected CPlace::Addr, found CPlace::Var"), - } + let (ptr, extra) = self.to_ptr_maybe_unsized(fx); + (ptr.get_addr(fx), extra) } pub fn write_cvalue(self, fx: &mut FunctionCx<'_, 'tcx, impl Backend>, from: CValue<'tcx>) { @@ -420,16 +432,16 @@ impl<'tcx> CPlace<'tcx> { assert_assignable(fx, from_ty, to_ty); let dst_layout = self.layout(); - let addr = match self.inner { + let to_ptr = match self.inner { CPlaceInner::Var(var) => { let data = from.load_scalar(fx); fx.bcx.set_val_label(data, cranelift::codegen::ir::ValueLabel::from_u32(var.as_u32())); fx.bcx.def_var(mir_var(var), data); return; } - CPlaceInner::Addr(addr, None) => addr, + CPlaceInner::Addr(ptr, None) => ptr, CPlaceInner::Stack(stack_slot) => { - fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0) + Pointer::new(fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0)) } CPlaceInner::NoPlace => { if dst_layout.abi != Abi::Uninhabited { @@ -442,27 +454,29 @@ impl<'tcx> CPlace<'tcx> { match from.0 { CValueInner::ByVal(val) => { - fx.bcx.ins().store(MemFlags::new(), val, addr, 0); + to_ptr.store(fx, val, MemFlags::new()); } CValueInner::ByValPair(value, extra) => match dst_layout.abi { Abi::ScalarPair(ref a_scalar, ref b_scalar) => { let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); - fx.bcx.ins().store(MemFlags::new(), value, addr, 0); - fx.bcx.ins().store(MemFlags::new(), extra, addr, b_offset); + to_ptr.store(fx, value, MemFlags::new()); + to_ptr.offset(fx, b_offset).store(fx, extra, MemFlags::new()); } _ => bug!( "Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi ), }, - CValueInner::ByRef(from_addr) => { + CValueInner::ByRef(from_ptr) => { + let from_addr = from_ptr.get_addr(fx); + let to_addr = to_ptr.get_addr(fx); let src_layout = from.1; let size = dst_layout.size.bytes(); let src_align = src_layout.align.abi.bytes() as u8; let dst_align = dst_layout.align.abi.bytes() as u8; fx.bcx.emit_small_memcpy( fx.module.target_config(), - addr, + to_addr, from_addr, size, dst_align, @@ -478,13 +492,13 @@ impl<'tcx> CPlace<'tcx> { field: mir::Field, ) -> CPlace<'tcx> { let layout = self.layout(); - let (base, extra) = self.to_addr_maybe_unsized(fx); + let (base, extra) = self.to_ptr_maybe_unsized(fx); let (field_ptr, field_layout) = codegen_field(fx, base, extra, layout, field); if field_layout.is_unsized() { - CPlace::for_addr_with_extra(field_ptr, extra.unwrap(), field_layout) + CPlace::for_ptr_with_extra(field_ptr, extra.unwrap(), field_layout) } else { - CPlace::for_addr(field_ptr, field_layout) + CPlace::for_ptr(field_ptr, field_layout) } } @@ -493,9 +507,9 @@ impl<'tcx> CPlace<'tcx> { fx: &mut FunctionCx<'_, 'tcx, impl Backend>, index: Value, ) -> CPlace<'tcx> { - let (elem_layout, addr) = match self.layout().ty.kind { - ty::Array(elem_ty, _) => (fx.layout_of(elem_ty), self.to_addr(fx)), - ty::Slice(elem_ty) => (fx.layout_of(elem_ty), self.to_addr_maybe_unsized(fx).0), + let (elem_layout, ptr) = match self.layout().ty.kind { + ty::Array(elem_ty, _) => (fx.layout_of(elem_ty), self.to_ptr(fx)), + ty::Slice(elem_ty) => (fx.layout_of(elem_ty), self.to_ptr_maybe_unsized(fx).0), _ => bug!("place_index({:?})", self.layout().ty), }; @@ -504,16 +518,16 @@ impl<'tcx> CPlace<'tcx> { .ins() .imul_imm(index, elem_layout.size.bytes() as i64); - CPlace::for_addr(fx.bcx.ins().iadd(addr, offset), elem_layout) + CPlace::for_ptr(ptr.offset_value(fx, offset), elem_layout) } pub fn place_deref(self, fx: &mut FunctionCx<'_, 'tcx, impl Backend>) -> CPlace<'tcx> { let inner_layout = fx.layout_of(self.layout().ty.builtin_deref(true).unwrap().ty); if has_ptr_meta(fx.tcx, inner_layout.ty) { let (addr, extra) = self.to_cvalue(fx).load_scalar_pair(fx); - CPlace::for_addr_with_extra(addr, extra, inner_layout) + CPlace::for_ptr_with_extra(Pointer::new(addr), extra, inner_layout) } else { - CPlace::for_addr(self.to_cvalue(fx).load_scalar(fx), inner_layout) + CPlace::for_ptr(Pointer::new(self.to_cvalue(fx).load_scalar(fx)), inner_layout) } } diff --git a/test.sh b/test.sh index 0d7fd71ff4a..88a86bcf404 100755 --- a/test.sh +++ b/test.sh @@ -78,8 +78,8 @@ hyperfine --runs ${RUN_RUNS:-10} ./raytracer_cg_llvm ./raytracer_cg_clif popd pushd build_sysroot/sysroot_src/src/libcore/tests -rm -r sysroot_src/src/**/*/target/ || true -cargo test +rm -r ./target || true +../../../../../cargo.sh test popd pushd regex