nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-22 06:44:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Auto merge of #130215 - RalfJung:interpret-simd, r=compiler-errors

Browse Source 
interpret: simplify SIMD type handling

This is possible as a follow-up to https://github.com/rust-lang/rust/pull/129403
This commit is contained in:
bors 2024-09-13 13:55:22 +00:00
commit 0307e401c2
15 changed files with 190 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
compiler/rustc_const_eval/src/interpret/intrinsics.rs
View File

@ -384,8 +384,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
sym::simd_insert => {
let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
let elem = &args[2];
let (input, input_len) = self.operand_to_simd(&args[0])?;
let (dest, dest_len) = self.mplace_to_simd(dest)?;
let (input, input_len) = self.project_to_simd(&args[0])?;
let (dest, dest_len) = self.project_to_simd(dest)?;
assert_eq!(input_len, dest_len, "Return vector length must match input length");
// Bounds are not checked by typeck so we have to do it ourselves.
if index >= input_len {
@ -406,7 +406,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
}
sym::simd_extract => {
let index = u64::from(self.read_scalar(&args[1])?.to_u32()?);
let (input, input_len) = self.operand_to_simd(&args[0])?;
let (input, input_len) = self.project_to_simd(&args[0])?;
// Bounds are not checked by typeck so we have to do it ourselves.
if index >= input_len {
throw_ub_format!(

24
compiler/rustc_const_eval/src/interpret/operand.rs
View File

@ -681,30 +681,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
Ok(str)
}
/// Converts a repr(simd) operand into an operand where `place_index` accesses the SIMD elements.
/// Also returns the number of elements.
///
/// Can (but does not always) trigger UB if `op` is uninitialized.
pub fn operand_to_simd(
&self,
op: &OpTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, (MPlaceTy<'tcx, M::Provenance>, u64)> {
// Basically we just transmute this place into an array following simd_size_and_type.
// This only works in memory, but repr(simd) types should never be immediates anyway.
assert!(op.layout.ty.is_simd());
match op.as_mplace_or_imm() {
Left(mplace) => self.mplace_to_simd(&mplace),
Right(imm) => match *imm {
Immediate::Uninit => {
throw_ub!(InvalidUninitBytes(None))
}
Immediate::Scalar(..) | Immediate::ScalarPair(..) => {
bug!("arrays/slices can never have Scalar/ScalarPair layout")
}
},
}
}
/// Read from a local of the current frame.
/// Will not access memory, instead an indirect `Operand` is returned.
///

45
compiler/rustc_const_eval/src/interpret/place.rs
View File

@ -377,13 +377,15 @@ where
Prov: Provenance,
M: Machine<'tcx, Provenance = Prov>,
{
pub fn ptr_with_meta_to_mplace(
fn ptr_with_meta_to_mplace(
&self,
ptr: Pointer<Option<M::Provenance>>,
meta: MemPlaceMeta<M::Provenance>,
layout: TyAndLayout<'tcx>,
unaligned: bool,
) -> MPlaceTy<'tcx, M::Provenance> {
let misaligned = self.is_ptr_misaligned(ptr, layout.align.abi);
let misaligned =
if unaligned { None } else { self.is_ptr_misaligned(ptr, layout.align.abi) };
MPlaceTy { mplace: MemPlace { ptr, meta, misaligned }, layout }
}
@ -393,7 +395,16 @@ where
layout: TyAndLayout<'tcx>,
) -> MPlaceTy<'tcx, M::Provenance> {
assert!(layout.is_sized());
self.ptr_with_meta_to_mplace(ptr, MemPlaceMeta::None, layout)
self.ptr_with_meta_to_mplace(ptr, MemPlaceMeta::None, layout, /*unaligned*/ false)
}
pub fn ptr_to_mplace_unaligned(
&self,
ptr: Pointer<Option<M::Provenance>>,
layout: TyAndLayout<'tcx>,
) -> MPlaceTy<'tcx, M::Provenance> {
assert!(layout.is_sized());
self.ptr_with_meta_to_mplace(ptr, MemPlaceMeta::None, layout, /*unaligned*/ true)
}
/// Take a value, which represents a (thin or wide) reference, and make it a place.
@ -414,7 +425,7 @@ where
// `ref_to_mplace` is called on raw pointers even if they don't actually get dereferenced;
// we hence can't call `size_and_align_of` since that asserts more validity than we want.
let ptr = ptr.to_pointer(self)?;
Ok(self.ptr_with_meta_to_mplace(ptr, meta, layout))
Ok(self.ptr_with_meta_to_mplace(ptr, meta, layout, /*unaligned*/ false))
}
/// Turn a mplace into a (thin or wide) mutable raw pointer, pointing to the same space.
@ -484,23 +495,6 @@ where
Ok(a)
}
/// Converts a repr(simd) place into a place where `place_index` accesses the SIMD elements.
/// Also returns the number of elements.
pub fn mplace_to_simd(
&self,
mplace: &MPlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, (MPlaceTy<'tcx, M::Provenance>, u64)> {
// Basically we want to transmute this place into an array following simd_size_and_type.
let (len, e_ty) = mplace.layout.ty.simd_size_and_type(*self.tcx);
// Some SIMD types have padding, so `len` many `e_ty` does not cover the entire place.
// Therefore we cannot transmute, and instead we project at offset 0, which side-steps
// the size check.
let array_layout = self.layout_of(Ty::new_array(self.tcx.tcx, e_ty, len))?;
assert!(array_layout.size <= mplace.layout.size);
let mplace = mplace.offset(Size::ZERO, array_layout, self)?;
Ok((mplace, len))
}
/// Turn a local in the current frame into a place.
pub fn local_to_place(
&self,
@ -986,7 +980,7 @@ where
span_bug!(self.cur_span(), "cannot allocate space for `extern` type, size is not known")
};
let ptr = self.allocate_ptr(size, align, kind)?;
Ok(self.ptr_with_meta_to_mplace(ptr.into(), meta, layout))
Ok(self.ptr_with_meta_to_mplace(ptr.into(), meta, layout, /*unaligned*/ false))
}
pub fn allocate(
@ -1021,7 +1015,12 @@ where
};
let meta = Scalar::from_target_usize(u64::try_from(str.len()).unwrap(), self);
let layout = self.layout_of(self.tcx.types.str_).unwrap();
Ok(self.ptr_with_meta_to_mplace(ptr.into(), MemPlaceMeta::Meta(meta), layout))
Ok(self.ptr_with_meta_to_mplace(
ptr.into(),
MemPlaceMeta::Meta(meta),
layout,
/*unaligned*/ false,
))
}
pub fn raw_const_to_mplace(

13
compiler/rustc_const_eval/src/interpret/projection.rs
View File

@ -244,6 +244,19 @@ where
base.offset(offset, field_layout, self)
}
/// Converts a repr(simd) value into an array of the right size, such that `project_index`
/// accesses the SIMD elements. Also returns the number of elements.
pub fn project_to_simd<P: Projectable<'tcx, M::Provenance>>(
&self,
base: &P,
) -> InterpResult<'tcx, (P, u64)> {
assert!(base.layout().ty.ty_adt_def().unwrap().repr().simd());
// SIMD types must be newtypes around arrays, so all we have to do is project to their only field.
let array = self.project_field(base, 0)?;
let len = array.len(self)?;
Ok((array, len))
}
fn project_constant_index<P: Projectable<'tcx, M::Provenance>>(
&self,
base: &P,

4
compiler/rustc_const_eval/src/interpret/util.rs
View File

@ -9,7 +9,7 @@ use rustc_middle::ty::{
};
use tracing::debug;
use super::{throw_inval, InterpCx, MPlaceTy, MemPlaceMeta, MemoryKind};
use super::{throw_inval, InterpCx, MPlaceTy, MemoryKind};
use crate::const_eval::{CompileTimeInterpCx, CompileTimeMachine, InterpretationResult};
/// Checks whether a type contains generic parameters which must be instantiated.
@ -103,5 +103,5 @@ pub(crate) fn create_static_alloc<'tcx>(
assert_eq!(ecx.machine.static_root_ids, None);
ecx.machine.static_root_ids = Some((alloc_id, static_def_id));
assert!(ecx.memory.alloc_map.insert(alloc_id, (MemoryKind::Stack, alloc)).is_none());
Ok(ecx.ptr_with_meta_to_mplace(Pointer::from(alloc_id).into(), MemPlaceMeta::None, layout))
Ok(ecx.ptr_to_mplace(Pointer::from(alloc_id).into(), layout))
}

82
src/tools/miri/src/intrinsics/simd.rs
View File

@ -50,8 +50,8 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
| "bitreverse"
=> {
let [op] = check_arg_count(args)?;
let (op, op_len) = this.operand_to_simd(op)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (op, op_len) = this.project_to_simd(op)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, op_len);
@ -200,9 +200,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
use mir::BinOp;
let [left, right] = check_arg_count(args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);
@ -291,10 +291,10 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
}
"fma" => {
let [a, b, c] = check_arg_count(args)?;
let (a, a_len) = this.operand_to_simd(a)?;
let (b, b_len) = this.operand_to_simd(b)?;
let (c, c_len) = this.operand_to_simd(c)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (a, a_len) = this.project_to_simd(a)?;
let (b, b_len) = this.project_to_simd(b)?;
let (c, c_len) = this.project_to_simd(c)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, a_len);
assert_eq!(dest_len, b_len);
@ -345,7 +345,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
use mir::BinOp;
let [op] = check_arg_count(args)?;
let (op, op_len) = this.operand_to_simd(op)?;
let (op, op_len) = this.project_to_simd(op)?;
let imm_from_bool =
|b| ImmTy::from_scalar(Scalar::from_bool(b), this.machine.layouts.bool);
@ -408,7 +408,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
use mir::BinOp;
let [op, init] = check_arg_count(args)?;
let (op, op_len) = this.operand_to_simd(op)?;
let (op, op_len) = this.project_to_simd(op)?;
let init = this.read_immediate(init)?;
let mir_op = match intrinsic_name {
@ -426,10 +426,10 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
}
"select" => {
let [mask, yes, no] = check_arg_count(args)?;
let (mask, mask_len) = this.operand_to_simd(mask)?;
let (yes, yes_len) = this.operand_to_simd(yes)?;
let (no, no_len) = this.operand_to_simd(no)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (mask, mask_len) = this.project_to_simd(mask)?;
let (yes, yes_len) = this.project_to_simd(yes)?;
let (no, no_len) = this.project_to_simd(no)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, mask_len);
assert_eq!(dest_len, yes_len);
@ -448,9 +448,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// Variant of `select` that takes a bitmask rather than a "vector of bool".
"select_bitmask" => {
let [mask, yes, no] = check_arg_count(args)?;
let (yes, yes_len) = this.operand_to_simd(yes)?;
let (no, no_len) = this.operand_to_simd(no)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (yes, yes_len) = this.project_to_simd(yes)?;
let (no, no_len) = this.project_to_simd(no)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
let bitmask_len = dest_len.next_multiple_of(8);
if bitmask_len > 64 {
throw_unsup_format!(
@ -522,7 +522,7 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// Converts a "vector of bool" into a bitmask.
"bitmask" => {
let [op] = check_arg_count(args)?;
let (op, op_len) = this.operand_to_simd(op)?;
let (op, op_len) = this.project_to_simd(op)?;
let bitmask_len = op_len.next_multiple_of(8);
if bitmask_len > 64 {
throw_unsup_format!(
@ -570,8 +570,8 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
}
"cast" | "as" | "cast_ptr" | "expose_provenance" | "with_exposed_provenance" => {
let [op] = check_arg_count(args)?;
let (op, op_len) = this.operand_to_simd(op)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (op, op_len) = this.project_to_simd(op)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, op_len);
@ -627,9 +627,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
}
"shuffle_generic" => {
let [left, right] = check_arg_count(args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
let index = generic_args[2]
.expect_const()
@ -662,15 +662,15 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
}
"shuffle" => {
let [left, right, index] = check_arg_count(args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
// `index` is an array or a SIMD type
let (index, index_len) = match index.layout.ty.kind() {
// FIXME: remove this once `index` must always be a SIMD vector.
ty::Array(..) => (index.assert_mem_place(), index.len(this)?),
_ => this.operand_to_simd(index)?,
ty::Array(..) => (index.clone(), index.len(this)?),
_ => this.project_to_simd(index)?,
};
assert_eq!(left_len, right_len);
@ -699,10 +699,10 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
}
"gather" => {
let [passthru, ptrs, mask] = check_arg_count(args)?;
let (passthru, passthru_len) = this.operand_to_simd(passthru)?;
let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
let (mask, mask_len) = this.operand_to_simd(mask)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (passthru, passthru_len) = this.project_to_simd(passthru)?;
let (ptrs, ptrs_len) = this.project_to_simd(ptrs)?;
let (mask, mask_len) = this.project_to_simd(mask)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, passthru_len);
assert_eq!(dest_len, ptrs_len);
@ -725,9 +725,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
}
"scatter" => {
let [value, ptrs, mask] = check_arg_count(args)?;
let (value, value_len) = this.operand_to_simd(value)?;
let (ptrs, ptrs_len) = this.operand_to_simd(ptrs)?;
let (mask, mask_len) = this.operand_to_simd(mask)?;
let (value, value_len) = this.project_to_simd(value)?;
let (ptrs, ptrs_len) = this.project_to_simd(ptrs)?;
let (mask, mask_len) = this.project_to_simd(mask)?;
assert_eq!(ptrs_len, value_len);
assert_eq!(ptrs_len, mask_len);
@ -745,10 +745,10 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
}
"masked_load" => {
let [mask, ptr, default] = check_arg_count(args)?;
let (mask, mask_len) = this.operand_to_simd(mask)?;
let (mask, mask_len) = this.project_to_simd(mask)?;
let ptr = this.read_pointer(ptr)?;
let (default, default_len) = this.operand_to_simd(default)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (default, default_len) = this.project_to_simd(default)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, mask_len);
assert_eq!(dest_len, default_len);
@ -772,9 +772,9 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
}
"masked_store" => {
let [mask, ptr, vals] = check_arg_count(args)?;
let (mask, mask_len) = this.operand_to_simd(mask)?;
let (mask, mask_len) = this.project_to_simd(mask)?;
let ptr = this.read_pointer(ptr)?;
let (vals, vals_len) = this.operand_to_simd(vals)?;
let (vals, vals_len) = this.project_to_simd(vals)?;
assert_eq!(mask_len, vals_len);

4
src/tools/miri/src/shims/foreign_items.rs
View File

@ -903,8 +903,8 @@ trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> {
name if name.starts_with("llvm.ctpop.v") => {
let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (op, op_len) = this.operand_to_simd(op)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (op, op_len) = this.project_to_simd(op)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, op_len);

12
src/tools/miri/src/shims/x86/avx.rs
View File

@ -159,9 +159,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let [data, control] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (data, data_len) = this.operand_to_simd(data)?;
let (control, control_len) = this.operand_to_simd(control)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (data, data_len) = this.project_to_simd(data)?;
let (control, control_len) = this.project_to_simd(control)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, data_len);
assert_eq!(dest_len, control_len);
@ -193,9 +193,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let [data, control] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (data, data_len) = this.operand_to_simd(data)?;
let (control, control_len) = this.operand_to_simd(control)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (data, data_len) = this.project_to_simd(data)?;
let (control, control_len) = this.project_to_simd(control)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, data_len);
assert_eq!(dest_len, control_len);

38
src/tools/miri/src/shims/x86/avx2.rs
View File

@ -62,10 +62,10 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
assert_eq!(dest.layout, src.layout);
let (src, _) = this.operand_to_simd(src)?;
let (offsets, offsets_len) = this.operand_to_simd(offsets)?;
let (mask, mask_len) = this.operand_to_simd(mask)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (src, _) = this.project_to_simd(src)?;
let (offsets, offsets_len) = this.project_to_simd(offsets)?;
let (mask, mask_len) = this.project_to_simd(mask)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
// There are cases like dest: i32x4, offsets: i64x2
// If dest has more elements than offset, extra dest elements are filled with zero.
@ -118,9 +118,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let [left, right] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(left_len, right_len);
assert_eq!(dest_len.strict_mul(2), left_len);
@ -155,9 +155,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let [left, right] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(left_len, right_len);
assert_eq!(dest_len.strict_mul(2), left_len);
@ -271,9 +271,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let [left, right] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);
@ -330,9 +330,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let [left, right] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(left_len, right_len);
assert_eq!(left_len, dest_len.strict_mul(8));
@ -363,9 +363,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let [left, right] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);

83
src/tools/miri/src/shims/x86/mod.rs
View File

@ -314,9 +314,9 @@ fn bin_op_simd_float_first<'tcx, F: rustc_apfloat::Float>(
right: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);
@ -344,9 +344,9 @@ fn bin_op_simd_float_all<'tcx, F: rustc_apfloat::Float>(
right: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);
@ -430,8 +430,8 @@ fn unary_op_ss<'tcx>(
op: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (op, op_len) = this.operand_to_simd(op)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (op, op_len) = this.project_to_simd(op)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, op_len);
@ -453,8 +453,8 @@ fn unary_op_ps<'tcx>(
op: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (op, op_len) = this.operand_to_simd(op)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (op, op_len) = this.project_to_simd(op)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, op_len);
@ -491,8 +491,8 @@ fn shift_simd_by_scalar<'tcx>(
which: ShiftOp,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (left, left_len) = this.operand_to_simd(left)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, left_len);
// `right` may have a different length, and we only care about its
@ -547,9 +547,9 @@ fn shift_simd_by_simd<'tcx>(
which: ShiftOp,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);
@ -613,9 +613,9 @@ fn round_first<'tcx, F: rustc_apfloat::Float>(
rounding: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);
@ -643,8 +643,8 @@ fn round_all<'tcx, F: rustc_apfloat::Float>(
rounding: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (op, op_len) = this.operand_to_simd(op)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (op, op_len) = this.project_to_simd(op)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, op_len);
@ -695,8 +695,8 @@ fn convert_float_to_int<'tcx>(
rnd: rustc_apfloat::Round,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (op, op_len) = this.operand_to_simd(op)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (op, op_len) = this.project_to_simd(op)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
// Output must be *signed* integers.
assert!(matches!(dest.layout.field(this, 0).ty.kind(), ty::Int(_)));
@ -729,8 +729,8 @@ fn int_abs<'tcx>(
op: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (op, op_len) = this.operand_to_simd(op)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (op, op_len) = this.project_to_simd(op)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(op_len, dest_len);
@ -906,8 +906,8 @@ fn test_bits_masked<'tcx>(
) -> InterpResult<'tcx, (bool, bool)> {
assert_eq!(op.layout, mask.layout);
let (op, op_len) = this.operand_to_simd(op)?;
let (mask, mask_len) = this.operand_to_simd(mask)?;
let (op, op_len) = this.project_to_simd(op)?;
let (mask, mask_len) = this.project_to_simd(mask)?;
assert_eq!(op_len, mask_len);
@ -937,8 +937,8 @@ fn test_high_bits_masked<'tcx>(
) -> InterpResult<'tcx, (bool, bool)> {
assert_eq!(op.layout, mask.layout);
let (op, op_len) = this.operand_to_simd(op)?;
let (mask, mask_len) = this.operand_to_simd(mask)?;
let (op, op_len) = this.project_to_simd(op)?;
let (mask, mask_len) = this.project_to_simd(mask)?;
assert_eq!(op_len, mask_len);
@ -967,8 +967,8 @@ fn mask_load<'tcx>(
mask: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (mask, mask_len) = this.operand_to_simd(mask)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (mask, mask_len) = this.project_to_simd(mask)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, mask_len);
@ -1000,8 +1000,8 @@ fn mask_store<'tcx>(
mask: &OpTy<'tcx>,
value: &OpTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (mask, mask_len) = this.operand_to_simd(mask)?;
let (value, value_len) = this.operand_to_simd(value)?;
let (mask, mask_len) = this.project_to_simd(mask)?;
let (value, value_len) = this.project_to_simd(value)?;
assert_eq!(value_len, mask_len);
@ -1014,9 +1014,12 @@ fn mask_store<'tcx>(
let value = this.project_index(&value, i)?;
if this.read_scalar(&mask)?.to_uint(mask_item_size)? >> high_bit_offset != 0 {
// *Non-inbounds* pointer arithmetic to compute the destination.
// (That's why we can't use a place projection.)
let ptr = ptr.wrapping_offset(value.layout.size * i, &this.tcx);
// Unaligned copy, which is what we want.
this.mem_copy(value.ptr(), ptr, value.layout.size, /*nonoverlapping*/ true)?;
// Deref the pointer *unaligned*, and do the copy.
let dest = this.ptr_to_mplace_unaligned(ptr, value.layout);
this.copy_op(&value, &dest)?;
}
}
@ -1095,9 +1098,9 @@ fn pmulhrsw<'tcx>(
right: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);
@ -1313,9 +1316,9 @@ fn psign<'tcx>(
right: &OpTy<'tcx>,
dest: &MPlaceTy<'tcx>,
) -> InterpResult<'tcx, ()> {
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);

22
src/tools/miri/src/shims/x86/sha.rs
View File

@ -23,7 +23,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// Prefix should have already been checked.
let unprefixed_name = link_name.as_str().strip_prefix("llvm.x86.sha").unwrap();
fn read<'c>(this: &mut MiriInterpCx<'c>, reg: &MPlaceTy<'c>) -> InterpResult<'c, [u32; 4]> {
fn read<'c>(this: &mut MiriInterpCx<'c>, reg: &OpTy<'c>) -> InterpResult<'c, [u32; 4]> {
let mut res = [0; 4];
// We reverse the order because x86 is little endian but the copied implementation uses
// big endian.
@ -53,10 +53,10 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
"256rnds2" => {
let [a, b, k] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (a_reg, a_len) = this.operand_to_simd(a)?;
let (b_reg, b_len) = this.operand_to_simd(b)?;
let (k_reg, k_len) = this.operand_to_simd(k)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (a_reg, a_len) = this.project_to_simd(a)?;
let (b_reg, b_len) = this.project_to_simd(b)?;
let (k_reg, k_len) = this.project_to_simd(k)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(a_len, 4);
assert_eq!(b_len, 4);
@ -74,9 +74,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
"256msg1" => {
let [a, b] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (a_reg, a_len) = this.operand_to_simd(a)?;
let (b_reg, b_len) = this.operand_to_simd(b)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (a_reg, a_len) = this.project_to_simd(a)?;
let (b_reg, b_len) = this.project_to_simd(b)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(a_len, 4);
assert_eq!(b_len, 4);
@ -92,9 +92,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
"256msg2" => {
let [a, b] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (a_reg, a_len) = this.operand_to_simd(a)?;
let (b_reg, b_len) = this.operand_to_simd(b)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (a_reg, a_len) = this.project_to_simd(a)?;
let (b_reg, b_len) = this.project_to_simd(b)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(a_len, 4);
assert_eq!(b_len, 4);

10
src/tools/miri/src/shims/x86/sse.rs
View File

@ -130,8 +130,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let [left, right] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
assert_eq!(left_len, right_len);
@ -157,7 +157,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// Converts the first component of `op` from f32 to i32/i64.
"cvtss2si" | "cvttss2si" | "cvtss2si64" | "cvttss2si64" => {
let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (op, _) = this.operand_to_simd(op)?;
let (op, _) = this.project_to_simd(op)?;
let op = this.read_immediate(&this.project_index(&op, 0)?)?;
@ -187,8 +187,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let [left, right] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, left_len);

24
src/tools/miri/src/shims/x86/sse2.rs
View File

@ -42,9 +42,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let [left, right] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(left_len, right_len);
assert_eq!(dest_len.strict_mul(2), left_len);
@ -81,9 +81,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let [left, right] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
// left and right are u8x16, dest is u64x2
assert_eq!(left_len, right_len);
@ -270,8 +270,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let [left, right] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
assert_eq!(left_len, right_len);
@ -297,7 +297,7 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// Converts the first component of `op` from f64 to i32/i64.
"cvtsd2si" | "cvttsd2si" | "cvtsd2si64" | "cvttsd2si64" => {
let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (op, _) = this.operand_to_simd(op)?;
let (op, _) = this.project_to_simd(op)?;
let op = this.read_immediate(&this.project_index(&op, 0)?)?;
@ -325,9 +325,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let [left, right] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, _) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, _) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, left_len);

10
src/tools/miri/src/shims/x86/sse41.rs
View File

@ -29,9 +29,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let [left, right, imm] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);
@ -118,8 +118,8 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
"phminposuw" => {
let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (op, op_len) = this.operand_to_simd(op)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (op, op_len) = this.project_to_simd(op)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
// Find minimum
let mut min_value = u16::MAX;

12
src/tools/miri/src/shims/x86/ssse3.rs
View File

@ -34,9 +34,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let [left, right] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(dest_len, left_len);
assert_eq!(dest_len, right_len);
@ -84,9 +84,9 @@ pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
let [left, right] =
this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
let (left, left_len) = this.operand_to_simd(left)?;
let (right, right_len) = this.operand_to_simd(right)?;
let (dest, dest_len) = this.mplace_to_simd(dest)?;
let (left, left_len) = this.project_to_simd(left)?;
let (right, right_len) = this.project_to_simd(right)?;
let (dest, dest_len) = this.project_to_simd(dest)?;
assert_eq!(left_len, right_len);
assert_eq!(dest_len.strict_mul(2), left_len);