nordic-dev.net/rust-gpu
2
0
Fork 0
mirror of https://github.com/EmbarkStudios/rust-gpu.git synced 2024-11-24 15:54:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

builder: aggressively pointercast/bitcast to paper over opaque pointers.

Browse Source
This commit is contained in:
Eduard-Mihai Burtescu 2023-11-18 00:48:18 +02:00
parent acf85064d0
commit 8858d5f7aa
13 changed files with 405 additions and 268 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

547
crates/rustc_codegen_spirv/src/builder/builder_methods.rs
View File

@ -373,31 +373,108 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
}
}
/// If possible, return the appropriate `OpAccessChain` indices for going from
/// a pointer to `ty`, to a pointer to `leaf_ty`, with an added `offset`.
/// Convenience wrapper for `adjust_pointer_for_sized_access`, falling back
/// on choosing `ty` as the leaf's type (and casting `ptr` to a pointer to it).
//
// HACK(eddyb) temporary workaround for untyped pointers upstream.
// FIXME(eddyb) replace with untyped memory SPIR-V + `qptr` or similar.
fn adjust_pointer_for_typed_access(
&mut self,
ptr: SpirvValue,
ty: <Self as BackendTypes>::Type,
) -> (SpirvValue, <Self as BackendTypes>::Type) {
self.lookup_type(ty)
.sizeof(self)
.and_then(|size| self.adjust_pointer_for_sized_access(ptr, size))
.unwrap_or_else(|| (self.pointercast(ptr, self.type_ptr_to(ty)), ty))
}
/// If `ptr`'s pointee type contains any prefix field/element of size `size`,
/// i.e. some leaf which can be used for all accesses of size `size`, return
/// `ptr` adjusted to point to the innermost such leaf, and the leaf's type.
//
// FIXME(eddyb) technically this duplicates `pointercast`, but the main use
// of `pointercast` is being replaced by this, and this can be more efficient.
//
// HACK(eddyb) temporary workaround for untyped pointers upstream.
// FIXME(eddyb) replace with untyped memory SPIR-V + `qptr` or similar.
fn adjust_pointer_for_sized_access(
&mut self,
ptr: SpirvValue,
size: Size,
) -> Option<(SpirvValue, <Self as BackendTypes>::Type)> {
let ptr = ptr.strip_ptrcasts();
let mut leaf_ty = match self.lookup_type(ptr.ty) {
SpirvType::Pointer { pointee } => pointee,
other => self.fatal(format!("non-pointer type: {other:?}")),
};
// FIXME(eddyb) this isn't efficient, `recover_access_chain_from_offset`
// could instead be doing all the extra digging itself.
let mut indices = SmallVec::<[_; 8]>::new();
while let Some((inner_indices, inner_ty)) =
self.recover_access_chain_from_offset(leaf_ty, Size::ZERO, Some(size), None)
{
indices.extend(inner_indices);
leaf_ty = inner_ty;
}
let leaf_ptr_ty = (self.lookup_type(leaf_ty).sizeof(self) == Some(size))
.then(|| self.type_ptr_to(leaf_ty))?;
let leaf_ptr = if indices.is_empty() {
assert_ty_eq!(self, ptr.ty, leaf_ptr_ty);
ptr
} else {
let indices = indices
.into_iter()
.map(|idx| self.constant_u32(self.span(), idx).def(self))
.collect::<Vec<_>>();
self.emit()
.access_chain(leaf_ptr_ty, None, ptr.def(self), indices)
.unwrap()
.with_type(leaf_ptr_ty)
};
Some((leaf_ptr, leaf_ty))
}
/// If possible, return the appropriate `OpAccessChain` indices for going
/// from a pointer to `ty`, to a pointer to some leaf field/element of size
/// `leaf_size` (and optionally type `leaf_ty`), while adding `offset` bytes.
///
/// That is, try to turn `((_: *T) as *u8).add(offset) as *Leaf` into a series
/// of struct field and array/vector element accesses.
fn recover_access_chain_from_offset(
&self,
mut ty: Word,
leaf_ty: Word,
mut ty: <Self as BackendTypes>::Type,
mut offset: Size,
) -> Option<Vec<u32>> {
assert_ne!(ty, leaf_ty);
// FIXME(eddyb) using `None` for "unsized" is a pretty bad design.
leaf_size_or_unsized: Option<Size>,
leaf_ty: Option<<Self as BackendTypes>::Type>,
) -> Option<(SmallVec<[u32; 8]>, <Self as BackendTypes>::Type)> {
assert_ne!(Some(ty), leaf_ty);
// NOTE(eddyb) `ty` and `ty_kind` should be kept in sync.
// HACK(eddyb) this has the correct ordering (`Sized(_) < Unsized`).
#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
enum MaybeSized {
Sized(Size),
Unsized,
}
let leaf_size = leaf_size_or_unsized.map_or(MaybeSized::Unsized, MaybeSized::Sized);
// NOTE(eddyb) `ty` and `ty_kind`/`ty_size` should be kept in sync.
let mut ty_kind = self.lookup_type(ty);
let mut indices = Vec::new();
let mut indices = SmallVec::new();
loop {
let ty_size;
match ty_kind {
SpirvType::Adt {
field_types,
field_offsets,
..
} => {
let (i, field_ty, field_ty_kind, offset_in_field) = field_offsets
let (i, field_ty, field_ty_kind, field_ty_size, offset_in_field) = field_offsets
.iter()
.enumerate()
.find_map(|(i, &field_offset)| {
@ -409,16 +486,16 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
// the leaf is somewhere inside the field.
let field_ty = field_types[i];
let field_ty_kind = self.lookup_type(field_ty);
let field_ty_size = field_ty_kind
.sizeof(self).map_or(MaybeSized::Unsized, MaybeSized::Sized);
let offset_in_field = offset - field_offset;
if field_ty_kind
.sizeof(self)
.map_or(true, |size| offset_in_field < size)
// If the field is a zero sized type, check the type to
// get the correct entry
|| offset_in_field == Size::ZERO && leaf_ty == field_ty
if MaybeSized::Sized(offset_in_field) < field_ty_size
// If the field is a zero sized type, check the
// expected size and type to get the correct entry
|| offset_in_field == Size::ZERO && leaf_size == MaybeSized::Sized(Size::ZERO) && leaf_ty == Some(field_ty)
{
Some((i, field_ty, field_ty_kind, offset_in_field))
Some((i, field_ty, field_ty_kind, field_ty_size, offset_in_field))
} else {
None
}
@ -426,6 +503,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
ty = field_ty;
ty_kind = field_ty_kind;
ty_size = field_ty_size;
indices.push(i as u32);
offset = offset_in_field;
@ -438,14 +516,24 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
ty_kind = self.lookup_type(ty);
let stride = ty_kind.sizeof(self)?;
ty_size = MaybeSized::Sized(stride);
indices.push((offset.bytes() / stride.bytes()).try_into().ok()?);
offset = Size::from_bytes(offset.bytes() % stride.bytes());
}
_ => return None,
}
if offset == Size::ZERO && ty == leaf_ty {
return Some(indices);
// Avoid digging beyond the point the leaf could actually fit.
if ty_size < leaf_size {
return None;
}
if offset == Size::ZERO
&& ty_size == leaf_size
&& leaf_ty.map_or(true, |leaf_ty| leaf_ty == ty)
{
return Some((indices, ty));
}
}
}
@ -765,6 +853,16 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
}
fn ret(&mut self, value: Self::Value) {
let func_ret_ty = {
let builder = self.emit();
let func = &builder.module_ref().functions[builder.selected_function().unwrap()];
func.def.as_ref().unwrap().result_type.unwrap()
};
// HACK(eddyb) temporary workaround for untyped pointers upstream.
// FIXME(eddyb) replace with untyped memory SPIR-V + `qptr` or similar.
let value = self.bitcast(value, func_ret_ty);
self.emit().ret_value(value.def(self)).unwrap();
}
@ -1055,7 +1153,7 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
}
fn alloca(&mut self, ty: Self::Type, _align: Align) -> Self::Value {
let ptr_ty = SpirvType::Pointer { pointee: ty }.def(self.span(), self);
let ptr_ty = self.type_ptr_to(ty);
// "All OpVariable instructions in a function must be the first instructions in the first block."
let mut builder = self.emit();
builder.select_block(Some(0)).unwrap();
@ -1092,22 +1190,14 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
}
fn load(&mut self, ty: Self::Type, ptr: Self::Value, _align: Align) -> Self::Value {
if let Some(value) = ptr.const_fold_load(self) {
return value;
}
let ty = match self.lookup_type(ptr.ty) {
SpirvType::Pointer { pointee } => {
assert_ty_eq!(self, ty, pointee);
pointee
}
ty => self.fatal(format!(
"load called on variable that wasn't a pointer: {ty:?}"
)),
};
self.emit()
.load(ty, None, ptr.def(self), None, empty())
.unwrap()
.with_type(ty)
let (ptr, access_ty) = self.adjust_pointer_for_typed_access(ptr, ty);
let loaded_val = ptr.const_fold_load(self).unwrap_or_else(|| {
self.emit()
.load(access_ty, None, ptr.def(self), None, empty())
.unwrap()
.with_type(access_ty)
});
self.bitcast(loaded_val, ty)
}
fn volatile_load(&mut self, ty: Self::Type, ptr: Self::Value) -> Self::Value {
@ -1124,31 +1214,24 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
order: AtomicOrdering,
_size: Size,
) -> Self::Value {
let ty = match self.lookup_type(ptr.ty) {
SpirvType::Pointer { pointee } => {
assert_ty_eq!(self, ty, pointee);
pointee
}
ty => self.fatal(format!(
"atomic_load called on variable that wasn't a pointer: {ty:?}"
)),
};
let (ptr, access_ty) = self.adjust_pointer_for_typed_access(ptr, ty);
// TODO: Default to device scope
let memory = self.constant_u32(self.span(), Scope::Device as u32);
let semantics = self.ordering_to_semantics_def(order);
let result = self
.emit()
.atomic_load(
ty,
access_ty,
None,
ptr.def(self),
memory.def(self),
semantics.def(self),
)
.unwrap()
.with_type(ty);
self.validate_atomic(ty, result.def(self));
result
.with_type(access_ty);
self.validate_atomic(access_ty, result.def(self));
self.bitcast(result, ty)
}
fn load_operand(
@ -1230,29 +1313,13 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
}
fn store(&mut self, val: Self::Value, ptr: Self::Value, _align: Align) -> Self::Value {
let ptr_elem_ty = match self.lookup_type(ptr.ty) {
SpirvType::Pointer { pointee } => pointee,
ty => self.fatal(format!(
"store called on variable that wasn't a pointer: {ty:?}"
)),
};
let (ptr, access_ty) = self.adjust_pointer_for_typed_access(ptr, val.ty);
let val = self.bitcast(val, access_ty);
// HACK(eddyb) https://github.com/rust-lang/rust/pull/101483 accidentally
// abused the fact that an `i1` LLVM value will be automatically `zext`'d
// to `i8` by `from_immediate`, and so you can pretend that, from the
// Rust perspective, a `bool` value has the type `u8`, as long as it will
// be stored to memory (which intrinsics all do, for historical reasons)
// - but we don't do that in `from_immediate`, so it's emulated here.
let val = match (self.lookup_type(val.ty), self.lookup_type(ptr_elem_ty)) {
(SpirvType::Bool, SpirvType::Integer(8, false)) => self.zext(val, ptr_elem_ty),
_ => val,
};
assert_ty_eq!(self, ptr_elem_ty, val.ty);
self.emit()
.store(ptr.def(self), val.def(self), None, empty())
.unwrap();
// FIXME(eddyb) this is meant to be a handle the store instruction itself.
val
}
@ -1276,13 +1343,9 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
order: AtomicOrdering,
_size: Size,
) {
let ptr_elem_ty = match self.lookup_type(ptr.ty) {
SpirvType::Pointer { pointee } => pointee,
ty => self.fatal(format!(
"atomic_store called on variable that wasn't a pointer: {ty:?}"
)),
};
assert_ty_eq!(self, ptr_elem_ty, val.ty);
let (ptr, access_ty) = self.adjust_pointer_for_typed_access(ptr, val.ty);
let val = self.bitcast(val, access_ty);
// TODO: Default to device scope
let memory = self.constant_u32(self.span(), Scope::Device as u32);
let semantics = self.ordering_to_semantics_def(order);
@ -1311,70 +1374,60 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
}
fn struct_gep(&mut self, ty: Self::Type, ptr: Self::Value, idx: u64) -> Self::Value {
let pointee = match self.lookup_type(ptr.ty) {
SpirvType::Pointer { pointee } => {
assert_ty_eq!(self, ty, pointee);
pointee
}
other => self.fatal(format!(
"struct_gep not on pointer type: {other:?}, index {idx}"
)),
};
let pointee_kind = self.lookup_type(pointee);
let result_pointee_type = match pointee_kind {
SpirvType::Adt { field_types, .. } => field_types[idx as usize],
let (offset, result_pointee_type) = match self.lookup_type(ty) {
SpirvType::Adt {
field_offsets,
field_types,
..
} => (field_offsets[idx as usize], field_types[idx as usize]),
SpirvType::Array { element, .. }
| SpirvType::RuntimeArray { element, .. }
| SpirvType::Vector { element, .. }
| SpirvType::Matrix { element, .. } => element,
| SpirvType::Matrix { element, .. } => (
self.lookup_type(element).sizeof(self).unwrap() * idx,
element,
),
SpirvType::InterfaceBlock { inner_type } => {
assert_eq!(idx, 0);
inner_type
(Size::ZERO, inner_type)
}
other => self.fatal(format!(
"struct_gep not on struct, array, or vector type: {other:?}, index {idx}"
)),
};
let result_type = SpirvType::Pointer {
pointee: result_pointee_type,
}
.def(self.span(), self);
let result_type = self.type_ptr_to(result_pointee_type);
// Special-case field accesses through a `pointercast`, to accesss the
// right field in the original type, for the `Logical` addressing model.
if let SpirvValueKind::LogicalPtrCast {
original_ptr,
let ptr = ptr.strip_ptrcasts();
let original_pointee_ty = match self.lookup_type(ptr.ty) {
SpirvType::Pointer { pointee } => pointee,
other => self.fatal(format!("struct_gep called on non-pointer type: {other:?}")),
};
if let Some((indices, _)) = self.recover_access_chain_from_offset(
original_pointee_ty,
bitcast_result_id: _,
} = ptr.kind
{
let offset = match pointee_kind {
SpirvType::Adt { field_offsets, .. } => field_offsets[idx as usize],
SpirvType::Array { element, .. }
| SpirvType::RuntimeArray { element, .. }
| SpirvType::Vector { element, .. }
| SpirvType::Matrix { element, .. } => {
self.lookup_type(element).sizeof(self).unwrap() * idx
}
_ => unreachable!(),
};
if let Some(indices) = self.recover_access_chain_from_offset(
original_pointee_ty,
result_pointee_type,
offset,
) {
let indices = indices
.into_iter()
.map(|idx| self.constant_u32(self.span(), idx).def(self))
.collect::<Vec<_>>();
return self
.emit()
.access_chain(result_type, None, original_ptr, indices)
.unwrap()
.with_type(result_type);
}
offset,
self.lookup_type(result_pointee_type).sizeof(self),
Some(result_pointee_type),
) {
let original_ptr = ptr.def(self);
let indices = indices
.into_iter()
.map(|idx| self.constant_u32(self.span(), idx).def(self))
.collect::<Vec<_>>();
return self
.emit()
.access_chain(result_type, None, original_ptr, indices)
.unwrap()
.with_type(result_type);
}
// FIXME(eddyb) can we even get to this point, with valid SPIR-V?
// HACK(eddyb) temporary workaround for untyped pointers upstream.
// FIXME(eddyb) replace with untyped memory SPIR-V + `qptr` or similar.
let ptr = self.pointercast(ptr, self.type_ptr_to(ty));
// Important! LLVM, and therefore intel-compute-runtime, require the `getelementptr` instruction (and therefore
// OpAccessChain) on structs to be a constant i32. Not i64! i32.
if idx > u32::MAX as u64 {
@ -1517,8 +1570,59 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
if val.ty == dest_ty {
val
} else {
let val_is_ptr = matches!(self.lookup_type(val.ty), SpirvType::Pointer { .. });
let dest_is_ptr = matches!(self.lookup_type(dest_ty), SpirvType::Pointer { .. });
let val_ty_kind = self.lookup_type(val.ty);
let dest_ty_kind = self.lookup_type(dest_ty);
// HACK(eddyb) account for bitcasts from/to aggregates not being legal
// in SPIR-V, but still being used to paper over untyped pointers,
// by unpacking/repacking newtype-shaped aggregates as-needed.
let unpack_newtype = |ty, kind| {
if !matches!(kind, SpirvType::Adt { .. } | SpirvType::Array { .. }) {
return None;
}
let size = kind.sizeof(self)?;
let mut leaf_ty = ty;
// FIXME(eddyb) this isn't efficient, `recover_access_chain_from_offset`
// could instead be doing all the extra digging itself.
let mut indices = SmallVec::<[_; 8]>::new();
while let Some((inner_indices, inner_ty)) =
self.recover_access_chain_from_offset(leaf_ty, Size::ZERO, Some(size), None)
{
indices.extend(inner_indices);
leaf_ty = inner_ty;
}
(!indices.is_empty()).then_some((indices, leaf_ty))
};
// Unpack input newtypes, and bitcast the leaf inside, instead.
if let Some((indices, in_leaf_ty)) = unpack_newtype(val.ty, val_ty_kind) {
let in_leaf = self
.emit()
.composite_extract(in_leaf_ty, None, val.def(self), indices)
.unwrap()
.with_type(in_leaf_ty);
return self.bitcast(in_leaf, dest_ty);
}
// Repack output newtypes, after bitcasting the leaf inside, instead.
if let Some((indices, out_leaf_ty)) = unpack_newtype(dest_ty, dest_ty_kind) {
let out_leaf = self.bitcast(val, out_leaf_ty);
let out_agg_undef = self.undef(dest_ty);
return self
.emit()
.composite_insert(
dest_ty,
None,
out_leaf.def(self),
out_agg_undef.def(self),
indices,
)
.unwrap()
.with_type(dest_ty);
}
let val_is_ptr = matches!(val_ty_kind, SpirvType::Pointer { .. });
let dest_is_ptr = matches!(dest_ty_kind, SpirvType::Pointer { .. });
// Reuse the pointer-specific logic in `pointercast` for `*T -> *U`.
if val_is_ptr && dest_is_ptr {
@ -1605,29 +1709,21 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
}
}
fn pointercast(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value {
let (val, val_pointee) = match val.kind {
// Strip a previous `pointercast`, to reveal the original pointer type.
SpirvValueKind::LogicalPtrCast {
original_ptr,
original_pointee_ty,
bitcast_result_id: _,
} => (
original_ptr.with_type(
SpirvType::Pointer {
pointee: original_pointee_ty,
}
.def(self.span(), self),
),
original_pointee_ty,
),
fn pointercast(&mut self, ptr: Self::Value, dest_ty: Self::Type) -> Self::Value {
// HACK(eddyb) reuse the special-casing in `const_bitcast`, which relies
// on adding a pointer type to an untyped pointer (to some const data).
if let SpirvValueKind::IllegalConst(_) = ptr.kind {
return self.const_bitcast(ptr, dest_ty);
}
_ => match self.lookup_type(val.ty) {
SpirvType::Pointer { pointee } => (val, pointee),
other => self.fatal(format!(
"pointercast called on non-pointer source type: {other:?}"
)),
},
// Strip a previous `pointercast`, to reveal the original pointer type.
let ptr = ptr.strip_ptrcasts();
let ptr_pointee = match self.lookup_type(ptr.ty) {
SpirvType::Pointer { pointee } => pointee,
other => self.fatal(format!(
"pointercast called on non-pointer source type: {other:?}"
)),
};
let dest_pointee = match self.lookup_type(dest_ty) {
SpirvType::Pointer { pointee } => pointee,
@ -1635,26 +1731,29 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
"pointercast called on non-pointer dest type: {other:?}"
)),
};
if val.ty == dest_ty {
val
} else if let Some(indices) =
self.recover_access_chain_from_offset(val_pointee, dest_pointee, Size::ZERO)
{
if ptr.ty == dest_ty {
ptr
} else if let Some((indices, _)) = self.recover_access_chain_from_offset(
ptr_pointee,
Size::ZERO,
self.lookup_type(dest_pointee).sizeof(self),
Some(dest_pointee),
) {
let indices = indices
.into_iter()
.map(|idx| self.constant_u32(self.span(), idx).def(self))
.collect::<Vec<_>>();
self.emit()
.access_chain(dest_ty, None, val.def(self), indices)
.access_chain(dest_ty, None, ptr.def(self), indices)
.unwrap()
.with_type(dest_ty)
} else {
// Defer the cast so that it has a chance to be avoided.
let original_ptr = val.def(self);
let original_ptr = ptr.def(self);
SpirvValue {
kind: SpirvValueKind::LogicalPtrCast {
original_ptr,
original_pointee_ty: val_pointee,
original_ptr_ty: ptr.ty,
bitcast_result_id: self.emit().bitcast(dest_ty, None, original_ptr).unwrap(),
},
ty: dest_ty,
@ -1930,17 +2029,33 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
"memcpy with mem flags is not supported yet: {flags:?}"
));
}
let const_size = self.builder.lookup_const_u64(size);
if const_size == Some(0) {
let const_size = self.builder.lookup_const_u64(size).map(Size::from_bytes);
if const_size == Some(Size::ZERO) {
// Nothing to do!
return;
}
let src_pointee = match self.lookup_type(src.ty) {
SpirvType::Pointer { pointee } => Some(pointee),
_ => None,
};
let src_element_size = src_pointee.and_then(|p| self.lookup_type(p).sizeof(self));
if src_element_size.is_some() && src_element_size == const_size.map(Size::from_bytes) {
let typed_copy_dst_src = const_size.and_then(|const_size| {
let dst_adj = self.adjust_pointer_for_sized_access(dst, const_size);
let src_adj = self.adjust_pointer_for_sized_access(src, const_size);
match (dst_adj, src_adj) {
// HACK(eddyb) fill in missing `dst`/`src` with the other side.
(Some((dst, access_ty)), None) => {
Some((dst, self.pointercast(src, self.type_ptr_to(access_ty))))
}
(None, Some((src, access_ty))) => {
Some((self.pointercast(dst, self.type_ptr_to(access_ty)), src))
}
(Some((dst, dst_access_ty)), Some((src, src_access_ty)))
if dst_access_ty == src_access_ty =>
{
Some((dst, src))
}
(None, None) | (Some(_), Some(_)) => None,
}
});
if let Some((dst, src)) = typed_copy_dst_src {
if let Some(const_value) = src.const_fold_load(self) {
self.store(const_value, dst, Align::from_bytes(0).unwrap());
} else {
@ -2095,12 +2210,15 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
}
fn insert_value(&mut self, agg_val: Self::Value, elt: Self::Value, idx: u64) -> Self::Value {
match self.lookup_type(agg_val.ty) {
SpirvType::Adt { field_types, .. } => {
assert_ty_eq!(self, field_types[idx as usize], elt.ty);
}
let field_type = match self.lookup_type(agg_val.ty) {
SpirvType::Adt { field_types, .. } => field_types[idx as usize],
other => self.fatal(format!("insert_value not implemented on type {other:?}")),
};
// HACK(eddyb) temporary workaround for untyped pointers upstream.
// FIXME(eddyb) replace with untyped memory SPIR-V + `qptr` or similar.
let elt = self.bitcast(elt, field_type);
self.emit()
.composite_insert(
agg_val.ty,
@ -2165,23 +2283,23 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
failure_order: AtomicOrdering,
_weak: bool,
) -> Self::Value {
let dst_pointee_ty = match self.lookup_type(dst.ty) {
SpirvType::Pointer { pointee } => pointee,
ty => self.fatal(format!(
"atomic_cmpxchg called on variable that wasn't a pointer: {ty:?}"
)),
};
assert_ty_eq!(self, dst_pointee_ty, cmp.ty);
assert_ty_eq!(self, dst_pointee_ty, src.ty);
self.validate_atomic(dst_pointee_ty, dst.def(self));
assert_ty_eq!(self, cmp.ty, src.ty);
let ty = src.ty;
let (dst, access_ty) = self.adjust_pointer_for_typed_access(dst, ty);
let cmp = self.bitcast(cmp, access_ty);
let src = self.bitcast(src, access_ty);
self.validate_atomic(access_ty, dst.def(self));
// TODO: Default to device scope
let memory = self.constant_u32(self.span(), Scope::Device as u32);
let semantics_equal = self.ordering_to_semantics_def(order);
let semantics_unequal = self.ordering_to_semantics_def(failure_order);
// Note: OpAtomicCompareExchangeWeak is deprecated, and has the same semantics
self.emit()
let result = self
.emit()
.atomic_compare_exchange(
src.ty,
access_ty,
None,
dst.def(self),
memory.def(self),
@ -2191,7 +2309,8 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
cmp.def(self),
)
.unwrap()
.with_type(src.ty)
.with_type(access_ty);
self.bitcast(result, ty)
}
fn atomic_rmw(
@ -2201,48 +2320,45 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
src: Self::Value,
order: AtomicOrdering,
) -> Self::Value {
let dst_pointee_ty = match self.lookup_type(dst.ty) {
SpirvType::Pointer { pointee } => pointee,
ty => self.fatal(format!(
"atomic_rmw called on variable that wasn't a pointer: {ty:?}"
)),
};
assert_ty_eq!(self, dst_pointee_ty, src.ty);
self.validate_atomic(dst_pointee_ty, dst.def(self));
let ty = src.ty;
let (dst, access_ty) = self.adjust_pointer_for_typed_access(dst, ty);
let src = self.bitcast(src, access_ty);
self.validate_atomic(access_ty, dst.def(self));
// TODO: Default to device scope
let memory = self
.constant_u32(self.span(), Scope::Device as u32)
.def(self);
let semantics = self.ordering_to_semantics_def(order).def(self);
let mut emit = self.emit();
use AtomicRmwBinOp::*;
match op {
AtomicXchg => emit.atomic_exchange(
src.ty,
let result = match op {
AtomicXchg => self.emit().atomic_exchange(
access_ty,
None,
dst.def(self),
memory,
semantics,
src.def(self),
),
AtomicAdd => emit.atomic_i_add(
src.ty,
AtomicAdd => self.emit().atomic_i_add(
access_ty,
None,
dst.def(self),
memory,
semantics,
src.def(self),
),
AtomicSub => emit.atomic_i_sub(
src.ty,
AtomicSub => self.emit().atomic_i_sub(
access_ty,
None,
dst.def(self),
memory,
semantics,
src.def(self),
),
AtomicAnd => emit.atomic_and(
src.ty,
AtomicAnd => self.emit().atomic_and(
access_ty,
None,
dst.def(self),
memory,
@ -2250,48 +2366,48 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
src.def(self),
),
AtomicNand => self.fatal("atomic nand is not supported"),
AtomicOr => emit.atomic_or(
src.ty,
AtomicOr => self.emit().atomic_or(
access_ty,
None,
dst.def(self),
memory,
semantics,
src.def(self),
),
AtomicXor => emit.atomic_xor(
src.ty,
AtomicXor => self.emit().atomic_xor(
access_ty,
None,
dst.def(self),
memory,
semantics,
src.def(self),
),
AtomicMax => emit.atomic_s_max(
src.ty,
AtomicMax => self.emit().atomic_s_max(
access_ty,
None,
dst.def(self),
memory,
semantics,
src.def(self),
),
AtomicMin => emit.atomic_s_min(
src.ty,
AtomicMin => self.emit().atomic_s_min(
access_ty,
None,
dst.def(self),
memory,
semantics,
src.def(self),
),
AtomicUMax => emit.atomic_u_max(
src.ty,
AtomicUMax => self.emit().atomic_u_max(
access_ty,
None,
dst.def(self),
memory,
semantics,
src.def(self),
),
AtomicUMin => emit.atomic_u_min(
src.ty,
AtomicUMin => self.emit().atomic_u_min(
access_ty,
None,
dst.def(self),
memory,
@ -2300,7 +2416,8 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
),
}
.unwrap()
.with_type(src.ty)
.with_type(access_ty);
self.bitcast(result, ty)
}
fn atomic_fence(&mut self, order: AtomicOrdering, _scope: SynchronizationScope) {
@ -2394,9 +2511,15 @@ impl<'a, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'tcx> {
),
};
for (argument, &argument_type) in args.iter().zip(argument_types) {
assert_ty_eq!(self, argument.ty, argument_type);
}
// HACK(eddyb) temporary workaround for untyped pointers upstream.
// FIXME(eddyb) replace with untyped memory SPIR-V + `qptr` or similar.
let args: SmallVec<[_; 8]> = args
.iter()
.zip_eq(argument_types)
.map(|(&arg, &expected_type)| self.bitcast(arg, expected_type))
.collect();
let args = &args[..];
let libm_intrinsic = self.libm_intrinsics.borrow().get(&callee_val).copied();
let buffer_load_intrinsic = self
.buffer_load_intrinsic_fn_id

24
crates/rustc_codegen_spirv/src/builder/mod.rs
View File

@ -102,6 +102,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
self.current_span.unwrap_or(DUMMY_SP)
}
// HACK(eddyb) like the `CodegenCx` method but with `self.span()` awareness.
pub fn type_ptr_to(&self, ty: Word) -> Word {
SpirvType::Pointer { pointee: ty }.def(self.span(), self)
}
// Given an ID, check if it's defined by an OpAccessChain, and if it is, return its ptr/indices
fn find_access_chain(&self, id: spirv::Word) -> Option<(spirv::Word, Vec<spirv::Word>)> {
let emit = self.emit();
@ -130,20 +135,16 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
indices: &[SpirvValue],
is_inbounds: bool,
) -> SpirvValue {
// HACK(eddyb) temporary workaround for untyped pointers upstream.
// FIXME(eddyb) replace with untyped memory SPIR-V + `qptr` or similar.
let ptr = self.pointercast(ptr, self.type_ptr_to(ty));
// The first index is an offset to the pointer, the rest are actual members.
// https://llvm.org/docs/GetElementPtr.html
// "An OpAccessChain instruction is the equivalent of an LLVM getelementptr instruction where the first index element is zero."
// https://github.com/gpuweb/gpuweb/issues/33
let mut result_indices = Vec::with_capacity(indices.len() - 1);
let mut result_pointee_type = match self.lookup_type(ptr.ty) {
SpirvType::Pointer { pointee } => {
assert_ty_eq!(self, ty, pointee);
pointee
}
other_type => self.fatal(format!(
"GEP first deref not implemented for type {other_type:?}"
)),
};
let mut result_pointee_type = ty;
for index in indices.iter().cloned().skip(1) {
result_indices.push(index.def(self));
result_pointee_type = match self.lookup_type(result_pointee_type) {
@ -154,10 +155,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
)),
};
}
let result_type = SpirvType::Pointer {
pointee: result_pointee_type,
}
.def(self.span(), self);
let result_type = self.type_ptr_to(result_pointee_type);
let ptr_id = ptr.def(self);
if let Some((original_ptr, mut original_indices)) = self.find_access_chain(ptr_id) {

22
crates/rustc_codegen_spirv/src/builder_spirv.rs
View File

@ -58,8 +58,8 @@ pub enum SpirvValueKind {
/// Pointer value being cast.
original_ptr: Word,
/// Pointee type of `original_ptr`.
original_pointee_ty: Word,
/// Pointer type of `original_ptr`.
original_ptr_ty: Word,
/// Result ID for the `OpBitcast` instruction representing the cast,
/// to attach zombies to.
@ -77,6 +77,18 @@ pub struct SpirvValue {
}
impl SpirvValue {
pub fn strip_ptrcasts(self) -> Self {
match self.kind {
SpirvValueKind::LogicalPtrCast {
original_ptr,
original_ptr_ty,
bitcast_result_id: _,
} => original_ptr.with_type(original_ptr_ty),
_ => self,
}
}
pub fn const_fold_load(self, cx: &CodegenCx<'_>) -> Option<Self> {
match self.kind {
SpirvValueKind::Def(id) | SpirvValueKind::IllegalConst(id) => {
@ -173,7 +185,7 @@ impl SpirvValue {
SpirvValueKind::LogicalPtrCast {
original_ptr: _,
original_pointee_ty,
original_ptr_ty,
bitcast_result_id,
} => {
cx.zombie_with_span(
@ -181,9 +193,9 @@ impl SpirvValue {
span,
&format!(
"cannot cast between pointer types\
\nfrom `*{}`\
\nfrom `{}`\
\n to `{}`",
cx.debug_type(original_pointee_ty),
cx.debug_type(original_ptr_ty),
cx.debug_type(self.ty)
),
);

17
crates/rustc_codegen_spirv/src/codegen_cx/constant.rs
View File

@ -370,7 +370,8 @@ impl<'tcx> ConstMethods<'tcx> for CodegenCx<'tcx> {
self.builder.lookup_const_by_id(pointee)
{
if let SpirvType::Pointer { pointee } = self.lookup_type(ty) {
let init = self.create_const_alloc(alloc, pointee);
let mut offset = Size::ZERO;
let init = self.read_from_const_alloc(alloc, &mut offset, pointee);
return self.static_addr_of(init, alloc.inner().align, None);
}
}
@ -422,7 +423,7 @@ impl<'tcx> CodegenCx<'tcx> {
// alloc.len()
// );
let mut offset = Size::ZERO;
let result = self.create_const_alloc2(alloc, &mut offset, ty);
let result = self.read_from_const_alloc(alloc, &mut offset, ty);
assert_eq!(
offset.bytes_usize(),
alloc.inner().len(),
@ -432,7 +433,7 @@ impl<'tcx> CodegenCx<'tcx> {
result
}
fn create_const_alloc2(
fn read_from_const_alloc(
&self,
alloc: ConstAllocation<'tcx>,
offset: &mut Size,
@ -515,7 +516,7 @@ impl<'tcx> CodegenCx<'tcx> {
let total_offset_start = base + field_offset;
let mut total_offset_end = total_offset_start;
values.push(
self.create_const_alloc2(alloc, &mut total_offset_end, ty)
self.read_from_const_alloc(alloc, &mut total_offset_end, ty)
.def_cx(self),
);
occupied_spaces.push(total_offset_start..total_offset_end);
@ -534,7 +535,7 @@ impl<'tcx> CodegenCx<'tcx> {
SpirvType::Array { element, count } => {
let count = self.builder.lookup_const_u64(count).unwrap() as usize;
let values = (0..count).map(|_| {
self.create_const_alloc2(alloc, offset, element)
self.read_from_const_alloc(alloc, offset, element)
.def_cx(self)
});
self.constant_composite(ty, values)
@ -545,7 +546,7 @@ impl<'tcx> CodegenCx<'tcx> {
.expect("create_const_alloc: Vectors must be sized");
let final_offset = *offset + total_size;
let values = (0..count).map(|_| {
self.create_const_alloc2(alloc, offset, element)
self.read_from_const_alloc(alloc, offset, element)
.def_cx(self)
});
let result = self.constant_composite(ty, values);
@ -560,7 +561,7 @@ impl<'tcx> CodegenCx<'tcx> {
.expect("create_const_alloc: Matrices must be sized");
let final_offset = *offset + total_size;
let values = (0..count).map(|_| {
self.create_const_alloc2(alloc, offset, element)
self.read_from_const_alloc(alloc, offset, element)
.def_cx(self)
});
let result = self.constant_composite(ty, values);
@ -573,7 +574,7 @@ impl<'tcx> CodegenCx<'tcx> {
let mut values = Vec::new();
while offset.bytes_usize() != alloc.inner().len() {
values.push(
self.create_const_alloc2(alloc, offset, element)
self.read_from_const_alloc(alloc, offset, element)
.def_cx(self),
);
}

2
crates/rustc_codegen_spirv/src/codegen_cx/type_.rs
View File

@ -194,7 +194,7 @@ impl<'tcx> BaseTypeMethods<'tcx> for CodegenCx<'tcx> {
align,
size,
field_types: els,
field_offsets: &field_offsets.as_slice(),
field_offsets: &field_offsets,
field_names: None,
}
.def(DUMMY_SP, self)

1
crates/rustc_codegen_spirv/src/lib.rs
View File

@ -16,6 +16,7 @@
//! [`spirv-tools`]: https://embarkstudios.github.io/rust-gpu/api/spirv_tools
//! [`spirv-tools-sys`]: https://embarkstudios.github.io/rust-gpu/api/spirv_tools_sys
#![feature(rustc_private)]
#![feature(array_methods)]
#![feature(assert_matches)]
#![feature(result_flattening)]
#![feature(lint_reasons)]

8
crates/spirv-std/macros/src/lib.rs
View File

@ -740,10 +740,14 @@ impl SampleImplRewriter {
fn add_regs(&self, t: &mut Vec<TokenTree>) {
for i in 0..SAMPLE_PARAM_COUNT {
if self.0 & (1 << i) != 0 {
// HACK(eddyb) the extra `{...}` force the pointers to be to
// fresh variables holding value copies, instead of the originals,
// allowing `OpLoad _` inference to pick the appropriate type.
let s = if is_grad(i) {
String::from("grad_x=in(reg) &params.grad.0.0,grad_y=in(reg) &params.grad.0.1,")
"grad_x=in(reg) &{params.grad.0.0},grad_y=in(reg) &{params.grad.0.1},"
.to_string()
} else {
format!("{0} = in(reg) &params.{0}.0,", SAMPLE_PARAM_NAMES[i])
format!("{0} = in(reg) &{{params.{0}.0}},", SAMPLE_PARAM_NAMES[i])
};
let ts: proc_macro2::TokenStream = s.parse().unwrap();
t.extend(ts);

36
tests/ui/arch/debug_printf_type_checking.stderr
View File

@ -75,9 +75,9 @@ help: the return type of this call is `u32` due to the type of the argument pass
| |
| this argument influences the return type of `spirv_std`
note: function defined here
--> $SPIRV_STD_SRC/lib.rs:135:8
--> $SPIRV_STD_SRC/lib.rs:136:8
|
135 | pub fn debug_printf_assert_is_type<T>(ty: T) -> T {
136 | pub fn debug_printf_assert_is_type<T>(ty: T) -> T {
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^
= note: this error originates in the macro `debug_printf` (in Nightly builds, run with -Z macro-backtrace for more info)
help: change the type of the numeric literal from `u32` to `f32`
@ -102,9 +102,9 @@ help: the return type of this call is `f32` due to the type of the argument pass
| |
| this argument influences the return type of `spirv_std`
note: function defined here
--> $SPIRV_STD_SRC/lib.rs:135:8
--> $SPIRV_STD_SRC/lib.rs:136:8
|
135 | pub fn debug_printf_assert_is_type<T>(ty: T) -> T {
136 | pub fn debug_printf_assert_is_type<T>(ty: T) -> T {
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^
= note: this error originates in the macro `debug_printf` (in Nightly builds, run with -Z macro-backtrace for more info)
help: change the type of the numeric literal from `f32` to `u32`
@ -113,32 +113,30 @@ help: change the type of the numeric literal from `f32` to `u32`
| ~~~
error[E0277]: the trait bound `{float}: Vector<f32, 2>` is not satisfied
--> $DIR/debug_printf_type_checking.rs:23:31
--> $DIR/debug_printf_type_checking.rs:23:9
|
23 | debug_printf!("%v2f", 11.0);
| ----------------------^^^^-
| | |
| | the trait `Vector<f32, 2>` is not implemented for `{float}`
| required by a bound introduced by this call
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^ the trait `Vector<f32, 2>` is not implemented for `{float}`
|
= help: the following other types implement trait `Vector<T, N>`:
<Vec2 as Vector<f32, 2>>
<Vec3 as Vector<f32, 3>>
<Vec3A as Vector<f32, 3>>
<Vec4 as Vector<f32, 4>>
<DVec2 as Vector<f64, 2>>
<DVec3 as Vector<f64, 3>>
<DVec4 as Vector<f64, 4>>
<IVec2 as Vector<i32, 2>>
<IVec3 as Vector<i32, 3>>
<IVec4 as Vector<i32, 4>>
<UVec2 as Vector<u32, 2>>
<UVec3 as Vector<u32, 3>>
and 5 others
note: required by a bound in `debug_printf_assert_is_vector`
--> $SPIRV_STD_SRC/lib.rs:142:8
--> $SPIRV_STD_SRC/lib.rs:143:8
|
140 | pub fn debug_printf_assert_is_vector<
141 | pub fn debug_printf_assert_is_vector<
| ----------------------------- required by a bound in this function
141 | TY: crate::scalar::Scalar,
142 | V: crate::vector::Vector<TY, SIZE>,
142 | TY: crate::scalar::Scalar,
143 | V: crate::vector::Vector<TY, SIZE>,
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `debug_printf_assert_is_vector`
= note: this error originates in the macro `debug_printf` (in Nightly builds, run with -Z macro-backtrace for more info)
error[E0308]: mismatched types
--> $DIR/debug_printf_type_checking.rs:24:29
@ -157,9 +155,9 @@ help: the return type of this call is `Vec2` due to the type of the argument pas
| |
| this argument influences the return type of `spirv_std`
note: function defined here
--> $SPIRV_STD_SRC/lib.rs:135:8
--> $SPIRV_STD_SRC/lib.rs:136:8
|
135 | pub fn debug_printf_assert_is_type<T>(ty: T) -> T {
136 | pub fn debug_printf_assert_is_type<T>(ty: T) -> T {
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^
= note: this error originates in the macro `debug_printf` (in Nightly builds, run with -Z macro-backtrace for more info)

8
tests/ui/dis/ptr_copy.normal.stderr
View File

@ -1,13 +1,13 @@
error: cannot memcpy dynamically sized data
--> $CORE_SRC/intrinsics.rs:2767:9
--> $CORE_SRC/intrinsics.rs:2771:9
|
2767 | copy(src, dst, count)
2771 | copy(src, dst, count)
| ^^^^^^^^^^^^^^^^^^^^^
|
note: used from within `core::intrinsics::copy::<f32>`
--> $CORE_SRC/intrinsics.rs:2753:21
--> $CORE_SRC/intrinsics.rs:2757:21
|
2753 | pub const unsafe fn copy<T>(src: *const T, dst: *mut T, count: usize) {
2757 | pub const unsafe fn copy<T>(src: *const T, dst: *mut T, count: usize) {
| ^^^^
note: called by `ptr_copy::copy_via_raw_ptr`
--> $DIR/ptr_copy.rs:28:18

2
tests/ui/dis/ptr_read.stderr
View File

@ -2,7 +2,7 @@
%4 = OpFunctionParameter %5
%6 = OpFunctionParameter %5
%7 = OpLabel
OpLine %8 1179 8
OpLine %8 1180 8
%9 = OpLoad %10 %4
OpLine %11 7 13
OpStore %6 %9

2
tests/ui/dis/ptr_read_method.stderr
View File

@ -2,7 +2,7 @@
%4 = OpFunctionParameter %5
%6 = OpFunctionParameter %5
%7 = OpLabel
OpLine %8 1179 8
OpLine %8 1180 8
%9 = OpLoad %10 %4
OpLine %11 7 13
OpStore %6 %9

2
tests/ui/dis/ptr_write.stderr
View File

@ -4,7 +4,7 @@
%7 = OpLabel
OpLine %8 7 35
%9 = OpLoad %10 %4
OpLine %11 1377 8
OpLine %11 1379 8
OpStore %6 %9
OpNoLine
OpReturn

2
tests/ui/dis/ptr_write_method.stderr
View File

@ -4,7 +4,7 @@
%7 = OpLabel
OpLine %8 7 37
%9 = OpLoad %10 %4
OpLine %11 1377 8
OpLine %11 1379 8
OpStore %6 %9
OpNoLine
OpReturn