From e69491ac6021572d4ac9cde1c14a04673b4ec859 Mon Sep 17 00:00:00 2001 From: Moulins Date: Fri, 7 Mar 2025 21:17:16 +0100 Subject: [PATCH] Move SIMD layout logic to `rustc_abi` --- compiler/rustc_abi/src/layout.rs | 97 ++++++++++++++++++++++---- compiler/rustc_abi/src/layout/ty.rs | 6 ++ compiler/rustc_abi/src/lib.rs | 7 ++ compiler/rustc_middle/src/ty/layout.rs | 7 +- compiler/rustc_ty_utils/src/layout.rs | 79 +++++---------------- 5 files changed, 117 insertions(+), 79 deletions(-) diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index 42ecbce8117..d0d7cc68a77 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -62,17 +62,28 @@ pub enum LayoutCalculatorError { /// The fields or variants have irreconcilable reprs ReprConflict, + + /// The length of an SIMD type is zero + ZeroLengthSimdType, + + /// The length of an SIMD type exceeds the maximum number of lanes + OversizedSimdType { max_lanes: u64 }, + + /// An element type of an SIMD type isn't a primitive + NonPrimitiveSimdType(F), } impl LayoutCalculatorError { pub fn without_payload(&self) -> LayoutCalculatorError<()> { - match self { - LayoutCalculatorError::UnexpectedUnsized(_) => { - LayoutCalculatorError::UnexpectedUnsized(()) - } - LayoutCalculatorError::SizeOverflow => LayoutCalculatorError::SizeOverflow, - LayoutCalculatorError::EmptyUnion => LayoutCalculatorError::EmptyUnion, - LayoutCalculatorError::ReprConflict => LayoutCalculatorError::ReprConflict, + use LayoutCalculatorError::*; + match *self { + UnexpectedUnsized(_) => UnexpectedUnsized(()), + SizeOverflow => SizeOverflow, + EmptyUnion => EmptyUnion, + ReprConflict => ReprConflict, + ZeroLengthSimdType => ZeroLengthSimdType, + OversizedSimdType { max_lanes } => OversizedSimdType { max_lanes }, + NonPrimitiveSimdType(_) => NonPrimitiveSimdType(()), } } @@ -80,13 +91,15 @@ impl LayoutCalculatorError { /// /// Intended for use by rust-analyzer, as neither it nor `rustc_abi` depend on fluent infra. pub fn fallback_fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use LayoutCalculatorError::*; f.write_str(match self { - LayoutCalculatorError::UnexpectedUnsized(_) => { - "an unsized type was found where a sized type was expected" + UnexpectedUnsized(_) => "an unsized type was found where a sized type was expected", + SizeOverflow => "size overflow", + EmptyUnion => "type is a union with no fields", + ReprConflict => "type has an invalid repr", + ZeroLengthSimdType | OversizedSimdType { .. } | NonPrimitiveSimdType(_) => { + "invalid simd type definition" } - LayoutCalculatorError::SizeOverflow => "size overflow", - LayoutCalculatorError::EmptyUnion => "type is a union with no fields", - LayoutCalculatorError::ReprConflict => "type has an invalid repr", }) } } @@ -127,6 +140,66 @@ impl LayoutCalculator { }) } + pub fn simd_type< + FieldIdx: Idx, + VariantIdx: Idx, + F: AsRef> + fmt::Debug, + >( + &self, + element: F, + count: u64, + repr_packed: bool, + ) -> LayoutCalculatorResult { + let elt = element.as_ref(); + if count == 0 { + return Err(LayoutCalculatorError::ZeroLengthSimdType); + } else if count > crate::MAX_SIMD_LANES { + return Err(LayoutCalculatorError::OversizedSimdType { + max_lanes: crate::MAX_SIMD_LANES, + }); + } + + let BackendRepr::Scalar(e_repr) = elt.backend_repr else { + return Err(LayoutCalculatorError::NonPrimitiveSimdType(element)); + }; + + // Compute the size and alignment of the vector + let dl = self.cx.data_layout(); + let size = + elt.size.checked_mul(count, dl).ok_or_else(|| LayoutCalculatorError::SizeOverflow)?; + let (repr, align) = if repr_packed && !count.is_power_of_two() { + // Non-power-of-two vectors have padding up to the next power-of-two. + // If we're a packed repr, remove the padding while keeping the alignment as close + // to a vector as possible. + ( + BackendRepr::Memory { sized: true }, + AbiAndPrefAlign { + abi: Align::max_aligned_factor(size), + pref: dl.llvmlike_vector_align(size).pref, + }, + ) + } else { + (BackendRepr::SimdVector { element: e_repr, count }, dl.llvmlike_vector_align(size)) + }; + let size = size.align_to(align.abi); + + Ok(LayoutData { + variants: Variants::Single { index: VariantIdx::new(0) }, + fields: FieldsShape::Arbitrary { + offsets: [Size::ZERO].into(), + memory_index: [0].into(), + }, + backend_repr: repr, + largest_niche: elt.largest_niche, + uninhabited: false, + size, + align, + max_repr_align: None, + unadjusted_abi_align: elt.align.abi, + randomization_seed: elt.randomization_seed.wrapping_add(Hash64::new(count)), + }) + } + pub fn univariant< 'a, FieldIdx: Idx, diff --git a/compiler/rustc_abi/src/layout/ty.rs b/compiler/rustc_abi/src/layout/ty.rs index 03f3f043c21..4f43c0e6f8e 100644 --- a/compiler/rustc_abi/src/layout/ty.rs +++ b/compiler/rustc_abi/src/layout/ty.rs @@ -150,6 +150,12 @@ impl<'a, Ty> Deref for TyAndLayout<'a, Ty> { } } +impl<'a, Ty> AsRef> for TyAndLayout<'a, Ty> { + fn as_ref(&self) -> &LayoutData { + &*self.layout.0.0 + } +} + /// Trait that needs to be implemented by the higher-level type representation /// (e.g. `rustc_middle::ty::Ty`), to provide `rustc_target::abi` functionality. pub trait TyAbiInterface<'a, C>: Sized + std::fmt::Debug { diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index db9a26c3ef7..1b737582009 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -205,6 +205,13 @@ impl ReprOptions { } } +/// The maximum supported number of lanes in a SIMD vector. +/// +/// This value is selected based on backend support: +/// * LLVM does not appear to have a vector width limit. +/// * Cranelift stores the base-2 log of the lane count in a 4 bit integer. +pub const MAX_SIMD_LANES: u64 = 1 << 0xF; + /// Parsed [Data layout](https://llvm.org/docs/LangRef.html#data-layout) /// for a target, which contains everything needed to compute layouts. #[derive(Debug, PartialEq, Eq)] diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index d32c524a427..ebb6a8c08a5 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -182,12 +182,7 @@ pub const WIDE_PTR_ADDR: usize = 0; /// - For a slice, this is the length. pub const WIDE_PTR_EXTRA: usize = 1; -/// The maximum supported number of lanes in a SIMD vector. -/// -/// This value is selected based on backend support: -/// * LLVM does not appear to have a vector width limit. -/// * Cranelift stores the base-2 log of the lane count in a 4 bit integer. -pub const MAX_SIMD_LANES: u64 = 1 << 0xF; +pub const MAX_SIMD_LANES: u64 = rustc_abi::MAX_SIMD_LANES; /// Used in `check_validity_requirement` to indicate the kind of initialization /// that is checked to be valid diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index 45c2639280f..fe30b3e1093 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -5,9 +5,9 @@ use hir::def_id::DefId; use rustc_abi::Integer::{I8, I32}; use rustc_abi::Primitive::{self, Float, Int, Pointer}; use rustc_abi::{ - AbiAndPrefAlign, AddressSpace, Align, BackendRepr, FIRST_VARIANT, FieldIdx, FieldsShape, - HasDataLayout, Layout, LayoutCalculatorError, LayoutData, Niche, ReprOptions, Scalar, Size, - StructKind, TagEncoding, VariantIdx, Variants, WrappingRange, + AddressSpace, BackendRepr, FIRST_VARIANT, FieldIdx, FieldsShape, HasDataLayout, Layout, + LayoutCalculatorError, LayoutData, Niche, ReprOptions, Scalar, Size, StructKind, TagEncoding, + VariantIdx, Variants, WrappingRange, }; use rustc_hashes::Hash64; use rustc_index::bit_set::DenseBitSet; @@ -16,7 +16,7 @@ use rustc_middle::bug; use rustc_middle::mir::{CoroutineLayout, CoroutineSavedLocal}; use rustc_middle::query::Providers; use rustc_middle::ty::layout::{ - FloatExt, HasTyCtxt, IntegerExt, LayoutCx, LayoutError, LayoutOf, MAX_SIMD_LANES, TyAndLayout, + FloatExt, HasTyCtxt, IntegerExt, LayoutCx, LayoutError, LayoutOf, TyAndLayout, }; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_middle::ty::{ @@ -124,6 +124,19 @@ fn map_error<'tcx>( .delayed_bug(format!("computed impossible repr (packed enum?): {ty:?}")); LayoutError::ReferencesError(guar) } + LayoutCalculatorError::ZeroLengthSimdType => { + // Can't be caught in typeck if the array length is generic. + cx.tcx().dcx().emit_fatal(ZeroLengthSimdType { ty }) + } + LayoutCalculatorError::OversizedSimdType { max_lanes } => { + // Can't be caught in typeck if the array length is generic. + cx.tcx().dcx().emit_fatal(OversizedSimdType { ty, max_lanes }) + } + LayoutCalculatorError::NonPrimitiveSimdType(field) => { + // This error isn't caught in typeck, e.g., if + // the element type of the vector is generic. + cx.tcx().dcx().emit_fatal(NonPrimitiveSimdType { ty, e_ty: field.ty }) + } }; error(cx, err) } @@ -423,65 +436,9 @@ fn layout_of_uncached<'tcx>( .try_to_target_usize(tcx) .ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?; - // SIMD vectors of zero length are not supported. - // Additionally, lengths are capped at 2^16 as a fixed maximum backends must - // support. - // - // Can't be caught in typeck if the array length is generic. - if e_len == 0 { - tcx.dcx().emit_fatal(ZeroLengthSimdType { ty }); - } else if e_len > MAX_SIMD_LANES { - tcx.dcx().emit_fatal(OversizedSimdType { ty, max_lanes: MAX_SIMD_LANES }); - } - - // Compute the ABI of the element type: let e_ly = cx.layout_of(e_ty)?; - let BackendRepr::Scalar(e_abi) = e_ly.backend_repr else { - // This error isn't caught in typeck, e.g., if - // the element type of the vector is generic. - tcx.dcx().emit_fatal(NonPrimitiveSimdType { ty, e_ty }); - }; - // Compute the size and alignment of the vector: - let size = e_ly - .size - .checked_mul(e_len, dl) - .ok_or_else(|| error(cx, LayoutError::SizeOverflow(ty)))?; - - let (abi, align) = if def.repr().packed() && !e_len.is_power_of_two() { - // Non-power-of-two vectors have padding up to the next power-of-two. - // If we're a packed repr, remove the padding while keeping the alignment as close - // to a vector as possible. - ( - BackendRepr::Memory { sized: true }, - AbiAndPrefAlign { - abi: Align::max_aligned_factor(size), - pref: dl.llvmlike_vector_align(size).pref, - }, - ) - } else { - ( - BackendRepr::SimdVector { element: e_abi, count: e_len }, - dl.llvmlike_vector_align(size), - ) - }; - let size = size.align_to(align.abi); - - tcx.mk_layout(LayoutData { - variants: Variants::Single { index: FIRST_VARIANT }, - fields: FieldsShape::Arbitrary { - offsets: [Size::ZERO].into(), - memory_index: [0].into(), - }, - backend_repr: abi, - largest_niche: e_ly.largest_niche, - uninhabited: false, - size, - align, - max_repr_align: None, - unadjusted_abi_align: align.abi, - randomization_seed: e_ly.randomization_seed.wrapping_add(Hash64::new(e_len)), - }) + map_layout(cx.calc.simd_type(e_ly, e_len, def.repr().packed()))? } // ADTs.