mirror of
https://github.com/rust-lang/rust.git
synced 2025-06-19 19:17:31 +00:00
881 lines
36 KiB
Rust
881 lines
36 KiB
Rust
use hir::def_id::DefId;
|
|
use rustc_abi::Integer::{I8, I32};
|
|
use rustc_abi::Primitive::{self, Float, Int, Pointer};
|
|
use rustc_abi::{
|
|
AddressSpace, BackendRepr, FIRST_VARIANT, FieldIdx, FieldsShape, HasDataLayout, Layout,
|
|
LayoutCalculatorError, LayoutData, Niche, ReprOptions, Scalar, Size, StructKind, TagEncoding,
|
|
VariantIdx, Variants, WrappingRange,
|
|
};
|
|
use rustc_hashes::Hash64;
|
|
use rustc_index::IndexVec;
|
|
use rustc_middle::bug;
|
|
use rustc_middle::query::Providers;
|
|
use rustc_middle::ty::layout::{
|
|
FloatExt, HasTyCtxt, IntegerExt, LayoutCx, LayoutError, LayoutOf, TyAndLayout,
|
|
};
|
|
use rustc_middle::ty::print::with_no_trimmed_paths;
|
|
use rustc_middle::ty::{
|
|
self, AdtDef, CoroutineArgsExt, EarlyBinder, PseudoCanonicalInput, Ty, TyCtxt, TypeVisitableExt,
|
|
};
|
|
use rustc_session::{DataTypeKind, FieldInfo, FieldKind, SizeKind, VariantInfo};
|
|
use rustc_span::{Symbol, sym};
|
|
use tracing::{debug, instrument};
|
|
use {rustc_abi as abi, rustc_hir as hir};
|
|
|
|
use crate::errors::{NonPrimitiveSimdType, OversizedSimdType, ZeroLengthSimdType};
|
|
|
|
mod invariant;
|
|
|
|
pub(crate) fn provide(providers: &mut Providers) {
|
|
*providers = Providers { layout_of, ..*providers };
|
|
}
|
|
|
|
#[instrument(skip(tcx, query), level = "debug")]
|
|
fn layout_of<'tcx>(
|
|
tcx: TyCtxt<'tcx>,
|
|
query: ty::PseudoCanonicalInput<'tcx, Ty<'tcx>>,
|
|
) -> Result<TyAndLayout<'tcx>, &'tcx LayoutError<'tcx>> {
|
|
let PseudoCanonicalInput { typing_env, value: ty } = query;
|
|
debug!(?ty);
|
|
|
|
// Optimization: We convert to TypingMode::PostAnalysis and convert opaque types in
|
|
// the where bounds to their hidden types. This reduces overall uncached invocations
|
|
// of `layout_of` and is thus a small performance improvement.
|
|
let typing_env = typing_env.with_post_analysis_normalized(tcx);
|
|
let unnormalized_ty = ty;
|
|
|
|
// FIXME: We might want to have two different versions of `layout_of`:
|
|
// One that can be called after typecheck has completed and can use
|
|
// `normalize_erasing_regions` here and another one that can be called
|
|
// before typecheck has completed and uses `try_normalize_erasing_regions`.
|
|
let ty = match tcx.try_normalize_erasing_regions(typing_env, ty) {
|
|
Ok(t) => t,
|
|
Err(normalization_error) => {
|
|
return Err(tcx
|
|
.arena
|
|
.alloc(LayoutError::NormalizationFailure(ty, normalization_error)));
|
|
}
|
|
};
|
|
|
|
if ty != unnormalized_ty {
|
|
// Ensure this layout is also cached for the normalized type.
|
|
return tcx.layout_of(typing_env.as_query_input(ty));
|
|
}
|
|
|
|
let cx = LayoutCx::new(tcx, typing_env);
|
|
|
|
let layout = layout_of_uncached(&cx, ty)?;
|
|
let layout = TyAndLayout { ty, layout };
|
|
|
|
// If we are running with `-Zprint-type-sizes`, maybe record layouts
|
|
// for dumping later.
|
|
if cx.tcx().sess.opts.unstable_opts.print_type_sizes {
|
|
record_layout_for_printing(&cx, layout);
|
|
}
|
|
|
|
invariant::layout_sanity_check(&cx, &layout);
|
|
|
|
Ok(layout)
|
|
}
|
|
|
|
fn error<'tcx>(cx: &LayoutCx<'tcx>, err: LayoutError<'tcx>) -> &'tcx LayoutError<'tcx> {
|
|
cx.tcx().arena.alloc(err)
|
|
}
|
|
|
|
fn map_error<'tcx>(
|
|
cx: &LayoutCx<'tcx>,
|
|
ty: Ty<'tcx>,
|
|
err: LayoutCalculatorError<TyAndLayout<'tcx>>,
|
|
) -> &'tcx LayoutError<'tcx> {
|
|
let err = match err {
|
|
LayoutCalculatorError::SizeOverflow => {
|
|
// This is sometimes not a compile error in `check` builds.
|
|
// See `tests/ui/limits/huge-enum.rs` for an example.
|
|
LayoutError::SizeOverflow(ty)
|
|
}
|
|
LayoutCalculatorError::UnexpectedUnsized(field) => {
|
|
// This is sometimes not a compile error if there are trivially false where clauses.
|
|
// See `tests/ui/layout/trivial-bounds-sized.rs` for an example.
|
|
assert!(field.layout.is_unsized(), "invalid layout error {err:#?}");
|
|
if cx.typing_env.param_env.caller_bounds().is_empty() {
|
|
cx.tcx().dcx().delayed_bug(format!(
|
|
"encountered unexpected unsized field in layout of {ty:?}: {field:#?}"
|
|
));
|
|
}
|
|
LayoutError::Unknown(ty)
|
|
}
|
|
LayoutCalculatorError::EmptyUnion => {
|
|
// This is always a compile error.
|
|
let guar =
|
|
cx.tcx().dcx().delayed_bug(format!("computed layout of empty union: {ty:?}"));
|
|
LayoutError::ReferencesError(guar)
|
|
}
|
|
LayoutCalculatorError::ReprConflict => {
|
|
// packed enums are the only known trigger of this, but others might arise
|
|
let guar = cx
|
|
.tcx()
|
|
.dcx()
|
|
.delayed_bug(format!("computed impossible repr (packed enum?): {ty:?}"));
|
|
LayoutError::ReferencesError(guar)
|
|
}
|
|
LayoutCalculatorError::ZeroLengthSimdType => {
|
|
// Can't be caught in typeck if the array length is generic.
|
|
cx.tcx().dcx().emit_fatal(ZeroLengthSimdType { ty })
|
|
}
|
|
LayoutCalculatorError::OversizedSimdType { max_lanes } => {
|
|
// Can't be caught in typeck if the array length is generic.
|
|
cx.tcx().dcx().emit_fatal(OversizedSimdType { ty, max_lanes })
|
|
}
|
|
LayoutCalculatorError::NonPrimitiveSimdType(field) => {
|
|
// This error isn't caught in typeck, e.g., if
|
|
// the element type of the vector is generic.
|
|
cx.tcx().dcx().emit_fatal(NonPrimitiveSimdType { ty, e_ty: field.ty })
|
|
}
|
|
};
|
|
error(cx, err)
|
|
}
|
|
|
|
fn extract_const_value<'tcx>(
|
|
cx: &LayoutCx<'tcx>,
|
|
ty: Ty<'tcx>,
|
|
ct: ty::Const<'tcx>,
|
|
) -> Result<ty::Value<'tcx>, &'tcx LayoutError<'tcx>> {
|
|
match ct.kind() {
|
|
ty::ConstKind::Value(cv) => Ok(cv),
|
|
ty::ConstKind::Param(_) | ty::ConstKind::Expr(_) => {
|
|
if !ct.has_param() {
|
|
bug!("failed to normalize const, but it is not generic: {ct:?}");
|
|
}
|
|
Err(error(cx, LayoutError::TooGeneric(ty)))
|
|
}
|
|
ty::ConstKind::Unevaluated(_) => {
|
|
let err = if ct.has_param() {
|
|
LayoutError::TooGeneric(ty)
|
|
} else {
|
|
// This case is reachable with unsatisfiable predicates and GCE (which will
|
|
// cause anon consts to inherit the unsatisfiable predicates). For example
|
|
// if we have an unsatisfiable `u8: Trait` bound, then it's not a compile
|
|
// error to mention `[u8; <u8 as Trait>::CONST]`, but we can't compute its
|
|
// layout.
|
|
LayoutError::Unknown(ty)
|
|
};
|
|
Err(error(cx, err))
|
|
}
|
|
ty::ConstKind::Infer(_)
|
|
| ty::ConstKind::Bound(..)
|
|
| ty::ConstKind::Placeholder(_)
|
|
| ty::ConstKind::Error(_) => {
|
|
// `ty::ConstKind::Error` is handled at the top of `layout_of_uncached`
|
|
// (via `ty.error_reported()`).
|
|
bug!("layout_of: unexpected const: {ct:?}");
|
|
}
|
|
}
|
|
}
|
|
|
|
fn layout_of_uncached<'tcx>(
|
|
cx: &LayoutCx<'tcx>,
|
|
ty: Ty<'tcx>,
|
|
) -> Result<Layout<'tcx>, &'tcx LayoutError<'tcx>> {
|
|
// Types that reference `ty::Error` pessimistically don't have a meaningful layout.
|
|
// The only side-effect of this is possibly worse diagnostics in case the layout
|
|
// was actually computable (like if the `ty::Error` showed up only in a `PhantomData`).
|
|
if let Err(guar) = ty.error_reported() {
|
|
return Err(error(cx, LayoutError::ReferencesError(guar)));
|
|
}
|
|
|
|
let tcx = cx.tcx();
|
|
let dl = cx.data_layout();
|
|
let map_layout = |result: Result<_, _>| match result {
|
|
Ok(layout) => Ok(tcx.mk_layout(layout)),
|
|
Err(err) => Err(map_error(cx, ty, err)),
|
|
};
|
|
let scalar_unit = |value: Primitive| {
|
|
let size = value.size(dl);
|
|
assert!(size.bits() <= 128);
|
|
Scalar::Initialized { value, valid_range: WrappingRange::full(size) }
|
|
};
|
|
let scalar = |value: Primitive| tcx.mk_layout(LayoutData::scalar(cx, scalar_unit(value)));
|
|
|
|
let univariant = |tys: &[Ty<'tcx>], kind| {
|
|
let fields = tys.iter().map(|ty| cx.layout_of(*ty)).try_collect::<IndexVec<_, _>>()?;
|
|
let repr = ReprOptions::default();
|
|
map_layout(cx.calc.univariant(&fields, &repr, kind))
|
|
};
|
|
debug_assert!(!ty.has_non_region_infer());
|
|
|
|
Ok(match *ty.kind() {
|
|
ty::Pat(ty, pat) => {
|
|
let layout = cx.layout_of(ty)?.layout;
|
|
let mut layout = LayoutData::clone(&layout.0);
|
|
match *pat {
|
|
ty::PatternKind::Range { start, end } => {
|
|
if let BackendRepr::Scalar(scalar) | BackendRepr::ScalarPair(scalar, _) =
|
|
&mut layout.backend_repr
|
|
{
|
|
scalar.valid_range_mut().start = extract_const_value(cx, ty, start)?
|
|
.try_to_bits(tcx, cx.typing_env)
|
|
.ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?;
|
|
|
|
scalar.valid_range_mut().end = extract_const_value(cx, ty, end)?
|
|
.try_to_bits(tcx, cx.typing_env)
|
|
.ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?;
|
|
|
|
// FIXME(pattern_types): create implied bounds from pattern types in signatures
|
|
// that require that the range end is >= the range start so that we can't hit
|
|
// this error anymore without first having hit a trait solver error.
|
|
// Very fuzzy on the details here, but pattern types are an internal impl detail,
|
|
// so we can just go with this for now
|
|
if scalar.is_signed() {
|
|
let range = scalar.valid_range_mut();
|
|
let start = layout.size.sign_extend(range.start);
|
|
let end = layout.size.sign_extend(range.end);
|
|
if end < start {
|
|
let guar = tcx.dcx().err(format!(
|
|
"pattern type ranges cannot wrap: {start}..={end}"
|
|
));
|
|
|
|
return Err(error(cx, LayoutError::ReferencesError(guar)));
|
|
}
|
|
} else {
|
|
let range = scalar.valid_range_mut();
|
|
if range.end < range.start {
|
|
let guar = tcx.dcx().err(format!(
|
|
"pattern type ranges cannot wrap: {}..={}",
|
|
range.start, range.end
|
|
));
|
|
|
|
return Err(error(cx, LayoutError::ReferencesError(guar)));
|
|
}
|
|
};
|
|
|
|
let niche = Niche {
|
|
offset: Size::ZERO,
|
|
value: scalar.primitive(),
|
|
valid_range: scalar.valid_range(cx),
|
|
};
|
|
|
|
layout.largest_niche = Some(niche);
|
|
|
|
tcx.mk_layout(layout)
|
|
} else {
|
|
bug!("pattern type with range but not scalar layout: {ty:?}, {layout:?}")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Basic scalars.
|
|
ty::Bool => tcx.mk_layout(LayoutData::scalar(
|
|
cx,
|
|
Scalar::Initialized {
|
|
value: Int(I8, false),
|
|
valid_range: WrappingRange { start: 0, end: 1 },
|
|
},
|
|
)),
|
|
ty::Char => tcx.mk_layout(LayoutData::scalar(
|
|
cx,
|
|
Scalar::Initialized {
|
|
value: Int(I32, false),
|
|
valid_range: WrappingRange { start: 0, end: 0x10FFFF },
|
|
},
|
|
)),
|
|
ty::Int(ity) => scalar(Int(abi::Integer::from_int_ty(dl, ity), true)),
|
|
ty::Uint(ity) => scalar(Int(abi::Integer::from_uint_ty(dl, ity), false)),
|
|
ty::Float(fty) => scalar(Float(abi::Float::from_float_ty(fty))),
|
|
ty::FnPtr(..) => {
|
|
let mut ptr = scalar_unit(Pointer(dl.instruction_address_space));
|
|
ptr.valid_range_mut().start = 1;
|
|
tcx.mk_layout(LayoutData::scalar(cx, ptr))
|
|
}
|
|
|
|
// The never type.
|
|
ty::Never => tcx.mk_layout(LayoutData::never_type(cx)),
|
|
|
|
// Potentially-wide pointers.
|
|
ty::Ref(_, pointee, _) | ty::RawPtr(pointee, _) => {
|
|
let mut data_ptr = scalar_unit(Pointer(AddressSpace::DATA));
|
|
if !ty.is_raw_ptr() {
|
|
data_ptr.valid_range_mut().start = 1;
|
|
}
|
|
|
|
if pointee.is_sized(tcx, cx.typing_env) {
|
|
return Ok(tcx.mk_layout(LayoutData::scalar(cx, data_ptr)));
|
|
}
|
|
|
|
let metadata = if let Some(metadata_def_id) = tcx.lang_items().metadata_type() {
|
|
let pointee_metadata = Ty::new_projection(tcx, metadata_def_id, [pointee]);
|
|
let metadata_ty =
|
|
match tcx.try_normalize_erasing_regions(cx.typing_env, pointee_metadata) {
|
|
Ok(metadata_ty) => metadata_ty,
|
|
Err(mut err) => {
|
|
// Usually `<Ty as Pointee>::Metadata` can't be normalized because
|
|
// its struct tail cannot be normalized either, so try to get a
|
|
// more descriptive layout error here, which will lead to less confusing
|
|
// diagnostics.
|
|
//
|
|
// We use the raw struct tail function here to get the first tail
|
|
// that is an alias, which is likely the cause of the normalization
|
|
// error.
|
|
match tcx.try_normalize_erasing_regions(
|
|
cx.typing_env,
|
|
tcx.struct_tail_raw(pointee, |ty| ty, || {}),
|
|
) {
|
|
Ok(_) => {}
|
|
Err(better_err) => {
|
|
err = better_err;
|
|
}
|
|
}
|
|
return Err(error(cx, LayoutError::NormalizationFailure(pointee, err)));
|
|
}
|
|
};
|
|
|
|
let metadata_layout = cx.layout_of(metadata_ty)?;
|
|
// If the metadata is a 1-zst, then the pointer is thin.
|
|
if metadata_layout.is_1zst() {
|
|
return Ok(tcx.mk_layout(LayoutData::scalar(cx, data_ptr)));
|
|
}
|
|
|
|
let BackendRepr::Scalar(metadata) = metadata_layout.backend_repr else {
|
|
return Err(error(cx, LayoutError::Unknown(pointee)));
|
|
};
|
|
|
|
metadata
|
|
} else {
|
|
let unsized_part = tcx.struct_tail_for_codegen(pointee, cx.typing_env);
|
|
|
|
match unsized_part.kind() {
|
|
ty::Foreign(..) => {
|
|
return Ok(tcx.mk_layout(LayoutData::scalar(cx, data_ptr)));
|
|
}
|
|
ty::Slice(_) | ty::Str => scalar_unit(Int(dl.ptr_sized_integer(), false)),
|
|
ty::Dynamic(..) => {
|
|
let mut vtable = scalar_unit(Pointer(AddressSpace::DATA));
|
|
vtable.valid_range_mut().start = 1;
|
|
vtable
|
|
}
|
|
_ => {
|
|
return Err(error(cx, LayoutError::Unknown(pointee)));
|
|
}
|
|
}
|
|
};
|
|
|
|
// Effectively a (ptr, meta) tuple.
|
|
tcx.mk_layout(LayoutData::scalar_pair(cx, data_ptr, metadata))
|
|
}
|
|
|
|
ty::Dynamic(_, _, ty::DynStar) => {
|
|
let mut data = scalar_unit(Pointer(AddressSpace::DATA));
|
|
data.valid_range_mut().start = 0;
|
|
let mut vtable = scalar_unit(Pointer(AddressSpace::DATA));
|
|
vtable.valid_range_mut().start = 1;
|
|
tcx.mk_layout(LayoutData::scalar_pair(cx, data, vtable))
|
|
}
|
|
|
|
// Arrays and slices.
|
|
ty::Array(element, count) => {
|
|
let count = extract_const_value(cx, ty, count)?
|
|
.try_to_target_usize(tcx)
|
|
.ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?;
|
|
|
|
let element = cx.layout_of(element)?;
|
|
map_layout(cx.calc.array_like(&element, Some(count)))?
|
|
}
|
|
ty::Slice(element) => {
|
|
let element = cx.layout_of(element)?;
|
|
map_layout(cx.calc.array_like(&element, None).map(|mut layout| {
|
|
// a randomly chosen value to distinguish slices
|
|
layout.randomization_seed = Hash64::new(0x2dcba99c39784102);
|
|
layout
|
|
}))?
|
|
}
|
|
ty::Str => {
|
|
let element = scalar(Int(I8, false));
|
|
map_layout(cx.calc.array_like(&element, None).map(|mut layout| {
|
|
// another random value
|
|
layout.randomization_seed = Hash64::new(0xc1325f37d127be22);
|
|
layout
|
|
}))?
|
|
}
|
|
|
|
// Odd unit types.
|
|
ty::FnDef(..) | ty::Dynamic(_, _, ty::Dyn) | ty::Foreign(..) => {
|
|
let sized = matches!(ty.kind(), ty::FnDef(..));
|
|
tcx.mk_layout(LayoutData::unit(cx, sized))
|
|
}
|
|
|
|
ty::Coroutine(def_id, args) => {
|
|
use rustc_middle::ty::layout::PrimitiveExt as _;
|
|
|
|
let Some(info) = tcx.coroutine_layout(def_id, args.as_coroutine().kind_ty()) else {
|
|
return Err(error(cx, LayoutError::Unknown(ty)));
|
|
};
|
|
|
|
let local_layouts = info
|
|
.field_tys
|
|
.iter()
|
|
.map(|local| {
|
|
let field_ty = EarlyBinder::bind(local.ty);
|
|
let uninit_ty = Ty::new_maybe_uninit(tcx, field_ty.instantiate(tcx, args));
|
|
cx.spanned_layout_of(uninit_ty, local.source_info.span)
|
|
})
|
|
.try_collect::<IndexVec<_, _>>()?;
|
|
|
|
let prefix_layouts = args
|
|
.as_coroutine()
|
|
.prefix_tys()
|
|
.iter()
|
|
.map(|ty| cx.layout_of(ty))
|
|
.try_collect::<IndexVec<_, _>>()?;
|
|
|
|
let layout = cx
|
|
.calc
|
|
.coroutine(
|
|
&local_layouts,
|
|
prefix_layouts,
|
|
&info.variant_fields,
|
|
&info.storage_conflicts,
|
|
|tag| TyAndLayout {
|
|
ty: tag.primitive().to_ty(tcx),
|
|
layout: tcx.mk_layout(LayoutData::scalar(cx, tag)),
|
|
},
|
|
)
|
|
.map(|mut layout| {
|
|
// this is similar to how ReprOptions populates its field_shuffle_seed
|
|
layout.randomization_seed = tcx.def_path_hash(def_id).0.to_smaller_hash();
|
|
debug!("coroutine layout ({:?}): {:#?}", ty, layout);
|
|
layout
|
|
});
|
|
map_layout(layout)?
|
|
}
|
|
|
|
ty::Closure(_, args) => univariant(args.as_closure().upvar_tys(), StructKind::AlwaysSized)?,
|
|
|
|
ty::CoroutineClosure(_, args) => {
|
|
univariant(args.as_coroutine_closure().upvar_tys(), StructKind::AlwaysSized)?
|
|
}
|
|
|
|
ty::Tuple(tys) => {
|
|
let kind =
|
|
if tys.len() == 0 { StructKind::AlwaysSized } else { StructKind::MaybeUnsized };
|
|
|
|
univariant(tys, kind)?
|
|
}
|
|
|
|
// SIMD vector types.
|
|
ty::Adt(def, args) if def.repr().simd() => {
|
|
// Supported SIMD vectors are ADTs with a single array field:
|
|
//
|
|
// * #[repr(simd)] struct S([T; 4])
|
|
//
|
|
// where T is a primitive scalar (integer/float/pointer).
|
|
let Some(ty::Array(e_ty, e_len)) = def
|
|
.is_struct()
|
|
.then(|| &def.variant(FIRST_VARIANT).fields)
|
|
.filter(|fields| fields.len() == 1)
|
|
.map(|fields| *fields[FieldIdx::ZERO].ty(tcx, args).kind())
|
|
else {
|
|
// Invalid SIMD types should have been caught by typeck by now.
|
|
let guar = tcx.dcx().delayed_bug("#[repr(simd)] was applied to an invalid ADT");
|
|
return Err(error(cx, LayoutError::ReferencesError(guar)));
|
|
};
|
|
|
|
let e_len = extract_const_value(cx, ty, e_len)?
|
|
.try_to_target_usize(tcx)
|
|
.ok_or_else(|| error(cx, LayoutError::Unknown(ty)))?;
|
|
|
|
let e_ly = cx.layout_of(e_ty)?;
|
|
|
|
map_layout(cx.calc.simd_type(e_ly, e_len, def.repr().packed()))?
|
|
}
|
|
|
|
// ADTs.
|
|
ty::Adt(def, args) => {
|
|
// Cache the field layouts.
|
|
let variants = def
|
|
.variants()
|
|
.iter()
|
|
.map(|v| {
|
|
v.fields
|
|
.iter()
|
|
.map(|field| cx.layout_of(field.ty(tcx, args)))
|
|
.try_collect::<IndexVec<_, _>>()
|
|
})
|
|
.try_collect::<IndexVec<VariantIdx, _>>()?;
|
|
|
|
if def.is_union() {
|
|
if def.repr().pack.is_some() && def.repr().align.is_some() {
|
|
let guar = tcx.dcx().span_delayed_bug(
|
|
tcx.def_span(def.did()),
|
|
"union cannot be packed and aligned",
|
|
);
|
|
return Err(error(cx, LayoutError::ReferencesError(guar)));
|
|
}
|
|
|
|
return map_layout(cx.calc.layout_of_union(&def.repr(), &variants));
|
|
}
|
|
|
|
let get_discriminant_type =
|
|
|min, max| abi::Integer::repr_discr(tcx, ty, &def.repr(), min, max);
|
|
|
|
let discriminants_iter = || {
|
|
def.is_enum()
|
|
.then(|| def.discriminants(tcx).map(|(v, d)| (v, d.val as i128)))
|
|
.into_iter()
|
|
.flatten()
|
|
};
|
|
|
|
let dont_niche_optimize_enum = def.repr().inhibit_enum_layout_opt()
|
|
|| def
|
|
.variants()
|
|
.iter_enumerated()
|
|
.any(|(i, v)| v.discr != ty::VariantDiscr::Relative(i.as_u32()));
|
|
|
|
let maybe_unsized = def.is_struct()
|
|
&& def.non_enum_variant().tail_opt().is_some_and(|last_field| {
|
|
let typing_env = ty::TypingEnv::post_analysis(tcx, def.did());
|
|
!tcx.type_of(last_field.did).instantiate_identity().is_sized(tcx, typing_env)
|
|
});
|
|
|
|
let layout = cx
|
|
.calc
|
|
.layout_of_struct_or_enum(
|
|
&def.repr(),
|
|
&variants,
|
|
def.is_enum(),
|
|
def.is_unsafe_cell(),
|
|
tcx.layout_scalar_valid_range(def.did()),
|
|
get_discriminant_type,
|
|
discriminants_iter(),
|
|
dont_niche_optimize_enum,
|
|
!maybe_unsized,
|
|
)
|
|
.map_err(|err| map_error(cx, ty, err))?;
|
|
|
|
if !maybe_unsized && layout.is_unsized() {
|
|
bug!("got unsized layout for type that cannot be unsized {ty:?}: {layout:#?}");
|
|
}
|
|
|
|
// If the struct tail is sized and can be unsized, check that unsizing doesn't move the fields around.
|
|
if cfg!(debug_assertions)
|
|
&& maybe_unsized
|
|
&& def.non_enum_variant().tail().ty(tcx, args).is_sized(tcx, cx.typing_env)
|
|
{
|
|
let mut variants = variants;
|
|
let tail_replacement = cx.layout_of(Ty::new_slice(tcx, tcx.types.u8)).unwrap();
|
|
*variants[FIRST_VARIANT].raw.last_mut().unwrap() = tail_replacement;
|
|
|
|
let Ok(unsized_layout) = cx.calc.layout_of_struct_or_enum(
|
|
&def.repr(),
|
|
&variants,
|
|
def.is_enum(),
|
|
def.is_unsafe_cell(),
|
|
tcx.layout_scalar_valid_range(def.did()),
|
|
get_discriminant_type,
|
|
discriminants_iter(),
|
|
dont_niche_optimize_enum,
|
|
!maybe_unsized,
|
|
) else {
|
|
bug!("failed to compute unsized layout of {ty:?}");
|
|
};
|
|
|
|
let FieldsShape::Arbitrary { offsets: sized_offsets, .. } = &layout.fields else {
|
|
bug!("unexpected FieldsShape for sized layout of {ty:?}: {:?}", layout.fields);
|
|
};
|
|
let FieldsShape::Arbitrary { offsets: unsized_offsets, .. } =
|
|
&unsized_layout.fields
|
|
else {
|
|
bug!(
|
|
"unexpected FieldsShape for unsized layout of {ty:?}: {:?}",
|
|
unsized_layout.fields
|
|
);
|
|
};
|
|
|
|
let (sized_tail, sized_fields) = sized_offsets.raw.split_last().unwrap();
|
|
let (unsized_tail, unsized_fields) = unsized_offsets.raw.split_last().unwrap();
|
|
|
|
if sized_fields != unsized_fields {
|
|
bug!("unsizing {ty:?} changed field order!\n{layout:?}\n{unsized_layout:?}");
|
|
}
|
|
|
|
if sized_tail < unsized_tail {
|
|
bug!("unsizing {ty:?} moved tail backwards!\n{layout:?}\n{unsized_layout:?}");
|
|
}
|
|
}
|
|
|
|
tcx.mk_layout(layout)
|
|
}
|
|
|
|
ty::UnsafeBinder(bound_ty) => {
|
|
let ty = tcx.instantiate_bound_regions_with_erased(bound_ty.into());
|
|
cx.layout_of(ty)?.layout
|
|
}
|
|
|
|
// Types with no meaningful known layout.
|
|
ty::Param(_) | ty::Placeholder(..) => {
|
|
return Err(error(cx, LayoutError::TooGeneric(ty)));
|
|
}
|
|
|
|
ty::Alias(..) => {
|
|
// NOTE(eddyb) `layout_of` query should've normalized these away,
|
|
// if that was possible, so there's no reason to try again here.
|
|
let err = if ty.has_param() {
|
|
LayoutError::TooGeneric(ty)
|
|
} else {
|
|
// This is only reachable with unsatisfiable predicates. For example, if we have
|
|
// `u8: Iterator`, then we can't compute the layout of `<u8 as Iterator>::Item`.
|
|
LayoutError::Unknown(ty)
|
|
};
|
|
return Err(error(cx, err));
|
|
}
|
|
|
|
ty::Bound(..) | ty::CoroutineWitness(..) | ty::Infer(_) | ty::Error(_) => {
|
|
// `ty::Error` is handled at the top of this function.
|
|
bug!("layout_of: unexpected type `{ty}`")
|
|
}
|
|
})
|
|
}
|
|
|
|
fn record_layout_for_printing<'tcx>(cx: &LayoutCx<'tcx>, layout: TyAndLayout<'tcx>) {
|
|
// Ignore layouts that are done with non-empty environments or
|
|
// non-monomorphic layouts, as the user only wants to see the stuff
|
|
// resulting from the final codegen session.
|
|
if layout.ty.has_non_region_param() || !cx.typing_env.param_env.caller_bounds().is_empty() {
|
|
return;
|
|
}
|
|
|
|
// (delay format until we actually need it)
|
|
let record = |kind, packed, opt_discr_size, variants| {
|
|
let type_desc = with_no_trimmed_paths!(format!("{}", layout.ty));
|
|
cx.tcx().sess.code_stats.record_type_size(
|
|
kind,
|
|
type_desc,
|
|
layout.align.abi,
|
|
layout.size,
|
|
packed,
|
|
opt_discr_size,
|
|
variants,
|
|
);
|
|
};
|
|
|
|
match *layout.ty.kind() {
|
|
ty::Adt(adt_def, _) => {
|
|
debug!("print-type-size t: `{:?}` process adt", layout.ty);
|
|
let adt_kind = adt_def.adt_kind();
|
|
let adt_packed = adt_def.repr().pack.is_some();
|
|
let (variant_infos, opt_discr_size) = variant_info_for_adt(cx, layout, adt_def);
|
|
record(adt_kind.into(), adt_packed, opt_discr_size, variant_infos);
|
|
}
|
|
|
|
ty::Coroutine(def_id, args) => {
|
|
debug!("print-type-size t: `{:?}` record coroutine", layout.ty);
|
|
// Coroutines always have a begin/poisoned/end state with additional suspend points
|
|
let (variant_infos, opt_discr_size) =
|
|
variant_info_for_coroutine(cx, layout, def_id, args);
|
|
record(DataTypeKind::Coroutine, false, opt_discr_size, variant_infos);
|
|
}
|
|
|
|
ty::Closure(..) => {
|
|
debug!("print-type-size t: `{:?}` record closure", layout.ty);
|
|
record(DataTypeKind::Closure, false, None, vec![]);
|
|
}
|
|
|
|
_ => {
|
|
debug!("print-type-size t: `{:?}` skip non-nominal", layout.ty);
|
|
}
|
|
};
|
|
}
|
|
|
|
fn variant_info_for_adt<'tcx>(
|
|
cx: &LayoutCx<'tcx>,
|
|
layout: TyAndLayout<'tcx>,
|
|
adt_def: AdtDef<'tcx>,
|
|
) -> (Vec<VariantInfo>, Option<Size>) {
|
|
let build_variant_info = |n: Option<Symbol>, flds: &[Symbol], layout: TyAndLayout<'tcx>| {
|
|
let mut min_size = Size::ZERO;
|
|
let field_info: Vec<_> = flds
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(i, &name)| {
|
|
let field_layout = layout.field(cx, i);
|
|
let offset = layout.fields.offset(i);
|
|
min_size = min_size.max(offset + field_layout.size);
|
|
FieldInfo {
|
|
kind: FieldKind::AdtField,
|
|
name,
|
|
offset: offset.bytes(),
|
|
size: field_layout.size.bytes(),
|
|
align: field_layout.align.abi.bytes(),
|
|
type_name: None,
|
|
}
|
|
})
|
|
.collect();
|
|
|
|
VariantInfo {
|
|
name: n,
|
|
kind: if layout.is_unsized() { SizeKind::Min } else { SizeKind::Exact },
|
|
align: layout.align.abi.bytes(),
|
|
size: if min_size.bytes() == 0 { layout.size.bytes() } else { min_size.bytes() },
|
|
fields: field_info,
|
|
}
|
|
};
|
|
|
|
match layout.variants {
|
|
Variants::Empty => (vec![], None),
|
|
|
|
Variants::Single { index } => {
|
|
debug!("print-type-size `{:#?}` variant {}", layout, adt_def.variant(index).name);
|
|
let variant_def = &adt_def.variant(index);
|
|
let fields: Vec<_> = variant_def.fields.iter().map(|f| f.name).collect();
|
|
(vec![build_variant_info(Some(variant_def.name), &fields, layout)], None)
|
|
}
|
|
|
|
Variants::Multiple { tag, ref tag_encoding, .. } => {
|
|
debug!(
|
|
"print-type-size `{:#?}` adt general variants def {}",
|
|
layout.ty,
|
|
adt_def.variants().len()
|
|
);
|
|
let variant_infos: Vec<_> = adt_def
|
|
.variants()
|
|
.iter_enumerated()
|
|
.map(|(i, variant_def)| {
|
|
let fields: Vec<_> = variant_def.fields.iter().map(|f| f.name).collect();
|
|
build_variant_info(Some(variant_def.name), &fields, layout.for_variant(cx, i))
|
|
})
|
|
.collect();
|
|
|
|
(
|
|
variant_infos,
|
|
match tag_encoding {
|
|
TagEncoding::Direct => Some(tag.size(cx)),
|
|
_ => None,
|
|
},
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
fn variant_info_for_coroutine<'tcx>(
|
|
cx: &LayoutCx<'tcx>,
|
|
layout: TyAndLayout<'tcx>,
|
|
def_id: DefId,
|
|
args: ty::GenericArgsRef<'tcx>,
|
|
) -> (Vec<VariantInfo>, Option<Size>) {
|
|
use itertools::Itertools;
|
|
|
|
let Variants::Multiple { tag, ref tag_encoding, tag_field, .. } = layout.variants else {
|
|
return (vec![], None);
|
|
};
|
|
|
|
let coroutine = cx.tcx().coroutine_layout(def_id, args.as_coroutine().kind_ty()).unwrap();
|
|
let upvar_names = cx.tcx().closure_saved_names_of_captured_variables(def_id);
|
|
|
|
let mut upvars_size = Size::ZERO;
|
|
let upvar_fields: Vec<_> = args
|
|
.as_coroutine()
|
|
.upvar_tys()
|
|
.iter()
|
|
.zip_eq(upvar_names)
|
|
.enumerate()
|
|
.map(|(field_idx, (_, name))| {
|
|
let field_layout = layout.field(cx, field_idx);
|
|
let offset = layout.fields.offset(field_idx);
|
|
upvars_size = upvars_size.max(offset + field_layout.size);
|
|
FieldInfo {
|
|
kind: FieldKind::Upvar,
|
|
name: *name,
|
|
offset: offset.bytes(),
|
|
size: field_layout.size.bytes(),
|
|
align: field_layout.align.abi.bytes(),
|
|
type_name: None,
|
|
}
|
|
})
|
|
.collect();
|
|
|
|
let mut variant_infos: Vec<_> = coroutine
|
|
.variant_fields
|
|
.iter_enumerated()
|
|
.map(|(variant_idx, variant_def)| {
|
|
let variant_layout = layout.for_variant(cx, variant_idx);
|
|
let mut variant_size = Size::ZERO;
|
|
let fields = variant_def
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(field_idx, local)| {
|
|
let field_name = coroutine.field_names[*local];
|
|
let field_layout = variant_layout.field(cx, field_idx);
|
|
let offset = variant_layout.fields.offset(field_idx);
|
|
// The struct is as large as the last field's end
|
|
variant_size = variant_size.max(offset + field_layout.size);
|
|
FieldInfo {
|
|
kind: FieldKind::CoroutineLocal,
|
|
name: field_name.unwrap_or(Symbol::intern(&format!(
|
|
".coroutine_field{}",
|
|
local.as_usize()
|
|
))),
|
|
offset: offset.bytes(),
|
|
size: field_layout.size.bytes(),
|
|
align: field_layout.align.abi.bytes(),
|
|
// Include the type name if there is no field name, or if the name is the
|
|
// __awaitee placeholder symbol which means a child future being `.await`ed.
|
|
type_name: (field_name.is_none() || field_name == Some(sym::__awaitee))
|
|
.then(|| Symbol::intern(&field_layout.ty.to_string())),
|
|
}
|
|
})
|
|
.chain(upvar_fields.iter().copied())
|
|
.collect();
|
|
|
|
// If the variant has no state-specific fields, then it's the size of the upvars.
|
|
if variant_size == Size::ZERO {
|
|
variant_size = upvars_size;
|
|
}
|
|
|
|
// This `if` deserves some explanation.
|
|
//
|
|
// The layout code has a choice of where to place the discriminant of this coroutine.
|
|
// If the discriminant of the coroutine is placed early in the layout (before the
|
|
// variant's own fields), then it'll implicitly be counted towards the size of the
|
|
// variant, since we use the maximum offset to calculate size.
|
|
// (side-note: I know this is a bit problematic given upvars placement, etc).
|
|
//
|
|
// This is important, since the layout printing code always subtracts this discriminant
|
|
// size from the variant size if the struct is "enum"-like, so failing to account for it
|
|
// will either lead to numerical underflow, or an underreported variant size...
|
|
//
|
|
// However, if the discriminant is placed past the end of the variant, then we need
|
|
// to factor in the size of the discriminant manually. This really should be refactored
|
|
// better, but this "works" for now.
|
|
if layout.fields.offset(tag_field) >= variant_size {
|
|
variant_size += match tag_encoding {
|
|
TagEncoding::Direct => tag.size(cx),
|
|
_ => Size::ZERO,
|
|
};
|
|
}
|
|
|
|
VariantInfo {
|
|
name: Some(Symbol::intern(&ty::CoroutineArgs::variant_name(variant_idx))),
|
|
kind: SizeKind::Exact,
|
|
size: variant_size.bytes(),
|
|
align: variant_layout.align.abi.bytes(),
|
|
fields,
|
|
}
|
|
})
|
|
.collect();
|
|
|
|
// The first three variants are hardcoded to be `UNRESUMED`, `RETURNED` and `POISONED`.
|
|
// We will move the `RETURNED` and `POISONED` elements to the end so we
|
|
// are left with a sorting order according to the coroutines yield points:
|
|
// First `Unresumed`, then the `SuspendN` followed by `Returned` and `Panicked` (POISONED).
|
|
let end_states = variant_infos.drain(1..=2);
|
|
let end_states: Vec<_> = end_states.collect();
|
|
variant_infos.extend(end_states);
|
|
|
|
(
|
|
variant_infos,
|
|
match tag_encoding {
|
|
TagEncoding::Direct => Some(tag.size(cx)),
|
|
_ => None,
|
|
},
|
|
)
|
|
}
|