nordic-dev.net/rust
nordic-dev.net/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2024-11-26 08:44:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Auto merge of #94075 - mikebenfield:wip-enum, r=oli-obk

Browse Source 
Use niche-filling optimization even when multiple variants have data.

Fixes #46213
This commit is contained in:
bors 2022-09-07 23:40:06 +00:00
commit 512bd84f51
22 changed files with 376 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
compiler/rustc_ast/src/ast.rs
View File

@ -3075,7 +3075,8 @@ mod size_asserts {
static_assert_size!(Block, 48);
static_assert_size!(Expr, 104);
static_assert_size!(ExprKind, 72);
static_assert_size!(Fn, 192);
#[cfg(not(bootstrap))]
static_assert_size!(Fn, 184);
static_assert_size!(ForeignItem, 96);
static_assert_size!(ForeignItemKind, 24);
static_assert_size!(GenericArg, 24);

11
compiler/rustc_codegen_cranelift/src/discriminant.rs
View File

@ -42,10 +42,10 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
Variants::Multiple {
tag: _,
tag_field,
tag_encoding: TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start },
tag_encoding: TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start },
variants: _,
} => {
if variant_index != dataful_variant {
if variant_index != untagged_variant {
let niche = place.place_field(fx, mir::Field::new(tag_field));
let niche_value = variant_index.as_u32() - niche_variants.start().as_u32();
let niche_value = ty::ScalarInt::try_from_uint(
@ -113,7 +113,7 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
let res = CValue::by_val(val, dest_layout);
dest.write_cvalue(fx, res);
}
TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => {
TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
// Rebase from niche values to discriminants, and check
// whether the result is in range for the niche variants.
@ -169,8 +169,9 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
fx.bcx.ins().iadd_imm(relative_discr, i64::from(niche_variants.start().as_u32()))
};
let dataful_variant = fx.bcx.ins().iconst(cast_to, i64::from(dataful_variant.as_u32()));
let discr = fx.bcx.ins().select(is_niche, niche_discr, dataful_variant);
let untagged_variant =
fx.bcx.ins().iconst(cast_to, i64::from(untagged_variant.as_u32()));
let discr = fx.bcx.ins().select(is_niche, niche_discr, untagged_variant);
let res = CValue::by_val(discr, dest_layout);
dest.write_cvalue(fx, res);
}

18
compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs
View File

@ -99,7 +99,7 @@ const SINGLE_VARIANT_VIRTUAL_DISR: u64 = 0;
/// compiler versions.
///
/// Niche-tag enums have one special variant, usually called the
/// "dataful variant". This variant has a field that
/// "untagged variant". This variant has a field that
/// doubles as the tag of the enum. The variant is active when the value of
/// that field is within a pre-defined range. Therefore the variant struct
/// has a `DISCR_BEGIN` and `DISCR_END` field instead of `DISCR_EXACT` in
@ -249,7 +249,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>(
None,
),
Variants::Multiple {
tag_encoding: TagEncoding::Niche { dataful_variant, .. },
tag_encoding: TagEncoding::Niche { untagged_variant, .. },
ref variants,
tag_field,
..
@ -260,7 +260,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>(
enum_type_di_node,
variants.indices(),
tag_field,
Some(dataful_variant),
Some(untagged_variant),
),
}
},
@ -391,7 +391,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>(
enum_type_di_node: &'ll DIType,
variant_indices: impl Iterator<Item = VariantIdx> + Clone,
tag_field: usize,
dataful_variant_index: Option<VariantIdx>,
untagged_variant_index: Option<VariantIdx>,
) -> SmallVec<&'ll DIType> {
let tag_base_type = super::tag_base_type(cx, enum_type_and_layout);
@ -436,7 +436,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>(
variant_names_type_di_node,
tag_base_type,
tag_field,
dataful_variant_index,
untagged_variant_index,
)
}
@ -472,7 +472,7 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>(
enum_or_generator_type_and_layout: TyAndLayout<'tcx>,
enum_or_generator_type_di_node: &'ll DIType,
variant_index: VariantIdx,
dataful_variant_index: Option<VariantIdx>,
untagged_variant_index: Option<VariantIdx>,
variant_struct_type_di_node: &'ll DIType,
variant_names_type_di_node: &'ll DIType,
tag_base_type_di_node: &'ll DIType,
@ -517,7 +517,7 @@ fn build_variant_struct_wrapper_type_di_node<'ll, 'tcx>(
}
}
DiscrResult::Range(min, max) => {
assert_eq!(Some(variant_index), dataful_variant_index);
assert_eq!(Some(variant_index), untagged_variant_index);
if is_128_bits {
DiscrKind::Range128(min, max)
} else {
@ -757,7 +757,7 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>(
discr_type_di_node: &'ll DIType,
tag_base_type: Ty<'tcx>,
tag_field: usize,
dataful_variant_index: Option<VariantIdx>,
untagged_variant_index: Option<VariantIdx>,
) -> SmallVec<&'ll DIType> {
let tag_base_type_di_node = type_di_node(cx, tag_base_type);
let mut unions_fields = SmallVec::with_capacity(variant_field_infos.len() + 1);
@ -776,7 +776,7 @@ fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>(
enum_type_and_layout,
enum_type_di_node,
variant_member_info.variant_index,
dataful_variant_index,
untagged_variant_index,
variant_member_info.variant_struct_type_di_node,
discr_type_di_node,
tag_base_type_di_node,

6
compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs
View File

@ -417,7 +417,7 @@ impl DiscrResult {
/// Returns the discriminant value corresponding to the variant index.
///
/// Will return `None` if there is less than two variants (because then the enum won't have)
/// a tag, and if this is the dataful variant of a niche-layout enum (because then there is no
/// a tag, and if this is the untagged variant of a niche-layout enum (because then there is no
/// single discriminant value).
fn compute_discriminant_value<'ll, 'tcx>(
cx: &CodegenCx<'ll, 'tcx>,
@ -430,11 +430,11 @@ fn compute_discriminant_value<'ll, 'tcx>(
enum_type_and_layout.ty.discriminant_for_variant(cx.tcx, variant_index).unwrap().val,
),
&Variants::Multiple {
tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, dataful_variant },
tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, untagged_variant },
tag,
..
} => {
if variant_index == dataful_variant {
if variant_index == untagged_variant {
let valid_range = enum_type_and_layout
.for_variant(cx, variant_index)
.largest_niche

2
compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
View File

@ -378,7 +378,7 @@ fn build_discr_member_di_node<'ll, 'tcx>(
///
/// The DW_AT_discr_value is optional, and is omitted if
/// - This is the only variant of a univariant enum (i.e. their is no discriminant)
/// - This is the "dataful" variant of a niche-layout enum
/// - This is the "untagged" variant of a niche-layout enum
/// (where only the other variants are identified by a single value)
///
/// There is only ever a single member, the type of which is a struct that describes the

8
compiler/rustc_codegen_ssa/src/mir/place.rs
View File

@ -244,7 +244,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
};
bx.intcast(tag.immediate(), cast_to, signed)
}
TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => {
TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
// Rebase from niche values to discriminants, and check
// whether the result is in range for the niche variants.
let niche_llty = bx.cx().immediate_backend_type(tag.layout);
@ -302,7 +302,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
bx.select(
is_niche,
niche_discr,
bx.cx().const_uint(cast_to, dataful_variant.as_u32() as u64),
bx.cx().const_uint(cast_to, untagged_variant.as_u32() as u64),
)
}
}
@ -337,11 +337,11 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
}
Variants::Multiple {
tag_encoding:
TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start },
TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start },
tag_field,
..
} => {
if variant_index != dataful_variant {
if variant_index != untagged_variant {
let niche = self.project_field(bx, tag_field);
let niche_llty = bx.cx().immediate_backend_type(niche.layout);
let niche_value = variant_index.as_u32() - niche_variants.start().as_u32();

16
compiler/rustc_const_eval/src/interpret/operand.rs
View File

@ -718,7 +718,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
// Return the cast value, and the index.
(discr_val, index.0)
}
TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => {
TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
let tag_val = tag_val.to_scalar();
// Compute the variant this niche value/"tag" corresponds to. With niche layout,
// discriminant (encoded in niche/tag) and variant index are the same.
@ -736,7 +736,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
if !ptr_valid {
throw_ub!(InvalidTag(dbg_val))
}
dataful_variant
untagged_variant
}
Ok(tag_bits) => {
let tag_bits = tag_bits.assert_bits(tag_layout.size);
@ -766,7 +766,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
assert!(usize::try_from(variant_index).unwrap() < variants_len);
VariantIdx::from_u32(variant_index)
} else {
dataful_variant
untagged_variant
}
}
};
@ -780,13 +780,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
// Some nodes are used a lot. Make sure they don't unintentionally get bigger.
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64", not(bootstrap)))]
mod size_asserts {
use super::*;
use rustc_data_structures::static_assert_size;
// These are in alphabetical order, which is easy to maintain.
static_assert_size!(Immediate, 56);
static_assert_size!(ImmTy<'_>, 72);
static_assert_size!(Operand, 64);
static_assert_size!(OpTy<'_>, 88);
static_assert_size!(Immediate, 48);
static_assert_size!(ImmTy<'_>, 64);
static_assert_size!(Operand, 56);
static_assert_size!(OpTy<'_>, 80);
}

10
compiler/rustc_const_eval/src/interpret/place.rs
View File

@ -817,7 +817,7 @@ where
}
abi::Variants::Multiple {
tag_encoding:
TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start },
TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start },
tag: tag_layout,
tag_field,
..
@ -825,7 +825,7 @@ where
// No need to validate that the discriminant here because the
// `TyAndLayout::for_variant()` call earlier already checks the variant is valid.
if variant_index != dataful_variant {
if variant_index != untagged_variant {
let variants_start = niche_variants.start().as_u32();
let variant_index_relative = variant_index
.as_u32()
@ -890,6 +890,8 @@ mod size_asserts {
static_assert_size!(MemPlaceMeta, 24);
static_assert_size!(MemPlace, 40);
static_assert_size!(MPlaceTy<'_>, 64);
static_assert_size!(Place, 48);
static_assert_size!(PlaceTy<'_>, 72);
#[cfg(not(bootstrap))]
static_assert_size!(Place, 40);
#[cfg(not(bootstrap))]
static_assert_size!(PlaceTy<'_>, 64);
}

4
compiler/rustc_data_structures/src/obligation_forest/mod.rs
View File

@ -117,6 +117,10 @@ pub trait ObligationProcessor {
}
/// The result type used by `process_obligation`.
// `repr(C)` to inhibit the niche filling optimization. Otherwise, the `match` appearing
// in `process_obligations` is significantly slower, which can substantially affect
// benchmarks like `rustc-perf`'s inflate and keccak.
#[repr(C)]
#[derive(Debug)]
pub enum ProcessResult<O, E> {
Unchanged,

4
compiler/rustc_errors/src/lib.rs
View File

@ -69,8 +69,8 @@ pub type PResult<'a, T> = Result<T, DiagnosticBuilder<'a, ErrorGuaranteed>>;
// (See also the comment on `DiagnosticBuilder`'s `diagnostic` field.)
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
rustc_data_structures::static_assert_size!(PResult<'_, ()>, 16);
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
rustc_data_structures::static_assert_size!(PResult<'_, bool>, 24);
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64", not(bootstrap)))]
rustc_data_structures::static_assert_size!(PResult<'_, bool>, 16);
#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Encodable, Decodable)]
pub enum SuggestionStyle {

15
compiler/rustc_hir/src/hir.rs
View File

@ -3473,12 +3473,15 @@ mod size_asserts {
static_assert_size!(FnDecl<'_>, 40);
static_assert_size!(ForeignItem<'_>, 72);
static_assert_size!(ForeignItemKind<'_>, 40);
static_assert_size!(GenericArg<'_>, 40);
#[cfg(not(bootstrap))]
static_assert_size!(GenericArg<'_>, 32);
static_assert_size!(GenericBound<'_>, 48);
static_assert_size!(Generics<'_>, 56);
static_assert_size!(Impl<'_>, 80);
static_assert_size!(ImplItem<'_>, 88);
static_assert_size!(ImplItemKind<'_>, 40);
#[cfg(not(bootstrap))]
static_assert_size!(ImplItem<'_>, 80);
#[cfg(not(bootstrap))]
static_assert_size!(ImplItemKind<'_>, 32);
static_assert_size!(Item<'_>, 80);
static_assert_size!(ItemKind<'_>, 48);
static_assert_size!(Local<'_>, 64);
@ -3490,8 +3493,10 @@ mod size_asserts {
static_assert_size!(QPath<'_>, 24);
static_assert_size!(Stmt<'_>, 32);
static_assert_size!(StmtKind<'_>, 16);
static_assert_size!(TraitItem<'_>, 96);
static_assert_size!(TraitItemKind<'_>, 56);
#[cfg(not(bootstrap))]
static_assert_size!(TraitItem<'static>, 88);
#[cfg(not(bootstrap))]
static_assert_size!(TraitItemKind<'_>, 48);
static_assert_size!(Ty<'_>, 72);
static_assert_size!(TyKind<'_>, 56);
}

3
compiler/rustc_middle/src/mir/syntax.rs
View File

@ -1231,7 +1231,8 @@ pub enum BinOp {
mod size_asserts {
use super::*;
// These are in alphabetical order, which is easy to maintain.
static_assert_size!(AggregateKind<'_>, 48);
#[cfg(not(bootstrap))]
static_assert_size!(AggregateKind<'_>, 40);
static_assert_size!(Operand<'_>, 24);
static_assert_size!(Place<'_>, 16);
static_assert_size!(PlaceElem<'_>, 24);

12
compiler/rustc_middle/src/thir.rs
View File

@ -825,8 +825,12 @@ mod size_asserts {
static_assert_size!(Block, 56);
static_assert_size!(Expr<'_>, 64);
static_assert_size!(ExprKind<'_>, 40);
static_assert_size!(Pat<'_>, 72);
static_assert_size!(PatKind<'_>, 56);
static_assert_size!(Stmt<'_>, 56);
static_assert_size!(StmtKind<'_>, 48);
#[cfg(not(bootstrap))]
static_assert_size!(Pat<'_>, 64);
#[cfg(not(bootstrap))]
static_assert_size!(PatKind<'_>, 48);
#[cfg(not(bootstrap))]
static_assert_size!(Stmt<'_>, 48);
#[cfg(not(bootstrap))]
static_assert_size!(StmtKind<'_>, 40);
}

316
compiler/rustc_middle/src/ty/layout.rs
View File

@ -22,7 +22,7 @@ use rustc_target::abi::call::{
use rustc_target::abi::*;
use rustc_target::spec::{abi::Abi as SpecAbi, HasTargetSpec, PanicStrategy, Target};
use std::cmp;
use std::cmp::{self, Ordering};
use std::fmt;
use std::iter;
use std::num::NonZeroUsize;
@ -1046,131 +1046,191 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
// that allow representation optimization.)
assert!(def.is_enum());
// The current code for niche-filling relies on variant indices
// instead of actual discriminants, so dataful enums with
// explicit discriminants (RFC #2363) would misbehave.
let no_explicit_discriminants = def
.variants()
.iter_enumerated()
.all(|(i, v)| v.discr == ty::VariantDiscr::Relative(i.as_u32()));
// Until we've decided whether to use the tagged or
// niche filling LayoutS, we don't want to intern the
// variant layouts, so we can't store them in the
// overall LayoutS. Store the overall LayoutS
// and the variant LayoutSs here until then.
struct TmpLayout<'tcx> {
layout: LayoutS<'tcx>,
variants: IndexVec<VariantIdx, LayoutS<'tcx>>,
}
let mut niche_filling_layout = None;
// Niche-filling enum optimization.
if !def.repr().inhibit_enum_layout_opt() && no_explicit_discriminants {
let mut dataful_variant = None;
let mut niche_variants = VariantIdx::MAX..=VariantIdx::new(0);
// Find one non-ZST variant.
'variants: for (v, fields) in variants.iter_enumerated() {
if absent(fields) {
continue 'variants;
let calculate_niche_filling_layout =
|| -> Result<Option<TmpLayout<'tcx>>, LayoutError<'tcx>> {
// The current code for niche-filling relies on variant indices
// instead of actual discriminants, so enums with
// explicit discriminants (RFC #2363) would misbehave.
if def.repr().inhibit_enum_layout_opt()
|| def
.variants()
.iter_enumerated()
.any(|(i, v)| v.discr != ty::VariantDiscr::Relative(i.as_u32()))
{
return Ok(None);
}
for f in fields {
if !f.is_zst() {
if dataful_variant.is_none() {
dataful_variant = Some(v);
continue 'variants;
} else {
dataful_variant = None;
break 'variants;
}
}
if variants.len() < 2 {
return Ok(None);
}
niche_variants = *niche_variants.start().min(&v)..=v;
}
if niche_variants.start() > niche_variants.end() {
dataful_variant = None;
}
let mut align = dl.aggregate_align;
let mut variant_layouts = variants
.iter_enumerated()
.map(|(j, v)| {
let mut st = self.univariant_uninterned(
ty,
v,
&def.repr(),
StructKind::AlwaysSized,
)?;
st.variants = Variants::Single { index: j };
if let Some(i) = dataful_variant {
let count = (niche_variants.end().as_u32()
- niche_variants.start().as_u32()
+ 1) as u128;
align = align.max(st.align);
Ok(st)
})
.collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
let largest_variant_index = match variant_layouts
.iter_enumerated()
.max_by_key(|(_i, layout)| layout.size.bytes())
.map(|(i, _layout)| i)
{
None => return Ok(None),
Some(i) => i,
};
let all_indices = VariantIdx::new(0)..=VariantIdx::new(variants.len() - 1);
let needs_disc = |index: VariantIdx| {
index != largest_variant_index && !absent(&variants[index])
};
let niche_variants = all_indices.clone().find(|v| needs_disc(*v)).unwrap()
..=all_indices.rev().find(|v| needs_disc(*v)).unwrap();
let count = niche_variants.size_hint().1.unwrap() as u128;
// Find the field with the largest niche
let niche_candidate = variants[i]
let (field_index, niche, (niche_start, niche_scalar)) = match variants
[largest_variant_index]
.iter()
.enumerate()
.filter_map(|(j, field)| Some((j, field.largest_niche?)))
.max_by_key(|(_, niche)| niche.available(dl));
if let Some((field_index, niche, (niche_start, niche_scalar))) =
niche_candidate.and_then(|(field_index, niche)| {
Some((field_index, niche, niche.reserve(self, count)?))
})
.max_by_key(|(_, niche)| niche.available(dl))
.and_then(|(j, niche)| Some((j, niche, niche.reserve(self, count)?)))
{
let mut align = dl.aggregate_align;
let st = variants
.iter_enumerated()
.map(|(j, v)| {
let mut st = self.univariant_uninterned(
ty,
v,
&def.repr(),
StructKind::AlwaysSized,
)?;
st.variants = Variants::Single { index: j };
None => return Ok(None),
Some(x) => x,
};
align = align.max(st.align);
let niche_offset = niche.offset
+ variant_layouts[largest_variant_index].fields.offset(field_index);
let niche_size = niche.value.size(dl);
let size = variant_layouts[largest_variant_index].size.align_to(align.abi);
Ok(tcx.intern_layout(st))
})
.collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
let all_variants_fit =
variant_layouts.iter_enumerated_mut().all(|(i, layout)| {
if i == largest_variant_index {
return true;
}
let offset = st[i].fields().offset(field_index) + niche.offset;
layout.largest_niche = None;
// Align the total size to the largest alignment.
let size = st[i].size().align_to(align.abi);
if layout.size <= niche_offset {
// This variant will fit before the niche.
return true;
}
let abi = if st.iter().all(|v| v.abi().is_uninhabited()) {
Abi::Uninhabited
} else if align == st[i].align() && size == st[i].size() {
// When the total alignment and size match, we can use the
// same ABI as the scalar variant with the reserved niche.
match st[i].abi() {
Abi::Scalar(_) => Abi::Scalar(niche_scalar),
Abi::ScalarPair(first, second) => {
// Only the niche is guaranteed to be initialised,
// so use union layout for the other primitive.
if offset.bytes() == 0 {
Abi::ScalarPair(niche_scalar, second.to_union())
} else {
Abi::ScalarPair(first.to_union(), niche_scalar)
// Determine if it'll fit after the niche.
let this_align = layout.align.abi;
let this_offset = (niche_offset + niche_size).align_to(this_align);
if this_offset + layout.size > size {
return false;
}
// It'll fit, but we need to make some adjustments.
match layout.fields {
FieldsShape::Arbitrary { ref mut offsets, .. } => {
for (j, offset) in offsets.iter_mut().enumerate() {
if !variants[i][j].is_zst() {
*offset += this_offset;
}
}
}
_ => Abi::Aggregate { sized: true },
_ => {
panic!("Layout of fields should be Arbitrary for variants")
}
}
} else {
Abi::Aggregate { sized: true }
};
let largest_niche = Niche::from_scalar(dl, offset, niche_scalar);
// It can't be a Scalar or ScalarPair because the offset isn't 0.
if !layout.abi.is_uninhabited() {
layout.abi = Abi::Aggregate { sized: true };
}
layout.size += this_offset;
niche_filling_layout = Some(LayoutS {
variants: Variants::Multiple {
tag: niche_scalar,
tag_encoding: TagEncoding::Niche {
dataful_variant: i,
niche_variants,
niche_start,
},
tag_field: 0,
variants: st,
},
fields: FieldsShape::Arbitrary {
offsets: vec![offset],
memory_index: vec![0],
},
abi,
largest_niche,
size,
align,
true
});
if !all_variants_fit {
return Ok(None);
}
}
}
let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar);
let others_zst = variant_layouts.iter_enumerated().all(|(i, layout)| {
i == largest_variant_index || layout.size == Size::ZERO
});
let same_size = size == variant_layouts[largest_variant_index].size;
let same_align = align == variant_layouts[largest_variant_index].align;
let abi = if variant_layouts.iter().all(|v| v.abi.is_uninhabited()) {
Abi::Uninhabited
} else if same_size && same_align && others_zst {
match variant_layouts[largest_variant_index].abi {
// When the total alignment and size match, we can use the
// same ABI as the scalar variant with the reserved niche.
Abi::Scalar(_) => Abi::Scalar(niche_scalar),
Abi::ScalarPair(first, second) => {
// Only the niche is guaranteed to be initialised,
// so use union layouts for the other primitive.
if niche_offset == Size::ZERO {
Abi::ScalarPair(niche_scalar, second.to_union())
} else {
Abi::ScalarPair(first.to_union(), niche_scalar)
}
}
_ => Abi::Aggregate { sized: true },
}
} else {
Abi::Aggregate { sized: true }
};
let layout = LayoutS {
variants: Variants::Multiple {
tag: niche_scalar,
tag_encoding: TagEncoding::Niche {
untagged_variant: largest_variant_index,
niche_variants,
niche_start,
},
tag_field: 0,
variants: IndexVec::new(),
},
fields: FieldsShape::Arbitrary {
offsets: vec![niche_offset],
memory_index: vec![0],
},
abi,
largest_niche,
size,
align,
};
Ok(Some(TmpLayout { layout, variants: variant_layouts }))
};
let niche_filling_layout = calculate_niche_filling_layout()?;
let (mut min, mut max) = (i128::MAX, i128::MIN);
let discr_type = def.repr().discr_type();
@ -1425,15 +1485,12 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag);
let layout_variants =
layout_variants.into_iter().map(|v| tcx.intern_layout(v)).collect();
let tagged_layout = LayoutS {
variants: Variants::Multiple {
tag,
tag_encoding: TagEncoding::Direct,
tag_field: 0,
variants: layout_variants,
variants: IndexVec::new(),
},
fields: FieldsShape::Arbitrary {
offsets: vec![Size::ZERO],
@ -1445,20 +1502,45 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
size,
};
let best_layout = match (tagged_layout, niche_filling_layout) {
(tagged_layout, Some(niche_filling_layout)) => {
let tagged_layout = TmpLayout { layout: tagged_layout, variants: layout_variants };
let mut best_layout = match (tagged_layout, niche_filling_layout) {
(tl, Some(nl)) => {
// Pick the smaller layout; otherwise,
// pick the layout with the larger niche; otherwise,
// pick tagged as it has simpler codegen.
cmp::min_by_key(tagged_layout, niche_filling_layout, |layout| {
let niche_size = layout.largest_niche.map_or(0, |n| n.available(dl));
(layout.size, cmp::Reverse(niche_size))
})
use Ordering::*;
let niche_size = |tmp_l: &TmpLayout<'_>| {
tmp_l.layout.largest_niche.map_or(0, |n| n.available(dl))
};
match (
tl.layout.size.cmp(&nl.layout.size),
niche_size(&tl).cmp(&niche_size(&nl)),
) {
(Greater, _) => nl,
(Equal, Less) => nl,
_ => tl,
}
}
(tagged_layout, None) => tagged_layout,
(tl, None) => tl,
};
tcx.intern_layout(best_layout)
// Now we can intern the variant layouts and store them in the enum layout.
best_layout.layout.variants = match best_layout.layout.variants {
Variants::Multiple { tag, tag_encoding, tag_field, .. } => Variants::Multiple {
tag,
tag_encoding,
tag_field,
variants: best_layout
.variants
.into_iter()
.map(|layout| tcx.intern_layout(layout))
.collect(),
},
_ => bug!(),
};
tcx.intern_layout(best_layout.layout)
}
// Types with no meaningful known layout.
@ -2559,11 +2641,11 @@ where
// using more niches than just null (e.g., the first page of
// the address space, or unaligned pointers).
Variants::Multiple {
tag_encoding: TagEncoding::Niche { dataful_variant, .. },
tag_encoding: TagEncoding::Niche { untagged_variant, .. },
tag_field,
..
} if this.fields.offset(tag_field) == offset => {
Some(this.for_variant(cx, dataful_variant))
Some(this.for_variant(cx, untagged_variant))
}
_ => Some(this),
};

4
compiler/rustc_target/src/abi/mod.rs
View File

@ -1130,7 +1130,7 @@ pub enum TagEncoding {
/// Niche (values invalid for a type) encoding the discriminant:
/// Discriminant and variant index coincide.
/// The variant `dataful_variant` contains a niche at an arbitrary
/// The variant `untagged_variant` contains a niche at an arbitrary
/// offset (field `tag_field` of the enum), which for a variant with
/// discriminant `d` is set to
/// `(d - niche_variants.start).wrapping_add(niche_start)`.
@ -1139,7 +1139,7 @@ pub enum TagEncoding {
/// `None` has a null pointer for the second tuple field, and
/// `Some` is the identity function (with a non-null reference).
Niche {
dataful_variant: VariantIdx,
untagged_variant: VariantIdx,
niche_variants: RangeInclusive<VariantIdx>,
niche_start: u128,
},

3
src/librustdoc/clean/types.rs
View File

@ -2532,7 +2532,8 @@ mod size_asserts {
// These are in alphabetical order, which is easy to maintain.
static_assert_size!(Crate, 72); // frequently moved by-value
static_assert_size!(DocFragment, 32);
static_assert_size!(GenericArg, 64);
#[cfg(not(bootstrap))]
static_assert_size!(GenericArg, 56);
static_assert_size!(GenericArgs, 32);
static_assert_size!(GenericParamDef, 56);
static_assert_size!(Item, 56);

6
src/test/debuginfo/msvc-pretty-enums.rs
View File

@ -53,8 +53,8 @@
// cdb-command: dx niche128_none
// cdb-check: niche128_none : None [Type: enum2$<core::option::Option<core::num::nonzero::NonZeroI128> >]
// cdb-command: dx wrapping_niche128_dataful
// cdb-check: wrapping_niche128_dataful : X [Type: enum2$<msvc_pretty_enums::Wrapping128Niche>]
// cdb-command: dx wrapping_niche128_untagged
// cdb-check: wrapping_niche128_untagged : X [Type: enum2$<msvc_pretty_enums::Wrapping128Niche>]
// cdb-check: [+0x[...]] __0 [Type: msvc_pretty_enums::Wrapping128]
// cdb-command: dx wrapping_niche128_none1
@ -213,7 +213,7 @@ fn main() {
let niche128_some = Some(NonZeroI128::new(123456).unwrap());
let niche128_none: Option<NonZeroI128> = None;
let wrapping_niche128_dataful =
let wrapping_niche128_untagged =
unsafe { Wrapping128Niche::X(Wrapping128(340282366920938463463374607431768211454)) };
let wrapping_niche128_none1 = Wrapping128Niche::Y;
let wrapping_niche128_none2 = Wrapping128Niche::Z;

4
src/test/ui/layout/issue-96158-scalarpair-payload-might-be-uninit.stderr
View File

@ -411,7 +411,7 @@ error: layout_of(NicheFirst) = Layout {
valid_range: 0..=4,
},
tag_encoding: Niche {
dataful_variant: 0,
untagged_variant: 0,
niche_variants: 1..=2,
niche_start: 3,
},
@ -555,7 +555,7 @@ error: layout_of(NicheSecond) = Layout {
valid_range: 0..=4,
},
tag_encoding: Niche {
dataful_variant: 0,
untagged_variant: 0,
niche_variants: 1..=2,
niche_start: 3,
},

2
src/test/ui/layout/zero-sized-array-enum-niche.stderr
View File

@ -353,7 +353,7 @@ error: layout_of(std::result::Result<[u32; 0], Packed<U16IsZero>>) = Layout {
valid_range: 0..=1,
},
tag_encoding: Niche {
dataful_variant: 1,
untagged_variant: 1,
niche_variants: 0..=0,
niche_start: 1,
},

1
src/test/ui/stats/hir-stats.rs
View File

@ -1,6 +1,7 @@
// check-pass
// compile-flags: -Zhir-stats
// only-x86_64
// ignore-stage1
// The aim here is to include at least one of every different type of top-level
// AST/HIR node reported by `-Zhir-stats`.

44
src/test/ui/stats/hir-stats.stderr
View File

@ -21,39 +21,39 @@ ast-stats-1 - MacCall 32 ( 0.4%) 1
ast-stats-1 - Expr 96 ( 1.1%) 3
ast-stats-1 Param 160 ( 1.9%) 4 40
ast-stats-1 FnDecl 200 ( 2.4%) 5 40
ast-stats-1 Variant 240 ( 2.8%) 2 120
ast-stats-1 Variant 240 ( 2.9%) 2 120
ast-stats-1 Block 288 ( 3.4%) 6 48
ast-stats-1 GenericBound 352 ( 4.2%) 4 88
ast-stats-1 - Trait 352 ( 4.2%) 4
ast-stats-1 AssocItem 416 ( 4.9%) 4 104
ast-stats-1 - TyAlias 208 ( 2.5%) 2
ast-stats-1 - Fn 208 ( 2.5%) 2
ast-stats-1 GenericParam 520 ( 6.1%) 5 104
ast-stats-1 PathSegment 720 ( 8.5%) 30 24
ast-stats-1 Expr 832 ( 9.8%) 8 104
ast-stats-1 GenericParam 480 ( 5.7%) 5 96
ast-stats-1 PathSegment 720 ( 8.6%) 30 24
ast-stats-1 Expr 832 ( 9.9%) 8 104
ast-stats-1 - Path 104 ( 1.2%) 1
ast-stats-1 - Match 104 ( 1.2%) 1
ast-stats-1 - Struct 104 ( 1.2%) 1
ast-stats-1 - Lit 208 ( 2.5%) 2
ast-stats-1 - Block 312 ( 3.7%) 3
ast-stats-1 Pat 840 ( 9.9%) 7 120
ast-stats-1 Pat 840 (10.0%) 7 120
ast-stats-1 - Struct 120 ( 1.4%) 1
ast-stats-1 - Wild 120 ( 1.4%) 1
ast-stats-1 - Ident 600 ( 7.1%) 5
ast-stats-1 Ty 1_344 (15.9%) 14 96
ast-stats-1 Ty 1_344 (16.0%) 14 96
ast-stats-1 - Rptr 96 ( 1.1%) 1
ast-stats-1 - Ptr 96 ( 1.1%) 1
ast-stats-1 - ImplicitSelf 192 ( 2.3%) 2
ast-stats-1 - Path 960 (11.4%) 10
ast-stats-1 Item 1_656 (19.6%) 9 184
ast-stats-1 Item 1_656 (19.7%) 9 184
ast-stats-1 - Trait 184 ( 2.2%) 1
ast-stats-1 - Enum 184 ( 2.2%) 1
ast-stats-1 - ForeignMod 184 ( 2.2%) 1
ast-stats-1 - Impl 184 ( 2.2%) 1
ast-stats-1 - Fn 368 ( 4.4%) 2
ast-stats-1 - Use 552 ( 6.5%) 3
ast-stats-1 - Use 552 ( 6.6%) 3
ast-stats-1 ----------------------------------------------------------------
ast-stats-1 Total 8_456
ast-stats-1 Total 8_416
ast-stats-1
ast-stats-2 POST EXPANSION AST STATS
ast-stats-2 Name Accumulated Size Count Item Size
@ -86,12 +86,12 @@ ast-stats-2 - Trait 352 ( 3.8%) 4
ast-stats-2 AssocItem 416 ( 4.5%) 4 104
ast-stats-2 - TyAlias 208 ( 2.3%) 2
ast-stats-2 - Fn 208 ( 2.3%) 2
ast-stats-2 GenericParam 520 ( 5.7%) 5 104
ast-stats-2 PathSegment 792 ( 8.6%) 33 24
ast-stats-2 Pat 840 ( 9.1%) 7 120
ast-stats-2 GenericParam 480 ( 5.2%) 5 96
ast-stats-2 PathSegment 792 ( 8.7%) 33 24
ast-stats-2 Pat 840 ( 9.2%) 7 120
ast-stats-2 - Struct 120 ( 1.3%) 1
ast-stats-2 - Wild 120 ( 1.3%) 1
ast-stats-2 - Ident 600 ( 6.5%) 5
ast-stats-2 - Ident 600 ( 6.6%) 5
ast-stats-2 Expr 936 (10.2%) 9 104
ast-stats-2 - Path 104 ( 1.1%) 1
ast-stats-2 - Match 104 ( 1.1%) 1
@ -99,12 +99,12 @@ ast-stats-2 - Struct 104 ( 1.1%) 1
ast-stats-2 - InlineAsm 104 ( 1.1%) 1
ast-stats-2 - Lit 208 ( 2.3%) 2
ast-stats-2 - Block 312 ( 3.4%) 3
ast-stats-2 Ty 1_344 (14.6%) 14 96
ast-stats-2 Ty 1_344 (14.7%) 14 96
ast-stats-2 - Rptr 96 ( 1.0%) 1
ast-stats-2 - Ptr 96 ( 1.0%) 1
ast-stats-2 - ImplicitSelf 192 ( 2.1%) 2
ast-stats-2 - Path 960 (10.5%) 10
ast-stats-2 Item 2_024 (22.0%) 11 184
ast-stats-2 Item 2_024 (22.1%) 11 184
ast-stats-2 - Trait 184 ( 2.0%) 1
ast-stats-2 - Enum 184 ( 2.0%) 1
ast-stats-2 - ExternCrate 184 ( 2.0%) 1
@ -113,7 +113,7 @@ ast-stats-2 - Impl 184 ( 2.0%) 1
ast-stats-2 - Fn 368 ( 4.0%) 2
ast-stats-2 - Use 736 ( 8.0%) 4
ast-stats-2 ----------------------------------------------------------------
ast-stats-2 Total 9_184
ast-stats-2 Total 9_144
ast-stats-2
hir-stats HIR STATS
hir-stats Name Accumulated Size Count Item Size
@ -121,7 +121,7 @@ hir-stats ----------------------------------------------------------------
hir-stats ForeignItemRef 24 ( 0.2%) 1 24
hir-stats Mod 32 ( 0.3%) 1 32
hir-stats ExprField 40 ( 0.4%) 1 40
hir-stats TraitItemRef 56 ( 0.5%) 2 28
hir-stats TraitItemRef 56 ( 0.6%) 2 28
hir-stats Param 64 ( 0.6%) 2 32
hir-stats Local 64 ( 0.6%) 1 64
hir-stats InlineAsm 72 ( 0.7%) 1 72
@ -135,11 +135,11 @@ hir-stats - Semi 32 ( 0.3%) 1
hir-stats - Expr 32 ( 0.3%) 1
hir-stats FnDecl 120 ( 1.2%) 3 40
hir-stats Attribute 128 ( 1.3%) 4 32
hir-stats GenericArg 128 ( 1.3%) 4 32
hir-stats - Type 32 ( 0.3%) 1
hir-stats - Lifetime 96 ( 0.9%) 3
hir-stats GenericArgs 144 ( 1.4%) 3 48
hir-stats Variant 160 ( 1.6%) 2 80
hir-stats GenericArg 160 ( 1.6%) 4 40
hir-stats - Type 40 ( 0.4%) 1
hir-stats - Lifetime 120 ( 1.2%) 3
hir-stats GenericBound 192 ( 1.9%) 4 48
hir-stats - Trait 192 ( 1.9%) 4
hir-stats WherePredicate 216 ( 2.1%) 3 72
@ -151,7 +151,7 @@ hir-stats - Wild 88 ( 0.9%) 1
hir-stats - Struct 88 ( 0.9%) 1
hir-stats - Binding 264 ( 2.6%) 3
hir-stats Generics 560 ( 5.5%) 10 56
hir-stats Expr 768 ( 7.5%) 12 64
hir-stats Expr 768 ( 7.6%) 12 64
hir-stats - Path 64 ( 0.6%) 1
hir-stats - Struct 64 ( 0.6%) 1
hir-stats - Match 64 ( 0.6%) 1
@ -173,5 +173,5 @@ hir-stats - Path 936 ( 9.2%) 13
hir-stats Path 1_536 (15.1%) 32 48
hir-stats PathSegment 2_240 (22.0%) 40 56
hir-stats ----------------------------------------------------------------
hir-stats Total 10_200
hir-stats Total 10_168
hir-stats

79
src/test/ui/structs-enums/type-sizes.rs
View File

@ -120,6 +120,54 @@ pub enum AlwaysTaggedBecauseItHasNoNiche {
B
}
pub enum NicheFilledMultipleFields {
A(bool, u8),
B(u8),
C(u8),
D(bool),
E,
F,
G,
}
struct BoolInTheMiddle(std::num::NonZeroU16, bool, u8);
enum NicheWithData {
A,
B([u16; 5]),
Largest { a1: u32, a2: BoolInTheMiddle, a3: u32 },
C,
D(u32, u32),
}
// A type with almost 2^16 invalid values.
#[repr(u16)]
pub enum NicheU16 {
_0,
}
pub enum EnumManyVariant<X> {
Dataful(u8, X),
// 0x100 niche variants.
_00, _01, _02, _03, _04, _05, _06, _07, _08, _09, _0A, _0B, _0C, _0D, _0E, _0F,
_10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _1A, _1B, _1C, _1D, _1E, _1F,
_20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _2A, _2B, _2C, _2D, _2E, _2F,
_30, _31, _32, _33, _34, _35, _36, _37, _38, _39, _3A, _3B, _3C, _3D, _3E, _3F,
_40, _41, _42, _43, _44, _45, _46, _47, _48, _49, _4A, _4B, _4C, _4D, _4E, _4F,
_50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _5A, _5B, _5C, _5D, _5E, _5F,
_60, _61, _62, _63, _64, _65, _66, _67, _68, _69, _6A, _6B, _6C, _6D, _6E, _6F,
_70, _71, _72, _73, _74, _75, _76, _77, _78, _79, _7A, _7B, _7C, _7D, _7E, _7F,
_80, _81, _82, _83, _84, _85, _86, _87, _88, _89, _8A, _8B, _8C, _8D, _8E, _8F,
_90, _91, _92, _93, _94, _95, _96, _97, _98, _99, _9A, _9B, _9C, _9D, _9E, _9F,
_A0, _A1, _A2, _A3, _A4, _A5, _A6, _A7, _A8, _A9, _AA, _AB, _AC, _AD, _AE, _AF,
_B0, _B1, _B2, _B3, _B4, _B5, _B6, _B7, _B8, _B9, _BA, _BB, _BC, _BD, _BE, _BF,
_C0, _C1, _C2, _C3, _C4, _C5, _C6, _C7, _C8, _C9, _CA, _CB, _CC, _CD, _CE, _CF,
_D0, _D1, _D2, _D3, _D4, _D5, _D6, _D7, _D8, _D9, _DA, _DB, _DC, _DD, _DE, _DF,
_E0, _E1, _E2, _E3, _E4, _E5, _E6, _E7, _E8, _E9, _EA, _EB, _EC, _ED, _EE, _EF,
_F0, _F1, _F2, _F3, _F4, _F5, _F6, _F7, _F8, _F9, _FA, _FB, _FC, _FD, _FE, _FF,
}
pub fn main() {
assert_eq!(size_of::<u8>(), 1 as usize);
assert_eq!(size_of::<u32>(), 4 as usize);
@ -170,4 +218,35 @@ pub fn main() {
assert_eq!(size_of::<AlwaysTaggedBecauseItHasNoNiche>(), 8);
assert_eq!(size_of::<Option<AlwaysTaggedBecauseItHasNoNiche>>(), 8);
assert_eq!(size_of::<Option<Option<AlwaysTaggedBecauseItHasNoNiche>>>(), 8);
assert_eq!(size_of::<NicheFilledMultipleFields>(), 2);
assert_eq!(size_of::<Option<NicheFilledMultipleFields>>(), 2);
assert_eq!(size_of::<Option<Option<NicheFilledMultipleFields>>>(), 2);
struct S1{ a: u16, b: std::num::NonZeroU16, c: u16, d: u8, e: u32, f: u64, g:[u8;2] }
assert_eq!(size_of::<S1>(), 24);
assert_eq!(size_of::<Option<S1>>(), 24);
assert_eq!(size_of::<NicheWithData>(), 12);
assert_eq!(size_of::<Option<NicheWithData>>(), 12);
assert_eq!(size_of::<Option<Option<NicheWithData>>>(), 12);
assert_eq!(
size_of::<Option<Option2<&(), Option<NicheWithData>>>>(),
size_of::<(&(), NicheWithData)>()
);
pub enum FillPadding { A(std::num::NonZeroU8, u32), B }
assert_eq!(size_of::<FillPadding>(), 8);
assert_eq!(size_of::<Option<FillPadding>>(), 8);
assert_eq!(size_of::<Option<Option<FillPadding>>>(), 8);
assert_eq!(size_of::<Result<(std::num::NonZeroU8, u8, u8), u16>>(), 4);
assert_eq!(size_of::<Option<Result<(std::num::NonZeroU8, u8, u8), u16>>>(), 4);
assert_eq!(size_of::<Result<(std::num::NonZeroU8, u8, u8, u8), u16>>(), 4);
assert_eq!(size_of::<EnumManyVariant<u16>>(), 6);
assert_eq!(size_of::<EnumManyVariant<NicheU16>>(), 4);
assert_eq!(size_of::<EnumManyVariant<Option<NicheU16>>>(), 4);
assert_eq!(size_of::<EnumManyVariant<Option2<NicheU16,u8>>>(), 6);
assert_eq!(size_of::<EnumManyVariant<Option<(NicheU16,u8)>>>(), 6);
}