diff --git a/src/librustc/ty/layout.rs b/src/librustc/ty/layout.rs index cb702176027..a08a9ddcd1a 100644 --- a/src/librustc/ty/layout.rs +++ b/src/librustc/ty/layout.rs @@ -836,7 +836,7 @@ impl<'a, 'tcx> Struct { } (&FatPointer { non_zero: true, .. }, _) => { - Ok(Some((layout.field_offset(tcx, FAT_PTR_ADDR), Pointer))) + Ok(Some((layout.fields.offset(FAT_PTR_ADDR), Pointer))) } // Is this the NonZero lang item wrapping a pointer or integer type? @@ -846,11 +846,11 @@ impl<'a, 'tcx> Struct { // FIXME(eddyb) also allow floating-point types here. Scalar { value: value @ Int(_), non_zero: false } | Scalar { value: value @ Pointer, non_zero: false } => { - Ok(Some((layout.field_offset(tcx, 0), value))) + Ok(Some((layout.fields.offset(0), value))) } FatPointer { non_zero: false, .. } => { - Ok(Some((layout.field_offset(tcx, 0) + - field.field_offset(tcx, FAT_PTR_ADDR), + Ok(Some((layout.fields.offset(0) + + field.fields.offset(FAT_PTR_ADDR), Pointer))) } _ => Ok(None) @@ -862,7 +862,7 @@ impl<'a, 'tcx> Struct { variant.non_zero_field( tcx, param_env, - (0..layout.field_count()).map(|i| layout.field(cx, i))) + (0..layout.fields.count()).map(|i| layout.field(cx, i))) } // Is this a fixed-size array of something non-zero @@ -991,6 +991,59 @@ pub const FAT_PTR_ADDR: usize = 0; /// - For a slice, this is the length. pub const FAT_PTR_EXTRA: usize = 1; +/// Describes how the fields of a type are located in memory. +#[derive(Copy, Clone, Debug)] +pub enum FieldPlacement<'a> { + /// Array-like placement. Can also express + /// unions, by using a stride of zero bytes. + Linear { + stride: Size, + count: u64 + }, + + /// Struct-like placement, with precomputed offsets. + /// + /// Fields are guaranteed to not overlap, but note that gaps + /// before, between and after all the fields are NOT always + /// padding, and as such their contents may not be discarded. + /// For example, enum variants leave a gap at the start, + /// where the discriminant field in the enum layout goes. + Arbitrary { + offsets: &'a [Size] + } +} + +impl<'a> FieldPlacement<'a> { + pub fn union(count: usize) -> Self { + FieldPlacement::Linear { + stride: Size::from_bytes(0), + count: count as u64 + } + } + + pub fn count(&self) -> usize { + match *self { + FieldPlacement::Linear { count, .. } => { + let usize_count = count as usize; + assert_eq!(usize_count as u64, count); + usize_count + } + FieldPlacement::Arbitrary { offsets } => offsets.len() + } + } + + pub fn offset(&self, i: usize) -> Size { + match *self { + FieldPlacement::Linear { stride, count, .. } => { + let i = i as u64; + assert!(i < count); + stride * i + } + FieldPlacement::Arbitrary { offsets } => offsets[i] + } + } +} + /// Type layout, from which size and alignment can be cheaply computed. /// For ADTs, it also includes field placement and enum optimizations. /// NOTE: Because Layout is interned, redundant information should be @@ -1105,9 +1158,15 @@ impl<'tcx> fmt::Display for LayoutError<'tcx> { } } +#[derive(Copy, Clone, Debug)] +pub struct CachedLayout<'tcx> { + pub layout: &'tcx Layout, + pub fields: FieldPlacement<'tcx>, +} + fn layout_raw<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) - -> Result<&'tcx Layout, LayoutError<'tcx>> + -> Result, LayoutError<'tcx>> { let (param_env, ty) = query.into_parts(); @@ -1136,10 +1195,59 @@ impl<'a, 'tcx> Layout { fn compute_uncached(tcx: TyCtxt<'a, 'tcx, 'tcx>, param_env: ty::ParamEnv<'tcx>, ty: Ty<'tcx>) - -> Result<&'tcx Layout, LayoutError<'tcx>> { - let success = |layout| Ok(tcx.intern_layout(layout)); + -> Result, LayoutError<'tcx>> { let cx = (tcx, param_env); let dl = cx.data_layout(); + let success = |layout| { + let layout = tcx.intern_layout(layout); + let fields = match *layout { + Scalar { .. } | + CEnum { .. } | + RawNullablePointer { .. } | + StructWrappedNullablePointer { .. } => { + FieldPlacement::union(0) + } + + Vector { element, count } => { + FieldPlacement::Linear { + stride: element.size(tcx), + count + } + } + + Array { element_size, count, .. } => { + FieldPlacement::Linear { + stride: element_size, + count + } + } + + FatPointer { .. } => { + FieldPlacement::Linear { + stride: Pointer.size(tcx), + count: 2 + } + } + + Univariant(ref variant) => { + FieldPlacement::Arbitrary { + offsets: &variant.offsets + } + } + + UntaggedUnion(_) => { + // Handle unions through the type rather than Layout. + let def = ty.ty_adt_def().unwrap(); + FieldPlacement::union(def.struct_variant().fields.len()) + } + + General { .. } => FieldPlacement::union(1) + }; + Ok(CachedLayout { + layout, + fields + }) + }; assert!(!ty.has_infer_types()); let ptr_layout = |pointee: Ty<'tcx>| { @@ -1536,7 +1644,11 @@ impl<'a, 'tcx> Layout { if ty == normalized { return Err(LayoutError::Unknown(ty)); } - return Ok(cx.layout_of(normalized)?.layout); + let layout = cx.layout_of(normalized)?; + return Ok(CachedLayout { + layout: layout.layout, + fields: layout.fields + }); } ty::TyParam(_) => { return Err(LayoutError::Unknown(ty)); @@ -1656,61 +1768,6 @@ impl<'a, 'tcx> Layout { } } - pub fn field_offset(&self, - cx: C, - i: usize, - variant_index: Option) - -> Size { - let dl = cx.data_layout(); - - match *self { - Scalar { .. } | - CEnum { .. } | - UntaggedUnion(_) | - RawNullablePointer { .. } => { - Size::from_bytes(0) - } - - Vector { element, count } => { - let element_size = element.size(dl); - let i = i as u64; - assert!(i < count); - Size::from_bytes(element_size.bytes() * count) - } - - Array { element_size, count, .. } => { - let i = i as u64; - assert!(i < count); - Size::from_bytes(element_size.bytes() * count) - } - - FatPointer { metadata, .. } => { - // Effectively a (ptr, meta) tuple. - assert!(i < 2); - if i == 0 { - Size::from_bytes(0) - } else { - Pointer.size(dl).abi_align(metadata.align(dl)) - } - } - - Univariant(ref variant) => variant.offsets[i], - - General { ref variants, .. } => { - let v = variant_index.expect("variant index required"); - variants[v].offsets[i] - } - - StructWrappedNullablePointer { nndiscr, ref nonnull, .. } => { - if Some(nndiscr as usize) == variant_index { - nonnull.offsets[i] - } else { - Size::from_bytes(0) - } - } - } - } - /// This is invoked by the `layout_raw` query to record the final /// layout of each type. #[inline] @@ -2077,6 +2134,7 @@ pub struct FullLayout<'tcx> { pub ty: Ty<'tcx>, pub variant_index: Option, pub layout: &'tcx Layout, + pub fields: FieldPlacement<'tcx>, } impl<'tcx> Deref for FullLayout<'tcx> { @@ -2130,93 +2188,97 @@ impl<'a, 'tcx> LayoutOf> for (TyCtxt<'a, 'tcx, 'tcx>, ty::ParamEnv<'tcx let (tcx, param_env) = self; let ty = tcx.normalize_associated_type_in_env(&ty, param_env.reveal_all()); - let layout = tcx.layout_raw(param_env.reveal_all().and(ty)); + let cached = tcx.layout_raw(param_env.reveal_all().and(ty))?; // NB: This recording is normally disabled; when enabled, it - // can however trigger recursive invocations of `layout()`. + // can however trigger recursive invocations of `layout_of`. // Therefore, we execute it *after* the main query has // completed, to avoid problems around recursive structures // and the like. (Admitedly, I wasn't able to reproduce a problem // here, but it seems like the right thing to do. -nmatsakis) - if let Ok(l) = layout { - Layout::record_layout_for_printing(tcx, ty, param_env, l); - } + Layout::record_layout_for_printing(tcx, ty, param_env, cached.layout); Ok(FullLayout { ty, variant_index: None, - layout: layout?, + layout: cached.layout, + fields: cached.fields + }) + } +} + +impl<'a, 'tcx> LayoutOf> for (ty::maps::TyCtxtAt<'a, 'tcx, 'tcx>, + ty::ParamEnv<'tcx>) { + type FullLayout = Result, LayoutError<'tcx>>; + + /// Computes the layout of a type. Note that this implicitly + /// executes in "reveal all" mode. + #[inline] + fn layout_of(self, ty: Ty<'tcx>) -> Self::FullLayout { + let (tcx_at, param_env) = self; + + let ty = tcx_at.tcx.normalize_associated_type_in_env(&ty, param_env.reveal_all()); + let cached = tcx_at.layout_raw(param_env.reveal_all().and(ty))?; + + // NB: This recording is normally disabled; when enabled, it + // can however trigger recursive invocations of `layout_of`. + // Therefore, we execute it *after* the main query has + // completed, to avoid problems around recursive structures + // and the like. (Admitedly, I wasn't able to reproduce a problem + // here, but it seems like the right thing to do. -nmatsakis) + Layout::record_layout_for_printing(tcx_at.tcx, ty, param_env, cached.layout); + + Ok(FullLayout { + ty, + variant_index: None, + layout: cached.layout, + fields: cached.fields }) } } impl<'a, 'tcx> FullLayout<'tcx> { pub fn for_variant(&self, variant_index: usize) -> Self { - let is_enum = match self.ty.sty { - ty::TyAdt(def, _) => def.is_enum(), - _ => false + let variants = match self.ty.sty { + ty::TyAdt(def, _) if def.is_enum() => &def.variants[..], + _ => &[] }; - assert!(is_enum); + let count = if variants.is_empty() { + 0 + } else { + variants[variant_index].fields.len() + }; + + let fields = match *self.layout { + Univariant(ref variant) => { + FieldPlacement::Arbitrary { + offsets: &variant.offsets + } + } + + General { ref variants, .. } => { + FieldPlacement::Arbitrary { + offsets: &variants[variant_index].offsets + } + } + + StructWrappedNullablePointer { nndiscr, ref nonnull, .. } + if nndiscr as usize == variant_index => { + FieldPlacement::Arbitrary { + offsets: &nonnull.offsets + } + } + + _ => FieldPlacement::union(count) + }; + FullLayout { variant_index: Some(variant_index), + fields, ..*self } } - pub fn field_offset(&self, cx: C, i: usize) -> Size { - self.layout.field_offset(cx, i, self.variant_index) - } - - pub fn field_count(&self) -> usize { - // Handle enum/union through the type rather than Layout. - if let ty::TyAdt(def, _) = self.ty.sty { - let v = if def.is_enum() { - if def.variants.is_empty() { - return 0; - } - match self.variant_index { - None => match *self.layout { - // Discriminant field for enums (where applicable). - General { .. } => return 1, - _ if def.variants.len() > 1 => return 0, - - // Enums with one variant behave like structs. - _ => 0 - }, - Some(v) => v - } - } else { - 0 - }; - - return def.variants[v].fields.len(); - } - - match *self.layout { - Scalar { .. } => { - bug!("FullLayout::field_count({:?}): not applicable", self) - } - - // Handled above (the TyAdt case). - CEnum { .. } | - General { .. } | - UntaggedUnion(_) | - RawNullablePointer { .. } | - StructWrappedNullablePointer { .. } => bug!(), - - FatPointer { .. } => 2, - - Vector { count, .. } | - Array { count, .. } => { - let usize_count = count as usize; - assert_eq!(usize_count as u64, count); - usize_count - } - - Univariant(ref variant) => variant.offsets.len(), - } - } - fn field_type_unnormalized(&self, tcx: TyCtxt<'a, 'tcx, 'tcx>, i: usize) -> Ty<'tcx> { let ptr_field_type = |pointee: Ty<'tcx>| { assert!(i < 2); @@ -2384,6 +2446,30 @@ impl<'gcx> HashStable> for Layout } } +impl<'gcx> HashStable> for FieldPlacement<'gcx> { + fn hash_stable(&self, + hcx: &mut StableHashingContext<'gcx>, + hasher: &mut StableHasher) { + use ty::layout::FieldPlacement::*; + mem::discriminant(self).hash_stable(hcx, hasher); + + match *self { + Linear { count, stride } => { + count.hash_stable(hcx, hasher); + stride.hash_stable(hcx, hasher); + } + Arbitrary { offsets } => { + offsets.hash_stable(hcx, hasher); + } + } + } +} + +impl_stable_hash_for!(struct ::ty::layout::CachedLayout<'tcx> { + layout, + fields +}); + impl_stable_hash_for!(enum ::ty::layout::Integer { I1, I8, diff --git a/src/librustc/ty/maps/mod.rs b/src/librustc/ty/maps/mod.rs index 320f6514849..67467763089 100644 --- a/src/librustc/ty/maps/mod.rs +++ b/src/librustc/ty/maps/mod.rs @@ -34,7 +34,6 @@ use session::config::OutputFilenames; use traits::Vtable; use traits::specialization_graph; use ty::{self, CrateInherentImpls, Ty, TyCtxt}; -use ty::layout::{Layout, LayoutError}; use ty::steal::Steal; use ty::subst::Substs; use util::nodemap::{DefIdSet, DefIdMap, ItemLocalSet}; @@ -265,7 +264,8 @@ define_maps! { <'tcx> [] fn is_freeze_raw: is_freeze_dep_node(ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool, [] fn needs_drop_raw: needs_drop_dep_node(ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool, [] fn layout_raw: layout_dep_node(ty::ParamEnvAnd<'tcx, Ty<'tcx>>) - -> Result<&'tcx Layout, LayoutError<'tcx>>, + -> Result, + ty::layout::LayoutError<'tcx>>, [] fn dylib_dependency_formats: DylibDepFormats(CrateNum) -> Rc>, diff --git a/src/librustc_const_eval/eval.rs b/src/librustc_const_eval/eval.rs index 657156902b5..7badea86c1a 100644 --- a/src/librustc_const_eval/eval.rs +++ b/src/librustc_const_eval/eval.rs @@ -17,6 +17,7 @@ use rustc::hir::map::blocks::FnLikeNode; use rustc::hir::def::{Def, CtorKind}; use rustc::hir::def_id::DefId; use rustc::ty::{self, Ty, TyCtxt}; +use rustc::ty::layout::LayoutOf; use rustc::ty::maps::Providers; use rustc::ty::util::IntTypeExt; use rustc::ty::subst::{Substs, Subst}; @@ -313,7 +314,7 @@ fn eval_const_expr_partial<'a, 'tcx>(cx: &ConstContext<'a, 'tcx>, if tcx.fn_sig(def_id).abi() == Abi::RustIntrinsic { let layout_of = |ty: Ty<'tcx>| { let ty = tcx.erase_regions(&ty); - tcx.at(e.span).layout_raw(cx.param_env.reveal_all().and(ty)).map_err(|err| { + (tcx.at(e.span), cx.param_env).layout_of(ty).map_err(|err| { ConstEvalErr { span: e.span, kind: LayoutError(err) } }) }; diff --git a/src/librustc_trans/abi.rs b/src/librustc_trans/abi.rs index 695fc9160c5..1b4cbd687df 100644 --- a/src/librustc_trans/abi.rs +++ b/src/librustc_trans/abi.rs @@ -334,7 +334,7 @@ impl<'tcx> LayoutExt<'tcx> for FullLayout<'tcx> { let mut unaligned_offset = Size::from_bytes(0); let mut result = None; - for i in 0..self.field_count() { + for i in 0..self.fields.count() { if unaligned_offset != variant.offsets[i] { return None; } @@ -371,7 +371,7 @@ impl<'tcx> LayoutExt<'tcx> for FullLayout<'tcx> { let mut max = Size::from_bytes(0); let mut result = None; - for i in 0..self.field_count() { + for i in 0..self.fields.count() { let field = self.field(ccx, i); match (result, field.homogeneous_aggregate(ccx)) { // The field itself must be a homogeneous aggregate. diff --git a/src/librustc_trans/adt.rs b/src/librustc_trans/adt.rs index 3d67d3df042..ec87429c2b2 100644 --- a/src/librustc_trans/adt.rs +++ b/src/librustc_trans/adt.rs @@ -209,7 +209,7 @@ pub fn memory_index_to_gep(index: u64) -> u64 { pub fn struct_llfields<'a, 'tcx>(cx: &CrateContext<'a, 'tcx>, layout: FullLayout<'tcx>, variant: &layout::Struct) -> Vec { - let field_count = layout.field_count(); + let field_count = layout.fields.count(); debug!("struct_llfields: variant: {:?}", variant); let mut offset = Size::from_bytes(0); let mut result: Vec = Vec::with_capacity(1 + field_count * 2); diff --git a/src/librustc_trans/cabi_s390x.rs b/src/librustc_trans/cabi_s390x.rs index 225919c6a3a..4db33b9204a 100644 --- a/src/librustc_trans/cabi_s390x.rs +++ b/src/librustc_trans/cabi_s390x.rs @@ -30,7 +30,7 @@ fn is_single_fp_element<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, Layout::Scalar { value: layout::F32, .. } | Layout::Scalar { value: layout::F64, .. } => true, Layout::Univariant { .. } => { - if layout.field_count() == 1 { + if layout.fields.count() == 1 { is_single_fp_element(ccx, layout.field(ccx, 0)) } else { false diff --git a/src/librustc_trans/cabi_x86.rs b/src/librustc_trans/cabi_x86.rs index 0f9f9b87b5c..362ceb60601 100644 --- a/src/librustc_trans/cabi_x86.rs +++ b/src/librustc_trans/cabi_x86.rs @@ -25,7 +25,7 @@ fn is_single_fp_element<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, Layout::Scalar { value: layout::F32, .. } | Layout::Scalar { value: layout::F64, .. } => true, Layout::Univariant { .. } => { - if layout.field_count() == 1 { + if layout.fields.count() == 1 { is_single_fp_element(ccx, layout.field(ccx, 0)) } else { false diff --git a/src/librustc_trans/cabi_x86_64.rs b/src/librustc_trans/cabi_x86_64.rs index 493a9f713fd..95470b07564 100644 --- a/src/librustc_trans/cabi_x86_64.rs +++ b/src/librustc_trans/cabi_x86_64.rs @@ -102,14 +102,14 @@ fn classify_arg<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, arg: &ArgType<'tcx>) } Layout::Univariant(ref variant) => { - for i in 0..layout.field_count() { + for i in 0..layout.fields.count() { let field_off = off + variant.offsets[i]; classify(ccx, layout.field(ccx, i), cls, field_off)?; } } Layout::UntaggedUnion { .. } => { - for i in 0..layout.field_count() { + for i in 0..layout.fields.count() { classify(ccx, layout.field(ccx, i), cls, off)?; } } diff --git a/src/librustc_trans/debuginfo/metadata.rs b/src/librustc_trans/debuginfo/metadata.rs index 44dc831e442..e08141f2fc3 100644 --- a/src/librustc_trans/debuginfo/metadata.rs +++ b/src/librustc_trans/debuginfo/metadata.rs @@ -429,7 +429,7 @@ fn trait_pointer_metadata<'a, 'tcx>(cx: &CrateContext<'a, 'tcx>, type_metadata: type_metadata(cx, cx.tcx().mk_mut_ptr(cx.tcx().types.u8), syntax_pos::DUMMY_SP), - offset: layout.field_offset(cx, 0), + offset: layout.fields.offset(0), size: data_ptr_field.size(cx), align: data_ptr_field.align(cx), flags: DIFlags::FlagArtificial, @@ -437,7 +437,7 @@ fn trait_pointer_metadata<'a, 'tcx>(cx: &CrateContext<'a, 'tcx>, MemberDescription { name: "vtable".to_string(), type_metadata: type_metadata(cx, vtable_field.ty, syntax_pos::DUMMY_SP), - offset: layout.field_offset(cx, 1), + offset: layout.fields.offset(1), size: vtable_field.size(cx), align: vtable_field.align(cx), flags: DIFlags::FlagArtificial, @@ -1321,8 +1321,8 @@ impl<'tcx> EnumMemberDescriptionFactory<'tcx> { layout: FullLayout<'tcx>, offset: Size, size: Size) { - for i in 0..layout.field_count() { - let field_offset = layout.field_offset(ccx, i); + for i in 0..layout.fields.count() { + let field_offset = layout.fields.offset(i); if field_offset > offset { continue; } @@ -1414,7 +1414,7 @@ fn describe_enum_variant<'a, 'tcx>(cx: &CrateContext<'a, 'tcx>, }; let layout = layout.for_variant(variant_index); - let mut field_tys = (0..layout.field_count()).map(|i| { + let mut field_tys = (0..layout.fields.count()).map(|i| { layout.field(cx, i).ty }).collect::>(); diff --git a/src/librustc_trans/glue.rs b/src/librustc_trans/glue.rs index 657f7da66b3..61c0820539d 100644 --- a/src/librustc_trans/glue.rs +++ b/src/librustc_trans/glue.rs @@ -74,7 +74,7 @@ pub fn size_and_align_of_dst<'a, 'tcx>(bcx: &Builder<'a, 'tcx>, t: Ty<'tcx>, inf // Recurse to get the size of the dynamically sized field (must be // the last field). - let field_ty = layout.field(ccx, layout.field_count() - 1).ty; + let field_ty = layout.field(ccx, layout.fields.count() - 1).ty; let (unsized_size, unsized_align) = size_and_align_of_dst(bcx, field_ty, info); // FIXME (#26403, #27023): We should be adding padding diff --git a/src/librustc_trans/mir/lvalue.rs b/src/librustc_trans/mir/lvalue.rs index 8f094cd5102..2bd76308c91 100644 --- a/src/librustc_trans/mir/lvalue.rs +++ b/src/librustc_trans/mir/lvalue.rs @@ -295,7 +295,7 @@ impl<'a, 'tcx> LvalueRef<'tcx> { let meta = self.llextra; - let offset = l.field_offset(ccx, ix).bytes(); + let offset = l.fields.offset(ix).bytes(); let unaligned_offset = C_usize(ccx, offset); // Get the alignment of the field