From 105ac1c26d5ea9de84f6b4af583e85eb8991e749 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 21 Jun 2023 16:04:48 +1000 Subject: [PATCH] Merge root and inlined item placement. There's no longer any need for them to be separate, and putting them together reduces the amount of code. --- .../rustc_monomorphize/src/partitioning.rs | 115 +++++++----------- 1 file changed, 44 insertions(+), 71 deletions(-) diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs index 3be0908fc7f..4e9a4f91ddf 100644 --- a/compiler/rustc_monomorphize/src/partitioning.rs +++ b/compiler/rustc_monomorphize/src/partitioning.rs @@ -125,7 +125,7 @@ struct PartitioningCx<'a, 'tcx> { usage_map: &'a UsageMap<'tcx>, } -struct PlacedRootMonoItems<'tcx> { +struct PlacedMonoItems<'tcx> { /// The codegen units, sorted by name to make things deterministic. codegen_units: Vec>, @@ -150,36 +150,20 @@ where let cx = &PartitioningCx { tcx, usage_map }; - // In the first step, we place all regular monomorphizations into their - // respective 'home' codegen unit. Regular monomorphizations are all - // functions and statics defined in the local crate. - let PlacedRootMonoItems { mut codegen_units, internalization_candidates, unique_inlined_stats } = { - let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_place_roots"); - let mut placed = place_root_mono_items(cx, mono_items); + // Place all mono items into a codegen unit. + let PlacedMonoItems { mut codegen_units, internalization_candidates, unique_inlined_stats } = { + let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_place_items"); + let mut placed = place_mono_items(cx, mono_items); for cgu in &mut placed.codegen_units { cgu.create_size_estimate(tcx); } - debug_dump(tcx, "ROOTS", &placed.codegen_units, placed.unique_inlined_stats); + debug_dump(tcx, "PLACE", &placed.codegen_units, placed.unique_inlined_stats); placed }; - // Use the usage map to put additional mono items in each codegen unit: - // drop-glue, functions from external crates, and local functions the - // definition of which is marked with `#[inline]`. - { - let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_place_inline_items"); - place_inlined_mono_items(cx, &mut codegen_units); - - for cgu in &mut codegen_units { - cgu.create_size_estimate(tcx); - } - - debug_dump(tcx, "INLINE", &codegen_units, unique_inlined_stats); - } - // Merge until we have at most `max_cgu_count` codegen units. // `merge_codegen_units` is responsible for updating the CGU size // estimates. @@ -211,10 +195,7 @@ where codegen_units } -fn place_root_mono_items<'tcx, I>( - cx: &PartitioningCx<'_, 'tcx>, - mono_items: I, -) -> PlacedRootMonoItems<'tcx> +fn place_mono_items<'tcx, I>(cx: &PartitioningCx<'_, 'tcx>, mono_items: I) -> PlacedMonoItems<'tcx> where I: Iterator>, { @@ -235,6 +216,8 @@ where let mut num_unique_inlined_items = 0; let mut unique_inlined_items_size = 0; for mono_item in mono_items { + // Handle only root items directly here. Inlined items are handled at + // the bottom of the loop based on reachability. match mono_item.instantiation_mode(cx.tcx) { InstantiationMode::GloballyShared { .. } => {} InstantiationMode::LocalCopy => { @@ -247,7 +230,7 @@ where let characteristic_def_id = characteristic_def_id_of_mono_item(cx.tcx, mono_item); let is_volatile = is_incremental_build && mono_item.is_generic_fn(); - let codegen_unit_name = match characteristic_def_id { + let cgu_name = match characteristic_def_id { Some(def_id) => compute_codegen_unit_name( cx.tcx, cgu_name_builder, @@ -258,9 +241,7 @@ where None => fallback_cgu_name(cgu_name_builder), }; - let codegen_unit = codegen_units - .entry(codegen_unit_name) - .or_insert_with(|| CodegenUnit::new(codegen_unit_name)); + let cgu = codegen_units.entry(cgu_name).or_insert_with(|| CodegenUnit::new(cgu_name)); let mut can_be_internalized = true; let (linkage, visibility) = mono_item_linkage_and_visibility( @@ -273,23 +254,52 @@ where internalization_candidates.insert(mono_item); } - codegen_unit.items_mut().insert(mono_item, (linkage, visibility)); + cgu.items_mut().insert(mono_item, (linkage, visibility)); + + // Get all inlined items that are reachable from `mono_item` without + // going via another root item. This includes drop-glue, functions from + // external crates, and local functions the definition of which is + // marked with `#[inline]`. + let mut reachable_inlined_items = FxHashSet::default(); + get_reachable_inlined_items(cx.tcx, mono_item, cx.usage_map, &mut reachable_inlined_items); + + // Add those inlined items. It's possible an inlined item is reachable + // from multiple root items within a CGU, which is fine, it just means + // the `insert` will be a no-op. + for inlined_item in reachable_inlined_items { + // This is a CGU-private copy. + cgu.items_mut().insert(inlined_item, (Linkage::Internal, Visibility::Default)); + } } // Always ensure we have at least one CGU; otherwise, if we have a // crate with just types (for example), we could wind up with no CGU. if codegen_units.is_empty() { - let codegen_unit_name = fallback_cgu_name(cgu_name_builder); - codegen_units.insert(codegen_unit_name, CodegenUnit::new(codegen_unit_name)); + let cgu_name = fallback_cgu_name(cgu_name_builder); + codegen_units.insert(cgu_name, CodegenUnit::new(cgu_name)); } let mut codegen_units: Vec<_> = codegen_units.into_values().collect(); codegen_units.sort_by(|a, b| a.name().as_str().cmp(b.name().as_str())); - PlacedRootMonoItems { + return PlacedMonoItems { codegen_units, internalization_candidates, unique_inlined_stats: (num_unique_inlined_items, unique_inlined_items_size), + }; + + fn get_reachable_inlined_items<'tcx>( + tcx: TyCtxt<'tcx>, + item: MonoItem<'tcx>, + usage_map: &UsageMap<'tcx>, + visited: &mut FxHashSet>, + ) { + usage_map.for_each_inlined_used_item(tcx, item, |inlined_item| { + let is_new = visited.insert(inlined_item); + if is_new { + get_reachable_inlined_items(tcx, inlined_item, usage_map, visited); + } + }); } } @@ -407,43 +417,6 @@ fn merge_codegen_units<'tcx>( codegen_units.sort_by(|a, b| a.name().as_str().cmp(b.name().as_str())); } -fn place_inlined_mono_items<'tcx>( - cx: &PartitioningCx<'_, 'tcx>, - codegen_units: &mut [CodegenUnit<'tcx>], -) { - for cgu in codegen_units.iter_mut() { - // Collect all inlined items that need to be available in this codegen unit. - let mut reachable_inlined_items = FxHashSet::default(); - for root in cgu.items().keys() { - // Get all inlined items that are reachable from it without going - // via another root item. - get_reachable_inlined_items(cx.tcx, *root, cx.usage_map, &mut reachable_inlined_items); - } - - // Add all monomorphizations that are not already there. - for inlined_item in reachable_inlined_items { - assert!(!cgu.items().contains_key(&inlined_item)); - - // This is a CGU-private copy. - cgu.items_mut().insert(inlined_item, (Linkage::Internal, Visibility::Default)); - } - } - - fn get_reachable_inlined_items<'tcx>( - tcx: TyCtxt<'tcx>, - item: MonoItem<'tcx>, - usage_map: &UsageMap<'tcx>, - visited: &mut FxHashSet>, - ) { - usage_map.for_each_inlined_used_item(tcx, item, |inlined_item| { - let is_new = visited.insert(inlined_item); - if is_new { - get_reachable_inlined_items(tcx, inlined_item, usage_map, visited); - } - }); - } -} - fn internalize_symbols<'tcx>( cx: &PartitioningCx<'_, 'tcx>, codegen_units: &mut [CodegenUnit<'tcx>],