Auto merge of #126784 - scottmcm:smaller-terminator, r=compiler-errors

Save 2 pointers in `TerminatorKind` (96 → 80 bytes)

These things don't need to be `Vec`s; boxed slices are enough.

The frequent one here is call arguments, but MIR building knows the number of arguments from the THIR, so the collect is always getting the allocation right in the first place, and thus this shouldn't ever add the shrink-in-place overhead.

compiler/rustc_middle/src/mir/mod.rs

@@ -1817,11 +1817,11 @@ mod size_asserts {
     use super::*;
     use rustc_data_structures::static_assert_size;
     // tidy-alphabetical-start
-    static_assert_size!(BasicBlockData<'_>, 144);
+    static_assert_size!(BasicBlockData<'_>, 128);
     static_assert_size!(LocalDecl<'_>, 40);
     static_assert_size!(SourceScopeData<'_>, 64);
     static_assert_size!(Statement<'_>, 32);
-    static_assert_size!(Terminator<'_>, 112);
+    static_assert_size!(Terminator<'_>, 96);
     static_assert_size!(VarDebugInfo<'_>, 88);
     // tidy-alphabetical-end
 }

compiler/rustc_middle/src/mir/syntax.rs

@@ -730,7 +730,7 @@ pub enum TerminatorKind<'tcx> {
         /// reused across function calls without duplicating the contents.
         /// The span for each arg is also included
         /// (e.g. `a` and `b` in `x.foo(a, b)`).
-        args: Vec<Spanned<Operand<'tcx>>>,
+        args: Box<[Spanned<Operand<'tcx>>]>,
         /// Where the returned value will be written
         destination: Place<'tcx>,
         /// Where to go after this call returns. If none, the call necessarily diverges.
@@ -837,7 +837,7 @@ pub enum TerminatorKind<'tcx> {
         template: &'tcx [InlineAsmTemplatePiece],
 
         /// The operands for the inline assembly, as `Operand`s or `Place`s.
-        operands: Vec<InlineAsmOperand<'tcx>>,
+        operands: Box<[InlineAsmOperand<'tcx>]>,
 
         /// Miscellaneous options for the inline assembly.
         options: InlineAsmOptions,
@@ -849,7 +849,7 @@ pub enum TerminatorKind<'tcx> {
         /// Valid targets for the inline assembly.
         /// The first element is the fallthrough destination, unless
         /// InlineAsmOptions::NORETURN is set.
-        targets: Vec<BasicBlock>,
+        targets: Box<[BasicBlock]>,
 
         /// Action to be taken if the inline assembly unwinds. This is present
         /// if and only if InlineAsmOptions::MAY_UNWIND is set.
@@ -1561,6 +1561,6 @@ mod size_asserts {
     static_assert_size!(PlaceElem<'_>, 24);
     static_assert_size!(Rvalue<'_>, 40);
     static_assert_size!(StatementKind<'_>, 16);
-    static_assert_size!(TerminatorKind<'_>, 96);
+    static_assert_size!(TerminatorKind<'_>, 80);
     // tidy-alphabetical-end
 }

compiler/rustc_mir_build/src/build/custom/parse/instruction.rs

@@ -170,7 +170,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
                     .map(|arg|
                         Ok(Spanned { node: self.parse_operand(*arg)?, span: self.thir.exprs[*arg].span  } )
                     )
-                    .collect::<PResult<Vec<_>>>()?;
+                    .collect::<PResult<Box<[_]>>>()?;
                 Ok(TerminatorKind::Call {
                     func: fun,
                     args,

compiler/rustc_mir_build/src/build/expr/as_rvalue.rs

@@ -152,10 +152,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     source_info,
                     TerminatorKind::Call {
                         func: exchange_malloc,
-                        args: vec![
+                        args: [
                             Spanned { node: Operand::Move(size), span: DUMMY_SP },
                             Spanned { node: Operand::Move(align), span: DUMMY_SP },
-                        ],
+                        ]
+                        .into(),
                         destination: storage,
                         target: Some(success),
                         unwind: UnwindAction::Continue,

compiler/rustc_mir_build/src/build/expr/into.rs

@@ -238,7 +238,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
             ExprKind::Call { ty: _, fun, ref args, from_hir_call, fn_span } => {
                 let fun = unpack!(block = this.as_local_operand(block, fun));
-                let args: Vec<_> = args
+                let args: Box<[_]> = args
                     .into_iter()
                     .copied()
                     .map(|arg| Spanned {
@@ -485,7 +485,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         operands,
                         options,
                         line_spans,
-                        targets,
+                        targets: targets.into_boxed_slice(),
                         unwind: if options.contains(InlineAsmOptions::MAY_UNWIND) {
                             UnwindAction::Continue
                         } else {

compiler/rustc_mir_build/src/build/matches/test.rs

@@ -324,7 +324,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     user_ty: None,
                     const_: method,
                 })),
-                args: vec![Spanned { node: Operand::Move(ref_src), span }],
+                args: [Spanned { node: Operand::Move(ref_src), span }].into(),
                 destination: temp,
                 target: Some(target_block),
                 unwind: UnwindAction::Continue,
@@ -486,10 +486,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                     const_: method,
                 })),
-                args: vec![
+                args: [
                     Spanned { node: Operand::Copy(val), span: DUMMY_SP },
                     Spanned { node: expect, span: DUMMY_SP },
-                ],
+                ]
+                .into(),
                 destination: eq_result,
                 target: Some(eq_block),
                 unwind: UnwindAction::Continue,

compiler/rustc_mir_dataflow/src/elaborate_drops.rs

@@ -650,10 +650,8 @@ where
                         [ty.into()],
                         self.source_info.span,
                     ),
-                    args: vec![Spanned {
+                    args: [Spanned { node: Operand::Move(Place::from(ref_place)), span: DUMMY_SP }]
-                        node: Operand::Move(Place::from(ref_place)),
+                        .into(),
-                        span: DUMMY_SP,
-                    }],
                     destination: unit_temp,
                     target: Some(succ),
                     unwind: unwind.into_action(),

compiler/rustc_mir_dataflow/src/framework/tests.rs

@@ -34,7 +34,7 @@ fn mock_body<'tcx>() -> mir::Body<'tcx> {
         2,
         mir::TerminatorKind::Call {
             func: mir::Operand::Copy(dummy_place.clone()),
-            args: vec![],
+            args: [].into(),
             destination: dummy_place.clone(),
             target: Some(mir::START_BLOCK),
             unwind: mir::UnwindAction::Continue,
@@ -48,7 +48,7 @@ fn mock_body<'tcx>() -> mir::Body<'tcx> {
         4,
         mir::TerminatorKind::Call {
             func: mir::Operand::Copy(dummy_place.clone()),
-            args: vec![],
+            args: [].into(),
             destination: dummy_place.clone(),
             target: Some(mir::START_BLOCK),
             unwind: mir::UnwindAction::Continue,

compiler/rustc_mir_transform/src/coroutine.rs

@@ -677,8 +677,8 @@ fn transform_async_context<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
 
 fn eliminate_get_context_call<'tcx>(bb_data: &mut BasicBlockData<'tcx>) -> Local {
     let terminator = bb_data.terminator.take().unwrap();
-    if let TerminatorKind::Call { mut args, destination, target, .. } = terminator.kind {
+    if let TerminatorKind::Call { args, destination, target, .. } = terminator.kind {
-        let arg = args.pop().unwrap();
+        let [arg] = *Box::try_from(args).unwrap();
         let local = arg.node.place().unwrap().local;
 
         let arg = Rvalue::Use(arg.node);

compiler/rustc_mir_transform/src/coverage/tests.rs

@@ -134,7 +134,7 @@ impl<'tcx> MockBlocks<'tcx> {
             some_from_block,
             TerminatorKind::Call {
                 func: Operand::Copy(self.dummy_place.clone()),
-                args: vec![],
+                args: [].into(),
                 destination: self.dummy_place.clone(),
                 target: Some(TEMP_BLOCK),
                 unwind: UnwindAction::Continue,

compiler/rustc_mir_transform/src/inline.rs

@@ -624,8 +624,7 @@ impl<'tcx> Inliner<'tcx> {
         };
 
         // Copy the arguments if needed.
-        let args: Vec<_> =
+        let args = self.make_call_args(args, &callsite, caller_body, &callee_body, return_block);
-            self.make_call_args(args, &callsite, caller_body, &callee_body, return_block);
 
         let mut integrator = Integrator {
             args: &args,
@@ -736,12 +735,12 @@ impl<'tcx> Inliner<'tcx> {
 
     fn make_call_args(
         &self,
-        args: Vec<Spanned<Operand<'tcx>>>,
+        args: Box<[Spanned<Operand<'tcx>>]>,
         callsite: &CallSite<'tcx>,
         caller_body: &mut Body<'tcx>,
         callee_body: &Body<'tcx>,
         return_block: Option<BasicBlock>,
-    ) -> Vec<Local> {
+    ) -> Box<[Local]> {
         let tcx = self.tcx;
 
         // There is a bit of a mismatch between the *caller* of a closure and the *callee*.
@@ -768,7 +767,8 @@ impl<'tcx> Inliner<'tcx> {
         //
         // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`.
         if callsite.fn_sig.abi() == Abi::RustCall && callee_body.spread_arg.is_none() {
-            let mut args = args.into_iter();
+            // FIXME(edition_2024): switch back to a normal method call.
+            let mut args = <_>::into_iter(args);
             let self_ = self.create_temp_if_necessary(
                 args.next().unwrap().node,
                 callsite,
@@ -802,7 +802,8 @@ impl<'tcx> Inliner<'tcx> {
 
             closure_ref_arg.chain(tuple_tmp_args).collect()
         } else {
-            args.into_iter()
+            // FIXME(edition_2024): switch back to a normal method call.
+            <_>::into_iter(args)
                 .map(|a| self.create_temp_if_necessary(a.node, callsite, caller_body, return_block))
                 .collect()
         }

compiler/rustc_mir_transform/src/instsimplify.rs

@@ -1,6 +1,7 @@
 //! Performs various peephole optimizations.
 
 use crate::simplify::simplify_duplicate_switch_targets;
+use crate::take_array;
 use rustc_ast::attr;
 use rustc_middle::bug;
 use rustc_middle::mir::*;
@@ -285,7 +286,8 @@ impl<'tcx> InstSimplifyContext<'tcx, '_> {
             return;
         }
 
-        let Some(arg_place) = args.pop().unwrap().node.place() else { return };
+        let Ok([arg]) = take_array(args) else { return };
+        let Some(arg_place) = arg.node.place() else { return };
 
         statements.push(Statement {
             source_info: terminator.source_info,

compiler/rustc_mir_transform/src/lib.rs

@@ -160,8 +160,9 @@ fn remap_mir_for_const_eval_select<'tcx>(
             } if let ty::FnDef(def_id, _) = *const_.ty().kind()
                 && tcx.is_intrinsic(def_id, sym::const_eval_select) =>
             {
-                let [tupled_args, called_in_const, called_at_rt]: [_; 3] =
+                let Ok([tupled_args, called_in_const, called_at_rt]) = take_array(args) else {
-                    std::mem::take(args).try_into().unwrap();
+                    unreachable!()
+                };
                 let ty = tupled_args.node.ty(&body.local_decls, tcx);
                 let fields = ty.tuple_fields();
                 let num_args = fields.len();
@@ -211,6 +212,11 @@ fn remap_mir_for_const_eval_select<'tcx>(
     body
 }
 
+fn take_array<T, const N: usize>(b: &mut Box<[T]>) -> Result<[T; N], Box<[T]>> {
+    let b: Box<[T; N]> = std::mem::take(b).try_into()?;
+    Ok(*b)
+}
+
 fn is_mir_available(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     tcx.mir_keys(()).contains(&def_id)
 }

compiler/rustc_mir_transform/src/lower_intrinsics.rs

@@ -1,5 +1,6 @@
 //! Lowers intrinsic calls
 
+use crate::take_array;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_middle::{bug, span_bug};
@@ -50,42 +51,34 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                     }
                     sym::copy_nonoverlapping => {
                         let target = target.unwrap();
-                        let mut args = args.drain(..);
+                        let Ok([src, dst, count]) = take_array(args) else {
+                            bug!("Wrong arguments for copy_non_overlapping intrinsic");
+                        };
                         block.statements.push(Statement {
                             source_info: terminator.source_info,
                             kind: StatementKind::Intrinsic(Box::new(
                                 NonDivergingIntrinsic::CopyNonOverlapping(
                                     rustc_middle::mir::CopyNonOverlapping {
-                                        src: args.next().unwrap().node,
+                                        src: src.node,
-                                        dst: args.next().unwrap().node,
+                                        dst: dst.node,
-                                        count: args.next().unwrap().node,
+                                        count: count.node,
                                     },
                                 ),
                             )),
                         });
-                        assert_eq!(
-                            args.next(),
-                            None,
-                            "Extra argument for copy_non_overlapping intrinsic"
-                        );
-                        drop(args);
                         terminator.kind = TerminatorKind::Goto { target };
                     }
                     sym::assume => {
                         let target = target.unwrap();
-                        let mut args = args.drain(..);
+                        let Ok([arg]) = take_array(args) else {
+                            bug!("Wrong arguments for assume intrinsic");
+                        };
                         block.statements.push(Statement {
                             source_info: terminator.source_info,
                             kind: StatementKind::Intrinsic(Box::new(
-                                NonDivergingIntrinsic::Assume(args.next().unwrap().node),
+                                NonDivergingIntrinsic::Assume(arg.node),
                             )),
                         });
-                        assert_eq!(
-                            args.next(),
-                            None,
-                            "Extra argument for copy_non_overlapping intrinsic"
-                        );
-                        drop(args);
                         terminator.kind = TerminatorKind::Goto { target };
                     }
                     sym::wrapping_add
@@ -100,13 +93,9 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                     | sym::unchecked_shl
                     | sym::unchecked_shr => {
                         let target = target.unwrap();
-                        let lhs;
+                        let Ok([lhs, rhs]) = take_array(args) else {
-                        let rhs;
+                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
-                        {
+                        };
-                            let mut args = args.drain(..);
-                            lhs = args.next().unwrap();
-                            rhs = args.next().unwrap();
-                        }
                         let bin_op = match intrinsic.name {
                             sym::wrapping_add => BinOp::Add,
                             sym::wrapping_sub => BinOp::Sub,
@@ -132,13 +121,9 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                     }
                     sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
                         if let Some(target) = *target {
-                            let lhs;
+                            let Ok([lhs, rhs]) = take_array(args) else {
-                            let rhs;
+                                bug!("Wrong arguments for {} intrinsic", intrinsic.name);
-                            {
+                            };
-                                let mut args = args.drain(..);
-                                lhs = args.next().unwrap();
-                                rhs = args.next().unwrap();
-                            }
                             let bin_op = match intrinsic.name {
                                 sym::add_with_overflow => BinOp::AddWithOverflow,
                                 sym::sub_with_overflow => BinOp::SubWithOverflow,
@@ -174,7 +159,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                         }
                     }
                     sym::read_via_copy => {
-                        let [arg] = args.as_slice() else {
+                        let Ok([arg]) = take_array(args) else {
                             span_bug!(terminator.source_info.span, "Wrong number of arguments");
                         };
                         let derefed_place = if let Some(place) = arg.node.place()
@@ -207,7 +192,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                     }
                     sym::write_via_move => {
                         let target = target.unwrap();
-                        let Ok([ptr, val]) = <[_; 2]>::try_from(std::mem::take(args)) else {
+                        let Ok([ptr, val]) = take_array(args) else {
                             span_bug!(
                                 terminator.source_info.span,
                                 "Wrong number of arguments for write_via_move intrinsic",
@@ -247,7 +232,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                     }
                     sym::offset => {
                         let target = target.unwrap();
-                        let Ok([ptr, delta]) = <[_; 2]>::try_from(std::mem::take(args)) else {
+                        let Ok([ptr, delta]) = take_array(args) else {
                             span_bug!(
                                 terminator.source_info.span,
                                 "Wrong number of arguments for offset intrinsic",
@@ -264,7 +249,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                     }
                     sym::transmute | sym::transmute_unchecked => {
                         let dst_ty = destination.ty(local_decls, tcx).ty;
-                        let Ok([arg]) = <[_; 1]>::try_from(std::mem::take(args)) else {
+                        let Ok([arg]) = take_array(args) else {
                             span_bug!(
                                 terminator.source_info.span,
                                 "Wrong number of arguments for transmute intrinsic",
@@ -289,7 +274,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                         }
                     }
                     sym::aggregate_raw_ptr => {
-                        let Ok([data, meta]) = <[_; 2]>::try_from(std::mem::take(args)) else {
+                        let Ok([data, meta]) = take_array(args) else {
                             span_bug!(
                                 terminator.source_info.span,
                                 "Wrong number of arguments for aggregate_raw_ptr intrinsic",
@@ -317,7 +302,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                         terminator.kind = TerminatorKind::Goto { target };
                     }
                     sym::ptr_metadata => {
-                        let Ok([ptr]) = <[_; 1]>::try_from(std::mem::take(args)) else {
+                        let Ok([ptr]) = take_array(args) else {
                             span_bug!(
                                 terminator.source_info.span,
                                 "Wrong number of arguments for ptr_metadata intrinsic",

compiler/rustc_mir_transform/src/shim.rs

@@ -562,7 +562,7 @@ impl<'tcx> CloneShimBuilder<'tcx> {
             vec![statement],
             TerminatorKind::Call {
                 func,
-                args: vec![Spanned { node: Operand::Move(ref_loc), span: DUMMY_SP }],
+                args: [Spanned { node: Operand::Move(ref_loc), span: DUMMY_SP }].into(),
                 destination: dest,
                 target: Some(next),
                 unwind: UnwindAction::Cleanup(cleanup),
@@ -843,7 +843,7 @@ fn build_call_shim<'tcx>(
     };
 
     // BB #0
-    let args = args.into_iter().map(|a| Spanned { node: a, span: DUMMY_SP }).collect::<Vec<_>>();
+    let args = args.into_iter().map(|a| Spanned { node: a, span: DUMMY_SP }).collect();
     block(
         &mut blocks,
         statements,

compiler/rustc_type_ir/src/fold.rs

@@ -323,6 +323,12 @@ impl<I: Interner, T: TypeFoldable<I>> TypeFoldable<I> for Vec<T> {
     }
 }
 
+impl<I: Interner, T: TypeFoldable<I>> TypeFoldable<I> for Box<[T]> {
+    fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
+        Vec::from(self).try_fold_with(folder).map(Vec::into_boxed_slice)
+    }
+}
+
 impl<I: Interner, T: TypeFoldable<I>, Ix: Idx> TypeFoldable<I> for IndexVec<Ix, T> {
     fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
         self.raw.try_fold_with(folder).map(IndexVec::from_raw)