Auto merge of #128299 - DianQK:clone-copy, r=cjgillot

Simplify the canonical clone method and the copy-like forms to copy

Fixes #128081.

The optimized clone method ends up as the following MIR:

```
_2 = copy ((*_1).0: i32);
_3 = copy ((*_1).1: u64);
_4 = copy ((*_1).2: [i8; 3]);
_0 = Foo { a: move _2, b: move _3, c: move _4 };
```

We can transform this to:

```
_0 = copy (*_1);
```

r? `@cjgillot`

compiler/rustc_mir_transform/src/gvn.rs

@@ -875,6 +875,95 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         None
     }
 
+    fn try_as_place_elem(
+        &mut self,
+        proj: ProjectionElem<VnIndex, Ty<'tcx>>,
+        loc: Location,
+    ) -> Option<PlaceElem<'tcx>> {
+        Some(match proj {
+            ProjectionElem::Deref => ProjectionElem::Deref,
+            ProjectionElem::Field(idx, ty) => ProjectionElem::Field(idx, ty),
+            ProjectionElem::Index(idx) => {
+                let Some(local) = self.try_as_local(idx, loc) else {
+                    return None;
+                };
+                self.reused_locals.insert(local);
+                ProjectionElem::Index(local)
+            }
+            ProjectionElem::ConstantIndex { offset, min_length, from_end } => {
+                ProjectionElem::ConstantIndex { offset, min_length, from_end }
+            }
+            ProjectionElem::Subslice { from, to, from_end } => {
+                ProjectionElem::Subslice { from, to, from_end }
+            }
+            ProjectionElem::Downcast(symbol, idx) => ProjectionElem::Downcast(symbol, idx),
+            ProjectionElem::OpaqueCast(idx) => ProjectionElem::OpaqueCast(idx),
+            ProjectionElem::Subtype(idx) => ProjectionElem::Subtype(idx),
+        })
+    }
+
+    fn simplify_aggregate_to_copy(
+        &mut self,
+        rvalue: &mut Rvalue<'tcx>,
+        location: Location,
+        fields: &[VnIndex],
+        variant_index: VariantIdx,
+    ) -> Option<VnIndex> {
+        let Some(&first_field) = fields.first() else {
+            return None;
+        };
+        let Value::Projection(copy_from_value, _) = *self.get(first_field) else {
+            return None;
+        };
+        // All fields must correspond one-to-one and come from the same aggregate value.
+        if fields.iter().enumerate().any(|(index, &v)| {
+            if let Value::Projection(pointer, ProjectionElem::Field(from_index, _)) = *self.get(v)
+                && copy_from_value == pointer
+                && from_index.index() == index
+            {
+                return false;
+            }
+            true
+        }) {
+            return None;
+        }
+
+        let mut copy_from_local_value = copy_from_value;
+        if let Value::Projection(pointer, proj) = *self.get(copy_from_value)
+            && let ProjectionElem::Downcast(_, read_variant) = proj
+        {
+            if variant_index == read_variant {
+                // When copying a variant, there is no need to downcast.
+                copy_from_local_value = pointer;
+            } else {
+                // The copied variant must be identical.
+                return None;
+            }
+        }
+
+        let tcx = self.tcx;
+        let mut projection = SmallVec::<[PlaceElem<'tcx>; 1]>::new();
+        loop {
+            if let Some(local) = self.try_as_local(copy_from_local_value, location) {
+                projection.reverse();
+                let place = Place { local, projection: tcx.mk_place_elems(projection.as_slice()) };
+                if rvalue.ty(self.local_decls, tcx) == place.ty(self.local_decls, tcx).ty {
+                    self.reused_locals.insert(local);
+                    *rvalue = Rvalue::Use(Operand::Copy(place));
+                    return Some(copy_from_value);
+                }
+                return None;
+            } else if let Value::Projection(pointer, proj) = *self.get(copy_from_local_value)
+                && let Some(proj) = self.try_as_place_elem(proj, location)
+            {
+                projection.push(proj);
+                copy_from_local_value = pointer;
+            } else {
+                return None;
+            }
+        }
+    }
+
     fn simplify_aggregate(
         &mut self,
         rvalue: &mut Rvalue<'tcx>,
@@ -971,6 +1060,13 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             }
         }
 
+        if let AggregateTy::Def(_, _) = ty
+            && let Some(value) =
+                self.simplify_aggregate_to_copy(rvalue, location, &fields, variant_index)
+        {
+            return Some(value);
+        }
+
         Some(self.insert(Value::Aggregate(ty, variant_index, fields)))
     }
 
@@ -1485,7 +1581,7 @@ impl<'tcx> VnState<'_, 'tcx> {
     }
 
     /// If there is a local which is assigned `index`, and its assignment strictly dominates `loc`,
-    /// return it.
+    /// return it. If you used this local, add it to `reused_locals` to remove storage statements.
     fn try_as_local(&mut self, index: VnIndex, loc: Location) -> Option<Local> {
         let other = self.rev_locals.get(index)?;
         other

tests/codegen/clone_as_copy.rs

@@ -0,0 +1,40 @@
+//@ revisions: DEBUGINFO NODEBUGINFO
+//@ compile-flags: -O -Cno-prepopulate-passes
+//@ [DEBUGINFO] compile-flags: -Cdebuginfo=full
+
+// From https://github.com/rust-lang/rust/issues/128081.
+// Ensure that we only generate a memcpy instruction.
+
+#![crate_type = "lib"]
+
+#[derive(Clone)]
+struct SubCloneAndCopy {
+    v1: u32,
+    v2: u32,
+}
+
+#[derive(Clone)]
+struct CloneOnly {
+    v1: u8,
+    v2: u8,
+    v3: u8,
+    v4: u8,
+    v5: u8,
+    v6: u8,
+    v7: u8,
+    v8: u8,
+    v9: u8,
+    v_sub: SubCloneAndCopy,
+    v_large: [u8; 256],
+}
+
+// CHECK-LABEL: define {{.*}}@clone_only(
+#[no_mangle]
+pub fn clone_only(v: &CloneOnly) -> CloneOnly {
+    // CHECK-NOT: call {{.*}}clone
+    // CHECK-NOT: store i8
+    // CHECK-NOT: store i32
+    // CHECK: call void @llvm.memcpy
+    // CHECK-NEXT: ret void
+    v.clone()
+}

tests/codegen/enum/unreachable_enum_default_branch.rs

@@ -28,11 +28,13 @@ pub fn implicit_match(x: Int) -> bool {
 // The code is from https://github.com/rust-lang/rust/issues/110097.
 // We expect it to generate the same optimized code as a full match.
 // CHECK-LABEL: @if_let(
-// CHECK-NEXT:  start:
+// CHECK: start:
+// CHECK-NOT: zext
+// CHECK: select
 // CHECK-NEXT: insertvalue
 // CHECK-NEXT: insertvalue
 // CHECK-NEXT: ret
 #[no_mangle]
 pub fn if_let(val: Result<i32, ()>) -> Result<i32, ()> {
-    if let Ok(x) = val { Ok(x) } else { Err(()) }
+    if let Ok(x) = val { Ok(x * 2) } else { Err(()) }
 }

tests/codegen/try_question_mark_nop.rs

@@ -1,5 +1,7 @@
 //@ compile-flags: -O -Z merge-functions=disabled --edition=2021
 //@ only-x86_64
+// FIXME: Remove the `min-llvm-version`.
+//@ min-llvm-version: 19
 
 #![crate_type = "lib"]
 #![feature(try_blocks)]
@@ -7,11 +9,14 @@
 use std::ops::ControlFlow::{self, Break, Continue};
 use std::ptr::NonNull;
 
+// FIXME: The `trunc` and `select` instructions can be eliminated.
 // CHECK-LABEL: @option_nop_match_32
 #[no_mangle]
 pub fn option_nop_match_32(x: Option<u32>) -> Option<u32> {
     // CHECK: start:
-    // CHECK-NEXT: insertvalue { i32, i32 }
+    // CHECK-NEXT: [[TRUNC:%.*]] = trunc nuw i32 %0 to i1
+    // CHECK-NEXT: [[FIRST:%.*]] = select i1 [[TRUNC]], i32 %0
+    // CHECK-NEXT: insertvalue { i32, i32 } poison, i32 [[FIRST]]
     // CHECK-NEXT: insertvalue { i32, i32 }
     // CHECK-NEXT: ret { i32, i32 }
     match x {

tests/mir-opt/gvn_clone.rs

@@ -0,0 +1,17 @@
+//@ test-mir-pass: GVN
+//@ compile-flags: -Zmir-enable-passes=+InstSimplify-before-inline
+
+// Check if we have transformed the default clone to copy in the specific pipeline.
+
+// EMIT_MIR gvn_clone.{impl#0}-clone.GVN.diff
+
+// CHECK-LABEL: ::clone(
+// CHECK-NOT: = AllCopy { {{.*}} };
+// CHECK: _0 = copy (*_1);
+// CHECK: return;
+#[derive(Clone)]
+struct AllCopy {
+    a: i32,
+    b: u64,
+    c: [i8; 3],
+}

tests/mir-opt/gvn_clone.{impl#0}-clone.GVN.diff

@@ -0,0 +1,71 @@
+- // MIR for `<impl at $DIR/gvn_clone.rs:12:10: 12:15>::clone` before GVN
++ // MIR for `<impl at $DIR/gvn_clone.rs:12:10: 12:15>::clone` after GVN
+  
+  fn <impl at $DIR/gvn_clone.rs:12:10: 12:15>::clone(_1: &AllCopy) -> AllCopy {
+      debug self => _1;
+      let mut _0: AllCopy;
+      let mut _2: i32;
+      let mut _3: &i32;
+      let _4: &i32;
+      let mut _5: u64;
+      let mut _6: &u64;
+      let _7: &u64;
+      let mut _8: [i8; 3];
+      let mut _9: &[i8; 3];
+      let _10: &[i8; 3];
+  
+      bb0: {
+          StorageLive(_2);
+          StorageLive(_3);
+-         StorageLive(_4);
++         nop;
+          _4 = &((*_1).0: i32);
+          _3 = copy _4;
+-         _2 = copy (*_3);
++         _2 = copy ((*_1).0: i32);
+          goto -> bb1;
+      }
+  
+      bb1: {
+          StorageDead(_3);
+          StorageLive(_5);
+          StorageLive(_6);
+-         StorageLive(_7);
++         nop;
+          _7 = &((*_1).1: u64);
+          _6 = copy _7;
+-         _5 = copy (*_6);
++         _5 = copy ((*_1).1: u64);
+          goto -> bb2;
+      }
+  
+      bb2: {
+          StorageDead(_6);
+          StorageLive(_8);
+          StorageLive(_9);
+-         StorageLive(_10);
++         nop;
+          _10 = &((*_1).2: [i8; 3]);
+          _9 = copy _10;
+-         _8 = copy (*_9);
++         _8 = copy ((*_1).2: [i8; 3]);
+          goto -> bb3;
+      }
+  
+      bb3: {
+          StorageDead(_9);
+-         _0 = AllCopy { a: move _2, b: move _5, c: move _8 };
++         _0 = copy (*_1);
+          StorageDead(_8);
+          StorageDead(_5);
+          StorageDead(_2);
+-         StorageDead(_10);
+-         StorageDead(_7);
+-         StorageDead(_4);
++         nop;
++         nop;
++         nop;
+          return;
+      }
+  }
+  

tests/mir-opt/gvn_copy_aggregate.all_copy.GVN.diff

@@ -0,0 +1,53 @@
+- // MIR for `all_copy` before GVN
++ // MIR for `all_copy` after GVN
+  
+  fn all_copy(_1: &AllCopy) -> AllCopy {
+      debug v => _1;
+      let mut _0: AllCopy;
+      let _2: i32;
+      let mut _5: i32;
+      let mut _6: u64;
+      let mut _7: [i8; 3];
+      scope 1 {
+          debug a => _2;
+          let _3: u64;
+          scope 2 {
+              debug b => _3;
+              let _4: [i8; 3];
+              scope 3 {
+                  debug c => _4;
+              }
+          }
+      }
+  
+      bb0: {
+-         StorageLive(_2);
++         nop;
+          _2 = copy ((*_1).0: i32);
+-         StorageLive(_3);
++         nop;
+          _3 = copy ((*_1).1: u64);
+-         StorageLive(_4);
++         nop;
+          _4 = copy ((*_1).2: [i8; 3]);
+          StorageLive(_5);
+          _5 = copy _2;
+          StorageLive(_6);
+          _6 = copy _3;
+          StorageLive(_7);
+          _7 = copy _4;
+-         _0 = AllCopy { a: move _5, b: move _6, c: move _7 };
++         _0 = copy (*_1);
+          StorageDead(_7);
+          StorageDead(_6);
+          StorageDead(_5);
+-         StorageDead(_4);
+-         StorageDead(_3);
+-         StorageDead(_2);
++         nop;
++         nop;
++         nop;
+          return;
+      }
+  }
+  

tests/mir-opt/gvn_copy_aggregate.all_copy_2.GVN.diff

@@ -0,0 +1,64 @@
+- // MIR for `all_copy_2` before GVN
++ // MIR for `all_copy_2` after GVN
+  
+  fn all_copy_2(_1: &&AllCopy) -> AllCopy {
+      debug v => _1;
+      let mut _0: AllCopy;
+      let _2: i32;
+      let mut _5: i32;
+      let mut _6: u64;
+      let mut _7: [i8; 3];
+      let mut _8: &AllCopy;
+      let mut _9: &AllCopy;
+      let mut _10: &AllCopy;
+      scope 1 {
+          debug a => _2;
+          let _3: u64;
+          scope 2 {
+              debug b => _3;
+              let _4: [i8; 3];
+              scope 3 {
+                  debug c => _4;
+              }
+          }
+      }
+  
+      bb0: {
+-         StorageLive(_2);
+-         _8 = deref_copy (*_1);
++         nop;
++         _8 = copy (*_1);
+          _2 = copy ((*_8).0: i32);
+-         StorageLive(_3);
+-         _9 = deref_copy (*_1);
+-         _3 = copy ((*_9).1: u64);
+-         StorageLive(_4);
+-         _10 = deref_copy (*_1);
+-         _4 = copy ((*_10).2: [i8; 3]);
++         nop;
++         _9 = copy _8;
++         _3 = copy ((*_8).1: u64);
++         nop;
++         _10 = copy _8;
++         _4 = copy ((*_8).2: [i8; 3]);
+          StorageLive(_5);
+          _5 = copy _2;
+          StorageLive(_6);
+          _6 = copy _3;
+          StorageLive(_7);
+          _7 = copy _4;
+-         _0 = AllCopy { a: move _5, b: move _6, c: move _7 };
++         _0 = copy (*_8);
+          StorageDead(_7);
+          StorageDead(_6);
+          StorageDead(_5);
+-         StorageDead(_4);
+-         StorageDead(_3);
+-         StorageDead(_2);
++         nop;
++         nop;
++         nop;
+          return;
+      }
+  }
+  

tests/mir-opt/gvn_copy_aggregate.all_copy_different_type.GVN.diff

@@ -0,0 +1,53 @@
+- // MIR for `all_copy_different_type` before GVN
++ // MIR for `all_copy_different_type` after GVN
+  
+  fn all_copy_different_type(_1: &AllCopy) -> AllCopy2 {
+      debug v => _1;
+      let mut _0: AllCopy2;
+      let _2: i32;
+      let mut _5: i32;
+      let mut _6: u64;
+      let mut _7: [i8; 3];
+      scope 1 {
+          debug a => _2;
+          let _3: u64;
+          scope 2 {
+              debug b => _3;
+              let _4: [i8; 3];
+              scope 3 {
+                  debug c => _4;
+              }
+          }
+      }
+  
+      bb0: {
+-         StorageLive(_2);
++         nop;
+          _2 = copy ((*_1).0: i32);
+-         StorageLive(_3);
++         nop;
+          _3 = copy ((*_1).1: u64);
+-         StorageLive(_4);
++         nop;
+          _4 = copy ((*_1).2: [i8; 3]);
+          StorageLive(_5);
+          _5 = copy _2;
+          StorageLive(_6);
+          _6 = copy _3;
+          StorageLive(_7);
+          _7 = copy _4;
+-         _0 = AllCopy2 { a: move _5, b: move _6, c: move _7 };
++         _0 = AllCopy2 { a: copy _2, b: copy _3, c: copy _4 };
+          StorageDead(_7);
+          StorageDead(_6);
+          StorageDead(_5);
+-         StorageDead(_4);
+-         StorageDead(_3);
+-         StorageDead(_2);
++         nop;
++         nop;
++         nop;
+          return;
+      }
+  }
+  

tests/mir-opt/gvn_copy_aggregate.all_copy_has_changed.GVN.diff

@@ -0,0 +1,54 @@
+- // MIR for `all_copy_has_changed` before GVN
++ // MIR for `all_copy_has_changed` after GVN
+  
+  fn all_copy_has_changed(_1: &mut AllCopy) -> AllCopy {
+      debug v => _1;
+      let mut _0: AllCopy;
+      let _2: i32;
+      let mut _5: i32;
+      let mut _6: u64;
+      let mut _7: [i8; 3];
+      scope 1 {
+          debug a => _2;
+          let _3: u64;
+          scope 2 {
+              debug b => _3;
+              let _4: [i8; 3];
+              scope 3 {
+                  debug c => _4;
+              }
+          }
+      }
+  
+      bb0: {
+-         StorageLive(_2);
++         nop;
+          _2 = copy ((*_1).0: i32);
+-         StorageLive(_3);
++         nop;
+          _3 = copy ((*_1).1: u64);
+-         StorageLive(_4);
++         nop;
+          _4 = copy ((*_1).2: [i8; 3]);
+          ((*_1).0: i32) = const 1_i32;
+          StorageLive(_5);
+          _5 = copy _2;
+          StorageLive(_6);
+          _6 = copy _3;
+          StorageLive(_7);
+          _7 = copy _4;
+-         _0 = AllCopy { a: move _5, b: move _6, c: move _7 };
++         _0 = AllCopy { a: copy _2, b: copy _3, c: copy _4 };
+          StorageDead(_7);
+          StorageDead(_6);
+          StorageDead(_5);
+-         StorageDead(_4);
+-         StorageDead(_3);
+-         StorageDead(_2);
++         nop;
++         nop;
++         nop;
+          return;
+      }
+  }
+  

tests/mir-opt/gvn_copy_aggregate.all_copy_move.GVN.diff

@@ -0,0 +1,53 @@
+- // MIR for `all_copy_move` before GVN
++ // MIR for `all_copy_move` after GVN
+  
+  fn all_copy_move(_1: AllCopy) -> AllCopy {
+      debug v => _1;
+      let mut _0: AllCopy;
+      let _2: i32;
+      let mut _5: i32;
+      let mut _6: u64;
+      let mut _7: [i8; 3];
+      scope 1 {
+          debug a => _2;
+          let _3: u64;
+          scope 2 {
+              debug b => _3;
+              let _4: [i8; 3];
+              scope 3 {
+                  debug c => _4;
+              }
+          }
+      }
+  
+      bb0: {
+-         StorageLive(_2);
++         nop;
+          _2 = copy (_1.0: i32);
+-         StorageLive(_3);
++         nop;
+          _3 = copy (_1.1: u64);
+-         StorageLive(_4);
++         nop;
+          _4 = copy (_1.2: [i8; 3]);
+          StorageLive(_5);
+          _5 = copy _2;
+          StorageLive(_6);
+          _6 = copy _3;
+          StorageLive(_7);
+          _7 = copy _4;
+-         _0 = AllCopy { a: move _5, b: move _6, c: move _7 };
++         _0 = copy _1;
+          StorageDead(_7);
+          StorageDead(_6);
+          StorageDead(_5);
+-         StorageDead(_4);
+-         StorageDead(_3);
+-         StorageDead(_2);
++         nop;
++         nop;
++         nop;
+          return;
+      }
+  }
+  

tests/mir-opt/gvn_copy_aggregate.all_copy_ret_2.GVN.diff

@@ -0,0 +1,77 @@
+- // MIR for `all_copy_ret_2` before GVN
++ // MIR for `all_copy_ret_2` after GVN
+  
+  fn all_copy_ret_2(_1: &AllCopy) -> (AllCopy, AllCopy) {
+      debug v => _1;
+      let mut _0: (AllCopy, AllCopy);
+      let _2: i32;
+      let mut _5: AllCopy;
+      let mut _6: i32;
+      let mut _7: u64;
+      let mut _8: [i8; 3];
+      let mut _9: AllCopy;
+      let mut _10: i32;
+      let mut _11: u64;
+      let mut _12: [i8; 3];
+      scope 1 {
+          debug a => _2;
+          let _3: u64;
+          scope 2 {
+              debug b => _3;
+              let _4: [i8; 3];
+              scope 3 {
+                  debug c => _4;
+              }
+          }
+      }
+  
+      bb0: {
+-         StorageLive(_2);
++         nop;
+          _2 = copy ((*_1).0: i32);
+-         StorageLive(_3);
++         nop;
+          _3 = copy ((*_1).1: u64);
+-         StorageLive(_4);
++         nop;
+          _4 = copy ((*_1).2: [i8; 3]);
+-         StorageLive(_5);
++         nop;
+          StorageLive(_6);
+          _6 = copy _2;
+          StorageLive(_7);
+          _7 = copy _3;
+          StorageLive(_8);
+          _8 = copy _4;
+-         _5 = AllCopy { a: move _6, b: move _7, c: move _8 };
++         _5 = copy (*_1);
+          StorageDead(_8);
+          StorageDead(_7);
+          StorageDead(_6);
+          StorageLive(_9);
+          StorageLive(_10);
+          _10 = copy _2;
+          StorageLive(_11);
+          _11 = copy _3;
+          StorageLive(_12);
+          _12 = copy _4;
+-         _9 = AllCopy { a: move _10, b: move _11, c: move _12 };
++         _9 = copy _5;
+          StorageDead(_12);
+          StorageDead(_11);
+          StorageDead(_10);
+-         _0 = (move _5, move _9);
++         _0 = (copy _5, copy _5);
+          StorageDead(_9);
+-         StorageDead(_5);
+-         StorageDead(_4);
+-         StorageDead(_3);
+-         StorageDead(_2);
++         nop;
++         nop;
++         nop;
++         nop;
+          return;
+      }
+  }
+  

tests/mir-opt/gvn_copy_aggregate.all_copy_use_changed.GVN.diff

@@ -0,0 +1,57 @@
+- // MIR for `all_copy_use_changed` before GVN
++ // MIR for `all_copy_use_changed` after GVN
+  
+  fn all_copy_use_changed(_1: &mut AllCopy) -> AllCopy {
+      debug v => _1;
+      let mut _0: AllCopy;
+      let mut _2: i32;
+      let mut _3: i32;
+      let mut _6: i32;
+      let mut _7: u64;
+      let mut _8: [i8; 3];
+      scope 1 {
+          debug a => _2;
+          let _4: u64;
+          scope 2 {
+              debug b => _4;
+              let _5: [i8; 3];
+              scope 3 {
+                  debug c => _5;
+              }
+          }
+      }
+  
+      bb0: {
+          StorageLive(_2);
+          _2 = copy ((*_1).0: i32);
+          ((*_1).0: i32) = const 1_i32;
+          StorageLive(_3);
+          _3 = copy ((*_1).0: i32);
+          _2 = move _3;
+          StorageDead(_3);
+-         StorageLive(_4);
++         nop;
+          _4 = copy ((*_1).1: u64);
+-         StorageLive(_5);
++         nop;
+          _5 = copy ((*_1).2: [i8; 3]);
+          StorageLive(_6);
+          _6 = copy _2;
+          StorageLive(_7);
+          _7 = copy _4;
+          StorageLive(_8);
+          _8 = copy _5;
+-         _0 = AllCopy { a: move _6, b: move _7, c: move _8 };
++         _0 = AllCopy { a: move _6, b: copy _4, c: copy _5 };
+          StorageDead(_8);
+          StorageDead(_7);
+          StorageDead(_6);
+-         StorageDead(_5);
+-         StorageDead(_4);
++         nop;
++         nop;
+          StorageDead(_2);
+          return;
+      }
+  }
+  

tests/mir-opt/gvn_copy_aggregate.all_copy_use_changed_2.GVN.diff

@@ -0,0 +1,57 @@
+- // MIR for `all_copy_use_changed_2` before GVN
++ // MIR for `all_copy_use_changed_2` after GVN
+  
+  fn all_copy_use_changed_2(_1: &mut AllCopy) -> AllCopy {
+      debug v => _1;
+      let mut _0: AllCopy;
+      let mut _2: i32;
+      let mut _5: i32;
+      let mut _6: i32;
+      let mut _7: u64;
+      let mut _8: [i8; 3];
+      scope 1 {
+          debug a => _2;
+          let _3: u64;
+          scope 2 {
+              debug b => _3;
+              let _4: [i8; 3];
+              scope 3 {
+                  debug c => _4;
+              }
+          }
+      }
+  
+      bb0: {
+          StorageLive(_2);
+          _2 = copy ((*_1).0: i32);
+-         StorageLive(_3);
++         nop;
+          _3 = copy ((*_1).1: u64);
+-         StorageLive(_4);
++         nop;
+          _4 = copy ((*_1).2: [i8; 3]);
+          ((*_1).0: i32) = const 1_i32;
+          StorageLive(_5);
+          _5 = copy ((*_1).0: i32);
+          _2 = move _5;
+          StorageDead(_5);
+          StorageLive(_6);
+          _6 = copy _2;
+          StorageLive(_7);
+          _7 = copy _3;
+          StorageLive(_8);
+          _8 = copy _4;
+-         _0 = AllCopy { a: move _6, b: move _7, c: move _8 };
++         _0 = AllCopy { a: move _6, b: copy _3, c: copy _4 };
+          StorageDead(_8);
+          StorageDead(_7);
+          StorageDead(_6);
+-         StorageDead(_4);
+-         StorageDead(_3);
++         nop;
++         nop;
+          StorageDead(_2);
+          return;
+      }
+  }
+  

tests/mir-opt/gvn_copy_aggregate.enum_different_variant.GVN.diff

@@ -0,0 +1,162 @@
+- // MIR for `enum_different_variant` before GVN
++ // MIR for `enum_different_variant` after GVN
+  
+  fn enum_different_variant(_1: &Enum1) -> Enum1 {
+      debug v => _1;
+      let mut _0: Enum1;
+      let mut _2: isize;
+      let _3: &AllCopy;
+      let mut _8: i32;
+      let mut _9: u64;
+      let mut _10: [i8; 3];
+      let mut _11: AllCopy;
+      let _12: &AllCopy;
+      let mut _17: i32;
+      let mut _18: u64;
+      let mut _19: [i8; 3];
+      let mut _20: AllCopy;
+      scope 1 {
+          debug v => _3;
+          let _4: i32;
+          scope 2 {
+              debug a => _4;
+              let _5: u64;
+              scope 3 {
+                  debug b => _5;
+                  let _6: [i8; 3];
+                  scope 4 {
+                      debug c => _6;
+                      let _7: AllCopy;
+                      scope 5 {
+                          debug all_copy => _7;
+                      }
+                  }
+              }
+          }
+      }
+      scope 6 {
+          debug v => _12;
+          let _13: i32;
+          scope 7 {
+              debug a => _13;
+              let _14: u64;
+              scope 8 {
+                  debug b => _14;
+                  let _15: [i8; 3];
+                  scope 9 {
+                      debug c => _15;
+                      let _16: AllCopy;
+                      scope 10 {
+                          debug all_copy => _16;
+                      }
+                  }
+              }
+          }
+      }
+  
+      bb0: {
+          _2 = discriminant((*_1));
+          switchInt(move _2) -> [0: bb3, 1: bb2, otherwise: bb1];
+      }
+  
+      bb1: {
+          unreachable;
+      }
+  
+      bb2: {
+          StorageLive(_12);
+          _12 = &(((*_1) as B).0: AllCopy);
+-         StorageLive(_13);
+-         _13 = copy ((*_12).0: i32);
+-         StorageLive(_14);
+-         _14 = copy ((*_12).1: u64);
+-         StorageLive(_15);
+-         _15 = copy ((*_12).2: [i8; 3]);
+-         StorageLive(_16);
++         nop;
++         _13 = copy ((((*_1) as B).0: AllCopy).0: i32);
++         nop;
++         _14 = copy ((((*_1) as B).0: AllCopy).1: u64);
++         nop;
++         _15 = copy ((((*_1) as B).0: AllCopy).2: [i8; 3]);
++         nop;
+          StorageLive(_17);
+          _17 = copy _13;
+          StorageLive(_18);
+          _18 = copy _14;
+          StorageLive(_19);
+          _19 = copy _15;
+-         _16 = AllCopy { a: move _17, b: move _18, c: move _19 };
++         _16 = copy (((*_1) as B).0: AllCopy);
+          StorageDead(_19);
+          StorageDead(_18);
+          StorageDead(_17);
+          StorageLive(_20);
+-         _20 = move _16;
+-         _0 = Enum1::A(move _20);
++         _20 = copy _16;
++         _0 = Enum1::A(copy _16);
+          StorageDead(_20);
+-         StorageDead(_16);
+-         StorageDead(_15);
+-         StorageDead(_14);
+-         StorageDead(_13);
++         nop;
++         nop;
++         nop;
++         nop;
+          StorageDead(_12);
+          goto -> bb4;
+      }
+  
+      bb3: {
+          StorageLive(_3);
+          _3 = &(((*_1) as A).0: AllCopy);
+-         StorageLive(_4);
+-         _4 = copy ((*_3).0: i32);
+-         StorageLive(_5);
+-         _5 = copy ((*_3).1: u64);
+-         StorageLive(_6);
+-         _6 = copy ((*_3).2: [i8; 3]);
+-         StorageLive(_7);
++         nop;
++         _4 = copy ((((*_1) as A).0: AllCopy).0: i32);
++         nop;
++         _5 = copy ((((*_1) as A).0: AllCopy).1: u64);
++         nop;
++         _6 = copy ((((*_1) as A).0: AllCopy).2: [i8; 3]);
++         nop;
+          StorageLive(_8);
+          _8 = copy _4;
+          StorageLive(_9);
+          _9 = copy _5;
+          StorageLive(_10);
+          _10 = copy _6;
+-         _7 = AllCopy { a: move _8, b: move _9, c: move _10 };
++         _7 = copy (((*_1) as A).0: AllCopy);
+          StorageDead(_10);
+          StorageDead(_9);
+          StorageDead(_8);
+          StorageLive(_11);
+-         _11 = move _7;
+-         _0 = Enum1::B(move _11);
++         _11 = copy _7;
++         _0 = Enum1::B(copy _7);
+          StorageDead(_11);
+-         StorageDead(_7);
+-         StorageDead(_6);
+-         StorageDead(_5);
+-         StorageDead(_4);
++         nop;
++         nop;
++         nop;
++         nop;
+          StorageDead(_3);
+          goto -> bb4;
+      }
+  
+      bb4: {
+          return;
+      }
+  }
+  

tests/mir-opt/gvn_copy_aggregate.enum_identical_variant.GVN.diff

@@ -0,0 +1,162 @@
+- // MIR for `enum_identical_variant` before GVN
++ // MIR for `enum_identical_variant` after GVN
+  
+  fn enum_identical_variant(_1: &Enum1) -> Enum1 {
+      debug v => _1;
+      let mut _0: Enum1;
+      let mut _2: isize;
+      let _3: &AllCopy;
+      let mut _8: i32;
+      let mut _9: u64;
+      let mut _10: [i8; 3];
+      let mut _11: AllCopy;
+      let _12: &AllCopy;
+      let mut _17: i32;
+      let mut _18: u64;
+      let mut _19: [i8; 3];
+      let mut _20: AllCopy;
+      scope 1 {
+          debug v => _3;
+          let _4: i32;
+          scope 2 {
+              debug a => _4;
+              let _5: u64;
+              scope 3 {
+                  debug b => _5;
+                  let _6: [i8; 3];
+                  scope 4 {
+                      debug c => _6;
+                      let _7: AllCopy;
+                      scope 5 {
+                          debug all_copy => _7;
+                      }
+                  }
+              }
+          }
+      }
+      scope 6 {
+          debug v => _12;
+          let _13: i32;
+          scope 7 {
+              debug a => _13;
+              let _14: u64;
+              scope 8 {
+                  debug b => _14;
+                  let _15: [i8; 3];
+                  scope 9 {
+                      debug c => _15;
+                      let _16: AllCopy;
+                      scope 10 {
+                          debug all_copy => _16;
+                      }
+                  }
+              }
+          }
+      }
+  
+      bb0: {
+          _2 = discriminant((*_1));
+          switchInt(move _2) -> [0: bb3, 1: bb2, otherwise: bb1];
+      }
+  
+      bb1: {
+          unreachable;
+      }
+  
+      bb2: {
+          StorageLive(_12);
+          _12 = &(((*_1) as B).0: AllCopy);
+-         StorageLive(_13);
+-         _13 = copy ((*_12).0: i32);
+-         StorageLive(_14);
+-         _14 = copy ((*_12).1: u64);
+-         StorageLive(_15);
+-         _15 = copy ((*_12).2: [i8; 3]);
+-         StorageLive(_16);
++         nop;
++         _13 = copy ((((*_1) as B).0: AllCopy).0: i32);
++         nop;
++         _14 = copy ((((*_1) as B).0: AllCopy).1: u64);
++         nop;
++         _15 = copy ((((*_1) as B).0: AllCopy).2: [i8; 3]);
++         nop;
+          StorageLive(_17);
+          _17 = copy _13;
+          StorageLive(_18);
+          _18 = copy _14;
+          StorageLive(_19);
+          _19 = copy _15;
+-         _16 = AllCopy { a: move _17, b: move _18, c: move _19 };
++         _16 = copy (((*_1) as B).0: AllCopy);
+          StorageDead(_19);
+          StorageDead(_18);
+          StorageDead(_17);
+          StorageLive(_20);
+-         _20 = move _16;
+-         _0 = Enum1::B(move _20);
++         _20 = copy _16;
++         _0 = copy (*_1);
+          StorageDead(_20);
+-         StorageDead(_16);
+-         StorageDead(_15);
+-         StorageDead(_14);
+-         StorageDead(_13);
++         nop;
++         nop;
++         nop;
++         nop;
+          StorageDead(_12);
+          goto -> bb4;
+      }
+  
+      bb3: {
+          StorageLive(_3);
+          _3 = &(((*_1) as A).0: AllCopy);
+-         StorageLive(_4);
+-         _4 = copy ((*_3).0: i32);
+-         StorageLive(_5);
+-         _5 = copy ((*_3).1: u64);
+-         StorageLive(_6);
+-         _6 = copy ((*_3).2: [i8; 3]);
+-         StorageLive(_7);
++         nop;
++         _4 = copy ((((*_1) as A).0: AllCopy).0: i32);
++         nop;
++         _5 = copy ((((*_1) as A).0: AllCopy).1: u64);
++         nop;
++         _6 = copy ((((*_1) as A).0: AllCopy).2: [i8; 3]);
++         nop;
+          StorageLive(_8);
+          _8 = copy _4;
+          StorageLive(_9);
+          _9 = copy _5;
+          StorageLive(_10);
+          _10 = copy _6;
+-         _7 = AllCopy { a: move _8, b: move _9, c: move _10 };
++         _7 = copy (((*_1) as A).0: AllCopy);
+          StorageDead(_10);
+          StorageDead(_9);
+          StorageDead(_8);
+          StorageLive(_11);
+-         _11 = move _7;
+-         _0 = Enum1::A(move _11);
++         _11 = copy _7;
++         _0 = copy (*_1);
+          StorageDead(_11);
+-         StorageDead(_7);
+-         StorageDead(_6);
+-         StorageDead(_5);
+-         StorageDead(_4);
++         nop;
++         nop;
++         nop;
++         nop;
+          StorageDead(_3);
+          goto -> bb4;
+      }
+  
+      bb4: {
+          return;
+      }
+  }
+  

tests/mir-opt/gvn_copy_aggregate.nest_copy.GVN.diff

@@ -0,0 +1,81 @@
+- // MIR for `nest_copy` before GVN
++ // MIR for `nest_copy` after GVN
+  
+  fn nest_copy(_1: &NestCopy) -> NestCopy {
+      debug v => _1;
+      let mut _0: NestCopy;
+      let _2: i32;
+      let mut _6: i32;
+      let mut _7: u64;
+      let mut _8: [i8; 3];
+      let mut _10: i32;
+      let mut _11: AllCopy;
+      scope 1 {
+          debug a => _2;
+          let _3: u64;
+          scope 2 {
+              debug b => _3;
+              let _4: [i8; 3];
+              scope 3 {
+                  debug c => _4;
+                  let _5: AllCopy;
+                  scope 4 {
+                      debug all_copy => _5;
+                      let _9: i32;
+                      scope 5 {
+                          debug d => _9;
+                      }
+                  }
+              }
+          }
+      }
+  
+      bb0: {
+-         StorageLive(_2);
++         nop;
+          _2 = copy (((*_1).1: AllCopy).0: i32);
+-         StorageLive(_3);
++         nop;
+          _3 = copy (((*_1).1: AllCopy).1: u64);
+-         StorageLive(_4);
++         nop;
+          _4 = copy (((*_1).1: AllCopy).2: [i8; 3]);
+-         StorageLive(_5);
++         nop;
+          StorageLive(_6);
+          _6 = copy _2;
+          StorageLive(_7);
+          _7 = copy _3;
+          StorageLive(_8);
+          _8 = copy _4;
+-         _5 = AllCopy { a: move _6, b: move _7, c: move _8 };
++         _5 = copy ((*_1).1: AllCopy);
+          StorageDead(_8);
+          StorageDead(_7);
+          StorageDead(_6);
+-         StorageLive(_9);
++         nop;
+          _9 = copy ((*_1).0: i32);
+          StorageLive(_10);
+          _10 = copy _9;
+          StorageLive(_11);
+-         _11 = move _5;
+-         _0 = NestCopy { d: move _10, all_copy: move _11 };
++         _11 = copy _5;
++         _0 = copy (*_1);
+          StorageDead(_11);
+          StorageDead(_10);
+-         StorageDead(_9);
+-         StorageDead(_5);
+-         StorageDead(_4);
+-         StorageDead(_3);
+-         StorageDead(_2);
++         nop;
++         nop;
++         nop;
++         nop;
++         nop;
+          return;
+      }
+  }
+  

tests/mir-opt/gvn_copy_aggregate.remove_storage_dead.GVN.diff

@@ -0,0 +1,54 @@
+- // MIR for `remove_storage_dead` before GVN
++ // MIR for `remove_storage_dead` after GVN
+  
+  fn remove_storage_dead(_1: fn() -> AlwaysSome<T>) -> AlwaysSome<T> {
+      debug f => _1;
+      let mut _0: AlwaysSome<T>;
+      let _2: T;
+      let mut _3: AlwaysSome<T>;
+      let mut _4: fn() -> AlwaysSome<T>;
+      let _5: T;
+      let mut _6: T;
+      let mut _7: isize;
+      let mut _8: isize;
+      scope 1 {
+          debug v => _2;
+      }
+      scope 2 {
+          debug v => _5;
+      }
+  
+      bb0: {
+          StorageLive(_2);
+-         StorageLive(_3);
++         nop;
+          StorageLive(_4);
+          _4 = copy _1;
+-         _3 = move _4() -> [return: bb1, unwind unreachable];
++         _3 = copy _1() -> [return: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+          StorageDead(_4);
+-         StorageLive(_5);
+-         _5 = move ((_3 as Some).0: T);
+-         _2 = move _5;
+-         StorageDead(_5);
++         nop;
++         _5 = copy ((_3 as Some).0: T);
++         _2 = copy _5;
++         nop;
+          _7 = discriminant(_3);
+-         StorageDead(_3);
++         nop;
+          StorageLive(_6);
+-         _6 = move _2;
+-         _0 = AlwaysSome::<T>::Some(move _6);
++         _6 = copy _5;
++         _0 = copy _3;
+          StorageDead(_6);
+          StorageDead(_2);
+          return;
+      }
+  }
+  

tests/mir-opt/gvn_copy_aggregate.remove_storage_dead_from_index.GVN.diff

@@ -0,0 +1,26 @@
+- // MIR for `remove_storage_dead_from_index` before GVN
++ // MIR for `remove_storage_dead_from_index` after GVN
+  
+  fn remove_storage_dead_from_index(_1: fn() -> usize, _2: [SameType; 5]) -> SameType {
+      let mut _0: SameType;
+      let mut _3: usize;
+      let mut _4: i32;
+      let mut _5: i32;
+  
+      bb0: {
+-         StorageLive(_3);
++         nop;
+          _3 = copy _1() -> [return: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+          _4 = copy (_2[_3].0: i32);
+          _5 = copy (_2[_3].1: i32);
+-         StorageDead(_3);
+-         _0 = SameType { a: copy _4, b: copy _5 };
++         nop;
++         _0 = copy _2[_3];
+          return;
+      }
+  }
+  

tests/mir-opt/gvn_copy_aggregate.rs

@@ -0,0 +1,261 @@
+//@ test-mir-pass: GVN
+//@ compile-flags: -Cpanic=abort
+
+#![feature(core_intrinsics, custom_mir)]
+#![allow(internal_features)]
+
+use std::intrinsics::mir::*;
+
+struct AllCopy {
+    a: i32,
+    b: u64,
+    c: [i8; 3],
+}
+
+// EMIT_MIR gvn_copy_aggregate.all_copy.GVN.diff
+fn all_copy(v: &AllCopy) -> AllCopy {
+    // CHECK-LABEL: fn all_copy(
+    // CHECK: bb0: {
+    // CHECK-NOT: = AllCopy { {{.*}} };
+    // CHECK: _0 = copy (*_1);
+    let a = v.a;
+    let b = v.b;
+    let c = v.c;
+    AllCopy { a, b, c }
+}
+
+// EMIT_MIR gvn_copy_aggregate.all_copy_2.GVN.diff
+fn all_copy_2(v: &&AllCopy) -> AllCopy {
+    // CHECK-LABEL: fn all_copy_2(
+    // CHECK: bb0: {
+    // CHECK-NOT: = AllCopy { {{.*}} };
+    // CHECK: [[V1:_.*]] = copy (*_1);
+    // CHECK: _0 = copy (*[[V1]]);
+    let a = v.a;
+    let b = v.b;
+    let c = v.c;
+    AllCopy { a, b, c }
+}
+
+// EMIT_MIR gvn_copy_aggregate.all_copy_move.GVN.diff
+fn all_copy_move(v: AllCopy) -> AllCopy {
+    // CHECK-LABEL: fn all_copy_move(
+    // CHECK: bb0: {
+    // CHECK-NOT: = AllCopy { {{.*}} };
+    // CHECK: _0 = copy _1;
+    let a = v.a;
+    let b = v.b;
+    let c = v.c;
+    AllCopy { a, b, c }
+}
+
+// EMIT_MIR gvn_copy_aggregate.all_copy_ret_2.GVN.diff
+fn all_copy_ret_2(v: &AllCopy) -> (AllCopy, AllCopy) {
+    // CHECK-LABEL: fn all_copy_ret_2(
+    // CHECK: bb0: {
+    // CHECK-NOT: = AllCopy { {{.*}} };
+    // CHECK: [[V1:_.*]] = copy (*_1);
+    // CHECK: [[V2:_.*]] = copy [[V1]];
+    // CHECK: _0 = (copy [[V1]], copy [[V1]]);
+    let a = v.a;
+    let b = v.b;
+    let c = v.c;
+    (AllCopy { a, b, c }, AllCopy { a, b, c })
+}
+
+struct AllCopy2 {
+    a: i32,
+    b: u64,
+    c: [i8; 3],
+}
+
+// EMIT_MIR gvn_copy_aggregate.all_copy_different_type.GVN.diff
+fn all_copy_different_type(v: &AllCopy) -> AllCopy2 {
+    // CHECK-LABEL: fn all_copy_different_type(
+    // CHECK: bb0: {
+    // CHECK: _0 = AllCopy2 { {{.*}} };
+    let a = v.a;
+    let b = v.b;
+    let c = v.c;
+    AllCopy2 { a, b, c }
+}
+
+struct SameType {
+    a: i32,
+    b: i32,
+}
+
+// EMIT_MIR gvn_copy_aggregate.same_type_different_index.GVN.diff
+fn same_type_different_index(v: &SameType) -> SameType {
+    // CHECK-LABEL: fn same_type_different_index(
+    // CHECK: bb0: {
+    // CHECK: _0 = SameType { {{.*}} };
+    let a = v.b;
+    let b = v.a;
+    SameType { a, b }
+}
+
+// EMIT_MIR gvn_copy_aggregate.all_copy_has_changed.GVN.diff
+fn all_copy_has_changed(v: &mut AllCopy) -> AllCopy {
+    // CHECK-LABEL: fn all_copy_has_changed(
+    // CHECK: bb0: {
+    // CHECK: _0 = AllCopy { {{.*}} };
+    let a = v.a;
+    let b = v.b;
+    let c = v.c;
+    v.a = 1;
+    AllCopy { a, b, c }
+}
+
+// FIXME: This can be simplified to `Copy`.
+// EMIT_MIR gvn_copy_aggregate.all_copy_use_changed.GVN.diff
+fn all_copy_use_changed(v: &mut AllCopy) -> AllCopy {
+    // CHECK-LABEL: fn all_copy_use_changed(
+    // CHECK: bb0: {
+    // CHECK-NOT: _0 = copy (*_1);
+    // CHECK: = AllCopy { {{.*}} };
+    let mut a = v.a;
+    v.a = 1;
+    a = v.a;
+    let b = v.b;
+    let c = v.c;
+    AllCopy { a, b, c }
+}
+
+// FIXME: This can be simplified to `Copy`.
+// EMIT_MIR gvn_copy_aggregate.all_copy_use_changed_2.GVN.diff
+fn all_copy_use_changed_2(v: &mut AllCopy) -> AllCopy {
+    // CHECK-LABEL: fn all_copy_use_changed_2(
+    // CHECK: bb0: {
+    // CHECK-NOT: _0 = (*_1);
+    // CHECK: = AllCopy { {{.*}} };
+    let mut a = v.a;
+    let b = v.b;
+    let c = v.c;
+    v.a = 1;
+    a = v.a;
+    AllCopy { a, b, c }
+}
+
+struct NestCopy {
+    d: i32,
+    all_copy: AllCopy,
+}
+
+// EMIT_MIR gvn_copy_aggregate.nest_copy.GVN.diff
+fn nest_copy(v: &NestCopy) -> NestCopy {
+    // CHECK-LABEL: fn nest_copy(
+    // CHECK: bb0: {
+    // CHECK-NOT: = AllCopy { {{.*}} };
+    // CHECK-NOT: = NestCopy { {{.*}} };
+    let a = v.all_copy.a;
+    let b = v.all_copy.b;
+    let c = v.all_copy.c;
+    let all_copy = AllCopy { a, b, c };
+    let d = v.d;
+    NestCopy { d, all_copy }
+}
+
+enum Enum1 {
+    A(AllCopy),
+    B(AllCopy),
+}
+
+// EMIT_MIR gvn_copy_aggregate.enum_identical_variant.GVN.diff
+fn enum_identical_variant(v: &Enum1) -> Enum1 {
+    // CHECK-LABEL: fn enum_identical_variant(
+    // CHECK-NOT: = AllCopy { {{.*}} };
+    // CHECK: _0 = copy (*_1);
+    // CHECK-NOT: = AllCopy { {{.*}} };
+    // CHECK: _0 = copy (*_1);
+    match v {
+        Enum1::A(v) => {
+            let a = v.a;
+            let b = v.b;
+            let c = v.c;
+            let all_copy = AllCopy { a, b, c };
+            Enum1::A(all_copy)
+        }
+        Enum1::B(v) => {
+            let a = v.a;
+            let b = v.b;
+            let c = v.c;
+            let all_copy = AllCopy { a, b, c };
+            Enum1::B(all_copy)
+        }
+    }
+}
+
+// EMIT_MIR gvn_copy_aggregate.enum_different_variant.GVN.diff
+fn enum_different_variant(v: &Enum1) -> Enum1 {
+    // CHECK-LABEL: fn enum_different_variant(
+    // CHECK-NOT: = AllCopy { {{.*}} };
+    // CHECK: [[V1:_.*]] = copy (((*_1) as [[VARIANT1:.*]]).0: AllCopy);
+    // CHECK: _0 = Enum1::[[VARIANT2:.*]](copy [[V1]]);
+    // CHECK-NOT: = AllCopy { {{.*}} };
+    // CHECK: [[V2:_.*]] = copy (((*_1) as [[VARIANT2]]).0: AllCopy);
+    // CHECK: _0 = Enum1::[[VARIANT1]](copy [[V2]]);
+    match v {
+        Enum1::A(v) => {
+            let a = v.a;
+            let b = v.b;
+            let c = v.c;
+            let all_copy = AllCopy { a, b, c };
+            Enum1::B(all_copy)
+        }
+        Enum1::B(v) => {
+            let a = v.a;
+            let b = v.b;
+            let c = v.c;
+            let all_copy = AllCopy { a, b, c };
+            Enum1::A(all_copy)
+        }
+    }
+}
+
+enum AlwaysSome<T> {
+    Some(T),
+}
+
+// Ensure that we do not access this local after `StorageDead`.
+// EMIT_MIR gvn_copy_aggregate.remove_storage_dead.GVN.diff
+fn remove_storage_dead<T>(f: fn() -> AlwaysSome<T>) -> AlwaysSome<T> {
+    // CHECK-LABEL: fn remove_storage_dead(
+    // CHECK: [[V1:_.*]] = copy _1() -> [return: [[BB1:bb.*]],
+    // CHECK: [[BB1]]: {
+    // CHECK-NOT: StorageDead([[V1]]);
+    // CHECK: _0 = copy [[V1]];
+    let v = {
+        match f() {
+            AlwaysSome::Some(v) => v,
+        }
+    };
+    AlwaysSome::Some(v)
+}
+
+// EMIT_MIR gvn_copy_aggregate.remove_storage_dead_from_index.GVN.diff
+#[custom_mir(dialect = "analysis")]
+fn remove_storage_dead_from_index(f: fn() -> usize, v: [SameType; 5]) -> SameType {
+    // CHECK-LABEL: fn remove_storage_dead_from_index(
+    // CHECK: [[V1:_.*]] = copy _1() -> [return: [[BB1:bb.*]],
+    // CHECK: [[BB1]]: {
+    // CHECK-NOT: StorageDead([[V1]]);
+    // CHECK-NOT: = SameType { {{.*}} };
+    // CHECK: _0 = copy _2[[[V1]]];
+    mir! {
+        let index: usize;
+        let a: i32;
+        let b: i32;
+        {
+            StorageLive(index);
+            Call(index = f(), ReturnTo(bb1), UnwindUnreachable())
+        }
+        bb1 = {
+            a = v[index].a;
+            b = v[index].b;
+            StorageDead(index);
+            RET = SameType { a, b };
+            Return()
+        }
+    }
+}

tests/mir-opt/gvn_copy_aggregate.same_type_different_index.GVN.diff

@@ -0,0 +1,40 @@
+- // MIR for `same_type_different_index` before GVN
++ // MIR for `same_type_different_index` after GVN
+  
+  fn same_type_different_index(_1: &SameType) -> SameType {
+      debug v => _1;
+      let mut _0: SameType;
+      let _2: i32;
+      let mut _4: i32;
+      let mut _5: i32;
+      scope 1 {
+          debug a => _2;
+          let _3: i32;
+          scope 2 {
+              debug b => _3;
+          }
+      }
+  
+      bb0: {
+-         StorageLive(_2);
++         nop;
+          _2 = copy ((*_1).1: i32);
+-         StorageLive(_3);
++         nop;
+          _3 = copy ((*_1).0: i32);
+          StorageLive(_4);
+          _4 = copy _2;
+          StorageLive(_5);
+          _5 = copy _3;
+-         _0 = SameType { a: move _4, b: move _5 };
++         _0 = SameType { a: copy _2, b: copy _3 };
+          StorageDead(_5);
+          StorageDead(_4);
+-         StorageDead(_3);
+-         StorageDead(_2);
++         nop;
++         nop;
+          return;
+      }
+  }
+  

tests/mir-opt/pre-codegen/clone_as_copy.clone_as_copy.PreCodegen.after.mir

@@ -0,0 +1,21 @@
+// MIR for `clone_as_copy` after PreCodegen
+
+fn clone_as_copy(_1: &NestCopy) -> NestCopy {
+    debug v => _1;
+    let mut _0: NestCopy;
+    scope 1 (inlined <NestCopy as Clone>::clone) {
+        debug self => _1;
+        let _2: &AllCopy;
+        scope 2 (inlined <AllCopy as Clone>::clone) {
+            debug self => _2;
+        }
+    }
+
+    bb0: {
+        StorageLive(_2);
+        _2 = &((*_1).1: AllCopy);
+        _0 = copy (*_1);
+        StorageDead(_2);
+        return;
+    }
+}

tests/mir-opt/pre-codegen/clone_as_copy.enum_clone_as_copy.PreCodegen.after.mir

@@ -0,0 +1,62 @@
+// MIR for `enum_clone_as_copy` after PreCodegen
+
+fn enum_clone_as_copy(_1: &Enum1) -> Enum1 {
+    debug v => _1;
+    let mut _0: Enum1;
+    scope 1 (inlined <Enum1 as Clone>::clone) {
+        debug self => _1;
+        let mut _2: isize;
+        let mut _3: &AllCopy;
+        let mut _4: &NestCopy;
+        scope 2 {
+            debug __self_0 => _3;
+            scope 6 (inlined <AllCopy as Clone>::clone) {
+                debug self => _3;
+            }
+        }
+        scope 3 {
+            debug __self_0 => _4;
+            scope 4 (inlined <NestCopy as Clone>::clone) {
+                debug self => _4;
+                let _5: &AllCopy;
+                scope 5 (inlined <AllCopy as Clone>::clone) {
+                    debug self => _5;
+                }
+            }
+        }
+    }
+
+    bb0: {
+        StorageLive(_2);
+        StorageLive(_3);
+        StorageLive(_4);
+        _2 = discriminant((*_1));
+        switchInt(move _2) -> [0: bb1, 1: bb2, otherwise: bb4];
+    }
+
+    bb1: {
+        _3 = &(((*_1) as A).0: AllCopy);
+        _0 = copy (*_1);
+        goto -> bb3;
+    }
+
+    bb2: {
+        _4 = &(((*_1) as B).0: NestCopy);
+        StorageLive(_5);
+        _5 = &((((*_1) as B).0: NestCopy).1: AllCopy);
+        StorageDead(_5);
+        _0 = copy (*_1);
+        goto -> bb3;
+    }
+
+    bb3: {
+        StorageDead(_4);
+        StorageDead(_3);
+        StorageDead(_2);
+        return;
+    }
+
+    bb4: {
+        unreachable;
+    }
+}

tests/mir-opt/pre-codegen/clone_as_copy.rs

@@ -0,0 +1,43 @@
+//@ compile-flags: -Cdebuginfo=full
+
+// Check if we have transformed the nested clone to the copy in the complete pipeline.
+
+#[derive(Clone)]
+struct AllCopy {
+    a: i32,
+    b: u64,
+    c: [i8; 3],
+}
+
+#[derive(Clone)]
+struct NestCopy {
+    a: i32,
+    b: AllCopy,
+    c: [i8; 3],
+}
+
+#[derive(Clone)]
+enum Enum1 {
+    A(AllCopy),
+    B(NestCopy),
+}
+
+// EMIT_MIR clone_as_copy.clone_as_copy.PreCodegen.after.mir
+fn clone_as_copy(v: &NestCopy) -> NestCopy {
+    // CHECK-LABEL: fn clone_as_copy(
+    // CHECK-NOT: = AllCopy { {{.*}} };
+    // CHECK-NOT: = NestCopy { {{.*}} };
+    // CHECK: _0 = copy (*_1);
+    // CHECK: return;
+    v.clone()
+}
+
+// FIXME: We can merge into exactly one assignment statement.
+// EMIT_MIR clone_as_copy.enum_clone_as_copy.PreCodegen.after.mir
+fn enum_clone_as_copy(v: &Enum1) -> Enum1 {
+    // CHECK-LABEL: fn enum_clone_as_copy(
+    // CHECK-NOT: = Enum1::
+    // CHECK: _0 = copy (*_1);
+    // CHECK: _0 = copy (*_1);
+    v.clone()
+}

tests/mir-opt/pre-codegen/no_inlined_clone.{impl#0}-clone.PreCodegen.after.mir

@@ -3,13 +3,9 @@
 fn <impl at $DIR/no_inlined_clone.rs:9:10: 9:15>::clone(_1: &Foo) -> Foo {
     debug self => _1;
     let mut _0: Foo;
-    let mut _2: i32;
 
     bb0: {
-        StorageLive(_2);
+        _0 = copy (*_1);
-        _2 = copy ((*_1).0: i32);
-        _0 = Foo { a: move _2 };
-        StorageDead(_2);
         return;
     }
 }

tests/mir-opt/pre-codegen/try_identity.old.PreCodegen.after.mir

@@ -19,14 +19,14 @@ fn old(_1: Result<T, E>) -> Result<T, E> {
     }
 
     bb1: {
-        _3 = move ((_1 as Ok).0: T);
+        _3 = copy ((_1 as Ok).0: T);
-        _0 = Result::<T, E>::Ok(copy _3);
+        _0 = copy _1;
         goto -> bb3;
     }
 
     bb2: {
-        _4 = move ((_1 as Err).0: E);
+        _4 = copy ((_1 as Err).0: E);
-        _0 = Result::<T, E>::Err(copy _4);
+        _0 = copy _1;
         goto -> bb3;
     }
 

tests/mir-opt/pre-codegen/vec_deref.vec_deref_to_slice.PreCodegen.after.panic-abort.mir

@@ -5,7 +5,7 @@ fn vec_deref_to_slice(_1: &Vec<u8>) -> &[u8] {
     let mut _0: &[u8];
     scope 1 (inlined <Vec<u8> as Deref>::deref) {
         debug self => _1;
-        let mut _7: usize;
+        let mut _6: usize;
         scope 2 (inlined Vec::<u8>::as_ptr) {
             debug self => _1;
             let mut _2: &alloc::raw_vec::RawVec<u8>;
@@ -14,7 +14,6 @@ fn vec_deref_to_slice(_1: &Vec<u8>) -> &[u8] {
                 let mut _3: &alloc::raw_vec::RawVecInner;
                 scope 4 (inlined alloc::raw_vec::RawVecInner::ptr::<u8>) {
                     debug self => _3;
-                    let mut _6: std::ptr::NonNull<u8>;
                     scope 5 (inlined alloc::raw_vec::RawVecInner::non_null::<u8>) {
                         debug self => _3;
                         let mut _4: std::ptr::NonNull<u8>;
@@ -30,28 +29,28 @@ fn vec_deref_to_slice(_1: &Vec<u8>) -> &[u8] {
                             }
                         }
                         scope 9 (inlined #[track_caller] <Unique<u8> as Into<NonNull<u8>>>::into) {
-                            debug ((self: Unique<u8>).0: std::ptr::NonNull<u8>) => _6;
+                            debug ((self: Unique<u8>).0: std::ptr::NonNull<u8>) => _4;
                             debug ((self: Unique<u8>).1: std::marker::PhantomData<u8>) => const PhantomData::<u8>;
                             scope 10 (inlined <NonNull<u8> as From<Unique<u8>>>::from) {
-                                debug ((unique: Unique<u8>).0: std::ptr::NonNull<u8>) => _6;
+                                debug ((unique: Unique<u8>).0: std::ptr::NonNull<u8>) => _4;
                                 debug ((unique: Unique<u8>).1: std::marker::PhantomData<u8>) => const PhantomData::<u8>;
                                 scope 11 (inlined Unique::<u8>::as_non_null_ptr) {
-                                    debug ((self: Unique<u8>).0: std::ptr::NonNull<u8>) => _6;
+                                    debug ((self: Unique<u8>).0: std::ptr::NonNull<u8>) => _4;
                                     debug ((self: Unique<u8>).1: std::marker::PhantomData<u8>) => const PhantomData::<u8>;
                                 }
                             }
                         }
                     }
                     scope 12 (inlined NonNull::<u8>::as_ptr) {
-                        debug self => _6;
+                        debug self => _4;
                     }
                 }
             }
         }
         scope 13 (inlined std::slice::from_raw_parts::<'_, u8>) {
             debug data => _5;
-            debug len => _7;
+            debug len => _6;
-            let _8: *const [u8];
+            let _7: *const [u8];
             scope 14 (inlined core::ub_checks::check_language_ub) {
                 scope 15 (inlined core::ub_checks::check_language_ub::runtime) {
                 }
@@ -62,10 +61,10 @@ fn vec_deref_to_slice(_1: &Vec<u8>) -> &[u8] {
             }
             scope 18 (inlined slice_from_raw_parts::<u8>) {
                 debug data => _5;
-                debug len => _7;
+                debug len => _6;
                 scope 19 (inlined std::ptr::from_raw_parts::<[u8], u8>) {
                     debug data_pointer => _5;
-                    debug metadata => _7;
+                    debug metadata => _6;
                 }
             }
         }
@@ -76,22 +75,17 @@ fn vec_deref_to_slice(_1: &Vec<u8>) -> &[u8] {
         _2 = &((*_1).0: alloc::raw_vec::RawVec<u8>);
         StorageLive(_3);
         _3 = &(((*_1).0: alloc::raw_vec::RawVec<u8>).0: alloc::raw_vec::RawVecInner);
-        StorageLive(_6);
-        StorageLive(_4);
         _4 = copy (((((*_1).0: alloc::raw_vec::RawVec<u8>).0: alloc::raw_vec::RawVecInner).0: std::ptr::Unique<u8>).0: std::ptr::NonNull<u8>);
         _5 = copy (_4.0: *const u8);
-        _6 = NonNull::<u8> { pointer: copy _5 };
-        StorageDead(_4);
-        StorageDead(_6);
         StorageDead(_3);
         StorageDead(_2);
+        StorageLive(_6);
+        _6 = copy ((*_1).1: usize);
         StorageLive(_7);
-        _7 = copy ((*_1).1: usize);
+        _7 = *const [u8] from (copy _5, copy _6);
-        StorageLive(_8);
+        _0 = &(*_7);
-        _8 = *const [u8] from (copy _5, copy _7);
-        _0 = &(*_8);
-        StorageDead(_8);
         StorageDead(_7);
+        StorageDead(_6);
         return;
     }
 }

tests/mir-opt/pre-codegen/vec_deref.vec_deref_to_slice.PreCodegen.after.panic-unwind.mir

@@ -5,7 +5,7 @@ fn vec_deref_to_slice(_1: &Vec<u8>) -> &[u8] {
     let mut _0: &[u8];
     scope 1 (inlined <Vec<u8> as Deref>::deref) {
         debug self => _1;
-        let mut _7: usize;
+        let mut _6: usize;
         scope 2 (inlined Vec::<u8>::as_ptr) {
             debug self => _1;
             let mut _2: &alloc::raw_vec::RawVec<u8>;
@@ -14,7 +14,6 @@ fn vec_deref_to_slice(_1: &Vec<u8>) -> &[u8] {
                 let mut _3: &alloc::raw_vec::RawVecInner;
                 scope 4 (inlined alloc::raw_vec::RawVecInner::ptr::<u8>) {
                     debug self => _3;
-                    let mut _6: std::ptr::NonNull<u8>;
                     scope 5 (inlined alloc::raw_vec::RawVecInner::non_null::<u8>) {
                         debug self => _3;
                         let mut _4: std::ptr::NonNull<u8>;
@@ -30,28 +29,28 @@ fn vec_deref_to_slice(_1: &Vec<u8>) -> &[u8] {
                             }
                         }
                         scope 9 (inlined #[track_caller] <Unique<u8> as Into<NonNull<u8>>>::into) {
-                            debug ((self: Unique<u8>).0: std::ptr::NonNull<u8>) => _6;
+                            debug ((self: Unique<u8>).0: std::ptr::NonNull<u8>) => _4;
                             debug ((self: Unique<u8>).1: std::marker::PhantomData<u8>) => const PhantomData::<u8>;
                             scope 10 (inlined <NonNull<u8> as From<Unique<u8>>>::from) {
-                                debug ((unique: Unique<u8>).0: std::ptr::NonNull<u8>) => _6;
+                                debug ((unique: Unique<u8>).0: std::ptr::NonNull<u8>) => _4;
                                 debug ((unique: Unique<u8>).1: std::marker::PhantomData<u8>) => const PhantomData::<u8>;
                                 scope 11 (inlined Unique::<u8>::as_non_null_ptr) {
-                                    debug ((self: Unique<u8>).0: std::ptr::NonNull<u8>) => _6;
+                                    debug ((self: Unique<u8>).0: std::ptr::NonNull<u8>) => _4;
                                     debug ((self: Unique<u8>).1: std::marker::PhantomData<u8>) => const PhantomData::<u8>;
                                 }
                             }
                         }
                     }
                     scope 12 (inlined NonNull::<u8>::as_ptr) {
-                        debug self => _6;
+                        debug self => _4;
                     }
                 }
             }
         }
         scope 13 (inlined std::slice::from_raw_parts::<'_, u8>) {
             debug data => _5;
-            debug len => _7;
+            debug len => _6;
-            let _8: *const [u8];
+            let _7: *const [u8];
             scope 14 (inlined core::ub_checks::check_language_ub) {
                 scope 15 (inlined core::ub_checks::check_language_ub::runtime) {
                 }
@@ -62,10 +61,10 @@ fn vec_deref_to_slice(_1: &Vec<u8>) -> &[u8] {
             }
             scope 18 (inlined slice_from_raw_parts::<u8>) {
                 debug data => _5;
-                debug len => _7;
+                debug len => _6;
                 scope 19 (inlined std::ptr::from_raw_parts::<[u8], u8>) {
                     debug data_pointer => _5;
-                    debug metadata => _7;
+                    debug metadata => _6;
                 }
             }
         }
@@ -76,22 +75,17 @@ fn vec_deref_to_slice(_1: &Vec<u8>) -> &[u8] {
         _2 = &((*_1).0: alloc::raw_vec::RawVec<u8>);
         StorageLive(_3);
         _3 = &(((*_1).0: alloc::raw_vec::RawVec<u8>).0: alloc::raw_vec::RawVecInner);
-        StorageLive(_6);
-        StorageLive(_4);
         _4 = copy (((((*_1).0: alloc::raw_vec::RawVec<u8>).0: alloc::raw_vec::RawVecInner).0: std::ptr::Unique<u8>).0: std::ptr::NonNull<u8>);
         _5 = copy (_4.0: *const u8);
-        _6 = NonNull::<u8> { pointer: copy _5 };
-        StorageDead(_4);
-        StorageDead(_6);
         StorageDead(_3);
         StorageDead(_2);
+        StorageLive(_6);
+        _6 = copy ((*_1).1: usize);
         StorageLive(_7);
-        _7 = copy ((*_1).1: usize);
+        _7 = *const [u8] from (copy _5, copy _6);
-        StorageLive(_8);
+        _0 = &(*_7);
-        _8 = *const [u8] from (copy _5, copy _7);
-        _0 = &(*_8);
-        StorageDead(_8);
         StorageDead(_7);
+        StorageDead(_6);
         return;
     }
 }