Rollup merge of #112810 - compiler-errors:dont-ice-on-bad-layout, r=wesleywiser

Don't ICE on unnormalized struct tail in layout computation

1. We try to compute a `SizeSkeleton` even if a layout error occurs, but we really only need to do this if we get `LayoutError::Unknown`, since that means our type is too polymorphic to actually compute the full layout. If we have other errors, like `LayoutError::NormalizationError` or `LayoutError::Cycle`, then we can't really make any progress, since this represents an actual error.
2. Avoid using `normalize_erasing_regions` and `struct_tail_erasing_lifetimes` since those ICE on normalization errors, and since we may call `layout_of` in HIR typeck, we don't know for certain that we're on the happy path.

Fixes #112736

compiler/rustc_middle/src/ty/layout.rs

@@ -318,7 +318,13 @@ impl<'tcx> SizeSkeleton<'tcx> {
             Ok(layout) => {
                 return Ok(SizeSkeleton::Known(layout.size));
             }
-            Err(err) => err,
+            Err(err @ LayoutError::Unknown(_)) => err,
+            // We can't extract SizeSkeleton info from other layout errors
+            Err(
+                e @ LayoutError::Cycle
+                | e @ LayoutError::SizeOverflow(_)
+                | e @ LayoutError::NormalizationFailure(..),
+            ) => return Err(e),
         };
 
         match *ty.kind() {

compiler/rustc_ty_utils/src/layout.rs

@@ -145,17 +145,35 @@ fn layout_of_uncached<'tcx>(
                 return Ok(tcx.mk_layout(LayoutS::scalar(cx, data_ptr)));
             }
 
-            let unsized_part = tcx.struct_tail_erasing_lifetimes(pointee, param_env);
-
             let metadata = if let Some(metadata_def_id) = tcx.lang_items().metadata_type()
                 // Projection eagerly bails out when the pointee references errors,
                 // fall back to structurally deducing metadata.
                 && !pointee.references_error()
             {
-                let metadata_ty = tcx.normalize_erasing_regions(
+                let pointee_metadata = tcx.mk_projection(metadata_def_id, [pointee]);
+                let metadata_ty = match tcx.try_normalize_erasing_regions(
                     param_env,
-                    tcx.mk_projection(metadata_def_id, [pointee]),
-                );
+                    pointee_metadata,
+                ) {
+                    Ok(metadata_ty) => metadata_ty,
+                    Err(mut err) => {
+                        // Usually `<Ty as Pointee>::Metadata` can't be normalized because
+                        // its struct tail cannot be normalized either, so try to get a
+                        // more descriptive layout error here, which will lead to less confusing
+                        // diagnostics.
+                        match tcx.try_normalize_erasing_regions(
+                            param_env,
+                            tcx.struct_tail_without_normalization(pointee),
+                        ) {
+                            Ok(_) => {},
+                            Err(better_err) => {
+                                err = better_err;
+                            }
+                        }
+                        return Err(LayoutError::NormalizationFailure(pointee, err));
+                    },
+                };
+
                 let metadata_layout = cx.layout_of(metadata_ty)?;
                 // If the metadata is a 1-zst, then the pointer is thin.
                 if metadata_layout.is_zst() && metadata_layout.align.abi.bytes() == 1 {
@@ -163,10 +181,13 @@ fn layout_of_uncached<'tcx>(
                 }
 
                 let Abi::Scalar(metadata) = metadata_layout.abi else {
-                    return Err(LayoutError::Unknown(unsized_part));
+                    return Err(LayoutError::Unknown(pointee));
                 };
+
                 metadata
             } else {
+                let unsized_part = tcx.struct_tail_erasing_lifetimes(pointee, param_env);
+
                 match unsized_part.kind() {
                     ty::Foreign(..) => {
                         return Ok(tcx.mk_layout(LayoutS::scalar(cx, data_ptr)));
@@ -178,7 +199,7 @@ fn layout_of_uncached<'tcx>(
                         vtable
                     }
                     _ => {
-                        return Err(LayoutError::Unknown(unsized_part));
+                        return Err(LayoutError::Unknown(pointee));
                     }
                 }
             };

tests/ui/const-generics/issue-112505-overflow.stderr

@@ -4,8 +4,8 @@ error[E0512]: cannot transmute between types of different sizes, or dependently-
 LL |     unsafe { std::mem::transmute(v) }
    |              ^^^^^^^^^^^^^^^^^^^
    |
-   = note: source type: `[[[u32; 8888888]; 9999999]; 777777777]` (values of the type `[[[u32; 8888888]; 9999999]; 777777777]` are too big for the current architecture)
-   = note: target type: `[[[u32; 9999999]; 777777777]; 239]` (59484438436515561504 bits)
+   = note: source type: `[[[u32; 8888888]; 9999999]; 777777777]` (values of the type `[[u32; 8888888]; 9999999]` are too big for the current architecture)
+   = note: target type: `[[[u32; 9999999]; 777777777]; 239]` (values of the type `[[u32; 9999999]; 777777777]` are too big for the current architecture)
 
 error: aborting due to previous error
 

tests/ui/const-generics/transmute-fail.stderr

@@ -43,8 +43,8 @@ error[E0512]: cannot transmute between types of different sizes, or dependently-
 LL |     std::mem::transmute(v)
    |     ^^^^^^^^^^^^^^^^^^^
    |
-   = note: source type: `[[[u32; 8888888]; 9999999]; 777777777]` (values of the type `[[[u32; 8888888]; 9999999]; 777777777]` are too big for the current architecture)
-   = note: target type: `[[[u32; 9999999]; 777777777]; 8888888]` (values of the type `[[[u32; 9999999]; 777777777]; 8888888]` are too big for the current architecture)
+   = note: source type: `[[[u32; 8888888]; 9999999]; 777777777]` (values of the type `[[u32; 8888888]; 9999999]` are too big for the current architecture)
+   = note: target type: `[[[u32; 9999999]; 777777777]; 8888888]` (values of the type `[[u32; 9999999]; 777777777]` are too big for the current architecture)
 
 error: aborting due to 6 previous errors
 

tests/ui/layout/cannot-transmute-unnormalizable-type.rs

@@ -0,0 +1,22 @@
+trait Trait {
+    type RefTarget;
+}
+
+impl Trait for ()
+where
+    Missing: Trait,
+    //~^ ERROR cannot find type `Missing` in this scope
+{
+    type RefTarget = ();
+}
+
+struct Other {
+    data: <() as Trait>::RefTarget,
+}
+
+fn main() {
+    unsafe {
+        std::mem::transmute::<Option<()>, Option<&Other>>(None);
+        //~^ ERROR cannot transmute between types of different sizes, or dependently-sized types
+    }
+}

tests/ui/layout/cannot-transmute-unnormalizable-type.stderr

@@ -0,0 +1,19 @@
+error[E0412]: cannot find type `Missing` in this scope
+  --> $DIR/cannot-transmute-unnormalizable-type.rs:7:5
+   |
+LL |     Missing: Trait,
+   |     ^^^^^^^ not found in this scope
+
+error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
+  --> $DIR/cannot-transmute-unnormalizable-type.rs:19:9
+   |
+LL |         std::mem::transmute::<Option<()>, Option<&Other>>(None);
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: source type: `Option<()>` (8 bits)
+   = note: target type: `Option<&Other>` (unable to determine layout for `Other` because `<() as Trait>::RefTarget` cannot be normalized)
+
+error: aborting due to 2 previous errors
+
+Some errors have detailed explanations: E0412, E0512.
+For more information about an error, try `rustc --explain E0412`.