turns out Layout has some more things to worry about -- move ABI comparison into helper function

like is_bool, and some special magic extra fields

compiler/rustc_abi/src/lib.rs

@@ -1300,12 +1300,18 @@ impl Abi {
         matches!(*self, Abi::Uninhabited)
     }
 
-    /// Returns `true` is this is a scalar type
+    /// Returns `true` if this is a scalar type
     #[inline]
     pub fn is_scalar(&self) -> bool {
         matches!(*self, Abi::Scalar(_))
     }
 
+    /// Returns `true` if this is a bool
+    #[inline]
+    pub fn is_bool(&self) -> bool {
+        matches!(*self, Abi::Scalar(s) if s.is_bool())
+    }
+
     /// Returns the fixed alignment of this ABI, if any is mandated.
     pub fn inherent_align<C: HasDataLayout>(&self, cx: &C) -> Option<AbiAndPrefAlign> {
         Some(match *self {
@@ -1703,6 +1709,22 @@ impl LayoutS {
             Abi::Aggregate { sized } => sized && self.size.bytes() == 0,
         }
     }
+
+    /// Checks if these two `Layout` are equal enough to be considered "the same for all function
+    /// call ABIs". Note however that real ABIs depend on more details that are not reflected in the
+    /// `Layout`; the `PassMode` need to be compared as well.
+    pub fn eq_abi(&self, other: &Self) -> bool {
+        // The one thing that we are not capturing here is that for unsized types, the metadata must
+        // also have the same ABI, and moreover that the same metadata leads to the same size. The
+        // 2nd point is quite hard to check though.
+        self.size == other.size
+            && self.is_sized() == other.is_sized()
+            && self.abi.eq_up_to_validity(&other.abi)
+            && self.abi.is_bool() == other.abi.is_bool()
+            && self.align.abi == other.align.abi
+            && self.max_repr_align == other.max_repr_align
+            && self.unadjusted_abi_align == other.unadjusted_abi_align
+    }
 }
 
 #[derive(Copy, Clone, Debug)]

compiler/rustc_const_eval/src/interpret/terminator.rs

@@ -332,12 +332,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         if self.layout_compat(caller_abi.layout, callee_abi.layout)
             && caller_abi.mode.eq_abi(&callee_abi.mode)
         {
-            // Something went very wrong if our checks don't even imply that the layout is the same.
-            assert!(
-                caller_abi.layout.size == callee_abi.layout.size
-                    && caller_abi.layout.align.abi == callee_abi.layout.align.abi
-                    && caller_abi.layout.is_sized() == callee_abi.layout.is_sized()
-            );
+            // Something went very wrong if our checks don't imply layout ABI compatibility.
+            assert!(caller_abi.layout.eq_abi(&callee_abi.layout));
             return true;
         } else {
             trace!(

compiler/rustc_passes/src/abi_test.rs

@@ -5,7 +5,7 @@ use rustc_middle::ty::layout::{FnAbiError, LayoutError};
 use rustc_middle::ty::{self, GenericArgs, Instance, Ty, TyCtxt};
 use rustc_span::source_map::Spanned;
 use rustc_span::symbol::sym;
-use rustc_target::abi::call::{ArgAbi, FnAbi};
+use rustc_target::abi::call::FnAbi;
 
 use crate::errors::{AbiInvalidAttribute, AbiNe, AbiOf, UnrecognizedField};
 
@@ -114,20 +114,6 @@ fn dump_abi_of_fn_item(tcx: TyCtxt<'_>, item_def_id: DefId, attr: &Attribute) {
     }
 }
 
-fn test_arg_abi_eq<'tcx>(
-    abi1: &'tcx ArgAbi<'tcx, Ty<'tcx>>,
-    abi2: &'tcx ArgAbi<'tcx, Ty<'tcx>>,
-) -> bool {
-    // Ideally we'd just compare the `mode`, but that is not enough -- for some modes LLVM will look
-    // at the type. Comparing the `mode` and `layout.abi` as well as size and alignment should catch
-    // basically everything though (except for tricky cases around unized types).
-    abi1.mode.eq_abi(&abi2.mode)
-        && abi1.layout.abi.eq_up_to_validity(&abi2.layout.abi)
-        && abi1.layout.size == abi2.layout.size
-        && abi1.layout.align.abi == abi2.layout.align.abi
-        && abi1.layout.is_sized() == abi2.layout.is_sized()
-}
-
 fn test_abi_eq<'tcx>(abi1: &'tcx FnAbi<'tcx, Ty<'tcx>>, abi2: &'tcx FnAbi<'tcx, Ty<'tcx>>) -> bool {
     if abi1.conv != abi2.conv
         || abi1.args.len() != abi2.args.len()
@@ -138,8 +124,8 @@ fn test_abi_eq<'tcx>(abi1: &'tcx FnAbi<'tcx, Ty<'tcx>>, abi2: &'tcx FnAbi<'tcx,
         return false;
     }
 
-    test_arg_abi_eq(&abi1.ret, &abi2.ret)
-        && abi1.args.iter().zip(abi2.args.iter()).all(|(arg1, arg2)| test_arg_abi_eq(arg1, arg2))
+    abi1.ret.eq_abi(&abi2.ret)
+        && abi1.args.iter().zip(abi2.args.iter()).all(|(arg1, arg2)| arg1.eq_abi(arg2))
 }
 
 fn dump_abi_of_fn_type(tcx: TyCtxt<'_>, item_def_id: DefId, attr: &Attribute) {

compiler/rustc_target/src/abi/call/mod.rs

@@ -60,7 +60,8 @@ pub enum PassMode {
 
 impl PassMode {
     /// Checks if these two `PassMode` are equal enough to be considered "the same for all
-    /// function call ABIs".
+    /// function call ABIs". However, the `Layout` can also impact ABI decisions,
+    /// so that needs to be compared as well!
     pub fn eq_abi(&self, other: &Self) -> bool {
         match (self, other) {
             (PassMode::Ignore, PassMode::Ignore) => true, // can still be reached for the return type
@@ -623,6 +624,14 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
     pub fn is_ignore(&self) -> bool {
         matches!(self.mode, PassMode::Ignore)
     }
+
+    /// Checks if these two `ArgAbi` are equal enough to be considered "the same for all
+    /// function call ABIs".
+    pub fn eq_abi(&self, other: &Self) -> bool {
+        // Ideally we'd just compare the `mode`, but that is not enough -- for some modes LLVM will look
+        // at the type.
+        self.layout.eq_abi(&other.layout) && self.mode.eq_abi(&other.mode)
+    }
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)]