Auto merge of #135618 - lcnr:coherence-unknown, r=compiler-errors

add cache to `AmbiguityCausesVisitor`

fixes #135457, alternative to #135524.

cc https://rust-lang.zulipchat.com/#narrow/channel/364551-t-types.2Ftrait-system-refactor/topic/new-solver.20hang.20.23135457

r? `@compiler-errors`

compiler/rustc_trait_selection/src/traits/coherence.rs

@@ -6,7 +6,7 @@
 
 use std::fmt::Debug;
 
-use rustc_data_structures::fx::FxIndexSet;
+use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
 use rustc_errors::{Diag, EmissionGuarantee};
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
@@ -116,28 +116,39 @@ pub fn overlapping_impls(
         return None;
     }
 
-    let _overlap_with_bad_diagnostics = overlap(
-        tcx,
-        TrackAmbiguityCauses::No,
-        skip_leak_check,
-        impl1_def_id,
-        impl2_def_id,
-        overlap_mode,
-    )?;
+    if tcx.next_trait_solver_in_coherence() {
+        overlap(
+            tcx,
+            TrackAmbiguityCauses::Yes,
+            skip_leak_check,
+            impl1_def_id,
+            impl2_def_id,
+            overlap_mode,
+        )
+    } else {
+        let _overlap_with_bad_diagnostics = overlap(
+            tcx,
+            TrackAmbiguityCauses::No,
+            skip_leak_check,
+            impl1_def_id,
+            impl2_def_id,
+            overlap_mode,
+        )?;
 
-    // In the case where we detect an error, run the check again, but
-    // this time tracking intercrate ambiguity causes for better
-    // diagnostics. (These take time and can lead to false errors.)
-    let overlap = overlap(
-        tcx,
-        TrackAmbiguityCauses::Yes,
-        skip_leak_check,
-        impl1_def_id,
-        impl2_def_id,
-        overlap_mode,
-    )
-    .unwrap();
-    Some(overlap)
+        // In the case where we detect an error, run the check again, but
+        // this time tracking intercrate ambiguity causes for better
+        // diagnostics. (These take time and can lead to false errors.)
+        let overlap = overlap(
+            tcx,
+            TrackAmbiguityCauses::Yes,
+            skip_leak_check,
+            impl1_def_id,
+            impl2_def_id,
+            overlap_mode,
+        )
+        .unwrap();
+        Some(overlap)
+    }
 }
 
 fn fresh_impl_header<'tcx>(infcx: &InferCtxt<'tcx>, impl_def_id: DefId) -> ty::ImplHeader<'tcx> {
@@ -615,6 +626,7 @@ fn compute_intercrate_ambiguity_causes<'tcx>(
 }
 
 struct AmbiguityCausesVisitor<'a, 'tcx> {
+    cache: FxHashSet<Goal<'tcx, ty::Predicate<'tcx>>>,
     causes: &'a mut FxIndexSet<IntercrateAmbiguityCause<'tcx>>,
 }
 
@@ -624,6 +636,10 @@ impl<'a, 'tcx> ProofTreeVisitor<'tcx> for AmbiguityCausesVisitor<'a, 'tcx> {
     }
 
     fn visit_goal(&mut self, goal: &InspectGoal<'_, 'tcx>) {
+        if !self.cache.insert(goal.goal()) {
+            return;
+        }
+
         let infcx = goal.infcx();
         for cand in goal.candidates() {
             cand.visit_nested_in_probe(self);
@@ -748,5 +764,10 @@ fn search_ambiguity_causes<'tcx>(
     goal: Goal<'tcx, ty::Predicate<'tcx>>,
     causes: &mut FxIndexSet<IntercrateAmbiguityCause<'tcx>>,
 ) {
-    infcx.probe(|_| infcx.visit_proof_tree(goal, &mut AmbiguityCausesVisitor { causes }));
+    infcx.probe(|_| {
+        infcx.visit_proof_tree(goal, &mut AmbiguityCausesVisitor {
+            cache: Default::default(),
+            causes,
+        })
+    });
 }

tests/ui/coherence/occurs-check/associated-type.next.stderr

@@ -1,7 +1,5 @@
  WARN rustc_infer::infer::relate::generalize may incompletely handle alias type: AliasTy { args: [*const ?1t, '^0.Named(DefId(0:27 ~ associated_type[f554]::{impl#3}::'a#1), "'a")], def_id: DefId(0:5 ~ associated_type[f554]::ToUnit::Unit), .. }
  WARN rustc_infer::infer::relate::generalize may incompletely handle alias type: AliasTy { args: [*const ?1t, '^0.Named(DefId(0:27 ~ associated_type[f554]::{impl#3}::'a#1), "'a")], def_id: DefId(0:5 ~ associated_type[f554]::ToUnit::Unit), .. }
- WARN rustc_infer::infer::relate::generalize may incompletely handle alias type: AliasTy { args: [*const ?1t, '^0.Named(DefId(0:27 ~ associated_type[f554]::{impl#3}::'a#1), "'a")], def_id: DefId(0:5 ~ associated_type[f554]::ToUnit::Unit), .. }
- WARN rustc_infer::infer::relate::generalize may incompletely handle alias type: AliasTy { args: [*const ?1t, '^0.Named(DefId(0:27 ~ associated_type[f554]::{impl#3}::'a#1), "'a")], def_id: DefId(0:5 ~ associated_type[f554]::ToUnit::Unit), .. }
 error[E0119]: conflicting implementations of trait `Overlap<for<'a> fn(&'a (), ())>` for type `for<'a> fn(&'a (), ())`
   --> $DIR/associated-type.rs:32:1
    |

tests/ui/coherence/occurs-check/associated-type.old.stderr

@@ -1,7 +1,5 @@
  WARN rustc_infer::infer::relate::generalize may incompletely handle alias type: AliasTy { args: [*const ?1t, '^0.Named(DefId(0:27 ~ associated_type[f554]::{impl#3}::'a#1), "'a")], def_id: DefId(0:5 ~ associated_type[f554]::ToUnit::Unit), .. }
  WARN rustc_infer::infer::relate::generalize may incompletely handle alias type: AliasTy { args: [*const ?1t, '^0.Named(DefId(0:27 ~ associated_type[f554]::{impl#3}::'a#1), "'a")], def_id: DefId(0:5 ~ associated_type[f554]::ToUnit::Unit), .. }
- WARN rustc_infer::infer::relate::generalize may incompletely handle alias type: AliasTy { args: [*const ?1t, '^0.Named(DefId(0:27 ~ associated_type[f554]::{impl#3}::'a#1), "'a")], def_id: DefId(0:5 ~ associated_type[f554]::ToUnit::Unit), .. }
- WARN rustc_infer::infer::relate::generalize may incompletely handle alias type: AliasTy { args: [*const ?1t, '^0.Named(DefId(0:27 ~ associated_type[f554]::{impl#3}::'a#1), "'a")], def_id: DefId(0:5 ~ associated_type[f554]::ToUnit::Unit), .. }
 error[E0119]: conflicting implementations of trait `Overlap<for<'a> fn(&'a (), ())>` for type `for<'a> fn(&'a (), ())`
   --> $DIR/associated-type.rs:32:1
    |

tests/ui/traits/next-solver/coherence/ambiguity-causes-visitor-hang.rs

@@ -0,0 +1,56 @@
+// Computing the ambiguity causes for the overlap ended up
+// causing an exponential blowup when recursing into the normalization
+// goals for `<Box<?t> as RecursiveSuper>::Assoc`. This test
+// takes multiple minutes when doing so and less than a second
+// otherwise.
+
+//@ compile-flags: -Znext-solver=coherence
+
+trait RecursiveSuper:
+    Super<
+        A0 = Self::Assoc,
+        A1 = Self::Assoc,
+        A2 = Self::Assoc,
+        A3 = Self::Assoc,
+        A4 = Self::Assoc,
+        A5 = Self::Assoc,
+        A6 = Self::Assoc,
+        A7 = Self::Assoc,
+        A8 = Self::Assoc,
+        A9 = Self::Assoc,
+        A10 = Self::Assoc,
+        A11 = Self::Assoc,
+        A12 = Self::Assoc,
+        A13 = Self::Assoc,
+        A14 = Self::Assoc,
+        A15 = Self::Assoc,
+    >
+{
+    type Assoc;
+}
+
+trait Super {
+    type A0;
+    type A1;
+    type A2;
+    type A3;
+    type A4;
+    type A5;
+    type A6;
+    type A7;
+    type A8;
+    type A9;
+    type A10;
+    type A11;
+    type A12;
+    type A13;
+    type A14;
+    type A15;
+}
+
+trait Overlap {}
+impl<T: RecursiveSuper> Overlap for T {}
+impl<T> Overlap for Box<T> {}
+//~^ ERROR conflicting implementations of trait `Overlap` for type `Box<_>`
+
+fn main() {}

tests/ui/traits/next-solver/coherence/ambiguity-causes-visitor-hang.stderr

@@ -0,0 +1,14 @@
+error[E0119]: conflicting implementations of trait `Overlap` for type `Box<_>`
+  --> $DIR/ambiguity-causes-visitor-hang.rs:53:1
+   |
+LL | impl<T: RecursiveSuper> Overlap for T {}
+   | ------------------------------------- first implementation here
+LL | impl<T> Overlap for Box<T> {}
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^ conflicting implementation for `Box<_>`
+   |
+   = note: downstream crates may implement trait `Super` for type `std::boxed::Box<_>`
+   = note: downstream crates may implement trait `RecursiveSuper` for type `std::boxed::Box<_>`
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0119`.