instantiate canonical vars eagerly

compiler/rustc_middle/src/ty/sty.rs

@@ -1106,17 +1106,6 @@ impl<'tcx, T> Binder<'tcx, T> {
         if self.0.has_escaping_bound_vars() { None } else { Some(self.skip_binder()) }
     }
 
-    pub fn no_bound_vars_ignoring_escaping(self, tcx: TyCtxt<'tcx>) -> Option<T>
-    where
-        T: TypeFoldable<'tcx>,
-    {
-        if !self.0.has_escaping_bound_vars() {
-            Some(self.skip_binder())
-        } else {
-            self.0.try_fold_with(&mut SkipBindersAt { index: ty::INNERMOST, tcx }).ok()
-        }
-    }
-
     /// Splits the contents into two things that share the same binder
     /// level as the original, returning two distinct binders.
     ///

compiler/rustc_trait_selection/src/solve/assembly.rs

@@ -1,20 +1,11 @@
 //! Code shared by trait and projection goals for candidate assembly.
 
 use super::infcx_ext::InferCtxtExt;
-use super::{
+use super::{CanonicalResponse, Certainty, EvalCtxt, Goal};
-    instantiate_canonical_query_response, CanonicalGoal, CanonicalResponse, Certainty, EvalCtxt,
-    Goal,
-};
 use rustc_hir::def_id::DefId;
-use rustc_infer::infer::TyCtxtInferExt;
-use rustc_infer::infer::{
-    canonical::{CanonicalVarValues, OriginalQueryValues},
-    InferCtxt,
-};
 use rustc_infer::traits::query::NoSolution;
 use rustc_middle::ty::TypeFoldable;
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_span::DUMMY_SP;
 use std::fmt::Debug;
 
 /// A candidate is a possible way to prove a goal.
@@ -40,7 +31,7 @@ pub(super) trait GoalKind<'tcx>: TypeFoldable<'tcx> + Copy {
     fn trait_def_id(self, tcx: TyCtxt<'tcx>) -> DefId;
 
     fn consider_impl_candidate(
-        acx: &mut AssemblyCtxt<'_, 'tcx, Self>,
+        acx: &mut AssemblyCtxt<'_, '_, 'tcx, Self>,
         goal: Goal<'tcx, Self>,
         impl_def_id: DefId,
     );
@@ -49,21 +40,17 @@ pub(super) trait GoalKind<'tcx>: TypeFoldable<'tcx> + Copy {
 /// An abstraction which correctly deals with the canonical results for candidates.
 ///
 /// It also deduplicates the behavior between trait and projection predicates.
-pub(super) struct AssemblyCtxt<'a, 'tcx, G: GoalKind<'tcx>> {
+pub(super) struct AssemblyCtxt<'a, 'b, 'tcx, G: GoalKind<'tcx>> {
-    pub(super) cx: &'a mut EvalCtxt<'tcx>,
+    pub(super) cx: &'a mut EvalCtxt<'b, 'tcx>,
-    pub(super) infcx: &'a InferCtxt<'tcx>,
-    var_values: CanonicalVarValues<'tcx>,
     candidates: Vec<Candidate<'tcx, G>>,
 }
 
-impl<'a, 'tcx, G: GoalKind<'tcx>> AssemblyCtxt<'a, 'tcx, G> {
+impl<'a, 'b, 'tcx, G: GoalKind<'tcx>> AssemblyCtxt<'a, 'b, 'tcx, G> {
     pub(super) fn assemble_and_evaluate_candidates(
-        cx: &'a mut EvalCtxt<'tcx>,
+        cx: &'a mut EvalCtxt<'b, 'tcx>,
-        goal: CanonicalGoal<'tcx, G>,
+        goal: Goal<'tcx, G>,
     ) -> Vec<Candidate<'tcx, G>> {
-        let (ref infcx, goal, var_values) =
+        let mut acx = AssemblyCtxt { cx, candidates: Vec::new() };
-            cx.tcx.infer_ctxt().build_with_canonical(DUMMY_SP, &goal);
-        let mut acx = AssemblyCtxt { cx, infcx, var_values, candidates: Vec::new() };
 
         acx.assemble_candidates_after_normalizing_self_ty(goal);
 
@@ -77,7 +64,7 @@ impl<'a, 'tcx, G: GoalKind<'tcx>> AssemblyCtxt<'a, 'tcx, G> {
         source: G::CandidateSource,
         certainty: Certainty,
     ) {
-        match self.infcx.make_canonical_response(self.var_values.clone(), certainty) {
+        match self.cx.make_canonical_response(certainty) {
             Ok(result) => self.candidates.push(Candidate { source, result }),
             Err(NoSolution) => debug!(?source, ?certainty, "failed leakcheck"),
         }
@@ -89,13 +76,14 @@ impl<'a, 'tcx, G: GoalKind<'tcx>> AssemblyCtxt<'a, 'tcx, G> {
     /// self type to the list of candidates in case that succeeds. Note that we can't just eagerly return in
     /// this case as projections as self types add `
     fn assemble_candidates_after_normalizing_self_ty(&mut self, goal: Goal<'tcx, G>) {
-        let tcx = self.cx.tcx;
+        let tcx = self.cx.tcx();
+        let infcx = self.cx.infcx;
         // FIXME: We also have to normalize opaque types, not sure where to best fit that in.
         let &ty::Alias(ty::Projection, projection_ty) = goal.predicate.self_ty().kind() else {
             return
         };
-        self.infcx.probe(|_| {
+        infcx.probe(|_| {
-            let normalized_ty = self.infcx.next_ty_infer();
+            let normalized_ty = infcx.next_ty_infer();
             let normalizes_to_goal = goal.with(
                 tcx,
                 ty::Binder::dummy(ty::ProjectionPredicate {
@@ -103,43 +91,31 @@ impl<'a, 'tcx, G: GoalKind<'tcx>> AssemblyCtxt<'a, 'tcx, G> {
                     term: normalized_ty.into(),
                 }),
             );
-            let normalization_certainty =
+            let normalization_certainty = match self.cx.evaluate_goal(normalizes_to_goal) {
-                match self.cx.evaluate_goal(&self.infcx, normalizes_to_goal) {
+                Ok((_, certainty)) => certainty,
-                    Ok((_, certainty)) => certainty,
+                Err(NoSolution) => return,
-                    Err(NoSolution) => return,
+            };
-                };
 
             // NOTE: Alternatively we could call `evaluate_goal` here and only have a `Normalized` candidate.
             // This doesn't work as long as we use `CandidateSource` in both winnowing and to resolve associated items.
             let goal = goal.with(tcx, goal.predicate.with_self_ty(tcx, normalized_ty));
-            let mut orig_values = OriginalQueryValues::default();
-            let goal = self.infcx.canonicalize_query(goal, &mut orig_values);
             let normalized_candidates =
                 AssemblyCtxt::assemble_and_evaluate_candidates(self.cx, goal);
-
+            for mut normalized_candidate in normalized_candidates {
-            // Map each candidate from being canonical wrt the current inference context to being
+                normalized_candidate.result =
-            // canonical wrt the caller.
+                    normalized_candidate.result.unchecked_map(|mut response| {
-            for Candidate { source, result } in normalized_candidates {
+                        response.certainty = response.certainty.unify_and(normalization_certainty);
-                self.infcx.probe(|_| {
+                        response
-                    let candidate_certainty =
+                    });
-                        instantiate_canonical_query_response(&self.infcx, &orig_values, result);
+                self.candidates.push(normalized_candidate);
-
-                    // FIXME: This is a bit scary if the `normalizes_to_goal` overflows.
-                    //
-                    // If we have an ambiguous candidate it hides that normalization
-                    // caused an overflow which may cause issues.
-                    self.try_insert_candidate(
-                        source,
-                        normalization_certainty.unify_and(candidate_certainty),
-                    )
-                })
             }
         })
     }
 
     fn assemble_impl_candidates(&mut self, goal: Goal<'tcx, G>) {
-        self.cx.tcx.for_each_relevant_impl(
+        let tcx = self.cx.tcx();
-            goal.predicate.trait_def_id(self.cx.tcx),
+        tcx.for_each_relevant_impl(
+            goal.predicate.trait_def_id(tcx),
             goal.predicate.self_ty(),
             |impl_def_id| G::consider_impl_candidate(self, goal, impl_def_id),
         );

compiler/rustc_trait_selection/src/solve/cache.rs

@@ -1,267 +0,0 @@
-//! This module both handles the global cache which stores "finished" goals,
-//! and the provisional cache which contains partially computed goals.
-//!
-//! The provisional cache is necessary when dealing with coinductive cycles.
-//!
-//! For more information about the provisional cache and coinduction in general,
-//! check out the relevant section of the rustc-dev-guide.
-//!
-//! FIXME(@lcnr): Write that section, feel free to ping me if you need help here
-//! before then or if I still haven't done that before January 2023.
-use super::overflow::OverflowData;
-use super::{CanonicalGoal, Certainty, MaybeCause, Response};
-use super::{EvalCtxt, QueryResult};
-use rustc_data_structures::fx::FxHashMap;
-use rustc_index::vec::IndexVec;
-use rustc_infer::infer::canonical::{Canonical, CanonicalVarKind, CanonicalVarValues};
-use rustc_middle::ty::{self, TyCtxt};
-use std::collections::hash_map::Entry;
-
-rustc_index::newtype_index! {
-    pub struct StackDepth {}
-}
-rustc_index::newtype_index! {
-    pub struct EntryIndex {}
-}
-
-#[derive(Debug, Clone)]
-struct ProvisionalEntry<'tcx> {
-    // In case we have a coinductive cycle, this is the
-    // the currently least restrictive result of this goal.
-    response: QueryResult<'tcx>,
-    // In case of a cycle, the depth of lowest stack entry involved
-    // in that cycle. This is monotonically decreasing in the stack as all
-    // elements between the current stack element in the lowest stack entry
-    // involved have to also be involved in that cycle.
-    //
-    // We can only move entries to the global cache once we're complete done
-    // with the cycle. If this entry has not been involved in a cycle,
-    // this is just its own depth.
-    depth: StackDepth,
-
-    // The goal for this entry. Should always be equal to the corresponding goal
-    // in the lookup table.
-    goal: CanonicalGoal<'tcx>,
-}
-
-struct StackElem<'tcx> {
-    goal: CanonicalGoal<'tcx>,
-    has_been_used: bool,
-}
-
-pub(super) struct ProvisionalCache<'tcx> {
-    stack: IndexVec<StackDepth, StackElem<'tcx>>,
-    entries: IndexVec<EntryIndex, ProvisionalEntry<'tcx>>,
-    // FIXME: This is only used to quickly check whether a given goal
-    // is in the cache. We should experiment with using something like
-    // `SsoHashSet` here because in most cases there are only a few entries.
-    lookup_table: FxHashMap<CanonicalGoal<'tcx>, EntryIndex>,
-}
-
-impl<'tcx> ProvisionalCache<'tcx> {
-    pub(super) fn empty() -> ProvisionalCache<'tcx> {
-        ProvisionalCache {
-            stack: Default::default(),
-            entries: Default::default(),
-            lookup_table: Default::default(),
-        }
-    }
-
-    pub(super) fn current_depth(&self) -> usize {
-        self.stack.len()
-    }
-}
-
-impl<'tcx> EvalCtxt<'tcx> {
-    /// Tries putting the new goal on the stack, returning an error if it is already cached.
-    ///
-    /// This correctly updates the provisional cache if there is a cycle.
-    pub(super) fn try_push_stack(
-        &mut self,
-        goal: CanonicalGoal<'tcx>,
-    ) -> Result<(), QueryResult<'tcx>> {
-        // FIXME: start by checking the global cache
-
-        // Look at the provisional cache to check for cycles.
-        let cache = &mut self.provisional_cache;
-        match cache.lookup_table.entry(goal) {
-            // No entry, simply push this goal on the stack after dealing with overflow.
-            Entry::Vacant(v) => {
-                if self.overflow_data.has_overflow(cache.stack.len()) {
-                    return Err(self.deal_with_overflow(goal));
-                }
-
-                let depth = cache.stack.push(StackElem { goal, has_been_used: false });
-                let response = response_no_constraints(self.tcx, goal, Certainty::Yes);
-                let entry_index = cache.entries.push(ProvisionalEntry { response, depth, goal });
-                v.insert(entry_index);
-                Ok(())
-            }
-            // We have a nested goal which relies on a goal `root` deeper in the stack.
-            //
-            // We first store that we may have to rerun `evaluate_goal` for `root` in case the
-            // provisional response is not equal to the final response. We also update the depth
-            // of all goals which recursively depend on our current goal to depend on `root`
-            // instead.
-            //
-            // Finally we can return either the provisional response for that goal if we have a
-            // coinductive cycle or an ambiguous result if the cycle is inductive.
-            Entry::Occupied(entry_index) => {
-                let entry_index = *entry_index.get();
-                // FIXME `ProvisionalEntry` should be `Copy`.
-                let entry = cache.entries.get(entry_index).unwrap().clone();
-                cache.stack[entry.depth].has_been_used = true;
-                for provisional_entry in cache.entries.iter_mut().skip(entry_index.index()) {
-                    provisional_entry.depth = provisional_entry.depth.min(entry.depth);
-                }
-
-                // NOTE: The goals on the stack aren't the only goals involved in this cycle.
-                // We can also depend on goals which aren't part of the stack but coinductively
-                // depend on the stack themselves. We already checked whether all the goals
-                // between these goals and their root on the stack. This means that as long as
-                // each goal in a cycle is checked for coinductivity by itself, simply checking
-                // the stack is enough.
-                if cache.stack.raw[entry.depth.index()..]
-                    .iter()
-                    .all(|g| g.goal.value.predicate.is_coinductive(self.tcx))
-                {
-                    Err(entry.response)
-                } else {
-                    Err(response_no_constraints(
-                        self.tcx,
-                        goal,
-                        Certainty::Maybe(MaybeCause::Overflow),
-                    ))
-                }
-            }
-        }
-    }
-
-    /// We cannot simply store the result of [EvalCtxt::compute_goal] as we have to deal with
-    /// coinductive cycles.
-    ///
-    /// When we encounter a coinductive cycle, we have to prove the final result of that cycle
-    /// while we are still computing that result. Because of this we continously recompute the
-    /// cycle until the result of the previous iteration is equal to the final result, at which
-    /// point we are done.
-    ///
-    /// This function returns `true` if we were able to finalize the goal and `false` if it has
-    /// updated the provisional cache and we have to recompute the current goal.
-    ///
-    /// FIXME: Refer to the rustc-dev-guide entry once it exists.
-    pub(super) fn try_finalize_goal(
-        &mut self,
-        actual_goal: CanonicalGoal<'tcx>,
-        response: QueryResult<'tcx>,
-    ) -> bool {
-        let cache = &mut self.provisional_cache;
-        let StackElem { goal, has_been_used } = cache.stack.pop().unwrap();
-        assert_eq!(goal, actual_goal);
-
-        let provisional_entry_index = *cache.lookup_table.get(&goal).unwrap();
-        let provisional_entry = &mut cache.entries[provisional_entry_index];
-        // Was the current goal the root of a cycle and was the provisional response
-        // different from the final one.
-        if has_been_used && provisional_entry.response != response {
-            // If so, update the provisional reponse for this goal...
-            provisional_entry.response = response;
-            // ...remove all entries whose result depends on this goal
-            // from the provisional cache...
-            //
-            // That's not completely correct, as a nested goal can also
-            // depend on a goal which is lower in the stack so it doesn't
-            // actually depend on the current goal. This should be fairly
-            // rare and is hopefully not relevant for performance.
-            #[allow(rustc::potential_query_instability)]
-            cache.lookup_table.retain(|_key, index| *index <= provisional_entry_index);
-            cache.entries.truncate(provisional_entry_index.index() + 1);
-
-            // ...and finally push our goal back on the stack and reevaluate it.
-            cache.stack.push(StackElem { goal, has_been_used: false });
-            false
-        } else {
-            // If not, we're done with this goal.
-            //
-            // Check whether that this goal doesn't depend on a goal deeper on the stack
-            // and if so, move it and all nested goals to the global cache.
-            //
-            // Note that if any nested goal were to depend on something deeper on the stack,
-            // this would have also updated the depth of this goal.
-            if provisional_entry.depth == cache.stack.next_index() {
-                for (i, entry) in cache.entries.drain_enumerated(provisional_entry_index.index()..)
-                {
-                    let actual_index = cache.lookup_table.remove(&entry.goal);
-                    debug_assert_eq!(Some(i), actual_index);
-                    Self::try_move_finished_goal_to_global_cache(
-                        self.tcx,
-                        &mut self.overflow_data,
-                        &cache.stack,
-                        entry.goal,
-                        entry.response,
-                    );
-                }
-            }
-            true
-        }
-    }
-
-    fn try_move_finished_goal_to_global_cache(
-        tcx: TyCtxt<'tcx>,
-        overflow_data: &mut OverflowData,
-        stack: &IndexVec<StackDepth, StackElem<'tcx>>,
-        goal: CanonicalGoal<'tcx>,
-        response: QueryResult<'tcx>,
-    ) {
-        // We move goals to the global cache if we either did not hit an overflow or if it's
-        // the root goal as that will now always hit the same overflow limit.
-        //
-        // NOTE: We cannot move any non-root goals to the global cache even if their final result
-        // isn't impacted by the overflow as that goal still has unstable query dependencies
-        // because it didn't go its full depth.
-        //
-        // FIXME(@lcnr): We could still cache subtrees which are not impacted by overflow though.
-        // Tracking that info correctly isn't trivial, so I haven't implemented it for now.
-        let should_cache_globally = !overflow_data.did_overflow() || stack.is_empty();
-        if should_cache_globally {
-            // FIXME: move the provisional entry to the global cache.
-            let _ = (tcx, goal, response);
-        }
-    }
-}
-
-pub(super) fn response_no_constraints<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    goal: Canonical<'tcx, impl Sized>,
-    certainty: Certainty,
-) -> QueryResult<'tcx> {
-    let var_values = goal
-        .variables
-        .iter()
-        .enumerate()
-        .map(|(i, info)| match info.kind {
-            CanonicalVarKind::Ty(_) | CanonicalVarKind::PlaceholderTy(_) => {
-                tcx.mk_ty(ty::Bound(ty::INNERMOST, ty::BoundVar::from_usize(i).into())).into()
-            }
-            CanonicalVarKind::Region(_) | CanonicalVarKind::PlaceholderRegion(_) => {
-                let br = ty::BoundRegion {
-                    var: ty::BoundVar::from_usize(i),
-                    kind: ty::BrAnon(i as u32, None),
-                };
-                tcx.mk_region(ty::ReLateBound(ty::INNERMOST, br)).into()
-            }
-            CanonicalVarKind::Const(_, ty) | CanonicalVarKind::PlaceholderConst(_, ty) => tcx
-                .mk_const(ty::ConstKind::Bound(ty::INNERMOST, ty::BoundVar::from_usize(i)), ty)
-                .into(),
-        })
-        .collect();
-
-    Ok(Canonical {
-        max_universe: goal.max_universe,
-        variables: goal.variables,
-        value: Response {
-            var_values: CanonicalVarValues { var_values },
-            external_constraints: Default::default(),
-            certainty,
-        },
-    })
-}

compiler/rustc_trait_selection/src/solve/fulfill.rs

@@ -2,7 +2,7 @@ use std::mem;
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_infer::{
-    infer::InferCtxt,
+    infer::{canonical::OriginalQueryValues, InferCtxt},
     traits::{
         query::NoSolution, FulfillmentError, FulfillmentErrorCode, PredicateObligation,
         SelectionError, TraitEngine,
@@ -67,10 +67,26 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentCtxt<'tcx> {
 
             let mut has_changed = false;
             for obligation in mem::take(&mut self.obligations) {
-                let mut cx = EvalCtxt::new(infcx.tcx);
+                let goal = obligation.clone().into();
-                let (changed, certainty) = match cx.evaluate_goal(infcx, obligation.clone().into())
+
-                {
+                // FIXME: Add a better API for that '^^
-                    Ok(result) => result,
+                let mut orig_values = OriginalQueryValues::default();
+                let canonical_goal = infcx.canonicalize_query(goal, &mut orig_values);
+                let (changed, certainty) = match EvalCtxt::evaluate_canonical_goal(
+                    infcx.tcx,
+                    &mut super::search_graph::SearchGraph::new(infcx.tcx),
+                    canonical_goal,
+                ) {
+                    Ok(canonical_response) => {
+                        (
+                            true, // FIXME: check whether `var_values` are an identity substitution.
+                            super::instantiate_canonical_query_response(
+                                infcx,
+                                &orig_values,
+                                canonical_response,
+                            ),
+                        )
+                    }
                     Err(NoSolution) => {
                         errors.push(FulfillmentError {
                             obligation: obligation.clone(),

compiler/rustc_trait_selection/src/solve/infcx_ext.rs

@@ -1,23 +1,11 @@
-use rustc_infer::infer::canonical::CanonicalVarValues;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use rustc_infer::infer::InferCtxt;
-use rustc_infer::traits::query::NoSolution;
 use rustc_middle::ty::Ty;
 use rustc_span::DUMMY_SP;
 
-use crate::solve::ExternalConstraints;
-
-use super::{Certainty, QueryResult, Response};
-
 /// Methods used inside of the canonical queries of the solver.
 pub(super) trait InferCtxtExt<'tcx> {
     fn next_ty_infer(&self) -> Ty<'tcx>;
-
-    fn make_canonical_response(
-        &self,
-        var_values: CanonicalVarValues<'tcx>,
-        certainty: Certainty,
-    ) -> QueryResult<'tcx>;
 }
 
 impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
@@ -27,29 +15,4 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
             span: DUMMY_SP,
         })
     }
-
-    fn make_canonical_response(
-        &self,
-        var_values: CanonicalVarValues<'tcx>,
-        certainty: Certainty,
-    ) -> QueryResult<'tcx> {
-        let external_constraints = take_external_constraints(self)?;
-
-        Ok(self.canonicalize_response(Response { var_values, external_constraints, certainty }))
-    }
-}
-
-#[instrument(level = "debug", skip(infcx), ret)]
-fn take_external_constraints<'tcx>(
-    infcx: &InferCtxt<'tcx>,
-) -> Result<ExternalConstraints<'tcx>, NoSolution> {
-    let region_obligations = infcx.take_registered_region_obligations();
-    let opaque_types = infcx.take_opaque_types_for_query_response();
-    Ok(ExternalConstraints {
-        // FIXME: Now that's definitely wrong :)
-        //
-        // Should also do the leak check here I think
-        regions: drop(region_obligations),
-        opaque_types,
-    })
 }

compiler/rustc_trait_selection/src/solve/mod.rs

@@ -19,27 +19,23 @@
 
 use std::mem;
 
+use rustc_infer::infer::canonical::{Canonical, CanonicalVarKind, CanonicalVarValues};
 use rustc_infer::infer::canonical::{OriginalQueryValues, QueryRegionConstraints, QueryResponse};
 use rustc_infer::infer::{InferCtxt, InferOk, TyCtxtInferExt};
 use rustc_infer::traits::query::NoSolution;
 use rustc_infer::traits::Obligation;
 use rustc_middle::infer::canonical::Certainty as OldCertainty;
-use rustc_middle::infer::canonical::{Canonical, CanonicalVarValues};
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::ty::{RegionOutlivesPredicate, ToPredicate, TypeOutlivesPredicate};
 use rustc_span::DUMMY_SP;
 
 use crate::traits::ObligationCause;
 
-use self::cache::response_no_constraints;
-use self::infcx_ext::InferCtxtExt;
-
 mod assembly;
-mod cache;
 mod fulfill;
 mod infcx_ext;
-mod overflow;
 mod project_goals;
+mod search_graph;
 mod trait_goals;
 
 pub use fulfill::FulfillmentCtxt;
@@ -146,45 +142,25 @@ pub trait TyCtxtExt<'tcx> {
 
 impl<'tcx> TyCtxtExt<'tcx> for TyCtxt<'tcx> {
     fn evaluate_goal(self, goal: CanonicalGoal<'tcx>) -> QueryResult<'tcx> {
-        let mut cx = EvalCtxt::new(self);
+        let mut search_graph = search_graph::SearchGraph::new(self);
-        cx.evaluate_canonical_goal(goal)
+        EvalCtxt::evaluate_canonical_goal(self, &mut search_graph, goal)
     }
 }
 
-struct EvalCtxt<'tcx> {
+struct EvalCtxt<'a, 'tcx> {
-    tcx: TyCtxt<'tcx>,
+    infcx: &'a InferCtxt<'tcx>,
+    var_values: CanonicalVarValues<'tcx>,
 
-    provisional_cache: cache::ProvisionalCache<'tcx>,
+    search_graph: &'a mut search_graph::SearchGraph<'tcx>,
-    overflow_data: overflow::OverflowData,
 }
 
-impl<'tcx> EvalCtxt<'tcx> {
+impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
-    fn new(tcx: TyCtxt<'tcx>) -> EvalCtxt<'tcx> {
+    fn evaluate_canonical_goal(
-        EvalCtxt {
+        tcx: TyCtxt<'tcx>,
-            tcx,
+        search_graph: &'a mut search_graph::SearchGraph<'tcx>,
-            provisional_cache: cache::ProvisionalCache::empty(),
+        canonical_goal: CanonicalGoal<'tcx>,
-            overflow_data: overflow::OverflowData::new(tcx),
+    ) -> QueryResult<'tcx> {
-        }
+        match search_graph.try_push_stack(tcx, canonical_goal) {
-    }
-
-    /// Recursively evaluates `goal`, returning whether any inference vars have
-    /// been constrained and the certainty of the result.
-    fn evaluate_goal(
-        &mut self,
-        infcx: &InferCtxt<'tcx>,
-        goal: Goal<'tcx, ty::Predicate<'tcx>>,
-    ) -> Result<(bool, Certainty), NoSolution> {
-        let mut orig_values = OriginalQueryValues::default();
-        let canonical_goal = infcx.canonicalize_query(goal, &mut orig_values);
-        let canonical_response = self.evaluate_canonical_goal(canonical_goal)?;
-        Ok((
-            !canonical_response.value.var_values.is_identity(),
-            instantiate_canonical_query_response(infcx, &orig_values, canonical_response),
-        ))
-    }
-
-    fn evaluate_canonical_goal(&mut self, goal: CanonicalGoal<'tcx>) -> QueryResult<'tcx> {
-        match self.try_push_stack(goal) {
             Ok(()) => {}
             // Our goal is already on the stack, eager return.
             Err(response) => return response,
@@ -195,41 +171,65 @@ impl<'tcx> EvalCtxt<'tcx> {
         //
         // FIXME: Similar to `evaluate_all`, this has to check for overflow.
         loop {
-            let result = self.compute_goal(goal);
+            let (ref infcx, goal, var_values) =
+                tcx.infer_ctxt().build_with_canonical(DUMMY_SP, &canonical_goal);
+            let mut ecx = EvalCtxt { infcx, var_values, search_graph };
+            let result = ecx.compute_goal(goal);
 
             // FIXME: `Response` should be `Copy`
-            if self.try_finalize_goal(goal, result.clone()) {
+            if search_graph.try_finalize_goal(tcx, canonical_goal, result.clone()) {
                 return result;
             }
         }
     }
 
-    fn compute_goal(&mut self, canonical_goal: CanonicalGoal<'tcx>) -> QueryResult<'tcx> {
+    fn tcx(&self) -> TyCtxt<'tcx> {
-        // WARNING: We're looking at a canonical value without instantiating it here.
+        self.infcx.tcx
-        //
+    }
-        // We have to be incredibly careful to not change the order of bound variables or
-        // remove any. As we go from `Goal<'tcx, Predicate>` to `Goal` with the variants
-        // of `PredicateKind` this is the case and it is and faster than instantiating and
-        // recanonicalizing.
-        let Goal { param_env, predicate } = canonical_goal.value;
 
-        if let Some(kind) = predicate.kind().no_bound_vars_ignoring_escaping(self.tcx) {
+    fn make_canonical_response(&self, certainty: Certainty) -> QueryResult<'tcx> {
+        let external_constraints = take_external_constraints(self.infcx)?;
+
+        Ok(self.infcx.canonicalize_response(Response {
+            var_values: self.var_values.clone(),
+            external_constraints,
+            certainty,
+        }))
+    }
+
+    /// Recursively evaluates `goal`, returning whether any inference vars have
+    /// been constrained and the certainty of the result.
+    fn evaluate_goal(
+        &mut self,
+        goal: Goal<'tcx, ty::Predicate<'tcx>>,
+    ) -> Result<(bool, Certainty), NoSolution> {
+        let mut orig_values = OriginalQueryValues::default();
+        let canonical_goal = self.infcx.canonicalize_query(goal, &mut orig_values);
+        let canonical_response =
+            EvalCtxt::evaluate_canonical_goal(self.tcx(), self.search_graph, canonical_goal)?;
+        Ok((
+            !canonical_response.value.var_values.is_identity(),
+            instantiate_canonical_query_response(self.infcx, &orig_values, canonical_response),
+        ))
+    }
+
+    fn compute_goal(&mut self, goal: Goal<'tcx, ty::Predicate<'tcx>>) -> QueryResult<'tcx> {
+        let Goal { param_env, predicate } = goal;
+        let kind = predicate.kind();
+        if let Some(kind) = kind.no_bound_vars() {
             match kind {
-                ty::PredicateKind::Clause(ty::Clause::Trait(predicate)) => self.compute_trait_goal(
+                ty::PredicateKind::Clause(ty::Clause::Trait(predicate)) => {
-                    canonical_goal.unchecked_rebind(Goal { param_env, predicate }),
+                    self.compute_trait_goal(Goal { param_env, predicate })
-                ),
+                }
-                ty::PredicateKind::Clause(ty::Clause::Projection(predicate)) => self
+                ty::PredicateKind::Clause(ty::Clause::Projection(predicate)) => {
-                    .compute_projection_goal(
+                    self.compute_projection_goal(Goal { param_env, predicate })
-                        canonical_goal.unchecked_rebind(Goal { param_env, predicate }),
+                }
-                    ),
+                ty::PredicateKind::Clause(ty::Clause::TypeOutlives(predicate)) => {
-                ty::PredicateKind::Clause(ty::Clause::TypeOutlives(predicate)) => self
+                    self.compute_type_outlives_goal(Goal { param_env, predicate })
-                    .compute_type_outlives_goal(
+                }
-                        canonical_goal.unchecked_rebind(Goal { param_env, predicate }),
+                ty::PredicateKind::Clause(ty::Clause::RegionOutlives(predicate)) => {
-                    ),
+                    self.compute_region_outlives_goal(Goal { param_env, predicate })
-                ty::PredicateKind::Clause(ty::Clause::RegionOutlives(predicate)) => self
+                }
-                    .compute_region_outlives_goal(
-                        canonical_goal.unchecked_rebind(Goal { param_env, predicate }),
-                    ),
                 // FIXME: implement these predicates :)
                 ty::PredicateKind::WellFormed(_)
                 | ty::PredicateKind::ObjectSafe(_)
@@ -239,49 +239,41 @@ impl<'tcx> EvalCtxt<'tcx> {
                 | ty::PredicateKind::ConstEvaluatable(_)
                 | ty::PredicateKind::ConstEquate(_, _)
                 | ty::PredicateKind::TypeWellFormedFromEnv(_)
-                | ty::PredicateKind::Ambiguous => {
+                | ty::PredicateKind::Ambiguous => self.make_canonical_response(Certainty::Yes),
-                    // FIXME
-                    response_no_constraints(self.tcx, canonical_goal, Certainty::Yes)
-                }
             }
         } else {
-            let (infcx, goal, var_values) =
+            let kind = self.infcx.replace_bound_vars_with_placeholders(kind);
-                self.tcx.infer_ctxt().build_with_canonical(DUMMY_SP, &canonical_goal);
+            let goal = goal.with(self.tcx(), ty::Binder::dummy(kind));
-            let kind = infcx.replace_bound_vars_with_placeholders(goal.predicate.kind());
+            let (_, certainty) = self.evaluate_goal(goal)?;
-            let goal = goal.with(self.tcx, ty::Binder::dummy(kind));
+            self.make_canonical_response(certainty)
-            let (_, certainty) = self.evaluate_goal(&infcx, goal)?;
-            infcx.make_canonical_response(var_values, certainty)
         }
     }
 
     fn compute_type_outlives_goal(
         &mut self,
-        goal: CanonicalGoal<'tcx, TypeOutlivesPredicate<'tcx>>,
+        _goal: Goal<'tcx, TypeOutlivesPredicate<'tcx>>,
     ) -> QueryResult<'tcx> {
-        // FIXME
+        self.make_canonical_response(Certainty::Yes)
-        response_no_constraints(self.tcx, goal, Certainty::Yes)
     }
 
     fn compute_region_outlives_goal(
         &mut self,
-        goal: CanonicalGoal<'tcx, RegionOutlivesPredicate<'tcx>>,
+        _goal: Goal<'tcx, RegionOutlivesPredicate<'tcx>>,
     ) -> QueryResult<'tcx> {
-        // FIXME
+        self.make_canonical_response(Certainty::Yes)
-        response_no_constraints(self.tcx, goal, Certainty::Yes)
     }
 }
 
-impl<'tcx> EvalCtxt<'tcx> {
+impl<'tcx> EvalCtxt<'_, 'tcx> {
     fn evaluate_all(
         &mut self,
-        infcx: &InferCtxt<'tcx>,
         mut goals: Vec<Goal<'tcx, ty::Predicate<'tcx>>>,
     ) -> Result<Certainty, NoSolution> {
         let mut new_goals = Vec::new();
         self.repeat_while_none(|this| {
             let mut has_changed = Err(Certainty::Yes);
             for goal in goals.drain(..) {
-                let (changed, certainty) = match this.evaluate_goal(infcx, goal) {
+                let (changed, certainty) = match this.evaluate_goal(goal) {
                     Ok(result) => result,
                     Err(NoSolution) => return Some(Err(NoSolution)),
                 };
@@ -310,6 +302,21 @@ impl<'tcx> EvalCtxt<'tcx> {
     }
 }
 
+#[instrument(level = "debug", skip(infcx), ret)]
+fn take_external_constraints<'tcx>(
+    infcx: &InferCtxt<'tcx>,
+) -> Result<ExternalConstraints<'tcx>, NoSolution> {
+    let region_obligations = infcx.take_registered_region_obligations();
+    let opaque_types = infcx.take_opaque_types_for_query_response();
+    Ok(ExternalConstraints {
+        // FIXME: Now that's definitely wrong :)
+        //
+        // Should also do the leak check here I think
+        regions: drop(region_obligations),
+        opaque_types,
+    })
+}
+
 fn instantiate_canonical_query_response<'tcx>(
     infcx: &InferCtxt<'tcx>,
     original_values: &OriginalQueryValues<'tcx>,
@@ -334,3 +341,40 @@ fn instantiate_canonical_query_response<'tcx>(
     assert!(obligations.is_empty());
     value
 }
+
+pub(super) fn response_no_constraints<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    goal: Canonical<'tcx, impl Sized>,
+    certainty: Certainty,
+) -> QueryResult<'tcx> {
+    let var_values = goal
+        .variables
+        .iter()
+        .enumerate()
+        .map(|(i, info)| match info.kind {
+            CanonicalVarKind::Ty(_) | CanonicalVarKind::PlaceholderTy(_) => {
+                tcx.mk_ty(ty::Bound(ty::INNERMOST, ty::BoundVar::from_usize(i).into())).into()
+            }
+            CanonicalVarKind::Region(_) | CanonicalVarKind::PlaceholderRegion(_) => {
+                let br = ty::BoundRegion {
+                    var: ty::BoundVar::from_usize(i),
+                    kind: ty::BrAnon(i as u32, None),
+                };
+                tcx.mk_region(ty::ReLateBound(ty::INNERMOST, br)).into()
+            }
+            CanonicalVarKind::Const(_, ty) | CanonicalVarKind::PlaceholderConst(_, ty) => tcx
+                .mk_const(ty::ConstKind::Bound(ty::INNERMOST, ty::BoundVar::from_usize(i)), ty)
+                .into(),
+        })
+        .collect();
+
+    Ok(Canonical {
+        max_universe: goal.max_universe,
+        variables: goal.variables,
+        value: Response {
+            var_values: CanonicalVarValues { var_values },
+            external_constraints: Default::default(),
+            certainty,
+        },
+    })
+}

compiler/rustc_trait_selection/src/solve/project_goals.rs

@@ -1,7 +1,7 @@
 use crate::traits::{specialization_graph, translate_substs};
 
 use super::assembly::{self, AssemblyCtxt};
-use super::{CanonicalGoal, EvalCtxt, Goal, QueryResult};
+use super::{EvalCtxt, Goal, QueryResult};
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
@@ -26,10 +26,10 @@ pub(super) enum CandidateSource {
 
 type Candidate<'tcx> = assembly::Candidate<'tcx, ProjectionPredicate<'tcx>>;
 
-impl<'tcx> EvalCtxt<'tcx> {
+impl<'tcx> EvalCtxt<'_, 'tcx> {
     pub(super) fn compute_projection_goal(
         &mut self,
-        goal: CanonicalGoal<'tcx, ProjectionPredicate<'tcx>>,
+        goal: Goal<'tcx, ProjectionPredicate<'tcx>>,
     ) -> QueryResult<'tcx> {
         let candidates = AssemblyCtxt::assemble_and_evaluate_candidates(self, goal);
         self.merge_project_candidates(candidates)
@@ -104,11 +104,13 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
     }
 
     fn consider_impl_candidate(
-        acx: &mut AssemblyCtxt<'_, 'tcx, ProjectionPredicate<'tcx>>,
+        acx: &mut AssemblyCtxt<'_, '_, 'tcx, ProjectionPredicate<'tcx>>,
         goal: Goal<'tcx, ProjectionPredicate<'tcx>>,
         impl_def_id: DefId,
     ) {
-        let tcx = acx.cx.tcx;
+        let tcx = acx.cx.tcx();
+        let infcx = acx.cx.infcx;
+
         let goal_trait_ref = goal.predicate.projection_ty.trait_ref(tcx);
         let impl_trait_ref = tcx.impl_trait_ref(impl_def_id).unwrap();
         let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::AsPlaceholder };
@@ -118,12 +120,11 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
             return;
         }
 
-        acx.infcx.probe(|_| {
+        infcx.probe(|_| {
-            let impl_substs = acx.infcx.fresh_substs_for_item(DUMMY_SP, impl_def_id);
+            let impl_substs = infcx.fresh_substs_for_item(DUMMY_SP, impl_def_id);
             let impl_trait_ref = impl_trait_ref.subst(tcx, impl_substs);
 
-            let Ok(InferOk { obligations, .. }) = acx
+            let Ok(InferOk { obligations, .. }) = infcx
-                .infcx
                 .at(&ObligationCause::dummy(), goal.param_env)
                 .define_opaque_types(false)
                 .eq(goal_trait_ref, impl_trait_ref)
@@ -138,11 +139,12 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                 .into_iter()
                 .map(|pred| goal.with(tcx, pred));
 
-            let nested_goals = obligations.into_iter().map(|o| o.into()).chain(where_clause_bounds).collect();
+            let nested_goals =
-            let Ok(trait_ref_certainty) = acx.cx.evaluate_all(acx.infcx, nested_goals) else { return };
+                obligations.into_iter().map(|o| o.into()).chain(where_clause_bounds).collect();
+            let Ok(trait_ref_certainty) = acx.cx.evaluate_all(nested_goals) else { return };
 
             let Some(assoc_def) = fetch_eligible_assoc_item_def(
-                acx.infcx,
+                infcx,
                 goal.param_env,
                 goal_trait_ref,
                 goal.predicate.def_id(),
@@ -174,7 +176,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                 impl_substs,
             );
             let substs = translate_substs(
-                acx.infcx,
+                infcx,
                 goal.param_env,
                 impl_def_id,
                 impl_substs_with_gat,
@@ -185,7 +187,8 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
             let is_const = matches!(tcx.def_kind(assoc_def.item.def_id), DefKind::AssocConst);
             let ty = tcx.bound_type_of(assoc_def.item.def_id);
             let term: ty::EarlyBinder<ty::Term<'tcx>> = if is_const {
-                let identity_substs = ty::InternalSubsts::identity_for_item(tcx, assoc_def.item.def_id);
+                let identity_substs =
+                    ty::InternalSubsts::identity_for_item(tcx, assoc_def.item.def_id);
                 let did = ty::WithOptConstParam::unknown(assoc_def.item.def_id);
                 let kind =
                     ty::ConstKind::Unevaluated(ty::UnevaluatedConst::new(did, identity_substs));
@@ -194,8 +197,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                 ty.map_bound(|ty| ty.into())
             };
 
-            let Ok(InferOk { obligations, .. }) = acx
+            let Ok(InferOk { obligations, .. }) = infcx
-                .infcx
                 .at(&ObligationCause::dummy(), goal.param_env)
                 .define_opaque_types(false)
                 .eq(goal.predicate.term,  term.subst(tcx, substs))
@@ -205,7 +207,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
             };
 
             let nested_goals = obligations.into_iter().map(|o| o.into()).collect();
-            let Ok(rhs_certainty) = acx.cx.evaluate_all(acx.infcx, nested_goals) else { return };
+            let Ok(rhs_certainty) = acx.cx.evaluate_all(nested_goals) else { return };
 
             let certainty = trait_ref_certainty.unify_and(rhs_certainty);
             acx.try_insert_candidate(CandidateSource::Impl(impl_def_id), certainty);

compiler/rustc_trait_selection/src/solve/search_graph/cache.rs

@@ -0,0 +1,115 @@
+//! This module both handles the global cache which stores "finished" goals,
+//! and the provisional cache which contains partially computed goals.
+//!
+//! The provisional cache is necessary when dealing with coinductive cycles.
+//!
+//! For more information about the provisional cache and coinduction in general,
+//! check out the relevant section of the rustc-dev-guide.
+//!
+//! FIXME(@lcnr): Write that section, feel free to ping me if you need help here
+//! before then or if I still haven't done that before January 2023.
+use super::overflow::OverflowData;
+use super::StackDepth;
+use crate::solve::{CanonicalGoal, QueryResult};
+use rustc_data_structures::fx::FxHashMap;
+use rustc_index::vec::IndexVec;
+use rustc_middle::ty::TyCtxt;
+
+rustc_index::newtype_index! {
+    pub struct EntryIndex {}
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct ProvisionalEntry<'tcx> {
+    // In case we have a coinductive cycle, this is the
+    // the currently least restrictive result of this goal.
+    pub(super) response: QueryResult<'tcx>,
+    // In case of a cycle, the position of deepest stack entry involved
+    // in that cycle. This is monotonically decreasing in the stack as all
+    // elements between the current stack element in the deepest stack entry
+    // involved have to also be involved in that cycle.
+    //
+    // We can only move entries to the global cache once we're complete done
+    // with the cycle. If this entry has not been involved in a cycle,
+    // this is just its own depth.
+    pub(super) depth: StackDepth,
+
+    // The goal for this entry. Should always be equal to the corresponding goal
+    // in the lookup table.
+    pub(super) goal: CanonicalGoal<'tcx>,
+}
+
+pub(super) struct ProvisionalCache<'tcx> {
+    pub(super) entries: IndexVec<EntryIndex, ProvisionalEntry<'tcx>>,
+    // FIXME: This is only used to quickly check whether a given goal
+    // is in the cache. We should experiment with using something like
+    // `SsoHashSet` here because in most cases there are only a few entries.
+    pub(super) lookup_table: FxHashMap<CanonicalGoal<'tcx>, EntryIndex>,
+}
+
+impl<'tcx> ProvisionalCache<'tcx> {
+    pub(super) fn empty() -> ProvisionalCache<'tcx> {
+        ProvisionalCache { entries: Default::default(), lookup_table: Default::default() }
+    }
+
+    /// Adds a dependency from the current leaf to `target` in the cache
+    /// to prevent us from moving any goals which depend on the current leaf
+    /// to the global cache while we're still computing `target`.
+    pub(super) fn add_dependency_of_leaf_on(&mut self, target: EntryIndex) {
+        let depth = self.entries[target].depth;
+        for provisional_entry in &mut self.entries.raw[target.index()..] {
+            // The depth of `target` is the position of the deepest goal in the stack
+            // on which `target` depends. That goal is the `root` of this cycle.
+            //
+            // Any entry which was added after `target` is either on the stack itself
+            // at which point its depth is definitely at least as high as the depth of
+            // `root`. If it's not on the stack itself it has to depend on a goal
+            // between `root` and `leaf`. If it were to depend on a goal deeper in the
+            // stack than `root`, then `root` would also depend on that goal, at which
+            // point `root` wouldn't be the root anymore.
+            debug_assert!(provisional_entry.depth >= depth);
+            provisional_entry.depth = depth;
+        }
+
+        // We only update entries which were added after `target` as no other
+        // entry should have a higher depth.
+        //
+        // Any entry which previously had a higher depth than target has to
+        // be between `target` and `root`. Because of this we would have updated
+        // its depth when calling `add_dependency_of_leaf_on(root)` for `target`.
+        if cfg!(debug_assertions) {
+            self.entries.iter().all(|e| e.depth <= depth);
+        }
+    }
+
+    pub(super) fn depth(&self, entry_index: EntryIndex) -> StackDepth {
+        self.entries[entry_index].depth
+    }
+
+    pub(super) fn provisional_result(&self, entry_index: EntryIndex) -> QueryResult<'tcx> {
+        self.entries[entry_index].response.clone()
+    }
+}
+
+pub(super) fn try_move_finished_goal_to_global_cache<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    overflow_data: &mut OverflowData,
+    stack: &IndexVec<super::StackDepth, super::StackElem<'tcx>>,
+    goal: CanonicalGoal<'tcx>,
+    response: QueryResult<'tcx>,
+) {
+    // We move goals to the global cache if we either did not hit an overflow or if it's
+    // the root goal as that will now always hit the same overflow limit.
+    //
+    // NOTE: We cannot move any non-root goals to the global cache even if their final result
+    // isn't impacted by the overflow as that goal still has unstable query dependencies
+    // because it didn't go its full depth.
+    //
+    // FIXME(@lcnr): We could still cache subtrees which are not impacted by overflow though.
+    // Tracking that info correctly isn't trivial, so I haven't implemented it for now.
+    let should_cache_globally = !overflow_data.did_overflow() || stack.is_empty();
+    if should_cache_globally {
+        // FIXME: move the provisional entry to the global cache.
+        let _ = (tcx, goal, response);
+    }
+}

compiler/rustc_trait_selection/src/solve/search_graph/mod.rs

@@ -0,0 +1,172 @@
+mod cache;
+mod overflow;
+
+use self::cache::ProvisionalEntry;
+use super::{CanonicalGoal, Certainty, MaybeCause, QueryResult};
+use cache::ProvisionalCache;
+use overflow::OverflowData;
+use rustc_index::vec::IndexVec;
+use rustc_middle::ty::TyCtxt;
+use std::collections::hash_map::Entry;
+
+rustc_index::newtype_index! {
+    pub struct StackDepth {}
+}
+
+struct StackElem<'tcx> {
+    goal: CanonicalGoal<'tcx>,
+    has_been_used: bool,
+}
+
+pub(super) struct SearchGraph<'tcx> {
+    /// The stack of goals currently being computed.
+    ///
+    /// An element is *deeper* in the stack if its index is *lower*.
+    stack: IndexVec<StackDepth, StackElem<'tcx>>,
+    overflow_data: OverflowData,
+    provisional_cache: ProvisionalCache<'tcx>,
+}
+
+impl<'tcx> SearchGraph<'tcx> {
+    pub(super) fn new(tcx: TyCtxt<'tcx>) -> SearchGraph<'tcx> {
+        Self {
+            stack: Default::default(),
+            overflow_data: OverflowData::new(tcx),
+            provisional_cache: ProvisionalCache::empty(),
+        }
+    }
+
+    /// Tries putting the new goal on the stack, returning an error if it is already cached.
+    ///
+    /// This correctly updates the provisional cache if there is a cycle.
+    pub(super) fn try_push_stack(
+        &mut self,
+        tcx: TyCtxt<'tcx>,
+        goal: CanonicalGoal<'tcx>,
+    ) -> Result<(), QueryResult<'tcx>> {
+        // FIXME: start by checking the global cache
+
+        // Look at the provisional cache to check for cycles.
+        let cache = &mut self.provisional_cache;
+        match cache.lookup_table.entry(goal) {
+            // No entry, simply push this goal on the stack after dealing with overflow.
+            Entry::Vacant(v) => {
+                if self.overflow_data.has_overflow(self.stack.len()) {
+                    return Err(self.deal_with_overflow(tcx, goal));
+                }
+
+                let depth = self.stack.push(StackElem { goal, has_been_used: false });
+                let response = super::response_no_constraints(tcx, goal, Certainty::Yes);
+                let entry_index = cache.entries.push(ProvisionalEntry { response, depth, goal });
+                v.insert(entry_index);
+                Ok(())
+            }
+            // We have a nested goal which relies on a goal `root` deeper in the stack.
+            //
+            // We first store that we may have to rerun `evaluate_goal` for `root` in case the
+            // provisional response is not equal to the final response. We also update the depth
+            // of all goals which recursively depend on our current goal to depend on `root`
+            // instead.
+            //
+            // Finally we can return either the provisional response for that goal if we have a
+            // coinductive cycle or an ambiguous result if the cycle is inductive.
+            Entry::Occupied(entry_index) => {
+                let entry_index = *entry_index.get();
+
+                cache.add_dependency_of_leaf_on(entry_index);
+                let stack_depth = cache.depth(entry_index);
+
+                self.stack[stack_depth].has_been_used = true;
+                // NOTE: The goals on the stack aren't the only goals involved in this cycle.
+                // We can also depend on goals which aren't part of the stack but coinductively
+                // depend on the stack themselves. We already checked whether all the goals
+                // between these goals and their root on the stack. This means that as long as
+                // each goal in a cycle is checked for coinductivity by itself, simply checking
+                // the stack is enough.
+                if self.stack.raw[stack_depth.index()..]
+                    .iter()
+                    .all(|g| g.goal.value.predicate.is_coinductive(tcx))
+                {
+                    Err(cache.provisional_result(entry_index))
+                } else {
+                    Err(super::response_no_constraints(
+                        tcx,
+                        goal,
+                        Certainty::Maybe(MaybeCause::Overflow),
+                    ))
+                }
+            }
+        }
+    }
+
+    /// We cannot simply store the result of [EvalCtxt::compute_goal] as we have to deal with
+    /// coinductive cycles.
+    ///
+    /// When we encounter a coinductive cycle, we have to prove the final result of that cycle
+    /// while we are still computing that result. Because of this we continously recompute the
+    /// cycle until the result of the previous iteration is equal to the final result, at which
+    /// point we are done.
+    ///
+    /// This function returns `true` if we were able to finalize the goal and `false` if it has
+    /// updated the provisional cache and we have to recompute the current goal.
+    ///
+    /// FIXME: Refer to the rustc-dev-guide entry once it exists.
+    pub(super) fn try_finalize_goal(
+        &mut self,
+        tcx: TyCtxt<'tcx>,
+        actual_goal: CanonicalGoal<'tcx>,
+        response: QueryResult<'tcx>,
+    ) -> bool {
+        let StackElem { goal, has_been_used } = self.stack.pop().unwrap();
+        assert_eq!(goal, actual_goal);
+
+        let cache = &mut self.provisional_cache;
+        let provisional_entry_index = *cache.lookup_table.get(&goal).unwrap();
+        let provisional_entry = &mut cache.entries[provisional_entry_index];
+        let depth = provisional_entry.depth;
+        // Was the current goal the root of a cycle and was the provisional response
+        // different from the final one.
+        if has_been_used && provisional_entry.response != response {
+            // If so, update the provisional reponse for this goal...
+            provisional_entry.response = response;
+            // ...remove all entries whose result depends on this goal
+            // from the provisional cache...
+            //
+            // That's not completely correct, as a nested goal can also
+            // depend on a goal which is lower in the stack so it doesn't
+            // actually depend on the current goal. This should be fairly
+            // rare and is hopefully not relevant for performance.
+            #[allow(rustc::potential_query_instability)]
+            cache.lookup_table.retain(|_key, index| *index <= provisional_entry_index);
+            cache.entries.truncate(provisional_entry_index.index() + 1);
+
+            // ...and finally push our goal back on the stack and reevaluate it.
+            self.stack.push(StackElem { goal, has_been_used: false });
+            false
+        } else {
+            // If not, we're done with this goal.
+            //
+            // Check whether that this goal doesn't depend on a goal deeper on the stack
+            // and if so, move it and all nested goals to the global cache.
+            //
+            // Note that if any nested goal were to depend on something deeper on the stack,
+            // this would have also updated the depth of the current goal.
+            if depth == self.stack.next_index() {
+                for (i, entry) in cache.entries.drain_enumerated(provisional_entry_index.index()..)
+                {
+                    let actual_index = cache.lookup_table.remove(&entry.goal);
+                    debug_assert_eq!(Some(i), actual_index);
+                    debug_assert!(entry.depth == depth);
+                    cache::try_move_finished_goal_to_global_cache(
+                        tcx,
+                        &mut self.overflow_data,
+                        &self.stack,
+                        entry.goal,
+                        entry.response,
+                    );
+                }
+            }
+            true
+        }
+    }
+}

compiler/rustc_trait_selection/src/solve/search_graph/overflow.rs

@@ -3,8 +3,8 @@ use rustc_infer::traits::query::NoSolution;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::Limit;
 
-use super::cache::response_no_constraints;
+use super::SearchGraph;
-use super::{Certainty, EvalCtxt, MaybeCause, QueryResult};
+use crate::solve::{response_no_constraints, Certainty, EvalCtxt, MaybeCause, QueryResult};
 
 /// When detecting a solver overflow, we return ambiguity. Overflow can be
 /// *hidden* by either a fatal error in an **AND** or a trivial success in an **OR**.
@@ -50,32 +50,35 @@ impl OverflowData {
     }
 }
 
-impl<'tcx> EvalCtxt<'tcx> {
+impl<'tcx> SearchGraph<'tcx> {
-    pub(super) fn deal_with_overflow(
+    pub fn deal_with_overflow(
         &mut self,
+        tcx: TyCtxt<'tcx>,
         goal: Canonical<'tcx, impl Sized>,
     ) -> QueryResult<'tcx> {
         self.overflow_data.deal_with_overflow();
-        response_no_constraints(self.tcx, goal, Certainty::Maybe(MaybeCause::Overflow))
+        response_no_constraints(tcx, goal, Certainty::Maybe(MaybeCause::Overflow))
     }
+}
 
+impl<'tcx> EvalCtxt<'_, 'tcx> {
     /// A `while`-loop which tracks overflow.
-    pub(super) fn repeat_while_none(
+    pub fn repeat_while_none(
         &mut self,
         mut loop_body: impl FnMut(&mut Self) -> Option<Result<Certainty, NoSolution>>,
     ) -> Result<Certainty, NoSolution> {
-        let start_depth = self.overflow_data.additional_depth;
+        let start_depth = self.search_graph.overflow_data.additional_depth;
-        let depth = self.provisional_cache.current_depth();
+        let depth = self.search_graph.stack.len();
-        while !self.overflow_data.has_overflow(depth) {
+        while !self.search_graph.overflow_data.has_overflow(depth) {
             if let Some(result) = loop_body(self) {
-                self.overflow_data.additional_depth = start_depth;
+                self.search_graph.overflow_data.additional_depth = start_depth;
                 return result;
             }
 
-            self.overflow_data.additional_depth += 1;
+            self.search_graph.overflow_data.additional_depth += 1;
         }
-        self.overflow_data.additional_depth = start_depth;
+        self.search_graph.overflow_data.additional_depth = start_depth;
-        self.overflow_data.deal_with_overflow();
+        self.search_graph.overflow_data.deal_with_overflow();
         Ok(Certainty::Maybe(MaybeCause::Overflow))
     }
 }

compiler/rustc_trait_selection/src/solve/trait_goals.rs

@@ -3,7 +3,7 @@
 use std::iter;
 
 use super::assembly::{self, AssemblyCtxt};
-use super::{CanonicalGoal, EvalCtxt, Goal, QueryResult};
+use super::{EvalCtxt, Goal, QueryResult};
 use rustc_hir::def_id::DefId;
 use rustc_infer::infer::InferOk;
 use rustc_infer::traits::query::NoSolution;
@@ -67,11 +67,12 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
     }
 
     fn consider_impl_candidate(
-        acx: &mut AssemblyCtxt<'_, 'tcx, Self>,
+        acx: &mut AssemblyCtxt<'_, '_, 'tcx, Self>,
         goal: Goal<'tcx, TraitPredicate<'tcx>>,
         impl_def_id: DefId,
     ) {
-        let tcx = acx.cx.tcx;
+        let tcx = acx.cx.tcx();
+        let infcx = acx.cx.infcx;
 
         let impl_trait_ref = tcx.impl_trait_ref(impl_def_id).unwrap();
         let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::AsPlaceholder };
@@ -81,12 +82,11 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
             return;
         }
 
-        acx.infcx.probe(|_| {
+        infcx.probe(|_| {
-            let impl_substs = acx.infcx.fresh_substs_for_item(DUMMY_SP, impl_def_id);
+            let impl_substs = infcx.fresh_substs_for_item(DUMMY_SP, impl_def_id);
             let impl_trait_ref = impl_trait_ref.subst(tcx, impl_substs);
 
-            let Ok(InferOk { obligations, .. }) = acx
+            let Ok(InferOk { obligations, .. }) = infcx
-                .infcx
                 .at(&ObligationCause::dummy(), goal.param_env)
                 .define_opaque_types(false)
                 .eq(goal.predicate.trait_ref, impl_trait_ref)
@@ -104,16 +104,16 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
             let nested_goals =
                 obligations.into_iter().map(|o| o.into()).chain(where_clause_bounds).collect();
 
-            let Ok(certainty) = acx.cx.evaluate_all(acx.infcx, nested_goals) else { return };
+            let Ok(certainty) = acx.cx.evaluate_all(nested_goals) else { return };
             acx.try_insert_candidate(CandidateSource::Impl(impl_def_id), certainty);
         })
     }
 }
 
-impl<'tcx> EvalCtxt<'tcx> {
+impl<'tcx> EvalCtxt<'_, 'tcx> {
     pub(super) fn compute_trait_goal(
         &mut self,
-        goal: CanonicalGoal<'tcx, TraitPredicate<'tcx>>,
+        goal: Goal<'tcx, TraitPredicate<'tcx>>,
     ) -> QueryResult<'tcx> {
         let candidates = AssemblyCtxt::assemble_and_evaluate_candidates(self, goal);
         self.merge_trait_candidates_discard_reservation_impls(candidates)
@@ -176,7 +176,7 @@ impl<'tcx> EvalCtxt<'tcx> {
 
     fn discard_reservation_impl(&self, candidate: Candidate<'tcx>) -> Candidate<'tcx> {
         if let CandidateSource::Impl(def_id) = candidate.source {
-            if let ty::ImplPolarity::Reservation = self.tcx.impl_polarity(def_id) {
+            if let ty::ImplPolarity::Reservation = self.tcx().impl_polarity(def_id) {
                 debug!("Selected reservation impl");
                 // FIXME: reduce candidate to ambiguous
                 // FIXME: replace `var_values` with identity, yeet external constraints.