Use dynamic dispatch for queries

compiler/rustc_interface/src/interface.rs

@@ -318,7 +318,7 @@ pub fn try_print_query_stack(handler: &Handler, num_frames: Option<usize>) {
     // state if it was responsible for triggering the panic.
     let i = ty::tls::with_context_opt(|icx| {
         if let Some(icx) = icx {
-            print_query_stack(QueryCtxt { tcx: icx.tcx }, icx.query, handler, num_frames)
+            print_query_stack(QueryCtxt::new(icx.tcx), icx.query, handler, num_frames)
         } else {
             0
         }

compiler/rustc_interface/src/passes.rs

@@ -700,9 +700,12 @@ pub fn create_global_ctxt<'tcx>(
                 hir_arena,
                 untracked,
                 dep_graph,
-                query_result_on_disk_cache,
                 rustc_query_impl::query_callbacks(arena),
-                rustc_query_impl::query_system_fns(local_providers, extern_providers),
+                rustc_query_impl::query_system(
+                    local_providers,
+                    extern_providers,
+                    query_result_on_disk_cache,
+                ),
             )
         })
     })

compiler/rustc_middle/Cargo.toml

@@ -11,6 +11,7 @@ chalk-ir = "0.87.0"
 derive_more = "0.99.17"
 either = "1.5.0"
 gsgdt = "0.1.2"
+field-offset = "0.3.5"
 measureme = "10.0.0"
 polonius-engine = "0.13.0"
 rustc_apfloat = { path = "../rustc_apfloat" }

compiler/rustc_middle/src/ty/context.rs

@@ -14,14 +14,12 @@ use crate::middle::resolve_bound_vars;
 use crate::middle::stability;
 use crate::mir::interpret::{self, Allocation, ConstAllocation};
 use crate::mir::{Body, Local, Place, PlaceElem, ProjectionKind, Promoted};
-use crate::query::on_disk_cache::OnDiskCache;
 use crate::query::LocalCrate;
 use crate::thir::Thir;
 use crate::traits;
 use crate::traits::solve;
 use crate::traits::solve::{ExternalConstraints, ExternalConstraintsData};
 use crate::ty::query::QuerySystem;
-use crate::ty::query::QuerySystemFns;
 use crate::ty::query::{self, TyCtxtAt};
 use crate::ty::{
     self, AdtDef, AdtDefData, AdtKind, Binder, Const, ConstData, FloatTy, FloatVar, FloatVid,
@@ -653,9 +651,8 @@ impl<'tcx> TyCtxt<'tcx> {
         hir_arena: &'tcx WorkerLocal<hir::Arena<'tcx>>,
         untracked: Untracked,
         dep_graph: DepGraph,
-        on_disk_cache: Option<OnDiskCache<'tcx>>,
         query_kinds: &'tcx [DepKindStruct<'tcx>],
-        query_system_fns: QuerySystemFns<'tcx>,
+        query_system: QuerySystem<'tcx>,
     ) -> GlobalCtxt<'tcx> {
         let data_layout = s.target.parse_data_layout().unwrap_or_else(|err| {
             s.emit_fatal(err);
@@ -677,7 +674,7 @@ impl<'tcx> TyCtxt<'tcx> {
             lifetimes: common_lifetimes,
             consts: common_consts,
             untracked,
-            query_system: QuerySystem::new(query_system_fns, on_disk_cache),
+            query_system,
             query_kinds,
             ty_rcache: Default::default(),
             pred_rcache: Default::default(),

compiler/rustc_middle/src/ty/query.rs

@@ -43,6 +43,7 @@ use crate::ty::subst::{GenericArg, SubstsRef};
 use crate::ty::util::AlwaysRequiresDrop;
 use crate::ty::GeneratorDiagnosticData;
 use crate::ty::{self, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt, UnusedGenericParams};
+use field_offset::FieldOffset;
 use measureme::StringId;
 use rustc_arena::TypedArena;
 use rustc_ast as ast;
@@ -66,9 +67,12 @@ use rustc_hir::hir_id::OwnerId;
 use rustc_hir::lang_items::{LangItem, LanguageItems};
 use rustc_hir::{Crate, ItemLocalId, TraitCandidate};
 use rustc_index::IndexVec;
+use rustc_query_system::dep_graph::DepNodeIndex;
+use rustc_query_system::dep_graph::SerializedDepNodeIndex;
 use rustc_query_system::ich::StableHashingContext;
 pub(crate) use rustc_query_system::query::QueryJobId;
 use rustc_query_system::query::*;
+use rustc_query_system::HandleCycleError;
 use rustc_session::config::{EntryFnType, OptLevel, OutputFilenames, SymbolManglingVersion};
 use rustc_session::cstore::{CrateDepKind, CrateSource};
 use rustc_session::cstore::{ExternCrate, ForeignModule, LinkagePreference, NativeLib};
@@ -78,8 +82,6 @@ use rustc_span::symbol::Symbol;
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi;
 use rustc_target::spec::PanicStrategy;
-
-use std::marker::PhantomData;
 use std::mem;
 use std::ops::Deref;
 use std::path::PathBuf;
@@ -103,6 +105,31 @@ pub struct QueryStruct<'tcx> {
         Option<fn(TyCtxt<'tcx>, &mut CacheEncoder<'_, 'tcx>, &mut EncodedDepNodeIndex)>,
 }
 
+pub struct DynamicQuery<'tcx, C: QueryCache> {
+    pub name: &'static str,
+    pub eval_always: bool,
+    pub dep_kind: rustc_middle::dep_graph::DepKind,
+    pub handle_cycle_error: HandleCycleError,
+    pub query_state: FieldOffset<QueryStates<'tcx>, QueryState<C::Key, crate::dep_graph::DepKind>>,
+    pub query_cache: FieldOffset<QueryCaches<'tcx>, C>,
+    pub cache_on_disk: fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool,
+    pub execute_query: fn(tcx: TyCtxt<'tcx>, k: C::Key) -> C::Value,
+    pub compute: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
+    pub can_load_from_disk: bool,
+    pub try_load_from_disk: fn(
+        tcx: TyCtxt<'tcx>,
+        key: &C::Key,
+        prev_index: SerializedDepNodeIndex,
+        index: DepNodeIndex,
+    ) -> Option<C::Value>,
+    pub loadable_from_disk:
+        fn(tcx: TyCtxt<'tcx>, key: &C::Key, index: SerializedDepNodeIndex) -> bool,
+    pub hash_result: HashResult<C::Value>,
+    pub value_from_cycle_error:
+        fn(tcx: TyCtxt<'tcx>, cycle: &[QueryInfo<crate::dep_graph::DepKind>]) -> C::Value,
+    pub format_value: fn(&C::Value) -> String,
+}
+
 pub struct QuerySystemFns<'tcx> {
     pub engine: QueryEngine,
     pub local_providers: Providers,
@@ -120,6 +147,7 @@ pub struct QuerySystem<'tcx> {
     pub states: QueryStates<'tcx>,
     pub arenas: QueryArenas<'tcx>,
     pub caches: QueryCaches<'tcx>,
+    pub dynamic_queries: DynamicQueries<'tcx>,
 
     /// This provides access to the incremental compilation on-disk cache for query results.
     /// Do not access this directly. It is only meant to be used by
@@ -130,23 +158,6 @@ pub struct QuerySystem<'tcx> {
     pub fns: QuerySystemFns<'tcx>,
 
     pub jobs: AtomicU64,
-
-    // Since we erase query value types we tell the typesystem about them with `PhantomData`.
-    _phantom_values: QueryPhantomValues<'tcx>,
-}
-
-impl<'tcx> QuerySystem<'tcx> {
-    pub fn new(fns: QuerySystemFns<'tcx>, on_disk_cache: Option<OnDiskCache<'tcx>>) -> Self {
-        QuerySystem {
-            states: Default::default(),
-            arenas: Default::default(),
-            caches: Default::default(),
-            on_disk_cache,
-            fns,
-            jobs: AtomicU64::new(1),
-            _phantom_values: Default::default(),
-        }
-    }
 }
 
 #[derive(Copy, Clone)]
@@ -427,11 +438,6 @@ macro_rules! define_callbacks {
             }
         }
 
-        #[derive(Default)]
-        pub struct QueryPhantomValues<'tcx> {
-            $($(#[$attr])* pub $name: PhantomData<query_values::$name<'tcx>>,)*
-        }
-
         #[derive(Default)]
         pub struct QueryCaches<'tcx> {
             $($(#[$attr])* pub $name: query_storage::$name<'tcx>,)*
@@ -490,6 +496,12 @@ macro_rules! define_callbacks {
             })*
         }
 
+        pub struct DynamicQueries<'tcx> {
+            $(
+                pub $name: DynamicQuery<'tcx, query_storage::$name<'tcx>>,
+            )*
+        }
+
         #[derive(Default)]
         pub struct QueryStates<'tcx> {
             $(

compiler/rustc_query_impl/Cargo.toml

@@ -7,6 +7,8 @@ edition = "2021"
 
 
 [dependencies]
+memoffset = { version = "0.6.0", features = ["unstable_const"] }
+field-offset = "0.3.5"
 measureme = "10.0.0"
 rustc_ast = { path = "../rustc_ast" }
 rustc_data_structures = { path = "../rustc_data_structures" }

compiler/rustc_query_impl/src/lib.rs

@@ -3,11 +3,12 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 // this shouldn't be necessary, but the check for `&mut _` is too naive and denies returning a function pointer that takes a mut ref
 #![feature(const_mut_refs)]
+#![feature(const_refs_to_cell)]
 #![feature(min_specialization)]
 #![feature(never_type)]
 #![feature(rustc_attrs)]
 #![recursion_limit = "256"]
-#![allow(rustc::potential_query_instability)]
+#![allow(rustc::potential_query_instability, unused_parens)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
@@ -15,16 +16,27 @@
 extern crate rustc_middle;
 
 use crate::plumbing::{encode_all_query_results, try_mark_green};
+use field_offset::offset_of;
+use rustc_data_structures::stable_hasher::HashStable;
+use rustc_data_structures::sync::AtomicU64;
 use rustc_middle::arena::Arena;
+use rustc_middle::dep_graph::DepNodeIndex;
 use rustc_middle::dep_graph::{self, DepKind, DepKindStruct};
 use rustc_middle::query::erase::{erase, restore, Erase};
+use rustc_middle::query::on_disk_cache::OnDiskCache;
 use rustc_middle::query::AsLocalKey;
 use rustc_middle::ty::query::{
     query_keys, query_provided, query_provided_to_value, query_storage, query_values,
+    DynamicQueries, DynamicQuery, ExternProviders, Providers, QueryCaches, QueryEngine,
+    QueryStates, QuerySystem, QuerySystemFns,
 };
-use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine, QuerySystemFns};
 use rustc_middle::ty::TyCtxt;
 use rustc_query_system::dep_graph::SerializedDepNodeIndex;
+use rustc_query_system::ich::StableHashingContext;
+use rustc_query_system::query::{
+    get_query, HashResult, QueryCache, QueryConfig, QueryInfo, QueryMap, QueryMode, QueryState,
+};
+use rustc_query_system::HandleCycleError;
 use rustc_query_system::Value;
 use rustc_span::Span;
 
@@ -32,31 +44,182 @@ use rustc_span::Span;
 mod plumbing;
 pub use crate::plumbing::QueryCtxt;
 
-pub use rustc_query_system::query::QueryConfig;
-use rustc_query_system::query::*;
-
 mod profiling_support;
 pub use self::profiling_support::alloc_self_profile_query_strings;
 
-/// This is implemented per query and restoring query values from their erased state.
+struct DynamicConfig<
-trait QueryConfigRestored<'tcx>: QueryConfig<QueryCtxt<'tcx>> + Default {
+    'tcx,
-    type RestoredValue;
+    C: QueryCache,
+    const ANON: bool,
+    const DEPTH_LIMIT: bool,
+    const FEEDABLE: bool,
+> {
+    dynamic: &'tcx DynamicQuery<'tcx, C>,
+}
 
-    fn restore(value: <Self as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue;
+impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Copy
+    for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
+{
+}
+impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Clone
+    for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
+{
+    fn clone(&self) -> Self {
+        DynamicConfig { dynamic: self.dynamic }
+    }
+}
+
+impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool>
+    QueryConfig<QueryCtxt<'tcx>> for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
+where
+    for<'a> C::Key: HashStable<StableHashingContext<'a>>,
+{
+    type Key = C::Key;
+    type Value = C::Value;
+    type Cache = C;
+
+    #[inline(always)]
+    fn name(self) -> &'static str {
+        self.dynamic.name
+    }
+
+    #[inline(always)]
+    fn cache_on_disk(self, tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool {
+        (self.dynamic.cache_on_disk)(tcx, key)
+    }
+
+    #[inline(always)]
+    fn query_state<'a>(self, qcx: QueryCtxt<'tcx>) -> &'a QueryState<Self::Key, DepKind>
+    where
+        QueryCtxt<'tcx>: 'a,
+    {
+        self.dynamic.query_state.apply(&qcx.tcx.query_system.states)
+    }
+
+    #[inline(always)]
+    fn query_cache<'a>(self, qcx: QueryCtxt<'tcx>) -> &'a Self::Cache
+    where
+        'tcx: 'a,
+    {
+        self.dynamic.query_cache.apply(&qcx.tcx.query_system.caches)
+    }
+
+    #[inline(always)]
+    fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
+        (self.dynamic.execute_query)(tcx, key)
+    }
+
+    #[inline(always)]
+    fn compute(self, qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value {
+        (self.dynamic.compute)(qcx.tcx, key)
+    }
+
+    #[inline(always)]
+    fn try_load_from_disk(
+        self,
+        qcx: QueryCtxt<'tcx>,
+        key: &Self::Key,
+        prev_index: SerializedDepNodeIndex,
+        index: DepNodeIndex,
+    ) -> Option<Self::Value> {
+        if self.dynamic.can_load_from_disk {
+            (self.dynamic.try_load_from_disk)(qcx.tcx, key, prev_index, index)
+        } else {
+            None
+        }
+    }
+
+    #[inline]
+    fn loadable_from_disk(
+        self,
+        qcx: QueryCtxt<'tcx>,
+        key: &Self::Key,
+        index: SerializedDepNodeIndex,
+    ) -> bool {
+        (self.dynamic.loadable_from_disk)(qcx.tcx, key, index)
+    }
+
+    fn value_from_cycle_error(
+        self,
+        tcx: TyCtxt<'tcx>,
+        cycle: &[QueryInfo<DepKind>],
+    ) -> Self::Value {
+        (self.dynamic.value_from_cycle_error)(tcx, cycle)
+    }
+
+    #[inline(always)]
+    fn format_value(self) -> fn(&Self::Value) -> String {
+        self.dynamic.format_value
+    }
+
+    #[inline(always)]
+    fn anon(self) -> bool {
+        ANON
+    }
+
+    #[inline(always)]
+    fn eval_always(self) -> bool {
+        self.dynamic.eval_always
+    }
+
+    #[inline(always)]
+    fn depth_limit(self) -> bool {
+        DEPTH_LIMIT
+    }
+
+    #[inline(always)]
+    fn feedable(self) -> bool {
+        FEEDABLE
+    }
+
+    #[inline(always)]
+    fn dep_kind(self) -> DepKind {
+        self.dynamic.dep_kind
+    }
+
+    #[inline(always)]
+    fn handle_cycle_error(self) -> HandleCycleError {
+        self.dynamic.handle_cycle_error
+    }
+
+    #[inline(always)]
+    fn hash_result(self) -> HashResult<Self::Value> {
+        self.dynamic.hash_result
+    }
+}
+
+/// This is implemented per query. It allows restoring query values from their erased state
+/// and constructing a QueryConfig.
+trait QueryConfigRestored<'tcx> {
+    type RestoredValue;
+    type Config: QueryConfig<QueryCtxt<'tcx>>;
+
+    fn config(tcx: TyCtxt<'tcx>) -> Self::Config;
+    fn restore(value: <Self::Config as QueryConfig<QueryCtxt<'tcx>>>::Value)
+    -> Self::RestoredValue;
+}
+
+pub fn query_system<'tcx>(
+    local_providers: Providers,
+    extern_providers: ExternProviders,
+    on_disk_cache: Option<OnDiskCache<'tcx>>,
+) -> QuerySystem<'tcx> {
+    QuerySystem {
+        states: Default::default(),
+        arenas: Default::default(),
+        caches: Default::default(),
+        dynamic_queries: dynamic_queries(),
+        on_disk_cache,
+        fns: QuerySystemFns {
+            engine: engine(),
+            local_providers,
+            extern_providers,
+            query_structs: make_dep_kind_array!(query_structs).to_vec(),
+            encode_query_results: encode_all_query_results,
+            try_mark_green: try_mark_green,
+        },
+        jobs: AtomicU64::new(1),
+    }
 }
 
 rustc_query_append! { define_queries! }
-
-pub fn query_system_fns<'tcx>(
-    local_providers: Providers,
-    extern_providers: ExternProviders,
-) -> QuerySystemFns<'tcx> {
-    QuerySystemFns {
-        engine: engine(),
-        local_providers,
-        extern_providers,
-        query_structs: make_dep_kind_array!(query_structs).to_vec(),
-        encode_query_results: encode_all_query_results,
-        try_mark_green: try_mark_green,
-    }
-}

compiler/rustc_query_impl/src/plumbing.rs

@@ -4,6 +4,7 @@
 
 use crate::rustc_middle::dep_graph::DepContext;
 use crate::rustc_middle::ty::TyEncoder;
+use crate::QueryConfigRestored;
 use rustc_data_structures::stable_hasher::{Hash64, HashStable, StableHasher};
 use rustc_data_structures::sync::Lock;
 use rustc_errors::Diagnostic;
@@ -265,14 +266,14 @@ macro_rules! hash_result {
 }
 
 macro_rules! call_provider {
-    ([][$qcx:expr, $name:ident, $key:expr]) => {{
+    ([][$tcx:expr, $name:ident, $key:expr]) => {{
-        ($qcx.query_system.fns.local_providers.$name)($qcx, $key)
+        ($tcx.query_system.fns.local_providers.$name)($tcx, $key)
     }};
-    ([(separate_provide_extern) $($rest:tt)*][$qcx:expr, $name:ident, $key:expr]) => {{
+    ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{
         if let Some(key) = $key.as_local_key() {
-            ($qcx.query_system.fns.local_providers.$name)($qcx, key)
+            ($tcx.query_system.fns.local_providers.$name)($tcx, key)
         } else {
-            ($qcx.query_system.fns.extern_providers.$name)($qcx, $key)
+            ($tcx.query_system.fns.extern_providers.$name)($tcx, $key)
         }
     }};
     ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
@@ -341,7 +342,7 @@ pub(crate) fn create_query_frame<
 }
 
 pub(crate) fn encode_query_results<'a, 'tcx, Q>(
-    query: Q,
+    query: Q::Config,
     qcx: QueryCtxt<'tcx>,
     encoder: &mut CacheEncoder<'a, 'tcx>,
     query_result_index: &mut EncodedDepNodeIndex,
@@ -392,12 +393,26 @@ pub(crate) fn loadable_from_disk<'tcx>(tcx: TyCtxt<'tcx>, id: SerializedDepNodeI
 
 pub(crate) fn try_load_from_disk<'tcx, V>(
     tcx: TyCtxt<'tcx>,
-    id: SerializedDepNodeIndex,
+    prev_index: SerializedDepNodeIndex,
+    index: DepNodeIndex,
 ) -> Option<V>
 where
     V: for<'a> Decodable<CacheDecoder<'a, 'tcx>>,
 {
-    tcx.query_system.on_disk_cache.as_ref()?.try_load_query_result(tcx, id)
+    let on_disk_cache = tcx.query_system.on_disk_cache.as_ref()?;
+
+    let prof_timer = tcx.prof.incr_cache_loading();
+
+    // The call to `with_query_deserialization` enforces that no new `DepNodes`
+    // are created during deserialization. See the docs of that method for more
+    // details.
+    let value = tcx
+        .dep_graph
+        .with_query_deserialization(|| on_disk_cache.try_load_query_result(tcx, prev_index));
+
+    prof_timer.finish_with_query_invocation_id(index.into());
+
+    value
 }
 
 fn force_from_dep_node<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode) -> bool
@@ -434,10 +449,9 @@ where
 
 pub(crate) fn query_callback<'tcx, Q>(is_anon: bool, is_eval_always: bool) -> DepKindStruct<'tcx>
 where
-    Q: QueryConfig<QueryCtxt<'tcx>> + Default,
+    Q: QueryConfigRestored<'tcx>,
-    Q::Key: DepNodeParams<TyCtxt<'tcx>>,
 {
-    let fingerprint_style = Q::Key::fingerprint_style();
+    let fingerprint_style = <Q::Config as QueryConfig<QueryCtxt<'tcx>>>::Key::fingerprint_style();
 
     if is_anon || !fingerprint_style.reconstructible() {
         return DepKindStruct {
@@ -453,9 +467,11 @@ where
         is_anon,
         is_eval_always,
         fingerprint_style,
-        force_from_dep_node: Some(|tcx, dep_node| force_from_dep_node(Q::default(), tcx, dep_node)),
+        force_from_dep_node: Some(|tcx, dep_node| {
+            force_from_dep_node(Q::config(tcx), tcx, dep_node)
+        }),
         try_load_from_on_disk_cache: Some(|tcx, dep_node| {
-            try_load_from_on_disk_cache(Q::default(), tcx, dep_node)
+            try_load_from_on_disk_cache(Q::config(tcx), tcx, dep_node)
         }),
     }
 }
@@ -491,7 +507,7 @@ macro_rules! define_queries {
                     mode: QueryMode,
                 ) -> Option<Erase<query_values::$name<'tcx>>> {
                     get_query(
-                        queries::$name::default(),
+                        queries::$name::config(tcx),
                         QueryCtxt::new(tcx),
                         span,
                         key,
@@ -519,147 +535,92 @@ macro_rules! define_queries {
             )*
         }
 
-        $(impl<'tcx> QueryConfig<QueryCtxt<'tcx>> for queries::$name<'tcx> {
+        #[allow(nonstandard_style)]
-            type Key = query_keys::$name<'tcx>;
+        mod dynamic_query {
-            type Value = Erase<query_values::$name<'tcx>>;
+            use super::*;
 
-            #[inline(always)]
+            $(
-            fn name(self) -> &'static str {
+                pub(super) fn $name<'tcx>() -> DynamicQuery<'tcx, query_storage::$name<'tcx>> {
-                stringify!($name)
+                    DynamicQuery {
-            }
+                        name: stringify!($name),
-
+                        eval_always: is_eval_always!([$($modifiers)*]),
-            #[inline]
+                        dep_kind: dep_graph::DepKind::$name,
-            fn format_value(self) -> fn(&Self::Value) -> String {
+                        handle_cycle_error: handle_cycle_error!([$($modifiers)*]),
-                |value| format!("{:?}", restore::<query_values::$name<'tcx>>(*value))
+                        query_state: offset_of!(QueryStates<'tcx> => $name),
-            }
+                        query_cache: offset_of!(QueryCaches<'tcx> => $name),
-
+                        cache_on_disk: |tcx, key| ::rustc_middle::query::cached::$name(tcx, key),
-            #[inline]
+                        execute_query: |tcx, key| erase(tcx.$name(key)),
-            fn cache_on_disk(self, tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool {
+                        compute: |tcx, key| query_provided_to_value::$name(
-                ::rustc_middle::query::cached::$name(tcx, key)
+                            tcx,
-            }
+                            call_provider!([$($modifiers)*][tcx, $name, key])
-
+                        ),
-            type Cache = query_storage::$name<'tcx>;
+                        can_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] true false),
-
+                        try_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] {
-            #[inline(always)]
+                            |tcx, key, prev_index, index| {
-            fn query_state<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a QueryState<Self::Key, crate::dep_graph::DepKind>
+                                if ::rustc_middle::query::cached::$name(tcx, key) {
-                where QueryCtxt<'tcx>: 'a
+                                    let value = $crate::plumbing::try_load_from_disk::<query_provided::$name<'tcx>>(
-            {
+                                        tcx,
-                &tcx.query_system.states.$name
+                                        prev_index,
-            }
+                                        index,
-
+                                    );
-            #[inline(always)]
+                                    value.map(|value| query_provided_to_value::$name(tcx, value))
-            fn query_cache<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a Self::Cache
+                                } else {
-                where 'tcx:'a
+                                    None
-            {
+                                }
-                &tcx.query_system.caches.$name
+                            }
-            }
+                        } {
-
+                            |_tcx, _key, _prev_index, _index| None
-            fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
+                        }),
-                erase(tcx.$name(key))
+                        value_from_cycle_error: |tcx, cycle| {
-            }
+                            let result: query_values::$name<'tcx> = Value::from_cycle_error(tcx, cycle);
-
+                            erase(result)
-            #[inline]
+                        },
-            #[allow(unused_variables)]
+                        loadable_from_disk: |_tcx, _key, _index| {
-            fn compute(self, qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value {
+                            should_ever_cache_on_disk!([$($modifiers)*] {
-                query_provided_to_value::$name(
+                                ::rustc_middle::query::cached::$name(_tcx, _key) &&
-                    qcx.tcx,
+                                    $crate::plumbing::loadable_from_disk(_tcx, _index)
-                    call_provider!([$($modifiers)*][qcx.tcx, $name, key])
+                            } {
-                )
+                                false
-            }
+                            })
-
+                        },
-            #[inline]
+                        hash_result: hash_result!([$($modifiers)*][query_values::$name<'tcx>]),
-            fn try_load_from_disk(
+                        format_value: |value| format!("{:?}", restore::<query_values::$name<'tcx>>(*value)),
-                self,
-                _qcx: QueryCtxt<'tcx>,
-                _key: &Self::Key
-            ) -> rustc_query_system::query::TryLoadFromDisk<QueryCtxt<'tcx>, Self::Value> {
-                should_ever_cache_on_disk!([$($modifiers)*] {
-                    if ::rustc_middle::query::cached::$name(_qcx.tcx, _key) {
-                        Some(|qcx: QueryCtxt<'tcx>, dep_node| {
-                            let value = $crate::plumbing::try_load_from_disk::<query_provided::$name<'tcx>>(
-                                qcx.tcx,
-                                dep_node
-                            );
-                            value.map(|value| query_provided_to_value::$name(qcx.tcx, value))
-                        })
-                    } else {
-                        None
                     }
-                } {
+                }
-                    None
+            )*
-                })
+        }
-            }
-
-            #[inline]
-            fn loadable_from_disk(
-                self,
-                _qcx: QueryCtxt<'tcx>,
-                _key: &Self::Key,
-                _index: SerializedDepNodeIndex,
-            ) -> bool {
-                should_ever_cache_on_disk!([$($modifiers)*] {
-                    self.cache_on_disk(_qcx.tcx, _key) &&
-                        $crate::plumbing::loadable_from_disk(_qcx.tcx, _index)
-                } {
-                    false
-                })
-            }
-
-            #[inline]
-            fn value_from_cycle_error(
-                self,
-                tcx: TyCtxt<'tcx>,
-                cycle: &[QueryInfo<DepKind>],
-            ) -> Self::Value {
-                let result: query_values::$name<'tcx> = Value::from_cycle_error(tcx, cycle);
-                erase(result)
-            }
-
-            #[inline(always)]
-            fn anon(self) -> bool {
-                is_anon!([$($modifiers)*])
-            }
-
-            #[inline(always)]
-            fn eval_always(self) -> bool {
-                is_eval_always!([$($modifiers)*])
-            }
-
-            #[inline(always)]
-            fn depth_limit(self) -> bool {
-                depth_limit!([$($modifiers)*])
-            }
-
-            #[inline(always)]
-            fn feedable(self) -> bool {
-                feedable!([$($modifiers)*])
-            }
-
-            #[inline(always)]
-            fn dep_kind(self) -> rustc_middle::dep_graph::DepKind {
-                dep_graph::DepKind::$name
-            }
-
-            #[inline(always)]
-            fn handle_cycle_error(self) -> rustc_query_system::HandleCycleError {
-                handle_cycle_error!([$($modifiers)*])
-            }
-
-            #[inline(always)]
-            fn hash_result(self) -> rustc_query_system::query::HashResult<Self::Value> {
-                hash_result!([$($modifiers)*][query_values::$name<'tcx>])
-            }
-        })*
 
         $(impl<'tcx> QueryConfigRestored<'tcx> for queries::$name<'tcx> {
             type RestoredValue = query_values::$name<'tcx>;
+            type Config = DynamicConfig<
+                'tcx,
+                query_storage::$name<'tcx>,
+                { is_anon!([$($modifiers)*]) },
+                { depth_limit!([$($modifiers)*]) },
+                { feedable!([$($modifiers)*]) },
+            >;
 
             #[inline(always)]
-            fn restore(value: <Self as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue {
+            fn config(tcx: TyCtxt<'tcx>) -> Self::Config {
+                DynamicConfig {
+                    dynamic: &tcx.query_system.dynamic_queries.$name,
+                }
+            }
+
+            #[inline(always)]
+            fn restore(value: <Self::Config as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue {
                 restore::<query_values::$name<'tcx>>(value)
             }
         })*
 
+        pub fn dynamic_queries<'tcx>() -> DynamicQueries<'tcx> {
+            DynamicQueries {
+                $(
+                    $name: dynamic_query::$name(),
+                )*
+            }
+        }
+
         #[allow(nonstandard_style)]
         mod query_callbacks {
             use super::*;
@@ -730,6 +691,7 @@ macro_rules! define_queries {
             use rustc_middle::ty::query::QueryStruct;
             use rustc_middle::ty::query::QueryKeyStringCache;
             use rustc_middle::dep_graph::DepKind;
+            use crate::QueryConfigRestored;
 
             pub(super) const fn dummy_query_struct<'tcx>() -> QueryStruct<'tcx> {
                 fn noop_try_collect_active_jobs(_: TyCtxt<'_>, _: &mut QueryMap<DepKind>) -> Option<()> {
@@ -774,7 +736,7 @@ macro_rules! define_queries {
                 },
                 encode_query_results: expand_if_cached!([$($modifiers)*], |tcx, encoder, query_result_index|
                     $crate::plumbing::encode_query_results::<super::queries::$name<'tcx>>(
-                        super::queries::$name::default(),
+                        super::queries::$name::config(tcx),
                         QueryCtxt::new(tcx),
                         encoder,
                         query_result_index,

compiler/rustc_query_system/src/query/config.rs

@@ -4,6 +4,7 @@ use crate::dep_graph::{DepNode, DepNodeParams, SerializedDepNodeIndex};
 use crate::error::HandleCycleError;
 use crate::ich::StableHashingContext;
 use crate::query::caches::QueryCache;
+use crate::query::DepNodeIndex;
 use crate::query::{QueryContext, QueryInfo, QueryState};
 
 use rustc_data_structures::fingerprint::Fingerprint;
@@ -12,8 +13,6 @@ use std::hash::Hash;
 
 pub type HashResult<V> = Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>;
 
-pub type TryLoadFromDisk<Qcx, V> = Option<fn(Qcx, SerializedDepNodeIndex) -> Option<V>>;
-
 pub trait QueryConfig<Qcx: QueryContext>: Copy {
     fn name(self) -> &'static str;
 
@@ -43,7 +42,13 @@ pub trait QueryConfig<Qcx: QueryContext>: Copy {
 
     fn compute(self, tcx: Qcx, key: Self::Key) -> Self::Value;
 
-    fn try_load_from_disk(self, qcx: Qcx, idx: &Self::Key) -> TryLoadFromDisk<Qcx, Self::Value>;
+    fn try_load_from_disk(
+        self,
+        tcx: Qcx,
+        key: &Self::Key,
+        prev_index: SerializedDepNodeIndex,
+        index: DepNodeIndex,
+    ) -> Option<Self::Value>;
 
     fn loadable_from_disk(self, qcx: Qcx, key: &Self::Key, idx: SerializedDepNodeIndex) -> bool;
 

compiler/rustc_query_system/src/query/mod.rs

@@ -12,7 +12,7 @@ pub use self::caches::{
 };
 
 mod config;
-pub use self::config::{HashResult, QueryConfig, TryLoadFromDisk};
+pub use self::config::{HashResult, QueryConfig};
 
 use crate::dep_graph::DepKind;
 use crate::dep_graph::{DepNodeIndex, HasDepContext, SerializedDepNodeIndex};

compiler/rustc_query_system/src/query/plumbing.rs

@@ -564,59 +564,44 @@ where
 
     // First we try to load the result from the on-disk cache.
     // Some things are never cached on disk.
-    if let Some(try_load_from_disk) = query.try_load_from_disk(qcx, &key) {
+    if let Some(result) = query.try_load_from_disk(qcx, key, prev_dep_node_index, dep_node_index) {
-        let prof_timer = qcx.dep_context().profiler().incr_cache_loading();
+        if std::intrinsics::unlikely(qcx.dep_context().sess().opts.unstable_opts.query_dep_graph) {
-
+            dep_graph_data.mark_debug_loaded_from_disk(*dep_node)
-        // The call to `with_query_deserialization` enforces that no new `DepNodes`
-        // are created during deserialization. See the docs of that method for more
-        // details.
-        let result = qcx
-            .dep_context()
-            .dep_graph()
-            .with_query_deserialization(|| try_load_from_disk(qcx, prev_dep_node_index));
-
-        prof_timer.finish_with_query_invocation_id(dep_node_index.into());
-
-        if let Some(result) = result {
-            if std::intrinsics::unlikely(
-                qcx.dep_context().sess().opts.unstable_opts.query_dep_graph,
-            ) {
-                dep_graph_data.mark_debug_loaded_from_disk(*dep_node)
-            }
-
-            let prev_fingerprint = dep_graph_data.prev_fingerprint_of(prev_dep_node_index);
-            // If `-Zincremental-verify-ich` is specified, re-hash results from
-            // the cache and make sure that they have the expected fingerprint.
-            //
-            // If not, we still seek to verify a subset of fingerprints loaded
-            // from disk. Re-hashing results is fairly expensive, so we can't
-            // currently afford to verify every hash. This subset should still
-            // give us some coverage of potential bugs though.
-            let try_verify = prev_fingerprint.split().1.as_u64() % 32 == 0;
-            if std::intrinsics::unlikely(
-                try_verify || qcx.dep_context().sess().opts.unstable_opts.incremental_verify_ich,
-            ) {
-                incremental_verify_ich(
-                    *qcx.dep_context(),
-                    dep_graph_data,
-                    &result,
-                    prev_dep_node_index,
-                    query.hash_result(),
-                    query.format_value(),
-                );
-            }
-
-            return Some((result, dep_node_index));
         }
 
-        // We always expect to find a cached result for things that
+        let prev_fingerprint = dep_graph_data.prev_fingerprint_of(prev_dep_node_index);
-        // can be forced from `DepNode`.
+        // If `-Zincremental-verify-ich` is specified, re-hash results from
-        debug_assert!(
+        // the cache and make sure that they have the expected fingerprint.
-            !qcx.dep_context().fingerprint_style(dep_node.kind).reconstructible(),
+        //
-            "missing on-disk cache entry for reconstructible {dep_node:?}"
+        // If not, we still seek to verify a subset of fingerprints loaded
-        );
+        // from disk. Re-hashing results is fairly expensive, so we can't
+        // currently afford to verify every hash. This subset should still
+        // give us some coverage of potential bugs though.
+        let try_verify = prev_fingerprint.split().1.as_u64() % 32 == 0;
+        if std::intrinsics::unlikely(
+            try_verify || qcx.dep_context().sess().opts.unstable_opts.incremental_verify_ich,
+        ) {
+            incremental_verify_ich(
+                *qcx.dep_context(),
+                dep_graph_data,
+                &result,
+                prev_dep_node_index,
+                query.hash_result(),
+                query.format_value(),
+            );
+        }
+
+        return Some((result, dep_node_index));
     }
 
+    // We always expect to find a cached result for things that
+    // can be forced from `DepNode`.
+    debug_assert!(
+        !query.cache_on_disk(*qcx.dep_context(), key)
+            || !qcx.dep_context().fingerprint_style(dep_node.kind).reconstructible(),
+        "missing on-disk cache entry for {dep_node:?}"
+    );
+
     // Sanity check for the logic in `ensure`: if the node is green and the result loadable,
     // we should actually be able to load it.
     debug_assert!(

src/tools/tidy/src/deps.rs

@@ -135,6 +135,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "expect-test",
     "fallible-iterator", // dependency of `thorin`
     "fastrand",
+    "field-offset",
     "fixedbitset",
     "flate2",
     "fluent-bundle",