Auto merge of #109333 - Zoxc:erase-query-cache-values, r=cjgillot

Erase query cache values

This replaces most concrete query values `V` with `MaybeUninit<[u8; { size_of::<V>() }]>` without introducing dynamic dispatch like https://github.com/rust-lang/rust/pull/108638 does. This is split out of https://github.com/rust-lang/rust/pull/108638 so the performance impact of only this change can be measured.

r? `@cjgillot`
This commit is contained in:
bors 2023-04-06 16:29:36 +00:00
commit f5b8f44e5d
11 changed files with 465 additions and 61 deletions

View File

@ -33,6 +33,7 @@
#![feature(generators)]
#![feature(get_mut_unchecked)]
#![feature(if_let_guard)]
#![feature(inline_const)]
#![feature(iter_from_generator)]
#![feature(local_key_cell_methods)]
#![feature(negative_impls)]

View File

@ -0,0 +1,337 @@
use crate::mir;
use crate::traits;
use crate::ty::{self, Ty};
use std::mem::{size_of, transmute_copy, MaybeUninit};
#[derive(Copy, Clone)]
pub struct Erased<T: Copy> {
// We use `MaybeUninit` here so we can store any value
// in `data` since we aren't actually storing a `T`.
data: MaybeUninit<T>,
}
pub trait EraseType: Copy {
type Result: Copy;
}
// Allow `type_alias_bounds` since compilation will fail without `EraseType`.
#[allow(type_alias_bounds)]
pub type Erase<T: EraseType> = Erased<impl Copy>;
#[inline(always)]
pub fn erase<T: EraseType>(src: T) -> Erase<T> {
// Ensure the sizes match
const {
if std::mem::size_of::<T>() != std::mem::size_of::<T::Result>() {
panic!("size of T must match erased type T::Result")
}
};
Erased::<<T as EraseType>::Result> {
// SAFETY: Is it safe to transmute to MaybeUninit for types with the same sizes.
data: unsafe { transmute_copy(&src) },
}
}
/// Restores an erased value.
#[inline(always)]
pub fn restore<T: EraseType>(value: Erase<T>) -> T {
let value: Erased<<T as EraseType>::Result> = value;
// SAFETY: Due to the use of impl Trait in `Erase` the only way to safetly create an instance
// of `Erase` is to call `erase`, so we know that `value.data` is a valid instance of `T` of
// the right size.
unsafe { transmute_copy(&value.data) }
}
impl<T> EraseType for &'_ T {
type Result = [u8; size_of::<*const ()>()];
}
impl<T> EraseType for &'_ [T] {
type Result = [u8; size_of::<*const [()]>()];
}
impl<T> EraseType for &'_ ty::List<T> {
type Result = [u8; size_of::<*const ()>()];
}
impl<T> EraseType for Result<&'_ T, traits::query::NoSolution> {
type Result = [u8; size_of::<Result<&'static (), traits::query::NoSolution>>()];
}
impl<T> EraseType for Result<&'_ T, rustc_errors::ErrorGuaranteed> {
type Result = [u8; size_of::<Result<&'static (), rustc_errors::ErrorGuaranteed>>()];
}
impl<T> EraseType for Result<&'_ T, traits::CodegenObligationError> {
type Result = [u8; size_of::<Result<&'static (), traits::CodegenObligationError>>()];
}
impl<T> EraseType for Result<&'_ T, ty::layout::FnAbiError<'_>> {
type Result = [u8; size_of::<Result<&'static (), ty::layout::FnAbiError<'static>>>()];
}
impl<T> EraseType for Result<(&'_ T, rustc_middle::thir::ExprId), rustc_errors::ErrorGuaranteed> {
type Result = [u8; size_of::<
Result<(&'static (), rustc_middle::thir::ExprId), rustc_errors::ErrorGuaranteed>,
>()];
}
impl EraseType for Result<Option<ty::Instance<'_>>, rustc_errors::ErrorGuaranteed> {
type Result =
[u8; size_of::<Result<Option<ty::Instance<'static>>, rustc_errors::ErrorGuaranteed>>()];
}
impl EraseType for Result<Option<ty::Const<'_>>, rustc_errors::ErrorGuaranteed> {
type Result =
[u8; size_of::<Result<Option<ty::Const<'static>>, rustc_errors::ErrorGuaranteed>>()];
}
impl EraseType for Result<ty::GenericArg<'_>, traits::query::NoSolution> {
type Result = [u8; size_of::<Result<ty::GenericArg<'static>, traits::query::NoSolution>>()];
}
impl EraseType for Result<bool, ty::layout::LayoutError<'_>> {
type Result = [u8; size_of::<Result<bool, ty::layout::LayoutError<'static>>>()];
}
impl EraseType for Result<rustc_target::abi::TyAndLayout<'_, Ty<'_>>, ty::layout::LayoutError<'_>> {
type Result = [u8; size_of::<
Result<
rustc_target::abi::TyAndLayout<'static, Ty<'static>>,
ty::layout::LayoutError<'static>,
>,
>()];
}
impl EraseType for Result<ty::Const<'_>, mir::interpret::LitToConstError> {
type Result = [u8; size_of::<Result<ty::Const<'static>, mir::interpret::LitToConstError>>()];
}
impl EraseType for Result<mir::ConstantKind<'_>, mir::interpret::LitToConstError> {
type Result =
[u8; size_of::<Result<mir::ConstantKind<'static>, mir::interpret::LitToConstError>>()];
}
impl EraseType for Result<mir::interpret::ConstAlloc<'_>, mir::interpret::ErrorHandled> {
type Result = [u8; size_of::<
Result<mir::interpret::ConstAlloc<'static>, mir::interpret::ErrorHandled>,
>()];
}
impl EraseType for Result<mir::interpret::ConstValue<'_>, mir::interpret::ErrorHandled> {
type Result = [u8; size_of::<
Result<mir::interpret::ConstValue<'static>, mir::interpret::ErrorHandled>,
>()];
}
impl EraseType for Result<Option<ty::ValTree<'_>>, mir::interpret::ErrorHandled> {
type Result =
[u8; size_of::<Result<Option<ty::ValTree<'static>>, mir::interpret::ErrorHandled>>()];
}
impl EraseType for Result<&'_ ty::List<Ty<'_>>, ty::util::AlwaysRequiresDrop> {
type Result =
[u8; size_of::<Result<&'static ty::List<Ty<'static>>, ty::util::AlwaysRequiresDrop>>()];
}
impl<T> EraseType for Option<&'_ T> {
type Result = [u8; size_of::<Option<&'static ()>>()];
}
impl<T> EraseType for Option<&'_ [T]> {
type Result = [u8; size_of::<Option<&'static [()]>>()];
}
impl EraseType for Option<rustc_middle::hir::Owner<'_>> {
type Result = [u8; size_of::<Option<rustc_middle::hir::Owner<'static>>>()];
}
impl EraseType for Option<mir::DestructuredConstant<'_>> {
type Result = [u8; size_of::<Option<mir::DestructuredConstant<'static>>>()];
}
impl EraseType for Option<ty::EarlyBinder<ty::TraitRef<'_>>> {
type Result = [u8; size_of::<Option<ty::EarlyBinder<ty::TraitRef<'static>>>>()];
}
impl EraseType for Option<ty::EarlyBinder<Ty<'_>>> {
type Result = [u8; size_of::<Option<ty::EarlyBinder<Ty<'static>>>>()];
}
impl<T> EraseType for rustc_hir::MaybeOwner<&'_ T> {
type Result = [u8; size_of::<rustc_hir::MaybeOwner<&'static ()>>()];
}
impl<T: EraseType> EraseType for ty::EarlyBinder<T> {
type Result = T::Result;
}
impl EraseType for ty::Binder<'_, ty::FnSig<'_>> {
type Result = [u8; size_of::<ty::Binder<'static, ty::FnSig<'static>>>()];
}
impl<T0, T1> EraseType for (&'_ T0, &'_ T1) {
type Result = [u8; size_of::<(&'static (), &'static ())>()];
}
impl<T0, T1> EraseType for (&'_ T0, &'_ [T1]) {
type Result = [u8; size_of::<(&'static (), &'static [()])>()];
}
macro_rules! trivial {
($($ty:ty),+ $(,)?) => {
$(
impl EraseType for $ty {
type Result = [u8; size_of::<$ty>()];
}
)*
}
}
trivial! {
(),
bool,
Option<(rustc_span::def_id::DefId, rustc_session::config::EntryFnType)>,
Option<rustc_ast::expand::allocator::AllocatorKind>,
Option<rustc_attr::ConstStability>,
Option<rustc_attr::DefaultBodyStability>,
Option<rustc_attr::Stability>,
Option<rustc_data_structures::svh::Svh>,
Option<rustc_hir::def::DefKind>,
Option<rustc_hir::GeneratorKind>,
Option<rustc_hir::HirId>,
Option<rustc_middle::middle::stability::DeprecationEntry>,
Option<rustc_middle::ty::Destructor>,
Option<rustc_middle::ty::ImplTraitInTraitData>,
Option<rustc_span::def_id::CrateNum>,
Option<rustc_span::def_id::DefId>,
Option<rustc_span::def_id::LocalDefId>,
Option<rustc_span::Span>,
Option<rustc_target::spec::PanicStrategy>,
Option<usize>,
Result<(), rustc_errors::ErrorGuaranteed>,
Result<(), rustc_middle::traits::query::NoSolution>,
Result<rustc_middle::traits::EvaluationResult, rustc_middle::traits::OverflowError>,
rustc_ast::expand::allocator::AllocatorKind,
rustc_attr::ConstStability,
rustc_attr::DefaultBodyStability,
rustc_attr::Deprecation,
rustc_attr::Stability,
rustc_data_structures::svh::Svh,
rustc_errors::ErrorGuaranteed,
rustc_hir::Constness,
rustc_hir::def_id::DefId,
rustc_hir::def_id::DefIndex,
rustc_hir::def_id::LocalDefId,
rustc_hir::def::DefKind,
rustc_hir::Defaultness,
rustc_hir::definitions::DefKey,
rustc_hir::GeneratorKind,
rustc_hir::HirId,
rustc_hir::IsAsync,
rustc_hir::ItemLocalId,
rustc_hir::LangItem,
rustc_hir::OwnerId,
rustc_hir::Upvar,
rustc_index::bit_set::FiniteBitSet<u32>,
rustc_middle::metadata::ModChild,
rustc_middle::middle::dependency_format::Linkage,
rustc_middle::middle::exported_symbols::SymbolExportInfo,
rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault,
rustc_middle::middle::resolve_bound_vars::ResolvedArg,
rustc_middle::middle::stability::DeprecationEntry,
rustc_middle::mir::ConstQualifs,
rustc_middle::mir::interpret::AllocId,
rustc_middle::mir::interpret::ErrorHandled,
rustc_middle::mir::interpret::LitToConstError,
rustc_middle::thir::ExprId,
rustc_middle::traits::CodegenObligationError,
rustc_middle::traits::EvaluationResult,
rustc_middle::traits::OverflowError,
rustc_middle::traits::query::NoSolution,
rustc_middle::traits::WellFormedLoc,
rustc_middle::ty::adjustment::CoerceUnsizedInfo,
rustc_middle::ty::AssocItem,
rustc_middle::ty::AssocItemContainer,
rustc_middle::ty::BoundVariableKind,
rustc_middle::ty::DeducedParamAttrs,
rustc_middle::ty::Destructor,
rustc_middle::ty::fast_reject::SimplifiedType,
rustc_middle::ty::ImplPolarity,
rustc_middle::ty::Representability,
rustc_middle::ty::ReprOptions,
rustc_middle::ty::UnusedGenericParams,
rustc_middle::ty::util::AlwaysRequiresDrop,
rustc_middle::ty::Visibility<rustc_span::def_id::DefId>,
rustc_session::config::CrateType,
rustc_session::config::EntryFnType,
rustc_session::config::OptLevel,
rustc_session::config::SymbolManglingVersion,
rustc_session::cstore::CrateDepKind,
rustc_session::cstore::ExternCrate,
rustc_session::cstore::LinkagePreference,
rustc_session::Limits,
rustc_session::lint::LintExpectationId,
rustc_span::def_id::CrateNum,
rustc_span::def_id::DefPathHash,
rustc_span::ExpnHash,
rustc_span::ExpnId,
rustc_span::Span,
rustc_span::Symbol,
rustc_span::symbol::Ident,
rustc_target::spec::PanicStrategy,
rustc_type_ir::Variance,
u32,
usize,
}
macro_rules! tcx_lifetime {
($($($fake_path:ident)::+),+ $(,)?) => {
$(
impl<'tcx> EraseType for $($fake_path)::+<'tcx> {
type Result = [u8; size_of::<$($fake_path)::+<'static>>()];
}
)*
}
}
tcx_lifetime! {
rustc_middle::hir::Owner,
rustc_middle::middle::exported_symbols::ExportedSymbol,
rustc_middle::mir::ConstantKind,
rustc_middle::mir::DestructuredConstant,
rustc_middle::mir::interpret::ConstAlloc,
rustc_middle::mir::interpret::ConstValue,
rustc_middle::mir::interpret::GlobalId,
rustc_middle::mir::interpret::LitToConstInput,
rustc_middle::traits::ChalkEnvironmentAndGoal,
rustc_middle::traits::query::MethodAutoderefStepsResult,
rustc_middle::traits::query::type_op::AscribeUserType,
rustc_middle::traits::query::type_op::Eq,
rustc_middle::traits::query::type_op::ProvePredicate,
rustc_middle::traits::query::type_op::Subtype,
rustc_middle::ty::AdtDef,
rustc_middle::ty::AliasTy,
rustc_middle::ty::Clause,
rustc_middle::ty::ClosureTypeInfo,
rustc_middle::ty::Const,
rustc_middle::ty::DestructuredConst,
rustc_middle::ty::ExistentialTraitRef,
rustc_middle::ty::FnSig,
rustc_middle::ty::GenericArg,
rustc_middle::ty::GenericPredicates,
rustc_middle::ty::inhabitedness::InhabitedPredicate,
rustc_middle::ty::Instance,
rustc_middle::ty::InstanceDef,
rustc_middle::ty::layout::FnAbiError,
rustc_middle::ty::layout::LayoutError,
rustc_middle::ty::ParamEnv,
rustc_middle::ty::Predicate,
rustc_middle::ty::SymbolName,
rustc_middle::ty::TraitRef,
rustc_middle::ty::Ty,
rustc_middle::ty::UnevaluatedConst,
rustc_middle::ty::ValTree,
rustc_middle::ty::VtblEntry,
}

View File

@ -7,6 +7,7 @@
use crate::ty::{self, print::describe_as_module, TyCtxt};
use rustc_span::def_id::LOCAL_CRATE;
pub mod erase;
mod keys;
pub use keys::{AsLocalKey, Key, LocalCrate};

View File

@ -17,6 +17,7 @@ use crate::mir::interpret::{
};
use crate::mir::interpret::{LitToConstError, LitToConstInput};
use crate::mir::mono::CodegenUnit;
use crate::query::erase::{erase, restore, Erase};
use crate::query::{AsLocalKey, Key};
use crate::thir;
use crate::traits::query::{
@ -57,6 +58,8 @@ use rustc_hir::hir_id::OwnerId;
use rustc_hir::lang_items::{LangItem, LanguageItems};
use rustc_hir::{Crate, ItemLocalId, TraitCandidate};
use rustc_index::vec::IndexVec;
pub(crate) use rustc_query_system::query::QueryJobId;
use rustc_query_system::query::*;
use rustc_session::config::{EntryFnType, OptLevel, OutputFilenames, SymbolManglingVersion};
use rustc_session::cstore::{CrateDepKind, CrateSource};
use rustc_session::cstore::{ExternCrate, ForeignModule, LinkagePreference, NativeLib};
@ -66,18 +69,19 @@ use rustc_span::symbol::Symbol;
use rustc_span::{Span, DUMMY_SP};
use rustc_target::abi;
use rustc_target::spec::PanicStrategy;
use std::marker::PhantomData;
use std::mem;
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;
pub(crate) use rustc_query_system::query::QueryJobId;
use rustc_query_system::query::*;
#[derive(Default)]
pub struct QuerySystem<'tcx> {
pub arenas: QueryArenas<'tcx>,
pub caches: QueryCaches<'tcx>,
// Since we erase query value types we tell the typesystem about them with `PhantomData`.
_phantom_values: QueryPhantomValues<'tcx>,
}
#[derive(Copy, Clone)]
@ -263,8 +267,8 @@ macro_rules! define_callbacks {
pub fn $name<'tcx>(
_tcx: TyCtxt<'tcx>,
value: query_provided::$name<'tcx>,
) -> query_values::$name<'tcx> {
query_if_arena!([$($modifiers)*]
) -> Erase<query_values::$name<'tcx>> {
erase(query_if_arena!([$($modifiers)*]
{
if mem::needs_drop::<query_provided::$name<'tcx>>() {
&*_tcx.query_system.arenas.$name.alloc(value)
@ -273,7 +277,7 @@ macro_rules! define_callbacks {
}
}
(value)
)
))
}
)*
}
@ -282,7 +286,7 @@ macro_rules! define_callbacks {
use super::*;
$(
pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache;
pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>>>::Cache;
)*
}
@ -334,6 +338,11 @@ macro_rules! define_callbacks {
}
}
#[derive(Default)]
pub struct QueryPhantomValues<'tcx> {
$($(#[$attr])* pub $name: PhantomData<query_values::$name<'tcx>>,)*
}
#[derive(Default)]
pub struct QueryCaches<'tcx> {
$($(#[$attr])* pub $name: query_storage::$name<'tcx>,)*
@ -395,10 +404,10 @@ macro_rules! define_callbacks {
let key = key.into_query_param();
opt_remap_env_constness!([$($modifiers)*][key]);
match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
restore::<$V>(match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
Some(value) => value,
None => self.tcx.queries.$name(self.tcx, self.span, key, QueryMode::Get).unwrap(),
}
})
})*
}
@ -459,7 +468,7 @@ macro_rules! define_callbacks {
span: Span,
key: query_keys::$name<'tcx>,
mode: QueryMode,
) -> Option<$V>;)*
) -> Option<Erase<$V>>;)*
}
};
}
@ -486,11 +495,13 @@ macro_rules! define_feedable {
opt_remap_env_constness!([$($modifiers)*][key]);
let tcx = self.tcx;
let value = query_provided_to_value::$name(tcx, value);
let erased = query_provided_to_value::$name(tcx, value);
let value = restore::<$V>(erased);
let cache = &tcx.query_system.caches.$name;
match try_get_cached(tcx, cache, &key) {
Some(old) => {
let old = restore::<$V>(old);
bug!(
"Trying to feed an already recorded value for query {} key={key:?}:\nold value: {old:?}\nnew value: {value:?}",
stringify!($name),
@ -505,7 +516,7 @@ macro_rules! define_feedable {
&value,
hash_result!([$($modifiers)*]),
);
cache.complete(key, value, dep_node_index);
cache.complete(key, erased, dep_node_index);
value
}
}

View File

@ -18,19 +18,21 @@ extern crate rustc_middle;
use rustc_data_structures::sync::AtomicU64;
use rustc_middle::arena::Arena;
use rustc_middle::dep_graph::{self, DepKindStruct};
use rustc_middle::dep_graph::{self, DepKind, DepKindStruct};
use rustc_middle::query::erase::{erase, restore, Erase};
use rustc_middle::query::AsLocalKey;
use rustc_middle::ty::query::{
query_keys, query_provided, query_provided_to_value, query_storage, query_values,
};
use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine};
use rustc_middle::ty::TyCtxt;
use rustc_query_system::dep_graph::SerializedDepNodeIndex;
use rustc_query_system::Value;
use rustc_span::Span;
#[macro_use]
mod plumbing;
pub use plumbing::QueryCtxt;
use rustc_query_system::dep_graph::SerializedDepNodeIndex;
use rustc_query_system::query::*;
#[cfg(parallel_compiler)]
pub use rustc_query_system::query::{deadlock, QueryContext};
@ -43,6 +45,13 @@ pub use on_disk_cache::OnDiskCache;
mod profiling_support;
pub use self::profiling_support::alloc_self_profile_query_strings;
/// This is implemented per query and restoring query values from their erased state.
trait QueryConfigRestored<'tcx>: QueryConfig<QueryCtxt<'tcx>> + Default {
type RestoredValue;
fn restore(value: <Self as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue;
}
rustc_query_append! { define_queries! }
impl<'tcx> Queries<'tcx> {

View File

@ -1064,14 +1064,14 @@ impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for [u8] {
}
}
pub fn encode_query_results<'a, 'tcx, Q>(
pub(crate) fn encode_query_results<'a, 'tcx, Q>(
query: Q,
qcx: QueryCtxt<'tcx>,
encoder: &mut CacheEncoder<'a, 'tcx>,
query_result_index: &mut EncodedDepNodeIndex,
) where
Q: super::QueryConfig<QueryCtxt<'tcx>>,
Q::Value: Encodable<CacheEncoder<'a, 'tcx>>,
Q: super::QueryConfigRestored<'tcx>,
Q::RestoredValue: Encodable<CacheEncoder<'a, 'tcx>>,
{
let _timer = qcx
.tcx
@ -1089,7 +1089,7 @@ pub fn encode_query_results<'a, 'tcx, Q>(
// Encode the type check tables with the `SerializedDepNodeIndex`
// as tag.
encoder.encode_tagged(dep_node, value);
encoder.encode_tagged(dep_node, &Q::restore(*value));
}
});
}

View File

@ -263,14 +263,14 @@ macro_rules! feedable {
}
macro_rules! hash_result {
([]) => {{
Some(dep_graph::hash_result)
([][$V:ty]) => {{
Some(|hcx, result| dep_graph::hash_result(hcx, &restore::<$V>(*result)))
}};
([(no_hash) $($rest:tt)*]) => {{
([(no_hash) $($rest:tt)*][$V:ty]) => {{
None
}};
([$other:tt $($modifiers:tt)*]) => {
hash_result!([$($modifiers)*])
([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
hash_result!([$($modifiers)*][$($args)*])
};
}
@ -479,13 +479,18 @@ macro_rules! define_queries {
$(impl<'tcx> QueryConfig<QueryCtxt<'tcx>> for queries::$name<'tcx> {
type Key = query_keys::$name<'tcx>;
type Value = query_values::$name<'tcx>;
type Value = Erase<query_values::$name<'tcx>>;
#[inline(always)]
fn name(self) -> &'static str {
stringify!($name)
}
#[inline]
fn format_value(self) -> fn(&Self::Value) -> String {
|value| format!("{:?}", restore::<query_values::$name<'tcx>>(*value))
}
#[inline]
fn cache_on_disk(self, tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool {
::rustc_middle::query::cached::$name(tcx, key)
@ -508,7 +513,7 @@ macro_rules! define_queries {
}
fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
tcx.$name(key)
erase(tcx.$name(key))
}
#[inline]
@ -558,6 +563,16 @@ macro_rules! define_queries {
})
}
#[inline]
fn value_from_cycle_error(
self,
tcx: TyCtxt<'tcx>,
cycle: &[QueryInfo<DepKind>],
) -> Self::Value {
let result: query_values::$name<'tcx> = Value::from_cycle_error(tcx, cycle);
erase(result)
}
#[inline(always)]
fn anon(self) -> bool {
is_anon!([$($modifiers)*])
@ -590,7 +605,16 @@ macro_rules! define_queries {
#[inline(always)]
fn hash_result(self) -> rustc_query_system::query::HashResult<Self::Value> {
hash_result!([$($modifiers)*])
hash_result!([$($modifiers)*][query_values::$name<'tcx>])
}
})*
$(impl<'tcx> QueryConfigRestored<'tcx> for queries::$name<'tcx> {
type RestoredValue = query_values::$name<'tcx>;
#[inline(always)]
fn restore(value: <Self as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue {
restore::<query_values::$name<'tcx>>(value)
}
})*
@ -708,7 +732,7 @@ macro_rules! define_queries {
)
},
encode_query_results: expand_if_cached!([$($modifiers)*], |qcx, encoder, query_result_index|
$crate::on_disk_cache::encode_query_results(
$crate::on_disk_cache::encode_query_results::<super::queries::$name<'tcx>>(
super::queries::$name::default(),
qcx,
encoder,
@ -793,14 +817,14 @@ macro_rules! define_queries_struct {
$($(#[$attr])*
#[inline(always)]
#[tracing::instrument(level = "trace", skip(self, tcx), ret)]
#[tracing::instrument(level = "trace", skip(self, tcx))]
fn $name(
&'tcx self,
tcx: TyCtxt<'tcx>,
span: Span,
key: <queries::$name<'tcx> as QueryConfig<QueryCtxt<'tcx>>>::Key,
key: query_keys::$name<'tcx>,
mode: QueryMode,
) -> Option<query_values::$name<'tcx>> {
) -> Option<Erase<query_values::$name<'tcx>>> {
let qcx = QueryCtxt { tcx, queries: self };
get_query(
queries::$name::default(),

View File

@ -538,7 +538,14 @@ impl<K: DepKind> DepGraph<K> {
if let Some(prev_index) = data.previous.node_to_index_opt(&node) {
let dep_node_index = data.current.prev_index_to_index.lock()[prev_index];
if let Some(dep_node_index) = dep_node_index {
crate::query::incremental_verify_ich(cx, data, result, prev_index, hash_result);
crate::query::incremental_verify_ich(
cx,
data,
result,
prev_index,
hash_result,
|value| format!("{:?}", value),
);
#[cfg(debug_assertions)]
if hash_result.is_some() {

View File

@ -18,7 +18,7 @@ pub trait CacheSelector<'tcx, V> {
pub trait QueryCache: Sized {
type Key: Hash + Eq + Copy + Debug;
type Value: Copy + Debug;
type Value: Copy;
/// Checks if the query is already computed and in the cache.
fn lookup(&self, key: &Self::Key) -> Option<(Self::Value, DepNodeIndex)>;
@ -52,7 +52,7 @@ impl<K, V> Default for DefaultCache<K, V> {
impl<K, V> QueryCache for DefaultCache<K, V>
where
K: Eq + Hash + Copy + Debug,
V: Copy + Debug,
V: Copy,
{
type Key = K;
type Value = V;
@ -120,7 +120,7 @@ impl<V> Default for SingleCache<V> {
impl<V> QueryCache for SingleCache<V>
where
V: Copy + Debug,
V: Copy,
{
type Key = ();
type Value = V;
@ -164,7 +164,7 @@ impl<K: Idx, V> Default for VecCache<K, V> {
impl<K, V> QueryCache for VecCache<K, V>
where
K: Eq + Idx + Copy + Debug,
V: Copy + Debug,
V: Copy,
{
type Key = K;
type Value = V;

View File

@ -4,7 +4,7 @@ use crate::dep_graph::{DepNode, DepNodeParams, SerializedDepNodeIndex};
use crate::error::HandleCycleError;
use crate::ich::StableHashingContext;
use crate::query::caches::QueryCache;
use crate::query::{QueryContext, QueryState};
use crate::query::{QueryContext, QueryInfo, QueryState};
use rustc_data_structures::fingerprint::Fingerprint;
use std::fmt::Debug;
@ -20,10 +20,12 @@ pub trait QueryConfig<Qcx: QueryContext>: Copy {
// `Key` and `Value` are `Copy` instead of `Clone` to ensure copying them stays cheap,
// but it isn't necessary.
type Key: DepNodeParams<Qcx::DepContext> + Eq + Hash + Copy + Debug;
type Value: Debug + Copy;
type Value: Copy;
type Cache: QueryCache<Key = Self::Key, Value = Self::Value>;
fn format_value(self) -> fn(&Self::Value) -> String;
// Don't use this method to access query results, instead use the methods on TyCtxt
fn query_state<'a>(self, tcx: Qcx) -> &'a QueryState<Self::Key, Qcx::DepKind>
where
@ -45,6 +47,13 @@ pub trait QueryConfig<Qcx: QueryContext>: Copy {
fn loadable_from_disk(self, qcx: Qcx, key: &Self::Key, idx: SerializedDepNodeIndex) -> bool;
/// Synthesize an error value to let compilation continue after a cycle.
fn value_from_cycle_error(
self,
tcx: Qcx::DepContext,
cycle: &[QueryInfo<Qcx::DepKind>],
) -> Self::Value;
fn anon(self) -> bool;
fn eval_always(self) -> bool;
fn depth_limit(self) -> bool;

View File

@ -11,7 +11,6 @@ use crate::query::job::QueryLatch;
use crate::query::job::{report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo};
use crate::query::SerializedDepNodeIndex;
use crate::query::{QueryContext, QueryMap, QuerySideEffects, QueryStackFrame};
use crate::values::Value;
use crate::HandleCycleError;
use rustc_data_structures::fingerprint::Fingerprint;
use rustc_data_structures::fx::FxHashMap;
@ -120,43 +119,45 @@ where
#[cold]
#[inline(never)]
fn mk_cycle<Qcx, R, D: DepKind>(
fn mk_cycle<Q, Qcx>(
query: Q,
qcx: Qcx,
cycle_error: CycleError<D>,
cycle_error: CycleError<Qcx::DepKind>,
handler: HandleCycleError,
) -> R
) -> Q::Value
where
Qcx: QueryContext + HasDepContext<DepKind = D>,
R: std::fmt::Debug + Value<Qcx::DepContext, Qcx::DepKind>,
Q: QueryConfig<Qcx>,
Qcx: QueryContext,
{
let error = report_cycle(qcx.dep_context().sess(), &cycle_error);
handle_cycle_error(*qcx.dep_context(), &cycle_error, error, handler)
handle_cycle_error(query, qcx, &cycle_error, error, handler)
}
fn handle_cycle_error<Tcx, V>(
tcx: Tcx,
cycle_error: &CycleError<Tcx::DepKind>,
fn handle_cycle_error<Q, Qcx>(
query: Q,
qcx: Qcx,
cycle_error: &CycleError<Qcx::DepKind>,
mut error: DiagnosticBuilder<'_, ErrorGuaranteed>,
handler: HandleCycleError,
) -> V
) -> Q::Value
where
Tcx: DepContext,
V: Value<Tcx, Tcx::DepKind>,
Q: QueryConfig<Qcx>,
Qcx: QueryContext,
{
use HandleCycleError::*;
match handler {
Error => {
error.emit();
Value::from_cycle_error(tcx, &cycle_error.cycle)
query.value_from_cycle_error(*qcx.dep_context(), &cycle_error.cycle)
}
Fatal => {
error.emit();
tcx.sess().abort_if_errors();
qcx.dep_context().sess().abort_if_errors();
unreachable!()
}
DelayBug => {
error.delay_as_bug();
Value::from_cycle_error(tcx, &cycle_error.cycle)
query.value_from_cycle_error(*qcx.dep_context(), &cycle_error.cycle)
}
}
}
@ -269,7 +270,7 @@ where
&qcx.current_query_job(),
span,
);
(mk_cycle(qcx, error, query.handle_cycle_error()), None)
(mk_cycle(query, qcx, error, query.handle_cycle_error()), None)
}
#[inline(always)]
@ -306,7 +307,7 @@ where
(v, Some(index))
}
Err(cycle) => (mk_cycle(qcx, cycle, query.handle_cycle_error()), None),
Err(cycle) => (mk_cycle(query, qcx, cycle, query.handle_cycle_error()), None),
}
}
@ -410,7 +411,8 @@ where
// get evaluated first, and re-feed the query.
if let Some((cached_result, _)) = cache.lookup(&key) {
panic!(
"fed query later has its value computed. The already cached value: {cached_result:?}"
"fed query later has its value computed. The already cached value: {}",
(query.format_value())(&cached_result)
);
}
}
@ -581,6 +583,7 @@ where
&result,
prev_dep_node_index,
query.hash_result(),
query.format_value(),
);
}
@ -626,19 +629,21 @@ where
&result,
prev_dep_node_index,
query.hash_result(),
query.format_value(),
);
Some((result, dep_node_index))
}
#[inline]
#[instrument(skip(tcx, dep_graph_data, result, hash_result), level = "debug")]
pub(crate) fn incremental_verify_ich<Tcx, V: Debug>(
#[instrument(skip(tcx, dep_graph_data, result, hash_result, format_value), level = "debug")]
pub(crate) fn incremental_verify_ich<Tcx, V>(
tcx: Tcx,
dep_graph_data: &DepGraphData<Tcx::DepKind>,
result: &V,
prev_index: SerializedDepNodeIndex,
hash_result: Option<fn(&mut StableHashingContext<'_>, &V) -> Fingerprint>,
format_value: fn(&V) -> String,
) where
Tcx: DepContext,
{
@ -653,7 +658,7 @@ pub(crate) fn incremental_verify_ich<Tcx, V: Debug>(
let old_hash = dep_graph_data.prev_fingerprint_of(prev_index);
if new_hash != old_hash {
incremental_verify_ich_failed(tcx, prev_index, result);
incremental_verify_ich_failed(tcx, prev_index, &|| format_value(&result));
}
}
@ -677,7 +682,7 @@ where
fn incremental_verify_ich_failed<Tcx>(
tcx: Tcx,
prev_index: SerializedDepNodeIndex,
result: &dyn Debug,
result: &dyn Fn() -> String,
) where
Tcx: DepContext,
{
@ -707,7 +712,7 @@ fn incremental_verify_ich_failed<Tcx>(
run_cmd,
dep_node: format!("{dep_node:?}"),
});
panic!("Found unstable fingerprints for {dep_node:?}: {result:?}");
panic!("Found unstable fingerprints for {dep_node:?}: {}", result());
}
INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.set(old_in_panic));