mirror of
https://github.com/rust-lang/rust.git
synced 2025-04-28 02:57:37 +00:00
Auto merge of #107833 - Zoxc:arena-query-clean, r=cjgillot
Factor query arena allocation out from query caches This moves the logic for arena allocation out from the query caches into conditional code in the query system. The specialized arena caches are removed. A new `QuerySystem` type is added in `rustc_middle` which contains the arenas, providers and query caches. Performance seems to be slightly regressed: <table><tr><td rowspan="2">Benchmark</td><td colspan="1"><b>Before</b></th><td colspan="2"><b>After</b></th></tr><tr><td align="right">Time</td><td align="right">Time</td><td align="right">%</th></tr><tr><td>🟣 <b>clap</b>:check</td><td align="right">1.8053s</td><td align="right">1.8109s</td><td align="right"> 0.31%</td></tr><tr><td>🟣 <b>hyper</b>:check</td><td align="right">0.2600s</td><td align="right">0.2597s</td><td align="right"> -0.10%</td></tr><tr><td>🟣 <b>regex</b>:check</td><td align="right">0.9973s</td><td align="right">1.0006s</td><td align="right"> 0.34%</td></tr><tr><td>🟣 <b>syn</b>:check</td><td align="right">1.6048s</td><td align="right">1.6051s</td><td align="right"> 0.02%</td></tr><tr><td>🟣 <b>syntex_syntax</b>:check</td><td align="right">6.2992s</td><td align="right">6.3159s</td><td align="right"> 0.26%</td></tr><tr><td>Total</td><td align="right">10.9664s</td><td align="right">10.9922s</td><td align="right"> 0.23%</td></tr><tr><td>Summary</td><td align="right">1.0000s</td><td align="right">1.0017s</td><td align="right"> 0.17%</td></tr></table> Incremental performance is a bit worse: <table><tr><td rowspan="2">Benchmark</td><td colspan="1"><b>Before</b></th><td colspan="2"><b>After</b></th></tr><tr><td align="right">Time</td><td align="right">Time</td><td align="right">%</th></tr><tr><td>🟣 <b>clap</b>:check:initial</td><td align="right">2.2103s</td><td align="right">2.2247s</td><td align="right"> 0.65%</td></tr><tr><td>🟣 <b>hyper</b>:check:initial</td><td align="right">0.3335s</td><td align="right">0.3349s</td><td align="right"> 0.41%</td></tr><tr><td>🟣 <b>regex</b>:check:initial</td><td align="right">1.2597s</td><td align="right">1.2650s</td><td align="right"> 0.42%</td></tr><tr><td>🟣 <b>syn</b>:check:initial</td><td align="right">2.0521s</td><td align="right">2.0613s</td><td align="right"> 0.45%</td></tr><tr><td>🟣 <b>syntex_syntax</b>:check:initial</td><td align="right">7.8275s</td><td align="right">7.8583s</td><td align="right"> 0.39%</td></tr><tr><td>Total</td><td align="right">13.6832s</td><td align="right">13.7442s</td><td align="right"> 0.45%</td></tr><tr><td>Summary</td><td align="right">1.0000s</td><td align="right">1.0046s</td><td align="right"> 0.46%</td></tr></table> It does seem like LLVM optimizers struggle a bit with the current state of the query system. Based on top of https://github.com/rust-lang/rust/pull/107782 and https://github.com/rust-lang/rust/pull/107802. r? `@cjgillot`
This commit is contained in:
bors
commit
947b696ce0
@ -114,7 +114,7 @@ macro_rules! provide_one {
|
||||
fn $name<'tcx>(
|
||||
$tcx: TyCtxt<'tcx>,
|
||||
def_id_arg: ty::query::query_keys::$name<'tcx>,
|
||||
) -> ty::query::query_values::$name<'tcx> {
|
||||
) -> ty::query::query_provided::$name<'tcx> {
|
||||
let _prof_timer =
|
||||
$tcx.prof.generic_activity(concat!("metadata_decode_entry_", stringify!($name)));
|
||||
|
||||
|
||||
@ -520,7 +520,7 @@ pub struct GlobalCtxt<'tcx> {
|
||||
pub on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
|
||||
|
||||
pub queries: &'tcx dyn query::QueryEngine<'tcx>,
|
||||
pub query_caches: query::QueryCaches<'tcx>,
|
||||
pub query_system: query::QuerySystem<'tcx>,
|
||||
pub(crate) query_kinds: &'tcx [DepKindStruct<'tcx>],
|
||||
|
||||
// Internal caches for metadata decoding. No need to track deps on this.
|
||||
@ -705,7 +705,7 @@ impl<'tcx> TyCtxt<'tcx> {
|
||||
untracked,
|
||||
on_disk_cache,
|
||||
queries,
|
||||
query_caches: query::QueryCaches::default(),
|
||||
query_system: Default::default(),
|
||||
query_kinds,
|
||||
ty_rcache: Default::default(),
|
||||
pred_rcache: Default::default(),
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
#![allow(unused_parens)]
|
||||
|
||||
use crate::dep_graph;
|
||||
use crate::infer::canonical::{self, Canonical};
|
||||
use crate::lint::LintExpectation;
|
||||
@ -34,6 +36,7 @@ use crate::ty::subst::{GenericArg, SubstsRef};
|
||||
use crate::ty::util::AlwaysRequiresDrop;
|
||||
use crate::ty::GeneratorDiagnosticData;
|
||||
use crate::ty::{self, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt, UnusedGenericParams};
|
||||
use rustc_arena::TypedArena;
|
||||
use rustc_ast as ast;
|
||||
use rustc_ast::expand::allocator::AllocatorKind;
|
||||
use rustc_attr as attr;
|
||||
@ -41,6 +44,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet};
|
||||
use rustc_data_structures::steal::Steal;
|
||||
use rustc_data_structures::svh::Svh;
|
||||
use rustc_data_structures::sync::Lrc;
|
||||
use rustc_data_structures::sync::WorkerLocal;
|
||||
use rustc_data_structures::unord::UnordSet;
|
||||
use rustc_errors::ErrorGuaranteed;
|
||||
use rustc_hir as hir;
|
||||
@ -59,6 +63,7 @@ use rustc_span::symbol::Symbol;
|
||||
use rustc_span::{Span, DUMMY_SP};
|
||||
use rustc_target::abi;
|
||||
use rustc_target::spec::PanicStrategy;
|
||||
use std::mem;
|
||||
use std::ops::Deref;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
@ -66,6 +71,12 @@ use std::sync::Arc;
|
||||
pub(crate) use rustc_query_system::query::QueryJobId;
|
||||
use rustc_query_system::query::*;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct QuerySystem<'tcx> {
|
||||
pub arenas: QueryArenas<'tcx>,
|
||||
pub caches: QueryCaches<'tcx>,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct TyCtxtAt<'tcx> {
|
||||
pub tcx: TyCtxt<'tcx>,
|
||||
@ -112,10 +123,10 @@ macro_rules! query_helper_param_ty {
|
||||
}
|
||||
|
||||
macro_rules! query_if_arena {
|
||||
([] $arena:ty, $no_arena:ty) => {
|
||||
([] $arena:tt $no_arena:tt) => {
|
||||
$no_arena
|
||||
};
|
||||
([(arena_cache) $($rest:tt)*] $arena:ty, $no_arena:ty) => {
|
||||
([(arena_cache) $($rest:tt)*] $arena:tt $no_arena:tt) => {
|
||||
$arena
|
||||
};
|
||||
([$other:tt $($modifiers:tt)*]$($args:tt)*) => {
|
||||
@ -131,7 +142,7 @@ macro_rules! separate_provide_extern_decl {
|
||||
for<'tcx> fn(
|
||||
TyCtxt<'tcx>,
|
||||
query_keys::$name<'tcx>,
|
||||
) -> query_values::$name<'tcx>
|
||||
) -> query_provided::$name<'tcx>
|
||||
};
|
||||
([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
|
||||
separate_provide_extern_decl!([$($modifiers)*][$($args)*])
|
||||
@ -183,30 +194,77 @@ macro_rules! define_callbacks {
|
||||
|
||||
$(pub type $name<'tcx> = $($K)*;)*
|
||||
}
|
||||
#[allow(nonstandard_style, unused_lifetimes, unused_parens)]
|
||||
#[allow(nonstandard_style, unused_lifetimes)]
|
||||
pub mod query_values {
|
||||
use super::*;
|
||||
|
||||
$(pub type $name<'tcx> = query_if_arena!([$($modifiers)*] <$V as Deref>::Target, $V);)*
|
||||
$(pub type $name<'tcx> = $V;)*
|
||||
}
|
||||
#[allow(nonstandard_style, unused_lifetimes, unused_parens)]
|
||||
|
||||
/// This module specifies the type returned from query providers and the type used for
|
||||
/// decoding. For regular queries this is the declared returned type `V`, but
|
||||
/// `arena_cache` will use `<V as Deref>::Target` instead.
|
||||
#[allow(nonstandard_style, unused_lifetimes)]
|
||||
pub mod query_provided {
|
||||
use super::*;
|
||||
|
||||
$(
|
||||
pub type $name<'tcx> = query_if_arena!([$($modifiers)*] (<$V as Deref>::Target) ($V));
|
||||
)*
|
||||
}
|
||||
|
||||
/// This module has a function per query which takes a `query_provided` value and coverts
|
||||
/// it to a regular `V` value by allocating it on an arena if the query has the
|
||||
/// `arena_cache` modifier. This will happen when computing the query using a provider or
|
||||
/// decoding a stored result.
|
||||
#[allow(nonstandard_style, unused_lifetimes)]
|
||||
pub mod query_provided_to_value {
|
||||
use super::*;
|
||||
|
||||
$(
|
||||
#[inline(always)]
|
||||
pub fn $name<'tcx>(
|
||||
_tcx: TyCtxt<'tcx>,
|
||||
value: query_provided::$name<'tcx>,
|
||||
) -> query_values::$name<'tcx> {
|
||||
query_if_arena!([$($modifiers)*]
|
||||
{
|
||||
if mem::needs_drop::<query_provided::$name<'tcx>>() {
|
||||
&*_tcx.query_system.arenas.$name.alloc(value)
|
||||
} else {
|
||||
&*_tcx.arena.dropless.alloc(value)
|
||||
}
|
||||
}
|
||||
(value)
|
||||
)
|
||||
}
|
||||
)*
|
||||
}
|
||||
#[allow(nonstandard_style, unused_lifetimes)]
|
||||
pub mod query_storage {
|
||||
use super::*;
|
||||
|
||||
$(
|
||||
pub type $name<'tcx> = query_if_arena!([$($modifiers)*]
|
||||
<<$($K)* as Key>::CacheSelector
|
||||
as CacheSelector<'tcx, <$V as Deref>::Target>>::ArenaCache,
|
||||
<<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache
|
||||
);
|
||||
pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache;
|
||||
)*
|
||||
}
|
||||
|
||||
#[allow(nonstandard_style, unused_lifetimes)]
|
||||
pub mod query_stored {
|
||||
use super::*;
|
||||
pub struct QueryArenas<'tcx> {
|
||||
$($(#[$attr])* pub $name: query_if_arena!([$($modifiers)*]
|
||||
(WorkerLocal<TypedArena<<$V as Deref>::Target>>)
|
||||
()
|
||||
),)*
|
||||
}
|
||||
|
||||
$(pub type $name<'tcx> = $V;)*
|
||||
impl Default for QueryArenas<'_> {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
$($name: query_if_arena!([$($modifiers)*]
|
||||
(WorkerLocal::new(|_| Default::default()))
|
||||
()
|
||||
),)*
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
@ -221,7 +279,7 @@ macro_rules! define_callbacks {
|
||||
let key = key.into_query_param();
|
||||
opt_remap_env_constness!([$($modifiers)*][key]);
|
||||
|
||||
match try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key) {
|
||||
match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
|
||||
Some(_) => return,
|
||||
None => self.tcx.queries.$name(self.tcx, DUMMY_SP, key, QueryMode::Ensure),
|
||||
};
|
||||
@ -246,7 +304,7 @@ macro_rules! define_callbacks {
|
||||
let key = key.into_query_param();
|
||||
opt_remap_env_constness!([$($modifiers)*][key]);
|
||||
|
||||
match try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key) {
|
||||
match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
|
||||
Some(value) => value,
|
||||
None => self.tcx.queries.$name(self.tcx, self.span, key, QueryMode::Get).unwrap(),
|
||||
}
|
||||
@ -257,7 +315,7 @@ macro_rules! define_callbacks {
|
||||
$(pub $name: for<'tcx> fn(
|
||||
TyCtxt<'tcx>,
|
||||
query_keys::$name<'tcx>,
|
||||
) -> query_values::$name<'tcx>,)*
|
||||
) -> query_provided::$name<'tcx>,)*
|
||||
}
|
||||
|
||||
pub struct ExternProviders {
|
||||
@ -334,12 +392,13 @@ macro_rules! define_feedable {
|
||||
$(impl<'tcx, K: IntoQueryParam<$($K)*> + Copy> TyCtxtFeed<'tcx, K> {
|
||||
$(#[$attr])*
|
||||
#[inline(always)]
|
||||
pub fn $name(self, value: query_values::$name<'tcx>) -> $V {
|
||||
pub fn $name(self, value: query_provided::$name<'tcx>) -> $V {
|
||||
let key = self.key().into_query_param();
|
||||
opt_remap_env_constness!([$($modifiers)*][key]);
|
||||
|
||||
let tcx = self.tcx;
|
||||
let cache = &tcx.query_caches.$name;
|
||||
let value = query_provided_to_value::$name(tcx, value);
|
||||
let cache = &tcx.query_system.caches.$name;
|
||||
|
||||
match try_get_cached(tcx, cache, &key) {
|
||||
Some(old) => {
|
||||
@ -357,7 +416,8 @@ macro_rules! define_feedable {
|
||||
&value,
|
||||
hash_result!([$($modifiers)*]),
|
||||
);
|
||||
cache.complete(key, value, dep_node_index)
|
||||
cache.complete(key, value, dep_node_index);
|
||||
value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -21,7 +21,9 @@ use rustc_data_structures::sync::AtomicU64;
|
||||
use rustc_middle::arena::Arena;
|
||||
use rustc_middle::dep_graph::{self, DepKindStruct};
|
||||
use rustc_middle::query::Key;
|
||||
use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_values};
|
||||
use rustc_middle::ty::query::{
|
||||
query_keys, query_provided, query_provided_to_value, query_storage, query_values,
|
||||
};
|
||||
use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine};
|
||||
use rustc_middle::ty::TyCtxt;
|
||||
use rustc_span::Span;
|
||||
|
||||
@ -293,14 +293,14 @@ macro_rules! get_provider {
|
||||
}
|
||||
|
||||
macro_rules! should_ever_cache_on_disk {
|
||||
([]) => {{
|
||||
None
|
||||
([]$yes:tt $no:tt) => {{
|
||||
$no
|
||||
}};
|
||||
([(cache) $($rest:tt)*]) => {{
|
||||
Some($crate::plumbing::try_load_from_disk::<Self::Value>)
|
||||
([(cache) $($rest:tt)*]$yes:tt $no:tt) => {{
|
||||
$yes
|
||||
}};
|
||||
([$other:tt $($modifiers:tt)*]) => {
|
||||
should_ever_cache_on_disk!([$($modifiers)*])
|
||||
([$other:tt $($modifiers:tt)*]$yes:tt $no:tt) => {
|
||||
should_ever_cache_on_disk!([$($modifiers)*]$yes $no)
|
||||
};
|
||||
}
|
||||
|
||||
@ -472,7 +472,6 @@ macro_rules! define_queries {
|
||||
$(impl<'tcx> QueryConfig<QueryCtxt<'tcx>> for queries::$name<'tcx> {
|
||||
type Key = query_keys::$name<'tcx>;
|
||||
type Value = query_values::$name<'tcx>;
|
||||
type Stored = query_stored::$name<'tcx>;
|
||||
const NAME: &'static str = stringify!($name);
|
||||
|
||||
#[inline]
|
||||
@ -493,24 +492,39 @@ macro_rules! define_queries {
|
||||
fn query_cache<'a>(tcx: QueryCtxt<'tcx>) -> &'a Self::Cache
|
||||
where 'tcx:'a
|
||||
{
|
||||
&tcx.query_caches.$name
|
||||
&tcx.query_system.caches.$name
|
||||
}
|
||||
|
||||
fn execute_query(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Stored {
|
||||
fn execute_query(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
|
||||
tcx.$name(key)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
// key is only sometimes used
|
||||
#[allow(unused_variables)]
|
||||
fn compute(qcx: QueryCtxt<'tcx>, key: &Self::Key) -> fn(TyCtxt<'tcx>, Self::Key) -> Self::Value {
|
||||
get_provider!([$($modifiers)*][qcx, $name, key])
|
||||
fn compute(qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value {
|
||||
query_provided_to_value::$name(
|
||||
qcx.tcx,
|
||||
get_provider!([$($modifiers)*][qcx, $name, key])(qcx.tcx, key)
|
||||
)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn try_load_from_disk(qcx: QueryCtxt<'tcx>, key: &Self::Key) -> rustc_query_system::query::TryLoadFromDisk<QueryCtxt<'tcx>, Self> {
|
||||
let cache_on_disk = Self::cache_on_disk(qcx.tcx, key);
|
||||
if cache_on_disk { should_ever_cache_on_disk!([$($modifiers)*]) } else { None }
|
||||
fn try_load_from_disk(_qcx: QueryCtxt<'tcx>, _key: &Self::Key) -> rustc_query_system::query::TryLoadFromDisk<QueryCtxt<'tcx>, Self> {
|
||||
should_ever_cache_on_disk!([$($modifiers)*] {
|
||||
if Self::cache_on_disk(_qcx.tcx, _key) {
|
||||
Some(|qcx: QueryCtxt<'tcx>, dep_node| {
|
||||
let value = $crate::plumbing::try_load_from_disk::<query_provided::$name<'tcx>>(
|
||||
qcx,
|
||||
dep_node
|
||||
);
|
||||
value.map(|value| query_provided_to_value::$name(qcx.tcx, value))
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} {
|
||||
None
|
||||
})
|
||||
}
|
||||
|
||||
const ANON: bool = is_anon!([$($modifiers)*]);
|
||||
@ -633,7 +647,7 @@ macro_rules! define_queries {
|
||||
$crate::profiling_support::alloc_self_profile_query_strings_for_query_cache(
|
||||
tcx,
|
||||
stringify!($name),
|
||||
&tcx.query_caches.$name,
|
||||
&tcx.query_system.caches.$name,
|
||||
string_cache,
|
||||
)
|
||||
},
|
||||
@ -725,7 +739,7 @@ macro_rules! define_queries_struct {
|
||||
span: Span,
|
||||
key: <queries::$name<'tcx> as QueryConfig<QueryCtxt<'tcx>>>::Key,
|
||||
mode: QueryMode,
|
||||
) -> Option<query_stored::$name<'tcx>> {
|
||||
) -> Option<query_values::$name<'tcx>> {
|
||||
let qcx = QueryCtxt { tcx, queries: self };
|
||||
get_query::<queries::$name<'tcx>, _, rustc_middle::dep_graph::DepKind>(qcx, span, key, mode)
|
||||
})*
|
||||
|
||||
@ -1,12 +1,10 @@
|
||||
use crate::dep_graph::DepNodeIndex;
|
||||
|
||||
use rustc_arena::TypedArena;
|
||||
use rustc_data_structures::fx::FxHashMap;
|
||||
use rustc_data_structures::sharded;
|
||||
#[cfg(parallel_compiler)]
|
||||
use rustc_data_structures::sharded::Sharded;
|
||||
use rustc_data_structures::sync::Lock;
|
||||
use rustc_data_structures::sync::WorkerLocal;
|
||||
use rustc_index::vec::{Idx, IndexVec};
|
||||
use std::fmt::Debug;
|
||||
use std::hash::Hash;
|
||||
@ -16,12 +14,10 @@ pub trait CacheSelector<'tcx, V> {
|
||||
type Cache
|
||||
where
|
||||
V: Copy;
|
||||
type ArenaCache;
|
||||
}
|
||||
|
||||
pub trait QueryStorage {
|
||||
type Value: Debug;
|
||||
type Stored: Copy;
|
||||
type Value: Copy;
|
||||
}
|
||||
|
||||
pub trait QueryCache: QueryStorage + Sized {
|
||||
@ -31,9 +27,9 @@ pub trait QueryCache: QueryStorage + Sized {
|
||||
/// It returns the shard index and a lock guard to the shard,
|
||||
/// which will be used if the query is not in the cache and we need
|
||||
/// to compute it.
|
||||
fn lookup(&self, key: &Self::Key) -> Option<(Self::Stored, DepNodeIndex)>;
|
||||
fn lookup(&self, key: &Self::Key) -> Option<(Self::Value, DepNodeIndex)>;
|
||||
|
||||
fn complete(&self, key: Self::Key, value: Self::Value, index: DepNodeIndex) -> Self::Stored;
|
||||
fn complete(&self, key: Self::Key, value: Self::Value, index: DepNodeIndex);
|
||||
|
||||
fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex));
|
||||
}
|
||||
@ -44,7 +40,6 @@ impl<'tcx, K: Eq + Hash, V: 'tcx> CacheSelector<'tcx, V> for DefaultCacheSelecto
|
||||
type Cache = DefaultCache<K, V>
|
||||
where
|
||||
V: Copy;
|
||||
type ArenaCache = ArenaCache<'tcx, K, V>;
|
||||
}
|
||||
|
||||
pub struct DefaultCache<K, V> {
|
||||
@ -62,7 +57,6 @@ impl<K, V> Default for DefaultCache<K, V> {
|
||||
|
||||
impl<K: Eq + Hash, V: Copy + Debug> QueryStorage for DefaultCache<K, V> {
|
||||
type Value = V;
|
||||
type Stored = V;
|
||||
}
|
||||
|
||||
impl<K, V> QueryCache for DefaultCache<K, V>
|
||||
@ -85,7 +79,7 @@ where
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
|
||||
fn complete(&self, key: K, value: V, index: DepNodeIndex) {
|
||||
#[cfg(parallel_compiler)]
|
||||
let mut lock = self.cache.get_shard_by_value(&key).lock();
|
||||
#[cfg(not(parallel_compiler))]
|
||||
@ -93,7 +87,6 @@ where
|
||||
// We may be overwriting another value. This is all right, since the dep-graph
|
||||
// will check that the fingerprint matches.
|
||||
lock.insert(key, (value, index));
|
||||
value
|
||||
}
|
||||
|
||||
fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
|
||||
@ -122,7 +115,6 @@ impl<'tcx, V: 'tcx> CacheSelector<'tcx, V> for SingleCacheSelector {
|
||||
type Cache = SingleCache<V>
|
||||
where
|
||||
V: Copy;
|
||||
type ArenaCache = ArenaCache<'tcx, (), V>;
|
||||
}
|
||||
|
||||
pub struct SingleCache<V> {
|
||||
@ -137,7 +129,6 @@ impl<V> Default for SingleCache<V> {
|
||||
|
||||
impl<V: Copy + Debug> QueryStorage for SingleCache<V> {
|
||||
type Value = V;
|
||||
type Stored = V;
|
||||
}
|
||||
|
||||
impl<V> QueryCache for SingleCache<V>
|
||||
@ -152,9 +143,8 @@ where
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn complete(&self, _key: (), value: V, index: DepNodeIndex) -> Self::Stored {
|
||||
fn complete(&self, _key: (), value: V, index: DepNodeIndex) {
|
||||
*self.cache.lock() = Some((value, index));
|
||||
value
|
||||
}
|
||||
|
||||
fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
|
||||
@ -162,85 +152,12 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ArenaCache<'tcx, K, V> {
|
||||
arena: WorkerLocal<TypedArena<(V, DepNodeIndex)>>,
|
||||
#[cfg(parallel_compiler)]
|
||||
cache: Sharded<FxHashMap<K, &'tcx (V, DepNodeIndex)>>,
|
||||
#[cfg(not(parallel_compiler))]
|
||||
cache: Lock<FxHashMap<K, &'tcx (V, DepNodeIndex)>>,
|
||||
}
|
||||
|
||||
impl<'tcx, K, V> Default for ArenaCache<'tcx, K, V> {
|
||||
fn default() -> Self {
|
||||
ArenaCache { arena: WorkerLocal::new(|_| TypedArena::default()), cache: Default::default() }
|
||||
}
|
||||
}
|
||||
|
||||
impl<'tcx, K: Eq + Hash, V: Debug + 'tcx> QueryStorage for ArenaCache<'tcx, K, V> {
|
||||
type Value = V;
|
||||
type Stored = &'tcx V;
|
||||
}
|
||||
|
||||
impl<'tcx, K, V: 'tcx> QueryCache for ArenaCache<'tcx, K, V>
|
||||
where
|
||||
K: Eq + Hash + Clone + Debug,
|
||||
V: Debug,
|
||||
{
|
||||
type Key = K;
|
||||
|
||||
#[inline(always)]
|
||||
fn lookup(&self, key: &K) -> Option<(&'tcx V, DepNodeIndex)> {
|
||||
let key_hash = sharded::make_hash(key);
|
||||
#[cfg(parallel_compiler)]
|
||||
let lock = self.cache.get_shard_by_hash(key_hash).lock();
|
||||
#[cfg(not(parallel_compiler))]
|
||||
let lock = self.cache.lock();
|
||||
let result = lock.raw_entry().from_key_hashed_nocheck(key_hash, key);
|
||||
|
||||
if let Some((_, value)) = result { Some((&value.0, value.1)) } else { None }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
|
||||
let value = self.arena.alloc((value, index));
|
||||
let value = unsafe { &*(value as *const _) };
|
||||
#[cfg(parallel_compiler)]
|
||||
let mut lock = self.cache.get_shard_by_value(&key).lock();
|
||||
#[cfg(not(parallel_compiler))]
|
||||
let mut lock = self.cache.lock();
|
||||
// We may be overwriting another value. This is all right, since the dep-graph
|
||||
// will check that the fingerprint matches.
|
||||
lock.insert(key, value);
|
||||
&value.0
|
||||
}
|
||||
|
||||
fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
|
||||
#[cfg(parallel_compiler)]
|
||||
{
|
||||
let shards = self.cache.lock_shards();
|
||||
for shard in shards.iter() {
|
||||
for (k, v) in shard.iter() {
|
||||
f(k, &v.0, v.1);
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(not(parallel_compiler))]
|
||||
{
|
||||
let map = self.cache.lock();
|
||||
for (k, v) in map.iter() {
|
||||
f(k, &v.0, v.1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct VecCacheSelector<K>(PhantomData<K>);
|
||||
|
||||
impl<'tcx, K: Idx, V: 'tcx> CacheSelector<'tcx, V> for VecCacheSelector<K> {
|
||||
type Cache = VecCache<K, V>
|
||||
where
|
||||
V: Copy;
|
||||
type ArenaCache = VecArenaCache<'tcx, K, V>;
|
||||
}
|
||||
|
||||
pub struct VecCache<K: Idx, V> {
|
||||
@ -258,7 +175,6 @@ impl<K: Idx, V> Default for VecCache<K, V> {
|
||||
|
||||
impl<K: Eq + Idx, V: Copy + Debug> QueryStorage for VecCache<K, V> {
|
||||
type Value = V;
|
||||
type Stored = V;
|
||||
}
|
||||
|
||||
impl<K, V> QueryCache for VecCache<K, V>
|
||||
@ -278,87 +194,12 @@ where
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
|
||||
fn complete(&self, key: K, value: V, index: DepNodeIndex) {
|
||||
#[cfg(parallel_compiler)]
|
||||
let mut lock = self.cache.get_shard_by_hash(key.index() as u64).lock();
|
||||
#[cfg(not(parallel_compiler))]
|
||||
let mut lock = self.cache.lock();
|
||||
lock.insert(key, (value, index));
|
||||
value
|
||||
}
|
||||
|
||||
fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
|
||||
#[cfg(parallel_compiler)]
|
||||
{
|
||||
let shards = self.cache.lock_shards();
|
||||
for shard in shards.iter() {
|
||||
for (k, v) in shard.iter_enumerated() {
|
||||
if let Some(v) = v {
|
||||
f(&k, &v.0, v.1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(not(parallel_compiler))]
|
||||
{
|
||||
let map = self.cache.lock();
|
||||
for (k, v) in map.iter_enumerated() {
|
||||
if let Some(v) = v {
|
||||
f(&k, &v.0, v.1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct VecArenaCache<'tcx, K: Idx, V> {
|
||||
arena: WorkerLocal<TypedArena<(V, DepNodeIndex)>>,
|
||||
#[cfg(parallel_compiler)]
|
||||
cache: Sharded<IndexVec<K, Option<&'tcx (V, DepNodeIndex)>>>,
|
||||
#[cfg(not(parallel_compiler))]
|
||||
cache: Lock<IndexVec<K, Option<&'tcx (V, DepNodeIndex)>>>,
|
||||
}
|
||||
|
||||
impl<'tcx, K: Idx, V> Default for VecArenaCache<'tcx, K, V> {
|
||||
fn default() -> Self {
|
||||
VecArenaCache {
|
||||
arena: WorkerLocal::new(|_| TypedArena::default()),
|
||||
cache: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'tcx, K: Eq + Idx, V: Debug + 'tcx> QueryStorage for VecArenaCache<'tcx, K, V> {
|
||||
type Value = V;
|
||||
type Stored = &'tcx V;
|
||||
}
|
||||
|
||||
impl<'tcx, K, V: 'tcx> QueryCache for VecArenaCache<'tcx, K, V>
|
||||
where
|
||||
K: Eq + Idx + Clone + Debug,
|
||||
V: Debug,
|
||||
{
|
||||
type Key = K;
|
||||
|
||||
#[inline(always)]
|
||||
fn lookup(&self, key: &K) -> Option<(&'tcx V, DepNodeIndex)> {
|
||||
#[cfg(parallel_compiler)]
|
||||
let lock = self.cache.get_shard_by_hash(key.index() as u64).lock();
|
||||
#[cfg(not(parallel_compiler))]
|
||||
let lock = self.cache.lock();
|
||||
if let Some(Some(value)) = lock.get(*key) { Some((&value.0, value.1)) } else { None }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored {
|
||||
let value = self.arena.alloc((value, index));
|
||||
let value = unsafe { &*(value as *const _) };
|
||||
#[cfg(parallel_compiler)]
|
||||
let mut lock = self.cache.get_shard_by_hash(key.index() as u64).lock();
|
||||
#[cfg(not(parallel_compiler))]
|
||||
let mut lock = self.cache.lock();
|
||||
lock.insert(key, value);
|
||||
&value.0
|
||||
}
|
||||
|
||||
fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
|
||||
|
||||
@ -20,10 +20,9 @@ pub trait QueryConfig<Qcx: QueryContext> {
|
||||
const NAME: &'static str;
|
||||
|
||||
type Key: DepNodeParams<Qcx::DepContext> + Eq + Hash + Clone + Debug;
|
||||
type Value: Debug;
|
||||
type Stored: Debug + Copy + std::borrow::Borrow<Self::Value>;
|
||||
type Value: Debug + Copy;
|
||||
|
||||
type Cache: QueryCache<Key = Self::Key, Stored = Self::Stored, Value = Self::Value>;
|
||||
type Cache: QueryCache<Key = Self::Key, Value = Self::Value>;
|
||||
|
||||
// Don't use this method to access query results, instead use the methods on TyCtxt
|
||||
fn query_state<'a>(tcx: Qcx) -> &'a QueryState<Self::Key, Qcx::DepKind>
|
||||
@ -38,9 +37,9 @@ pub trait QueryConfig<Qcx: QueryContext> {
|
||||
fn cache_on_disk(tcx: Qcx::DepContext, key: &Self::Key) -> bool;
|
||||
|
||||
// Don't use this method to compute query results, instead use the methods on TyCtxt
|
||||
fn execute_query(tcx: Qcx::DepContext, k: Self::Key) -> Self::Stored;
|
||||
fn execute_query(tcx: Qcx::DepContext, k: Self::Key) -> Self::Value;
|
||||
|
||||
fn compute(tcx: Qcx, key: &Self::Key) -> fn(Qcx::DepContext, Self::Key) -> Self::Value;
|
||||
fn compute(tcx: Qcx, key: Self::Key) -> Self::Value;
|
||||
|
||||
fn try_load_from_disk(qcx: Qcx, idx: &Self::Key) -> TryLoadFromDisk<Qcx, Self>;
|
||||
|
||||
|
||||
@ -246,7 +246,7 @@ where
|
||||
|
||||
/// Completes the query by updating the query cache with the `result`,
|
||||
/// signals the waiter and forgets the JobOwner, so it won't poison the query
|
||||
fn complete<C>(self, cache: &C, result: C::Value, dep_node_index: DepNodeIndex) -> C::Stored
|
||||
fn complete<C>(self, cache: &C, result: C::Value, dep_node_index: DepNodeIndex)
|
||||
where
|
||||
C: QueryCache<Key = K>,
|
||||
{
|
||||
@ -257,23 +257,19 @@ where
|
||||
// Forget ourself so our destructor won't poison the query
|
||||
mem::forget(self);
|
||||
|
||||
let (job, result) = {
|
||||
let job = {
|
||||
#[cfg(parallel_compiler)]
|
||||
let mut lock = state.active.get_shard_by_value(&key).lock();
|
||||
#[cfg(not(parallel_compiler))]
|
||||
let mut lock = state.active.lock();
|
||||
match lock.remove(&key).unwrap() {
|
||||
QueryResult::Started(job) => job,
|
||||
QueryResult::Poisoned => panic!(),
|
||||
}
|
||||
};
|
||||
let result = cache.complete(key, result, dep_node_index);
|
||||
(job, result)
|
||||
let job = {
|
||||
#[cfg(parallel_compiler)]
|
||||
let mut lock = state.active.get_shard_by_value(&key).lock();
|
||||
#[cfg(not(parallel_compiler))]
|
||||
let mut lock = state.active.lock();
|
||||
match lock.remove(&key).unwrap() {
|
||||
QueryResult::Started(job) => job,
|
||||
QueryResult::Poisoned => panic!(),
|
||||
}
|
||||
};
|
||||
cache.complete(key, result, dep_node_index);
|
||||
|
||||
job.signal_complete();
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
@ -336,7 +332,7 @@ where
|
||||
/// which will be used if the query is not in the cache and we need
|
||||
/// to compute it.
|
||||
#[inline]
|
||||
pub fn try_get_cached<Tcx, C>(tcx: Tcx, cache: &C, key: &C::Key) -> Option<C::Stored>
|
||||
pub fn try_get_cached<Tcx, C>(tcx: Tcx, cache: &C, key: &C::Key) -> Option<C::Value>
|
||||
where
|
||||
C: QueryCache,
|
||||
Tcx: DepContext,
|
||||
@ -358,7 +354,7 @@ fn try_execute_query<Q, Qcx>(
|
||||
span: Span,
|
||||
key: Q::Key,
|
||||
dep_node: Option<DepNode<Qcx::DepKind>>,
|
||||
) -> (Q::Stored, Option<DepNodeIndex>)
|
||||
) -> (Q::Value, Option<DepNodeIndex>)
|
||||
where
|
||||
Q: QueryConfig<Qcx>,
|
||||
Qcx: QueryContext,
|
||||
@ -390,7 +386,7 @@ where
|
||||
);
|
||||
}
|
||||
}
|
||||
let result = job.complete(cache, result, dep_node_index);
|
||||
job.complete(cache, result, dep_node_index);
|
||||
(result, Some(dep_node_index))
|
||||
}
|
||||
TryGetJob::Cycle(error) => {
|
||||
@ -426,9 +422,7 @@ where
|
||||
// Fast path for when incr. comp. is off.
|
||||
if !dep_graph.is_fully_enabled() {
|
||||
let prof_timer = qcx.dep_context().profiler().query_provider();
|
||||
let result = qcx.start_query(job_id, Q::DEPTH_LIMIT, None, || {
|
||||
Q::compute(qcx, &key)(*qcx.dep_context(), key)
|
||||
});
|
||||
let result = qcx.start_query(job_id, Q::DEPTH_LIMIT, None, || Q::compute(qcx, key));
|
||||
let dep_node_index = dep_graph.next_virtual_depnode_index();
|
||||
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
|
||||
return (result, dep_node_index);
|
||||
@ -454,17 +448,15 @@ where
|
||||
let (result, dep_node_index) =
|
||||
qcx.start_query(job_id, Q::DEPTH_LIMIT, Some(&diagnostics), || {
|
||||
if Q::ANON {
|
||||
return dep_graph.with_anon_task(*qcx.dep_context(), Q::DEP_KIND, || {
|
||||
Q::compute(qcx, &key)(*qcx.dep_context(), key)
|
||||
});
|
||||
return dep_graph
|
||||
.with_anon_task(*qcx.dep_context(), Q::DEP_KIND, || Q::compute(qcx, key));
|
||||
}
|
||||
|
||||
// `to_dep_node` is expensive for some `DepKind`s.
|
||||
let dep_node =
|
||||
dep_node_opt.unwrap_or_else(|| Q::construct_dep_node(*qcx.dep_context(), &key));
|
||||
|
||||
let task = Q::compute(qcx, &key);
|
||||
dep_graph.with_task(dep_node, *qcx.dep_context(), key, task, Q::HASH_RESULT)
|
||||
dep_graph.with_task(dep_node, qcx, key, Q::compute, Q::HASH_RESULT)
|
||||
});
|
||||
|
||||
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
|
||||
@ -555,7 +547,7 @@ where
|
||||
let prof_timer = qcx.dep_context().profiler().query_provider();
|
||||
|
||||
// The dep-graph for this computation is already in-place.
|
||||
let result = dep_graph.with_ignore(|| Q::compute(qcx, key)(*qcx.dep_context(), key.clone()));
|
||||
let result = dep_graph.with_ignore(|| Q::compute(qcx, key.clone()));
|
||||
|
||||
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
|
||||
|
||||
@ -727,7 +719,7 @@ pub enum QueryMode {
|
||||
Ensure,
|
||||
}
|
||||
|
||||
pub fn get_query<Q, Qcx, D>(qcx: Qcx, span: Span, key: Q::Key, mode: QueryMode) -> Option<Q::Stored>
|
||||
pub fn get_query<Q, Qcx, D>(qcx: Qcx, span: Span, key: Q::Key, mode: QueryMode) -> Option<Q::Value>
|
||||
where
|
||||
D: DepKind,
|
||||
Q: QueryConfig<Qcx>,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user