Auto merge of #116109 - Zoxc:no-dep-graph-thread, r=oli-obk

Don't use a thread to load the dep graph

This removes the use of a thread to load the dep graph. It's not currently useful as we immediately block on it.

r? `@oli-obk`

compiler/rustc_incremental/src/lib.rs

@@ -28,8 +28,8 @@ pub use persist::load_query_result_cache;
 pub use persist::prepare_session_directory;
 pub use persist::save_dep_graph;
 pub use persist::save_work_product_index;
+pub use persist::setup_dep_graph;
 pub use persist::LoadResult;
-pub use persist::{build_dep_graph, load_dep_graph, DepGraphFuture};
 
 use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
 use rustc_fluent_macro::fluent_messages;

compiler/rustc_incremental/src/persist/load.rs

@@ -3,17 +3,19 @@
 use crate::errors;
 use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::unord::UnordMap;
-use rustc_middle::dep_graph::{DepsType, SerializedDepGraph, WorkProductMap};
+use rustc_middle::dep_graph::{DepGraph, DepsType, SerializedDepGraph, WorkProductMap};
 use rustc_middle::query::on_disk_cache::OnDiskCache;
 use rustc_serialize::opaque::MemDecoder;
 use rustc_serialize::Decodable;
 use rustc_session::config::IncrementalStateAssertion;
-use rustc_session::Session;
+use rustc_session::{Session, StableCrateId};
+use rustc_span::{ErrorGuaranteed, Symbol};
 use std::path::{Path, PathBuf};
 
 use super::data::*;
 use super::file_format;
 use super::fs::*;
+use super::save::build_dep_graph;
 use super::work_product;
 
 #[derive(Debug)]
@@ -72,21 +74,12 @@ impl<T: Default> LoadResult<T> {
 }
 
 fn load_data(path: &Path, sess: &Session) -> LoadResult<(Mmap, usize)> {
-    load_data_no_sess(
+    match file_format::read_file(
         path,
         sess.opts.unstable_opts.incremental_info,
         sess.is_nightly_build(),
         sess.cfg_version,
-    )
-}
-
-fn load_data_no_sess(
-    path: &Path,
-    report_incremental_info: bool,
-    is_nightly_build: bool,
-    cfg_version: &'static str,
-) -> LoadResult<(Mmap, usize)> {
-    match file_format::read_file(path, report_incremental_info, is_nightly_build, cfg_version) {
+    ) {
         Ok(Some(data_and_pos)) => LoadResult::Ok { data: data_and_pos },
         Ok(None) => {
             // The file either didn't exist or was produced by an incompatible
@@ -102,39 +95,12 @@ fn delete_dirty_work_product(sess: &Session, swp: SerializedWorkProduct) {
     work_product::delete_workproduct_files(sess, &swp.work_product);
 }
 
-/// Either a result that has already be computed or a
-/// handle that will let us wait until it is computed
-/// by a background thread.
-pub enum MaybeAsync<T> {
-    Sync(T),
-    Async(std::thread::JoinHandle<T>),
-}
-
-impl<T> MaybeAsync<LoadResult<T>> {
-    /// Accesses the data returned in [`LoadResult::Ok`] in an asynchronous way if possible.
-    pub fn open(self) -> LoadResult<T> {
-        match self {
-            MaybeAsync::Sync(result) => result,
-            MaybeAsync::Async(handle) => {
-                handle.join().unwrap_or_else(|e| LoadResult::DecodeIncrCache(e))
-            }
-        }
-    }
-}
-
-/// An asynchronous type for computing the dependency graph.
-pub type DepGraphFuture = MaybeAsync<LoadResult<(SerializedDepGraph, WorkProductMap)>>;
-
-/// Launch a thread and load the dependency graph in the background.
-pub fn load_dep_graph(sess: &Session) -> DepGraphFuture {
-    // Since `sess` isn't `Sync`, we perform all accesses to `sess`
-    // before we fire the background thread.
-
+fn load_dep_graph(sess: &Session) -> LoadResult<(SerializedDepGraph, WorkProductMap)> {
     let prof = sess.prof.clone();
 
     if sess.opts.incremental.is_none() {
         // No incremental compilation.
-        return MaybeAsync::Sync(LoadResult::Ok { data: Default::default() });
+        return LoadResult::Ok { data: Default::default() };
     }
 
     let _timer = sess.prof.generic_activity("incr_comp_prepare_load_dep_graph");
@@ -142,7 +108,6 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture {
     // Calling `sess.incr_comp_session_dir()` will panic if `sess.opts.incremental.is_none()`.
     // Fortunately, we just checked that this isn't the case.
     let path = dep_graph_path(&sess);
-    let report_incremental_info = sess.opts.unstable_opts.incremental_info;
     let expected_hash = sess.opts.dep_tracking_hash(false);
 
     let mut prev_work_products = UnordMap::default();
@@ -180,40 +145,35 @@ pub fn load_dep_graph(sess: &Session) -> DepGraphFuture {
         }
     }
 
-    let is_nightly_build = sess.is_nightly_build();
-    let cfg_version = sess.cfg_version;
+    let _prof_timer = prof.generic_activity("incr_comp_load_dep_graph");
 
-    MaybeAsync::Async(std::thread::spawn(move || {
-        let _prof_timer = prof.generic_activity("incr_comp_load_dep_graph");
+    match load_data(&path, sess) {
+        LoadResult::DataOutOfDate => LoadResult::DataOutOfDate,
+        LoadResult::LoadDepGraph(path, err) => LoadResult::LoadDepGraph(path, err),
+        LoadResult::DecodeIncrCache(err) => LoadResult::DecodeIncrCache(err),
+        LoadResult::Ok { data: (bytes, start_pos) } => {
+            let mut decoder = MemDecoder::new(&bytes, start_pos);
+            let prev_commandline_args_hash = u64::decode(&mut decoder);
 
-        match load_data_no_sess(&path, report_incremental_info, is_nightly_build, cfg_version) {
-            LoadResult::DataOutOfDate => LoadResult::DataOutOfDate,
-            LoadResult::LoadDepGraph(path, err) => LoadResult::LoadDepGraph(path, err),
-            LoadResult::DecodeIncrCache(err) => LoadResult::DecodeIncrCache(err),
-            LoadResult::Ok { data: (bytes, start_pos) } => {
-                let mut decoder = MemDecoder::new(&bytes, start_pos);
-                let prev_commandline_args_hash = u64::decode(&mut decoder);
-
-                if prev_commandline_args_hash != expected_hash {
-                    if report_incremental_info {
-                        eprintln!(
-                            "[incremental] completely ignoring cache because of \
+            if prev_commandline_args_hash != expected_hash {
+                if sess.opts.unstable_opts.incremental_info {
+                    eprintln!(
+                        "[incremental] completely ignoring cache because of \
                                     differing commandline arguments"
-                        );
-                    }
-                    // We can't reuse the cache, purge it.
-                    debug!("load_dep_graph_new: differing commandline arg hashes");
-
-                    // No need to do any further work
-                    return LoadResult::DataOutOfDate;
+                    );
                 }
+                // We can't reuse the cache, purge it.
+                debug!("load_dep_graph_new: differing commandline arg hashes");
 
-                let dep_graph = SerializedDepGraph::decode::<DepsType>(&mut decoder);
-
-                LoadResult::Ok { data: (dep_graph, prev_work_products) }
+                // No need to do any further work
+                return LoadResult::DataOutOfDate;
             }
+
+            let dep_graph = SerializedDepGraph::decode::<DepsType>(&mut decoder);
+
+            LoadResult::Ok { data: (dep_graph, prev_work_products) }
         }
-    }))
+    }
 }
 
 /// Attempts to load the query result cache from disk
@@ -235,3 +195,35 @@ pub fn load_query_result_cache(sess: &Session) -> Option<OnDiskCache<'_>> {
         _ => Some(OnDiskCache::new_empty(sess.source_map())),
     }
 }
+
+/// Setups the dependency graph by loading an existing graph from disk and set up streaming of a
+/// new graph to an incremental session directory.
+pub fn setup_dep_graph(
+    sess: &Session,
+    crate_name: Symbol,
+    stable_crate_id: StableCrateId,
+) -> Result<DepGraph, ErrorGuaranteed> {
+    // `load_dep_graph` can only be called after `prepare_session_directory`.
+    prepare_session_directory(sess, crate_name, stable_crate_id)?;
+
+    let res = sess.opts.build_dep_graph().then(|| load_dep_graph(sess));
+
+    if sess.opts.incremental.is_some() {
+        sess.time("incr_comp_garbage_collect_session_directories", || {
+            if let Err(e) = garbage_collect_session_directories(sess) {
+                warn!(
+                    "Error while trying to garbage collect incremental \
+                     compilation cache directory: {}",
+                    e
+                );
+            }
+        });
+    }
+
+    Ok(res
+        .and_then(|result| {
+            let (prev_graph, prev_work_products) = result.open(sess);
+            build_dep_graph(sess, prev_graph, prev_work_products)
+        })
+        .unwrap_or_else(DepGraph::new_disabled))
+}

compiler/rustc_incremental/src/persist/mod.rs

@@ -16,9 +16,8 @@ pub use fs::in_incr_comp_dir;
 pub use fs::in_incr_comp_dir_sess;
 pub use fs::prepare_session_directory;
 pub use load::load_query_result_cache;
+pub use load::setup_dep_graph;
 pub use load::LoadResult;
-pub use load::{load_dep_graph, DepGraphFuture};
-pub use save::build_dep_graph;
 pub use save::save_dep_graph;
 pub use save::save_work_product_index;
 pub use work_product::copy_cgu_workproduct_to_incr_comp_cache_dir;

compiler/rustc_incremental/src/persist/save.rs

@@ -147,7 +147,7 @@ fn encode_query_cache(tcx: TyCtxt<'_>, encoder: FileEncoder) -> FileEncodeResult
 /// execution, the new dependency information is not kept in memory but directly
 /// output to this file. `save_dep_graph` then finalizes the staging dep-graph
 /// and moves it to the permanent dep-graph path
-pub fn build_dep_graph(
+pub(crate) fn build_dep_graph(
     sess: &Session,
     prev_graph: SerializedDepGraph,
     prev_work_products: WorkProductMap,

compiler/rustc_interface/src/queries.rs

@@ -10,7 +10,7 @@ use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::{AppendOnlyIndexVec, FreezeLock, Lrc, OnceLock, WorkerLocal};
 use rustc_hir::def_id::{StableCrateId, CRATE_DEF_ID, LOCAL_CRATE};
 use rustc_hir::definitions::Definitions;
-use rustc_incremental::DepGraphFuture;
+use rustc_incremental::setup_dep_graph;
 use rustc_metadata::creader::CStore;
 use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::DepGraph;
@@ -19,7 +19,6 @@ use rustc_session::config::{self, CrateType, OutputFilenames, OutputType};
 use rustc_session::cstore::Untracked;
 use rustc_session::{output::find_crate_name, Session};
 use rustc_span::symbol::sym;
-use rustc_span::Symbol;
 use std::any::Any;
 use std::cell::{RefCell, RefMut};
 use std::sync::Arc;
@@ -132,43 +131,6 @@ impl<'tcx> Queries<'tcx> {
         })
     }
 
-    fn dep_graph_future(
-        &self,
-        crate_name: Symbol,
-        stable_crate_id: StableCrateId,
-    ) -> Result<Option<DepGraphFuture>> {
-        let sess = self.session();
-
-        // `load_dep_graph` can only be called after `prepare_session_directory`.
-        rustc_incremental::prepare_session_directory(sess, crate_name, stable_crate_id)?;
-        let res = sess.opts.build_dep_graph().then(|| rustc_incremental::load_dep_graph(sess));
-
-        if sess.opts.incremental.is_some() {
-            sess.time("incr_comp_garbage_collect_session_directories", || {
-                if let Err(e) = rustc_incremental::garbage_collect_session_directories(sess) {
-                    warn!(
-                        "Error while trying to garbage collect incremental \
-                         compilation cache directory: {}",
-                        e
-                    );
-                }
-            });
-        }
-
-        Ok(res)
-    }
-
-    fn dep_graph(&self, dep_graph_future: Option<DepGraphFuture>) -> DepGraph {
-        dep_graph_future
-            .and_then(|future| {
-                let sess = self.session();
-                let (prev_graph, prev_work_products) =
-                    sess.time("blocked_on_dep_graph_loading", || future.open().open(sess));
-                rustc_incremental::build_dep_graph(sess, prev_graph, prev_work_products)
-            })
-            .unwrap_or_else(DepGraph::new_disabled)
-    }
-
     pub fn global_ctxt(&'tcx self) -> Result<QueryResult<'_, &'tcx GlobalCtxt<'tcx>>> {
         self.gcx.compute(|| {
             let sess = self.session();
@@ -184,10 +146,7 @@ impl<'tcx> Queries<'tcx> {
                 sess.opts.cg.metadata.clone(),
                 sess.cfg_version,
             );
-
-            // Compute the dependency graph (in the background). We want to do this as early as
-            // possible, to give the DepGraph maximum time to load before `dep_graph` is called.
-            let dep_graph_future = self.dep_graph_future(crate_name, stable_crate_id)?;
+            let dep_graph = setup_dep_graph(sess, crate_name, stable_crate_id)?;
 
             let lint_store = Lrc::new(passes::create_lint_store(
                 sess,
@@ -210,7 +169,7 @@ impl<'tcx> Queries<'tcx> {
                 crate_types,
                 stable_crate_id,
                 lint_store,
-                self.dep_graph(dep_graph_future),
+                dep_graph,
                 untracked,
                 &self.gcx_cell,
                 &self.arena,