diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
index 5d7f385c6e4..d09c026c4b4 100644
--- a/compiler/rustc_data_structures/src/lib.rs
+++ b/compiler/rustc_data_structures/src/lib.rs
@@ -127,6 +127,9 @@ impl<F: FnOnce()> Drop for OnDrop<F> {
     }
 }
 
+/// This is a marker for a fatal compiler error used with `resume_unwind`.
+pub struct FatalErrorMarker;
+
 /// Turns a closure that takes an `&mut Formatter` into something that can be display-formatted.
 pub fn make_display(f: impl Fn(&mut fmt::Formatter<'_>) -> fmt::Result) -> impl fmt::Display {
     struct Printer<F> {
diff --git a/compiler/rustc_data_structures/src/sync/parallel.rs b/compiler/rustc_data_structures/src/sync/parallel.rs
index 5695d839d3e..7783de57fba 100644
--- a/compiler/rustc_data_structures/src/sync/parallel.rs
+++ b/compiler/rustc_data_structures/src/sync/parallel.rs
@@ -3,6 +3,8 @@
 
 #![allow(dead_code)]
 
+use crate::sync::IntoDynSyncSend;
+use crate::FatalErrorMarker;
 use parking_lot::Mutex;
 use std::any::Any;
 use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe};
@@ -18,14 +20,17 @@ pub use enabled::*;
 /// continuing with unwinding. It's also used for the non-parallel code to ensure error message
 /// output match the parallel compiler for testing purposes.
 pub struct ParallelGuard {
-    panic: Mutex<Option<Box<dyn Any + Send + 'static>>>,
+    panic: Mutex<Option<IntoDynSyncSend<Box<dyn Any + Send + 'static>>>>,
 }
 
 impl ParallelGuard {
     pub fn run<R>(&self, f: impl FnOnce() -> R) -> Option<R> {
         catch_unwind(AssertUnwindSafe(f))
             .map_err(|err| {
-                *self.panic.lock() = Some(err);
+                let mut panic = self.panic.lock();
+                if panic.is_none() || !(*err).is::<FatalErrorMarker>() {
+                    *panic = Some(IntoDynSyncSend(err));
+                }
             })
             .ok()
     }
@@ -37,7 +42,7 @@ impl ParallelGuard {
 pub fn parallel_guard<R>(f: impl FnOnce(&ParallelGuard) -> R) -> R {
     let guard = ParallelGuard { panic: Mutex::new(None) };
     let ret = f(&guard);
-    if let Some(panic) = guard.panic.into_inner() {
+    if let Some(IntoDynSyncSend(panic)) = guard.panic.into_inner() {
         resume_unwind(panic);
     }
     ret
@@ -106,14 +111,20 @@ mod enabled {
             parallel!(impl $fblock [$block, $($c,)*] [$($rest),*])
         };
         (impl $fblock:block [$($blocks:expr,)*] []) => {
-            ::rustc_data_structures::sync::scope(|s| {
-                $(let block = rustc_data_structures::sync::FromDyn::from(|| $blocks);
-                s.spawn(move |_| block.into_inner()());)*
-                (|| $fblock)();
+            $crate::sync::parallel_guard(|guard| {
+                $crate::sync::scope(|s| {
+                    $(
+                        let block = $crate::sync::FromDyn::from(|| $blocks);
+                        s.spawn(move |_| {
+                            guard.run(move || block.into_inner()());
+                        });
+                    )*
+                    guard.run(|| $fblock);
+                });
             });
         };
         ($fblock:block, $($blocks:block),*) => {
-            if rustc_data_structures::sync::is_dyn_thread_safe() {
+            if $crate::sync::is_dyn_thread_safe() {
                 // Reverse the order of the later blocks since Rayon executes them in reverse order
                 // when using a single thread. This ensures the execution order matches that
                 // of a single threaded rustc.
@@ -146,11 +157,13 @@ mod enabled {
         if mode::is_dyn_thread_safe() {
             let oper_a = FromDyn::from(oper_a);
             let oper_b = FromDyn::from(oper_b);
-            let (a, b) = rayon::join(
-                move || FromDyn::from(oper_a.into_inner()()),
-                move || FromDyn::from(oper_b.into_inner()()),
-            );
-            (a.into_inner(), b.into_inner())
+            let (a, b) = parallel_guard(|guard| {
+                rayon::join(
+                    move || guard.run(move || FromDyn::from(oper_a.into_inner()())),
+                    move || guard.run(move || FromDyn::from(oper_b.into_inner()())),
+                )
+            });
+            (a.unwrap().into_inner(), b.unwrap().into_inner())
         } else {
             super::disabled::join(oper_a, oper_b)
         }
diff --git a/compiler/rustc_span/src/fatal_error.rs b/compiler/rustc_span/src/fatal_error.rs
index 8b4f77a0566..26c5711099c 100644
--- a/compiler/rustc_span/src/fatal_error.rs
+++ b/compiler/rustc_span/src/fatal_error.rs
@@ -3,7 +3,7 @@
 #[must_use]
 pub struct FatalError;
 
-pub struct FatalErrorMarker;
+pub use rustc_data_structures::FatalErrorMarker;
 
 // Don't implement Send on FatalError. This makes it impossible to `panic_any!(FatalError)`.
 // We don't want to invoke the panic handler and print a backtrace for fatal errors.