Auto merge of #127397 - jyn514:multi-thread-panic-hook, r=workingjubilee

fix interleaved output in the default panic hook when multiple threads panic simultaneously

previously, we only held a lock for printing the backtrace itself. since all threads were printing to the same file descriptor, that meant random output in the default panic hook from one thread would be interleaved with the backtrace from another. now, we hold the lock for the full duration of the hook, and the output is ordered.

---

i noticed some odd things while working on this you may or may not already be aware of.

- libbacktrace is included as a submodule instead of a normal rustc crate, and as a result uses `cfg(backtrace_in_std)` instead of a more normal `cfg(feature = "rustc-dep-of-std")`. probably this is left over from before rust used a cargo-based build system?
- the default panic handler uses `trace_unsynchronized`, etc, in `sys::backtrace::print`. as a result, the lock only applies to concurrent *panic handlers*, not concurrent *threads*.  in other words, if another, non-panicking, thread tried to print a backtrace at the same time as the panic handler, we may have UB, especially on windows.
    - we have the option of changing backtrace to enable locking when `backtrace_in_std` is set so we can reuse their lock instead of trying to add our own.

library/std/src/panicking.rs

@@ -253,16 +253,20 @@ fn default_hook(info: &PanicHookInfo<'_>) {
     let name = thread.as_ref().and_then(|t| t.name()).unwrap_or("<unnamed>");
 
     let write = |err: &mut dyn crate::io::Write| {
+        // Use a lock to prevent mixed output in multithreading context.
+        // Some platforms also require it when printing a backtrace, like `SymFromAddr` on Windows.
+        let mut lock = backtrace::lock();
         let _ = writeln!(err, "thread '{name}' panicked at {location}:\n{msg}");
 
         static FIRST_PANIC: AtomicBool = AtomicBool::new(true);
 
         match backtrace {
+            // SAFETY: we took out a lock just a second ago.
             Some(BacktraceStyle::Short) => {
-                drop(backtrace::print(err, crate::backtrace_rs::PrintFmt::Short))
+                drop(lock.print(err, crate::backtrace_rs::PrintFmt::Short))
             }
             Some(BacktraceStyle::Full) => {
-                drop(backtrace::print(err, crate::backtrace_rs::PrintFmt::Full))
+                drop(lock.print(err, crate::backtrace_rs::PrintFmt::Full))
             }
             Some(BacktraceStyle::Off) => {
                 if FIRST_PANIC.swap(false, Ordering::Relaxed) {

library/std/src/sys/backtrace.rs

@@ -7,44 +7,41 @@ use crate::fmt;
 use crate::io;
 use crate::io::prelude::*;
 use crate::path::{self, Path, PathBuf};
-use crate::sync::{Mutex, PoisonError};
+use crate::sync::{Mutex, MutexGuard, PoisonError};
 
 /// Max number of frames to print.
 const MAX_NB_FRAMES: usize = 100;
 
-pub fn lock() -> impl Drop {
+pub(crate) struct BacktraceLock<'a>(#[allow(dead_code)] MutexGuard<'a, ()>);
+
+pub(crate) fn lock<'a>() -> BacktraceLock<'a> {
     static LOCK: Mutex<()> = Mutex::new(());
-    LOCK.lock().unwrap_or_else(PoisonError::into_inner)
+    BacktraceLock(LOCK.lock().unwrap_or_else(PoisonError::into_inner))
 }
 
-/// Prints the current backtrace.
-pub fn print(w: &mut dyn Write, format: PrintFmt) -> io::Result<()> {
-    // There are issues currently linking libbacktrace into tests, and in
-    // general during std's own unit tests we're not testing this path. In
-    // test mode immediately return here to optimize away any references to the
-    // libbacktrace symbols
-    if cfg!(test) {
-        return Ok(());
-    }
-
-    // Use a lock to prevent mixed output in multithreading context.
-    // Some platforms also requires it, like `SymFromAddr` on Windows.
-    unsafe {
-        let _lock = lock();
-        _print(w, format)
-    }
-}
-
-unsafe fn _print(w: &mut dyn Write, format: PrintFmt) -> io::Result<()> {
-    struct DisplayBacktrace {
-        format: PrintFmt,
-    }
-    impl fmt::Display for DisplayBacktrace {
-        fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
-            unsafe { _print_fmt(fmt, self.format) }
+impl BacktraceLock<'_> {
+    /// Prints the current backtrace.
+    ///
+    /// NOTE: this function is not Sync. The caller must hold a mutex lock, or there must be only one thread in the program.
+    pub(crate) fn print(&mut self, w: &mut dyn Write, format: PrintFmt) -> io::Result<()> {
+        // There are issues currently linking libbacktrace into tests, and in
+        // general during std's own unit tests we're not testing this path. In
+        // test mode immediately return here to optimize away any references to the
+        // libbacktrace symbols
+        if cfg!(test) {
+            return Ok(());
         }
+
+        struct DisplayBacktrace {
+            format: PrintFmt,
+        }
+        impl fmt::Display for DisplayBacktrace {
+            fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+                unsafe { _print_fmt(fmt, self.format) }
+            }
+        }
+        write!(w, "{}", DisplayBacktrace { format })
     }
-    write!(w, "{}", DisplayBacktrace { format })
 }
 
 unsafe fn _print_fmt(fmt: &mut fmt::Formatter<'_>, print_fmt: PrintFmt) -> fmt::Result {

tests/ui/backtrace/synchronized-panic-handler.rs

@@ -0,0 +1,17 @@
+//@ run-pass
+//@ check-run-results
+//@ edition:2021
+//@ exec-env:RUST_BACKTRACE=0
+//@ needs-threads
+use std::thread;
+const PANIC_MESSAGE: &str = "oops oh no woe is me";
+
+fn entry() {
+    panic!("{PANIC_MESSAGE}")
+}
+
+fn main() {
+    let (a, b) = (thread::spawn(entry), thread::spawn(entry));
+    assert_eq!(&**a.join().unwrap_err().downcast::<String>().unwrap(), PANIC_MESSAGE);
+    assert_eq!(&**b.join().unwrap_err().downcast::<String>().unwrap(), PANIC_MESSAGE);
+}

tests/ui/backtrace/synchronized-panic-handler.run.stderr

@@ -0,0 +1,5 @@
+thread '<unnamed>' panicked at $DIR/synchronized-panic-handler.rs:10:5:
+oops oh no woe is me
+note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
+thread '<unnamed>' panicked at $DIR/synchronized-panic-handler.rs:10:5:
+oops oh no woe is me