Rollup merge of #138273 - petrochenkov:nonatroot, r=bjorn3

metadata: Ignore sysroot when doing the manual native lib search in rustc

This is the opposite alternative to https://github.com/rust-lang/rust/pull/138170 and another way to make native library search consistent between rustc and linker.

This way the directory list searched by rustc is still a prefix of the directory list considered by linker, but it's a shorter prefix than in #138170.
We can include the sysroot directories into rustc's search again later if the issues with #138170 are resolved, it will be a backward compatible change.

This may break some code doing weird things on unstable rustc, or tier 2-3 targets, like bundling `libunwind.a` or sanitizers into something.
Note that this doesn't affect shipped `libc.a`, because it lives in `self-contained` directories in sysroot, and `self-contained` sysroot is already not included into the rustc's search. All libunwind and sanitizer libs should be moved to `self-contained` sysroot too eventually.

With the consistent search directory list between rustc and linker we can make rustc own the native library search (at least for static libs) and use linker search only as a fallback (like in #123436). This will allow addressing issues like https://github.com/rust-lang/rust/pull/132394 once and for all on all targets.

r? ``@bjorn3``

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -22,7 +22,9 @@ use rustc_fs_util::{fix_windows_verbatim_for_gcc, try_canonicalize};
 use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
 use rustc_macros::LintDiagnostic;
 use rustc_metadata::fs::{METADATA_FILENAME, copy_to_stdout, emit_wrapper_file};
-use rustc_metadata::{find_native_static_library, walk_native_lib_search_dirs};
+use rustc_metadata::{
+    NativeLibSearchFallback, find_native_static_library, walk_native_lib_search_dirs,
+};
 use rustc_middle::bug;
 use rustc_middle::lint::lint_level;
 use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerFile;
@@ -2129,19 +2131,15 @@ fn add_library_search_dirs(
         return;
     }
 
-    walk_native_lib_search_dirs(
-        sess,
-        self_contained_components,
-        apple_sdk_root,
-        |dir, is_framework| {
-            if is_framework {
-                cmd.framework_path(dir);
-            } else {
-                cmd.include_path(&fix_windows_verbatim_for_gcc(dir));
-            }
-            ControlFlow::<()>::Continue(())
-        },
-    );
+    let fallback = Some(NativeLibSearchFallback { self_contained_components, apple_sdk_root });
+    walk_native_lib_search_dirs(sess, fallback, |dir, is_framework| {
+        if is_framework {
+            cmd.framework_path(dir);
+        } else {
+            cmd.include_path(&fix_windows_verbatim_for_gcc(dir));
+        }
+        ControlFlow::<()>::Continue(())
+    });
 }
 
 /// Add options making relocation sections in the produced ELF files read-only

compiler/rustc_metadata/src/lib.rs

@@ -35,8 +35,8 @@ pub mod locator;
 pub use creader::{DylibError, load_symbol_from_dylib};
 pub use fs::{METADATA_FILENAME, emit_wrapper_file};
 pub use native_libs::{
-    find_native_static_library, try_find_native_dynamic_library, try_find_native_static_library,
-    walk_native_lib_search_dirs,
+    NativeLibSearchFallback, find_native_static_library, try_find_native_dynamic_library,
+    try_find_native_static_library, walk_native_lib_search_dirs,
 };
 pub use rmeta::{EncodedMetadata, METADATA_HEADER, encode_metadata, rendered_const};
 

compiler/rustc_metadata/src/native_libs.rs

@@ -21,10 +21,17 @@ use rustc_target::spec::{BinaryFormat, LinkSelfContainedComponents};
 
 use crate::{errors, fluent_generated};
 
+/// The fallback directories are passed to linker, but not used when rustc does the search,
+/// because in the latter case the set of fallback directories cannot always be determined
+/// consistently at the moment.
+pub struct NativeLibSearchFallback<'a> {
+    pub self_contained_components: LinkSelfContainedComponents,
+    pub apple_sdk_root: Option<&'a Path>,
+}
+
 pub fn walk_native_lib_search_dirs<R>(
     sess: &Session,
-    self_contained_components: LinkSelfContainedComponents,
-    apple_sdk_root: Option<&Path>,
+    fallback: Option<NativeLibSearchFallback<'_>>,
     mut f: impl FnMut(&Path, bool /*is_framework*/) -> ControlFlow<R>,
 ) -> ControlFlow<R> {
     // Library search paths explicitly supplied by user (`-L` on the command line).
@@ -38,6 +45,11 @@ pub fn walk_native_lib_search_dirs<R>(
         }
     }
 
+    let Some(NativeLibSearchFallback { self_contained_components, apple_sdk_root }) = fallback
+    else {
+        return ControlFlow::Continue(());
+    };
+
     // The toolchain ships some native library components and self-contained linking was enabled.
     // Add the self-contained library directory to search paths.
     if self_contained_components.intersects(
@@ -93,23 +105,17 @@ pub fn try_find_native_static_library(
         if os == unix { vec![os] } else { vec![os, unix] }
     };
 
-    // FIXME: Account for self-contained linking settings and Apple SDK.
-    walk_native_lib_search_dirs(
-        sess,
-        LinkSelfContainedComponents::empty(),
-        None,
-        |dir, is_framework| {
-            if !is_framework {
-                for (prefix, suffix) in &formats {
-                    let test = dir.join(format!("{prefix}{name}{suffix}"));
-                    if test.exists() {
-                        return ControlFlow::Break(test);
-                    }
+    walk_native_lib_search_dirs(sess, None, |dir, is_framework| {
+        if !is_framework {
+            for (prefix, suffix) in &formats {
+                let test = dir.join(format!("{prefix}{name}{suffix}"));
+                if test.exists() {
+                    return ControlFlow::Break(test);
                 }
             }
-            ControlFlow::Continue(())
-        },
-    )
+        }
+        ControlFlow::Continue(())
+    })
     .break_value()
 }
 
@@ -132,22 +138,17 @@ pub fn try_find_native_dynamic_library(
         vec![os, meson, mingw]
     };
 
-    walk_native_lib_search_dirs(
-        sess,
-        LinkSelfContainedComponents::empty(),
-        None,
-        |dir, is_framework| {
-            if !is_framework {
-                for (prefix, suffix) in &formats {
-                    let test = dir.join(format!("{prefix}{name}{suffix}"));
-                    if test.exists() {
-                        return ControlFlow::Break(test);
-                    }
+    walk_native_lib_search_dirs(sess, None, |dir, is_framework| {
+        if !is_framework {
+            for (prefix, suffix) in &formats {
+                let test = dir.join(format!("{prefix}{name}{suffix}"));
+                if test.exists() {
+                    return ControlFlow::Break(test);
                 }
             }
-            ControlFlow::Continue(())
-        },
-    )
+        }
+        ControlFlow::Continue(())
+    })
     .break_value()
 }