[link] enable packed bundled lib in non stable cases

This commit is contained in:
Daniil Belov 2023-01-26 17:10:13 +03:00 committed by BelovDV
parent dffea43fc1
commit 601fc8b36b
19 changed files with 145 additions and 126 deletions

View File

@ -270,10 +270,9 @@ pub fn each_linked_rlib(
/// Create an 'rlib'. /// Create an 'rlib'.
/// ///
/// An rlib in its current incarnation is essentially a renamed .a file. The rlib primarily contains /// An rlib in its current incarnation is essentially a renamed .a file (with "dummy" object files).
/// the object file of the crate, but it also contains all of the object files from native /// The rlib primarily contains the object file of the crate, but it also some of the object files
/// libraries. This is done by unzipping native libraries and inserting all of the contents into /// from native libraries.
/// this archive.
fn link_rlib<'a>( fn link_rlib<'a>(
sess: &'a Session, sess: &'a Session,
archive_builder_builder: &dyn ArchiveBuilderBuilder, archive_builder_builder: &dyn ArchiveBuilderBuilder,
@ -347,44 +346,23 @@ fn link_rlib<'a>(
// loaded from the libraries found here and then encode that into the // loaded from the libraries found here and then encode that into the
// metadata of the rlib we're generating somehow. // metadata of the rlib we're generating somehow.
for lib in codegen_results.crate_info.used_libraries.iter() { for lib in codegen_results.crate_info.used_libraries.iter() {
match lib.kind { let NativeLibKind::Static { bundle: None | Some(true), whole_archive } = lib.kind else {
NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) } continue;
if flavor == RlibFlavor::Normal && sess.opts.unstable_opts.packed_bundled_libs => {} };
NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) } if whole_archive == Some(true) && !codegen_results.crate_info.feature_packed_bundled_libs {
if flavor == RlibFlavor::Normal => sess.emit_err(errors::IncompatibleLinkingModifiers);
{
// Don't allow mixing +bundle with +whole_archive since an rlib may contain
// multiple native libs, some of which are +whole-archive and some of which are
// -whole-archive and it isn't clear how we can currently handle such a
// situation correctly.
// See https://github.com/rust-lang/rust/issues/88085#issuecomment-901050897
sess.emit_err(errors::IncompatibleLinkingModifiers);
}
NativeLibKind::Static { bundle: None | Some(true), .. } => {}
NativeLibKind::Static { bundle: Some(false), .. }
| NativeLibKind::Dylib { .. }
| NativeLibKind::Framework { .. }
| NativeLibKind::RawDylib
| NativeLibKind::LinkArg
| NativeLibKind::Unspecified => continue,
} }
if let Some(name) = lib.name { if flavor == RlibFlavor::Normal && let Some(filename) = lib.filename {
let location = let path = find_native_static_library(filename.as_str(), true, &lib_search_paths, sess);
let src = read(path).map_err(|e| sess.emit_fatal(errors::ReadFileError {message: e }))?;
let (data, _) = create_wrapper_file(sess, b".bundled_lib".to_vec(), &src);
let wrapper_file = emit_wrapper_file(sess, &data, tmpdir, filename.as_str());
packed_bundled_libs.push(wrapper_file);
} else if let Some(name) = lib.name {
let path =
find_native_static_library(name.as_str(), lib.verbatim, &lib_search_paths, sess); find_native_static_library(name.as_str(), lib.verbatim, &lib_search_paths, sess);
if sess.opts.unstable_opts.packed_bundled_libs && flavor == RlibFlavor::Normal { ab.add_archive(&path, Box::new(|_| false)).unwrap_or_else(|error| {
let filename = lib.filename.unwrap(); sess.emit_fatal(errors::AddNativeLibrary { library_path: path, error })});
let lib_path =
find_native_static_library(filename.as_str(), true, &lib_search_paths, sess);
let src = read(lib_path)
.map_err(|e| sess.emit_fatal(errors::ReadFileError { message: e }))?;
let (data, _) = create_wrapper_file(sess, b".bundled_lib".to_vec(), &src);
let wrapper_file = emit_wrapper_file(sess, &data, tmpdir, filename.as_str());
packed_bundled_libs.push(wrapper_file);
continue;
}
ab.add_archive(&location, Box::new(|_| false)).unwrap_or_else(|error| {
sess.emit_fatal(errors::AddNativeLibrary { library_path: location, error });
});
} }
} }
@ -516,36 +494,14 @@ fn link_staticlib<'a>(
&codegen_results.crate_info, &codegen_results.crate_info,
Some(CrateType::Staticlib), Some(CrateType::Staticlib),
&mut |cnum, path| { &mut |cnum, path| {
let name = codegen_results.crate_info.crate_name[&cnum];
let native_libs = &codegen_results.crate_info.native_libraries[&cnum];
// Here when we include the rlib into our staticlib we need to make a
// decision whether to include the extra object files along the way.
// These extra object files come from statically included native
// libraries, but they may be cfg'd away with #[link(cfg(..))].
//
// This unstable feature, though, only needs liblibc to work. The only
// use case there is where musl is statically included in liblibc.rlib,
// so if we don't want the included version we just need to skip it. As
// a result the logic here is that if *any* linked library is cfg'd away
// we just skip all object files.
//
// Clearly this is not sufficient for a general purpose feature, and
// we'd want to read from the library's metadata to determine which
// object files come from where and selectively skip them.
let skip_object_files = native_libs.iter().any(|lib| {
matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. })
&& !relevant_lib(sess, lib)
});
let lto = are_upstream_rust_objects_already_included(sess) let lto = are_upstream_rust_objects_already_included(sess)
&& !ignored_for_lto(sess, &codegen_results.crate_info, cnum); && !ignored_for_lto(sess, &codegen_results.crate_info, cnum);
// Ignoring obj file starting with the crate name let native_libs = codegen_results.crate_info.native_libraries[&cnum].iter();
// as simple comparison is not enough - there let relevant = native_libs.clone().filter(|lib| relevant_lib(sess, &lib));
// might be also an extra name suffix let relevant_libs: FxHashSet<_> = relevant.filter_map(|lib| lib.filename).collect();
let obj_start = name.as_str().to_owned();
let bundled_libs: FxHashSet<_> = native_libs.filter_map(|lib| lib.filename).collect();
ab.add_archive( ab.add_archive(
path, path,
Box::new(move |fname: &str| { Box::new(move |fname: &str| {
@ -559,20 +515,25 @@ fn link_staticlib<'a>(
return true; return true;
} }
// Otherwise if this is *not* a rust object and we're skipping // Skip objects for bundled libs.
// objects then skip this file if bundled_libs.contains(&Symbol::intern(fname)) {
if skip_object_files
&& (!fname.starts_with(&obj_start) || !fname.ends_with(".o"))
{
return true; return true;
} }
// ok, don't skip this
false false
}), }),
) )
.unwrap(); .unwrap();
archive_builder_builder
.extract_bundled_libs(path, tempdir.as_ref(), &relevant_libs)
.unwrap_or_else(|e| sess.emit_fatal(e));
for filename in relevant_libs {
let joined = tempdir.as_ref().join(filename.as_str());
let path = joined.as_path();
ab.add_archive(path, Box::new(|_| false)).unwrap();
}
all_native_libs all_native_libs
.extend(codegen_results.crate_info.native_libraries[&cnum].iter().cloned()); .extend(codegen_results.crate_info.native_libraries[&cnum].iter().cloned());
}, },
@ -2590,18 +2551,8 @@ fn add_static_crate<'a>(
cmd.link_rlib(&fix_windows_verbatim_for_gcc(path)); cmd.link_rlib(&fix_windows_verbatim_for_gcc(path));
}; };
// See the comment above in `link_staticlib` and `link_rlib` for why if if !are_upstream_rust_objects_already_included(sess)
// there's a static library that's not relevant we skip all object || ignored_for_lto(sess, &codegen_results.crate_info, cnum)
// files.
let native_libs = &codegen_results.crate_info.native_libraries[&cnum];
let skip_native = native_libs.iter().any(|lib| {
matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. })
&& !relevant_lib(sess, lib)
});
if (!are_upstream_rust_objects_already_included(sess)
|| ignored_for_lto(sess, &codegen_results.crate_info, cnum))
&& !skip_native
{ {
link_upstream(cratepath); link_upstream(cratepath);
return; return;
@ -2632,17 +2583,13 @@ fn add_static_crate<'a>(
let is_rust_object = let is_rust_object =
canonical.starts_with(&canonical_name) && looks_like_rust_object_file(&f); canonical.starts_with(&canonical_name) && looks_like_rust_object_file(&f);
// If we've been requested to skip all native object files
// (those not generated by the rust compiler) then we can skip
// this file. See above for why we may want to do this.
let skip_because_cfg_say_so = skip_native && !is_rust_object;
// If we're performing LTO and this is a rust-generated object // If we're performing LTO and this is a rust-generated object
// file, then we don't need the object file as it's part of the // file, then we don't need the object file as it's part of the
// LTO module. Note that `#![no_builtins]` is excluded from LTO, // LTO module. Note that `#![no_builtins]` is excluded from LTO,
// though, so we let that object file slide. // though, so we let that object file slide.
let skip_because_lto = if upstream_rust_objects_already_included && is_rust_object && is_builtins {
upstream_rust_objects_already_included && is_rust_object && is_builtins; return true;
}
// We skip native libraries because: // We skip native libraries because:
// 1. This native libraries won't be used from the generated rlib, // 1. This native libraries won't be used from the generated rlib,
@ -2653,10 +2600,6 @@ fn add_static_crate<'a>(
return true; return true;
} }
if skip_because_cfg_say_so || skip_because_lto {
return true;
}
false false
}), }),
) { ) {

View File

@ -858,6 +858,7 @@ impl CrateInfo {
dependency_formats: tcx.dependency_formats(()).clone(), dependency_formats: tcx.dependency_formats(()).clone(),
windows_subsystem, windows_subsystem,
natvis_debugger_visualizers: Default::default(), natvis_debugger_visualizers: Default::default(),
feature_packed_bundled_libs: tcx.features().packed_bundled_libs,
}; };
let crates = tcx.crates(()); let crates = tcx.crates(());

View File

@ -159,6 +159,7 @@ pub struct CrateInfo {
pub dependency_formats: Lrc<Dependencies>, pub dependency_formats: Lrc<Dependencies>,
pub windows_subsystem: Option<String>, pub windows_subsystem: Option<String>,
pub natvis_debugger_visualizers: BTreeSet<DebuggerVisualizerFile>, pub natvis_debugger_visualizers: BTreeSet<DebuggerVisualizerFile>,
pub feature_packed_bundled_libs: bool, // unstable feature flag.
} }
#[derive(Encodable, Decodable)] #[derive(Encodable, Decodable)]

View File

@ -22,7 +22,7 @@ codegen_ssa_ignoring_output = ignoring -o because multiple .{$extension} files w
codegen_ssa_create_temp_dir = couldn't create a temp dir: {$error} codegen_ssa_create_temp_dir = couldn't create a temp dir: {$error}
codegen_ssa_incompatible_linking_modifiers = the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs codegen_ssa_incompatible_linking_modifiers = link modifiers combination `+bundle,+whole-archive` is unstable when generating rlibs
codegen_ssa_add_native_library = failed to add native library {$library_path}: {$error} codegen_ssa_add_native_library = failed to add native library {$library_path}: {$error}

View File

@ -164,6 +164,8 @@ declare_features! (
(active, multiple_supertrait_upcastable, "CURRENT_RUSTC_VERSION", None, None), (active, multiple_supertrait_upcastable, "CURRENT_RUSTC_VERSION", None, None),
/// Allows using `#[omit_gdb_pretty_printer_section]`. /// Allows using `#[omit_gdb_pretty_printer_section]`.
(active, omit_gdb_pretty_printer_section, "1.5.0", None, None), (active, omit_gdb_pretty_printer_section, "1.5.0", None, None),
/// Allows using `+bundled,+whole-archive` native libs.
(active, packed_bundled_libs, "1.67.0", None, None),
/// Allows using `#[prelude_import]` on glob `use` items. /// Allows using `#[prelude_import]` on glob `use` items.
(active, prelude_import, "1.2.0", None, None), (active, prelude_import, "1.2.0", None, None),
/// Used to identify crates that contain the profiler runtime. /// Used to identify crates that contain the profiler runtime.

View File

@ -49,20 +49,21 @@ fn find_bundled_library(
name: Option<Symbol>, name: Option<Symbol>,
verbatim: Option<bool>, verbatim: Option<bool>,
kind: NativeLibKind, kind: NativeLibKind,
has_cfg: bool,
sess: &Session, sess: &Session,
) -> Option<Symbol> { ) -> Option<Symbol> {
if sess.opts.unstable_opts.packed_bundled_libs && if let NativeLibKind::Static { bundle: Some(true) | None, whole_archive } = kind
sess.crate_types().iter().any(|ct| ct == &CrateType::Rlib || ct == &CrateType::Staticlib) && && sess.crate_types().iter().any(|t| matches!(t, &CrateType::Rlib | CrateType::Staticlib))
let NativeLibKind::Static { bundle: Some(true) | None, .. } = kind { && (sess.opts.unstable_opts.packed_bundled_libs || has_cfg || whole_archive == Some(true))
find_native_static_library( {
name.unwrap().as_str(), let verbatim = verbatim.unwrap_or(false);
verbatim.unwrap_or(false), let search_paths = &sess.target_filesearch(PathKind::Native).search_path_dirs();
&sess.target_filesearch(PathKind::Native).search_path_dirs(), return find_native_static_library(name.unwrap().as_str(), verbatim, search_paths, sess)
sess, .file_name()
).file_name().and_then(|s| s.to_str()).map(Symbol::intern) .and_then(|s| s.to_str())
} else { .map(Symbol::intern);
None
} }
None
} }
pub(crate) fn collect(tcx: TyCtxt<'_>) -> Vec<NativeLib> { pub(crate) fn collect(tcx: TyCtxt<'_>) -> Vec<NativeLib> {
@ -385,7 +386,7 @@ impl<'tcx> Collector<'tcx> {
let name = name.map(|(name, _)| name); let name = name.map(|(name, _)| name);
let kind = kind.unwrap_or(NativeLibKind::Unspecified); let kind = kind.unwrap_or(NativeLibKind::Unspecified);
let filename = find_bundled_library(name, verbatim, kind, sess); let filename = find_bundled_library(name, verbatim, kind, cfg.is_some(), sess);
self.libs.push(NativeLib { self.libs.push(NativeLib {
name, name,
filename, filename,
@ -475,7 +476,7 @@ impl<'tcx> Collector<'tcx> {
let name = Some(Symbol::intern(new_name.unwrap_or(&passed_lib.name))); let name = Some(Symbol::intern(new_name.unwrap_or(&passed_lib.name)));
let sess = self.tcx.sess; let sess = self.tcx.sess;
let filename = let filename =
find_bundled_library(name, passed_lib.verbatim, passed_lib.kind, sess); find_bundled_library(name, passed_lib.verbatim, passed_lib.kind, false, sess);
self.libs.push(NativeLib { self.libs.push(NativeLib {
name, name,
filename, filename,

View File

@ -1049,6 +1049,7 @@ symbols! {
overlapping_marker_traits, overlapping_marker_traits,
owned_box, owned_box,
packed, packed,
packed_bundled_libs,
panic, panic,
panic_2015, panic_2015,
panic_2021, panic_2021,

View File

@ -0,0 +1,35 @@
-include ../../run-make-fulldeps/tools.mk
# ignore-cross-compile
# only-linux
# Make sure -Zpacked_bundled_libs-like behavior activates with whole-archive.
# We're using the llvm-nm instead of the system nm to ensure it is compatible
# with the LLVM bitcode generated by rustc.
NM = "$(LLVM_BIN_DIR)"/llvm-nm
all: $(call NATIVE_STATICLIB,native_dep_1) $(call NATIVE_STATICLIB,native_dep_2) $(call NATIVE_STATICLIB,native_dep_3) $(call NATIVE_STATICLIB,native_dep_4)
# test cfg with packed bundle
$(RUSTC) rust_dep_cfg.rs --crate-type=rlib -Zpacked_bundled_libs
$(RUSTC) main.rs --extern rust_dep=$(TMPDIR)/librust_dep_cfg.rlib --crate-type=staticlib --cfg should_add
$(AR) t $(TMPDIR)/librust_dep_cfg.rlib | $(CGREP) -e "libnative_dep_1.a"
$(AR) t $(TMPDIR)/librust_dep_cfg.rlib | $(CGREP) -e "libnative_dep_2.a"
$(AR) t $(TMPDIR)/libmain.a | $(CGREP) -e "libnative_dep_1.o"
$(AR) t $(TMPDIR)/libmain.a | $(CGREP) -ev "libnative_dep_2.o"
# test bundle with whole_archive
$(RUSTC) rust_dep.rs --crate-type=rlib
$(AR) t $(TMPDIR)/librust_dep.rlib | $(CGREP) -e "native_dep_1"
$(AR) t $(TMPDIR)/librust_dep.rlib | $(CGREP) -e "native_dep_3"
$(AR) t $(TMPDIR)/librust_dep.rlib | $(CGREP) -ev "native_dep_2"
$(AR) t $(TMPDIR)/librust_dep.rlib | $(CGREP) -ev "native_dep_4"
# Make sure compiler doesn't use files, that it shouldn't know about.
rm $(TMPDIR)/libnative_dep_1.a
rm $(TMPDIR)/libnative_dep_3.a
$(RUSTC) main.rs --extern rust_dep=$(TMPDIR)/librust_dep.rlib --print link-args > $(TMPDIR)/link_args
cat $(TMPDIR)/link_args | $(CGREP) -ev "native_dep_3"
cat $(TMPDIR)/link_args | $(CGREP) -e "--whole-archive.*native_dep_1.*--whole-archive.*lnative_dep_2.*no-whole-archive.*lnative_dep_4"

View File

@ -0,0 +1,5 @@
extern crate rust_dep;
pub fn main() {
rust_dep::rust_dep();
}

View File

@ -0,0 +1 @@
int native_f1() { return 1; }

View File

@ -0,0 +1 @@
int native_f2() { return 2; }

View File

@ -0,0 +1 @@
int native_f3() { return 3; }

View File

@ -0,0 +1 @@
int native_f4() { return 4; }

View File

@ -0,0 +1,16 @@
#![feature(packed_bundled_libs)]
#[link(name = "native_dep_1", kind = "static", modifiers = "+whole-archive,+bundle")]
extern "C" {}
#[link(name = "native_dep_2", kind = "static", modifiers = "+whole-archive,-bundle")]
extern "C" {}
#[link(name = "native_dep_3", kind = "static", modifiers = "+bundle")]
extern "C" {}
#[link(name = "native_dep_4", kind = "static", modifiers = "-bundle")]
extern "C" {}
#[no_mangle]
pub fn rust_dep() {}

View File

@ -0,0 +1,10 @@
#![feature(link_cfg)]
#[link(name = "native_dep_1", kind = "static", cfg(should_add))]
extern "C" {}
#[link(name = "native_dep_2", kind = "static", cfg(should_not_add))]
extern "C" {}
#[no_mangle]
pub fn rust_dep() {}

View File

@ -1,8 +1,11 @@
// compile-flags: -Zunstable-options --crate-type rlib // gate-test-packed_bundled_libs
// ignore-wasm32-bare
// compile-flags: --crate-type rlib
// error-pattern: link modifiers combination `+bundle,+whole-archive` is unstable when generating rlibs
// build-fail // build-fail
// error-pattern: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs
#[link(name = "mylib", kind = "static", modifiers = "+bundle,+whole-archive")] #[link(name = "rust_test_helpers", kind = "static", modifiers = "+bundle,+whole-archive")]
extern "C" { } extern "C" {}
fn main() { } fn main() {}

View File

@ -1,6 +1,4 @@
error: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs error: link modifiers combination `+bundle,+whole-archive` is unstable when generating rlibs
error: could not find native static library `mylib`, perhaps an -L flag is missing? error: aborting due to previous error
error: aborting due to 2 previous errors

View File

@ -1,7 +1,8 @@
// Mixing +bundle and +whole-archive is not allowed // gate-test-packed_bundled_libs
// compile-flags: -l static:+bundle,+whole-archive=mylib -Zunstable-options --crate-type rlib // ignore-wasm32-bare
// compile-flags: -l static:+bundle,+whole-archive=rust_test_helpers --crate-type rlib
// error-pattern: link modifiers combination `+bundle,+whole-archive` is unstable when generating rlibs
// build-fail // build-fail
// error-pattern: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs
fn main() { } fn main() {}

View File

@ -1,6 +1,4 @@
error: the linking modifiers `+bundle` and `+whole-archive` are not compatible with each other when generating rlibs error: link modifiers combination `+bundle,+whole-archive` is unstable when generating rlibs
error: could not find native static library `mylib`, perhaps an -L flag is missing? error: aborting due to previous error
error: aborting due to 2 previous errors