doctests: build test bundle and harness separately

This prevents the included test case from getting at nightly-only
features when run on stable. The harness builds with
RUSTC_BOOTSTRAP, but the bundle doesn't.
This commit is contained in:
Michael Howell 2025-03-02 19:37:40 -07:00 committed by León Orell Valerian Liehr
parent 5d6eeea5f9
commit 9cf531d26f
No known key found for this signature in database
GPG Key ID: D17A07215F68E713
7 changed files with 255 additions and 98 deletions

View File

@ -96,7 +96,7 @@ pub(crate) fn generate_args_file(file_path: &Path, options: &RustdocOptions) ->
.map_err(|error| format!("failed to create args file: {error:?}"))?;
// We now put the common arguments into the file we created.
let mut content = vec!["--crate-type=bin".to_string()];
let mut content = vec![];
for cfg in &options.cfgs {
content.push(format!("--cfg={cfg}"));
@ -513,12 +513,18 @@ pub(crate) struct RunnableDocTest {
line: usize,
edition: Edition,
no_run: bool,
is_multiple_tests: bool,
merged_test_code: Option<String>,
}
impl RunnableDocTest {
fn path_for_merged_doctest(&self) -> PathBuf {
self.test_opts.outdir.path().join(format!("doctest_{}.rs", self.edition))
fn path_for_merged_doctest_bundle(&self) -> PathBuf {
self.test_opts.outdir.path().join(format!("doctest_bundle_{}.rs", self.edition))
}
fn path_for_merged_doctest_runner(&self) -> PathBuf {
self.test_opts.outdir.path().join(format!("doctest_runner_{}.rs", self.edition))
}
fn is_multiple_tests(&self) -> bool {
self.merged_test_code.is_some()
}
}
@ -537,96 +543,108 @@ fn run_test(
let rust_out = add_exe_suffix("rust_out".to_owned(), &rustdoc_options.target);
let output_file = doctest.test_opts.outdir.path().join(rust_out);
// Common arguments used for compiling the doctest runner.
// On merged doctests, the compiler is invoked twice: once for the test code itself,
// and once for the runner wrapper (which needs to use `#![feature]` on stable).
let mut compiler_args = vec![];
compiler_args.push(format!("@{}", doctest.global_opts.args_file.display()));
if let Some(sysroot) = &rustdoc_options.maybe_sysroot {
compiler_args.push(format!("--sysroot={}", sysroot.display()));
}
compiler_args.extend_from_slice(&["--edition".to_owned(), doctest.edition.to_string()]);
if langstr.test_harness {
compiler_args.push("--test".to_owned());
}
if rustdoc_options.json_unused_externs.is_enabled() && !langstr.compile_fail {
compiler_args.push("--error-format=json".to_owned());
compiler_args.extend_from_slice(&["--json".to_owned(), "unused-externs".to_owned()]);
compiler_args.extend_from_slice(&["-W".to_owned(), "unused_crate_dependencies".to_owned()]);
compiler_args.extend_from_slice(&["-Z".to_owned(), "unstable-options".to_owned()]);
}
if doctest.no_run && !langstr.compile_fail && rustdoc_options.persist_doctests.is_none() {
// FIXME: why does this code check if it *shouldn't* persist doctests
// -- shouldn't it be the negation?
compiler_args.push("--emit=metadata".to_owned());
}
compiler_args.extend_from_slice(&[
"--target".to_owned(),
match &rustdoc_options.target {
TargetTuple::TargetTuple(s) => s.clone(),
TargetTuple::TargetJson { path_for_rustdoc, .. } => {
path_for_rustdoc.to_str().expect("target path must be valid unicode").to_owned()
}
},
]);
if let ErrorOutputType::HumanReadable { kind, color_config } = rustdoc_options.error_format {
let short = kind.short();
let unicode = kind == HumanReadableErrorType::Unicode;
if short {
compiler_args.extend_from_slice(&["--error-format".to_owned(), "short".to_owned()]);
}
if unicode {
compiler_args
.extend_from_slice(&["--error-format".to_owned(), "human-unicode".to_owned()]);
}
match color_config {
ColorConfig::Never => {
compiler_args.extend_from_slice(&["--color".to_owned(), "never".to_owned()]);
}
ColorConfig::Always => {
compiler_args.extend_from_slice(&["--color".to_owned(), "always".to_owned()]);
}
ColorConfig::Auto => {
compiler_args.extend_from_slice(&[
"--color".to_owned(),
if supports_color { "always" } else { "never" }.to_owned(),
]);
}
}
}
let rustc_binary = rustdoc_options
.test_builder
.as_deref()
.unwrap_or_else(|| rustc_interface::util::rustc_path().expect("found rustc"));
let mut compiler = wrapped_rustc_command(&rustdoc_options.test_builder_wrappers, rustc_binary);
compiler.arg(format!("@{}", doctest.global_opts.args_file.display()));
compiler.args(&compiler_args);
if let Some(sysroot) = &rustdoc_options.maybe_sysroot {
compiler.arg(format!("--sysroot={}", sysroot.display()));
}
compiler.arg("--edition").arg(doctest.edition.to_string());
if doctest.is_multiple_tests {
// The merged test harness uses the `test` crate, so we need to actually allow it.
// This will not expose nightly features on stable, because crate attrs disable
// merging, and `#![feature]` is required to be a crate attr.
compiler.env("RUSTC_BOOTSTRAP", "1");
// If this is a merged doctest, we need to write it into a file instead of using stdin
// because if the size of the merged doctests is too big, it'll simply break stdin.
if doctest.is_multiple_tests() {
// It makes the compilation failure much faster if it is for a combined doctest.
compiler.arg("--error-format=short");
let input_file = doctest.path_for_merged_doctest_bundle();
if std::fs::write(&input_file, &doctest.full_test_code).is_err() {
// If we cannot write this file for any reason, we leave. All combined tests will be
// tested as standalone tests.
return Err(TestFailure::CompileError);
}
if !rustdoc_options.nocapture {
// If `nocapture` is disabled, then we don't display rustc's output when compiling
// the merged doctests.
compiler.stderr(Stdio::null());
}
// bundled tests are an rlib, loaded by a separate runner executable
compiler
.arg("--crate-type=lib")
.arg("--out-dir")
.arg(doctest.test_opts.outdir.path())
.arg(input_file);
} else {
compiler.arg("--crate-type=bin").arg("-o").arg(&output_file);
// Setting these environment variables is unneeded if this is a merged doctest.
compiler.env("UNSTABLE_RUSTDOC_TEST_PATH", &doctest.test_opts.path);
compiler.env(
"UNSTABLE_RUSTDOC_TEST_LINE",
format!("{}", doctest.line as isize - doctest.full_test_line_offset as isize),
);
}
compiler.arg("-o").arg(&output_file);
if langstr.test_harness {
compiler.arg("--test");
}
if rustdoc_options.json_unused_externs.is_enabled() && !langstr.compile_fail {
compiler.arg("--error-format=json");
compiler.arg("--json").arg("unused-externs");
compiler.arg("-W").arg("unused_crate_dependencies");
compiler.arg("-Z").arg("unstable-options");
}
if doctest.no_run && !langstr.compile_fail && rustdoc_options.persist_doctests.is_none() {
// FIXME: why does this code check if it *shouldn't* persist doctests
// -- shouldn't it be the negation?
compiler.arg("--emit=metadata");
}
compiler.arg("--target").arg(match &rustdoc_options.target {
TargetTuple::TargetTuple(s) => s,
TargetTuple::TargetJson { path_for_rustdoc, .. } => {
path_for_rustdoc.to_str().expect("target path must be valid unicode")
}
});
if let ErrorOutputType::HumanReadable { kind, color_config } = rustdoc_options.error_format {
let short = kind.short();
let unicode = kind == HumanReadableErrorType::Unicode;
if short {
compiler.arg("--error-format").arg("short");
}
if unicode {
compiler.arg("--error-format").arg("human-unicode");
}
match color_config {
ColorConfig::Never => {
compiler.arg("--color").arg("never");
}
ColorConfig::Always => {
compiler.arg("--color").arg("always");
}
ColorConfig::Auto => {
compiler.arg("--color").arg(if supports_color { "always" } else { "never" });
}
}
}
// If this is a merged doctest, we need to write it into a file instead of using stdin
// because if the size of the merged doctests is too big, it'll simply break stdin.
if doctest.is_multiple_tests {
// It makes the compilation failure much faster if it is for a combined doctest.
compiler.arg("--error-format=short");
let input_file = doctest.path_for_merged_doctest();
if std::fs::write(&input_file, &doctest.full_test_code).is_err() {
// If we cannot write this file for any reason, we leave. All combined tests will be
// tested as standalone tests.
return Err(TestFailure::CompileError);
}
compiler.arg(input_file);
if !rustdoc_options.nocapture {
// If `nocapture` is disabled, then we don't display rustc's output when compiling
// the merged doctests.
compiler.stderr(Stdio::null());
}
} else {
compiler.arg("-");
compiler.stdin(Stdio::piped());
compiler.stderr(Stdio::piped());
@ -635,8 +653,65 @@ fn run_test(
debug!("compiler invocation for doctest: {compiler:?}");
let mut child = compiler.spawn().expect("Failed to spawn rustc process");
let output = if doctest.is_multiple_tests {
let output = if let Some(merged_test_code) = &doctest.merged_test_code {
// compile-fail tests never get merged, so this should always pass
let status = child.wait().expect("Failed to wait");
// the actual test runner is a separate component, built with nightly-only features;
// build it now
let runner_input_file = doctest.path_for_merged_doctest_runner();
let mut runner_compiler =
wrapped_rustc_command(&rustdoc_options.test_builder_wrappers, rustc_binary);
// the test runner does not contain any user-written code, so this doesn't allow
// the user to exploit nightly-only features on stable
runner_compiler.env("RUSTC_BOOTSTRAP", "1");
runner_compiler.args(compiler_args);
runner_compiler.args(&["--crate-type=bin", "-o"]).arg(&output_file);
let mut extern_path = std::ffi::OsString::from(format!(
"--extern=doctest_bundle_{edition}=",
edition = doctest.edition
));
for extern_str in &rustdoc_options.extern_strs {
if let Some((_cratename, path)) = extern_str.split_once('=') {
// Direct dependencies of the tests themselves are
// indirect dependencies of the test runner.
// They need to be in the library search path.
let dir = Path::new(path)
.parent()
.filter(|x| x.components().count() > 0)
.unwrap_or(Path::new("."));
runner_compiler.arg("-L").arg(dir);
}
}
let output_bundle_file = doctest
.test_opts
.outdir
.path()
.join(format!("libdoctest_bundle_{edition}.rlib", edition = doctest.edition));
extern_path.push(&output_bundle_file);
runner_compiler.arg(extern_path);
runner_compiler.arg(&runner_input_file);
if std::fs::write(&runner_input_file, &merged_test_code).is_err() {
// If we cannot write this file for any reason, we leave. All combined tests will be
// tested as standalone tests.
return Err(TestFailure::CompileError);
}
if !rustdoc_options.nocapture {
// If `nocapture` is disabled, then we don't display rustc's output when compiling
// the merged doctests.
runner_compiler.stderr(Stdio::null());
}
runner_compiler.arg("--error-format=short");
debug!("compiler invocation for doctest runner: {runner_compiler:?}");
let status = if !status.success() {
status
} else {
let mut child_runner = runner_compiler.spawn().expect("Failed to spawn rustc process");
child_runner.wait().expect("Failed to wait")
};
process::Output { status, stdout: Vec::new(), stderr: Vec::new() }
} else {
let stdin = child.stdin.as_mut().expect("Failed to open stdin");
@ -713,7 +788,7 @@ fn run_test(
cmd.arg(&output_file);
} else {
cmd = Command::new(&output_file);
if doctest.is_multiple_tests {
if doctest.is_multiple_tests() {
cmd.env("RUSTDOC_DOCTEST_BIN_PATH", &output_file);
}
}
@ -721,7 +796,7 @@ fn run_test(
cmd.current_dir(run_directory);
}
let result = if doctest.is_multiple_tests || rustdoc_options.nocapture {
let result = if doctest.is_multiple_tests() || rustdoc_options.nocapture {
cmd.status().map(|status| process::Output {
status,
stdout: Vec::new(),
@ -1008,7 +1083,7 @@ fn doctest_run_fn(
line: scraped_test.line,
edition: scraped_test.edition(&rustdoc_options),
no_run: scraped_test.no_run(&rustdoc_options),
is_multiple_tests: false,
merged_test_code: None,
};
let res =
run_test(runnable_test, &rustdoc_options, doctest.supports_color, report_unused_externs);

View File

@ -14,6 +14,7 @@ pub(crate) struct DocTestRunner {
crate_attrs: FxIndexSet<String>,
ids: String,
output: String,
output_merged_tests: String,
supports_color: bool,
nb_tests: usize,
}
@ -24,6 +25,7 @@ impl DocTestRunner {
crate_attrs: FxIndexSet::default(),
ids: String::new(),
output: String::new(),
output_merged_tests: String::new(),
supports_color: true,
nb_tests: 0,
}
@ -55,7 +57,8 @@ impl DocTestRunner {
scraped_test,
ignore,
self.nb_tests,
&mut self.output
&mut self.output,
&mut self.output_merged_tests,
),
));
self.supports_color &= doctest.supports_color;
@ -78,9 +81,11 @@ impl DocTestRunner {
"
.to_string();
let mut code_prefix = String::new();
for crate_attr in &self.crate_attrs {
code.push_str(crate_attr);
code.push('\n');
code_prefix.push_str(crate_attr);
code_prefix.push('\n');
}
if opts.attrs.is_empty() {
@ -88,15 +93,16 @@ impl DocTestRunner {
// lints that are commonly triggered in doctests. The crate-level test attributes are
// commonly used to make tests fail in case they trigger warnings, so having this there in
// that case may cause some tests to pass when they shouldn't have.
code.push_str("#![allow(unused)]\n");
code_prefix.push_str("#![allow(unused)]\n");
}
// Next, any attributes that came from the crate root via #![doc(test(attr(...)))].
for attr in &opts.attrs {
code.push_str(&format!("#![{attr}]\n"));
code_prefix.push_str(&format!("#![{attr}]\n"));
}
code.push_str("extern crate test;\n");
writeln!(code, "extern crate doctest_bundle_{edition} as doctest_bundle;").unwrap();
let test_args = test_args.iter().fold(String::new(), |mut x, arg| {
write!(x, "{arg:?}.to_string(),").unwrap();
@ -161,12 +167,12 @@ the same process\");
std::process::Termination::report(test::test_main(test_args, Vec::from(TESTS), None))
}}",
nb_tests = self.nb_tests,
output = self.output,
output = self.output_merged_tests,
ids = self.ids,
)
.expect("failed to generate test code");
let runnable_test = RunnableDocTest {
full_test_code: code,
full_test_code: format!("{code_prefix}{code}", code = self.output),
full_test_line_offset: 0,
test_opts: test_options,
global_opts: opts.clone(),
@ -174,7 +180,7 @@ std::process::Termination::report(test::test_main(test_args, Vec::from(TESTS), N
line: 0,
edition,
no_run: false,
is_multiple_tests: true,
merged_test_code: Some(code),
};
let ret =
run_test(runnable_test, rustdoc_options, self.supports_color, |_: UnusedExterns| {});
@ -189,14 +195,15 @@ fn generate_mergeable_doctest(
ignore: bool,
id: usize,
output: &mut String,
output_merged_tests: &mut String,
) -> String {
let test_id = format!("__doctest_{id}");
if ignore {
// We generate nothing else.
writeln!(output, "mod {test_id} {{\n").unwrap();
writeln!(output, "pub mod {test_id} {{}}\n").unwrap();
} else {
writeln!(output, "mod {test_id} {{\n{}{}", doctest.crates, doctest.maybe_crate_attrs)
writeln!(output, "pub mod {test_id} {{\n{}{}", doctest.crates, doctest.maybe_crate_attrs)
.unwrap();
if doctest.has_main_fn {
output.push_str(&doctest.everything_else);
@ -216,11 +223,17 @@ fn main() {returns_result} {{
)
.unwrap();
}
writeln!(
output,
"\npub fn __main_fn() -> impl std::process::Termination {{ main() }} \n}}\n"
)
.unwrap();
}
let not_running = ignore || scraped_test.langstr.no_run;
writeln!(
output,
output_merged_tests,
"
mod {test_id} {{
pub const TEST: test::TestDescAndFn = test::TestDescAndFn::new_doctest(
{test_name:?}, {ignore}, {file:?}, {line}, {no_run}, {should_panic},
test::StaticTestFn(
@ -242,7 +255,7 @@ test::StaticTestFn(
if let Some(bin_path) = crate::__doctest_mod::doctest_path() {{
test::assert_test_result(crate::__doctest_mod::doctest_runner(bin_path, {id}))
}} else {{
test::assert_test_result(self::main())
test::assert_test_result(doctest_bundle::{test_id}::__main_fn())
}}
",
)

View File

@ -8,7 +8,6 @@ fn test_and_compare(input_file: &str, stdout_file: &str, edition: &str, dep: &Pa
let output = cmd
.input(input_file)
.arg("--test")
.arg("-Zunstable-options")
.edition(edition)
.arg("--test-args=--test-threads=1")
.extern_("foo", dep.display().to_string())

View File

@ -2,7 +2,7 @@
//@[edition2015]edition:2015
//@[edition2015]aux-build:extern_macros.rs
//@[edition2015]compile-flags:--test --test-args=--test-threads=1
//@[edition2024]edition:2015
//@[edition2024]edition:2024
//@[edition2024]aux-build:extern_macros.rs
//@[edition2024]compile-flags:--test --test-args=--test-threads=1
//@ normalize-stdout: "tests/rustdoc-ui/doctest" -> "$$DIR"

View File

@ -0,0 +1,28 @@
running 1 test
test $DIR/failed-doctest-test-crate.rs - m (line 14) ... FAILED
failures:
---- $DIR/failed-doctest-test-crate.rs - m (line 14) stdout ----
error[E0432]: unresolved import `test`
--> $DIR/failed-doctest-test-crate.rs:15:5
|
LL | use test::*;
| ^^^^ use of unresolved module or unlinked crate `test`
|
help: you might be missing a crate named `test`, add it to your project and import it in your code
|
LL + extern crate test;
|
error: aborting due to 1 previous error
For more information about this error, try `rustc --explain E0432`.
Couldn't compile the test.
failures:
$DIR/failed-doctest-test-crate.rs - m (line 14)
test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME

View File

@ -0,0 +1,25 @@
running 1 test
test $DIR/failed-doctest-test-crate.rs - m (line 14) ... FAILED
failures:
---- $DIR/failed-doctest-test-crate.rs - m (line 14) stdout ----
error[E0432]: unresolved import `test`
--> $DIR/failed-doctest-test-crate.rs:15:5
|
LL | use test::*;
| ^^^^ use of unresolved module or unlinked crate `test`
|
= help: you might be missing a crate named `test`
error: aborting due to 1 previous error
For more information about this error, try `rustc --explain E0432`.
Couldn't compile the test.
failures:
$DIR/failed-doctest-test-crate.rs - m (line 14)
test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out; finished in $TIME

View File

@ -0,0 +1,17 @@
// FIXME: if/when the output of the test harness can be tested on its own, this test should be
// adapted to use that, and that normalize line can go away
//@ revisions: edition2015 edition2024
//@[edition2015]edition:2015
//@[edition2024]edition:2024
//@ compile-flags:--test
//@ normalize-stdout: "tests/rustdoc-ui/doctest" -> "$$DIR"
//@ normalize-stdout: "finished in \d+\.\d+s" -> "finished in $$TIME"
//@ failure-status: 101
/// <https://github.com/rust-lang/rust/pull/137899#discussion_r1976743383>
///
/// ```rust
/// use test::*;
/// ```
pub mod m {}