Rollup merge of #110651 - durin42:xunit-stdout, r=cuviper

libtest: include test output in junit xml reports

Fixes #110336.

library/test/src/formatters/junit.rs

@@ -11,7 +11,7 @@ use crate::{
 
 pub struct JunitFormatter<T> {
     out: OutputLocation<T>,
-    results: Vec<(TestDesc, TestResult, Duration)>,
+    results: Vec<(TestDesc, TestResult, Duration, Vec<u8>)>,
 }
 
 impl<T: Write> JunitFormatter<T> {
@@ -26,6 +26,18 @@ impl<T: Write> JunitFormatter<T> {
     }
 }
 
+fn str_to_cdata(s: &str) -> String {
+    // Drop the stdout in a cdata. Unfortunately, you can't put either of `]]>` or
+    // `<?'` in a CDATA block, so the escaping gets a little weird.
+    let escaped_output = s.replace("]]>", "]]]]><![CDATA[>");
+    let escaped_output = escaped_output.replace("<?", "<]]><![CDATA[?");
+    // We also smuggle newlines as &#xa so as to keep all the output on one line
+    let escaped_output = escaped_output.replace("\n", "]]>&#xA;<![CDATA[");
+    // Prune empty CDATA blocks resulting from any escaping
+    let escaped_output = escaped_output.replace("<![CDATA[]]>", "");
+    format!("<![CDATA[{}]]>", escaped_output)
+}
+
 impl<T: Write> OutputFormatter for JunitFormatter<T> {
     fn write_discovery_start(&mut self) -> io::Result<()> {
         Err(io::Error::new(io::ErrorKind::NotFound, "Not yet implemented!"))
@@ -63,14 +75,14 @@ impl<T: Write> OutputFormatter for JunitFormatter<T> {
         desc: &TestDesc,
         result: &TestResult,
         exec_time: Option<&time::TestExecTime>,
-        _stdout: &[u8],
+        stdout: &[u8],
         _state: &ConsoleTestState,
     ) -> io::Result<()> {
         // Because the testsuite node holds some of the information as attributes, we can't write it
         // until all of the tests have finished. Instead of writing every result as they come in, we add
         // them to a Vec and write them all at once when run is complete.
         let duration = exec_time.map(|t| t.0).unwrap_or_default();
-        self.results.push((desc.clone(), result.clone(), duration));
+        self.results.push((desc.clone(), result.clone(), duration, stdout.to_vec()));
         Ok(())
     }
     fn write_run_finish(&mut self, state: &ConsoleTestState) -> io::Result<bool> {
@@ -85,7 +97,7 @@ impl<T: Write> OutputFormatter for JunitFormatter<T> {
              >",
             state.failed, state.total, state.ignored
         ))?;
-        for (desc, result, duration) in std::mem::take(&mut self.results) {
+        for (desc, result, duration, stdout) in std::mem::take(&mut self.results) {
             let (class_name, test_name) = parse_class_name(&desc);
             match result {
                 TestResult::TrIgnored => { /* no-op */ }
@@ -98,6 +110,11 @@ impl<T: Write> OutputFormatter for JunitFormatter<T> {
                         duration.as_secs_f64()
                     ))?;
                     self.write_message("<failure type=\"assert\"/>")?;
+                    if !stdout.is_empty() {
+                        self.write_message("<system-out>")?;
+                        self.write_message(&str_to_cdata(&String::from_utf8_lossy(&stdout)))?;
+                        self.write_message("</system-out>")?;
+                    }
                     self.write_message("</testcase>")?;
                 }
 
@@ -110,6 +127,11 @@ impl<T: Write> OutputFormatter for JunitFormatter<T> {
                         duration.as_secs_f64()
                     ))?;
                     self.write_message(&format!("<failure message=\"{m}\" type=\"assert\"/>"))?;
+                    if !stdout.is_empty() {
+                        self.write_message("<system-out>")?;
+                        self.write_message(&str_to_cdata(&String::from_utf8_lossy(&stdout)))?;
+                        self.write_message("</system-out>")?;
+                    }
                     self.write_message("</testcase>")?;
                 }
 
@@ -136,11 +158,19 @@ impl<T: Write> OutputFormatter for JunitFormatter<T> {
                 TestResult::TrOk => {
                     self.write_message(&format!(
                         "<testcase classname=\"{}\" \
-                         name=\"{}\" time=\"{}\"/>",
+                         name=\"{}\" time=\"{}\"",
                         class_name,
                         test_name,
                         duration.as_secs_f64()
                     ))?;
+                    if stdout.is_empty() || !state.options.display_output {
+                        self.write_message("/>")?;
+                    } else {
+                        self.write_message("><system-out>")?;
+                        self.write_message(&str_to_cdata(&String::from_utf8_lossy(&stdout)))?;
+                        self.write_message("</system-out>")?;
+                        self.write_message("</testcase>")?;
+                    }
                 }
             }
         }

tests/run-make/libtest-junit/Makefile

@@ -0,0 +1,19 @@
+# ignore-cross-compile
+include ../tools.mk
+
+# Test expected libtest's junit output
+
+OUTPUT_FILE_DEFAULT := $(TMPDIR)/libtest-junit-output-default.xml
+OUTPUT_FILE_STDOUT_SUCCESS := $(TMPDIR)/libtest-junit-output-stdout-success.xml
+
+all: f.rs validate_junit.py output-default.xml output-stdout-success.xml
+	$(RUSTC) --test f.rs
+	RUST_BACKTRACE=0 $(call RUN,f) -Z unstable-options --test-threads=1 --format=junit > $(OUTPUT_FILE_DEFAULT) || true
+	RUST_BACKTRACE=0 $(call RUN,f) -Z unstable-options --test-threads=1 --format=junit --show-output > $(OUTPUT_FILE_STDOUT_SUCCESS) || true
+
+	cat $(OUTPUT_FILE_DEFAULT) | "$(PYTHON)" validate_junit.py
+	cat $(OUTPUT_FILE_STDOUT_SUCCESS) | "$(PYTHON)" validate_junit.py
+
+	# Normalize the actual output and compare to expected output file
+	cat $(OUTPUT_FILE_DEFAULT) | sed 's/time="[0-9.]*"/time="$$TIME"/g' | diff output-default.xml -
+	cat $(OUTPUT_FILE_STDOUT_SUCCESS) | sed 's/time="[0-9.]*"/time="$$TIME"/g' | diff output-stdout-success.xml -

tests/run-make/libtest-junit/f.rs

@@ -0,0 +1,23 @@
+#[test]
+fn a() {
+    println!("print from successful test");
+    // Should pass
+}
+
+#[test]
+fn b() {
+    println!("print from failing test");
+    assert!(false);
+}
+
+#[test]
+#[should_panic]
+fn c() {
+    assert!(false);
+}
+
+#[test]
+#[ignore = "msg"]
+fn d() {
+    assert!(false);
+}

tests/run-make/libtest-junit/output-default.xml

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8"?><testsuites><testsuite name="test" package="test" id="0" errors="0" failures="1" tests="4" skipped="1" ><testcase classname="unknown" name="a" time="$TIME"/><testcase classname="unknown" name="b" time="$TIME"><failure type="assert"/><system-out><![CDATA[print from failing test]]>&#xA;<![CDATA[thread 'b' panicked at 'assertion failed: false', f.rs:10:5]]>&#xA;<![CDATA[note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace]]>&#xA;<![CDATA[]]></system-out></testcase><testcase classname="unknown" name="c" time="$TIME"/><system-out/><system-err/></testsuite></testsuites>

tests/run-make/libtest-junit/output-stdout-success.xml

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8"?><testsuites><testsuite name="test" package="test" id="0" errors="0" failures="1" tests="4" skipped="1" ><testcase classname="unknown" name="a" time="$TIME"><system-out><![CDATA[print from successful test]]>&#xA;<![CDATA[]]></system-out></testcase><testcase classname="unknown" name="b" time="$TIME"><failure type="assert"/><system-out><![CDATA[print from failing test]]>&#xA;<![CDATA[thread 'b' panicked at 'assertion failed: false', f.rs:10:5]]>&#xA;<![CDATA[note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace]]>&#xA;<![CDATA[]]></system-out></testcase><testcase classname="unknown" name="c" time="$TIME"><system-out><![CDATA[thread 'c' panicked at 'assertion failed: false', f.rs:16:5]]>&#xA;<![CDATA[]]></system-out></testcase><system-out/><system-err/></testsuite></testsuites>

tests/run-make/libtest-junit/validate_junit.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+
+import sys
+import xml.etree.ElementTree as ET
+
+# Try to decode line in order to ensure it is a valid XML document
+for line in sys.stdin:
+    try:
+        ET.fromstring(line)
+    except ET.ParseError as pe:
+        print("Invalid xml: %r" % line)
+        raise