Auto merge of #83529 - richkadel:demangler, r=tmandry

Make rust-demangler installable

Adds bootstrap rules to support installing rust-demangler, as an optional, in-tree `extended` tool. It can be included by updating `config.toml`, setting `extended = true`, and then either (a) adding `"rust-demangler"` to the `tools` array, or by enabling `profiler = true`. In other words, it is a _default_ `extended` tool if `profiler = true`.

When compiling with `-Z instrument-coverage`, the coverage reports are
generated by `llvm-cov`. `llvm-cov` includes a built-in demangler for
C++, and an option to supply an alternate demangler. For Rust, we have
`rust-demangler`, currently used in `rustc` coverage tests.

Fuchsia's toolchain for Rust is built via `./x.py install`. Fuchsia is
adding support for Rust coverage, and we need to include the
`rust-demangler` in the installed `bin` directory.

r? `@tmandry`
This commit is contained in:
bors 2021-04-04 02:30:18 +00:00
commit 0850c37bd3
12 changed files with 327 additions and 46 deletions

View File

@ -259,10 +259,11 @@ changelog-seen = 2
# be built if `extended = true`.
#extended = false
# Installs chosen set of extended tools if `extended = true`. By default builds all.
# If chosen tool failed to build the installation fails. If `extended = false`, this
# option is ignored.
#tools = ["cargo", "rls", "clippy", "rustfmt", "analysis", "src"]
# Installs chosen set of extended tools if `extended = true`. By default builds
# all extended tools except `rust-demangler`, unless the target is also being
# built with `profiler = true`. If chosen tool failed to build the installation
# fails. If `extended = false`, this option is ignored.
#tools = ["cargo", "rls", "clippy", "rustfmt", "analysis", "src"] # + "rust-demangler" if `profiler`
# Verbosity level: 0 == not verbose, 1 == verbose, 2 == very verbose
#verbose = 0

View File

@ -420,6 +420,7 @@ impl<'a> Builder<'a> {
test::Rustfmt,
test::Miri,
test::Clippy,
test::RustDemangler,
test::CompiletestTest,
test::RustdocJSStd,
test::RustdocJSNotStd,
@ -466,6 +467,7 @@ impl<'a> Builder<'a> {
dist::Rls,
dist::RustAnalyzer,
dist::Rustfmt,
dist::RustDemangler,
dist::Clippy,
dist::Miri,
dist::LlvmTools,
@ -481,6 +483,7 @@ impl<'a> Builder<'a> {
install::Rls,
install::RustAnalyzer,
install::Rustfmt,
install::RustDemangler,
install::Clippy,
install::Miri,
install::Analysis,

View File

@ -1246,6 +1246,56 @@ impl Step for Rustfmt {
}
}
#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
pub struct RustDemangler {
pub compiler: Compiler,
pub target: TargetSelection,
}
impl Step for RustDemangler {
type Output = Option<GeneratedTarball>;
const ONLY_HOSTS: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("rust-demangler")
}
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(RustDemangler {
compiler: run.builder.compiler_for(
run.builder.top_stage,
run.builder.config.build,
run.target,
),
target: run.target,
});
}
fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
let compiler = self.compiler;
let target = self.target;
assert!(builder.config.extended);
// Only build this extended tool if explicitly included in `tools`, or if `profiler = true`
let profiler = builder.config.profiler_enabled(target);
if !builder.config.tools.as_ref().map_or(profiler, |t| t.contains("rust-demangler")) {
return None;
}
let rust_demangler = builder
.ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
.expect("rust-demangler expected to build - in-tree tool");
// Prepare the image directory
let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple);
tarball.set_overlay(OverlayKind::RustDemangler);
tarball.is_preview(true);
tarball.add_file(&rust_demangler, "bin", 0o755);
tarball.add_legal_and_readme_to("share/doc/rust-demangler");
Some(tarball.generate())
}
}
#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
pub struct Extended {
stage: u32,
@ -1282,6 +1332,7 @@ impl Step for Extended {
let rustc_installer = builder.ensure(Rustc { compiler: builder.compiler(stage, target) });
let cargo_installer = builder.ensure(Cargo { compiler, target });
let rustfmt_installer = builder.ensure(Rustfmt { compiler, target });
let rust_demangler_installer = builder.ensure(RustDemangler { compiler, target });
let rls_installer = builder.ensure(Rls { compiler, target });
let rust_analyzer_installer = builder.ensure(RustAnalyzer { compiler, target });
let llvm_tools_installer = builder.ensure(LlvmTools { target });
@ -1307,9 +1358,10 @@ impl Step for Extended {
let mut tarballs = Vec::new();
tarballs.push(rustc_installer);
tarballs.push(cargo_installer);
tarballs.push(clippy_installer);
tarballs.extend(rust_demangler_installer.clone());
tarballs.extend(rls_installer.clone());
tarballs.extend(rust_analyzer_installer.clone());
tarballs.push(clippy_installer);
tarballs.extend(miri_installer.clone());
tarballs.extend(rustfmt_installer.clone());
tarballs.extend(llvm_tools_installer);
@ -1366,6 +1418,9 @@ impl Step for Extended {
let xform = |p: &Path| {
let mut contents = t!(fs::read_to_string(p));
if rust_demangler_installer.is_none() {
contents = filter(&contents, "rust-demangler");
}
if rls_installer.is_none() {
contents = filter(&contents, "rls");
}
@ -1414,7 +1469,9 @@ impl Step for Extended {
prepare("rust-std");
prepare("rust-analysis");
prepare("clippy");
if rust_demangler_installer.is_some() {
prepare("rust-demangler");
}
if rls_installer.is_some() {
prepare("rls");
}
@ -1462,6 +1519,8 @@ impl Step for Extended {
"rust-analyzer-preview".to_string()
} else if name == "clippy" {
"clippy-preview".to_string()
} else if name == "rust-demangler" {
"rust-demangler-preview".to_string()
} else if name == "miri" {
"miri-preview".to_string()
} else {
@ -1479,6 +1538,9 @@ impl Step for Extended {
prepare("rust-docs");
prepare("rust-std");
prepare("clippy");
if rust_demangler_installer.is_some() {
prepare("rust-demangler");
}
if rls_installer.is_some() {
prepare("rls");
}
@ -1620,6 +1682,25 @@ impl Step for Extended {
.arg("-t")
.arg(etc.join("msi/remove-duplicates.xsl")),
);
if rust_demangler_installer.is_some() {
builder.run(
Command::new(&heat)
.current_dir(&exe)
.arg("dir")
.arg("rust-demangler")
.args(&heat_flags)
.arg("-cg")
.arg("RustDemanglerGroup")
.arg("-dr")
.arg("RustDemangler")
.arg("-var")
.arg("var.RustDemanglerDir")
.arg("-out")
.arg(exe.join("RustDemanglerGroup.wxs"))
.arg("-t")
.arg(etc.join("msi/remove-duplicates.xsl")),
);
}
if miri_installer.is_some() {
builder.run(
Command::new(&heat)
@ -1693,6 +1774,9 @@ impl Step for Extended {
.arg(&input);
add_env(builder, &mut cmd, target);
if rust_demangler_installer.is_some() {
cmd.arg("-dRustDemanglerDir=rust-demangler");
}
if rls_installer.is_some() {
cmd.arg("-dRlsDir=rls");
}
@ -1715,6 +1799,9 @@ impl Step for Extended {
candle("CargoGroup.wxs".as_ref());
candle("StdGroup.wxs".as_ref());
candle("ClippyGroup.wxs".as_ref());
if rust_demangler_installer.is_some() {
candle("RustDemanglerGroup.wxs".as_ref());
}
if rls_installer.is_some() {
candle("RlsGroup.wxs".as_ref());
}
@ -1761,6 +1848,9 @@ impl Step for Extended {
if rust_analyzer_installer.is_some() {
cmd.arg("RustAnalyzerGroup.wixobj");
}
if rust_demangler_installer.is_some() {
cmd.arg("RustDemanglerGroup.wixobj");
}
if miri_installer.is_some() {
cmd.arg("MiriGroup.wixobj");
}

View File

@ -190,6 +190,22 @@ install!((self, builder, _config),
);
}
};
RustDemangler, "rust-demangler", Self::should_build(_config), only_hosts: true, {
// Note: Even though `should_build` may return true for `extended` default tools,
// dist::RustDemangler may still return None, unless the target-dependent `profiler` config
// is also true, or the `tools` array explicitly includes "rust-demangler".
if let Some(tarball) = builder.ensure(dist::RustDemangler {
compiler: self.compiler,
target: self.target
}) {
install_sh(builder, "rust-demangler", self.compiler.stage, Some(self.target), &tarball);
} else {
builder.info(
&format!("skipping Install RustDemangler stage{} ({})",
self.compiler.stage, self.target),
);
}
};
Analysis, "analysis", Self::should_build(_config), only_hosts: false, {
let tarball = builder.ensure(dist::Analysis {
// Find the actual compiler (handling the full bootstrap option) which

View File

@ -15,6 +15,7 @@ pub(crate) enum OverlayKind {
Clippy,
Miri,
Rustfmt,
RustDemangler,
RLS,
RustAnalyzer,
}
@ -47,6 +48,9 @@ impl OverlayKind {
"src/tools/rustfmt/LICENSE-APACHE",
"src/tools/rustfmt/LICENSE-MIT",
],
OverlayKind::RustDemangler => {
&["src/tools/rust-demangler/README.md", "LICENSE-APACHE", "LICENSE-MIT"]
}
OverlayKind::RLS => &[
"src/tools/rls/README.md",
"src/tools/rls/LICENSE-APACHE",
@ -64,6 +68,7 @@ impl OverlayKind {
match self {
OverlayKind::Rust => builder.rust_version(),
OverlayKind::LLVM => builder.rust_version(),
OverlayKind::RustDemangler => builder.release_num("rust-demangler"),
OverlayKind::Cargo => {
builder.cargo_info.version(builder, &builder.release_num("cargo"))
}

View File

@ -351,6 +351,57 @@ impl Step for Rustfmt {
}
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct RustDemangler {
stage: u32,
host: TargetSelection,
}
impl Step for RustDemangler {
type Output = ();
const ONLY_HOSTS: bool = true;
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
run.path("src/tools/rust-demangler")
}
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(RustDemangler { stage: run.builder.top_stage, host: run.target });
}
/// Runs `cargo test` for rust-demangler.
fn run(self, builder: &Builder<'_>) {
let stage = self.stage;
let host = self.host;
let compiler = builder.compiler(stage, host);
let rust_demangler = builder
.ensure(tool::RustDemangler { compiler, target: self.host, extra_features: Vec::new() })
.expect("in-tree tool");
let mut cargo = tool::prepare_tool_cargo(
builder,
compiler,
Mode::ToolRustc,
host,
"test",
"src/tools/rust-demangler",
SourceType::InTree,
&[],
);
let dir = testdir(builder, compiler.host);
t!(fs::create_dir_all(&dir));
cargo.env("RUST_DEMANGLER_DRIVER_PATH", rust_demangler);
cargo.arg("--").args(builder.config.cmd.test_args());
cargo.add_rustc_lib_path(builder, compiler);
builder.run(&mut cargo.into());
}
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
pub struct Miri {
stage: u32,
@ -1126,7 +1177,10 @@ note: if you're sure you want to do this, please open an issue as to why. In the
}
if mode == "run-make" && suite.ends_with("fulldeps") {
cmd.arg("--rust-demangler-path").arg(builder.tool_exe(Tool::RustDemangler));
let rust_demangler = builder
.ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() })
.expect("in-tree tool");
cmd.arg("--rust-demangler-path").arg(rust_demangler);
}
cmd.arg("--src-base").arg(builder.src.join("src/test").join(suite));

View File

@ -368,7 +368,6 @@ bootstrap_tool!(
Compiletest, "src/tools/compiletest", "compiletest", is_unstable_tool = true;
BuildManifest, "src/tools/build-manifest", "build-manifest";
RemoteTestClient, "src/tools/remote-test-client", "remote-test-client";
RustDemangler, "src/tools/rust-demangler", "rust-demangler";
RustInstaller, "src/tools/rust-installer", "fabricate", is_external_tool = true;
RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors";
@ -719,6 +718,7 @@ tool_extended!((self, builder),
});
self.extra_features.push("clippy".to_owned());
};
RustDemangler, rust_demangler, "src/tools/rust-demangler", "rust-demangler", stable=false, in_tree=true, {};
Rustfmt, rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, {};
RustAnalyzer, rust_analyzer, "src/tools/rust-analyzer/crates/rust-analyzer", "rust-analyzer", stable=false, {};
);

View File

@ -8,6 +8,10 @@ edition = "2018"
regex = "1.0"
rustc-demangle = "0.1.17"
[lib]
name = "rust_demangler"
doctest = false
[[bin]]
name = "rust-demangler"
path = "main.rs"
test = false

View File

@ -0,0 +1,36 @@
# rust-demangler
_Demangles rustc mangled names._
`rust-demangler` supports the requirements of the [`llvm-cov show -Xdemangler`
option](https://llvm.org/docs/CommandGuide/llvm-cov.html#cmdoption-llvm-cov-show-xdemangler),
to perform Rust-specific symbol demangling:
> _The demangler is expected to read a newline-separated list of symbols from
> stdin and write a newline-separated list of the same length to stdout._
To use `rust-demangler` with `llvm-cov` for example:
```shell
$ TARGET="${PWD}/build/x86_64-unknown-linux-gnu"
$ "${TARGET}"/llvm/bin/llvm-cov show \
--Xdemangler=path/to/rust-demangler \
--instr-profile=main.profdata ./main --show-line-counts-or-regions
```
`rust-demangler` is a Rust "extended tool", used in Rust compiler tests, and
optionally included in Rust distributions that enable coverage profiling. Symbol
demangling is implemented using the
[rustc-demangle](https://crates.io/crates/rustc-demangle) crate.
_(Note, for Rust developers, the third-party tool
[`rustfilt`](https://crates.io/crates/rustfilt) also supports `llvm-cov` symbol
demangling. `rustfilt` is a more generalized tool that searches any body of
text, using pattern matching, to find and demangle Rust symbols.)_
## License
Rust-demangler is distributed under the terms of both the MIT license and the
Apache License (Version 2.0).
See [LICENSE-APACHE](/LICENSE-APACHE) and [LICENSE-MIT](/LICENSE-MIT) for details.

View File

@ -0,0 +1,21 @@
use regex::Regex;
use rustc_demangle::demangle;
use std::str::Lines;
const REPLACE_COLONS: &str = "::";
pub fn create_disambiguator_re() -> Regex {
Regex::new(r"\[[a-f0-9]{5,16}\]::").unwrap()
}
pub fn demangle_lines(lines: Lines<'_>, strip_crate_disambiguators: Option<Regex>) -> Vec<String> {
let mut demangled_lines = Vec::new();
for mangled in lines {
let mut demangled = demangle(mangled).to_string();
if let Some(re) = &strip_crate_disambiguators {
demangled = re.replace_all(&demangled, REPLACE_COLONS).to_string();
}
demangled_lines.push(demangled);
}
demangled_lines
}

View File

@ -1,27 +1,5 @@
//! Demangles rustc mangled names.
//!
//! This tool uses https://crates.io/crates/rustc-demangle to convert an input buffer of
//! newline-separated mangled names into their demangled translations.
//!
//! This tool can be leveraged by other applications that support third-party demanglers.
//! It takes a list of mangled names (one per line) on standard input, and prints a corresponding
//! list of demangled names. The tool is designed to support other programs that can leverage a
//! third-party demangler, such as `llvm-cov`, via the `-Xdemangler=<path-to-demangler>` option.
//!
//! To use `rust-demangler`, first build the tool with:
//!
//! ```shell
//! $ ./x.py build rust-demangler
//! ```
//!
//! Then, with `llvm-cov` for example, add the `-Xdemangler=...` option:
//!
//! ```shell
//! $ TARGET="${PWD}/build/x86_64-unknown-linux-gnu"
//! $ "${TARGET}"/llvm/bin/llvm-cov show --Xdemangler="${TARGET}"/stage0-tools-bin/rust-demangler \
//! --instr-profile=main.profdata ./main --show-line-counts-or-regions
//! ```
//!
//! Note regarding crate disambiguators:
//!
//! Some demangled symbol paths can include "crate disambiguator" suffixes, represented as a large
@ -57,12 +35,9 @@
//! These disambiguators seem to have more analytical value (for instance, in coverage analysis), so
//! they are not removed.
use regex::Regex;
use rustc_demangle::demangle;
use rust_demangler::*;
use std::io::{self, Read, Write};
const REPLACE_COLONS: &str = "::";
fn main() -> io::Result<()> {
// FIXME(richkadel): In Issue #77615 discussed updating the `rustc-demangle` library, to provide
// an option to generate demangled names without including crate disambiguators. If that
@ -82,7 +57,7 @@ fn main() -> io::Result<()> {
// and more than three leading zeros should be extremely unlikely. Conversely, it should be
// sufficient to assume the zero-based indexes for closures and anonymous scopes will never
// exceed the value 9999.
let mut strip_crate_disambiguators = Some(Regex::new(r"\[[a-f0-9]{5,16}\]::").unwrap());
let mut strip_crate_disambiguators = Some(create_disambiguator_re());
let mut args = std::env::args();
let progname = args.next().unwrap();
@ -115,16 +90,8 @@ fn main() -> io::Result<()> {
let mut buffer = String::new();
io::stdin().read_to_string(&mut buffer)?;
let lines = buffer.lines();
let mut demangled_lines = Vec::new();
for mangled in lines {
let mut demangled = demangle(mangled).to_string();
if let Some(re) = &strip_crate_disambiguators {
demangled = re.replace_all(&demangled, REPLACE_COLONS).to_string();
}
demangled_lines.push(demangled);
}
demangled_lines.push("".to_string());
let mut demangled_lines = demangle_lines(buffer.lines(), strip_crate_disambiguators);
demangled_lines.push("".to_string()); // ensure a trailing newline
io::stdout().write_all(demangled_lines.join("\n").as_bytes())?;
Ok(())
}

View File

@ -0,0 +1,84 @@
use rust_demangler::*;
const MANGLED_INPUT: &str = r"
_RNvC6_123foo3bar
_RNqCs4fqI2P2rA04_11utf8_identsu30____7hkackfecea1cbdathfdh9hlq6y
_RNCNCNgCs6DXkGYLi8lr_2cc5spawn00B5_
_RNCINkXs25_NgCsbmNqQUJIY6D_4core5sliceINyB9_4IterhENuNgNoBb_4iter8iterator8Iterator9rpositionNCNgNpB9_6memchr7memrchrs_0E0Bb_
_RINbNbCskIICzLVDPPb_5alloc5alloc8box_freeDINbNiB4_5boxed5FnBoxuEp6OutputuEL_ECs1iopQbuBiw2_3std
INtC8arrayvec8ArrayVechKj7b_E
_RMCs4fqI2P2rA04_13const_genericINtB0_8UnsignedKhb_E
_RMCs4fqI2P2rA04_13const_genericINtB0_6SignedKs98_E
_RMCs4fqI2P2rA04_13const_genericINtB0_6SignedKanb_E
_RMCs4fqI2P2rA04_13const_genericINtB0_4BoolKb0_E
_RMCs4fqI2P2rA04_13const_genericINtB0_4BoolKb1_E
_RMCs4fqI2P2rA04_13const_genericINtB0_4CharKc76_E
_RMCs4fqI2P2rA04_13const_genericINtB0_4CharKca_E
_RMCs4fqI2P2rA04_13const_genericINtB0_4CharKc2202_E
_RNvNvMCs4fqI2P2rA04_13const_genericINtB4_3FooKpE3foo3FOO
_RC3foo.llvm.9D1C9369
_RC3foo.llvm.9D1C9369@@16
_RNvC9backtrace3foo.llvm.A5310EB9
_RNvNtNtNtNtCs92dm3009vxr_4rand4rngs7adapter9reseeding4fork23FORK_HANDLER_REGISTERED.0.0
";
const DEMANGLED_OUTPUT: &str = r"
123foo[0]::bar
utf8_idents[317d481089b8c8fe]::_გემრიელი_სადილი
cc[4d6468d6c9fd4bb3]::spawn::{closure#0}::{closure#0}
<core[846817f741e54dfd]::slice::Iter<u8> as core[846817f741e54dfd]::iter::iterator::Iterator>::rposition::<core[846817f741e54dfd]::slice::memchr::memrchr::{closure#1}>::{closure#0}
alloc[f15a878b47eb696b]::alloc::box_free::<dyn alloc[f15a878b47eb696b]::boxed::FnBox<(), Output = ()>>
INtC8arrayvec8ArrayVechKj7b_E
<const_generic[317d481089b8c8fe]::Unsigned<11: u8>>
<const_generic[317d481089b8c8fe]::Signed<152: i16>>
<const_generic[317d481089b8c8fe]::Signed<-11: i8>>
<const_generic[317d481089b8c8fe]::Bool<false: bool>>
<const_generic[317d481089b8c8fe]::Bool<true: bool>>
<const_generic[317d481089b8c8fe]::Char<'v': char>>
<const_generic[317d481089b8c8fe]::Char<'\n': char>>
<const_generic[317d481089b8c8fe]::Char<'∂': char>>
<const_generic[317d481089b8c8fe]::Foo<_>>::foo::FOO
foo[0]
foo[0]
backtrace[0]::foo
rand[693ea8e72247470f]::rngs::adapter::reseeding::fork::FORK_HANDLER_REGISTERED.0.0
";
const DEMANGLED_OUTPUT_NO_CRATE_DISAMBIGUATORS: &str = r"
123foo[0]::bar
utf8_idents::_გემრიელი_სადილი
cc::spawn::{closure#0}::{closure#0}
<core::slice::Iter<u8> as core::iter::iterator::Iterator>::rposition::<core::slice::memchr::memrchr::{closure#1}>::{closure#0}
alloc::alloc::box_free::<dyn alloc::boxed::FnBox<(), Output = ()>>
INtC8arrayvec8ArrayVechKj7b_E
<const_generic::Unsigned<11: u8>>
<const_generic::Signed<152: i16>>
<const_generic::Signed<-11: i8>>
<const_generic::Bool<false: bool>>
<const_generic::Bool<true: bool>>
<const_generic::Char<'v': char>>
<const_generic::Char<'\n': char>>
<const_generic::Char<'∂': char>>
<const_generic::Foo<_>>::foo::FOO
foo[0]
foo[0]
backtrace[0]::foo
rand::rngs::adapter::reseeding::fork::FORK_HANDLER_REGISTERED.0.0
";
#[test]
fn test_demangle_lines() {
let demangled_lines = demangle_lines(MANGLED_INPUT.lines(), None);
for (expected, actual) in DEMANGLED_OUTPUT.lines().zip(demangled_lines) {
assert_eq!(expected, actual);
}
}
#[test]
fn test_demangle_lines_no_crate_disambiguators() {
let demangled_lines = demangle_lines(MANGLED_INPUT.lines(), Some(create_disambiguator_re()));
for (expected, actual) in DEMANGLED_OUTPUT_NO_CRATE_DISAMBIGUATORS.lines().zip(demangled_lines)
{
assert_eq!(expected, actual);
}
}