Generating the coverage map

rustc now generates the coverage map and can support (limited)
coverage report generation, at the function level.

Example:

$ BUILD=$HOME/rust/build/x86_64-unknown-linux-gnu
$ $BUILD/stage1/bin/rustc -Zinstrument-coverage \
$HOME/rust/src/test/run-make-fulldeps/instrument-coverage/main.rs
$ LLVM_PROFILE_FILE="main.profraw" ./main
called
$ $BUILD/llvm/bin/llvm-profdata merge -sparse main.profraw -o main.profdata
$ $BUILD/llvm/bin/llvm-cov show --instr-profile=main.profdata main
    1|      1|pub fn will_be_called() {
    2|      1|    println!("called");
    3|      1|}
    4|       |
    5|      0|pub fn will_not_be_called() {
    6|      0|    println!("should not have been called");
    7|      0|}
    8|       |
    9|      1|fn main() {
   10|      1|    let less = 1;
   11|      1|    let more = 100;
   12|      1|
   13|      1|    if less < more {
   14|      1|        will_be_called();
   15|      1|    } else {
   16|      1|        will_not_be_called();
   17|      1|    }
   18|      1|}

Cargo.lock

@@ -2788,6 +2788,13 @@ dependencies = [
  "rls-span",
 ]
 
+[[package]]
+name = "rust-demangler"
+version = "0.0.0"
+dependencies = [
+ "rustc-demangle",
+]
+
 [[package]]
 name = "rustbook"
 version = "0.1.0"

Cargo.toml

@@ -17,6 +17,7 @@ members = [
   "src/tools/remote-test-client",
   "src/tools/remote-test-server",
   "src/tools/rust-installer",
+  "src/tools/rust-demangler",
   "src/tools/cargo",
   "src/tools/rustdoc",
   "src/tools/rls",

src/bootstrap/builder.rs

@@ -369,6 +369,7 @@ impl<'a> Builder<'a> {
                 tool::Cargo,
                 tool::Rls,
                 tool::RustAnalyzer,
+                tool::RustDemangler,
                 tool::Rustdoc,
                 tool::Clippy,
                 tool::CargoClippy,

src/bootstrap/test.rs

@@ -1019,6 +1019,10 @@ impl Step for Compiletest {
             cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler));
         }
 
+        if mode == "run-make" && suite.ends_with("fulldeps") {
+            cmd.arg("--rust-demangler-path").arg(builder.tool_exe(Tool::RustDemangler));
+        }
+
         cmd.arg("--src-base").arg(builder.src.join("src/test").join(suite));
         cmd.arg("--build-base").arg(testdir(builder, compiler.host).join(suite));
         cmd.arg("--stage-id").arg(format!("stage{}-{}", compiler.stage, target));

src/bootstrap/tool.rs

@@ -361,6 +361,7 @@ bootstrap_tool!(
     Compiletest, "src/tools/compiletest", "compiletest", is_unstable_tool = true;
     BuildManifest, "src/tools/build-manifest", "build-manifest";
     RemoteTestClient, "src/tools/remote-test-client", "remote-test-client";
+    RustDemangler, "src/tools/rust-demangler", "rust-demangler";
     RustInstaller, "src/tools/rust-installer", "fabricate", is_external_tool = true;
     RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
     ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors";

src/libcore/intrinsics.rs

@@ -1957,8 +1957,14 @@ extern "rust-intrinsic" {
     /// Internal placeholder for injecting code coverage counters when the "instrument-coverage"
     /// option is enabled. The placeholder is replaced with `llvm.instrprof.increment` during code
     /// generation.
+    #[cfg(not(bootstrap))]
     #[lang = "count_code_region"]
-    pub fn count_code_region(index: u32, start_byte_pos: u32, end_byte_pos: u32);
+    pub fn count_code_region(
+        function_source_hash: u64,
+        index: u32,
+        start_byte_pos: u32,
+        end_byte_pos: u32,
+    );
 
     /// Internal marker for code coverage expressions, injected into the MIR when the
     /// "instrument-coverage" option is enabled. This intrinsic is not converted into a
@@ -1966,6 +1972,8 @@ extern "rust-intrinsic" {
     /// "coverage map", which is injected into the generated code, as additional data.
     /// This marker identifies a code region and two other counters or counter expressions
     /// whose sum is the number of times the code region was executed.
+    #[cfg(not(bootstrap))]
+    #[lang = "coverage_counter_add"]
     pub fn coverage_counter_add(
         index: u32,
         left_index: u32,
@@ -1977,6 +1985,8 @@ extern "rust-intrinsic" {
     /// This marker identifies a code region and two other counters or counter expressions
     /// whose difference is the number of times the code region was executed.
     /// (See `coverage_counter_add` for more information.)
+    #[cfg(not(bootstrap))]
+    #[lang = "coverage_counter_subtract"]
     pub fn coverage_counter_subtract(
         index: u32,
         left_index: u32,

src/librustc_codegen_llvm/attributes.rs

@@ -133,6 +133,9 @@ fn set_probestack(cx: &CodegenCx<'ll, '_>, llfn: &'ll Value) {
         return;
     }
 
+    // FIXME(richkadel): Make sure probestack plays nice with `-Z instrument-coverage`
+    // or disable it if not, similar to above early exits.
+
     // Flag our internal `__rust_probestack` function as the stack probe symbol.
     // This is defined in the `compiler-builtins` crate for each architecture.
     llvm::AddFunctionAttrStringValue(

src/librustc_codegen_llvm/base.rs

@@ -144,17 +144,18 @@ pub fn compile_codegen_unit(
                 }
             }
 
+            // Finalize code coverage by injecting the coverage map. Note, the coverage map will
+            // also be added to the `llvm.used` variable, created next.
+            if cx.sess().opts.debugging_opts.instrument_coverage {
+                cx.coverageinfo_finalize();
+            }
+
             // Create the llvm.used variable
             // This variable has type [N x i8*] and is stored in the llvm.metadata section
             if !cx.used_statics().borrow().is_empty() {
                 cx.create_used_variable()
             }
 
-            // Finalize code coverage by injecting the coverage map
-            if cx.sess().opts.debugging_opts.instrument_coverage {
-                cx.coverageinfo_finalize();
-            }
-
             // Finalize debuginfo
             if cx.sess().opts.debuginfo != DebugInfo::None {
                 cx.debuginfo_finalize();

src/librustc_codegen_llvm/builder.rs

@@ -1060,7 +1060,7 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
             fn_name, hash, num_counters, index
         );
 
-        let llfn = unsafe { llvm::LLVMRustGetInstrprofIncrementIntrinsic(self.cx().llmod) };
+        let llfn = unsafe { llvm::LLVMRustGetInstrProfIncrementIntrinsic(self.cx().llmod) };
         let args = &[fn_name, hash, num_counters, index];
         let args = self.check_call("call", llfn, args);
 

src/librustc_codegen_llvm/consts.rs

@@ -493,10 +493,14 @@ impl StaticMethods for CodegenCx<'ll, 'tcx> {
             }
 
             if attrs.flags.contains(CodegenFnAttrFlags::USED) {
-                // This static will be stored in the llvm.used variable which is an array of i8*
-                let cast = llvm::LLVMConstPointerCast(g, self.type_i8p());
-                self.used_statics.borrow_mut().push(cast);
+                self.add_used_global(g);
             }
         }
     }
+
+    /// Add a global value to a list to be stored in the `llvm.used` variable, an array of i8*.
+    fn add_used_global(&self, global: &'ll Value) {
+        let cast = unsafe { llvm::LLVMConstPointerCast(global, self.type_i8p()) };
+        self.used_statics.borrow_mut().push(cast);
+    }
 }

src/librustc_codegen_llvm/coverageinfo/mapgen.rs

@@ -0,0 +1,274 @@
+use crate::llvm;
+
+use crate::common::CodegenCx;
+use crate::coverageinfo;
+
+use log::debug;
+use rustc_codegen_ssa::coverageinfo::map::*;
+use rustc_codegen_ssa::traits::{BaseTypeMethods, ConstMethods, MiscMethods};
+use rustc_data_structures::fx::FxHashMap;
+use rustc_llvm::RustString;
+use rustc_middle::ty::Instance;
+use rustc_middle::{bug, mir};
+
+use std::collections::BTreeMap;
+use std::ffi::CString;
+use std::path::PathBuf;
+
+// FIXME(richkadel): Complete all variations of generating and exporting the coverage map to LLVM.
+// The current implementation is an initial foundation with basic capabilities (Counters, but not
+// CounterExpressions, etc.).
+
+/// Generates and exports the Coverage Map.
+///
+/// This Coverage Map complies with Coverage Mapping Format version 3 (zero-based encoded as 2),
+/// as defined at [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/llvmorg-8.0.0/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format)
+/// and published in Rust's current (July 2020) fork of LLVM. This version is supported by the
+/// LLVM coverage tools (`llvm-profdata` and `llvm-cov`) bundled with Rust's fork of LLVM.
+///
+/// Consequently, Rust's bundled version of Clang also generates Coverage Maps compliant with
+/// version 3. Clang's implementation of Coverage Map generation was referenced when implementing
+/// this Rust version, and though the format documentation is very explicit and detailed, some
+/// undocumented details in Clang's implementation (that may or may not be important) were also
+/// replicated for Rust's Coverage Map.
+pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) {
+    let mut coverage_writer = CoverageMappingWriter::new(cx);
+
+    let function_coverage_map = cx.coverage_context().take_function_coverage_map();
+
+    // Encode coverage mappings and generate function records
+    let mut function_records = Vec::<&'ll llvm::Value>::new();
+    let coverage_mappings_buffer = llvm::build_byte_buffer(|coverage_mappings_buffer| {
+        for (instance, function_coverage) in function_coverage_map.into_iter() {
+            if let Some(function_record) = coverage_writer.write_function_mappings_and_record(
+                instance,
+                function_coverage,
+                coverage_mappings_buffer,
+            ) {
+                function_records.push(function_record);
+            }
+        }
+    });
+
+    // Encode all filenames covered in this module, ordered by `file_id`
+    let filenames_buffer = llvm::build_byte_buffer(|filenames_buffer| {
+        coverageinfo::write_filenames_section_to_buffer(
+            &coverage_writer.filenames,
+            filenames_buffer,
+        );
+    });
+
+    if coverage_mappings_buffer.len() > 0 {
+        // Generate the LLVM IR representation of the coverage map and store it in a well-known
+        // global constant.
+        coverage_writer.write_coverage_map(
+            function_records,
+            filenames_buffer,
+            coverage_mappings_buffer,
+        );
+    }
+}
+
+struct CoverageMappingWriter<'a, 'll, 'tcx> {
+    cx: &'a CodegenCx<'ll, 'tcx>,
+    filenames: Vec<CString>,
+    filename_to_index: FxHashMap<CString, u32>,
+}
+
+impl<'a, 'll, 'tcx> CoverageMappingWriter<'a, 'll, 'tcx> {
+    fn new(cx: &'a CodegenCx<'ll, 'tcx>) -> Self {
+        Self { cx, filenames: Vec::new(), filename_to_index: FxHashMap::<CString, u32>::default() }
+    }
+
+    /// For the given function, get the coverage region data, stream it to the given buffer, and
+    /// then generate and return a new function record.
+    fn write_function_mappings_and_record(
+        &mut self,
+        instance: Instance<'tcx>,
+        mut function_coverage: FunctionCoverage,
+        coverage_mappings_buffer: &RustString,
+    ) -> Option<&'ll llvm::Value> {
+        let cx = self.cx;
+        let coverageinfo: &mir::CoverageInfo = cx.tcx.coverageinfo(instance.def_id());
+        debug!(
+            "Generate coverage map for: {:?}, num_counters: {}, num_expressions: {}",
+            instance, coverageinfo.num_counters, coverageinfo.num_expressions
+        );
+        debug_assert!(coverageinfo.num_counters > 0);
+
+        let regions_in_file_order = function_coverage.regions_in_file_order(cx.sess().source_map());
+        if regions_in_file_order.len() == 0 {
+            return None;
+        }
+
+        // Stream the coverage mapping regions for the function (`instance`) to the buffer, and
+        // compute the data byte size used.
+        let old_len = coverage_mappings_buffer.len();
+        self.regions_to_mappings(regions_in_file_order, coverage_mappings_buffer);
+        let mapping_data_size = coverage_mappings_buffer.len() - old_len;
+        debug_assert!(mapping_data_size > 0);
+
+        let mangled_function_name = cx.tcx.symbol_name(instance).to_string();
+        let name_ref = coverageinfo::compute_hash(&mangled_function_name);
+        let function_source_hash = function_coverage.source_hash();
+
+        // Generate and return the function record
+        let name_ref_val = cx.const_u64(name_ref);
+        let mapping_data_size_val = cx.const_u32(mapping_data_size as u32);
+        let func_hash_val = cx.const_u64(function_source_hash);
+        Some(cx.const_struct(
+            &[name_ref_val, mapping_data_size_val, func_hash_val],
+            /*packed=*/ true,
+        ))
+    }
+
+    /// For each coverage region, extract its coverage data from the earlier coverage analysis.
+    /// Use LLVM APIs to convert the data into buffered bytes compliant with the LLVM Coverage
+    /// Mapping format.
+    fn regions_to_mappings(
+        &mut self,
+        regions_in_file_order: BTreeMap<PathBuf, BTreeMap<CoverageLoc, (usize, CoverageKind)>>,
+        coverage_mappings_buffer: &RustString,
+    ) {
+        let mut virtual_file_mapping = Vec::new();
+        let mut mapping_regions = coverageinfo::SmallVectorCounterMappingRegion::new();
+        let mut expressions = coverageinfo::SmallVectorCounterExpression::new();
+
+        for (file_id, (file_path, file_coverage_regions)) in
+            regions_in_file_order.into_iter().enumerate()
+        {
+            let file_id = file_id as u32;
+            let filename = CString::new(file_path.to_string_lossy().to_string())
+                .expect("null error converting filename to C string");
+            debug!("  file_id: {} = '{:?}'", file_id, filename);
+            let filenames_index = match self.filename_to_index.get(&filename) {
+                Some(index) => *index,
+                None => {
+                    let index = self.filenames.len() as u32;
+                    self.filenames.push(filename.clone());
+                    self.filename_to_index.insert(filename, index);
+                    index
+                }
+            };
+            virtual_file_mapping.push(filenames_index);
+
+            let mut mapping_indexes = vec![0 as u32; file_coverage_regions.len()];
+            for (mapping_index, (region_id, _)) in file_coverage_regions.values().enumerate() {
+                mapping_indexes[*region_id] = mapping_index as u32;
+            }
+
+            for (region_loc, (region_id, region_kind)) in file_coverage_regions.into_iter() {
+                let mapping_index = mapping_indexes[region_id];
+                match region_kind {
+                    CoverageKind::Counter => {
+                        debug!(
+                            "  Counter {}, file_id: {}, region_loc: {}",
+                            mapping_index, file_id, region_loc
+                        );
+                        mapping_regions.push_from(
+                            mapping_index,
+                            file_id,
+                            region_loc.start_line,
+                            region_loc.start_col,
+                            region_loc.end_line,
+                            region_loc.end_col,
+                        );
+                    }
+                    CoverageKind::CounterExpression(lhs, op, rhs) => {
+                        debug!(
+                            "  CounterExpression {} = {} {:?} {}, file_id: {}, region_loc: {:?}",
+                            mapping_index, lhs, op, rhs, file_id, region_loc,
+                        );
+                        mapping_regions.push_from(
+                            mapping_index,
+                            file_id,
+                            region_loc.start_line,
+                            region_loc.start_col,
+                            region_loc.end_line,
+                            region_loc.end_col,
+                        );
+                        expressions.push_from(op, lhs, rhs);
+                    }
+                    CoverageKind::Unreachable => {
+                        debug!(
+                            "  Unreachable region, file_id: {}, region_loc: {:?}",
+                            file_id, region_loc,
+                        );
+                        bug!("Unreachable region not expected and not yet handled!")
+                        // FIXME(richkadel): implement and call
+                        //   mapping_regions.push_from(...) for unreachable regions
+                    }
+                }
+            }
+        }
+
+        // Encode and append the current function's coverage mapping data
+        coverageinfo::write_mapping_to_buffer(
+            virtual_file_mapping,
+            expressions,
+            mapping_regions,
+            coverage_mappings_buffer,
+        );
+    }
+
+    fn write_coverage_map(
+        self,
+        function_records: Vec<&'ll llvm::Value>,
+        filenames_buffer: Vec<u8>,
+        mut coverage_mappings_buffer: Vec<u8>,
+    ) {
+        let cx = self.cx;
+
+        // Concatenate the encoded filenames and encoded coverage mappings, and add additional zero
+        // bytes as-needed to ensure 8-byte alignment.
+        let mut coverage_size = coverage_mappings_buffer.len();
+        let filenames_size = filenames_buffer.len();
+        let remaining_bytes =
+            (filenames_size + coverage_size) % coverageinfo::COVMAP_VAR_ALIGN_BYTES;
+        if remaining_bytes > 0 {
+            let pad = coverageinfo::COVMAP_VAR_ALIGN_BYTES - remaining_bytes;
+            coverage_mappings_buffer.append(&mut [0].repeat(pad));
+            coverage_size += pad;
+        }
+        let filenames_and_coverage_mappings = [filenames_buffer, coverage_mappings_buffer].concat();
+        let filenames_and_coverage_mappings_val =
+            cx.const_bytes(&filenames_and_coverage_mappings[..]);
+
+        debug!(
+            "cov map: n_records = {}, filenames_size = {}, coverage_size = {}, 0-based version = {}",
+            function_records.len(),
+            filenames_size,
+            coverage_size,
+            coverageinfo::mapping_version()
+        );
+
+        // Create the coverage data header
+        let n_records_val = cx.const_u32(function_records.len() as u32);
+        let filenames_size_val = cx.const_u32(filenames_size as u32);
+        let coverage_size_val = cx.const_u32(coverage_size as u32);
+        let version_val = cx.const_u32(coverageinfo::mapping_version());
+        let cov_data_header_val = cx.const_struct(
+            &[n_records_val, filenames_size_val, coverage_size_val, version_val],
+            /*packed=*/ false,
+        );
+
+        // Create the function records array
+        let name_ref_from_u64 = cx.type_i64();
+        let mapping_data_size_from_u32 = cx.type_i32();
+        let func_hash_from_u64 = cx.type_i64();
+        let function_record_ty = cx.type_struct(
+            &[name_ref_from_u64, mapping_data_size_from_u32, func_hash_from_u64],
+            /*packed=*/ true,
+        );
+        let function_records_val = cx.const_array(function_record_ty, &function_records[..]);
+
+        // Create the complete LLVM coverage data value to add to the LLVM IR
+        let cov_data_val = cx.const_struct(
+            &[cov_data_header_val, function_records_val, filenames_and_coverage_mappings_val],
+            /*packed=*/ false,
+        );
+
+        // Save the coverage data value to LLVM IR
+        coverageinfo::save_map_to_mod(cx, cov_data_val);
+    }
+}

src/librustc_codegen_llvm/coverageinfo/mod.rs

@@ -1,67 +1,44 @@
+use crate::llvm;
+
 use crate::builder::Builder;
 use crate::common::CodegenCx;
+
+use libc::c_uint;
 use log::debug;
 use rustc_codegen_ssa::coverageinfo::map::*;
-use rustc_codegen_ssa::traits::{CoverageInfoBuilderMethods, CoverageInfoMethods};
+use rustc_codegen_ssa::traits::{
+    BaseTypeMethods, CoverageInfoBuilderMethods, CoverageInfoMethods, StaticMethods,
+};
 use rustc_data_structures::fx::FxHashMap;
+use rustc_llvm::RustString;
 use rustc_middle::ty::Instance;
 
 use std::cell::RefCell;
+use std::ffi::CString;
+
+pub mod mapgen;
+
+const COVMAP_VAR_ALIGN_BYTES: usize = 8;
 
 /// A context object for maintaining all state needed by the coverageinfo module.
 pub struct CrateCoverageContext<'tcx> {
     // Coverage region data for each instrumented function identified by DefId.
-    pub(crate) coverage_regions: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverageRegions>>,
+    pub(crate) function_coverage_map: RefCell<FxHashMap<Instance<'tcx>, FunctionCoverage>>,
 }
 
 impl<'tcx> CrateCoverageContext<'tcx> {
     pub fn new() -> Self {
-        Self { coverage_regions: Default::default() }
+        Self { function_coverage_map: Default::default() }
     }
-}
 
-/// Generates and exports the Coverage Map.
-// FIXME(richkadel): Actually generate and export the coverage map to LLVM.
-// The current implementation is actually just debug messages to show the data is available.
-pub fn finalize(cx: &CodegenCx<'_, '_>) {
-    let coverage_regions = &*cx.coverage_context().coverage_regions.borrow();
-    for instance in coverage_regions.keys() {
-        let coverageinfo = cx.tcx.coverageinfo(instance.def_id());
-        debug_assert!(coverageinfo.num_counters > 0);
-        debug!(
-            "Generate coverage map for: {:?}, hash: {}, num_counters: {}",
-            instance, coverageinfo.hash, coverageinfo.num_counters
-        );
-        let function_coverage_regions = &coverage_regions[instance];
-        for (index, region) in function_coverage_regions.indexed_regions() {
-            match region.kind {
-                CoverageKind::Counter => debug!(
-                    "  Counter {}, for {}..{}",
-                    index, region.coverage_span.start_byte_pos, region.coverage_span.end_byte_pos
-                ),
-                CoverageKind::CounterExpression(lhs, op, rhs) => debug!(
-                    "  CounterExpression {} = {} {:?} {}, for {}..{}",
-                    index,
-                    lhs,
-                    op,
-                    rhs,
-                    region.coverage_span.start_byte_pos,
-                    region.coverage_span.end_byte_pos
-                ),
-            }
-        }
-        for unreachable in function_coverage_regions.unreachable_regions() {
-            debug!(
-                "  Unreachable code region: {}..{}",
-                unreachable.start_byte_pos, unreachable.end_byte_pos
-            );
-        }
+    pub fn take_function_coverage_map(&self) -> FxHashMap<Instance<'tcx>, FunctionCoverage> {
+        self.function_coverage_map.replace(FxHashMap::default())
     }
 }
 
 impl CoverageInfoMethods for CodegenCx<'ll, 'tcx> {
     fn coverageinfo_finalize(&self) {
-        finalize(self)
+        mapgen::finalize(self)
     }
 }
 
@@ -69,20 +46,22 @@ impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
     fn add_counter_region(
         &mut self,
         instance: Instance<'tcx>,
+        function_source_hash: u64,
         index: u32,
         start_byte_pos: u32,
         end_byte_pos: u32,
     ) {
         debug!(
-            "adding counter to coverage map: instance={:?}, index={}, byte range {}..{}",
-            instance, index, start_byte_pos, end_byte_pos,
-        );
-        let mut coverage_regions = self.coverage_context().coverage_regions.borrow_mut();
-        coverage_regions.entry(instance).or_default().add_counter(
-            index,
-            start_byte_pos,
-            end_byte_pos,
+            "adding counter to coverage_regions: instance={:?}, function_source_hash={}, index={}, byte range {}..{}",
+            instance, function_source_hash, index, start_byte_pos, end_byte_pos,
         );
+        let mut coverage_regions = self.coverage_context().function_coverage_map.borrow_mut();
+        coverage_regions
+            .entry(instance)
+            .or_insert_with(|| {
+                FunctionCoverage::with_coverageinfo(self.tcx.coverageinfo(instance.def_id()))
+            })
+            .add_counter(function_source_hash, index, start_byte_pos, end_byte_pos);
     }
 
     fn add_counter_expression_region(
@@ -96,18 +75,16 @@ impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
         end_byte_pos: u32,
     ) {
         debug!(
-            "adding counter expression to coverage map: instance={:?}, index={}, {} {:?} {}, byte range {}..{}",
+            "adding counter expression to coverage_regions: instance={:?}, index={}, {} {:?} {}, byte range {}..{}",
             instance, index, lhs, op, rhs, start_byte_pos, end_byte_pos,
         );
-        let mut coverage_regions = self.coverage_context().coverage_regions.borrow_mut();
-        coverage_regions.entry(instance).or_default().add_counter_expression(
-            index,
-            lhs,
-            op,
-            rhs,
-            start_byte_pos,
-            end_byte_pos,
-        );
+        let mut coverage_regions = self.coverage_context().function_coverage_map.borrow_mut();
+        coverage_regions
+            .entry(instance)
+            .or_insert_with(|| {
+                FunctionCoverage::with_coverageinfo(self.tcx.coverageinfo(instance.def_id()))
+            })
+            .add_counter_expression(index, lhs, op, rhs, start_byte_pos, end_byte_pos);
     }
 
     fn add_unreachable_region(
@@ -117,10 +94,175 @@ impl CoverageInfoBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
         end_byte_pos: u32,
     ) {
         debug!(
-            "adding unreachable code to coverage map: instance={:?}, byte range {}..{}",
+            "adding unreachable code to coverage_regions: instance={:?}, byte range {}..{}",
             instance, start_byte_pos, end_byte_pos,
         );
-        let mut coverage_regions = self.coverage_context().coverage_regions.borrow_mut();
-        coverage_regions.entry(instance).or_default().add_unreachable(start_byte_pos, end_byte_pos);
+        let mut coverage_regions = self.coverage_context().function_coverage_map.borrow_mut();
+        coverage_regions
+            .entry(instance)
+            .or_insert_with(|| {
+                FunctionCoverage::with_coverageinfo(self.tcx.coverageinfo(instance.def_id()))
+            })
+            .add_unreachable(start_byte_pos, end_byte_pos);
     }
 }
+
+/// This struct wraps an opaque reference to the C++ template instantiation of
+/// `llvm::SmallVector<coverage::CounterExpression>`. Each `coverage::CounterExpression` object is
+/// constructed from primative-typed arguments, and pushed to the `SmallVector`, in the C++
+/// implementation of `LLVMRustCoverageSmallVectorCounterExpressionAdd()` (see
+/// `src/rustllvm/CoverageMappingWrapper.cpp`).
+pub struct SmallVectorCounterExpression<'a> {
+    pub raw: &'a mut llvm::coverageinfo::SmallVectorCounterExpression<'a>,
+}
+
+impl SmallVectorCounterExpression<'a> {
+    pub fn new() -> Self {
+        SmallVectorCounterExpression {
+            raw: unsafe { llvm::LLVMRustCoverageSmallVectorCounterExpressionCreate() },
+        }
+    }
+
+    pub fn as_ptr(&self) -> *const llvm::coverageinfo::SmallVectorCounterExpression<'a> {
+        self.raw
+    }
+
+    pub fn push_from(
+        &mut self,
+        kind: rustc_codegen_ssa::coverageinfo::CounterOp,
+        left_index: u32,
+        right_index: u32,
+    ) {
+        unsafe {
+            llvm::LLVMRustCoverageSmallVectorCounterExpressionAdd(
+                &mut *(self.raw as *mut _),
+                kind,
+                left_index,
+                right_index,
+            )
+        }
+    }
+}
+
+impl Drop for SmallVectorCounterExpression<'a> {
+    fn drop(&mut self) {
+        unsafe {
+            llvm::LLVMRustCoverageSmallVectorCounterExpressionDispose(&mut *(self.raw as *mut _));
+        }
+    }
+}
+
+/// This struct wraps an opaque reference to the C++ template instantiation of
+/// `llvm::SmallVector<coverage::CounterMappingRegion>`. Each `coverage::CounterMappingRegion`
+/// object is constructed from primative-typed arguments, and pushed to the `SmallVector`, in the
+/// C++ implementation of `LLVMRustCoverageSmallVectorCounterMappingRegionAdd()` (see
+/// `src/rustllvm/CoverageMappingWrapper.cpp`).
+pub struct SmallVectorCounterMappingRegion<'a> {
+    pub raw: &'a mut llvm::coverageinfo::SmallVectorCounterMappingRegion<'a>,
+}
+
+impl SmallVectorCounterMappingRegion<'a> {
+    pub fn new() -> Self {
+        SmallVectorCounterMappingRegion {
+            raw: unsafe { llvm::LLVMRustCoverageSmallVectorCounterMappingRegionCreate() },
+        }
+    }
+
+    pub fn as_ptr(&self) -> *const llvm::coverageinfo::SmallVectorCounterMappingRegion<'a> {
+        self.raw
+    }
+
+    pub fn push_from(
+        &mut self,
+        index: u32,
+        file_id: u32,
+        line_start: u32,
+        column_start: u32,
+        line_end: u32,
+        column_end: u32,
+    ) {
+        unsafe {
+            llvm::LLVMRustCoverageSmallVectorCounterMappingRegionAdd(
+                &mut *(self.raw as *mut _),
+                index,
+                file_id,
+                line_start,
+                column_start,
+                line_end,
+                column_end,
+            )
+        }
+    }
+}
+
+impl Drop for SmallVectorCounterMappingRegion<'a> {
+    fn drop(&mut self) {
+        unsafe {
+            llvm::LLVMRustCoverageSmallVectorCounterMappingRegionDispose(
+                &mut *(self.raw as *mut _),
+            );
+        }
+    }
+}
+
+pub(crate) fn write_filenames_section_to_buffer(filenames: &Vec<CString>, buffer: &RustString) {
+    let c_str_vec = filenames.iter().map(|cstring| cstring.as_ptr()).collect::<Vec<_>>();
+    unsafe {
+        llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer(
+            c_str_vec.as_ptr(),
+            c_str_vec.len(),
+            buffer,
+        );
+    }
+}
+
+pub(crate) fn write_mapping_to_buffer(
+    virtual_file_mapping: Vec<u32>,
+    expressions: SmallVectorCounterExpression<'_>,
+    mapping_regions: SmallVectorCounterMappingRegion<'_>,
+    buffer: &RustString,
+) {
+    unsafe {
+        llvm::LLVMRustCoverageWriteMappingToBuffer(
+            virtual_file_mapping.as_ptr(),
+            virtual_file_mapping.len() as c_uint,
+            expressions.as_ptr(),
+            mapping_regions.as_ptr(),
+            buffer,
+        );
+    }
+}
+
+pub(crate) fn compute_hash(name: &str) -> u64 {
+    let name = CString::new(name).expect("null error converting hashable name to C string");
+    unsafe { llvm::LLVMRustCoverageComputeHash(name.as_ptr()) }
+}
+
+pub(crate) fn mapping_version() -> u32 {
+    unsafe { llvm::LLVMRustCoverageMappingVersion() }
+}
+
+pub(crate) fn save_map_to_mod<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    cov_data_val: &'ll llvm::Value,
+) {
+    let covmap_var_name = llvm::build_string(|s| unsafe {
+        llvm::LLVMRustCoverageWriteMappingVarNameToString(s);
+    })
+    .expect("Rust Coverage Mapping var name failed UTF-8 conversion");
+    debug!("covmap var name: {:?}", covmap_var_name);
+
+    let covmap_section_name = llvm::build_string(|s| unsafe {
+        llvm::LLVMRustCoverageWriteSectionNameToString(cx.llmod, s);
+    })
+    .expect("Rust Coverage section name failed UTF-8 conversion");
+    debug!("covmap section name: {:?}", covmap_section_name);
+
+    let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name);
+    llvm::set_initializer(llglobal, cov_data_val);
+    llvm::set_global_constant(llglobal, true);
+    llvm::set_linkage(llglobal, llvm::Linkage::InternalLinkage);
+    llvm::set_section(llglobal, &covmap_section_name);
+    llvm::set_alignment(llglobal, COVMAP_VAR_ALIGN_BYTES);
+    cx.add_used_global(llglobal);
+}

src/librustc_codegen_llvm/intrinsic.rs

@@ -90,45 +90,64 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
         args: &Vec<Operand<'tcx>>,
         caller_instance: ty::Instance<'tcx>,
     ) -> bool {
-        match intrinsic {
-            sym::count_code_region => {
-                use coverage::count_code_region_args::*;
-                self.add_counter_region(
-                    caller_instance,
-                    op_to_u32(&args[COUNTER_INDEX]),
-                    op_to_u32(&args[START_BYTE_POS]),
-                    op_to_u32(&args[END_BYTE_POS]),
-                );
-                true // Also inject the counter increment in the backend
+        if self.tcx.sess.opts.debugging_opts.instrument_coverage {
+            // Add the coverage information from the MIR to the Codegen context. Some coverage
+            // intrinsics are used only to pass along the coverage information (returns `false`
+            // for `is_codegen_intrinsic()`), but `count_code_region` is also converted into an
+            // LLVM intrinsic to increment a coverage counter.
+            match intrinsic {
+                sym::count_code_region => {
+                    use coverage::count_code_region_args::*;
+                    self.add_counter_region(
+                        caller_instance,
+                        op_to_u64(&args[FUNCTION_SOURCE_HASH]),
+                        op_to_u32(&args[COUNTER_INDEX]),
+                        op_to_u32(&args[START_BYTE_POS]),
+                        op_to_u32(&args[END_BYTE_POS]),
+                    );
+                    return true; // Also inject the counter increment in the backend
+                }
+                sym::coverage_counter_add | sym::coverage_counter_subtract => {
+                    use coverage::coverage_counter_expression_args::*;
+                    self.add_counter_expression_region(
+                        caller_instance,
+                        op_to_u32(&args[COUNTER_EXPRESSION_INDEX]),
+                        op_to_u32(&args[LEFT_INDEX]),
+                        if intrinsic == sym::coverage_counter_add {
+                            CounterOp::Add
+                        } else {
+                            CounterOp::Subtract
+                        },
+                        op_to_u32(&args[RIGHT_INDEX]),
+                        op_to_u32(&args[START_BYTE_POS]),
+                        op_to_u32(&args[END_BYTE_POS]),
+                    );
+                    return false; // Does not inject backend code
+                }
+                sym::coverage_unreachable => {
+                    use coverage::coverage_unreachable_args::*;
+                    self.add_unreachable_region(
+                        caller_instance,
+                        op_to_u32(&args[START_BYTE_POS]),
+                        op_to_u32(&args[END_BYTE_POS]),
+                    );
+                    return false; // Does not inject backend code
+                }
+                _ => {}
             }
-            sym::coverage_counter_add | sym::coverage_counter_subtract => {
-                use coverage::coverage_counter_expression_args::*;
-                self.add_counter_expression_region(
-                    caller_instance,
-                    op_to_u32(&args[COUNTER_EXPRESSION_INDEX]),
-                    op_to_u32(&args[LEFT_INDEX]),
-                    if intrinsic == sym::coverage_counter_add {
-                        CounterOp::Add
-                    } else {
-                        CounterOp::Subtract
-                    },
-                    op_to_u32(&args[RIGHT_INDEX]),
-                    op_to_u32(&args[START_BYTE_POS]),
-                    op_to_u32(&args[END_BYTE_POS]),
-                );
-                false // Does not inject backend code
+        } else {
+            // NOT self.tcx.sess.opts.debugging_opts.instrument_coverage
+            if intrinsic == sym::count_code_region {
+                // An external crate may have been pre-compiled with coverage instrumentation, and
+                // some references from the current crate to the external crate might carry along
+                // the call terminators to coverage intrinsics, like `count_code_region` (for
+                // example, when instantiating a generic function). If the current crate has
+                // `instrument_coverage` disabled, the `count_code_region` call terminators should
+                // be ignored.
+                return false; // Do not inject coverage counters inlined from external crates
             }
-            sym::coverage_unreachable => {
-                use coverage::coverage_unreachable_args::*;
-                self.add_unreachable_region(
-                    caller_instance,
-                    op_to_u32(&args[START_BYTE_POS]),
-                    op_to_u32(&args[END_BYTE_POS]),
-                );
-                false // Does not inject backend code
-            }
-            _ => true, // Unhandled intrinsics should be passed to `codegen_intrinsic_call()`
         }
+        true // Unhandled intrinsics should be passed to `codegen_intrinsic_call()`
     }
 
     fn codegen_intrinsic_call(
@@ -197,12 +216,13 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                 let coverageinfo = tcx.coverageinfo(caller_instance.def_id());
                 let mangled_fn = tcx.symbol_name(caller_instance);
                 let (mangled_fn_name, _len_val) = self.const_str(Symbol::intern(mangled_fn.name));
-                let hash = self.const_u64(coverageinfo.hash);
                 let num_counters = self.const_u32(coverageinfo.num_counters);
                 use coverage::count_code_region_args::*;
+                let hash = args[FUNCTION_SOURCE_HASH].immediate();
                 let index = args[COUNTER_INDEX].immediate();
                 debug!(
-                    "count_code_region to LLVM intrinsic instrprof.increment(fn_name={}, hash={:?}, num_counters={:?}, index={:?})",
+                    "translating Rust intrinsic `count_code_region()` to LLVM intrinsic: \
+                    instrprof.increment(fn_name={}, hash={:?}, num_counters={:?}, index={:?})",
                     mangled_fn.name, hash, num_counters, index,
                 );
                 self.instrprof_increment(mangled_fn_name, hash, num_counters, index)
@@ -2222,3 +2242,7 @@ fn float_type_width(ty: Ty<'_>) -> Option<u64> {
 fn op_to_u32<'tcx>(op: &Operand<'tcx>) -> u32 {
     Operand::scalar_from_const(op).to_u32().expect("Scalar is u32")
 }
+
+fn op_to_u64<'tcx>(op: &Operand<'tcx>) -> u64 {
+    Operand::scalar_from_const(op).to_u64().expect("Scalar is u64")
+}

src/librustc_codegen_llvm/llvm/ffi.rs

@@ -1,6 +1,8 @@
 #![allow(non_camel_case_types)]
 #![allow(non_upper_case_globals)]
 
+use super::coverageinfo::{SmallVectorCounterExpression, SmallVectorCounterMappingRegion};
+
 use super::debuginfo::{
     DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator,
     DIFile, DIFlags, DIGlobalVariableExpression, DILexicalBlock, DINameSpace, DISPFlags, DIScope,
@@ -650,6 +652,16 @@ pub struct Linker<'a>(InvariantOpaque<'a>);
 pub type DiagnosticHandler = unsafe extern "C" fn(&DiagnosticInfo, *mut c_void);
 pub type InlineAsmDiagHandler = unsafe extern "C" fn(&SMDiagnostic, *const c_void, c_uint);
 
+pub mod coverageinfo {
+    use super::InvariantOpaque;
+
+    #[repr(C)]
+    pub struct SmallVectorCounterExpression<'a>(InvariantOpaque<'a>);
+
+    #[repr(C)]
+    pub struct SmallVectorCounterMappingRegion<'a>(InvariantOpaque<'a>);
+}
+
 pub mod debuginfo {
     use super::{InvariantOpaque, Metadata};
     use bitflags::bitflags;
@@ -1365,7 +1377,7 @@ extern "C" {
 
     // Miscellaneous instructions
     pub fn LLVMBuildPhi(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value;
-    pub fn LLVMRustGetInstrprofIncrementIntrinsic(M: &Module) -> &'a Value;
+    pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &'a Value;
     pub fn LLVMRustBuildCall(
         B: &Builder<'a>,
         Fn: &'a Value,
@@ -1633,6 +1645,58 @@ extern "C" {
         ConstraintsLen: size_t,
     ) -> bool;
 
+    pub fn LLVMRustCoverageSmallVectorCounterExpressionCreate()
+    -> &'a mut SmallVectorCounterExpression<'a>;
+    pub fn LLVMRustCoverageSmallVectorCounterExpressionDispose(
+        Container: &'a mut SmallVectorCounterExpression<'a>,
+    );
+    pub fn LLVMRustCoverageSmallVectorCounterExpressionAdd(
+        Container: &mut SmallVectorCounterExpression<'a>,
+        Kind: rustc_codegen_ssa::coverageinfo::CounterOp,
+        LeftIndex: c_uint,
+        RightIndex: c_uint,
+    );
+
+    pub fn LLVMRustCoverageSmallVectorCounterMappingRegionCreate()
+    -> &'a mut SmallVectorCounterMappingRegion<'a>;
+    pub fn LLVMRustCoverageSmallVectorCounterMappingRegionDispose(
+        Container: &'a mut SmallVectorCounterMappingRegion<'a>,
+    );
+    pub fn LLVMRustCoverageSmallVectorCounterMappingRegionAdd(
+        Container: &mut SmallVectorCounterMappingRegion<'a>,
+        Index: c_uint,
+        FileID: c_uint,
+        LineStart: c_uint,
+        ColumnStart: c_uint,
+        LineEnd: c_uint,
+        ColumnEnd: c_uint,
+    );
+
+    #[allow(improper_ctypes)]
+    pub fn LLVMRustCoverageWriteFilenamesSectionToBuffer(
+        Filenames: *const *const c_char,
+        FilenamesLen: size_t,
+        BufferOut: &RustString,
+    );
+
+    #[allow(improper_ctypes)]
+    pub fn LLVMRustCoverageWriteMappingToBuffer(
+        VirtualFileMappingIDs: *const c_uint,
+        NumVirtualFileMappingIDs: c_uint,
+        Expressions: *const SmallVectorCounterExpression<'_>,
+        MappingRegions: *const SmallVectorCounterMappingRegion<'_>,
+        BufferOut: &RustString,
+    );
+
+    pub fn LLVMRustCoverageComputeHash(Name: *const c_char) -> u64;
+
+    #[allow(improper_ctypes)]
+    pub fn LLVMRustCoverageWriteSectionNameToString(M: &Module, Str: &RustString);
+
+    #[allow(improper_ctypes)]
+    pub fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString);
+
+    pub fn LLVMRustCoverageMappingVersion() -> u32;
     pub fn LLVMRustDebugMetadataVersion() -> u32;
     pub fn LLVMRustVersionMajor() -> u32;
     pub fn LLVMRustVersionMinor() -> u32;

src/librustc_codegen_llvm/llvm/mod.rs

@@ -12,7 +12,7 @@ use libc::c_uint;
 use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_llvm::RustString;
 use std::cell::RefCell;
-use std::ffi::CStr;
+use std::ffi::{CStr, CString};
 use std::str::FromStr;
 use std::string::FromUtf8Error;
 
@@ -189,6 +189,42 @@ pub fn mk_section_iter(llof: &ffi::ObjectFile) -> SectionIter<'_> {
     unsafe { SectionIter { llsi: LLVMGetSections(llof) } }
 }
 
+pub fn set_section(llglobal: &Value, section_name: &str) {
+    let section_name_cstr = CString::new(section_name).expect("unexpected CString error");
+    unsafe {
+        LLVMSetSection(llglobal, section_name_cstr.as_ptr());
+    }
+}
+
+pub fn add_global<'a>(llmod: &'a Module, ty: &'a Type, name: &str) -> &'a Value {
+    let name_cstr = CString::new(name).expect("unexpected CString error");
+    unsafe { LLVMAddGlobal(llmod, ty, name_cstr.as_ptr()) }
+}
+
+pub fn set_initializer(llglobal: &Value, constant_val: &Value) {
+    unsafe {
+        LLVMSetInitializer(llglobal, constant_val);
+    }
+}
+
+pub fn set_global_constant(llglobal: &Value, is_constant: bool) {
+    unsafe {
+        LLVMSetGlobalConstant(llglobal, if is_constant { ffi::True } else { ffi::False });
+    }
+}
+
+pub fn set_linkage(llglobal: &Value, linkage: Linkage) {
+    unsafe {
+        LLVMRustSetLinkage(llglobal, linkage);
+    }
+}
+
+pub fn set_alignment(llglobal: &Value, bytes: usize) {
+    unsafe {
+        ffi::LLVMSetAlignment(llglobal, bytes as c_uint);
+    }
+}
+
 /// Safe wrapper around `LLVMGetParam`, because segfaults are no fun.
 pub fn get_param(llfn: &Value, index: c_uint) -> &Value {
     unsafe {
@@ -225,6 +261,12 @@ pub fn build_string(f: impl FnOnce(&RustString)) -> Result<String, FromUtf8Error
     String::from_utf8(sr.bytes.into_inner())
 }
 
+pub fn build_byte_buffer(f: impl FnOnce(&RustString)) -> Vec<u8> {
+    let sr = RustString { bytes: RefCell::new(Vec::new()) };
+    f(&sr);
+    sr.bytes.into_inner()
+}
+
 pub fn twine_to_string(tr: &Twine) -> String {
     unsafe {
         build_string(|s| LLVMRustWriteTwineToString(tr, s)).expect("got a non-UTF8 Twine from LLVM")

src/librustc_codegen_ssa/back/link.rs

@@ -1659,7 +1659,7 @@ fn linker_with_args<'a, B: ArchiveBuilder<'a>>(
     // FIXME: Order dependent, applies to the following objects. Where should it be placed?
     // Try to strip as much out of the generated object by removing unused
     // sections if possible. See more comments in linker.rs
-    if !sess.opts.cg.link_dead_code {
+    if sess.opts.cg.link_dead_code != Some(true) {
         let keep_metadata = crate_type == CrateType::Dylib;
         cmd.gc_sections(keep_metadata);
     }
@@ -1695,7 +1695,7 @@ fn linker_with_args<'a, B: ArchiveBuilder<'a>>(
     );
 
     // OBJECT-FILES-NO, AUDIT-ORDER
-    if sess.opts.cg.profile_generate.enabled() {
+    if sess.opts.cg.profile_generate.enabled() || sess.opts.debugging_opts.instrument_coverage {
         cmd.pgo_gen();
     }
 

src/librustc_codegen_ssa/back/symbol_export.rs

@@ -203,6 +203,17 @@ fn exported_symbols_provider_local(
         }));
     }
 
+    if tcx.sess.opts.debugging_opts.instrument_coverage {
+        // Similar to PGO profiling, preserve symbols used by LLVM InstrProf coverage profiling.
+        const COVERAGE_WEAK_SYMBOLS: [&str; 3] =
+            ["__llvm_profile_filename", "__llvm_coverage_mapping", "__llvm_covmap"];
+
+        symbols.extend(COVERAGE_WEAK_SYMBOLS.iter().map(|sym| {
+            let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, sym));
+            (exported_symbol, SymbolExportLevel::C)
+        }));
+    }
+
     if tcx.sess.opts.debugging_opts.sanitizer.contains(SanitizerSet::MEMORY) {
         // Similar to profiling, preserve weak msan symbol during LTO.
         const MSAN_WEAK_SYMBOLS: [&str; 2] = ["__msan_track_origins", "__msan_keep_going"];

src/librustc_codegen_ssa/coverageinfo/map.rs

@@ -1,32 +1,154 @@
-use rustc_data_structures::fx::FxHashMap;
-use std::collections::hash_map;
-use std::slice;
+use rustc_data_structures::sync::Lrc;
+use rustc_middle::mir;
+use rustc_span::source_map::{Pos, SourceFile, SourceMap};
+use rustc_span::{BytePos, FileName, RealFileName};
+
+use std::cmp::{Ord, Ordering};
+use std::collections::BTreeMap;
+use std::fmt;
+use std::path::PathBuf;
 
 #[derive(Copy, Clone, Debug)]
+#[repr(C)]
 pub enum CounterOp {
-    Add,
+    // Note the order (and therefore the default values) is important. With the attribute
+    // `#[repr(C)]`, this enum matches the layout of the LLVM enum defined for the nested enum,
+    // `llvm::coverage::CounterExpression::ExprKind`, as shown in the following source snippet:
+    // https://github.com/rust-lang/llvm-project/blob/f208b70fbc4dee78067b3c5bd6cb92aa3ba58a1e/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L146
     Subtract,
+    Add,
 }
 
+#[derive(Copy, Clone, Debug)]
 pub enum CoverageKind {
     Counter,
     CounterExpression(u32, CounterOp, u32),
+    Unreachable,
 }
 
-pub struct CoverageSpan {
+#[derive(Clone, Debug)]
+pub struct CoverageRegion {
+    pub kind: CoverageKind,
     pub start_byte_pos: u32,
     pub end_byte_pos: u32,
 }
 
-pub struct CoverageRegion {
-    pub kind: CoverageKind,
-    pub coverage_span: CoverageSpan,
+impl CoverageRegion {
+    pub fn source_loc(&self, source_map: &SourceMap) -> Option<(Lrc<SourceFile>, CoverageLoc)> {
+        let (start_file, start_line, start_col) =
+            lookup_file_line_col(source_map, BytePos::from_u32(self.start_byte_pos));
+        let (end_file, end_line, end_col) =
+            lookup_file_line_col(source_map, BytePos::from_u32(self.end_byte_pos));
+        let start_file_path = match &start_file.name {
+            FileName::Real(RealFileName::Named(path)) => path,
+            _ => {
+                bug!("start_file_path should be a RealFileName, but it was: {:?}", start_file.name)
+            }
+        };
+        let end_file_path = match &end_file.name {
+            FileName::Real(RealFileName::Named(path)) => path,
+            _ => bug!("end_file_path should be a RealFileName, but it was: {:?}", end_file.name),
+        };
+        if start_file_path == end_file_path {
+            Some((start_file, CoverageLoc { start_line, start_col, end_line, end_col }))
+        } else {
+            None
+            // FIXME(richkadel): There seems to be a problem computing the file location in
+            // some cases. I need to investigate this more. When I generate and show coverage
+            // for the example binary in the crates.io crate `json5format`, I had a couple of
+            // notable problems:
+            //
+            //   1. I saw a lot of coverage spans in `llvm-cov show` highlighting regions in
+            //      various comments (not corresponding to rustdoc code), indicating a possible
+            //      problem with the byte_pos-to-source-map implementation.
+            //
+            //   2. And (perhaps not related) when I build the aforementioned example binary with:
+            //      `RUST_FLAGS="-Zinstrument-coverage" cargo build --example formatjson5`
+            //      and then run that binary with
+            //      `LLVM_PROFILE_FILE="formatjson5.profraw" ./target/debug/examples/formatjson5 \
+            //      some.json5` for some reason the binary generates *TWO* `.profraw` files. One
+            //      named `default.profraw` and the other named `formatjson5.profraw` (the expected
+            //      name, in this case).
+            //
+            // If the byte range conversion is wrong, fix it. But if it
+            // is right, then it is possible for the start and end to be in different files.
+            // Can I do something other than ignore coverages that span multiple files?
+            //
+            // If I can resolve this, remove the "Option<>" result type wrapper
+            // `regions_in_file_order()` accordingly.
+        }
+    }
+}
+
+impl Default for CoverageRegion {
+    fn default() -> Self {
+        Self {
+            // The default kind (Unreachable) is a placeholder that will be overwritten before
+            // backend codegen.
+            kind: CoverageKind::Unreachable,
+            start_byte_pos: 0,
+            end_byte_pos: 0,
+        }
+    }
+}
+
+/// A source code region used with coverage information.
+#[derive(Debug, Eq, PartialEq)]
+pub struct CoverageLoc {
+    /// The (1-based) line number of the region start.
+    pub start_line: u32,
+    /// The (1-based) column number of the region start.
+    pub start_col: u32,
+    /// The (1-based) line number of the region end.
+    pub end_line: u32,
+    /// The (1-based) column number of the region end.
+    pub end_col: u32,
+}
+
+impl Ord for CoverageLoc {
+    fn cmp(&self, other: &Self) -> Ordering {
+        (self.start_line, &self.start_col, &self.end_line, &self.end_col).cmp(&(
+            other.start_line,
+            &other.start_col,
+            &other.end_line,
+            &other.end_col,
+        ))
+    }
+}
+
+impl PartialOrd for CoverageLoc {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl fmt::Display for CoverageLoc {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Customize debug format, and repeat the file name, so generated location strings are
+        // "clickable" in many IDEs.
+        write!(f, "{}:{} - {}:{}", self.start_line, self.start_col, self.end_line, self.end_col)
+    }
+}
+
+fn lookup_file_line_col(source_map: &SourceMap, byte_pos: BytePos) -> (Lrc<SourceFile>, u32, u32) {
+    let found = source_map
+        .lookup_line(byte_pos)
+        .expect("should find coverage region byte position in source");
+    let file = found.sf;
+    let line_pos = file.line_begin_pos(byte_pos);
+
+    // Use 1-based indexing.
+    let line = (found.line + 1) as u32;
+    let col = (byte_pos - line_pos).to_u32() + 1;
+
+    (file, line, col)
 }
 
 /// Collects all of the coverage regions associated with (a) injected counters, (b) counter
 /// expressions (additions or subtraction), and (c) unreachable regions (always counted as zero),
 /// for a given Function. Counters and counter expressions are indexed because they can be operands
-/// in an expression.
+/// in an expression. This struct also stores the `function_source_hash`, computed during
+/// instrumentation and forwarded with counters.
 ///
 /// Note, it's important to distinguish the `unreachable` region type from what LLVM's refers to as
 /// a "gap region" (or "gap area"). A gap region is a code region within a counted region (either
@@ -34,50 +156,134 @@ pub struct CoverageRegion {
 /// lines with only whitespace or comments). According to LLVM Code Coverage Mapping documentation,
 /// "A count for a gap area is only used as the line execution count if there are no other regions
 /// on a line."
-#[derive(Default)]
-pub struct FunctionCoverageRegions {
-    indexed: FxHashMap<u32, CoverageRegion>,
-    unreachable: Vec<CoverageSpan>,
+pub struct FunctionCoverage {
+    source_hash: u64,
+    counters: Vec<CoverageRegion>,
+    expressions: Vec<CoverageRegion>,
+    unreachable: Vec<CoverageRegion>,
+    translated: bool,
 }
 
-impl FunctionCoverageRegions {
-    pub fn add_counter(&mut self, index: u32, start_byte_pos: u32, end_byte_pos: u32) {
-        self.indexed.insert(
-            index,
-            CoverageRegion {
-                kind: CoverageKind::Counter,
-                coverage_span: CoverageSpan { start_byte_pos, end_byte_pos },
-            },
-        );
+impl FunctionCoverage {
+    pub fn with_coverageinfo<'tcx>(coverageinfo: &'tcx mir::CoverageInfo) -> Self {
+        Self {
+            source_hash: 0, // will be set with the first `add_counter()`
+            counters: vec![CoverageRegion::default(); coverageinfo.num_counters as usize],
+            expressions: vec![CoverageRegion::default(); coverageinfo.num_expressions as usize],
+            unreachable: Vec::new(),
+            translated: false,
+        }
+    }
+
+    /// Adds a code region to be counted by an injected counter intrinsic. Return a counter ID
+    /// for the call.
+    pub fn add_counter(
+        &mut self,
+        source_hash: u64,
+        index: u32,
+        start_byte_pos: u32,
+        end_byte_pos: u32,
+    ) {
+        self.source_hash = source_hash;
+        self.counters[index as usize] =
+            CoverageRegion { kind: CoverageKind::Counter, start_byte_pos, end_byte_pos };
     }
 
     pub fn add_counter_expression(
         &mut self,
-        index: u32,
+        translated_index: u32,
         lhs: u32,
         op: CounterOp,
         rhs: u32,
         start_byte_pos: u32,
         end_byte_pos: u32,
     ) {
-        self.indexed.insert(
-            index,
-            CoverageRegion {
-                kind: CoverageKind::CounterExpression(lhs, op, rhs),
-                coverage_span: CoverageSpan { start_byte_pos, end_byte_pos },
-            },
-        );
+        let index = u32::MAX - translated_index;
+        // Counter expressions start with "translated indexes", descending from `u32::MAX`, so
+        // the range of expression indexes is disjoint from the range of counter indexes. This way,
+        // both counters and expressions can be operands in other expressions.
+        //
+        // Once all counters have been added, the final "region index" for an expression is
+        // `counters.len() + expression_index` (where `expression_index` is its index in
+        // `self.expressions`), and the expression operands (`lhs` and `rhs`) can be converted to
+        // final "region index" references by the same conversion, after subtracting from
+        // `u32::MAX`.
+        self.expressions[index as usize] = CoverageRegion {
+            kind: CoverageKind::CounterExpression(lhs, op, rhs),
+            start_byte_pos,
+            end_byte_pos,
+        };
     }
 
     pub fn add_unreachable(&mut self, start_byte_pos: u32, end_byte_pos: u32) {
-        self.unreachable.push(CoverageSpan { start_byte_pos, end_byte_pos });
+        self.unreachable.push(CoverageRegion {
+            kind: CoverageKind::Unreachable,
+            start_byte_pos,
+            end_byte_pos,
+        });
     }
 
-    pub fn indexed_regions(&self) -> hash_map::Iter<'_, u32, CoverageRegion> {
-        self.indexed.iter()
+    pub fn source_hash(&self) -> u64 {
+        self.source_hash
     }
 
-    pub fn unreachable_regions(&self) -> slice::Iter<'_, CoverageSpan> {
-        self.unreachable.iter()
+    fn regions(&'a mut self) -> impl Iterator<Item = &'a CoverageRegion> {
+        assert!(self.source_hash != 0);
+        self.ensure_expressions_translated();
+        self.counters.iter().chain(self.expressions.iter().chain(self.unreachable.iter()))
+    }
+
+    pub fn regions_in_file_order(
+        &'a mut self,
+        source_map: &SourceMap,
+    ) -> BTreeMap<PathBuf, BTreeMap<CoverageLoc, (usize, CoverageKind)>> {
+        let mut regions_in_file_order = BTreeMap::new();
+        for (region_id, region) in self.regions().enumerate() {
+            if let Some((source_file, region_loc)) = region.source_loc(source_map) {
+                // FIXME(richkadel): `region.source_loc()` sometimes fails with two different
+                // filenames for the start and end byte position. This seems wrong, but for
+                // now, if encountered, the region is skipped. If resolved, convert the result
+                // to a non-option value so regions are never skipped.
+                let real_file_path = match &(*source_file).name {
+                    FileName::Real(RealFileName::Named(path)) => path.clone(),
+                    _ => bug!("coverage mapping expected only real, named files"),
+                };
+                let file_coverage_regions =
+                    regions_in_file_order.entry(real_file_path).or_insert_with(|| BTreeMap::new());
+                file_coverage_regions.insert(region_loc, (region_id, region.kind));
+            }
+        }
+        regions_in_file_order
+    }
+
+    /// A one-time translation of expression operands is needed, for any operands referencing
+    /// other CounterExpressions. CounterExpression operands get an initial operand ID that is
+    /// computed by the simple translation: `u32::max - expression_index` because, when created,
+    /// the total number of Counters is not yet known. This function recomputes region indexes
+    /// for expressions so they start with the next region index after the last counter index.
+    fn ensure_expressions_translated(&mut self) {
+        if !self.translated {
+            self.translated = true;
+            let start = self.counters.len() as u32;
+            assert!(
+                (start as u64 + self.expressions.len() as u64) < u32::MAX as u64,
+                "the number of counters and counter expressions in a single function exceeds {}",
+                u32::MAX
+            );
+            for region in self.expressions.iter_mut() {
+                match region.kind {
+                    CoverageKind::CounterExpression(lhs, op, rhs) => {
+                        let lhs = to_region_index(start, lhs);
+                        let rhs = to_region_index(start, rhs);
+                        region.kind = CoverageKind::CounterExpression(lhs, op, rhs);
+                    }
+                    _ => bug!("expressions must only contain CounterExpression kinds"),
+                }
+            }
+        }
     }
 }
+
+fn to_region_index(start: u32, index: u32) -> u32 {
+    if index < start { index } else { start + (u32::MAX - index) }
+}

src/librustc_codegen_ssa/traits/coverageinfo.rs

@@ -10,6 +10,7 @@ pub trait CoverageInfoBuilderMethods<'tcx>: BackendTypes {
     fn add_counter_region(
         &mut self,
         instance: Instance<'tcx>,
+        function_source_hash: u64,
         index: u32,
         start_byte_pos: u32,
         end_byte_pos: u32,

src/librustc_codegen_ssa/traits/statics.rs

@@ -5,6 +5,18 @@ use rustc_target::abi::Align;
 pub trait StaticMethods: BackendTypes {
     fn static_addr_of(&self, cv: Self::Value, align: Align, kind: Option<&str>) -> Self::Value;
     fn codegen_static(&self, def_id: DefId, is_mutable: bool);
+
+    /// Mark the given global value as "used", to prevent a backend from potentially removing a
+    /// static variable that may otherwise appear unused.
+    ///
+    /// Static variables in Rust can be annotated with the `#[used]` attribute to direct the `rustc`
+    /// compiler to mark the variable as a "used global".
+    ///
+    /// ```no_run
+    /// #[used]
+    /// static FOO: u32 = 0;
+    /// ```
+    fn add_used_global(&self, global: Self::Value);
 }
 
 pub trait StaticBuilderMethods: BackendTypes {

src/librustc_hir/fake_lang_items.rs

@@ -0,0 +1,37 @@
+//! Validity checking for fake lang items
+
+use crate::def_id::DefId;
+use crate::{lang_items, LangItem, LanguageItems};
+
+use rustc_data_structures::fx::FxHashMap;
+use rustc_span::symbol::{sym, Symbol};
+
+use lazy_static::lazy_static;
+
+macro_rules! fake_lang_items {
+    ($($item:ident, $name:ident, $method:ident;)*) => (
+
+lazy_static! {
+    pub static ref FAKE_ITEMS_REFS: FxHashMap<Symbol, LangItem> = {
+        let mut map = FxHashMap::default();
+        $(map.insert(sym::$name, lang_items::$item);)*
+        map
+    };
+}
+
+impl LanguageItems {
+    pub fn is_fake_lang_item(&self, item_def_id: DefId) -> bool {
+        let did = Some(item_def_id);
+
+        $(self.$method() == did)||*
+    }
+}
+
+) }
+
+fake_lang_items! {
+//  Variant name,                      Symbol,                    Method name,
+    CountCodeRegionFnLangItem,         count_code_region,         count_code_region_fn;
+    CoverageCounterAddFnLangItem,      coverage_counter_add,      coverage_counter_add_fn;
+    CoverageCounterSubtractFnLangItem, coverage_counter_subtract, coverage_counter_subtract_fn;
+}

src/librustc_hir/lang_items.rs

@@ -276,8 +276,6 @@ language_item_table! {
 
     StartFnLangItem,               sym::start,              start_fn,                Target::Fn;
 
-    CountCodeRegionFnLangItem,     sym::count_code_region,  count_code_region_fn,    Target::Fn;
-
     EhPersonalityLangItem,         sym::eh_personality,     eh_personality,          Target::Fn;
     EhCatchTypeinfoLangItem,       sym::eh_catch_typeinfo,  eh_catch_typeinfo,       Target::Static;
 
@@ -295,4 +293,9 @@ language_item_table! {
     TerminationTraitLangItem,      sym::termination,        termination,             Target::Trait;
 
     TryTraitLangItem,              kw::Try,                 try_trait,               Target::Trait;
+
+    // language items related to source code coverage instrumentation (-Zinstrument-coverage)
+    CountCodeRegionFnLangItem,         sym::count_code_region,         count_code_region_fn,         Target::Fn;
+    CoverageCounterAddFnLangItem,      sym::coverage_counter_add,      coverage_counter_add_fn,      Target::Fn;
+    CoverageCounterSubtractFnLangItem, sym::coverage_counter_subtract, coverage_counter_subtract_fn, Target::Fn;
 }

src/librustc_hir/lib.rs

@@ -17,6 +17,7 @@ mod arena;
 pub mod def;
 pub mod definitions;
 pub use rustc_span::def_id;
+pub mod fake_lang_items;
 mod hir;
 pub mod hir_id;
 pub mod intravisit;

src/librustc_interface/tests.rs

@@ -401,7 +401,7 @@ fn test_codegen_options_tracking_hash() {
     untracked!(incremental, Some(String::from("abc")));
     // `link_arg` is omitted because it just forwards to `link_args`.
     untracked!(link_args, vec![String::from("abc"), String::from("def")]);
-    untracked!(link_dead_code, true);
+    untracked!(link_dead_code, Some(true));
     untracked!(linker, Some(PathBuf::from("linker")));
     untracked!(linker_flavor, Some(LinkerFlavor::Gcc));
     untracked!(no_stack_check, true);

src/librustc_llvm/build.rs

@@ -104,8 +104,16 @@ fn main() {
         optional_components.push("riscv");
     }
 
-    let required_components =
-        &["ipo", "bitreader", "bitwriter", "linker", "asmparser", "lto", "instrumentation"];
+    let required_components = &[
+        "ipo",
+        "bitreader",
+        "bitwriter",
+        "linker",
+        "asmparser",
+        "lto",
+        "coverage",
+        "instrumentation",
+    ];
 
     let components = output(Command::new(&llvm_config).arg("--components"));
     let mut components = components.split_whitespace().collect::<Vec<_>>();
@@ -169,6 +177,7 @@ fn main() {
     cfg.file("../rustllvm/PassWrapper.cpp")
         .file("../rustllvm/RustWrapper.cpp")
         .file("../rustllvm/ArchiveWrapper.cpp")
+        .file("../rustllvm/CoverageMappingWrapper.cpp")
         .file("../rustllvm/Linker.cpp")
         .cpp(true)
         .cpp_link_stdlib(None) // we handle this below

src/librustc_llvm/lib.rs

@@ -13,6 +13,12 @@ pub struct RustString {
     pub bytes: RefCell<Vec<u8>>,
 }
 
+impl RustString {
+    pub fn len(&self) -> usize {
+        self.bytes.borrow().len()
+    }
+}
+
 /// Appending to a Rust string -- used by RawRustStringOstream.
 #[no_mangle]
 #[allow(improper_ctypes_definitions)]

src/librustc_middle/mir/coverage/mod.rs

@@ -2,9 +2,10 @@
 
 /// Positional arguments to `libcore::count_code_region()`
 pub mod count_code_region_args {
-    pub const COUNTER_INDEX: usize = 0;
-    pub const START_BYTE_POS: usize = 1;
-    pub const END_BYTE_POS: usize = 2;
+    pub const FUNCTION_SOURCE_HASH: usize = 0;
+    pub const COUNTER_INDEX: usize = 1;
+    pub const START_BYTE_POS: usize = 2;
+    pub const END_BYTE_POS: usize = 3;
 }
 
 /// Positional arguments to `libcore::coverage_counter_add()` and

src/librustc_middle/mir/mono.rs

@@ -86,7 +86,7 @@ impl<'tcx> MonoItem<'tcx> {
             .debugging_opts
             .inline_in_all_cgus
             .unwrap_or_else(|| tcx.sess.opts.optimize != OptLevel::No)
-            && !tcx.sess.opts.cg.link_dead_code;
+            && tcx.sess.opts.cg.link_dead_code != Some(true);
 
         match *self {
             MonoItem::Fn(ref instance) => {

src/librustc_middle/mir/query.rs

@@ -400,13 +400,11 @@ pub struct DestructuredConst<'tcx> {
 /// `InstrumentCoverage` MIR pass and can be retrieved via the `coverageinfo` query.
 #[derive(Clone, RustcEncodable, RustcDecodable, Debug, HashStable)]
 pub struct CoverageInfo {
-    /// A hash value that can be used by the consumer of the coverage profile data to detect
-    /// changes to the instrumented source of the associated MIR body (typically, for an
-    /// individual function).
-    pub hash: u64,
-
     /// The total number of coverage region counters added to the MIR `Body`.
     pub num_counters: u32,
+
+    /// The total number of coverage region counter expressions added to the MIR `Body`.
+    pub num_expressions: u32,
 }
 
 impl<'tcx> TyCtxt<'tcx> {

src/librustc_mir/monomorphize/partitioning.rs

@@ -161,7 +161,7 @@ where
 
     // Next we try to make as many symbols "internal" as possible, so LLVM has
     // more freedom to optimize.
-    if !tcx.sess.opts.cg.link_dead_code {
+    if tcx.sess.opts.cg.link_dead_code != Some(true) {
         let _prof_timer = tcx.prof.generic_activity("cgu_partitioning_internalize_symbols");
         internalize_symbols(tcx, &mut post_inlining, inlining_map);
     }
@@ -906,7 +906,7 @@ fn collect_and_partition_mono_items(
             }
         }
         None => {
-            if tcx.sess.opts.cg.link_dead_code {
+            if tcx.sess.opts.cg.link_dead_code == Some(true) {
                 MonoItemCollectionMode::Eager
             } else {
                 MonoItemCollectionMode::Lazy

src/librustc_mir/transform/instrument_coverage.rs

@@ -35,46 +35,64 @@ fn coverageinfo_from_mir<'tcx>(tcx: TyCtxt<'tcx>, mir_def_id: DefId) -> Coverage
     // represents a single function. Validate and/or correct if inlining (which should be disabled
     // if -Zinstrument-coverage is enabled) and/or monomorphization invalidates these assumptions.
     let count_code_region_fn = tcx.require_lang_item(lang_items::CountCodeRegionFnLangItem, None);
+    let coverage_counter_add_fn =
+        tcx.require_lang_item(lang_items::CoverageCounterAddFnLangItem, None);
+    let coverage_counter_subtract_fn =
+        tcx.require_lang_item(lang_items::CoverageCounterSubtractFnLangItem, None);
 
     // The `num_counters` argument to `llvm.instrprof.increment` is the number of injected
     // counters, with each counter having an index from `0..num_counters-1`. MIR optimization
     // may split and duplicate some BasicBlock sequences. Simply counting the calls may not
     // not work; but computing the num_counters by adding `1` to the highest index (for a given
     // instrumented function) is valid.
+    //
+    // `num_expressions` is the number of counter expressions added to the MIR body. Both
+    // `num_counters` and `num_expressions` are used to initialize new vectors, during backend
+    // code generate, to lookup counters and expressions by their simple u32 indexes.
     let mut num_counters: u32 = 0;
-    for terminator in traversal::preorder(mir_body)
-        .map(|(_, data)| (data, count_code_region_fn))
-        .filter_map(terminators_that_call_given_fn)
+    let mut num_expressions: u32 = 0;
+    for terminator in
+        traversal::preorder(mir_body).map(|(_, data)| data).filter_map(call_terminators)
     {
-        if let TerminatorKind::Call { args, .. } = &terminator.kind {
-            let index_arg = args.get(count_code_region_args::COUNTER_INDEX).expect("arg found");
-            let index =
-                mir::Operand::scalar_from_const(index_arg).to_u32().expect("index arg is u32");
-            num_counters = std::cmp::max(num_counters, index + 1);
-        }
-    }
-    let hash = if num_counters > 0 { hash_mir_source(tcx, mir_def_id) } else { 0 };
-    CoverageInfo { num_counters, hash }
-}
-
-fn terminators_that_call_given_fn(
-    (data, fn_def_id): (&'tcx BasicBlockData<'tcx>, DefId),
-) -> Option<&'tcx Terminator<'tcx>> {
-    if let Some(terminator) = &data.terminator {
-        if let TerminatorKind::Call { func: Operand::Constant(func), .. } = &terminator.kind {
-            if let FnDef(called_fn_def_id, _) = func.literal.ty.kind {
-                if called_fn_def_id == fn_def_id {
-                    return Some(&terminator);
+        if let TerminatorKind::Call { func: Operand::Constant(func), args, .. } = &terminator.kind {
+            match func.literal.ty.kind {
+                FnDef(id, _) if id == count_code_region_fn => {
+                    let index_arg =
+                        args.get(count_code_region_args::COUNTER_INDEX).expect("arg found");
+                    let counter_index = mir::Operand::scalar_from_const(index_arg)
+                        .to_u32()
+                        .expect("index arg is u32");
+                    num_counters = std::cmp::max(num_counters, counter_index + 1);
                 }
+                FnDef(id, _)
+                    if id == coverage_counter_add_fn || id == coverage_counter_subtract_fn =>
+                {
+                    let index_arg = args
+                        .get(coverage_counter_expression_args::COUNTER_EXPRESSION_INDEX)
+                        .expect("arg found");
+                    let translated_index = mir::Operand::scalar_from_const(index_arg)
+                        .to_u32()
+                        .expect("index arg is u32");
+                    // Counter expressions start with "translated indexes", descending from
+                    // `u32::MAX`, so the range of expression indexes is disjoint from the range of
+                    // counter indexes. This way, both counters and expressions can be operands in
+                    // other expressions.
+                    let expression_index = u32::MAX - translated_index;
+                    num_expressions = std::cmp::max(num_expressions, expression_index + 1);
+                }
+                _ => {}
             }
         }
     }
-    None
+    CoverageInfo { num_counters, num_expressions }
 }
 
-struct Instrumentor<'tcx> {
-    tcx: TyCtxt<'tcx>,
-    num_counters: u32,
+fn call_terminators(data: &'tcx BasicBlockData<'tcx>) -> Option<&'tcx Terminator<'tcx>> {
+    let terminator = data.terminator();
+    match terminator.kind {
+        TerminatorKind::Call { .. } => Some(terminator),
+        _ => None,
+    }
 }
 
 impl<'tcx> MirPass<'tcx> for InstrumentCoverage {
@@ -83,42 +101,106 @@ impl<'tcx> MirPass<'tcx> for InstrumentCoverage {
             // If the InstrumentCoverage pass is called on promoted MIRs, skip them.
             // See: https://github.com/rust-lang/rust/pull/73011#discussion_r438317601
             if src.promoted.is_none() {
-                debug!(
-                    "instrumenting {:?}, span: {}",
-                    src.def_id(),
-                    tcx.sess.source_map().span_to_string(mir_body.span)
-                );
-                Instrumentor::new(tcx).inject_counters(mir_body);
+                Instrumentor::new(tcx, src, mir_body).inject_counters();
             }
         }
     }
 }
 
-impl<'tcx> Instrumentor<'tcx> {
-    fn new(tcx: TyCtxt<'tcx>) -> Self {
-        Self { tcx, num_counters: 0 }
+/// Distinguishes the expression operators.
+enum Op {
+    Add,
+    Subtract,
+}
+
+struct Instrumentor<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    mir_def_id: DefId,
+    mir_body: &'a mut mir::Body<'tcx>,
+    hir_body: &'tcx rustc_hir::Body<'tcx>,
+    function_source_hash: Option<u64>,
+    num_counters: u32,
+    num_expressions: u32,
+}
+
+impl<'a, 'tcx> Instrumentor<'a, 'tcx> {
+    fn new(tcx: TyCtxt<'tcx>, src: MirSource<'tcx>, mir_body: &'a mut mir::Body<'tcx>) -> Self {
+        let mir_def_id = src.def_id();
+        let hir_body = hir_body(tcx, mir_def_id);
+        Self {
+            tcx,
+            mir_def_id,
+            mir_body,
+            hir_body,
+            function_source_hash: None,
+            num_counters: 0,
+            num_expressions: 0,
+        }
     }
 
+    /// Counter IDs start from zero and go up.
     fn next_counter(&mut self) -> u32 {
+        assert!(self.num_counters < u32::MAX - self.num_expressions);
         let next = self.num_counters;
         self.num_counters += 1;
         next
     }
 
-    fn inject_counters(&mut self, mir_body: &mut mir::Body<'tcx>) {
-        // FIXME(richkadel): As a first step, counters are only injected at the top of each
-        // function. The complete solution will inject counters at each conditional code branch.
-        let code_region = mir_body.span;
-        let next_block = START_BLOCK;
-        self.inject_counter(mir_body, code_region, next_block);
+    /// Expression IDs start from u32::MAX and go down because a CounterExpression can reference
+    /// (add or subtract counts) of both Counter regions and CounterExpression regions. The indexes
+    /// of each type of region must be contiguous, but also must be unique across both sets.
+    /// The expression IDs are eventually translated into region indexes (starting after the last
+    /// counter index, for the given function), during backend code generation, by the helper method
+    /// `rustc_codegen_ssa::coverageinfo::map::FunctionCoverage::translate_expressions()`.
+    fn next_expression(&mut self) -> u32 {
+        assert!(self.num_counters < u32::MAX - self.num_expressions);
+        let next = u32::MAX - self.num_expressions;
+        self.num_expressions += 1;
+        next
     }
 
-    fn inject_counter(
-        &mut self,
-        mir_body: &mut mir::Body<'tcx>,
-        code_region: Span,
-        next_block: BasicBlock,
-    ) {
+    fn function_source_hash(&mut self) -> u64 {
+        match self.function_source_hash {
+            Some(hash) => hash,
+            None => {
+                let hash = hash_mir_source(self.tcx, self.hir_body);
+                self.function_source_hash.replace(hash);
+                hash
+            }
+        }
+    }
+
+    fn inject_counters(&mut self) {
+        let body_span = self.hir_body.value.span;
+        debug!(
+            "instrumenting {:?}, span: {}",
+            self.mir_def_id,
+            self.tcx.sess.source_map().span_to_string(body_span)
+        );
+
+        // FIXME(richkadel): As a first step, counters are only injected at the top of each
+        // function. The complete solution will inject counters at each conditional code branch.
+        let next_block = START_BLOCK;
+        self.inject_counter(body_span, next_block);
+
+        // FIXME(richkadel): The next step to implement source based coverage analysis will be
+        // instrumenting branches within functions, and some regions will be counted by "counter
+        // expression". The function to inject counter expression is implemented. Replace this
+        // "fake use" with real use.
+        let fake_use = false;
+        if fake_use {
+            let add = false;
+            if add {
+                self.inject_counter_expression(body_span, next_block, 1, Op::Add, 2);
+            } else {
+                self.inject_counter_expression(body_span, next_block, 1, Op::Subtract, 2);
+            }
+        }
+    }
+
+    fn inject_counter(&mut self, code_region: Span, next_block: BasicBlock) -> u32 {
+        let counter_id = self.next_counter();
+        let function_source_hash = self.function_source_hash();
         let injection_point = code_region.shrink_to_lo();
 
         let count_code_region_fn = function_handle(
@@ -127,13 +209,14 @@ impl<'tcx> Instrumentor<'tcx> {
             injection_point,
         );
 
-        let index = self.next_counter();
-
         let mut args = Vec::new();
 
         use count_code_region_args::*;
+        debug_assert_eq!(FUNCTION_SOURCE_HASH, args.len());
+        args.push(self.const_u64(function_source_hash, injection_point));
+
         debug_assert_eq!(COUNTER_INDEX, args.len());
-        args.push(self.const_u32(index, injection_point));
+        args.push(self.const_u32(counter_id, injection_point));
 
         debug_assert_eq!(START_BYTE_POS, args.len());
         args.push(self.const_u32(code_region.lo().to_u32(), injection_point));
@@ -141,36 +224,98 @@ impl<'tcx> Instrumentor<'tcx> {
         debug_assert_eq!(END_BYTE_POS, args.len());
         args.push(self.const_u32(code_region.hi().to_u32(), injection_point));
 
-        let mut patch = MirPatch::new(mir_body);
+        self.inject_call(count_code_region_fn, args, injection_point, next_block);
 
-        let temp = patch.new_temp(self.tcx.mk_unit(), code_region);
-        let new_block = patch.new_block(placeholder_block(code_region));
+        counter_id
+    }
+
+    fn inject_counter_expression(
+        &mut self,
+        code_region: Span,
+        next_block: BasicBlock,
+        lhs: u32,
+        op: Op,
+        rhs: u32,
+    ) -> u32 {
+        let expression_id = self.next_expression();
+        let injection_point = code_region.shrink_to_lo();
+
+        let count_code_region_fn = function_handle(
+            self.tcx,
+            self.tcx.require_lang_item(
+                match op {
+                    Op::Add => lang_items::CoverageCounterAddFnLangItem,
+                    Op::Subtract => lang_items::CoverageCounterSubtractFnLangItem,
+                },
+                None,
+            ),
+            injection_point,
+        );
+
+        let mut args = Vec::new();
+
+        use coverage_counter_expression_args::*;
+        debug_assert_eq!(COUNTER_EXPRESSION_INDEX, args.len());
+        args.push(self.const_u32(expression_id, injection_point));
+
+        debug_assert_eq!(LEFT_INDEX, args.len());
+        args.push(self.const_u32(lhs, injection_point));
+
+        debug_assert_eq!(RIGHT_INDEX, args.len());
+        args.push(self.const_u32(rhs, injection_point));
+
+        debug_assert_eq!(START_BYTE_POS, args.len());
+        args.push(self.const_u32(code_region.lo().to_u32(), injection_point));
+
+        debug_assert_eq!(END_BYTE_POS, args.len());
+        args.push(self.const_u32(code_region.hi().to_u32(), injection_point));
+
+        self.inject_call(count_code_region_fn, args, injection_point, next_block);
+
+        expression_id
+    }
+
+    fn inject_call(
+        &mut self,
+        func: Operand<'tcx>,
+        args: Vec<Operand<'tcx>>,
+        fn_span: Span,
+        next_block: BasicBlock,
+    ) {
+        let mut patch = MirPatch::new(self.mir_body);
+
+        let temp = patch.new_temp(self.tcx.mk_unit(), fn_span);
+        let new_block = patch.new_block(placeholder_block(fn_span));
         patch.patch_terminator(
             new_block,
             TerminatorKind::Call {
-                func: count_code_region_fn,
+                func,
                 args,
                 // new_block will swapped with the next_block, after applying patch
                 destination: Some((Place::from(temp), new_block)),
                 cleanup: None,
                 from_hir_call: false,
-                fn_span: injection_point,
+                fn_span,
             },
         );
 
         patch.add_statement(new_block.start_location(), StatementKind::StorageLive(temp));
         patch.add_statement(next_block.start_location(), StatementKind::StorageDead(temp));
 
-        patch.apply(mir_body);
+        patch.apply(self.mir_body);
 
         // To insert the `new_block` in front of the first block in the counted branch (the
         // `next_block`), just swap the indexes, leaving the rest of the graph unchanged.
-        mir_body.basic_blocks_mut().swap(next_block, new_block);
+        self.mir_body.basic_blocks_mut().swap(next_block, new_block);
     }
 
     fn const_u32(&self, value: u32, span: Span) -> Operand<'tcx> {
         Operand::const_from_scalar(self.tcx, self.tcx.types.u32, Scalar::from_u32(value), span)
     }
+
+    fn const_u64(&self, value: u64, span: Span) -> Operand<'tcx> {
+        Operand::const_from_scalar(self.tcx, self.tcx.types.u64, Scalar::from_u64(value), span)
+    }
 }
 
 fn function_handle<'tcx>(tcx: TyCtxt<'tcx>, fn_def_id: DefId, span: Span) -> Operand<'tcx> {
@@ -192,10 +337,13 @@ fn placeholder_block(span: Span) -> BasicBlockData<'tcx> {
     }
 }
 
-fn hash_mir_source<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> u64 {
+fn hir_body<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> &'tcx rustc_hir::Body<'tcx> {
     let hir_node = tcx.hir().get_if_local(def_id).expect("DefId is local");
     let fn_body_id = hir::map::associated_body(hir_node).expect("HIR node is a function with body");
-    let hir_body = tcx.hir().body(fn_body_id);
+    tcx.hir().body(fn_body_id)
+}
+
+fn hash_mir_source<'tcx>(tcx: TyCtxt<'tcx>, hir_body: &'tcx rustc_hir::Body<'tcx>) -> u64 {
     let mut hcx = tcx.create_no_span_stable_hashing_context();
     hash(&mut hcx, &hir_body.value).to_smaller_hash()
 }

src/librustc_passes/weak_lang_items.rs

@@ -3,14 +3,13 @@
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::struct_span_err;
 use rustc_hir as hir;
+use rustc_hir::fake_lang_items::FAKE_ITEMS_REFS;
 use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor};
 use rustc_hir::lang_items;
-use rustc_hir::lang_items::ITEM_REFS;
 use rustc_hir::weak_lang_items::WEAK_ITEMS_REFS;
 use rustc_middle::middle::lang_items::required;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::config::CrateType;
-use rustc_span::symbol::sym;
 use rustc_span::symbol::Symbol;
 use rustc_span::Span;
 
@@ -77,15 +76,14 @@ impl<'a, 'tcx> Context<'a, 'tcx> {
             if self.items.require(item).is_err() {
                 self.items.missing.push(item);
             }
-        } else if name == sym::count_code_region {
-            // `core::intrinsics::code_count_region()` is (currently) the only `extern` lang item
-            // that is never actually linked. It is not a `weak_lang_item` that can be registered
-            // when used, and should be registered here instead.
-            if let Some((item_index, _)) = ITEM_REFS.get(&name).cloned() {
-                if self.items.items[item_index].is_none() {
-                    let item_def_id = self.tcx.hir().local_def_id(hir_id).to_def_id();
-                    self.items.items[item_index] = Some(item_def_id);
-                }
+        } else if let Some(&item) = FAKE_ITEMS_REFS.get(&name) {
+            // Ensure "fake lang items" are registered. These are `extern` lang items that are
+            // injected into the MIR automatically (such as source code coverage counters), but are
+            // never actually linked; therefore, unlike "weak lang items", they cannot by registered
+            // when used, because they never appear to be used.
+            if self.items.items[item as usize].is_none() {
+                let item_def_id = self.tcx.hir().local_def_id(hir_id).to_def_id();
+                self.items.items[item as usize] = Some(item_def_id);
             }
         } else {
             struct_span_err!(self.tcx.sess, span, E0264, "unknown external lang item: `{}`", name)

src/librustc_session/config.rs

@@ -1707,6 +1707,31 @@ pub fn build_session_options(matches: &getopts::Matches) -> Options {
         );
     }
 
+    if debugging_opts.instrument_coverage {
+        if cg.profile_generate.enabled() || cg.profile_use.is_some() {
+            early_error(
+                error_format,
+                "option `-Z instrument-coverage` is not compatible with either `-C profile-use` \
+                or `-C profile-generate`",
+            );
+        }
+
+        // `-Z instrument-coverage` implies:
+        //   * `-Z symbol-mangling-version=v0` - to ensure consistent and reversable name mangling.
+        //     Note, LLVM coverage tools can analyze coverage over multiple runs, including some
+        //     changes to source code; so mangled names must be consistent across compilations.
+        //   * `-C link-dead-code` - so unexecuted code is still counted as zero, rather than be
+        //     optimized out. Note that instrumenting dead code can be explicitly disabled with:
+        //         `-Z instrument-coverage -C link-dead-code=no`.
+        debugging_opts.symbol_mangling_version = SymbolManglingVersion::V0;
+        if cg.link_dead_code == None {
+            // FIXME(richkadel): Investigate if the `instrument-coverage` implementation can
+            // inject ["zero counters"](https://llvm.org/docs/CoverageMappingFormat.html#counter)
+            // in the coverage map when "dead code" is removed, rather than forcing `link-dead-code`.
+            cg.link_dead_code = Some(true);
+        }
+    }
+
     if !cg.embed_bitcode {
         match cg.lto {
             LtoCli::No | LtoCli::Unspecified => {}

src/librustc_session/options.rs

@@ -715,7 +715,7 @@ options! {CodegenOptions, CodegenSetter, basic_codegen_options,
         "a single extra argument to append to the linker invocation (can be used several times)"),
     link_args: Vec<String> = (Vec::new(), parse_list, [UNTRACKED],
         "extra arguments to append to the linker invocation (space separated)"),
-    link_dead_code: bool = (false, parse_bool, [UNTRACKED],
+    link_dead_code: Option<bool> = (None, parse_opt_bool, [UNTRACKED],
         "keep dead code at link time (useful for code coverage) (default: no)"),
     linker: Option<PathBuf> = (None, parse_opt_pathbuf, [UNTRACKED],
         "system linker to link outputs with"),
@@ -880,10 +880,12 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
         (such as entering an empty infinite loop) by inserting llvm.sideeffect \
         (default: no)"),
     instrument_coverage: bool = (false, parse_bool, [TRACKED],
-        "instrument the generated code with LLVM code region counters to (in the \
-        future) generate coverage reports; disables/overrides some optimization \
-        options (note, the compiler build config must include `profiler = true`) \
-        (default: no)"),
+        "instrument the generated code to support LLVM source-based code coverage \
+        reports (note, the compiler build config must include `profiler = true`, \
+        and is mutually exclusive with `-C profile-generate`/`-C profile-use`); \
+        implies `-C link-dead-code` (unless explicitly disabled)` and
+        `-Z symbol-mangling-version=v0`; and disables/overrides some optimization \
+        options (default: no)"),
     instrument_mcount: bool = (false, parse_bool, [TRACKED],
         "insert function instrument code for mcount-based tracing (default: no)"),
     keep_hygiene_data: bool = (false, parse_bool, [UNTRACKED],

src/librustc_session/session.rs

@@ -1357,6 +1357,20 @@ fn validate_commandline_args_with_session_available(sess: &Session) {
         );
     }
 
+    // FIXME(richkadel): See `src/test/run-make-fulldeps/instrument-coverage/Makefile`. After
+    // compiling with `-Zinstrument-coverage`, the resulting binary generates a segfault during
+    // the program's exit process (likely while attempting to generate the coverage stats in
+    // the "*.profraw" file). An investigation to resolve the problem on Windows is ongoing,
+    // but until this is resolved, the option is disabled on Windows, and the test is skipped
+    // when targeting `MSVC`.
+    if sess.opts.debugging_opts.instrument_coverage && sess.target.target.options.is_like_msvc {
+        sess.warn(
+            "Rust source-based code coverage instrumentation (with `-Z instrument-coverage`) \
+            is not yet supported on Windows when targeting MSVC. The resulting binaries will \
+            still be instrumented for experimentation purposes, but may not execute correctly.",
+        );
+    }
+
     const ASAN_SUPPORTED_TARGETS: &[&str] = &[
         "aarch64-fuchsia",
         "aarch64-unknown-linux-gnu",

src/librustc_typeck/check/intrinsic.rs

@@ -386,7 +386,7 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
             }
 
             sym::count_code_region => {
-                (0, vec![tcx.types.u32, tcx.types.u32, tcx.types.u32], tcx.mk_unit())
+                (0, vec![tcx.types.u64, tcx.types.u32, tcx.types.u32, tcx.types.u32], tcx.mk_unit())
             }
 
             sym::coverage_counter_add | sym::coverage_counter_subtract => (

src/rustllvm/CoverageMappingWrapper.cpp

@@ -0,0 +1,115 @@
+#include "rustllvm.h"
+#include "llvm/ProfileData/Coverage/CoverageMapping.h"
+#include "llvm/ProfileData/Coverage/CoverageMappingWriter.h"
+#include "llvm/ProfileData/InstrProf.h"
+#include "llvm/ADT/ArrayRef.h"
+
+#include <iostream>
+
+using namespace llvm;
+
+extern "C" SmallVectorTemplateBase<coverage::CounterExpression>
+    *LLVMRustCoverageSmallVectorCounterExpressionCreate() {
+  return new SmallVector<coverage::CounterExpression, 32>();
+}
+
+extern "C" void LLVMRustCoverageSmallVectorCounterExpressionDispose(
+    SmallVectorTemplateBase<coverage::CounterExpression> *Vector) {
+  delete Vector;
+}
+
+extern "C" void LLVMRustCoverageSmallVectorCounterExpressionAdd(
+    SmallVectorTemplateBase<coverage::CounterExpression> *Expressions,
+    coverage::CounterExpression::ExprKind Kind,
+    unsigned LeftIndex,
+    unsigned RightIndex) {
+  auto LHS = coverage::Counter::getCounter(LeftIndex);
+  auto RHS = coverage::Counter::getCounter(RightIndex);
+  Expressions->push_back(coverage::CounterExpression { Kind, LHS, RHS });
+}
+
+extern "C" SmallVectorTemplateBase<coverage::CounterMappingRegion>
+    *LLVMRustCoverageSmallVectorCounterMappingRegionCreate() {
+  return new SmallVector<coverage::CounterMappingRegion, 32>();
+}
+
+extern "C" void LLVMRustCoverageSmallVectorCounterMappingRegionDispose(
+    SmallVectorTemplateBase<coverage::CounterMappingRegion> *Vector) {
+  delete Vector;
+}
+
+extern "C" void LLVMRustCoverageSmallVectorCounterMappingRegionAdd(
+    SmallVectorTemplateBase<coverage::CounterMappingRegion> *MappingRegions,
+    unsigned Index,
+    unsigned FileID,
+    unsigned LineStart,
+    unsigned ColumnStart,
+    unsigned LineEnd,
+    unsigned ColumnEnd) {
+  auto Counter = coverage::Counter::getCounter(Index);
+  MappingRegions->push_back(coverage::CounterMappingRegion::makeRegion(
+           Counter, FileID, LineStart,
+           ColumnStart, LineEnd, ColumnEnd));
+
+  // FIXME(richkadel): As applicable, implement additional CounterMappingRegion types using the
+  // static method alternatives to `coverage::CounterMappingRegion::makeRegion`:
+  //
+  //   makeExpansion(unsigned FileID, unsigned ExpandedFileID, unsigned LineStart,
+  //                 unsigned ColumnStart, unsigned LineEnd, unsigned ColumnEnd) {
+  //   makeSkipped(unsigned FileID, unsigned LineStart, unsigned ColumnStart,
+  //               unsigned LineEnd, unsigned ColumnEnd) {
+  //   makeGapRegion(Counter Count, unsigned FileID, unsigned LineStart,
+  //                 unsigned ColumnStart, unsigned LineEnd, unsigned ColumnEnd) {
+}
+
+extern "C" void LLVMRustCoverageWriteFilenamesSectionToBuffer(
+    const char* const Filenames[],
+    size_t FilenamesLen,
+    RustStringRef BufferOut) {
+  SmallVector<StringRef,32> FilenameRefs;
+  for (size_t i = 0; i < FilenamesLen; i++) {
+    FilenameRefs.push_back(StringRef(Filenames[i]));
+  }
+  auto FilenamesWriter = coverage::CoverageFilenamesSectionWriter(
+    makeArrayRef(FilenameRefs));
+  RawRustStringOstream OS(BufferOut);
+  FilenamesWriter.write(OS);
+}
+
+extern "C" void LLVMRustCoverageWriteMappingToBuffer(
+    const unsigned *VirtualFileMappingIDs,
+    unsigned NumVirtualFileMappingIDs,
+    const SmallVectorImpl<coverage::CounterExpression> *Expressions,
+    SmallVectorImpl<coverage::CounterMappingRegion> *MappingRegions,
+    RustStringRef BufferOut) {
+  auto CoverageMappingWriter = coverage::CoverageMappingWriter(
+    makeArrayRef(VirtualFileMappingIDs, NumVirtualFileMappingIDs),
+    makeArrayRef(*Expressions),
+    MutableArrayRef<coverage::CounterMappingRegion> { *MappingRegions });
+  RawRustStringOstream OS(BufferOut);
+  CoverageMappingWriter.write(OS);
+}
+
+extern "C" uint64_t LLVMRustCoverageComputeHash(const char *Name) {
+  StringRef NameRef(Name);
+  return IndexedInstrProf::ComputeHash(NameRef);
+}
+
+extern "C" void LLVMRustCoverageWriteSectionNameToString(LLVMModuleRef M,
+                                                         RustStringRef Str) {
+  Triple TargetTriple(unwrap(M)->getTargetTriple());
+  auto name = getInstrProfSectionName(IPSK_covmap,
+                                      TargetTriple.getObjectFormat());
+  RawRustStringOstream OS(Str);
+  OS << name;
+}
+
+extern "C" void LLVMRustCoverageWriteMappingVarNameToString(RustStringRef Str) {
+  auto name = getCoverageMappingVarName();
+  RawRustStringOstream OS(Str);
+  OS << name;
+}
+
+extern "C" uint32_t LLVMRustCoverageMappingVersion() {
+  return coverage::CovMapVersion::CurrentVersion;
+}

src/rustllvm/RustWrapper.cpp

@@ -1395,7 +1395,7 @@ extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMValueRef Fn,
       FTy, Callee, makeArrayRef(unwrap(Args), NumArgs), Bundles));
 }
 
-extern "C" LLVMValueRef LLVMRustGetInstrprofIncrementIntrinsic(LLVMModuleRef M) {
+extern "C" LLVMValueRef LLVMRustGetInstrProfIncrementIntrinsic(LLVMModuleRef M) {
   return wrap(llvm::Intrinsic::getDeclaration(unwrap(M),
               (llvm::Intrinsic::ID)llvm::Intrinsic::instrprof_increment));
 }

src/rustllvm/rustllvm.h

@@ -3,6 +3,7 @@
 #include "llvm-c/Object.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Analysis/Lint.h"
 #include "llvm/Analysis/Passes.h"

src/test/mir-opt/instrument_coverage/rustc.bar.InstrumentCoverage.diff

@@ -3,34 +3,40 @@
   
   fn bar() -> bool {
       let mut _0: bool;                    // return place in scope 0 at $DIR/instrument_coverage.rs:18:13: 18:17
-+     let mut _1: ();                      // in scope 0 at $DIR/instrument_coverage.rs:18:1: 20:2
++     let mut _1: ();                      // in scope 0 at $DIR/instrument_coverage.rs:18:18: 18:18
   
       bb0: {
-+         StorageLive(_1);                 // scope 0 at $DIR/instrument_coverage.rs:18:1: 20:2
-+         _1 = const std::intrinsics::count_code_region(const 0_u32, const 484_u32, const 513_u32) -> bb2; // scope 0 at $DIR/instrument_coverage.rs:18:1: 20:2
++         StorageLive(_1);                 // scope 0 at $DIR/instrument_coverage.rs:18:18: 18:18
++         _1 = const std::intrinsics::count_code_region(const 10208505205182607101_u64, const 0_u32, const 501_u32, const 513_u32) -> bb2; // scope 0 at $DIR/instrument_coverage.rs:18:18: 18:18
 +                                          // ty::Const
-+                                          // + ty: unsafe extern "rust-intrinsic" fn(u32, u32, u32) {std::intrinsics::count_code_region}
++                                          // + ty: unsafe extern "rust-intrinsic" fn(u64, u32, u32, u32) {std::intrinsics::count_code_region}
 +                                          // + val: Value(Scalar(<ZST>))
 +                                          // mir::Constant
-+                                          // + span: $DIR/instrument_coverage.rs:18:1: 18:1
-+                                          // + literal: Const { ty: unsafe extern "rust-intrinsic" fn(u32, u32, u32) {std::intrinsics::count_code_region}, val: Value(Scalar(<ZST>)) }
++                                          // + span: $DIR/instrument_coverage.rs:18:18: 18:18
++                                          // + literal: Const { ty: unsafe extern "rust-intrinsic" fn(u64, u32, u32, u32) {std::intrinsics::count_code_region}, val: Value(Scalar(<ZST>)) }
++                                          // ty::Const
++                                          // + ty: u64
++                                          // + val: Value(Scalar(0x8dabe565aaa2aefd))
++                                          // mir::Constant
++                                          // + span: $DIR/instrument_coverage.rs:18:18: 18:18
++                                          // + literal: Const { ty: u64, val: Value(Scalar(0x8dabe565aaa2aefd)) }
 +                                          // ty::Const
 +                                          // + ty: u32
 +                                          // + val: Value(Scalar(0x00000000))
 +                                          // mir::Constant
-+                                          // + span: $DIR/instrument_coverage.rs:18:1: 18:1
++                                          // + span: $DIR/instrument_coverage.rs:18:18: 18:18
 +                                          // + literal: Const { ty: u32, val: Value(Scalar(0x00000000)) }
 +                                          // ty::Const
 +                                          // + ty: u32
-+                                          // + val: Value(Scalar(0x000001e4))
++                                          // + val: Value(Scalar(0x000001f5))
 +                                          // mir::Constant
-+                                          // + span: $DIR/instrument_coverage.rs:18:1: 18:1
-+                                          // + literal: Const { ty: u32, val: Value(Scalar(0x000001e4)) }
++                                          // + span: $DIR/instrument_coverage.rs:18:18: 18:18
++                                          // + literal: Const { ty: u32, val: Value(Scalar(0x000001f5)) }
 +                                          // ty::Const
 +                                          // + ty: u32
 +                                          // + val: Value(Scalar(0x00000201))
 +                                          // mir::Constant
-+                                          // + span: $DIR/instrument_coverage.rs:18:1: 18:1
++                                          // + span: $DIR/instrument_coverage.rs:18:18: 18:18
 +                                          // + literal: Const { ty: u32, val: Value(Scalar(0x00000201)) }
 +     }
 + 

src/test/mir-opt/instrument_coverage/rustc.main.InstrumentCoverage.diff

@@ -6,35 +6,41 @@
       let mut _1: ();                      // in scope 0 at $DIR/instrument_coverage.rs:9:1: 15:2
       let mut _2: bool;                    // in scope 0 at $DIR/instrument_coverage.rs:11:12: 11:17
       let mut _3: !;                       // in scope 0 at $DIR/instrument_coverage.rs:11:18: 13:10
-+     let mut _4: ();                      // in scope 0 at $DIR/instrument_coverage.rs:9:1: 15:2
++     let mut _4: ();                      // in scope 0 at $DIR/instrument_coverage.rs:9:11: 9:11
   
       bb0: {
 -         falseUnwind -> [real: bb1, cleanup: bb2]; // scope 0 at $DIR/instrument_coverage.rs:10:5: 14:6
-+         StorageLive(_4);                 // scope 0 at $DIR/instrument_coverage.rs:9:1: 15:2
-+         _4 = const std::intrinsics::count_code_region(const 0_u32, const 387_u32, const 465_u32) -> bb7; // scope 0 at $DIR/instrument_coverage.rs:9:1: 15:2
++         StorageLive(_4);                 // scope 0 at $DIR/instrument_coverage.rs:9:11: 9:11
++         _4 = const std::intrinsics::count_code_region(const 16004455475339839479_u64, const 0_u32, const 397_u32, const 465_u32) -> bb7; // scope 0 at $DIR/instrument_coverage.rs:9:11: 9:11
 +                                          // ty::Const
-+                                          // + ty: unsafe extern "rust-intrinsic" fn(u32, u32, u32) {std::intrinsics::count_code_region}
++                                          // + ty: unsafe extern "rust-intrinsic" fn(u64, u32, u32, u32) {std::intrinsics::count_code_region}
 +                                          // + val: Value(Scalar(<ZST>))
 +                                          // mir::Constant
-+                                          // + span: $DIR/instrument_coverage.rs:9:1: 9:1
-+                                          // + literal: Const { ty: unsafe extern "rust-intrinsic" fn(u32, u32, u32) {std::intrinsics::count_code_region}, val: Value(Scalar(<ZST>)) }
++                                          // + span: $DIR/instrument_coverage.rs:9:11: 9:11
++                                          // + literal: Const { ty: unsafe extern "rust-intrinsic" fn(u64, u32, u32, u32) {std::intrinsics::count_code_region}, val: Value(Scalar(<ZST>)) }
++                                          // ty::Const
++                                          // + ty: u64
++                                          // + val: Value(Scalar(0xde1b3f75a72fc7f7))
++                                          // mir::Constant
++                                          // + span: $DIR/instrument_coverage.rs:9:11: 9:11
++                                          // + literal: Const { ty: u64, val: Value(Scalar(0xde1b3f75a72fc7f7)) }
 +                                          // ty::Const
 +                                          // + ty: u32
 +                                          // + val: Value(Scalar(0x00000000))
 +                                          // mir::Constant
-+                                          // + span: $DIR/instrument_coverage.rs:9:1: 9:1
++                                          // + span: $DIR/instrument_coverage.rs:9:11: 9:11
 +                                          // + literal: Const { ty: u32, val: Value(Scalar(0x00000000)) }
 +                                          // ty::Const
 +                                          // + ty: u32
-+                                          // + val: Value(Scalar(0x00000183))
++                                          // + val: Value(Scalar(0x0000018d))
 +                                          // mir::Constant
-+                                          // + span: $DIR/instrument_coverage.rs:9:1: 9:1
-+                                          // + literal: Const { ty: u32, val: Value(Scalar(0x00000183)) }
++                                          // + span: $DIR/instrument_coverage.rs:9:11: 9:11
++                                          // + literal: Const { ty: u32, val: Value(Scalar(0x0000018d)) }
 +                                          // ty::Const
 +                                          // + ty: u32
 +                                          // + val: Value(Scalar(0x000001d1))
 +                                          // mir::Constant
-+                                          // + span: $DIR/instrument_coverage.rs:9:1: 9:1
++                                          // + span: $DIR/instrument_coverage.rs:9:11: 9:11
 +                                          // + literal: Const { ty: u32, val: Value(Scalar(0x000001d1)) }
       }
   

src/test/run-make-fulldeps/instrument-coverage/Makefile

@@ -0,0 +1,57 @@
+# needs-profiler-support
+# ignore-msvc
+
+# FIXME(richkadel): Debug the following problem, and reenable on Windows (by
+# removing the `# ignore-msvc` directive above). The current implementation
+# generates a segfault when running the instrumented `main` executable,
+# after the `main` program code executes, but before the process terminates.
+# This most likely points to a problem generating the LLVM "main.profraw"
+# file.
+
+-include ../tools.mk
+
+# This test makes sure that LLVM coverage maps are genereated in LLVM IR.
+
+COMMON_FLAGS=-Zinstrument-coverage
+
+all:
+	# Compile the test program with instrumentation, and also generate LLVM IR
+	$(RUSTC) $(COMMON_FLAGS) main.rs
+
+	# Run it in order to generate some profiling data,
+	# with `LLVM_PROFILE_FILE=<profdata_file>` environment variable set to
+	# output the coverage stats for this run.
+	LLVM_PROFILE_FILE="$(TMPDIR)"/main.profraw \
+	  $(call RUN,main)
+
+	# Postprocess the profiling data so it can be used by the llvm-cov tool
+	"$(LLVM_BIN_DIR)"/llvm-profdata merge --sparse \
+	  "$(TMPDIR)"/main.profraw \
+		-o "$(TMPDIR)"/main.profdata
+
+	# Generate a coverage report using `llvm-cov show`. The output ordering
+	# can be non-deterministic, so ignore the return status. If the test fails
+	# when comparing the JSON `export`, the `show` output may be useful when
+	# debugging.
+	"$(LLVM_BIN_DIR)"/llvm-cov show \
+	  --Xdemangler="$(RUST_DEMANGLER)" \
+	  --show-line-counts-or-regions \
+	  --instr-profile="$(TMPDIR)"/main.profdata \
+		$(call BIN,"$(TMPDIR)"/main) \
+		> "$(TMPDIR)"/actual_show_coverage.txt
+
+	# Compare the show coverage output
+	$(DIFF) typical_show_coverage.txt "$(TMPDIR)"/actual_show_coverage.txt || \
+	  >&2 echo 'diff failed for `llvm-cov show` (might not be an error)'
+
+	# Generate a coverage report in JSON, using `llvm-cov export`, and fail if
+	# there are differences from the expected output.
+	"$(LLVM_BIN_DIR)"/llvm-cov export \
+	  --summary-only \
+	  --instr-profile="$(TMPDIR)"/main.profdata \
+		$(call BIN,"$(TMPDIR)"/main) \
+		| "$(PYTHON)" prettify_json.py \
+		> "$(TMPDIR)"/actual_export_coverage.json
+
+	# Check that the exported JSON coverage data matches what we expect
+	$(DIFF) expected_export_coverage.json "$(TMPDIR)"/actual_export_coverage.json

src/test/run-make-fulldeps/instrument-coverage/expected_export_coverage.json

@@ -0,0 +1,59 @@
+{
+  "data": [
+    {
+      "files": [
+        {
+          "filename": "main.rs",
+          "summary": {
+            "functions": {
+              "count": 7,
+              "covered": 5,
+              "percent": 71.42857142857143
+            },
+            "instantiations": {
+              "count": 8,
+              "covered": 6,
+              "percent": 75
+            },
+            "lines": {
+              "count": 30,
+              "covered": 25,
+              "percent": 83.33333333333334
+            },
+            "regions": {
+              "count": 7,
+              "covered": 5,
+              "notcovered": 2,
+              "percent": 71.42857142857143
+            }
+          }
+        }
+      ],
+      "totals": {
+        "functions": {
+          "count": 7,
+          "covered": 5,
+          "percent": 71.42857142857143
+        },
+        "instantiations": {
+          "count": 8,
+          "covered": 6,
+          "percent": 75
+        },
+        "lines": {
+          "count": 30,
+          "covered": 25,
+          "percent": 83.33333333333334
+        },
+        "regions": {
+          "count": 7,
+          "covered": 5,
+          "notcovered": 2,
+          "percent": 71.42857142857143
+        }
+      }
+    }
+  ],
+  "type": "llvm.coverage.json.export",
+  "version": "2.0.0"
+}

src/test/run-make-fulldeps/instrument-coverage/main.rs

@@ -0,0 +1,38 @@
+pub fn will_be_called() -> &'static str {
+    let val = "called";
+    println!("{}", val);
+    val
+}
+
+pub fn will_not_be_called() -> bool {
+    println!("should not have been called");
+    false
+}
+
+pub fn print<T>(left: &str, value: T, right: &str)
+where
+    T: std::fmt::Display,
+{
+    println!("{}{}{}", left, value, right);
+}
+
+pub fn wrap_with<F, T>(inner: T, should_wrap: bool, wrapper: F)
+where
+    F: FnOnce(&T)
+{
+    if should_wrap {
+        wrapper(&inner)
+    }
+}
+
+fn main() {
+    let less = 1;
+    let more = 100;
+
+    if less < more {
+        wrap_with(will_be_called(), less < more, |inner| print(" ***", inner, "*** "));
+        wrap_with(will_be_called(), more < less, |inner| print(" ***", inner, "*** "));
+    } else {
+        wrap_with(will_not_be_called(), true, |inner| print("wrapped result is: ", inner, ""));
+    }
+}

src/test/run-make-fulldeps/instrument-coverage/prettify_json.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+import sys
+import json
+
+# Try to decode line in order to ensure it is a valid JSON document
+for line in sys.stdin:
+    parsed = json.loads(line)
+    print (json.dumps(parsed, indent=2, separators=(',', ': '), sort_keys=True))

src/test/run-make-fulldeps/instrument-coverage/typical_show_coverage.txt

@@ -0,0 +1,55 @@
+    1|      2|pub fn will_be_called() -> &'static str {
+    2|      2|    let val = "called";
+    3|      2|    println!("{}", val);
+    4|      2|    val
+    5|      2|}
+    6|       |
+    7|      0|pub fn will_not_be_called() -> bool {
+    8|      0|    println!("should not have been called");
+    9|      0|    false
+   10|      0|}
+   11|       |
+   12|       |pub fn print<T>(left: &str, value: T, right: &str)
+   13|       |where
+   14|       |    T: std::fmt::Display,
+   15|      1|{
+   16|      1|    println!("{}{}{}", left, value, right);
+   17|      1|}
+   18|       |
+   19|       |pub fn wrap_with<F, T>(inner: T, should_wrap: bool, wrapper: F)
+   20|       |where
+   21|       |    F: FnOnce(&T)
+   22|      2|{
+   23|      2|    if should_wrap {
+   24|      2|        wrapper(&inner)
+   25|      2|    }
+   26|      2|}
+  ------------------
+  | main[317d481089b8c8fe]::wrap_with::<main[317d481089b8c8fe]::main::{closure#0}, &str>:
+  |   22|      1|{
+  |   23|      1|    if should_wrap {
+  |   24|      1|        wrapper(&inner)
+  |   25|      1|    }
+  |   26|      1|}
+  ------------------
+  | main[317d481089b8c8fe]::wrap_with::<main[317d481089b8c8fe]::main::{closure#1}, &str>:
+  |   22|      1|{
+  |   23|      1|    if should_wrap {
+  |   24|      1|        wrapper(&inner)
+  |   25|      1|    }
+  |   26|      1|}
+  ------------------
+   27|       |
+   28|      1|fn main() {
+   29|      1|    let less = 1;
+   30|      1|    let more = 100;
+   31|      1|
+   32|      1|    if less < more {
+   33|      1|        wrap_with(will_be_called(), less < more, |inner| print(" ***", inner, "*** "));
+   34|      1|        wrap_with(will_be_called(), more < less, |inner| print(" ***", inner, "*** "));
+                                                                       ^0
+   35|      1|    } else {
+   36|      1|        wrap_with(will_not_be_called(), true, |inner| print("wrapped result is: ", inner, ""));
+   37|      1|    }
+   38|      1|}
+

src/test/run-make-fulldeps/tools.mk

@@ -18,6 +18,9 @@ endif
 HTMLDOCCK := '$(PYTHON)' '$(S)/src/etc/htmldocck.py'
 CGREP := "$(S)/src/etc/cat-and-grep.sh"
 
+# diff with common flags for multi-platform diffs against text output
+DIFF := diff -u --strip-trailing-cr
+
 # This is the name of the binary we will generate and run; use this
 # e.g. for `$(CC) -o $(RUN_BINFILE)`.
 RUN_BINFILE = $(TMPDIR)/$(1)

src/tools/compiletest/src/common.rs

@@ -186,6 +186,9 @@ pub struct Config {
     /// The rustdoc executable.
     pub rustdoc_path: Option<PathBuf>,
 
+    /// The rust-demangler executable.
+    pub rust_demangler_path: Option<PathBuf>,
+
     /// The Python executable to use for LLDB.
     pub lldb_python: String,
 

src/tools/compiletest/src/main.rs

@@ -53,6 +53,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         .reqopt("", "run-lib-path", "path to target shared libraries", "PATH")
         .reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH")
         .optopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH")
+        .optopt("", "rust-demangler-path", "path to rust-demangler to use in tests", "PATH")
         .reqopt("", "lldb-python", "path to python to use for doc tests", "PATH")
         .reqopt("", "docck-python", "path to python to use for doc tests", "PATH")
         .optopt("", "valgrind-path", "path to Valgrind executable for Valgrind tests", "PROGRAM")
@@ -182,6 +183,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         run_lib_path: make_absolute(opt_path(matches, "run-lib-path")),
         rustc_path: opt_path(matches, "rustc-path"),
         rustdoc_path: matches.opt_str("rustdoc-path").map(PathBuf::from),
+        rust_demangler_path: matches.opt_str("rust-demangler-path").map(PathBuf::from),
         lldb_python: matches.opt_str("lldb-python").unwrap(),
         docck_python: matches.opt_str("docck-python").unwrap(),
         valgrind_path: matches.opt_str("valgrind-path"),
@@ -246,6 +248,7 @@ pub fn log_config(config: &Config) {
     logv(c, format!("run_lib_path: {:?}", config.run_lib_path));
     logv(c, format!("rustc_path: {:?}", config.rustc_path.display()));
     logv(c, format!("rustdoc_path: {:?}", config.rustdoc_path));
+    logv(c, format!("rust_demangler_path: {:?}", config.rust_demangler_path));
     logv(c, format!("src_base: {:?}", config.src_base.display()));
     logv(c, format!("build_base: {:?}", config.build_base.display()));
     logv(c, format!("stage_id: {}", config.stage_id));
@@ -479,6 +482,8 @@ fn common_inputs_stamp(config: &Config) -> Stamp {
         stamp.add_path(&rustdoc_path);
         stamp.add_path(&rust_src_dir.join("src/etc/htmldocck.py"));
     }
+    // FIXME(richkadel): Do I need to add an `if let Some(rust_demangler_path) contribution to the
+    // stamp here as well?
 
     // Compiletest itself.
     stamp.add_dir(&rust_src_dir.join("src/tools/compiletest/"));

src/tools/compiletest/src/runtest.rs

@@ -2739,6 +2739,10 @@ impl<'test> TestCx<'test> {
             cmd.env("RUSTDOC", cwd.join(rustdoc));
         }
 
+        if let Some(ref rust_demangler) = self.config.rust_demangler_path {
+            cmd.env("RUST_DEMANGLER", cwd.join(rust_demangler));
+        }
+
         if let Some(ref node) = self.config.nodejs {
             cmd.env("NODE", node);
         }

src/tools/rust-demangler/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+authors = ["The Rust Project Developers"]
+name = "rust-demangler"
+version = "0.0.0"
+edition = "2018"
+
+[dependencies]
+rustc-demangle = "0.1"
+
+[[bin]]
+name = "rust-demangler"
+path = "main.rs"

src/tools/rust-demangler/main.rs

@@ -0,0 +1,39 @@
+//! Demangles rustc mangled names.
+//!
+//! This tool uses https://crates.io/crates/rustc-demangle to convert an input buffer of
+//! newline-separated mangled names into their demangled translations.
+//!
+//! This tool can be leveraged by other applications that support third-party demanglers.
+//! It takes a list of mangled names (one per line) on standard input, and prints a corresponding
+//! list of demangled names. The tool is designed to support other programs that can leverage a
+//! third-party demangler, such as `llvm-cov`, via the `-Xdemangler=<path-to-demangler>` option.
+//!
+//! To use `rust-demangler`, first build the tool with:
+//!
+//! ```shell
+//! $ ./x.py build rust-demangler
+//! ```
+//!
+//! Then, with `llvm-cov` for example, add the `-Xdemangler=...` option:
+//!
+//! ```shell
+//! $ TARGET="${PWD}/build/x86_64-unknown-linux-gnu"
+//! $ "${TARGET}"/llvm/bin/llvm-cov show --Xdemangler="${TARGET}"/stage0-tools-bin/rust-demangler \
+//!   --instr-profile=main.profdata ./main --show-line-counts-or-regions
+//! ```
+
+use rustc_demangle::demangle;
+use std::io::{self, Read, Write};
+
+fn main() -> io::Result<()> {
+    let mut buffer = String::new();
+    io::stdin().read_to_string(&mut buffer)?;
+    let lines = buffer.lines();
+    let mut demangled = Vec::new();
+    for mangled in lines {
+        demangled.push(demangle(mangled).to_string());
+    }
+    demangled.push("".to_string());
+    io::stdout().write_all(demangled.join("\n").as_bytes())?;
+    Ok(())
+}