Add lldb to the build

This optionally adds lldb (and clang, which it needs) to the build.

Because rust uses LLVM 7, and because clang 7 is not yet released, a
recent git master version of clang is used.

The lldb that is used includes the Rust plugin.

lldb is only built when asked for, or when doing a nightly build on
macOS.  Only macOS is done for now due to difficulties with the Python
dependency.

.gitmodules

@@ -56,3 +56,11 @@
 [submodule "src/libbacktrace"]
 	path = src/libbacktrace
 	url = https://github.com/rust-lang-nursery/libbacktrace
+[submodule "src/tools/lldb"]
+	path = src/tools/lldb
+	url = https://github.com/rust-lang-nursery/lldb/
+	branch = rust-release-70
+[submodule "src/tools/clang"]
+	path = src/tools/clang
+	url = https://github.com/rust-lang-nursery/clang/
+	branch = release_70

.travis.yml

@@ -30,7 +30,7 @@ matrix:
 
     - env: >
         RUST_CHECK_TARGET=dist
-        RUST_CONFIGURE_ARGS="--enable-extended --enable-profiler"
+        RUST_CONFIGURE_ARGS="--enable-extended --enable-profiler --enable-lldb"
         SRC=.
         DEPLOY_ALT=1
         RUSTC_RETRY_LINKER_ON_SEGFAULT=1
@@ -85,7 +85,7 @@ matrix:
     # OSX 10.7 and `xcode7` is the latest Xcode able to compile LLVM for 10.7.
     - env: >
         RUST_CHECK_TARGET=dist
-        RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-full-tools --enable-profiler"
+        RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb"
         SRC=.
         DEPLOY=1
         RUSTC_RETRY_LINKER_ON_SEGFAULT=1
@@ -99,7 +99,7 @@ matrix:
 
     - env: >
         RUST_CHECK_TARGET=dist
-        RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler"
+        RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --enable-lldb"
         SRC=.
         DEPLOY=1
         RUSTC_RETRY_LINKER_ON_SEGFAULT=1
@@ -233,7 +233,8 @@ install:
         osx)
           if [[ "$RUST_CHECK_TARGET" == dist ]]; then
             travis_retry brew update &&
-            travis_retry brew install xz;
+            travis_retry brew install xz &&
+            travis_retry brew install swig;
           fi &&
           travis_retry curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin &&
             chmod +x /usr/local/bin/sccache &&

config.toml.example

@@ -354,6 +354,10 @@
 # sysroot.
 #llvm-tools = false
 
+# Indicates whether LLDB will be made available in the sysroot.
+# This is only built if LLVM is also being built.
+#lldb = false
+
 # Whether to deny warnings in crates
 #deny-warnings = true
 

src/Cargo.lock

@@ -1015,7 +1015,7 @@ name = "installer"
 version = "0.0.0"
 dependencies = [
  "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "flate2 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "rayon 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",

src/bootstrap/bootstrap.py

@@ -721,6 +721,10 @@ class RustBuild(object):
                 config = self.get_toml('lld')
                 if config is None or config == 'false':
                     continue
+            if module.endswith("lldb") or module.endswith("clang"):
+                config = self.get_toml('lldb')
+                if config is None or config == 'false':
+                    continue
             check = self.check_submodule(module, slow_submodules)
             filtered_submodules.append((module, check))
             submodules_names.append(module)

src/bootstrap/builder.rs

@@ -461,6 +461,7 @@ impl<'a> Builder<'a> {
                 dist::Rustfmt,
                 dist::Clippy,
                 dist::LlvmTools,
+                dist::Lldb,
                 dist::Extended,
                 dist::HashSign
             ),

src/bootstrap/config.rs

@@ -87,6 +87,7 @@ pub struct Config {
     pub llvm_link_jobs: Option<u32>,
 
     pub lld_enabled: bool,
+    pub lldb_enabled: bool,
     pub llvm_tools_enabled: bool,
 
     // rust codegen options
@@ -310,6 +311,7 @@ struct Rust {
     codegen_backends_dir: Option<String>,
     wasm_syscall: Option<bool>,
     lld: Option<bool>,
+    lldb: Option<bool>,
     llvm_tools: Option<bool>,
     deny_warnings: Option<bool>,
     backtrace_on_ice: Option<bool>,
@@ -538,6 +540,7 @@ impl Config {
             }
             set(&mut config.wasm_syscall, rust.wasm_syscall);
             set(&mut config.lld_enabled, rust.lld);
+            set(&mut config.lldb_enabled, rust.lldb);
             set(&mut config.llvm_tools_enabled, rust.llvm_tools);
             config.rustc_parallel_queries = rust.experimental_parallel_queries.unwrap_or(false);
             config.rustc_default_linker = rust.default_linker.clone();

src/bootstrap/configure.py

@@ -68,6 +68,7 @@ o("cargo-openssl-static", "build.openssl-static", "static openssl in cargo")
 o("profiler", "build.profiler", "build the profiler runtime")
 o("emscripten", None, "compile the emscripten backend as well as LLVM")
 o("full-tools", None, "enable all tools")
+o("lldb", "rust.lldb", "build lldb")
 
 # Optimization and debugging options. These may be overridden by the release
 # channel, etc.

src/bootstrap/dist.rs

@@ -47,6 +47,8 @@ pub fn pkgname(builder: &Builder, component: &str) -> String {
         format!("{}-{}", component, builder.rustfmt_package_vers())
     } else if component == "llvm-tools" {
         format!("{}-{}", component, builder.llvm_tools_package_vers())
+    } else if component == "lldb" {
+        format!("{}-{}", component, builder.lldb_package_vers())
     } else {
         assert!(component.starts_with("rust"));
         format!("{}-{}", component, builder.rust_package_vers())
@@ -1396,6 +1398,7 @@ impl Step for Extended {
         let rls_installer = builder.ensure(Rls { stage, target });
         let llvm_tools_installer = builder.ensure(LlvmTools { stage, target });
         let clippy_installer = builder.ensure(Clippy { stage, target });
+        let lldb_installer = builder.ensure(Lldb { target });
         let mingw_installer = builder.ensure(Mingw { host: target });
         let analysis_installer = builder.ensure(Analysis {
             compiler: builder.compiler(stage, self.host),
@@ -1435,6 +1438,7 @@ impl Step for Extended {
         tarballs.extend(clippy_installer.clone());
         tarballs.extend(rustfmt_installer.clone());
         tarballs.extend(llvm_tools_installer.clone());
+        tarballs.extend(lldb_installer.clone());
         tarballs.push(analysis_installer);
         tarballs.push(std_installer);
         if builder.config.docs {
@@ -1869,6 +1873,7 @@ impl Step for HashSign {
         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
         cmd.arg(builder.llvm_tools_package_vers());
+        cmd.arg(builder.lldb_package_vers());
         cmd.arg(addr);
 
         builder.create_dir(&distdir(builder));
@@ -1963,3 +1968,121 @@ impl Step for LlvmTools {
         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
     }
 }
+
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct Lldb {
+    pub target: Interned<String>,
+}
+
+impl Step for Lldb {
+    type Output = Option<PathBuf>;
+    const ONLY_HOSTS: bool = true;
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        run.path("src/tools/lldb")
+    }
+
+    fn make_run(run: RunConfig) {
+        run.builder.ensure(Lldb {
+            target: run.target,
+        });
+    }
+
+    fn run(self, builder: &Builder) -> Option<PathBuf> {
+        let target = self.target;
+
+        if builder.config.dry_run {
+            return None;
+        }
+
+        let bindir = builder
+            .llvm_out(target)
+            .join("bin");
+        let lldb_exe = bindir.join(exe("lldb", &target));
+        if !lldb_exe.exists() {
+            return None;
+        }
+
+        builder.info(&format!("Dist Lldb ({})", target));
+        let src = builder.src.join("src/tools/lldb");
+        let name = pkgname(builder, "lldb");
+
+        let tmp = tmpdir(builder);
+        let image = tmp.join("lldb-image");
+        drop(fs::remove_dir_all(&image));
+
+        // Prepare the image directory
+        let dst = image.join("bin");
+        t!(fs::create_dir_all(&dst));
+        for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
+            let exe = bindir.join(exe(program, &target));
+            builder.install(&exe, &dst, 0o755);
+        }
+
+        // The libraries.
+        let libdir = builder.llvm_out(target).join("lib");
+        let dst = image.join("lib");
+        t!(fs::create_dir_all(&dst));
+        for entry in t!(fs::read_dir(&libdir)) {
+            // let entry = t!(entry);
+            let entry = entry.unwrap();
+            if let Ok(name) = entry.file_name().into_string() {
+                if name.starts_with("liblldb.") && !name.ends_with(".a") {
+                    if t!(entry.file_type()).is_symlink() {
+                        builder.copy_to_folder(&entry.path(), &dst);
+                    } else {
+                       builder.install(&entry.path(), &dst, 0o755);
+                    }
+                }
+            }
+        }
+
+        // The lldb scripts might be installed in lib/python$version
+        // or in lib64/python$version.  If lib64 exists, use it;
+        // otherwise lib.
+        let libdir = builder.llvm_out(target).join("lib64");
+        let (libdir, libdir_name) = if libdir.exists() {
+            (libdir, "lib64")
+        } else {
+            (builder.llvm_out(target).join("lib"), "lib")
+        };
+        for entry in t!(fs::read_dir(&libdir)) {
+            let entry = t!(entry);
+            if let Ok(name) = entry.file_name().into_string() {
+                if name.starts_with("python") {
+                    let dst = image.join(libdir_name)
+                        .join(entry.file_name());
+                    t!(fs::create_dir_all(&dst));
+                    builder.cp_r(&entry.path(), &dst);
+                    break;
+                }
+            }
+        }
+
+        // Prepare the overlay
+        let overlay = tmp.join("lldb-overlay");
+        drop(fs::remove_dir_all(&overlay));
+        builder.create_dir(&overlay);
+        builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
+        builder.create(&overlay.join("version"), &builder.lldb_vers());
+
+        // Generate the installer tarball
+        let mut cmd = rust_installer(builder);
+        cmd.arg("generate")
+            .arg("--product-name=Rust")
+            .arg("--rel-manifest-dir=rustlib")
+            .arg("--success-message=lldb-installed.")
+            .arg("--image-dir").arg(&image)
+            .arg("--work-dir").arg(&tmpdir(builder))
+            .arg("--output-dir").arg(&distdir(builder))
+            .arg("--non-installed-overlay").arg(&overlay)
+            .arg(format!("--package-name={}-{}", name, target))
+            .arg("--legacy-manifest-dirs=rustlib,cargo")
+            .arg("--component-name=lldb-preview");
+
+
+        builder.run(&mut cmd);
+        Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
+    }
+}

src/bootstrap/lib.rs

@@ -151,6 +151,11 @@ use std::process::{self, Command};
 use std::slice;
 use std::str;
 
+#[cfg(unix)]
+use std::os::unix::fs::symlink as symlink_file;
+#[cfg(windows)]
+use std::os::windows::fs::symlink_file;
+
 use build_helper::{run_silent, run_suppressed, try_run_silent, try_run_suppressed, output, mtime};
 use filetime::FileTime;
 
@@ -1005,6 +1010,14 @@ impl Build {
         self.rust_version()
     }
 
+    fn lldb_package_vers(&self) -> String {
+        self.package_vers(&self.rust_version())
+    }
+
+    fn lldb_vers(&self) -> String {
+        self.rust_version()
+    }
+
     /// Returns the `version` string associated with this compiler for Rust
     /// itself.
     ///
@@ -1123,21 +1136,25 @@ impl Build {
     pub fn copy(&self, src: &Path, dst: &Path) {
         if self.config.dry_run { return; }
         let _ = fs::remove_file(&dst);
-        // Attempt to "easy copy" by creating a hard link (symlinks don't work on
+        let metadata = t!(src.symlink_metadata());
-        // windows), but if that fails just fall back to a slow `copy` operation.
+        if metadata.file_type().is_symlink() {
-        if let Ok(()) = fs::hard_link(src, dst) {
+            let link = t!(fs::read_link(src));
-            return
+            t!(symlink_file(link, dst));
-        }
+        } else if let Ok(()) = fs::hard_link(src, dst) {
+            // Attempt to "easy copy" by creating a hard link
+            // (symlinks don't work on windows), but if that fails
+            // just fall back to a slow `copy` operation.
+        } else {
             if let Err(e) = fs::copy(src, dst) {
                 panic!("failed to copy `{}` to `{}`: {}", src.display(),
                        dst.display(), e)
             }
-        let metadata = t!(src.metadata());
             t!(fs::set_permissions(dst, metadata.permissions()));
             let atime = FileTime::from_last_access_time(&metadata);
             let mtime = FileTime::from_last_modification_time(&metadata);
             t!(filetime::set_file_times(dst, atime, mtime));
         }
+    }
 
     /// Search-and-replaces within a file. (Not maximally efficiently: allocates a
     /// new string for each replacement.)

src/bootstrap/native.rs

@@ -149,7 +149,6 @@ impl Step for Llvm {
            .define("WITH_POLLY", "OFF")
            .define("LLVM_ENABLE_TERMINFO", "OFF")
            .define("LLVM_ENABLE_LIBEDIT", "OFF")
-           .define("LLVM_ENABLE_LIBXML2", "OFF")
            .define("LLVM_PARALLEL_COMPILE_JOBS", builder.jobs().to_string())
            .define("LLVM_TARGET_ARCH", target.split('-').next().unwrap())
            .define("LLVM_DEFAULT_TARGET_TRIPLE", target);
@@ -163,6 +162,8 @@ impl Step for Llvm {
         cfg.define("LLVM_OCAML_INSTALL_PATH",
             env::var_os("LLVM_OCAML_INSTALL_PATH").unwrap_or_else(|| "usr/lib/ocaml".into()));
 
+        let want_lldb = builder.config.lldb_enabled && !self.emscripten;
+
         // This setting makes the LLVM tools link to the dynamic LLVM library,
         // which saves both memory during parallel links and overall disk space
         // for the tools.  We don't distribute any of those tools, so this is
@@ -170,12 +171,13 @@ impl Step for Llvm {
         //
         // If we are shipping llvm tools then we statically link them LLVM
         if (target.contains("linux-gnu") || target.contains("apple-darwin")) &&
-            !builder.config.llvm_tools_enabled {
+            !builder.config.llvm_tools_enabled &&
+            !want_lldb {
                 cfg.define("LLVM_LINK_LLVM_DYLIB", "ON");
         }
 
         // For distribution we want the LLVM tools to be *statically* linked to libstdc++
-        if builder.config.llvm_tools_enabled {
+        if builder.config.llvm_tools_enabled || want_lldb {
             if !target.contains("windows") {
                 if target.contains("apple") {
                     cfg.define("CMAKE_EXE_LINKER_FLAGS", "-static-libstdc++");
@@ -196,6 +198,17 @@ impl Step for Llvm {
             cfg.define("LLVM_BUILD_32_BITS", "ON");
         }
 
+        if want_lldb {
+            cfg.define("LLVM_EXTERNAL_CLANG_SOURCE_DIR", builder.src.join("src/tools/clang"));
+            cfg.define("LLVM_EXTERNAL_LLDB_SOURCE_DIR", builder.src.join("src/tools/lldb"));
+            // For the time being, disable code signing.
+            cfg.define("LLDB_CODESIGN_IDENTITY", "");
+        } else {
+            // LLDB requires libxml2; but otherwise we want it to be disabled.
+            // See https://github.com/rust-lang/rust/pull/50104
+            cfg.define("LLVM_ENABLE_LIBXML2", "OFF");
+        }
+
         if let Some(num_linkers) = builder.config.llvm_link_jobs {
             if num_linkers > 0 {
                 cfg.define("LLVM_PARALLEL_LINK_JOBS", num_linkers.to_string());

src/tools/build-manifest/src/main.rs

@@ -189,6 +189,7 @@ struct Builder {
     clippy_release: String,
     rustfmt_release: String,
     llvm_tools_release: String,
+    lldb_release: String,
 
     input: PathBuf,
     output: PathBuf,
@@ -203,6 +204,7 @@ struct Builder {
     clippy_version: Option<String>,
     rustfmt_version: Option<String>,
     llvm_tools_version: Option<String>,
+    lldb_version: Option<String>,
 
     rust_git_commit_hash: Option<String>,
     cargo_git_commit_hash: Option<String>,
@@ -210,6 +212,7 @@ struct Builder {
     clippy_git_commit_hash: Option<String>,
     rustfmt_git_commit_hash: Option<String>,
     llvm_tools_git_commit_hash: Option<String>,
+    lldb_git_commit_hash: Option<String>,
 }
 
 fn main() {
@@ -223,6 +226,7 @@ fn main() {
     let clippy_release = args.next().unwrap();
     let rustfmt_release = args.next().unwrap();
     let llvm_tools_release = args.next().unwrap();
+    let lldb_release = args.next().unwrap();
     let s3_address = args.next().unwrap();
     let mut passphrase = String::new();
     t!(io::stdin().read_to_string(&mut passphrase));
@@ -234,6 +238,7 @@ fn main() {
         clippy_release,
         rustfmt_release,
         llvm_tools_release,
+        lldb_release,
 
         input,
         output,
@@ -248,6 +253,7 @@ fn main() {
         clippy_version: None,
         rustfmt_version: None,
         llvm_tools_version: None,
+        lldb_version: None,
 
         rust_git_commit_hash: None,
         cargo_git_commit_hash: None,
@@ -255,6 +261,7 @@ fn main() {
         clippy_git_commit_hash: None,
         rustfmt_git_commit_hash: None,
         llvm_tools_git_commit_hash: None,
+        lldb_git_commit_hash: None,
     }.build();
 }
 
@@ -266,6 +273,7 @@ impl Builder {
         self.clippy_version = self.version("clippy", "x86_64-unknown-linux-gnu");
         self.rustfmt_version = self.version("rustfmt", "x86_64-unknown-linux-gnu");
         self.llvm_tools_version = self.version("llvm-tools", "x86_64-unknown-linux-gnu");
+        self.lldb_version = self.version("lldb", "x86_64-unknown-linux-gnu");
 
         self.rust_git_commit_hash = self.git_commit_hash("rust", "x86_64-unknown-linux-gnu");
         self.cargo_git_commit_hash = self.git_commit_hash("cargo", "x86_64-unknown-linux-gnu");
@@ -274,6 +282,7 @@ impl Builder {
         self.rustfmt_git_commit_hash = self.git_commit_hash("rustfmt", "x86_64-unknown-linux-gnu");
         self.llvm_tools_git_commit_hash = self.git_commit_hash("llvm-tools",
                                                                "x86_64-unknown-linux-gnu");
+        self.lldb_git_commit_hash = self.git_commit_hash("lldb", "x86_64-unknown-linux-gnu");
 
         self.digest_and_sign();
         let manifest = self.build_manifest();
@@ -312,11 +321,13 @@ impl Builder {
         self.package("rustfmt-preview", &mut manifest.pkg, HOSTS);
         self.package("rust-analysis", &mut manifest.pkg, TARGETS);
         self.package("llvm-tools-preview", &mut manifest.pkg, TARGETS);
+        self.package("lldb-preview", &mut manifest.pkg, TARGETS);
 
         let clippy_present = manifest.pkg.contains_key("clippy-preview");
         let rls_present = manifest.pkg.contains_key("rls-preview");
         let rustfmt_present = manifest.pkg.contains_key("rustfmt-preview");
         let llvm_tools_present = manifest.pkg.contains_key("llvm-tools-preview");
+        let lldb_present = manifest.pkg.contains_key("lldb-preview");
 
         if rls_present {
             manifest.renames.insert("rls".to_owned(), Rename { to: "rls-preview".to_owned() });
@@ -383,6 +394,12 @@ impl Builder {
                     target: host.to_string(),
                 });
             }
+            if lldb_present {
+                extensions.push(Component {
+                    pkg: "lldb-preview".to_string(),
+                    target: host.to_string(),
+                });
+            }
             extensions.push(Component {
                 pkg: "rust-analysis".to_string(),
                 target: host.to_string(),
@@ -496,6 +513,8 @@ impl Builder {
             format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target)
         } else if component == "llvm-tools" || component == "llvm-tools-preview" {
             format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target)
+        } else if component == "lldb" || component == "lldb-preview" {
+            format!("lldb-{}-{}.tar.gz", self.lldb_release, target)
         } else {
             format!("{}-{}-{}.tar.gz", component, self.rust_release, target)
         }
@@ -512,6 +531,8 @@ impl Builder {
             &self.rustfmt_version
         } else if component == "llvm-tools" || component == "llvm-tools-preview" {
             &self.llvm_tools_version
+        } else if component == "lldb" || component == "lldb-preview" {
+            &self.lldb_version
         } else {
             &self.rust_version
         }
@@ -528,6 +549,8 @@ impl Builder {
             &self.rustfmt_git_commit_hash
         } else if component == "llvm-tools" || component == "llvm-tools-preview" {
             &self.llvm_tools_git_commit_hash
+        } else if component == "lldb" || component == "lldb-preview" {
+            &self.lldb_git_commit_hash
         } else {
             &self.rust_git_commit_hash
         }

src/tools/clang

@@ -0,0 +1 @@
+Subproject commit 2a284a70e26997273c296afe06586ffdf3a142fd

src/tools/lldb

@@ -0,0 +1 @@
+Subproject commit 3dbe998969d457c5cef245f61b48bdaed0f5c059

src/tools/rust-installer

@@ -1 +1 @@
-Subproject commit 89414e44dc94844888e59c08bc31dcccb1792800
+Subproject commit 27dec6cae3a8132d8a073aad6775425c85095c99

src/tools/tidy/src/lib.rs

@@ -68,12 +68,14 @@ fn filter_dirs(path: &Path) -> bool {
         "src/vendor",
         "src/rt/hoedown",
         "src/tools/cargo",
+        "src/tools/clang",
         "src/tools/rls",
         "src/tools/clippy",
         "src/tools/rust-installer",
         "src/tools/rustfmt",
         "src/tools/miri",
         "src/tools/lld",
+        "src/tools/lldb",
         "src/librustc/mir/interpret",
         "src/librustc_mir/interpret",
         "src/target",