Auto merge of #52716 - tromey:rustup-lldb, r=alexcrichton

Add lldb to the build

This optionally adds lldb (and clang, which it needs) to the build.

Because rust uses LLVM 7, and because clang 7 is not yet released, a
recent git master version of clang is used.

The lldb that is used includes the Rust plugin.

lldb is only built when asked for, or when doing a nightly build on
macOS.  Only macOS is done for now due to difficulties with the Python
dependency.

.gitmodules

@@ -56,3 +56,11 @@
 [submodule "src/libbacktrace"]
 	path = src/libbacktrace
 	url = https://github.com/rust-lang-nursery/libbacktrace
+[submodule "src/tools/lldb"]
+	path = src/tools/lldb
+	url = https://github.com/rust-lang-nursery/lldb/
+	branch = rust-release-70
+[submodule "src/tools/clang"]
+	path = src/tools/clang
+	url = https://github.com/rust-lang-nursery/clang/
+	branch = release_70

.travis.yml

@@ -30,7 +30,7 @@ matrix:
 
     - env: >
         RUST_CHECK_TARGET=dist
-        RUST_CONFIGURE_ARGS="--enable-extended --enable-profiler"
+        RUST_CONFIGURE_ARGS="--enable-extended --enable-profiler --enable-lldb"
         SRC=.
         DEPLOY_ALT=1
         RUSTC_RETRY_LINKER_ON_SEGFAULT=1
@@ -85,7 +85,7 @@ matrix:
     # OSX 10.7 and `xcode7` is the latest Xcode able to compile LLVM for 10.7.
     - env: >
         RUST_CHECK_TARGET=dist
-        RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-full-tools --enable-profiler"
+        RUST_CONFIGURE_ARGS="--build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb"
         SRC=.
         DEPLOY=1
         RUSTC_RETRY_LINKER_ON_SEGFAULT=1
@@ -99,7 +99,7 @@ matrix:
 
     - env: >
         RUST_CHECK_TARGET=dist
-        RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler"
+        RUST_CONFIGURE_ARGS="--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --enable-lldb"
         SRC=.
         DEPLOY=1
         RUSTC_RETRY_LINKER_ON_SEGFAULT=1
@@ -233,7 +233,8 @@ install:
         osx)
           if [[ "$RUST_CHECK_TARGET" == dist ]]; then
             travis_retry brew update &&
-            travis_retry brew install xz;
+            travis_retry brew install xz &&
+            travis_retry brew install swig;
           fi &&
           travis_retry curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin &&
             chmod +x /usr/local/bin/sccache &&

config.toml.example

@@ -354,6 +354,10 @@
 # sysroot.
 #llvm-tools = false
 
+# Indicates whether LLDB will be made available in the sysroot.
+# This is only built if LLVM is also being built.
+#lldb = false
+
 # Whether to deny warnings in crates
 #deny-warnings = true
 

src/Cargo.lock

@@ -1015,7 +1015,7 @@ name = "installer"
 version = "0.0.0"
 dependencies = [
  "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "flate2 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "rayon 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",

src/bootstrap/bootstrap.py

@@ -721,6 +721,10 @@ def update_submodules(self):
                 config = self.get_toml('lld')
                 if config is None or config == 'false':
                     continue
+            if module.endswith("lldb") or module.endswith("clang"):
+                config = self.get_toml('lldb')
+                if config is None or config == 'false':
+                    continue
             check = self.check_submodule(module, slow_submodules)
             filtered_submodules.append((module, check))
             submodules_names.append(module)

src/bootstrap/builder.rs

@@ -461,6 +461,7 @@ impl<'a> Builder<'a> {
                 dist::Rustfmt,
                 dist::Clippy,
                 dist::LlvmTools,
+                dist::Lldb,
                 dist::Extended,
                 dist::HashSign
             ),

src/bootstrap/config.rs

@@ -87,6 +87,7 @@ pub struct Config {
     pub llvm_link_jobs: Option<u32>,
 
     pub lld_enabled: bool,
+    pub lldb_enabled: bool,
     pub llvm_tools_enabled: bool,
 
     // rust codegen options
@@ -310,6 +311,7 @@ struct Rust {
     codegen_backends_dir: Option<String>,
     wasm_syscall: Option<bool>,
     lld: Option<bool>,
+    lldb: Option<bool>,
     llvm_tools: Option<bool>,
     deny_warnings: Option<bool>,
     backtrace_on_ice: Option<bool>,
@@ -538,6 +540,7 @@ impl Config {
             }
             set(&mut config.wasm_syscall, rust.wasm_syscall);
             set(&mut config.lld_enabled, rust.lld);
+            set(&mut config.lldb_enabled, rust.lldb);
             set(&mut config.llvm_tools_enabled, rust.llvm_tools);
             config.rustc_parallel_queries = rust.experimental_parallel_queries.unwrap_or(false);
             config.rustc_default_linker = rust.default_linker.clone();

src/bootstrap/configure.py

@@ -68,6 +68,7 @@ def v(*args):
 o("profiler", "build.profiler", "build the profiler runtime")
 o("emscripten", None, "compile the emscripten backend as well as LLVM")
 o("full-tools", None, "enable all tools")
+o("lldb", "rust.lldb", "build lldb")
 
 # Optimization and debugging options. These may be overridden by the release
 # channel, etc.
@@ -350,7 +351,7 @@ def set(key, value):
 # all the various comments and whatnot.
 #
 # Note that the `target` section is handled separately as we'll duplicate it
-# per configure dtarget, so there's a bit of special handling for that here.
+# per configured target, so there's a bit of special handling for that here.
 sections = {}
 cur_section = None
 sections[None] = []

src/bootstrap/dist.rs

@@ -47,6 +47,8 @@ pub fn pkgname(builder: &Builder, component: &str) -> String {
         format!("{}-{}", component, builder.rustfmt_package_vers())
     } else if component == "llvm-tools" {
         format!("{}-{}", component, builder.llvm_tools_package_vers())
+    } else if component == "lldb" {
+        format!("{}-{}", component, builder.lldb_package_vers())
     } else {
         assert!(component.starts_with("rust"));
         format!("{}-{}", component, builder.rust_package_vers())
@@ -1396,6 +1398,7 @@ impl Step for Extended {
         let rls_installer = builder.ensure(Rls { stage, target });
         let llvm_tools_installer = builder.ensure(LlvmTools { stage, target });
         let clippy_installer = builder.ensure(Clippy { stage, target });
+        let lldb_installer = builder.ensure(Lldb { target });
         let mingw_installer = builder.ensure(Mingw { host: target });
         let analysis_installer = builder.ensure(Analysis {
             compiler: builder.compiler(stage, self.host),
@@ -1435,6 +1438,7 @@ impl Step for Extended {
         tarballs.extend(clippy_installer.clone());
         tarballs.extend(rustfmt_installer.clone());
         tarballs.extend(llvm_tools_installer.clone());
+        tarballs.extend(lldb_installer.clone());
         tarballs.push(analysis_installer);
         tarballs.push(std_installer);
         if builder.config.docs {
@@ -1869,6 +1873,7 @@ impl Step for HashSign {
         cmd.arg(builder.package_vers(&builder.release_num("clippy")));
         cmd.arg(builder.package_vers(&builder.release_num("rustfmt")));
         cmd.arg(builder.llvm_tools_package_vers());
+        cmd.arg(builder.lldb_package_vers());
         cmd.arg(addr);
 
         builder.create_dir(&distdir(builder));
@@ -1963,3 +1968,121 @@ impl Step for LlvmTools {
         Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
     }
 }
+
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct Lldb {
+    pub target: Interned<String>,
+}
+
+impl Step for Lldb {
+    type Output = Option<PathBuf>;
+    const ONLY_HOSTS: bool = true;
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun) -> ShouldRun {
+        run.path("src/tools/lldb")
+    }
+
+    fn make_run(run: RunConfig) {
+        run.builder.ensure(Lldb {
+            target: run.target,
+        });
+    }
+
+    fn run(self, builder: &Builder) -> Option<PathBuf> {
+        let target = self.target;
+
+        if builder.config.dry_run {
+            return None;
+        }
+
+        let bindir = builder
+            .llvm_out(target)
+            .join("bin");
+        let lldb_exe = bindir.join(exe("lldb", &target));
+        if !lldb_exe.exists() {
+            return None;
+        }
+
+        builder.info(&format!("Dist Lldb ({})", target));
+        let src = builder.src.join("src/tools/lldb");
+        let name = pkgname(builder, "lldb");
+
+        let tmp = tmpdir(builder);
+        let image = tmp.join("lldb-image");
+        drop(fs::remove_dir_all(&image));
+
+        // Prepare the image directory
+        let dst = image.join("bin");
+        t!(fs::create_dir_all(&dst));
+        for program in &["lldb", "lldb-argdumper", "lldb-mi", "lldb-server"] {
+            let exe = bindir.join(exe(program, &target));
+            builder.install(&exe, &dst, 0o755);
+        }
+
+        // The libraries.
+        let libdir = builder.llvm_out(target).join("lib");
+        let dst = image.join("lib");
+        t!(fs::create_dir_all(&dst));
+        for entry in t!(fs::read_dir(&libdir)) {
+            // let entry = t!(entry);
+            let entry = entry.unwrap();
+            if let Ok(name) = entry.file_name().into_string() {
+                if name.starts_with("liblldb.") && !name.ends_with(".a") {
+                    if t!(entry.file_type()).is_symlink() {
+                        builder.copy_to_folder(&entry.path(), &dst);
+                    } else {
+                       builder.install(&entry.path(), &dst, 0o755);
+                    }
+                }
+            }
+        }
+
+        // The lldb scripts might be installed in lib/python$version
+        // or in lib64/python$version.  If lib64 exists, use it;
+        // otherwise lib.
+        let libdir = builder.llvm_out(target).join("lib64");
+        let (libdir, libdir_name) = if libdir.exists() {
+            (libdir, "lib64")
+        } else {
+            (builder.llvm_out(target).join("lib"), "lib")
+        };
+        for entry in t!(fs::read_dir(&libdir)) {
+            let entry = t!(entry);
+            if let Ok(name) = entry.file_name().into_string() {
+                if name.starts_with("python") {
+                    let dst = image.join(libdir_name)
+                        .join(entry.file_name());
+                    t!(fs::create_dir_all(&dst));
+                    builder.cp_r(&entry.path(), &dst);
+                    break;
+                }
+            }
+        }
+
+        // Prepare the overlay
+        let overlay = tmp.join("lldb-overlay");
+        drop(fs::remove_dir_all(&overlay));
+        builder.create_dir(&overlay);
+        builder.install(&src.join("LICENSE.TXT"), &overlay, 0o644);
+        builder.create(&overlay.join("version"), &builder.lldb_vers());
+
+        // Generate the installer tarball
+        let mut cmd = rust_installer(builder);
+        cmd.arg("generate")
+            .arg("--product-name=Rust")
+            .arg("--rel-manifest-dir=rustlib")
+            .arg("--success-message=lldb-installed.")
+            .arg("--image-dir").arg(&image)
+            .arg("--work-dir").arg(&tmpdir(builder))
+            .arg("--output-dir").arg(&distdir(builder))
+            .arg("--non-installed-overlay").arg(&overlay)
+            .arg(format!("--package-name={}-{}", name, target))
+            .arg("--legacy-manifest-dirs=rustlib,cargo")
+            .arg("--component-name=lldb-preview");
+
+
+        builder.run(&mut cmd);
+        Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target)))
+    }
+}

src/bootstrap/lib.rs

@@ -151,6 +151,11 @@ use std::process::{self, Command};
 use std::slice;
 use std::str;
 
+#[cfg(unix)]
+use std::os::unix::fs::symlink as symlink_file;
+#[cfg(windows)]
+use std::os::windows::fs::symlink_file;
+
 use build_helper::{run_silent, run_suppressed, try_run_silent, try_run_suppressed, output, mtime};
 use filetime::FileTime;
 
@@ -1005,6 +1010,14 @@ impl Build {
         self.rust_version()
     }
 
+    fn lldb_package_vers(&self) -> String {
+        self.package_vers(&self.rust_version())
+    }
+
+    fn lldb_vers(&self) -> String {
+        self.rust_version()
+    }
+
     /// Returns the `version` string associated with this compiler for Rust
     /// itself.
     ///
@@ -1123,20 +1136,24 @@ impl Build {
     pub fn copy(&self, src: &Path, dst: &Path) {
         if self.config.dry_run { return; }
         let _ = fs::remove_file(&dst);
-        // Attempt to "easy copy" by creating a hard link (symlinks don't work on
-        // windows), but if that fails just fall back to a slow `copy` operation.
-        if let Ok(()) = fs::hard_link(src, dst) {
-            return
-        }
-        if let Err(e) = fs::copy(src, dst) {
-            panic!("failed to copy `{}` to `{}`: {}", src.display(),
-                dst.display(), e)
+        let metadata = t!(src.symlink_metadata());
+        if metadata.file_type().is_symlink() {
+            let link = t!(fs::read_link(src));
+            t!(symlink_file(link, dst));
+        } else if let Ok(()) = fs::hard_link(src, dst) {
+            // Attempt to "easy copy" by creating a hard link
+            // (symlinks don't work on windows), but if that fails
+            // just fall back to a slow `copy` operation.
+        } else {
+            if let Err(e) = fs::copy(src, dst) {
+                panic!("failed to copy `{}` to `{}`: {}", src.display(),
+                       dst.display(), e)
+            }
+            t!(fs::set_permissions(dst, metadata.permissions()));
+            let atime = FileTime::from_last_access_time(&metadata);
+            let mtime = FileTime::from_last_modification_time(&metadata);
+            t!(filetime::set_file_times(dst, atime, mtime));
         }
-        let metadata = t!(src.metadata());
-        t!(fs::set_permissions(dst, metadata.permissions()));
-        let atime = FileTime::from_last_access_time(&metadata);
-        let mtime = FileTime::from_last_modification_time(&metadata);
-        t!(filetime::set_file_times(dst, atime, mtime));
     }
 
     /// Search-and-replaces within a file. (Not maximally efficiently: allocates a

src/bootstrap/native.rs

@@ -149,7 +149,6 @@ impl Step for Llvm {
            .define("WITH_POLLY", "OFF")
            .define("LLVM_ENABLE_TERMINFO", "OFF")
            .define("LLVM_ENABLE_LIBEDIT", "OFF")
-           .define("LLVM_ENABLE_LIBXML2", "OFF")
            .define("LLVM_PARALLEL_COMPILE_JOBS", builder.jobs().to_string())
            .define("LLVM_TARGET_ARCH", target.split('-').next().unwrap())
            .define("LLVM_DEFAULT_TARGET_TRIPLE", target);
@@ -163,19 +162,22 @@ impl Step for Llvm {
         cfg.define("LLVM_OCAML_INSTALL_PATH",
             env::var_os("LLVM_OCAML_INSTALL_PATH").unwrap_or_else(|| "usr/lib/ocaml".into()));
 
+        let want_lldb = builder.config.lldb_enabled && !self.emscripten;
+
         // This setting makes the LLVM tools link to the dynamic LLVM library,
         // which saves both memory during parallel links and overall disk space
         // for the tools.  We don't distribute any of those tools, so this is
         // just a local concern.  However, it doesn't work well everywhere.
         //
         // If we are shipping llvm tools then we statically link them LLVM
         if (target.contains("linux-gnu") || target.contains("apple-darwin")) &&
-            !builder.config.llvm_tools_enabled {
+            !builder.config.llvm_tools_enabled &&
+            !want_lldb {
                 cfg.define("LLVM_LINK_LLVM_DYLIB", "ON");
         }
 
         // For distribution we want the LLVM tools to be *statically* linked to libstdc++
-        if builder.config.llvm_tools_enabled {
+        if builder.config.llvm_tools_enabled || want_lldb {
             if !target.contains("windows") {
                 if target.contains("apple") {
                     cfg.define("CMAKE_EXE_LINKER_FLAGS", "-static-libstdc++");
@@ -196,6 +198,17 @@ impl Step for Llvm {
             cfg.define("LLVM_BUILD_32_BITS", "ON");
         }
 
+        if want_lldb {
+            cfg.define("LLVM_EXTERNAL_CLANG_SOURCE_DIR", builder.src.join("src/tools/clang"));
+            cfg.define("LLVM_EXTERNAL_LLDB_SOURCE_DIR", builder.src.join("src/tools/lldb"));
+            // For the time being, disable code signing.
+            cfg.define("LLDB_CODESIGN_IDENTITY", "");
+        } else {
+            // LLDB requires libxml2; but otherwise we want it to be disabled.
+            // See https://github.com/rust-lang/rust/pull/50104
+            cfg.define("LLVM_ENABLE_LIBXML2", "OFF");
+        }
+
         if let Some(num_linkers) = builder.config.llvm_link_jobs {
             if num_linkers > 0 {
                 cfg.define("LLVM_PARALLEL_LINK_JOBS", num_linkers.to_string());