Auto merge of #42105 - Mark-Simulacrum:rollup, r=Mark-Simulacrum

Rollup of 17 pull requests

- Successful merges: #41870, #41910, #41958, #41971, #42006, #42024, #42037, #42056, #42067, #42070, #42079, #42080, #42082, #42089, #42092, #42096, #42100
- Failed merges:

src/bootstrap/bootstrap.py

@@ -398,13 +398,14 @@ def run(self, args, env=None, cwd=None):
             sys.exit(ret)
 
     def output(self, args, env=None, cwd=None):
+        default_encoding = sys.getdefaultencoding()
         proc = subprocess.Popen(args, stdout=subprocess.PIPE, env=env, cwd=cwd)
         (out, err) = proc.communicate()
         ret = proc.wait()
         if ret != 0:
             print(out)
             sys.exit(ret)
-        return out
+        return out.decode(default_encoding)
 
     def build_triple(self):
         default_encoding = sys.getdefaultencoding()
@@ -570,10 +571,10 @@ def update_submodules(self):
 
         for submod in submodules:
             path, status = submod
-            if path.endswith(b"llvm") and \
+            if path.endswith('llvm') and \
                 (self.get_toml('llvm-config') or self.get_mk('CFG_LLVM_ROOT')):
                 continue
-            if path.endswith(b"jemalloc") and \
+            if path.endswith('jemalloc') and \
                 (self.get_toml('jemalloc') or self.get_mk('CFG_JEMALLOC_ROOT')):
                 continue
             submod_path = os.path.join(self.rust_root, path)

src/bootstrap/install.rs

@@ -21,83 +21,110 @@ use std::process::Command;
 use Build;
 use dist::{sanitize_sh, tmpdir};
 
-/// Installs everything.
-pub fn install(build: &Build, stage: u32, host: &str) {
-    let prefix_default = PathBuf::from("/usr/local");
-    let sysconfdir_default = PathBuf::from("/etc");
-    let docdir_default = PathBuf::from("share/doc/rust");
-    let bindir_default = PathBuf::from("bin");
-    let libdir_default = PathBuf::from("lib");
-    let mandir_default = PathBuf::from("share/man");
-    let prefix = build.config.prefix.as_ref().unwrap_or(&prefix_default);
-    let sysconfdir = build.config.sysconfdir.as_ref().unwrap_or(&sysconfdir_default);
-    let docdir = build.config.docdir.as_ref().unwrap_or(&docdir_default);
-    let bindir = build.config.bindir.as_ref().unwrap_or(&bindir_default);
-    let libdir = build.config.libdir.as_ref().unwrap_or(&libdir_default);
-    let mandir = build.config.mandir.as_ref().unwrap_or(&mandir_default);
-
-    let sysconfdir = prefix.join(sysconfdir);
-    let docdir = prefix.join(docdir);
-    let bindir = prefix.join(bindir);
-    let libdir = prefix.join(libdir);
-    let mandir = prefix.join(mandir);
-
-    let destdir = env::var_os("DESTDIR").map(PathBuf::from);
-
-    let prefix = add_destdir(&prefix, &destdir);
-    let sysconfdir = add_destdir(&sysconfdir, &destdir);
-    let docdir = add_destdir(&docdir, &destdir);
-    let bindir = add_destdir(&bindir, &destdir);
-    let libdir = add_destdir(&libdir, &destdir);
-    let mandir = add_destdir(&mandir, &destdir);
-
-    let empty_dir = build.out.join("tmp/empty_dir");
-    t!(fs::create_dir_all(&empty_dir));
-    if build.config.docs {
-        install_sh(&build, "docs", "rust-docs", &build.rust_package_vers(),
-                   stage, host, &prefix, &sysconfdir, &docdir, &bindir, &libdir,
-                   &mandir, &empty_dir);
-    }
+pub struct Installer<'a> {
+    build: &'a Build,
+    prefix: PathBuf,
+    sysconfdir: PathBuf,
+    docdir: PathBuf,
+    bindir: PathBuf,
+    libdir: PathBuf,
+    mandir: PathBuf,
+}
 
-    for target in build.config.target.iter() {
-        install_sh(&build, "std", "rust-std", &build.rust_package_vers(),
-                   stage, target, &prefix, &sysconfdir, &docdir, &bindir, &libdir,
-                   &mandir, &empty_dir);
-    }
+impl<'a> Installer<'a> {
+    pub fn new(build: &'a Build) -> Installer<'a> {
+        let prefix_default = PathBuf::from("/usr/local");
+        let sysconfdir_default = PathBuf::from("/etc");
+        let docdir_default = PathBuf::from("share/doc/rust");
+        let bindir_default = PathBuf::from("bin");
+        let libdir_default = PathBuf::from("lib");
+        let mandir_default = PathBuf::from("share/man");
+        let prefix = build.config.prefix.as_ref().unwrap_or(&prefix_default);
+        let sysconfdir = build.config.sysconfdir.as_ref().unwrap_or(&sysconfdir_default);
+        let docdir = build.config.docdir.as_ref().unwrap_or(&docdir_default);
+        let bindir = build.config.bindir.as_ref().unwrap_or(&bindir_default);
+        let libdir = build.config.libdir.as_ref().unwrap_or(&libdir_default);
+        let mandir = build.config.mandir.as_ref().unwrap_or(&mandir_default);
+
+        let sysconfdir = prefix.join(sysconfdir);
+        let docdir = prefix.join(docdir);
+        let bindir = prefix.join(bindir);
+        let libdir = prefix.join(libdir);
+        let mandir = prefix.join(mandir);
+
+        let destdir = env::var_os("DESTDIR").map(PathBuf::from);
 
-    if build.config.extended {
-        install_sh(&build, "cargo", "cargo", &build.cargo_package_vers(),
-                   stage, host, &prefix, &sysconfdir, &docdir, &bindir, &libdir,
-                   &mandir, &empty_dir);
-        install_sh(&build, "rls", "rls", &build.rls_package_vers(),
-                   stage, host, &prefix, &sysconfdir, &docdir, &bindir, &libdir,
-                   &mandir, &empty_dir);
+        let prefix = add_destdir(&prefix, &destdir);
+        let sysconfdir = add_destdir(&sysconfdir, &destdir);
+        let docdir = add_destdir(&docdir, &destdir);
+        let bindir = add_destdir(&bindir, &destdir);
+        let libdir = add_destdir(&libdir, &destdir);
+        let mandir = add_destdir(&mandir, &destdir);
+
+        Installer {
+            build,
+            prefix,
+            sysconfdir,
+            docdir,
+            bindir,
+            libdir,
+            mandir,
+        }
     }
 
-    install_sh(&build, "rustc", "rustc", &build.rust_package_vers(),
-               stage, host, &prefix, &sysconfdir, &docdir, &bindir, &libdir,
-               &mandir, &empty_dir);
+    /// Installs everything.
+    pub fn install(&self, stage: u32, host: &str) {
+        let empty_dir = self.build.out.join("tmp/empty_dir");
+        t!(fs::create_dir_all(&empty_dir));
 
-    t!(fs::remove_dir_all(&empty_dir));
-}
+        if self.build.config.docs {
+            self.install_sh("docs", "rust-docs", &self.build.rust_package_vers(),
+                            stage, Some(host), &empty_dir);
+        }
 
-fn install_sh(build: &Build, package: &str, name: &str, version: &str, stage: u32, host: &str,
-              prefix: &Path, sysconfdir: &Path, docdir: &Path, bindir: &Path, libdir: &Path,
-              mandir: &Path, empty_dir: &Path) {
-    println!("Install {} stage{} ({})", package, stage, host);
-    let package_name = format!("{}-{}-{}", name, version, host);
-
-    let mut cmd = Command::new("sh");
-    cmd.current_dir(empty_dir)
-       .arg(sanitize_sh(&tmpdir(build).join(&package_name).join("install.sh")))
-       .arg(format!("--prefix={}", sanitize_sh(prefix)))
-       .arg(format!("--sysconfdir={}", sanitize_sh(sysconfdir)))
-       .arg(format!("--docdir={}", sanitize_sh(docdir)))
-       .arg(format!("--bindir={}", sanitize_sh(bindir)))
-       .arg(format!("--libdir={}", sanitize_sh(libdir)))
-       .arg(format!("--mandir={}", sanitize_sh(mandir)))
-       .arg("--disable-ldconfig");
-    build.run(&mut cmd);
+        for target in self.build.config.target.iter() {
+            self.install_sh("std", "rust-std", &self.build.rust_package_vers(),
+                            stage, Some(target), &empty_dir);
+        }
+
+        if self.build.config.extended {
+            self.install_sh("cargo", "cargo", &self.build.cargo_package_vers(),
+                            stage, Some(host), &empty_dir);
+            self.install_sh("rls", "rls", &self.build.rls_package_vers(),
+                            stage, Some(host), &empty_dir);
+            self.install_sh("analysis", "rust-analysis", &self.build.rust_package_vers(),
+                            stage, Some(host), &empty_dir);
+            self.install_sh("src", "rust-src", &self.build.rust_package_vers(),
+                            stage, None, &empty_dir);
+        }
+
+        self.install_sh("rustc", "rustc", &self.build.rust_package_vers(),
+                        stage, Some(host), &empty_dir);
+
+        t!(fs::remove_dir_all(&empty_dir));
+    }
+
+    fn install_sh(&self, package: &str, name: &str, version: &str,
+                  stage: u32, host: Option<&str>,  empty_dir: &Path) {
+        println!("Install {} stage{} ({:?})", package, stage, host);
+        let package_name = if let Some(host) = host {
+            format!("{}-{}-{}", name, version, host)
+        } else {
+            format!("{}-{}", name, version)
+        };
+
+        let mut cmd = Command::new("sh");
+        cmd.current_dir(empty_dir)
+           .arg(sanitize_sh(&tmpdir(self.build).join(&package_name).join("install.sh")))
+           .arg(format!("--prefix={}", sanitize_sh(&self.prefix)))
+           .arg(format!("--sysconfdir={}", sanitize_sh(&self.sysconfdir)))
+           .arg(format!("--docdir={}", sanitize_sh(&self.docdir)))
+           .arg(format!("--bindir={}", sanitize_sh(&self.bindir)))
+           .arg(format!("--libdir={}", sanitize_sh(&self.libdir)))
+           .arg(format!("--mandir={}", sanitize_sh(&self.mandir)))
+           .arg("--disable-ldconfig");
+        self.build.run(&mut cmd);
+    }
 }
 
 fn add_destdir(path: &Path, destdir: &Option<PathBuf>) -> PathBuf {

src/bootstrap/step.rs

@@ -761,7 +761,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .run(move |s| dist::rls(build, s.stage, s.target));
     rules.dist("install", "path/to/nowhere")
          .dep(|s| s.name("default:dist"))
-         .run(move |s| install::install(build, s.stage, s.target));
+         .run(move |s| install::Installer::new(build).install(s.stage, s.target));
     rules.dist("dist-cargo", "cargo")
          .host(true)
          .only_host_build(true)

src/ci/docker/README.md

@@ -16,6 +16,12 @@ for example:
 
 Images will output artifacts in an `obj` dir at the root of a repository.
 
+## Filesystem layout
+
+- Each directory, excluding `scripts` and `disabled`, corresponds to a docker image
+- `scripts` contains files shared by docker images
+- `disabled` contains images that are not build travis
+
 ## Cross toolchains
 
 A number of these images take quite a long time to compile as they're building

src/ci/docker/arm-android/Dockerfile

@@ -2,52 +2,44 @@ FROM ubuntu:16.04
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
+  ca-certificates \
+  cmake \
+  curl \
+  file \
   g++ \
+  git \
+  libssl-dev \
   make \
-  file \
-  curl \
-  ca-certificates \
+  pkg-config \
   python2.7 \
-  git \
-  cmake \
-  unzip \
   sudo \
-  xz-utils \
-  libssl-dev \
-  pkg-config
-
-RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
-    dpkg -i dumb-init_*.deb && \
-    rm dumb-init_*.deb
+  unzip \
+  xz-utils
 
-RUN curl -o /usr/local/bin/sccache \
-      https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-05-12-sccache-x86_64-unknown-linux-musl && \
-    chmod +x /usr/local/bin/sccache
+# dumb-init
+COPY scripts/dumb-init.sh /scripts/
+RUN sh /scripts/dumb-init.sh
 
-# Install NDK
-COPY install-ndk.sh /tmp
-RUN . /tmp/install-ndk.sh && \
-    download_ndk android-ndk-r13b-linux-x86_64.zip && \
-    make_standalone_toolchain arm 9 && \
-    remove_ndk
+# ndk
+COPY scripts/android-ndk.sh /scripts/
+RUN . /scripts/android-ndk.sh && \
+    download_and_make_toolchain android-ndk-r13b-linux-x86_64.zip arm 9
 
-# Install SDK
+# sdk
 RUN dpkg --add-architecture i386 && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
-  openjdk-9-jre-headless \
-  tzdata \
-  libstdc++6:i386 \
   libgl1-mesa-glx \
-  libpulse0
+  libpulse0 \
+  libstdc++6:i386 \
+  openjdk-9-jre-headless \
+  tzdata
 
-COPY install-sdk.sh /tmp
-RUN . /tmp/install-sdk.sh && \
-    download_sdk tools_r25.2.5-linux.zip && \
-    download_sysimage armeabi-v7a 18 && \
-    create_avd armeabi-v7a 18
+COPY scripts/android-sdk.sh /scripts/
+RUN . /scripts/android-sdk.sh && \
+    download_and_create_avd tools_r25.2.5-linux.zip armeabi-v7a 18
 
-# Setup env
+# env
 ENV PATH=$PATH:/android/sdk/tools
 ENV PATH=$PATH:/android/sdk/platform-tools
 
@@ -57,8 +49,12 @@ ENV RUST_CONFIGURE_ARGS \
       --target=$TARGETS \
       --arm-linux-androideabi-ndk=/android/ndk/arm-9
 
-ENV SCRIPT python2.7 ../x.py test --target $TARGETS --verbose
+ENV SCRIPT python2.7 ../x.py test --target $TARGETS
+
+# sccache
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
 
-# Entrypoint
-COPY start-emulator.sh /android/
-ENTRYPOINT ["/usr/bin/dumb-init", "--", "/android/start-emulator.sh"]
+# init
+COPY scripts/android-start-emulator.sh /scripts/
+ENTRYPOINT ["/usr/bin/dumb-init", "--", "/scripts/android-start-emulator.sh"]

src/ci/docker/armhf-gnu/Dockerfile

@@ -31,7 +31,7 @@ WORKDIR /build
 # The `vexpress_config` config file was a previously generated config file for
 # the kernel. This file was generated by running `make vexpress_defconfig`
 # followed by `make menuconfig` and then enabling the IPv6 protocol page.
-COPY vexpress_config /build/.config
+COPY armhf-gnu/vexpress_config /build/.config
 RUN curl https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.4.42.tar.xz | \
       tar xJf - && \
       cd /build/linux-4.4.42 && \
@@ -63,11 +63,11 @@ RUN curl http://cdimage.ubuntu.com/ubuntu-base/releases/16.04/release/ubuntu-bas
 
 # Copy over our init script, which starts up our test server and also a few
 # other misc tasks.
-COPY rcS rootfs/etc/init.d/rcS
+COPY armhf-gnu/rcS rootfs/etc/init.d/rcS
 RUN chmod +x rootfs/etc/init.d/rcS
 
 # Helper to quickly fill the entropy pool in the kernel.
-COPY addentropy.c /tmp/
+COPY armhf-gnu/addentropy.c /tmp/
 RUN arm-linux-gnueabihf-gcc addentropy.c -o rootfs/addentropy -static
 
 # TODO: What is this?!

src/ci/docker/cross/Dockerfile

@@ -32,10 +32,10 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 WORKDIR /tmp
 
-COPY build-rumprun.sh /tmp/
+COPY cross/build-rumprun.sh /tmp/
 RUN ./build-rumprun.sh
 
-COPY build-arm-musl.sh /tmp/
+COPY cross/build-arm-musl.sh /tmp/
 RUN ./build-arm-musl.sh
 
 # originally from