travis: Start uploading artifacts on commits

.travis.yml

@@ -13,20 +13,20 @@ osx_image: xcode8.2
 matrix:
   include:
     # Linux builders, all docker images
-    - env: IMAGE=arm-android
-    - env: IMAGE=cross
-    - env: IMAGE=dist-arm-unknown-linux-gnueabi
-    - env: IMAGE=dist-x86_64-unknown-freebsd
-    - env: IMAGE=i686-gnu
+    - env: IMAGE=arm-android DEPLOY=1
+    - env: IMAGE=cross DEPLOY=1
+    - env: IMAGE=dist-arm-unknown-linux-gnueabi DEPLOY=1
+    - env: IMAGE=dist-x86_64-unknown-freebsd DEPLOY=1
+    - env: IMAGE=i686-gnu DEPLOY=1
     - env: IMAGE=i686-gnu-nopt
-    - env: IMAGE=x86_64-gnu
+    - env: IMAGE=x86_64-gnu DEPLOY=1
     - env: IMAGE=x86_64-gnu-full-bootstrap
     - env: IMAGE=x86_64-gnu-aux
     - env: IMAGE=x86_64-gnu-debug
     - env: IMAGE=x86_64-gnu-nopt
     - env: IMAGE=x86_64-gnu-make
     - env: IMAGE=x86_64-gnu-llvm-3.7 ALLOW_PR=1 RUST_BACKTRACE=1
-    - env: IMAGE=x86_64-musl
+    - env: IMAGE=x86_64-musl DEPLOY=1
     - env: IMAGE=x86_64-gnu-distcheck
 
     # OSX builders
@@ -39,9 +39,10 @@ matrix:
         curl -L https://api.pub.build.mozilla.org/tooltool/sha512/d0025b286468cc5ada83b23d3fafbc936b9f190eaa7d4a981715b18e8e3bf720a7bcee7bfe758cfdeb8268857f6098fd52dcdd8818232692a30ce91039936596 |
           tar xJf - -C /usr/local/bin --strip-components=1
     - env: >
-        RUST_CHECK_TARGET=check
+        SCRIPT="./x.py test && ./x.py dist"
         RUST_CONFIGURE_ARGS=--build=i686-apple-darwin
         SRC=.
+        DEPLOY=1
       os: osx
       install: *osx_install_sccache
     - env: >
@@ -51,9 +52,10 @@ matrix:
       os: osx
       install: *osx_install_sccache
     - env: >
-        RUST_CHECK_TARGET=
+        RUST_CHECK_TARGET=dist
         RUST_CONFIGURE_ARGS=--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios
         SRC=.
+        DEPLOY=1
       os: osx
       install: *osx_install_sccache
 
@@ -91,3 +93,29 @@ notifications:
 cache:
   directories:
     - $HOME/docker
+
+before_deploy:
+  - mkdir -p deploy/$TRAVIS_COMMIT
+  - >
+      if [ "$TRAVIS_OS_NAME" == "osx" ]; then
+          cp build/dist/*.tar.gz deploy/$TRAVIS_COMMIT &&
+          find "deploy/$TRAVIS_COMMIT" -maxdepth 1 -type f -exec sh -c 'shasum -a 256 -b "{}" > "{}.sha256"' \;;
+      else
+          cp obj/build/dist/*.tar.gz deploy/$TRAVIS_COMMIT &&
+          find "deploy/$TRAVIS_COMMIT" -maxdepth 1 -type f -exec sh -c 'sha256sum -b "{}" > "{}.sha256"' \;;
+      fi
+
+deploy:
+  - provider: s3
+    bucket: rust-lang-ci
+    skip_cleanup: true
+    local_dir: deploy
+    upload_dir: rustc-builds
+    acl: public_read
+    region: us-east-1
+    access_key_id: AKIAIPQVNYF2T3DTYIWQ
+    secret_access_key:
+      secure: "FBqDqOTeIPMu6v/WYPf4CFSlh9rLRZGKVtpLa5KkyuOhXRTrnEzBduEtS8/FMIxdQImvurhSvxWvqRybMOi4qoVfjMqqpHAI7uBbidbrvAcJoHNsx6BgUNVCIoH6a0UsAjTUtm6/YPIpzbHoLZXPL0GrHPMk6Mu04qVSmcYNWn4="
+    on:
+      branch: auto
+      condition: $DEPLOY = 1

appveyor.yml

@@ -8,10 +8,12 @@ environment:
   # 32/64 bit MSVC
   - MSYS_BITS: 64
     RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
-    RUST_CHECK_TARGET: check
+    SCRIPT: python x.py test && python x.py dist
+    DEPLOY: 1
   - MSYS_BITS: 32
     RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
-    RUST_CHECK_TARGET: check
+    SCRIPT: python x.py test && python x.py dist
+    DEPLOY: 1
 
   # MSVC makefiles
   - MSYS_BITS: 64
@@ -50,10 +52,11 @@ environment:
   # too long on appveyor and this is tested by rustbuild below.
   - MSYS_BITS: 32
     RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
-    RUST_CHECK_TARGET: check
+    SCRIPT: python x.py test && python x.py dist
     MINGW_URL: https://s3.amazonaws.com/rust-lang-ci
     MINGW_ARCHIVE: i686-4.9.2-release-win32-dwarf-rt_v4-rev4.7z
     MINGW_DIR: mingw32
+    DEPLOY: 1
 
   - MSYS_BITS: 32
     RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --disable-rustbuild
@@ -63,11 +66,12 @@ environment:
     MINGW_DIR: mingw32
 
   - MSYS_BITS: 64
-    RUST_CHECK_TARGET: check
+    SCRIPT: python x.py test && python x.py dist
     RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
     MINGW_URL: https://s3.amazonaws.com/rust-lang-ci
     MINGW_ARCHIVE: x86_64-4.9.2-release-win32-seh-rt_v4-rev4.7z
     MINGW_DIR: mingw64
+    DEPLOY: 1
 
 clone_depth: 1
 build: false
@@ -123,6 +127,32 @@ branches:
   only:
     - auto
 
+before_deploy:
+  - ps: |
+        New-Item -Path deploy -ItemType directory
+        Get-ChildItem -Path build\dist -Filter '*.tar.gz' | Move-Item -Destination deploy
+        Get-FileHash .\deploy\* | ForEach-Object {
+          [io.file]::WriteAllText($_.Path + ".sha256", $_.Hash.ToLower() + "`n")
+        }
+        Get-ChildItem -Path deploy | Foreach-Object {
+          Push-AppveyorArtifact $_.FullName -FileName ${env:APPVEYOR_REPO_COMMIT}/$_
+        }
+
+deploy:
+  - provider: S3
+    skip_cleanup: true
+    access_key_id: AKIAIPQVNYF2T3DTYIWQ
+    secret_access_key:
+      secure: +11jsUNFTQ9dq5Ad1i2+PeUJaXluFJ0zIJAXESE1dFT3Kdjku4/eDdgyjgsB6GnV
+    bucket: rust-lang-ci
+    set_public: true
+    region: us-east-1
+    artifact: /.*\.(tar.gz|sha256)/
+    folder: rustc-builds
+    on:
+      branch: auto
+      DEPLOY: 1
+
 # init:
 #   - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
 # on_finish:

src/bootstrap/check.rs

@@ -457,6 +457,8 @@ fn krate_android(build: &Build,
 
         let output = output(Command::new("adb").arg("shell").arg(&program));
         println!("{}", output);
+
+        t!(fs::create_dir_all(build.out.join("tmp")));
         build.run(Command::new("adb")
                           .arg("pull")
                           .arg(&log)
@@ -516,6 +518,7 @@ pub fn android_copy_libs(build: &Build,
     }
 
     println!("Android copy libs to emulator ({})", target);
+    build.run(Command::new("adb").arg("wait-for-device"));
     build.run(Command::new("adb").arg("remount"));
     build.run(Command::new("adb").args(&["shell", "rm", "-r", ADB_TEST_DIR]));
     build.run(Command::new("adb").args(&["shell", "mkdir", ADB_TEST_DIR]));

src/bootstrap/compile.rs

@@ -16,7 +16,6 @@
 //! compiler. This module is also responsible for assembling the sysroot as it
 //! goes along from the output of the previous stage.
 
-use std::cmp;
 use std::collections::HashMap;
 use std::fs::{self, File};
 use std::path::{Path, PathBuf};
@@ -59,7 +58,7 @@ pub fn std(build: &Build, target: &str, compiler: &Compiler) {
     }
 
     build.run(&mut cargo);
-    update_mtime(&libstd_stamp(build, &compiler, target));
+    update_mtime(build, &libstd_stamp(build, &compiler, target));
 }
 
 /// Link all libstd rlibs/dylibs into the sysroot location.
@@ -145,7 +144,7 @@ pub fn test(build: &Build, target: &str, compiler: &Compiler) {
     cargo.arg("--manifest-path")
          .arg(build.src.join("src/rustc/test_shim/Cargo.toml"));
     build.run(&mut cargo);
-    update_mtime(&libtest_stamp(build, compiler, target));
+    update_mtime(build, &libtest_stamp(build, compiler, target));
 }
 
 /// Same as `std_link`, only for libtest
@@ -390,26 +389,39 @@ pub fn tool(build: &Build, stage: u32, host: &str, tool: &str) {
 }
 
 /// Updates the mtime of a stamp file if necessary, only changing it if it's
-/// older than some other file in the same directory.
+/// older than some other library file in the same directory.
 ///
 /// We don't know what file Cargo is going to output (because there's a hash in
 /// the file name) but we know where it's going to put it. We use this helper to
 /// detect changes to that output file by looking at the modification time for
 /// all files in a directory and updating the stamp if any are newer.
-fn update_mtime(path: &Path) {
-    let mut max = None;
-    if let Ok(entries) = path.parent().unwrap().join("deps").read_dir() {
-        for entry in entries.map(|e| t!(e)) {
-            if t!(entry.file_type()).is_file() {
-                let meta = t!(entry.metadata());
-                let time = FileTime::from_last_modification_time(&meta);
-                max = cmp::max(max, Some(time));
-            }
-        }
-    }
-
-    if !max.is_none() && max <= Some(mtime(path)) {
-        return
+///
+/// Note that we only consider Rust libraries as that's what we're interested in
+/// propagating changes from. Files like executables are tracked elsewhere.
+fn update_mtime(build: &Build, path: &Path) {
+    let entries = match path.parent().unwrap().join("deps").read_dir() {
+        Ok(entries) => entries,
+        Err(_) => return,
+    };
+    let files = entries.map(|e| t!(e)).filter(|e| t!(e.file_type()).is_file());
+    let files = files.filter(|e| {
+        let filename = e.file_name();
+        let filename = filename.to_str().unwrap();
+        filename.ends_with(".rlib") ||
+            filename.ends_with(".lib") ||
+            is_dylib(&filename)
+    });
+    let max = files.max_by_key(|entry| {
+        let meta = t!(entry.metadata());
+        FileTime::from_last_modification_time(&meta)
+    });
+    let max = match max {
+        Some(max) => max,
+        None => return,
+    };
+
+    if mtime(&max.path()) > mtime(path) {
+        build.verbose(&format!("updating {:?} as {:?} changed", path, max.path()));
+        t!(File::create(path));
     }
-    t!(File::create(path));
 }

src/bootstrap/dist.rs

@@ -357,14 +357,12 @@ pub fn analysis(build: &Build, compiler: &Compiler, target: &str) {
 pub fn rust_src(build: &Build) {
     println!("Dist src");
 
-    let plain_name = format!("rustc-{}-src", package_vers(build));
     let name = format!("rust-src-{}", package_vers(build));
     let image = tmpdir(build).join(format!("{}-image", name));
     let _ = fs::remove_dir_all(&image);
 
     let dst = image.join("lib/rustlib/src");
     let dst_src = dst.join("rust");
-    let plain_dst_src = dst.join(&plain_name);
     t!(fs::create_dir_all(&dst_src));
 
     // This is the set of root paths which will become part of the source package
@@ -444,7 +442,11 @@ pub fn rust_src(build: &Build) {
     build.run(&mut cmd);
 
     // Rename directory, so that root folder of tarball has the correct name
-    t!(fs::rename(&dst_src, &plain_dst_src));
+    let plain_name = format!("rustc-{}-src", package_vers(build));
+    let plain_dst_src = tmpdir(build).join(&plain_name);
+    let _ = fs::remove_dir_all(&plain_dst_src);
+    t!(fs::create_dir_all(&plain_dst_src));
+    cp_r(&dst_src, &plain_dst_src);
 
     // Create the version file
     write_file(&plain_dst_src.join("version"), build.version.as_bytes());
@@ -453,10 +455,11 @@ pub fn rust_src(build: &Build) {
     let mut cmd = Command::new("tar");
     cmd.arg("-czf").arg(sanitize_sh(&rust_src_location(build)))
        .arg(&plain_name)
-       .current_dir(&dst);
+       .current_dir(tmpdir(build));
     build.run(&mut cmd);
 
     t!(fs::remove_dir_all(&image));
+    t!(fs::remove_dir_all(&plain_dst_src));
 }
 
 fn install(src: &Path, dstdir: &Path, perms: u32) {

src/bootstrap/flags.rs

@@ -67,6 +67,7 @@ pub enum Subcommand {
     },
     Clean,
     Dist {
+        paths: Vec<PathBuf>,
         install: bool,
     },
 }
@@ -249,6 +250,7 @@ To learn more about a subcommand, run `./x.py <command> -h`
                 opts.optflag("", "install", "run installer as well");
                 m = parse(&opts);
                 Subcommand::Dist {
+                    paths: remaining_as_path(&m),
                     install: m.opt_present("install"),
                 }
             }

src/bootstrap/step.rs

@@ -615,7 +615,7 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
          .default(true)
          .only_host_build(true)
          .run(move |s| dist::analysis(build, &s.compiler(), s.target));
-    rules.dist("install", "src")
+    rules.dist("install", "path/to/nowhere")
          .dep(|s| s.name("default:dist"))
          .run(move |s| install::install(build, s.stage, s.target));
 
@@ -932,11 +932,11 @@ invalid rule dependency graph detected, was a rule added and maybe typo'd?
             Subcommand::Doc { ref paths } => (Kind::Doc, &paths[..]),
             Subcommand::Test { ref paths, test_args: _ } => (Kind::Test, &paths[..]),
             Subcommand::Bench { ref paths, test_args: _ } => (Kind::Bench, &paths[..]),
-            Subcommand::Dist { install } => {
+            Subcommand::Dist { ref paths, install } => {
                 if install {
                     return vec![self.sbuild.name("install")]
                 } else {
-                    (Kind::Dist, &[][..])
+                    (Kind::Dist, &paths[..])
                 }
             }
             Subcommand::Clean => panic!(),

src/ci/docker/arm-android/Dockerfile

@@ -48,4 +48,9 @@ ENV RUST_CONFIGURE_ARGS \
       --armv7-linux-androideabi-ndk=/android/ndk-arm-9 \
       --i686-linux-android-ndk=/android/ndk-x86-9 \
       --aarch64-linux-android-ndk=/android/ndk-aarch64
-ENV XPY_CHECK test --target arm-linux-androideabi
+
+# Just a smoke test in dist to see if this works for now, we should expand this
+# to all the targets above eventually.
+ENV SCRIPT \
+  python2.7 ../x.py test --target arm-linux-androideabi && \
+  python2.7 ../x.py dist --target arm-linux-androideabi

src/ci/docker/arm-android/start-emulator.sh

@@ -13,5 +13,4 @@ set -ex
 ANDROID_EMULATOR_FORCE_32BIT=true \
   nohup nohup emulator @arm-18 -no-window -partition-size 2047 \
   0<&- &>/dev/null &
-adb wait-for-device
 exec "$@"

src/ci/docker/cross/Dockerfile

@@ -56,7 +56,12 @@ ENV TARGETS=$TARGETS,wasm32-unknown-emscripten
 ENV RUST_CONFIGURE_ARGS \
       --target=$TARGETS \
       --enable-rustbuild
-ENV RUST_CHECK_TARGET ""
+
+# Just a smoke test in dist to see if this works for now, we should expand this
+# to all the targets above eventually.
+ENV SCRIPT \
+  python2.7 ../x.py build && \
+  python2.7 ../x.py dist --target wasm32-unknown-emscripten
 
 ENV AR_s390x_unknown_linux_gnu=s390x-linux-gnu-ar \
     CC_s390x_unknown_linux_gnu=s390x-linux-gnu-gcc \

src/ci/docker/dist-arm-unknown-linux-gnueabi/Dockerfile

@@ -24,7 +24,7 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 ENV RUST_CONFIGURE_ARGS --host=arm-unknown-linux-gnueabi
-ENV XPY_RUN \
-      dist \
-      --host arm-unknown-linux-gnueabi \
-      --target arm-unknown-linux-gnueabi
+ENV SCRIPT \
+      python2.7 ../x.py dist \
+        --host arm-unknown-linux-gnueabi \
+        --target arm-unknown-linux-gnueabi

src/ci/docker/dist-x86_64-unknown-freebsd/Dockerfile

@@ -32,7 +32,7 @@ ENV \
     CXX_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-g++
 
 ENV RUST_CONFIGURE_ARGS --host=x86_64-unknown-freebsd
-ENV XPY_RUN \
-      dist \
-      --host x86_64-unknown-freebsd \
-      --target x86_64-unknown-freebsd
+ENV SCRIPT \
+      python2.7 ../x.py dist \
+        --host x86_64-unknown-freebsd \
+        --target x86_64-unknown-freebsd

src/ci/docker/i686-gnu/Dockerfile

@@ -23,4 +23,4 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
-ENV RUST_CHECK_TARGET check
+ENV SCRIPT python2.7 ../x.py test && python2.7 ../x.py dist