Auto merge of rust-lang#117771 - tmandry:fuchsia-gha, r=<try>

Build Fuchsia in CI

This is very much in a draft state but I wanted to put it up now to get early feedback. It would also be nice if we could test it on actual CI builders.

Fittingly, it is failing right now due to discovering an ICE in clippy, probably more fallout from recent type system changes. Other recent regressions this would have caught include

- rust-lang#117455 and rust-lang#117493
- rust-lang#117602

This ends up not sharing very much at all with cargotest. Fuchsia has its own tool to manage checkouts and its own build system. What it requires is a fully "install"ed toolchain with a host and fuchsia target. We share logic from the dist-various-2 builder to build the fuchsia target.

Right now this runs clippy and skips linking a bunch of targets, since most issues we catch are in the frontend. In theory we could probably get the build CPU time down quite a bit with this approach, but right now some linked targets are creeping into the dependencies anyway and we don't have a good way of preventing that yet.

The approach is basically to get a checkout at a pinned commit and then run a [script](https://fuchsia-review.git.corp.google.com/c/fuchsia/+/943833/6/scripts/rust/build_fuchsia_from_rust_ci.sh) at a predetermined location. I would like to update that pin every few weeks. For now we cherry-pick some build changes but that's temporary. Partial checkouts are used to minimize clone time, but we don't filter out prebuilt packages.

r? `@Mark-Simulacrum`

Based on discussion in [this Zulip thread](https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra/topic/Putting.20Fuchsia.20in.20crater).

.github/workflows/ci.yml

@@ -288,6 +288,9 @@ jobs:
           - name: x86_64-gnu-aux
             os: ubuntu-20.04-4core-16gb
             env: {}
+          - name: x86_64-gnu-experimental
+            os: ubuntu-20.04-16core-64gb
+            env: {}
           - name: x86_64-gnu-debug
             os: ubuntu-20.04-8core-32gb
             env: {}
@@ -583,6 +586,9 @@ jobs:
             env:
               CODEGEN_BACKENDS: "llvm,cranelift"
             os: ubuntu-20.04-16core-64gb
+          - name: x86_64-gnu-experimental
+            os: ubuntu-20.04-16core-64gb
+            env: {}
     timeout-minutes: 600
     runs-on: "${{ matrix.os }}"
     steps:

src/bootstrap/mk/Makefile.in

@@ -52,6 +52,15 @@ check-aux:
 		src/tools/cargo \
 		src/tools/cargotest \
 		$(BOOTSTRAP_ARGS)
+check-fuchsia:
+	mkdir -p $(RUST_INSTALL_DIR)/etc
+	$(Q)$(BOOTSTRAP) install --stage 1 \
+		compiler/rustc \
+		library/std
+	$(Q)$(BOOTSTRAP) test --stage 2 \
+		src/tools/cargotest \
+		--test-args=fuchsia \
+		$(BOOTSTRAP_ARGS)
 dist:
 	$(Q)$(BOOTSTRAP) dist $(BOOTSTRAP_ARGS)
 distcheck:

src/bootstrap/src/core/build_steps/compile.rs

@@ -1711,7 +1711,7 @@ impl Step for Assemble {
             let dst_exe = exe("rust-lld", target_compiler.host);
             builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe));
             let self_contained_lld_dir = libdir_bin.join("gcc-ld");
-            t!(fs::create_dir(&self_contained_lld_dir));
+            t!(fs::create_dir_all(&self_contained_lld_dir));
             let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper {
                 compiler: build_compiler,
                 target: target_compiler.host,

src/ci/docker/host-x86_64/dist-various-2/Dockerfile

@@ -80,7 +80,7 @@ RUN env \
 
 WORKDIR /tmp
 COPY host-x86_64/dist-various-2/shared.sh /tmp/
-COPY host-x86_64/dist-various-2/build-fuchsia-toolchain.sh /tmp/
+COPY scripts/build-fuchsia-toolchain.sh /tmp/
 RUN /tmp/build-fuchsia-toolchain.sh
 COPY host-x86_64/dist-various-2/build-solaris-toolchain.sh /tmp/
 RUN /tmp/build-solaris-toolchain.sh x86_64  amd64   solaris-i386  pc

src/ci/docker/host-x86_64/x86_64-gnu-experimental/Dockerfile

@@ -0,0 +1,71 @@
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  build-essential \
+  gawk \
+  g++ \
+  make \
+  ninja-build \
+  file \
+  curl \
+  ca-certificates \
+  python3 \
+  git \
+  cmake \
+  libssl-dev \
+  sudo \
+  time \
+  xz-utils \
+  pkg-config \
+  unzip \
+  && rm -rf /var/lib/apt/lists/*
+
+# Copied from dist-various-2 Dockerfile.
+# TODO move to canonical triple and deduplicate this list
+ENV \
+    AR_x86_64_unknown_fuchsia=x86_64-unknown-fuchsia-ar \
+    CC_x86_64_unknown_fuchsia=x86_64-unknown-fuchsia-clang \
+    CFLAGS_x86_64_unknown_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \
+    CXX_x86_64_unknown_fuchsia=x86_64-unknown-fuchsia-clang++ \
+    CXXFLAGS_x86_64_unknown_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \
+    LDFLAGS_x86_64_unknown_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -L/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib" \
+    AR_x86_64_fuchsia=x86_64-unknown-fuchsia-ar \
+    CC_x86_64_fuchsia=x86_64-unknown-fuchsia-clang \
+    CFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \
+    CXX_x86_64_fuchsia=x86_64-unknown-fuchsia-clang++ \
+    CXXFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -I/usr/local/core-linux-amd64-fuchsia-sdk/pkg/fdio/include" \
+    LDFLAGS_x86_64_fuchsia="--target=x86_64-unknown-fuchsia --sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot -L/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib"
+
+WORKDIR /tmp
+COPY scripts/build-fuchsia-toolchain.sh /tmp/
+RUN /tmp/build-fuchsia-toolchain.sh
+
+# TODO move to canonical triple
+ENV CARGO_TARGET_X86_64_FUCHSIA_AR /usr/local/bin/llvm-ar
+ENV CARGO_TARGET_X86_64_FUCHSIA_RUSTFLAGS \
+-C link-arg=--sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot \
+-Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/sysroot/lib \
+-Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/x64/lib
+ENV CARGO_TARGET_AARCH64_FUCHSIA_AR /usr/local/bin/llvm-ar
+ENV CARGO_TARGET_AARCH64_FUCHSIA_RUSTFLAGS \
+-C link-arg=--sysroot=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/sysroot \
+-Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/sysroot/lib \
+-Lnative=/usr/local/core-linux-amd64-fuchsia-sdk/arch/arm64/lib
+
+ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS -Clink-arg=-static-libstdc++
+
+ENV TARGETS=x86_64-fuchsia
+ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnu
+
+COPY scripts/sccache.sh /scripts/
+RUN sh /scripts/sccache.sh
+
+ENV RUST_INSTALL_DIR /checkout/obj/install
+RUN mkdir -p $RUST_INSTALL_DIR/etc
+
+ENV RUST_CONFIGURE_ARGS --prefix=$RUST_INSTALL_DIR --sysconfdir=etc --enable-lld --llvm-libunwind=in-tree --enable-extended --disable-docs
+ENV SCRIPT \
+    python3 ../x.py install --target $TARGETS compiler/rustc library/std clippy && \
+    python3 ../x.py test src/tools/cargotest
+

src/ci/docker/host-x86_64/dist-various-2/build-fuchsia-toolchain.sh → src/ci/docker/scripts/build-fuchsia-toolchain.sh

@@ -1,16 +1,33 @@
 #!/usr/bin/env bash
 
+# TODO: copied from shared.sh
+hide_output() {
+  { set +x; } 2>/dev/null
+  on_err="
+echo ERROR: An error was encountered with the build.
+cat /tmp/build.log
+exit 1
+"
+  trap "$on_err" ERR
+  bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
+  PING_LOOP_PID=$!
+  "$@" &> /tmp/build.log
+  trap - ERR
+  kill $PING_LOOP_PID
+  set -x
+}
+
 set -ex
-source shared.sh
+#source shared.sh
 
 FUCHSIA_SDK_URL=https://chrome-infra-packages.appspot.com/dl/fuchsia/sdk/core/linux-amd64
-FUCHSIA_SDK_ID=4xjxrGUrDbQ6_zJwj6cDN1IbWsWV5aCQXC_zO_Hu0XkC
-FUCHSIA_SDK_SHA256=e318f1ac652b0db43aff32708fa70337521b5ac595e5a0905c2ff33bf1eed179
+FUCHSIA_SDK_ID=MrhQwtmP8CpZre-i_PNOREcThbUcrX3bA-45d6WQr-cC
+FUCHSIA_SDK_SHA256=32b850c2d98ff02a59adefa2fcf34e44471385b51cad7ddb03ee3977a590afe7
 FUCHSIA_SDK_USR_DIR=/usr/local/core-linux-amd64-fuchsia-sdk
 CLANG_DOWNLOAD_URL=\
 https://chrome-infra-packages.appspot.com/dl/fuchsia/third_party/clang/linux-amd64
-CLANG_DOWNLOAD_ID=vU0vNjSihOV4Q6taQYCpy03JXGiCyVwxen3rFMNMIgsC
-CLANG_DOWNLOAD_SHA256=bd4d2f3634a284e57843ab5a4180a9cb4dc95c6882c95c317a7deb14c34c220b
+CLANG_DOWNLOAD_ID=Tpc85d1ZwSlZ6UKl2d96GRUBGNA5JKholOKe24sRDr0C
+CLANG_DOWNLOAD_SHA256=4e973ce5dd59c12959e942a5d9df7a19150118d03924a86894e29edb8b110ebd
 
 install_clang() {
   mkdir -p clang_download

src/ci/github-actions/ci.yml

@@ -470,6 +470,9 @@ jobs:
           - name: x86_64-gnu-aux
             <<: *job-linux-4c
 
+          - name: x86_64-gnu-experimental
+            <<: *job-linux-16c
+
           - name: x86_64-gnu-debug
             <<: *job-linux-8c
 
@@ -734,6 +737,9 @@ jobs:
             env:
               CODEGEN_BACKENDS: llvm,cranelift
             <<: *job-linux-16c
+          - name: x86_64-gnu-experimental
+            <<: *job-linux-16c
+
 
   master:
     name: master

src/tools/cargotest/main.rs

@@ -106,9 +106,16 @@ fn main() {
     let out_dir = Path::new(&args[2]);
     let cargo = &Path::new(cargo);
 
-    for test in TEST_REPOS.iter().rev() {
-        if args[3..].is_empty() || args[3..].iter().any(|s| s.contains(test.name)) {
-            test_repo(cargo, out_dir, test);
+    if false {
+        for test in TEST_REPOS.iter().rev() {
+            if args[3..].is_empty() || args[3..].iter().any(|s| s.contains(test.name)) {
+                test_repo(cargo, out_dir, test);
+            }
+        }
+    } else {
+        // For now, let Fuchsia tests fail.
+        if std::panic::catch_unwind(|| test_fuchsia(out_dir)).is_err() {
+            eprintln!("Fuchsia tests failed; continuing.");
         }
     }
 }
@@ -216,3 +223,99 @@ fn run_cargo_test(
 
     status.success()
 }
+
+fn test_fuchsia(out_dir: &Path) {
+    const INTEGRATION_SHA: &str = "06ae16d18bd8e4db9a3fc062f678a170025d9f1a";
+    const PICK_REFS: &[&str] = &["refs/changes/58/938058/3", "refs/changes/33/943833/6"];
+
+    // This script will:
+    // - create a directory named "fuchsia" if it does not exist
+    // - download "jiri" to "fuchsia/.jiri_root/bin"
+    const BOOTSTRAP: &'static str = r#"
+        curl -s "https://fuchsia.googlesource.com/jiri/+/HEAD/scripts/bootstrap_jiri?format=TEXT" | base64 --decode | bash -s fuchsia
+    "#;
+    let status = Command::new("sh").arg("-c").arg(BOOTSTRAP).current_dir(out_dir).status().unwrap();
+    assert!(status.success(), "bootstrap_jiri failed");
+
+    let checkout_dir = &out_dir.join("fuchsia");
+    let jiri = || {
+        let mut cmd = Command::new(".jiri_root/bin/jiri");
+        cmd.current_dir(checkout_dir);
+        cmd
+    };
+
+    let status = jiri()
+        .arg("init")
+        .arg("-partial=true")
+        .arg("-analytics-opt=false")
+        .arg(checkout_dir)
+        .status()
+        .unwrap();
+    assert!(status.success(), "jiri init failed");
+
+    let status = jiri()
+        .args([
+            "import",
+            "-name=integration",
+            &format!("-revision={INTEGRATION_SHA}"),
+            "-overwrite=true",
+            "flower",
+            "https://fuchsia.googlesource.com/integration",
+        ])
+        .status()
+        .unwrap();
+    assert!(status.success(), "jiri import failed");
+
+    if checkout_dir.join(".git").is_dir() {
+        // Wipe out any local changes if we're reusing a checkout.
+        let status = Command::new("git")
+            .arg("checkout")
+            .arg("--force")
+            .arg("JIRI_HEAD")
+            .current_dir(checkout_dir)
+            .status()
+            .unwrap();
+        assert!(status.success(), "checkout JIRI_HEAD failed");
+    }
+
+    let status = jiri()
+        .arg("update")
+        .arg("-autoupdate=false")
+        // .arg("-v")
+        .status()
+        .unwrap();
+    assert!(status.success(), "jiri update failed");
+
+    let integration = Command::new("git")
+        .current_dir(checkout_dir.join("integration"))
+        .arg("rev-parse")
+        .arg("HEAD")
+        .output()
+        .unwrap()
+        .stdout;
+    let integration = String::from_utf8_lossy(&integration);
+    println!("integration commit = {integration}");
+
+    for git_ref in PICK_REFS {
+        let status = Command::new("git")
+            .current_dir(checkout_dir)
+            .args(["fetch", "https://fuchsia.googlesource.com/fuchsia", git_ref])
+            .status()
+            .unwrap();
+        assert!(status.success(), "fetching ref '{git_ref}' failed");
+
+        let status = Command::new("git")
+            .current_dir(checkout_dir)
+            .args(["cherry-pick", "--no-commit", "FETCH_HEAD"])
+            .status()
+            .unwrap();
+        assert!(status.success(), "picking ref '{git_ref}' failed");
+    }
+
+    let status = Command::new("bash")
+        .current_dir(checkout_dir)
+        .arg("scripts/rust/build_fuchsia_from_rust_ci.sh")
+        .status()
+        .unwrap();
+    assert!(status.success(), "fuchsia build failed");
+}