Achieve perfect reproducibility for dev setups

This commit changes the default C++ toolchain to the custom nix-backed clang toolchain. - Update custom stdenv to use Clang/LLVM 18 - Regenerate LRE toolchains. - Remove obsolete zig_cc toolchain - Expand LRE CI to build C++ parts of NativeLink remotely in K8s and the rest of rules_rust locally. Fixes TraceMachina#694 Closes TraceMachina#477
aaronmondal · Apr 23, 2024 · f988d53 · f988d53
1 parent f79b59b
commit f988d53
Show file tree

Hide file tree

Showing 20 changed files with 2,870 additions and 2,864 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -25,7 +25,7 @@ build --incompatible_strict_action_env
 # Don't use legacy toolchain resolution.
 build --incompatible_enable_cc_toolchain_resolution
 
-# Use the earliest supported C++ version for protoc.
+# Use the latest possible C++ as it might influence implicit rust dependencies.
 build --cxxopt=-std=c++14 --host_cxxopt=-std=c++14
 
 build --test_output=errors
@@ -75,24 +75,6 @@ build:windows --enable_runfiles
 # toolchains are resolved before the rust toolchains.
 build --extra_toolchains=@rust_toolchains//:all
 
-# Option to test the zig toolchain on Linux. Prefer the default `linux`
-# toolchain which builds cc targets roughly twice as fast.
-#
-# WARNING:
-#
-# We're using an incredibly old target glibc here. Builds created with this
-# toolchain have maximum compatibility (theoretically down to Ubuntu 18), but
-# miss out on half a decade of optimizations. Don't use this for production
-# builds if you're running a non-ancient OS and care about performance.
-#
-# TODO(aaronmondal): Migrate to a statically linked musl as soon as rules_rust
-#                    supports it. This way we get to keep (or even improve)
-#                    backwards compatibility without sacrificing performance.
-build:linux_zig --host_platform=@zig_sdk//platform:linux_amd64
-build:linux_zig --extra_toolchains=@zig_sdk//libc_aware/toolchain:linux_amd64_gnu.2.28
-build:linux_zig --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
-
-
 # Local Remote Execution.
 build:lre --extra_execution_platforms=@local-remote-execution//generated-cc/config:platform
 build:lre --extra_toolchains=@local-remote-execution//generated-cc/config:cc-toolchain"
@@ -105,5 +87,19 @@ build:lre --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1
 #                    is resolved.
 build:lre --define=EXECUTOR=remote
 
+# rules_rust doesn't properly work with remote execution.
+#
+# Note(aaronmondal): It's technically possible to run this build in some remote
+# execution images, such as ubuntu-based images. Those images only pretend to be
+# reproducible (think /usr/bin/clang which could be any version of clang). This
+# disables anything that doesn't "properly" execute remotely.
+common:lre --strategy=Rustc=local
+common:lre --strategy=CargoBuildScriptRun=local
+common:lre --strategy=Rustfmt=local
+common:lre --strategy=Clippy=local
+common:lre --strategy=TestRunner=local
+common:lre --strategy=Rustdoc=local
+common:lre --strategy=Genrule=local # Genrules break remotely as well.
+
 # Allow user-side customization.
 try-import %workspace%/.bazelrc.user
diff --git a/.github/workflows/lre.yaml b/.github/workflows/lre.yaml
@@ -42,6 +42,15 @@ jobs:
            --verbose_failures \
            @local-remote-execution//examples:hello_lre"
 
+      - name: Build NativeLink with LRE.
+        run: >
+          nix develop --impure --command
+          bash -c "bazel test \
+           --config=lre \
+           --verbose_failures \
+           ...
+
+
   remote:
     strategy:
       fail-fast: false
@@ -109,3 +118,14 @@ jobs:
             --remote_executor=grpc://$scheduler_ip:50052 \
             --verbose_failures \
             @local-remote-execution//examples:hello_lre"
+
+      - name: Build NativeLink with LRE.
+        run: >
+          nix develop --impure --command
+          bash -c "bazel test \
+            --config=lre \
+            --remote_instance_name=main \
+            --remote_cache=grpc://$cache_ip:50051 \
+            --remote_executor=grpc://$scheduler_ip:50052 \
+            --verbose_failures \
+            ...
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -58,15 +58,15 @@ jobs:
         cd ../../ && \
         docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
           bazel clean && \
-          bazel test --config=linux_zig //... \
+          bazel test //... \
           --remote_instance_name=main \
           --remote_cache=grpc://127.0.0.1:50051 \
           --remote_executor=grpc://127.0.0.1:50052 \
           --remote_default_exec_properties=cpu_count=1 \
         ' && \
         docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
           bazel clean && \
-          bazel test --config=linux_zig //... \
+          bazel test //... \
           --remote_instance_name=main \
           --remote_cache=grpc://127.0.0.1:50051 \
           --remote_executor=grpc://127.0.0.1:50052 \

diff --git a/.github/workflows/native-bazel.yaml b/.github/workflows/native-bazel.yaml
@@ -69,37 +69,3 @@ jobs:
             exit 1
           fi
         shell: bash
-
-  zig-cc-unit-tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-20.04, ubuntu-22.04]
-    name: zig-cc ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    steps:
-
-      - name: Checkout
-        uses: >- # v4.1.1
-          actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
-
-      - name: Setup Bazelisk
-        uses: >- # v0.8.1
-          bazel-contrib/setup-bazel@b388b84bb637e50cdae241d0f255670d4bd79f29
-        with:
-          bazelisk-cache: true
-
-      - name: Mount bazel cache
-        uses: >- # v4.0.1
-          actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319
-        with:
-          path: |
-            ~/.cache/bazel
-          key: |
-            ${{ matrix.os }}-bazel-legacy-${{ hashFiles('.bazelversion', '.bazelrc', 'WORKSPACE.bazel', 'MODULE.bazel') }}
-          restore-keys: |
-            ${{ matrix.os }}-bazel-legacy-
-
-      - name: Run Bazel tests
-        run: |
-          bazel test --config=linux_zig //... --verbose_failures
diff --git a/.github/workflows/nix.yaml b/.github/workflows/nix.yaml
@@ -47,7 +47,7 @@ jobs:
         run: |
           if [ "$RUNNER_OS" == "Linux" ]; then
             nix develop --impure --command \
-            bash -c "bazel test --config=linux_zig ... --verbose_failures"
+            bash -c "bazel test ... --verbose_failures"
           elif [ "$RUNNER_OS" == "macOS" ]; then
             nix develop --impure --command \
             bash -c "bazel test //... --verbose_failures"

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -18,14 +18,6 @@ python.toolchain(
 )
 use_repo(python, python = "python_versions")
 
-bazel_dep(name = "hermetic_cc_toolchain", version = "3.0.1")
-
-zig_toolchains = use_extension(
-    "@hermetic_cc_toolchain//toolchain:ext.bzl",
-    "toolchains",
-)
-use_repo(zig_toolchains, "zig_sdk")
-
 bazel_dep(name = "rules_rust", version = "0.42.1")
 
 rust = use_extension("@rules_rust//rust:extensions.bzl", "rust")
@@ -72,7 +64,7 @@ crate.from_cargo(
 )
 use_repo(crate, "crates")
 
-bazel_dep(name = "protobuf", version = "23.1")
+bazel_dep(name = "protobuf", version = "26.0")
 
 # Local remote execution.
 bazel_dep(name = "local-remote-execution", version = "0.0.0")

diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -73,8 +73,6 @@
 
         customStdenv = import ./tools/llvmStdenv.nix {inherit pkgs;};
 
-        # TODO(aaronmondal): This doesn't work with rules_rust yet.
-        # Tracked in https://github.com/TraceMachina/nativelink/issues/477.
         customClang = pkgs.callPackage ./tools/customClang.nix {
           inherit pkgs;
           stdenv = customStdenv;
@@ -154,10 +152,17 @@
           };
         };
       in rec {
-        _module.args.pkgs = import self.inputs.nixpkgs {
-          inherit system;
-          overlays = [(import rust-overlay)];
-        };
+        _module.args.pkgs = let
+          nixpkgs-patched = (import self.inputs.nixpkgs {inherit system;}).applyPatches {
+            name = "nixpkgs-patched";
+            src = self.inputs.nixpkgs;
+            patches = [./tools/nixpkgs_link_libunwind_and_libcxx.diff];
+          };
+        in
+          import nixpkgs-patched {
+            inherit system;
+            overlays = [(import rust-overlay)];
+          };
         apps = {
           default = {
             type = "app";
@@ -206,49 +211,48 @@
           # });
         };
         pre-commit.settings = {inherit hooks;};
-        devShells.default = pkgs.mkShell {
-          nativeBuildInputs =
-            [
-              # Development tooling goes here.
-              stable-rust-native.default
-              pkgs.pre-commit
-              pkgs.bazel_7
-              pkgs.awscli2
-              pkgs.skopeo
-              pkgs.dive
-              pkgs.cosign
-              pkgs.kubectl
-              pkgs.kubernetes-helm
-              pkgs.cilium-cli
-              pkgs.yarn
-              pkgs.vale
-              pkgs.trivy
-              pkgs.docker-client
-              pkgs.kind
-              pkgs.tektoncd-cli
-              (pkgs.pulumi.withPackages (ps: [ps.pulumi-language-go]))
-              pkgs.go
-
-              # Additional tools from within our development environment.
-              local-image-test
-              generate-toolchains
-              customClang
-              native-cli
-            ]
-            ++ maybeDarwinDeps;
-          shellHook = ''
-            # Generate the .pre-commit-config.yaml symlink when entering the
-            # development shell.
-            ${config.pre-commit.installationScript}
-
-            # The Bazel and Cargo builds in nix require a Clang toolchain.
-            # TODO(aaronmondal): The Bazel build currently uses the
-            #                    irreproducible host C++ toolchain. Provide
-            #                    this toolchain via nix for bitwise identical
-            #                    binaries across machines.
-            export CC=clang
-          '';
-        };
+        devShells.default =
+          pkgs.mkShell.override {
+            stdenv = customStdenv;
+          } {
+            nativeBuildInputs =
+              [
+                # Development tooling goes here.
+                stable-rust-native.default
+                pkgs.pre-commit
+                pkgs.bazel_7
+                pkgs.awscli2
+                pkgs.skopeo
+                pkgs.dive
+                pkgs.cosign
+                pkgs.kubectl
+                pkgs.kubernetes-helm
+                pkgs.cilium-cli
+                pkgs.yarn
+                pkgs.vale
+                pkgs.trivy
+                pkgs.docker-client
+                pkgs.kind
+                pkgs.tektoncd-cli
+                (pkgs.pulumi.withPackages (ps: [ps.pulumi-language-go]))
+                pkgs.go
+
+                # Additional tools from within our development environment.
+                local-image-test
+                generate-toolchains
+                customClang
+                native-cli
+              ]
+              ++ maybeDarwinDeps;
+            shellHook = ''
+              # Generate the .pre-commit-config.yaml symlink when entering the
+              # development shell.
+              ${config.pre-commit.installationScript}
+
+              # The Bazel and Cargo builds in nix require a Clang toolchain.
+              export CC=customClang
+            '';
+          };
       };
     };
 }
diff --git a/local-remote-execution/cpp_env.json b/local-remote-execution/cpp_env.json
@@ -1,5 +1,5 @@
 {
-    "ABI_LIBC_VERSION": "glibc_2.35",
-    "BAZEL_TARGET_LIBC": "glibc_2.35",
+    "ABI_LIBC_VERSION": "glibc_2.39",
+    "BAZEL_TARGET_LIBC": "glibc_2.39",
     "BAZEL_HOST_SYSTEM": "x86_64-unknown-linux-gnu"
 }