Parallel GH actions workflow for Nixpkgs eval

Motivated by ofborg struggling [1] and its evaluations taking too long, inspired by Jörg's initial PR [2] and Adam's previous attempt to parallelise Nixpkgs evaluation [3], this PR contains initial work to relief ofborg from its evaluation duty by using GitHub Actions to evaluate Nixpkgs. For now this doesn't take care of all of what ofborg does, such as requesting appropriate reviewers or labeling mass rebuilds, but this can be follow-up work. [1]: https://discourse.nixos.org/t/infrastructure-announcement-the-future-of-ofborg-your-help-needed/56025?u=infinisil [2]: NixOS#352808 [3]: NixOS#269403 Co-Authored-By: Jörg Thalheim <joerg@thalheim.io> Co-Authored-By: Adam Joseph <adam@westernsemico.com>
Bot-wxt1221 · Nov 19, 2024 · 17fc6c9 · 17fc6c9
1 parent 55f94d9
commit 17fc6c9
Show file tree

Hide file tree

Showing 10 changed files with 364 additions and 76 deletions.
diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml
@@ -0,0 +1,110 @@
+name: Eval
+
+on: pull_request_target
+
+permissions:
+  contents: read
+
+jobs:
+  attrs:
+    name: Attrs
+    runs-on: ubuntu-latest
+    outputs:
+      systems: ${{ steps.systems.outputs.systems }}
+      mergedSha: ${{ steps.merged.outputs.mergedSha }}
+    steps:
+      # Important: Because of `pull_request_target`, this doesn't check out the PR,
+      # but rather the base branch of the PR, which is needed so we don't run untrusted code
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          path: base
+          sparse-checkout: ci
+      - name: Resolving the merge commit
+        id: merged
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then
+            echo "Checking the merge commit $mergedSha"
+            echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT"
+          else
+            # Skipping so that no notifications are sent
+            echo "Skipping the rest..."
+          fi
+          rm -rf base
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        # Add this to _all_ subsequent steps to skip them
+        if: steps.merged.outputs.mergedSha
+        with:
+          ref: ${{ env.mergedSha }}
+          path: nixpkgs
+
+      - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
+        if: steps.merged.outputs.mergedSha
+
+      - id: systems
+        if: steps.merged.outputs.mergedSha
+        run: |
+          nix-build nixpkgs/ci -A eval.attrpathsSuperset
+          echo "systems=$(<result/systems.json)" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        if: steps.merged.outputs.mergedSha
+        with:
+          name: paths
+          path: result/paths.json
+
+  eval:
+    name: Eval
+    runs-on: ubuntu-latest
+    needs: attrs
+    if: needs.attrs.outputs.mergedSha
+    strategy:
+      matrix:
+        system: ${{ fromJSON(needs.attrs.outputs.systems) }}
+    steps:
+      - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          name: paths
+          path: paths
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ needs.attrs.outputs.mergedSha }}
+          path: nixpkgs
+
+      - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
+
+      - name: Check eval
+        run: nix-build nixpkgs/ci -A eval.singleSystem --argstr evalSystem ${{ matrix.system }} --arg attrpathFile ./paths/paths.json
+
+      - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        if: needs.attrs.outputs.mergedSha
+        with:
+          name: result-${{ matrix.system }}
+          path: result/paths
+
+  combine:
+    name: Combined
+    runs-on: ubuntu-latest
+    needs: eval
+    steps:
+      - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+        with:
+          pattern: result-*
+          path: results
+
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          ref: ${{ needs.attrs.outputs.mergedSha }}
+          path: nixpkgs
+
+      - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
+
+      - name: Check eval
+        run: nix-build nixpkgs/ci -A eval.combine --arg resultsDir ./results
+
+      - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          name: outpaths
+          path: result/outpaths.json
diff --git a/ci/default.nix b/ci/default.nix
@@ -26,4 +26,5 @@ in
   inherit pkgs;
   requestReviews = pkgs.callPackage ./request-reviews { };
   codeownersValidator = pkgs.callPackage ./codeowners-validator { };
+  eval = pkgs.callPackage ./eval { };
 }
diff --git a/ci/eval/default.nix b/ci/eval/default.nix
@@ -0,0 +1,196 @@
+{
+  lib,
+  runCommand,
+  writeShellScript,
+  linkFarm,
+  time,
+  procps,
+  nix,
+  jq,
+}:
+
+let
+  nixpkgs =
+    with lib.fileset;
+    toSource {
+      root = ../..;
+      fileset = unions (
+        map (lib.path.append ../..) [
+          "default.nix"
+          "doc"
+          "lib"
+          "maintainers"
+          "nixos"
+          "pkgs"
+          ".version"
+          "ci/eval/parallel.nix"
+        ]
+      );
+    };
+
+  supportedSystems = import ../supportedSystems.nix;
+
+  attrpathsSuperset =
+    runCommand "attrpaths-superset.json"
+      {
+        src = nixpkgs;
+        nativeBuildInputs = [
+          nix
+        ];
+        env.supportedSystems = builtins.toJSON supportedSystems;
+        passAsFile = [ "supportedSystems" ];
+      }
+      ''
+        export NIX_STATE_DIR=$(mktemp -d)
+        mkdir $out
+        nix-instantiate --eval --strict --json --arg enableWarnings false $src/pkgs/top-level/release-attrpaths-superset.nix -A paths > $out/paths.json
+        mv "$supportedSystemsPath" $out/systems.json
+      '';
+
+  singleSystem =
+    {
+      evalSystem,
+      attrpathFile,
+      checkMeta ? true,
+      includeBroken ? true,
+      # How many attributes to be evaluating at any single time.
+      # This effectively limits the maximum memory usage.
+      # Decrease this if too much memory is used
+      simultaneousAttrsPerSystem ? 100000,
+      quickTest ? false,
+    }:
+    let
+      singleChunk = writeShellScript "chunk" ''
+        set -euo pipefail
+        chunkSize=$1
+        myChunk=$2
+        outputDir=$3
+        system=$4
+
+        nix-env -f "${nixpkgs}/ci/eval/parallel.nix" \
+          --query --available \
+          --no-name --attr-path --out-path \
+          --show-trace \
+          --arg chunkSize "$chunkSize" \
+          --arg myChunk "$myChunk" \
+          --arg attrpathFile "${attrpathFile}" \
+          --arg systems "[ \"$system\" ]" \
+          --arg checkMeta ${lib.boolToString checkMeta} \
+          --arg includeBroken ${lib.boolToString includeBroken} \
+          > "$outputDir/$myChunk"
+      '';
+    in
+    runCommand "nixpkgs-eval-${evalSystem}"
+      {
+        nativeBuildInputs = [
+          nix
+          time
+          procps
+          jq
+        ];
+        env = {
+          inherit evalSystem;
+        };
+      }
+      ''
+        set -x
+        export NIX_STATE_DIR=$(mktemp -d)
+        nix-store --init
+
+        echo "System: $evalSystem"
+        cores=$NIX_BUILD_CORES
+        echo "Cores: $cores"
+        num_attrs=$(jq length "${attrpathFile}")
+        echo "Attribute count: $num_attrs"
+        chunk_size=$(( ${toString simultaneousAttrsPerSystem} / cores ))
+        echo "Chunk size: $chunk_size"
+        # Same as `num_attrs / chunk_size` but rounded up
+        num_chunks=$(( (num_attrs - 1) / chunk_size + 1 ))
+        echo "Chunk count: $num_chunks"
+
+        (
+          while true; do
+            free -g
+            sleep 20
+          done
+        ) &
+
+        seq_end=$(( num_chunks - 1 ))
+
+        ${lib.optionalString quickTest ''
+          seq_end=0
+        ''}
+
+        chunkOutputDir=$(mktemp -d)
+        seq -w 0 "$seq_end" |
+          command time -v xargs -t -I{} -P"$cores" \
+          ${singleChunk} "$chunk_size" {} "$chunkOutputDir" "$evalSystem"
+
+        mkdir $out
+        cat "$chunkOutputDir"/* > $out/paths
+      '';
+
+  combine =
+    {
+      resultsDir,
+    }:
+    runCommand "combined-result"
+      {
+        nativeBuildInputs = [
+          jq
+        ];
+        passAsFile = [ "jqScript" ];
+        jqScript = # jq
+          ''
+            split("\n") |
+            map(select(. != "") | split(" ") | map(select(. != ""))) |
+            map(
+              {
+                key: .[0],
+                value: .[1] | split(";") | map(split("=") |
+                  if length == 1 then
+                    { key: "out", value: .[0] }
+                  else
+                    { key: .[0], value: .[1] }
+                  end) | from_entries}
+            ) | from_entries
+          '';
+      }
+      ''
+        mkdir -p $out
+        cat ${resultsDir}/*/paths |
+          jq --sort-keys --raw-input --slurp -f "$jqScriptPath" \
+          > $out/outpaths.json
+      '';
+
+  together =
+    {
+      quickTest ? false,
+    }:
+    let
+      systems = if quickTest then [ "x86_64-linux" ] else supportedSystems;
+      results = linkFarm "results" (
+        map (system: {
+          name = system;
+          path = singleSystem {
+            system = system;
+            attrpathFile = attrpathsSuperset + "/paths.json";
+            inherit quickTest;
+          };
+        }) systems
+      );
+      final = combine {
+        resultsDir = results;
+      };
+    in
+    final;
+
+in
+{
+  inherit
+    attrpathsSuperset
+    singleSystem
+    combine
+    together
+    ;
+}
diff --git a/ci/eval/parallel.nix b/ci/eval/parallel.nix
@@ -0,0 +1,46 @@
+{
+  lib ? import ../../lib,
+  path ? ../..,
+  attrpathFile,
+  chunkSize,
+  myChunk,
+  checkMeta,
+  includeBroken,
+  systems,
+}:
+
+let
+  attrpaths = lib.importJSON attrpathFile;
+  myAttrpaths = lib.sublist (chunkSize * myChunk) chunkSize attrpaths;
+
+  unfiltered = import ../../pkgs/top-level/release-outpaths.nix {
+    inherit path;
+    inherit checkMeta includeBroken systems;
+  };
+
+  filtered =
+    let
+      recurse =
+        index: paths: attrs:
+        lib.mapAttrs (
+          name: values:
+          if attrs ? ${name} then
+            if lib.any (value: lib.length value <= index + 1) values then
+              attrs.${name}
+            else
+              recurse (index + 1) values attrs.${name}
+          else
+            null
+        ) (lib.groupBy (a: lib.elemAt a index) paths);
+    in
+    recurse 0 myAttrpaths unfiltered;
+
+  recurseEverywhere =
+    val:
+    if lib.isDerivation val || !(lib.isAttrs val) then
+      val
+    else
+      (lib.mapAttrs (_: v: recurseEverywhere v) val) // { recurseForDerivations = true; };
+
+in
+recurseEverywhere filtered
diff --git a/ci/supportedSystems.nix b/ci/supportedSystems.nix
@@ -0,0 +1,7 @@
+[
+  "aarch64-linux"
+  "aarch64-darwin"
+  #"i686-linux"  # !!!
+  "x86_64-linux"
+  "x86_64-darwin"
+]
diff --git a/lib/tests/release.nix b/lib/tests/release.nix
@@ -14,19 +14,5 @@ let
 in
   pkgs.symlinkJoin {
     name = "nixpkgs-lib-tests";
-    paths = map testWithNix nixVersions ++
-
-      #
-      # TEMPORARY MIGRATION MECHANISM
-      #
-      # This comment and the expression which follows it should be
-      # removed as part of resolving this issue:
-      #
-      #   https://github.com/NixOS/nixpkgs/issues/272591
-      #
-      [(import ../../pkgs/test/release {
-        inherit pkgs lib nix;
-      })]
-    ;
-
+    paths = map testWithNix nixVersions;
   }