diff --git a/.github/workflows/check-by-name.yml b/.github/workflows/check-by-name.yml
index 18efd181e0499..2daffc9f20186 100644
--- a/.github/workflows/check-by-name.yml
+++ b/.github/workflows/check-by-name.yml
@@ -32,27 +32,47 @@ jobs:
           fetch-depth: 2
       - name: Determining PR git hashes
         run: |
+          # Please don't put any logic here.  To ensure local
+          # reproducibility, this file should only print diagnostics
+          # (`env`, `set -x`) and execute scripts found elsewhere in
+          # nixpkgs.
           env
           set -x
           exec pkgs/test/nixpkgs-check-by-name/workflows/check-by-name/determine-pr-hashes.sh
       - uses: cachix/install-nix-action@v23
       - name: Determining channel to use for dependencies
         run: |
+          # Please don't put any logic here.  To ensure local
+          # reproducibility, this file should only print diagnostics
+          # (`env`, `set -x`) and execute scripts found elsewhere in
+          # nixpkgs.
           env
           set -x
           exec pkgs/test/nixpkgs-check-by-name/workflows/check-by-name/determine-channel-for-dependencies.sh
       - name: Fetching latest version of channel
         run: |
+          # Please don't put any logic here.  To ensure local
+          # reproducibility, this file should only print diagnostics
+          # (`env`, `set -x`) and execute scripts found elsewhere in
+          # nixpkgs.
           env
           set -x
           exec pkgs/test/nixpkgs-check-by-name/workflows/check-by-name/fetch-latest-version-of-channel.sh
       - name: Fetching pre-built nixpkgs-check-by-name from the channel
         run: |
+          # Please don't put any logic here.  To ensure local
+          # reproducibility, this file should only print diagnostics
+          # (`env`, `set -x`) and execute scripts found elsewhere in
+          # nixpkgs.
           env
           set -x
           exec pkgs/test/nixpkgs-check-by-name/workflows/check-by-name/fetch-pre-built-by-name-from-channel.sh
       - name: Running nixpkgs-check-by-name
         run: |
+          # Please don't put any logic here.  To ensure local
+          # reproducibility, this file should only print diagnostics
+          # (`env`, `set -x`) and execute scripts found elsewhere in
+          # nixpkgs.
           env
           set -x
-          exec pkgs/test/nixpkgs-check-by-name/workflows/check-by-name/run-nixpkgs-check-by-name.sh
+          exec pkgs/test/nixpkgs-check-by-name/workflows/check-by-name/reproduce.sh "$baseSha" "$toolingSha" "$headSha"
diff --git a/pkgs/test/nixpkgs-check-by-name/workflows/check-by-name/reproduce.sh b/pkgs/test/nixpkgs-check-by-name/workflows/check-by-name/reproduce.sh
index c85fd63acc871..63b00daebef37 100755
--- a/pkgs/test/nixpkgs-check-by-name/workflows/check-by-name/reproduce.sh
+++ b/pkgs/test/nixpkgs-check-by-name/workflows/check-by-name/reproduce.sh
@@ -1,4 +1,28 @@
 #!/usr/bin/env bash
+
+set -e
+# Usage: pkgs/test/nixpkgs-check-by-name/reproduce.sh <base SHA> <tooling SHA> <merged SHA>
+
+# TODO(amjoseph): allow omitting the final argument, since it is
+# often a commit which exists only in github (and is difficult to
+# `git fetch` from github before the PR is merged).
+
+baseSha="$1"
+toolingBinariesSha="$2"
+mergedSha="$3"
+
+nixpkgs_for_tooling_binaries=$(nix-instantiate --eval --expr "builtins.fetchTarball \"https://github.com/nixos/nixpkgs/archive/$toolingBinariesSha.tar.gz\"" | tr -d '"')
+
+nix-store --realise --add-root nixpkgs-for-tooling-binaries $nixpkgs_for_tooling_binaries
+
+echo "Fetching pre-built nixpkgs-check-by-name from channel $channel at revision $toolingBinariesSha"
+nix-build \
+  --option extra-substituters https://cache.nixos.org \
+  --option trusted-public-keys 'cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=' \
+  nixpkgs-for-tooling-binaries \
+  -A tests.nixpkgs-check-by-name \
+  --max-jobs 0
+
 echo "Checking whether the check succeeds on the base branch $GITHUB_BASE_REF"
 git checkout -q "$baseSha"
 if baseOutput=$(result/bin/nixpkgs-check-by-name . 2>&1); then
@@ -70,10 +94,10 @@ resultToEmoji() {
     echo "  - Tooling binaries built at nixpkgs commit: [$toolingBinariesSha](https://github.com/${GITHUB_REPOSITORY}/commit/$toolingBinariesSha)"
     echo "  - Store path: \`$(realpath result)\`"
     echo "- Tested Nixpkgs:"
-    echo "  - Base branch: $GITHUB_BASE_REF"
+    echo "  - Base branch: $BASE_SHA"
     echo "  - Latest base branch commit: [$baseSha](https://github.com/${GITHUB_REPOSITORY}/commit/$baseSha)"
     echo "  - Latest PR commit: [$headSha](https://github.com/${GITHUB_REPOSITORY}/commit/$headSha)"
     echo "  - Merge commit: [$mergedSha](https://github.com/${GITHUB_REPOSITORY}/commit/$mergedSha)"
-} >> "$GITHUB_STEP_SUMMARY"
+} | tee -a "${GITHUB_STEP_SUMMARY:-/dev/null}"
 
 exit "$exitCode"