reproducible.yml

name: Test compiler build reproducibility

on:
  push:
    # Only run on bors branches
    branches:
      - staging
      - trying

  # This is for passing required checks
  pull_request:
  merge_group:

jobs:
  reprotest:
    # This job is meant for testing whether the compiler can be built
    # reproducibly given the same build environment.
    #
    # There are two tools used for this test:
    #
    # - reprotest: This tool varies the environment in multiple ways, like
    #              adjusting time, build user, locale, etc. then run the build.
    #              If the binary matches the control build (build without any
    #              variations), then it's a pass. Otherwise, diffoscope is
    #              employed to show the differences.
    #
    # - diffoscope: This tool visualize differences in binaries in a
    #               human-readable fashion. This would allow developers to
    #               figure out what of their changes caused the build to
    #               varies based on outside environment.

    # Skip this for PRs
    if: github.event_name != 'pull_request'

    strategy:
      fail-fast: false

      matrix:
        test:
          - name: Source archive
            command: "./koch.py boot -d:danger && ./koch.py csource -d:danger && ./koch.py archive"
            pattern: "build/*.tar.zst"

          - name: Unix binary archive
            command: "./koch.py boot -d:danger && ./koch.py docs --docCmd:skip && ./koch.py unixrelease"
            pattern: "build/*.tar.zst"

          # Note: this tests the zip generation and not exe generation determinism.
          #
          # Testing exe generation will be done when cross-bootstrap is possible.
          - name: Windows binary archive
            command: "./koch.py boot -d:danger && ./koch.py docs --docCmd:skip && ./koch.py winrelease"
            pattern: "build/*.zip"

    name: "${{ matrix.test.name }} reproducibility tests"
    runs-on: ubuntu-latest

    steps:
      - name: Install reprotest
        run: |
          sudo apt-get update -qq
          # This tool is required for archive generation
          sudo apt-get install -yqq libarchive-tools
          # Diffoscope have a /lot/ of optional dependencies, but we don't need
          # all of them
          sudo apt-get install -yqq --no-install-recommends diffoscope
          # On the contrary, reprotest needs all of those deps to work
          sudo apt-get install -yqq reprotest

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          filter: tree:0

        # reprotest will manipulate the time which may cause bootstrapping
        # source download to fail due to SSL errors. Download this beforehand
        # as a workaround.
      - name: Download bootstrapping source
        run: ./koch.py fetch-bootstrap

      - id: reprotest
        name: Run reproducibility build
        run: |
          # Add a guest user for reprotest
          sudo useradd -m guest-builder

          # The path to output diffoscope HTML to
          output_html=$RUNNER_TEMP/diffoscope.html
          run_reprotest() {
            # Disabled kernel variation as it messes with csources architecture
            # detection.
            #
            # Can be re-enabled once reprotest is >= 0.7.18, where a fix is added
            # to prevent 32-bit architectures from being selected.
            reprotest \
              --vary=domain_host.use_sudo=1 \
              --vary=user_group.available+=guest-builder:guest-builder \
              --vary=-kernel \
              --diffoscope-arg="--html=$output_html" \
              'export XDG_CACHE_HOME=$PWD/build/nimcache \
                && ${{ matrix.test.command }}' \
              '${{ matrix.test.pattern }}'
          }

          if ! run_reprotest; then
            echo "::error::Reproducibility test failed, check the diffoscope output uploaded to artifacts for more details"
            echo "result=$output_html" >> $GITHUB_OUTPUT
            exit 1
          fi

      - name: Upload diffoscope output
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: ${{ matrix.test.name }} reproducibility test diffoscope output
          path: ${{ steps.reprotest.outputs.result }}

  passed:
    name: All reproducibility tests passed
    needs: [reprotest]
    if: always()
    runs-on: ubuntu-latest

    steps:
      - name: Raise failure
        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
        run: |
          echo "::error::There are failing required jobs"
          exit 1