ci.yml

# beacon_chain
# Copyright (c) 2020-2024 Status Research & Development GmbH
# Licensed and distributed under either of
#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.

name: CI
on:
  push:
    paths-ignore: ['media/**', 'docs/**', '**/*.md']
    branches:
      - stable
      - testing
      - unstable
  pull_request:
    paths-ignore: ['media/**', 'docs/**', '**/*.md']
    branches-ignore:
      - stable
  workflow_dispatch:

concurrency: # Cancel stale PR builds (but not push builds)
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

jobs:
  build:
    strategy:
      fail-fast: false
      matrix:
        target:
          - os: linux
            cpu: amd64
          - os: macos
            cpu: amd64
          - os: macos
            cpu: arm64
          - os: windows
            cpu: amd64
        branch: [~, upstream/version-2-0]
        exclude:
          - target:
              os: macos
            branch: upstream/version-2-0
          - target:
              os: windows
            branch: upstream/version-2-0
        include:
          - branch: upstream/version-2-0
            branch-short: version-2-0
            nimflags-extra: --mm:refc
          - target:
              os: linux
            builder: ['self-hosted','ubuntu-22.04']
          - target:
              os: macos
            builder: macos-13
          - target:
              os: windows
            builder: windows-2019

    defaults:
      run:
        shell: bash

    name: ${{ matrix.target.os }}-${{ matrix.target.cpu }}${{ matrix.branch != '' && ' (Nim ' || '' }}${{ matrix.branch-short }}${{ matrix.branch != '' && ')' || '' }}
    runs-on: ${{ matrix.builder }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Restore llvm-mingw (Windows) from cache
        if: runner.os == 'Windows'
        id: windows-mingw-cache
        uses: actions/cache@v4
        with:
          path: external/mingw-${{ matrix.target.cpu }}
          key: 'mingw-llvm-17-${{ matrix.target.cpu }}'

      - name: Install llvm-mingw dependency (Windows)
        if: >
          steps.windows-mingw-cache.outputs.cache-hit != 'true' &&
          runner.os == 'Windows'
        run: |
          mkdir -p external
          MINGW_BASE="https://github.com/mstorsjo/llvm-mingw/releases/download/20230905"
          MINGW_URL="$MINGW_BASE/llvm-mingw-20230905-ucrt-x86_64.zip"
          curl -L "$MINGW_URL" -o "external/mingw-${{ matrix.target.cpu }}.zip"
          7z x -y "external/mingw-${{ matrix.target.cpu }}.zip" -oexternal/mingw-${{ matrix.target.cpu }}/
          mv external/mingw-${{ matrix.target.cpu }}/**/* ./external/mingw-${{ matrix.target.cpu }}

      - name: Restore Nim DLLs dependencies (Windows) from cache
        if: runner.os == 'Windows'
        id: windows-dlls-cache
        uses: actions/cache@v4
        with:
          path: external/dlls-${{ matrix.target.cpu }}
          key: 'dlls-${{ matrix.target.cpu }}'

      - name: Install DLLs dependencies (Windows)
        if: >
          steps.windows-dlls-cache.outputs.cache-hit != 'true' &&
          runner.os == 'Windows'
        run: |
          mkdir -p external
          curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip
          7z x -y external/windeps.zip -oexternal/dlls-${{ matrix.target.cpu }}

      - name: Path to cached dependencies (Windows)
        if: >
          runner.os == 'Windows'
        run: |
          echo '${{ github.workspace }}'"/external/mingw-${{ matrix.target.cpu }}/bin" >> $GITHUB_PATH
          echo "${{ github.workspace }}/external/dlls-${{ matrix.target.cpu }}" >> $GITHUB_PATH
          # for miniupnp that runs "wingenminiupnpcstrings.exe" from the current dir
          echo "." >> $GITHUB_PATH

      - name: Derive environment variables
        run: |
          if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
            PLATFORM=x64
          elif [[ '${{ matrix.target.cpu }}' == 'arm64' ]]; then
            PLATFORM=arm64
          else
            PLATFORM=x86
          fi
          echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV

          # Stack usage test and UBSAN on recent enough gcc:
          if [[ '${{ runner.os }}' == 'Linux' && '${{ matrix.target.cpu }}' == 'amd64' ]]; then
            if [[ '${{ github.sha }}' =~ ^7 ]]; then
              export WITH_UBSAN=1
              echo "WITH_UBSAN=1" >> $GITHUB_ENV
              export NIMFLAGS="${NIMFLAGS} -d:limitStackUsage --passC:-fsanitize=undefined --passL:-fsanitize=undefined"
            else
              export NIMFLAGS="${NIMFLAGS} -d:limitStackUsage"
            fi
          fi

          if [[ '${{ runner.os }}' == 'Windows' ]]; then
            export NIMFLAGS="${NIMFLAGS} --cc:clang"
          fi

          export NIMFLAGS="${NIMFLAGS} ${{ matrix.nimflags-extra }}"
          echo "NIMFLAGS=${NIMFLAGS}" >> $GITHUB_ENV

          ncpu=""
          make_cmd="make"
          case '${{ runner.os }}' in
          'Linux')
            ncpu=$(nproc)
            ;;
          'macOS')
            ncpu=$(sysctl -n hw.ncpu)
            ;;
          'Windows')
            ncpu=${NUMBER_OF_PROCESSORS}
            make_cmd="mingw32-make"
            echo "Number of cores: ${NUMBER_OF_PROCESSORS}"
            echo "Physical memory: $(wmic ComputerSystem get TotalPhysicalMemory)"
            echo "Partition sizes: $(wmic partition get name,size,type)"
            ;;
          esac
          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
          echo "ncpu=${ncpu}" >> $GITHUB_ENV
          echo "make_cmd=${make_cmd}" >> $GITHUB_ENV

      - name: Build Nim and Nimbus dependencies
        run: |
          ${make_cmd} -j ${ncpu} NIM_COMMIT=${{ matrix.branch }} ARCH_OVERRIDE=${PLATFORM} QUICK_AND_DIRTY_COMPILER=1 update
          ./env.sh nim --version

      - name: Get latest fixtures commit hash
        id: fixtures_version
        run: |
          getHash() {
            git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1
          }
          fixturesHash=$(getHash status-im/nim-eth2-scenarios)
          echo "fixtures=${fixturesHash}" >> $GITHUB_OUTPUT

      - name: Build binaries (with trace logging enabled)
        run: |
          ${make_cmd} -j ${ncpu} V=1 NIM_COMMIT=${{ matrix.branch }} LOG_LEVEL=TRACE NIMFLAGS="-u:release --opt:none ${NIMFLAGS}"
          # The Windows image runs out of disk space, so make some room
          rm -rf build nimcache

      - name: Restore Ethereum Foundation fixtures from cache
        id: fixtures-cache
        uses: actions/cache@v4
        with:
          path: fixturesCache
          key: 'eth2-scenarios-${{ steps.fixtures_version.outputs.fixtures }}'

      # Important: even with a cache hit, this should be run
      # as it symlinks the cached items in their proper place
      - name: Get the Ethereum Foundation fixtures
        run: |
          scripts/setup_scenarios.sh fixturesCache

      - name: Run tests
        run: |
          ${make_cmd} -j ${ncpu} V=1 NIM_COMMIT=${{ matrix.branch }} DISABLE_TEST_FIXTURES_SCRIPT=1 test

      # The upload creates a combined report that gets posted as a comment on the PR
      # https://github.com/EnricoMi/publish-unit-test-result-action
      - name: Upload combined results
        uses: actions/upload-artifact@v3
        with:
          name: Unit Test Results ${{ matrix.target.os }}-${{ matrix.target.cpu }}
          path: build/*.xml

  devbuild:
    name: "Developer builds"
    runs-on: ['self-hosted','ubuntu-22.04']
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 1

      - name: Build with developer flags
        run: |
          make -j nimbus_beacon_node LOG_LEVEL=TRACE NIMFLAGS="-d:has_deposit_root_checks=1"

      - name: Build files with isMainModule
        run: |
          executables=(
            "beacon_chain/el/deposit_contract"
            "beacon_chain/fork_choice/fork_choice"
            "beacon_chain/fork_choice/proto_array"
            "beacon_chain/networking/network_metadata_downloads"
            "beacon_chain/era_db"
            "beacon_chain/trusted_node_sync"
            "benchmarks/rest_api_benchmark"
            "tests/mocking/mock_genesis"
          )
          source env.sh
          for executable in "${executables[@]}"; do
            nim c --passC:-fsyntax-only --noLinking:on -d:chronicles_log_level=TRACE "${executable}"
          done

  lint:
    name: "Lint"
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 2  # In PR, has extra merge commit: ^1 = PR, ^2 = base
          submodules: 'recursive'

      - name: Check copyright year
        if: ${{ !cancelled() }} && github.event_name == 'pull_request'
        run: |
          excluded_files="config.yaml"
          excluded_extensions="ans|bin|cfg|json|json\\.template|md|png|service|ssz|txt|lock|nix"

          current_year=$(date +"%Y")
          problematic_files=()
          while read -r file; do
            if ! grep -qE 'Copyright \(c\) .*'$current_year' Status Research & Development GmbH' "$file"; then
              problematic_files+=("$file")
            fi
          done < <(git diff --name-only --diff-filter=AM --ignore-submodules HEAD^ HEAD | grep -vE '(\.('$excluded_extensions')|'$excluded_files')$' || true)

          if (( ${#problematic_files[@]} )); then
            echo "The following files do not have an up-to-date copyright year:"
            for file in "${problematic_files[@]}"; do
              echo "- $file"
            done
            exit 2
          fi

      - name: Check exception tracking
        if: ${{ !cancelled() }} && github.event_name == 'pull_request'
        run: |
          problematic_files=()
          while read -r file; do
            if ! grep -qE '^{\.push raises: \[\]\.}$' "$file"; then
              problematic_files+=("$file")
            fi
          done < <(git diff --name-only --diff-filter=AM --ignore-submodules HEAD^ HEAD | grep -E '\.nim$' || true)

          if (( ${#problematic_files[@]} )); then
            echo "The following files do not have '{.push raises: [].}':"
            for file in "${problematic_files[@]}"; do
              echo "- $file"
            done
            echo "See https://status-im.github.io/nim-style-guide/errors.exceptions.html"
            exit 2
          fi

      - name: Check submodules
        if: ${{ !cancelled() }} && github.event_name == 'pull_request'
        run: |
          while read -r file; do
            commit="$(git -C "$file" rev-parse HEAD)"
            commit_date=$(TZ=UTC0 git -C "$file" show -s --format='%cd' --date=iso-local HEAD)
            if ! branch="$(git config -f .gitmodules --get "submodule.$file.branch")"; then
              echo "Submodule '$file': '.gitmodules' lacks 'branch' entry"
              exit 2
            fi
            # Without the `--depth=1` fetch, may run into 'error processing shallow info: 4'
            if ! error="$(git -C "$file" fetch -q --depth=1 origin "+refs/heads/${branch}:refs/remotes/origin/${branch}")"; then
              echo "Submodule '$file': Failed to fetch '$branch': $error (1)"
              exit 2
            fi
            branch_commit_date=$(TZ=UTC0 git -C "$file" show -s --format='%cd' --date=iso-local "refs/remotes/origin/${branch}")
            if [[ "${commit_date}" > "${branch_commit_date}" ]]; then
              echo "Submodule '$file': '$commit' ($commit_date) is more recent than latest '$branch' ($branch_commit_date) (branch config: '.gitmodules')"
              exit 2
            fi
            if ! error="$(git -C "$file" fetch -q --shallow-since="$commit_date" origin "+refs/heads/${branch}:refs/remotes/origin/${branch}")"; then
              echo "Submodule '$file': Failed to fetch '$branch': $error (2)"
              exit 2
            fi
            if ! git -C "$file" merge-base --is-ancestor "$commit" "refs/remotes/origin/$branch"; then
              echo "Submodule '$file': '$commit' is not on '$branch' as of $commit_date (branch config: '.gitmodules')"
              exit 2
            fi
          done < <(git diff --name-only --diff-filter=AM HEAD^ HEAD | grep -f <(git config --file .gitmodules --get-regexp path | awk '{ print $2 }') || true)

  # https://github.com/EnricoMi/publish-unit-test-result-action
  event_file:
    name: "Event File"
    runs-on: ubuntu-latest
    steps:
      - name: Upload
        uses: actions/upload-artifact@v3
        with:
          name: Event File
          path: ${{ github.event_path }}