benchmark.yml

name: Benchmarks

on:
  pull_request:
  push:
    tags:
      - "v*.*.*"
    branches:
      - master
      - develop

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  benchmark-unix:
    runs-on: ${{ matrix.os }}
    strategy:
        fail-fast: false
        matrix:
          os: [macos-latest, ubuntu-latest]

    env:
      LTTNG_UST_REGISTER_TIMEOUT: 0
      NUGET_XMLDOC_MODE: skip
      DOTNET_CLI_TELEMETRY_OPTOUT: "true"

    steps:
      - name: Check out the repository
        uses: actions/checkout@v2
        with:
          fetch-depth: 0

      # TODO: Add support for NetCore Bench
      - name: Set up the environment
        run: |
          sh ./tools/metacall-environment.sh $METACALL_INSTALL_OPTIONS
        env:
          METACALL_INSTALL_OPTIONS: base python nodejs ruby # TODO: Implement NetCore once it works with MacOS

      - name: Configure 
        run: |
          mkdir -p build
          cd build
          if [ "$(uname)" == "Darwin" ]; then
            . .env
          fi
          bash ../tools/metacall-configure.sh $METACALL_CONFIGURE_OPTIONS
        env:
          METACALL_CONFIGURE_OPTIONS: release scripts python nodejs ruby benchmarks # TODO: Implement NetCore once it works with MacOS

      - name: Build 
        working-directory: ./build
        # TODO: Remove the disable option for fork safe once funchook problem is solved
        run: |
          if [ "$(uname)" == "Darwin" ]; then
            . .env
          fi
          cmake -DOPTION_FORK_SAFE=OFF  ..
          bash ../tools/metacall-build.sh $METACALL_BUILD_OPTIONS
        env:
          METACALL_BUILD_OPTIONS: release benchmarks

      - name: Merge benchmarks
        if: ${{ github.event_name != 'pull_request' }}
        run: python3 ./tools/metacall-benchmarks-merge.py ./build/benchmarks

      # Download previous benchmark result from cache (if exists)
      - name: Download previous benchmark data
        uses: actions/cache@v1
        if: ${{ github.event_name != 'pull_request' }}
        with:
          path: ./cache
          key: ${{ matrix.os }}-benchmark

      - name: Store benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        if: ${{ github.event_name != 'pull_request' }}
        with:
            name: MetaCall Benchmark (${{ matrix.os }})
            tool: 'googlecpp'
            output-file-path: ./build/benchmarks/metacall-benchmarks.json
            # Access token to deploy GitHub Pages branch
            github-token: ""
            # Disable push and deploy GitHub pages branch automatically
            auto-push: false
            skip-fetch-gh-pages: true
            # Github Pages branch name
            gh-pages-branch: ${{ github.head_ref || github.ref_name }}
            # Output directory
            benchmark-data-dir-path: bench/${{ matrix.os }}
            # Where the previous data file is stored
            external-data-json-path: ./cache/metacall-benchmarks.json

      # Upload benchmark artifacts
      - name: Upload benchmark artifact
        uses: actions/upload-artifact@master
        if: ${{ github.event_name != 'pull_request' }}
        with:
          name: ${{ matrix.os }}-benchmark
          path: ./bench/

  benchmark-windows:
    runs-on: ${{ matrix.os }}
    strategy:
        fail-fast: false
        matrix:
          os: [windows-2019]

    env:
      LTTNG_UST_REGISTER_TIMEOUT: 0
      NUGET_XMLDOC_MODE: skip
      DOTNET_CLI_TELEMETRY_OPTOUT: "true"

    steps:
      - name: Check out the repository
        uses: actions/checkout@v2
        with:
          fetch-depth: 0

      - name: Activate the Visual Studio Developer Prompt
        uses: ilammy/msvc-dev-cmd@v1
        with:
          arch: amd64

      - name: Set up the environment
        run: cmd.exe /c "powershell .\tools\metacall-environment.ps1 $Env:METACALL_INSTALL_OPTIONS"
        env:
          METACALL_INSTALL_OPTIONS: python nodejs ruby # TODO: Implement NetCore once it works with Windows

      # TODO: Use release when this bug is solved: https://github.com/metacall/core/issues/461
      - name: Configure
        run: |
          $METACALL_PATH = $PWD
          md -Force "$METACALL_PATH\build"
          cd "$METACALL_PATH\build"
          cmd.exe /c "powershell ..\tools\metacall-configure.ps1 $Env:METACALL_BUILD_OPTIONS"
        env:
          METACALL_BUILD_OPTIONS: debug scripts python nodejs ruby benchmarks # TODO: Implement NetCore once it works with Windows

      # TODO: Use release when this bug is solved: https://github.com/metacall/core/issues/461
      - name: Build
        working-directory: ./build
        run: cmd.exe /c "powershell ..\tools\metacall-build.ps1 $Env:METACALL_BUILD_OPTIONS"
        env:
          METACALL_BUILD_OPTIONS: debug benchmarks

      - name: Merge benchmarks
        if: ${{ github.event_name != 'pull_request' }}
        run: python3 ./tools/metacall-benchmarks-merge.py ./build/benchmarks

      # Download previous benchmark result from cache (if exists)
      - name: Download previous benchmark data
        uses: actions/cache@v1
        if: ${{ github.event_name != 'pull_request' }}
        with:
          path: ./cache
          key: ${{ matrix.os }}-benchmark

      - name: Store benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        if: ${{ github.event_name != 'pull_request' }}
        with:
            name: MetaCall Benchmark (${{ matrix.os }})
            tool: 'googlecpp'
            output-file-path: ./build/benchmarks/metacall-benchmarks.json
            # Access token to deploy GitHub Pages branch
            github-token: ""
            # Disable push and deploy GitHub pages branch automatically
            auto-push: false
            skip-fetch-gh-pages: true
            # Github Pages branch name
            gh-pages-branch: ${{ github.head_ref || github.ref_name }}
            # Output directory
            benchmark-data-dir-path: bench/${{ matrix.os }}
            # Where the previous data file is stored
            external-data-json-path: ./cache/metacall-benchmarks.json

      # Upload benchmark artifacts
      - name: Upload benchmark artifact
        uses: actions/upload-artifact@master
        if: ${{ github.event_name != 'pull_request' }}
        with:
          name: ${{ matrix.os }}-benchmark
          path: ./bench/

  # This job will merge all benchmarks into one artifact
  benchmark-upload:
    runs-on: ubuntu-latest
    if: ${{ github.event_name != 'pull_request' }}
    needs: [benchmark-unix, benchmark-windows]
    steps:
      # Ubuntu
      - uses: actions/download-artifact@master
        with:
          name: ubuntu-latest-benchmark
          path: ./bench/

      # MacOS
      - uses: actions/download-artifact@master
        with:
          name: macos-latest-benchmark
          path: ./bench/

      # Windows
      - uses: actions/download-artifact@master
        with:
          name: windows-2019-benchmark
          path: ./bench/

      # Upload benchmark website artifacts
      - name: Upload benchmark artifact
        uses: actions/upload-pages-artifact@v2
        with:
          path: ./bench/
          name: benchmarks

  benchmark-deploy:
    runs-on: ubuntu-latest
    if: ${{ github.event_name != 'pull_request' }}
    needs: [benchmark-upload]
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    permissions:
      contents: read
      pages: write
      id-token: write
    steps:
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v2
        with:
          artifact_name: benchmarks