release.yaml

name: Atomic Build and Release

on:
  schedule:
    # Run every Monday at 12:00
    - cron: 0 12 * * 1
  workflow_dispatch:
  push:
    tags:
      - "*"

env:
  PYTHON_VERSION: "3.11"
  MATURIN_VERSION: "1.4.0"
  MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
  MATURIN_USERNAME: "seanglaredb"
  PROTOC: "${{ github.workspace }}/deps/protoc/bin/protoc"
  SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
  SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK

concurrency:
  group: release-${{ github.ref }}
  cancel-in-progress: true

jobs:
  atomic-release:
    # This creates a node in the DAG so that all of the "push" tasks
    # can depend on all build tasks.
    name: Atomic Release
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
    needs:
      - python-linux
      - python-windows
      - python-macos
      - kernel-dist
      - node-build
    steps:
      - run: sleep 1

  ########################################################################
  ##
  ## Kernel Release
  ##
  ########################################################################

  kernel-dist:
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: windows-latest-8-cores
            target: x86_64-pc-windows-msvc
          - os: ubuntu-2004-8-cores
            target: x86_64-unknown-linux-gnu
          - os: macos-12-large
            target: x86_64-apple-darwin
          - os: macos-13-xlarge
            target: aarch64-apple-darwin
    name: Build Kernel Release (${{ matrix.target }})
    runs-on: ${{ matrix.os }}
    container: ${{ matrix.container }}
    env:
      DIST_TARGET_TRIPLE: ${{ matrix.target }}
    steps:
      - uses: actions/checkout@v4
      - uses: ilammy/setup-nasm@v1
        if: matrix.target == 'x86_64-pc-windows-msvc'
      - uses: egor-tensin/setup-gcc@v1
        if: matrix.target == 'x86_64-unknown-linux-gnu'
        with:
          version: 11
      - uses: dtolnay/rust-toolchain@stable
        with:
          targets: ${{ matrix.target }}
      - uses: extractions/setup-just@v2
        with:
          just-version: "1.35.0"
      - run: just dist
      - uses: actions/upload-artifact@v3
        with:
          name: dist-${{ matrix.target }}.zip
          path: ./target/dist/
      - uses: slackapi/slack-github-action@v1.27.0
        if: ${{ always() }}
        with:
          payload: |
            {
              "text": "Build Kernel Binary: ${{ job.status }}",
              "attachments": [
                 {
                   "fields": [
                      { "title": "target", "value": "${{ matrix.target }}", "short": true },
                      { "title": "status", "value": "${{ job.status }}", "short": true },
                      { "title": "trigger", "value": "${{ github.triggering_actor}}", "short": true }
                   ]
                 }
              ]
            }

  docker-push:
    name: Build and Push Docker Image
    runs-on: ubuntu-latest-8-cores
    needs: [atomic-release]
    permissions:
      id-token: write # required to use OIDC auth
      contents: write # required to checkout code
    env:
      GCP_PROJECT_ID: glaredb-artifacts
    steps:
      - uses: actions/checkout@v4
      - uses: google-github-actions/auth@v2
        with:
          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}
      - uses: google-github-actions/setup-gcloud@v2
      - run: gcloud auth configure-docker -q
      - run: ./scripts/build-and-push-images.sh
        # TODO: this should pull the binary that was built in kernel-dist.

  kernel-push:
    name: Push Kernel Release
    runs-on: ubuntu-latest
    needs: [atomic-release]
    steps:
      - uses: actions/download-artifact@v3
        with:
          name: dist-aarch64-apple-darwin.zip
          path: dist
      - uses: actions/download-artifact@v3
        with:
          name: dist-x86_64-apple-darwin.zip
          path: dist
      - uses: actions/download-artifact@v3
        with:
          name: dist-x86_64-unknown-linux-gnu.zip
          path: dist
      - uses: actions/download-artifact@v3
        with:
          name: dist-x86_64-pc-windows-msvc.zip
          path: dist
      - run: ls -al ./dist
      - uses: actions/upload-artifact@v3
        with:
          name: dist-kernel-releases.zip
          path: ./dist
      - uses: slackapi/slack-github-action@v1.27.0
        if: ${{ always() }}
        with:
          payload: |
            {
              "text": "Push Kernel Release: ${{ job.status }}",
              "attachments": [
                 {
                   "fields": [
                      { "title": "status", "value": "${{ job.status }}", "short": true },
                      { "title": "trigger", "value": "${{ github.triggering_actor}}", "short": true }
                   ]
                 }
              ]
            }

  ########################################################################
  ##
  ## Python Bindings
  ##
  ########################################################################

  python-linux:
    name: Build Python Bindings (linux ${{ matrix.target }})
    runs-on: ubuntu-2004-8-cores
    permissions:
      contents: read
    strategy:
      fail-fast: true
      matrix:
        target: [x86_64]
    steps:
      - uses: actions/checkout@v4
      - uses: egor-tensin/setup-gcc@v1
        with:
          version: 11
      - uses: extractions/setup-just@v2
        with:
          just-version: "1.35.0"
      - run: just protoc
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: maturin build
        uses: PyO3/maturin-action@v1
        with:
          manylinux: "auto"
          docker-options: "--env PROTOC=${{ env.PROTOC }}"
          target: ${{ matrix.target }}
          args: --release --out dist -m bindings/python/Cargo.toml
      - uses: actions/upload-artifact@v3
        with:
          name: wheels.zip
          path: dist
      - uses: slackapi/slack-github-action@v1.27.0
        if: ${{ always() }}
        with:
          payload: |
            {
              "text": "Build Linux Python Bindings: ${{ job.status }}",
              "attachments": [
                 {
                   "fields": [
                      { "title": "target", "value": "${{ matrix.target }}", "short": true },
                      { "title": "status", "value": "${{ job.status }}", "short": true },
                      { "title": "trigger", "value": "${{ github.triggering_actor}}", "short": true }
                   ]
                 }
              ]
            }

  python-windows:
    name: Build Python Bindings (windows ${{ matrix.target }})
    runs-on: windows-latest-8-cores
    permissions:
      contents: read
    strategy:
      fail-fast: true
      matrix:
        target: [x64]
    steps:
      - uses: actions/checkout@v4
      - uses: ilammy/setup-nasm@v1
      - uses: extractions/setup-just@v2
        with:
          just-version: "1.35.0"
      - run: just protoc
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          architecture: ${{ matrix.target }}
      - name: maturin build
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.target }}
          args: --release --out dist
          working-directory: bindings/python
          container: "off"
      - uses: actions/upload-artifact@v3
        with:
          name: wheels
          path: bindings/python/dist
      - uses: slackapi/slack-github-action@v1.27.0
        if: ${{ always() }}
        with:
          payload: |
            {
              "text": "Build Windows Python Bindings: ${{ job.status }}",
              "attachments": [
                 {
                   "fields": [
                      { "title": "target", "value": "Windows ${{ matrix.target }}", "short": true },
                      { "title": "trigger", "value": "${{ github.triggering_actor}}", "short": true },
                      { "title": "status", "value": "${{ job.status }}", "short": true }
                   ]
                 }
              ]
            }

  python-macos:
    name: Build Python Bindings (macos ${{ matrix.target }})
    runs-on: ${{ matrix.runner }}
    permissions:
      contents: read
    strategy:
      fail-fast: true
      matrix:
        include:
          - target: x86_64-apple-darwin
            runner: macos-13-large
          - target: aarch64-apple-darwin
            runner: macos-13-xlarge
    steps:
      - uses: actions/checkout@v4
      - uses: extractions/setup-just@v2
        with:
          just-version: "1.35.0"
      - run: just protoc
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: maturin build
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.target }}
          args: --release --out dist
          working-directory: bindings/python
          container: "off"
      - uses: actions/upload-artifact@v3
        with:
          name: wheels
          path: bindings/python/dist
      - uses: slackapi/slack-github-action@v1.27.0
        if: ${{ always() }}
        with:
          payload: |
            {
              "text": "Build macOS Python Bindings: ${{ job.status }}",
              "attachments": [
                 {
                   "fields": [
                      { "title": "status", "value": "${{ job.status }}", "short": true },
                      { "title": "target", "value": "${{ matrix.target }}", "short": true },
                      { "title": "trigger", "value": "${{ github.triggering_actor}}", "short": true }
                   ]
                 }
              ]
            }

  python-push:
    name: Push Python Bindings
    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
    runs-on: ubuntu-latest
    needs: [atomic-release]
    permissions:
      contents: read
    steps:
      - uses: actions/download-artifact@v3
        with:
          name: wheels
      - name: maturin publish
        uses: PyO3/maturin-action@v1
        env:
          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
        with:
          command: upload
          args: --skip-existing *
      - uses: slackapi/slack-github-action@v1.27.0
        if: ${{ always() }}
        with:
          payload: |
            {
              "text": "Push Python Bindings: ${{ job.status }}",
              "attachments": [
                 {
                   "fields": [
                      { "title": "status", "value": "${{ job.status }}", "short": true },
                      { "title": "trigger", "value": "${{ github.triggering_actor}}", "short": true }
                   ]
                 }
              ]
            }

  ########################################################################
  ##
  ## Node.js Release Process
  ##
  ########################################################################

  node-build:
    defaults:
      run:
        working-directory: bindings/nodejs
    permissions:
      contents: write
      id-token: write
    strategy:
      fail-fast: true
      matrix:
        include:
          - host: macos-12-xl
            target: x86_64-apple-darwin
            build: |
              yarn build
              strip -x *.node
          - host: windows-latest-8-cores
            build: yarn build
            target: x86_64-pc-windows-msvc
          - host: ubuntu-2004-8-cores
            target: x86_64-unknown-linux-gnu
            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
            build: |-
              set -eux &&
              curl -L https://github.com/protocolbuffers/protobuf/releases/download/v23.1/protoc-23.1-linux-x86_64.zip -o protoc.zip
              mkdir -p deps &&
              unzip -o protoc.zip -d deps/protoc &&
              rm protoc.zip
              export PROTOC="$PWD/deps/protoc/bin/protoc" &&
              cd bindings/nodejs &&
              yarn build --target x86_64-unknown-linux-gnu &&
              strip *.node
          - host: macos-12-xl
            target: aarch64-apple-darwin
            build: |
              rustup target add aarch64-apple-darwin
              sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/*;
              export CC=$(xcrun -f clang);
              export CXX=$(xcrun -f clang++);
              SYSROOT=$(xcrun --sdk macosx --show-sdk-path);
              export CFLAGS="-isysroot $SYSROOT -isystem $SYSROOT";
              yarn build --target aarch64-apple-darwin
              strip -x *.node
    name: Build Node.JS Bindings ${{ matrix.target }}
    runs-on: ${{ matrix.host }}
    env:
      DEBUG: napi:*
      APP_NAME: "glaredb"
      MACOSX_DEPLOYMENT_TARGET: "10.13"
      PROTOC: "${{ github.workspace }}/deps/protoc/bin/protoc"
    steps:
      - uses: actions/checkout@v4
      - uses: ilammy/setup-nasm@v1
        if: matrix.target == 'x86_64-pc-windows-msvc'
      - uses: egor-tensin/setup-gcc@v1
        if: matrix.target == 'x86_64-unknown-linux-gnu'
        with:
          version: 11
      - uses: extractions/setup-just@v2
        with:
          just-version: "1.35.0"
      - run: just protoc
      - uses: actions/setup-node@v4
        if: ${{ !matrix.docker }}
        with:
          node-version: 20
          check-latest: true
      - uses: dtolnay/rust-toolchain@stable
        if: ${{ !matrix.docker }}
        with:
          toolchain: stable
          targets: ${{ matrix.target }}
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            .cargo-cache
            target/
          key: ${{ matrix.target }}-cargo-${{ matrix.host }}
      - uses: goto-bus-stop/setup-zig@v2
        if: ${{ matrix.target == 'armv7-unknown-linux-gnueabihf' }}
        with:
          version: 0.10.1
      - run: ${{ matrix.setup }}
        if: ${{ matrix.setup }}
        shell: bash
      - if: matrix.target == 'i686-pc-windows-msvc'
        run: yarn config set supportedArchitectures.cpu "ia32"
        shell: bash
      - run: yarn install --frozen-lockfile
      - uses: actions/setup-node@v4
        if: matrix.target == 'i686-pc-windows-msvc'
        with:
          node-version: 20
          check-latest: true
          architecture: x86
      - uses: addnab/docker-run-action@v3
        if: ${{ matrix.docker }}
        with:
          image: ${{ matrix.docker }}
          options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build"
          run: ${{ matrix.build }}
      - run: ${{ matrix.build }}
        if: ${{ !matrix.docker }}
        shell: bash
      - uses: actions/upload-artifact@v3
        with:
          name: bindings-${{ matrix.target }}
          path: bindings/nodejs/${{ env.APP_NAME }}.*.node
          if-no-files-found: error
      - uses: slackapi/slack-github-action@v1.27.0
        if: ${{ always() }}
        with:
          payload: |
            {
              "text": "Build Node Bindings ${{ matrix.target }}: ${{ job.status }}",
              "attachments": [
                 {
                   "fields": [
                      { "title": "target", "value": "${{ matrix.target }}", "short": true },
                      { "title": "status", "value": "${{ job.status }}", "short": true },
                      { "title": "trigger", "value": "${{ github.triggering_actor}}", "short": true }
                   ]
                 }
              ]
            }

  node-push:
    name: Push Node.js Bindings
    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
    defaults:
      run:
        working-directory: bindings/nodejs
    runs-on: ubuntu-2004-8-cores
    needs: [atomic-release]
    env:
      DEBUG: napi:*
      APP_NAME: "glaredb"
      MACOSX_DEPLOYMENT_TARGET: "10.13"
      PROTOC: "${{ github.workspace }}/deps/protoc/bin/protoc"
    permissions:
      contents: write
      id-token: write
    steps:
      - uses: actions/checkout@v4
      - uses: extractions/setup-just@v2
      - run: just protoc
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          check-latest: true
          registry-url: "https://registry.npmjs.org"
      - run: yarn install --frozen-lockfile
      - uses: actions/download-artifact@v3
        with:
          path: bindings/nodejs/artifacts
      - run: yarn artifacts
      - run: ls -R ./npm
        shell: bash
      - name: publish
        run: |
          echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc
          npm publish
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
      - uses: slackapi/slack-github-action@v1.27.0
        if: ${{ always() }}
        with:
          payload: |
            {
              "text": "Push Node Bindings: ${{ job.status }}",
              "attachments": [
                 {
                   "fields": [
                      { "title": "status", "value": "${{ job.status }}", "short": true },
                      { "title": "trigger", "value": "${{ github.triggering_actor}}", "short": true }
                   ]
                 }
              ]
            }