Atomic Build and Release #275
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Atomic Build and Release | |
| on: | |
| schedule: | |
| # Run every Monday at 12:00 | |
| - cron: 0 12 * * 1 | |
| workflow_dispatch: | |
| push: | |
| tags: | |
| - "*" | |
| env: | |
| PYTHON_VERSION: "3.11" | |
| MATURIN_VERSION: "1.4.0" | |
| MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} | |
| MATURIN_USERNAME: "seanglaredb" | |
| PROTOC: "${{ github.workspace }}/deps/protoc/bin/protoc" | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| concurrency: | |
| group: release-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| atomic-release: | |
| # This creates a node in the DAG so that all of the "push" tasks | |
| # can depend on all build tasks. | |
| name: Atomic Release | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') | |
| needs: | |
| - python-linux | |
| - python-windows | |
| - python-macos | |
| - kernel-dist | |
| - node-build | |
| steps: | |
| - run: sleep 1 | |
| ######################################################################## | |
| ## | |
| ## Kernel Release | |
| ## | |
| ######################################################################## | |
| kernel-dist: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: windows-latest-8-cores | |
| target: x86_64-pc-windows-msvc | |
| - os: ubuntu-2004-8-cores | |
| target: x86_64-unknown-linux-gnu | |
| - os: macos-12-large | |
| target: x86_64-apple-darwin | |
| - os: macos-13-xlarge | |
| target: aarch64-apple-darwin | |
| name: Build Kernel Release (${{ matrix.target }}) | |
| runs-on: ${{ matrix.os }} | |
| container: ${{ matrix.container }} | |
| env: | |
| DIST_TARGET_TRIPLE: ${{ matrix.target }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ilammy/setup-nasm@v1 | |
| if: matrix.target == 'x86_64-pc-windows-msvc' | |
| - uses: egor-tensin/setup-gcc@v1 | |
| if: matrix.target == 'x86_64-unknown-linux-gnu' | |
| with: | |
| version: 11 | |
| - uses: dtolnay/rust-toolchain@stable | |
| with: | |
| targets: ${{ matrix.target }} | |
| - uses: extractions/setup-just@v2 | |
| with: | |
| just-version: "1.35.0" | |
| - run: just dist | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: dist-${{ matrix.target }}.zip | |
| path: ./target/dist/ | |
| - uses: slackapi/slack-github-action@v1.27.0 | |
| if: ${{ always() }} | |
| with: | |
| payload: | | |
| { | |
| "text": "Build Kernel Binary: ${{ job.status }}", | |
| "attachments": [ | |
| { | |
| "fields": [ | |
| { "title": "target", "value": "${{ matrix.target }}", "short": true }, | |
| { "title": "status", "value": "${{ job.status }}", "short": true }, | |
| { "title": "trigger", "value": "${{ github.triggering_actor}}", "short": true } | |
| ] | |
| } | |
| ] | |
| } | |
| docker-push: | |
| name: Build and Push Docker Image | |
| runs-on: ubuntu-latest-8-cores | |
| needs: [atomic-release] | |
| permissions: | |
| id-token: write # required to use OIDC auth | |
| contents: write # required to checkout code | |
| env: | |
| GCP_PROJECT_ID: glaredb-artifacts | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }} | |
| - uses: google-github-actions/setup-gcloud@v2 | |
| - run: gcloud auth configure-docker -q | |
| - run: ./scripts/build-and-push-images.sh | |
| # TODO: this should pull the binary that was built in kernel-dist. | |
| kernel-push: | |
| name: Push Kernel Release | |
| runs-on: ubuntu-latest | |
| needs: [atomic-release] | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: dist-aarch64-apple-darwin.zip | |
| path: dist | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: dist-x86_64-apple-darwin.zip | |
| path: dist | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: dist-x86_64-unknown-linux-gnu.zip | |
| path: dist | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: dist-x86_64-pc-windows-msvc.zip | |
| path: dist | |
| - run: ls -al ./dist | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: dist-kernel-releases.zip | |
| path: ./dist | |
| - uses: slackapi/slack-github-action@v1.27.0 | |
| if: ${{ always() }} | |
| with: | |
| payload: | | |
| { | |
| "text": "Push Kernel Release: ${{ job.status }}", | |
| "attachments": [ | |
| { | |
| "fields": [ | |
| { "title": "status", "value": "${{ job.status }}", "short": true }, | |
| { "title": "trigger", "value": "${{ github.triggering_actor}}", "short": true } | |
| ] | |
| } | |
| ] | |
| } | |
| ######################################################################## | |
| ## | |
| ## Python Bindings | |
| ## | |
| ######################################################################## | |
| python-linux: | |
| name: Build Python Bindings (linux ${{ matrix.target }}) | |
| runs-on: ubuntu-2004-8-cores | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| target: [x86_64] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: egor-tensin/setup-gcc@v1 | |
| with: | |
| version: 11 | |
| - uses: extractions/setup-just@v2 | |
| with: | |
| just-version: "1.35.0" | |
| - run: just protoc | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: maturin build | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| manylinux: "auto" | |
| docker-options: "--env PROTOC=${{ env.PROTOC }}" | |
| target: ${{ matrix.target }} | |
| args: --release --out dist -m bindings/python/Cargo.toml | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheels.zip | |
| path: dist | |
| - uses: slackapi/slack-github-action@v1.27.0 | |
| if: ${{ always() }} | |
| with: | |
| payload: | | |
| { | |
| "text": "Build Linux Python Bindings: ${{ job.status }}", | |
| "attachments": [ | |
| { | |
| "fields": [ | |
| { "title": "target", "value": "${{ matrix.target }}", "short": true }, | |
| { "title": "status", "value": "${{ job.status }}", "short": true }, | |
| { "title": "trigger", "value": "${{ github.triggering_actor}}", "short": true } | |
| ] | |
| } | |
| ] | |
| } | |
| python-windows: | |
| name: Build Python Bindings (windows ${{ matrix.target }}) | |
| runs-on: windows-latest-8-cores | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| target: [x64] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ilammy/setup-nasm@v1 | |
| - uses: extractions/setup-just@v2 | |
| with: | |
| just-version: "1.35.0" | |
| - run: just protoc | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| architecture: ${{ matrix.target }} | |
| - name: maturin build | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.target }} | |
| args: --release --out dist | |
| working-directory: bindings/python | |
| container: "off" | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheels | |
| path: bindings/python/dist | |
| - uses: slackapi/slack-github-action@v1.27.0 | |
| if: ${{ always() }} | |
| with: | |
| payload: | | |
| { | |
| "text": "Build Windows Python Bindings: ${{ job.status }}", | |
| "attachments": [ | |
| { | |
| "fields": [ | |
| { "title": "target", "value": "Windows ${{ matrix.target }}", "short": true }, | |
| { "title": "trigger", "value": "${{ github.triggering_actor}}", "short": true }, | |
| { "title": "status", "value": "${{ job.status }}", "short": true } | |
| ] | |
| } | |
| ] | |
| } | |
| python-macos: | |
| name: Build Python Bindings (macos ${{ matrix.target }}) | |
| runs-on: ${{ matrix.runner }} | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| include: | |
| - target: x86_64-apple-darwin | |
| runner: macos-13-large | |
| - target: aarch64-apple-darwin | |
| runner: macos-13-xlarge | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: extractions/setup-just@v2 | |
| with: | |
| just-version: "1.35.0" | |
| - run: just protoc | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: maturin build | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.target }} | |
| args: --release --out dist | |
| working-directory: bindings/python | |
| container: "off" | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheels | |
| path: bindings/python/dist | |
| - uses: slackapi/slack-github-action@v1.27.0 | |
| if: ${{ always() }} | |
| with: | |
| payload: | | |
| { | |
| "text": "Build macOS Python Bindings: ${{ job.status }}", | |
| "attachments": [ | |
| { | |
| "fields": [ | |
| { "title": "status", "value": "${{ job.status }}", "short": true }, | |
| { "title": "target", "value": "${{ matrix.target }}", "short": true }, | |
| { "title": "trigger", "value": "${{ github.triggering_actor}}", "short": true } | |
| ] | |
| } | |
| ] | |
| } | |
| python-push: | |
| name: Push Python Bindings | |
| if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') | |
| runs-on: ubuntu-latest | |
| needs: [atomic-release] | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: wheels | |
| - name: maturin publish | |
| uses: PyO3/maturin-action@v1 | |
| env: | |
| MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} | |
| with: | |
| command: upload | |
| args: --skip-existing * | |
| - uses: slackapi/slack-github-action@v1.27.0 | |
| if: ${{ always() }} | |
| with: | |
| payload: | | |
| { | |
| "text": "Push Python Bindings: ${{ job.status }}", | |
| "attachments": [ | |
| { | |
| "fields": [ | |
| { "title": "status", "value": "${{ job.status }}", "short": true }, | |
| { "title": "trigger", "value": "${{ github.triggering_actor}}", "short": true } | |
| ] | |
| } | |
| ] | |
| } | |
| ######################################################################## | |
| ## | |
| ## Node.js Release Process | |
| ## | |
| ######################################################################## | |
| node-build: | |
| defaults: | |
| run: | |
| working-directory: bindings/nodejs | |
| permissions: | |
| contents: write | |
| id-token: write | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| include: | |
| - host: macos-12-xl | |
| target: x86_64-apple-darwin | |
| build: | | |
| yarn build | |
| strip -x *.node | |
| - host: windows-latest-8-cores | |
| build: yarn build | |
| target: x86_64-pc-windows-msvc | |
| - host: ubuntu-2004-8-cores | |
| target: x86_64-unknown-linux-gnu | |
| docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian | |
| build: |- | |
| set -eux && | |
| curl -L https://github.com/protocolbuffers/protobuf/releases/download/v23.1/protoc-23.1-linux-x86_64.zip -o protoc.zip | |
| mkdir -p deps && | |
| unzip -o protoc.zip -d deps/protoc && | |
| rm protoc.zip | |
| export PROTOC="$PWD/deps/protoc/bin/protoc" && | |
| cd bindings/nodejs && | |
| yarn build --target x86_64-unknown-linux-gnu && | |
| strip *.node | |
| - host: macos-12-xl | |
| target: aarch64-apple-darwin | |
| build: | | |
| rustup target add aarch64-apple-darwin | |
| sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/*; | |
| export CC=$(xcrun -f clang); | |
| export CXX=$(xcrun -f clang++); | |
| SYSROOT=$(xcrun --sdk macosx --show-sdk-path); | |
| export CFLAGS="-isysroot $SYSROOT -isystem $SYSROOT"; | |
| yarn build --target aarch64-apple-darwin | |
| strip -x *.node | |
| name: Build Node.JS Bindings ${{ matrix.target }} | |
| runs-on: ${{ matrix.host }} | |
| env: | |
| DEBUG: napi:* | |
| APP_NAME: "glaredb" | |
| MACOSX_DEPLOYMENT_TARGET: "10.13" | |
| PROTOC: "${{ github.workspace }}/deps/protoc/bin/protoc" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: ilammy/setup-nasm@v1 | |
| if: matrix.target == 'x86_64-pc-windows-msvc' | |
| - uses: egor-tensin/setup-gcc@v1 | |
| if: matrix.target == 'x86_64-unknown-linux-gnu' | |
| with: | |
| version: 11 | |
| - uses: extractions/setup-just@v2 | |
| with: | |
| just-version: "1.35.0" | |
| - run: just protoc | |
| - uses: actions/setup-node@v4 | |
| if: ${{ !matrix.docker }} | |
| with: | |
| node-version: 20 | |
| check-latest: true | |
| - uses: dtolnay/rust-toolchain@stable | |
| if: ${{ !matrix.docker }} | |
| with: | |
| toolchain: stable | |
| targets: ${{ matrix.target }} | |
| - uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| .cargo-cache | |
| target/ | |
| key: ${{ matrix.target }}-cargo-${{ matrix.host }} | |
| - uses: goto-bus-stop/setup-zig@v2 | |
| if: ${{ matrix.target == 'armv7-unknown-linux-gnueabihf' }} | |
| with: | |
| version: 0.10.1 | |
| - run: ${{ matrix.setup }} | |
| if: ${{ matrix.setup }} | |
| shell: bash | |
| - if: matrix.target == 'i686-pc-windows-msvc' | |
| run: yarn config set supportedArchitectures.cpu "ia32" | |
| shell: bash | |
| - run: yarn install --frozen-lockfile | |
| - uses: actions/setup-node@v4 | |
| if: matrix.target == 'i686-pc-windows-msvc' | |
| with: | |
| node-version: 20 | |
| check-latest: true | |
| architecture: x86 | |
| - uses: addnab/docker-run-action@v3 | |
| if: ${{ matrix.docker }} | |
| with: | |
| image: ${{ matrix.docker }} | |
| options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build" | |
| run: ${{ matrix.build }} | |
| - run: ${{ matrix.build }} | |
| if: ${{ !matrix.docker }} | |
| shell: bash | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: bindings-${{ matrix.target }} | |
| path: bindings/nodejs/${{ env.APP_NAME }}.*.node | |
| if-no-files-found: error | |
| - uses: slackapi/slack-github-action@v1.27.0 | |
| if: ${{ always() }} | |
| with: | |
| payload: | | |
| { | |
| "text": "Build Node Bindings ${{ matrix.target }}: ${{ job.status }}", | |
| "attachments": [ | |
| { | |
| "fields": [ | |
| { "title": "target", "value": "${{ matrix.target }}", "short": true }, | |
| { "title": "status", "value": "${{ job.status }}", "short": true }, | |
| { "title": "trigger", "value": "${{ github.triggering_actor}}", "short": true } | |
| ] | |
| } | |
| ] | |
| } | |
| node-push: | |
| name: Push Node.js Bindings | |
| if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') | |
| defaults: | |
| run: | |
| working-directory: bindings/nodejs | |
| runs-on: ubuntu-2004-8-cores | |
| needs: [atomic-release] | |
| env: | |
| DEBUG: napi:* | |
| APP_NAME: "glaredb" | |
| MACOSX_DEPLOYMENT_TARGET: "10.13" | |
| PROTOC: "${{ github.workspace }}/deps/protoc/bin/protoc" | |
| permissions: | |
| contents: write | |
| id-token: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: extractions/setup-just@v2 | |
| - run: just protoc | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| check-latest: true | |
| registry-url: "https://registry.npmjs.org" | |
| - run: yarn install --frozen-lockfile | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| path: bindings/nodejs/artifacts | |
| - run: yarn artifacts | |
| - run: ls -R ./npm | |
| shell: bash | |
| - name: publish | |
| run: | | |
| echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc | |
| npm publish | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| NPM_TOKEN: ${{ secrets.NPM_TOKEN }} | |
| NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} | |
| - uses: slackapi/slack-github-action@v1.27.0 | |
| if: ${{ always() }} | |
| with: | |
| payload: | | |
| { | |
| "text": "Push Node Bindings: ${{ job.status }}", | |
| "attachments": [ | |
| { | |
| "fields": [ | |
| { "title": "status", "value": "${{ job.status }}", "short": true }, | |
| { "title": "trigger", "value": "${{ github.triggering_actor}}", "short": true } | |
| ] | |
| } | |
| ] | |
| } |