Skip to content

chore: remove metastore client cruft #6664

chore: remove metastore client cruft

chore: remove metastore client cruft #6664

Workflow file for this run

name: GlareDB CI
on:
push:
branches:
- main
pull_request:
merge_group:
workflow_dispatch:
concurrency:
group: build-ci-${{ github.ref }}
cancel-in-progress: true
jobs:
build:
name: Build
runs-on: ubuntu-latest-8-cores
steps:
- name: checkout
uses: actions/checkout@v4
- uses: extractions/setup-just@v2
with:
just-version: "1.23.0"
- uses: actions/cache@v4
name: toolchain cache
with:
path: ~/.rustup/toolchains/
key: ${{ runner.os }}-toolchain-build-${{ hashFiles('**/rust-toolchain.toml') }}
- uses: actions/cache@v4
name: cargo cache
with:
path: ~/.cargo/
key: ${{ runner.os }}-cargo-pkg-build-${{ hashFiles('**/Cargo.lock') }}
restore-keys: ${{ runner.os }}-cargo-pkg-build-
- uses: actions/cache@v4
name: build-cache
with:
path: |
target/
!target/**/glaredb
!target/**/pgprototest
key: ${{ runner.os }}-cargo-glaredb-build-${{ github.ref_name }}
restore-keys: ${{ runner.os }}-cargo-glaredb-build-${{ hashFiles('**/Cargo.lock') }}
- run: just build
- run: cargo build --bin pgprototest
- uses: actions/cache/save@v4
name: glaredb cache
with:
path: target/debug/glaredb
key: ${{ runner.os }}-glaredb-bin-${{ github.run_id }}
- uses: actions/cache/save@v4
name: pgprototest cache
with:
path: target/debug/pgprototest
key: ${{ runner.os }}-pgprototest-bin-${{ github.run_id }}
fmt:
name: Format (rustfmt +nightly)
needs: ["build"]
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
- uses: extractions/setup-just@v2
with:
just-version: "1.23.0"
- uses: actions/cache@v4
name: nightly toolchain cache
id: lint-toolchain
with:
path: ~/.rustup/toolchains/
key: ${{ runner.os }}-toolchain-lint-${{ hashFiles('**/rustfmt.toml') }}
- run: rustup install nightly
if: ${{ steps.lint-toolchain.outputs.cache-hit != 'true' }}
- run: rustup component add rustfmt --toolchain nightly
if: ${{ steps.lint-toolchain.outputs.cache-hit != 'true' }}
- run: just fmt-check
lint:
name: Lint (clippy)
# emperically runtimes are the same for big/small hosts:
runs-on: ubuntu-latest
needs: ["build"]
steps:
- name: checkout
uses: actions/checkout@v4
- uses: extractions/setup-just@v2
with:
just-version: "1.23.0"
- uses: actions/cache@v4
name: toolchain cache
with:
path: ~/.rustup/toolchains/
key: ${{ runner.os }}-toolchain-build-${{ hashFiles('**/rust-toolchain.toml') }}
- uses: actions/cache@v4
name: cargo cache
with:
path: ~/.cargo/
key: ${{ runner.os }}-cargo-pkg-lint-${{ hashFiles('**/Cargo.lock') }}
restore-keys: ${{ runner.os }}-cargo-pkg-build-${{ hashFiles('**/Cargo.lock') }}
- uses: actions/cache/restore@v4
name: build cache
with:
path: |
target/
!target/**/glaredb
!target/**/pgprototest
key: ${{ runner.os }}-cargo-glaredb-build-${{ github.ref_name }}
- run: just clippy
python-binding-tests:
name: Python Binding Tests
runs-on: ubuntu-latest-4-cores
needs: ["build"]
steps:
- name: checkout
uses: actions/checkout@v4
- uses: extractions/setup-just@v2
with:
just-version: "1.23.0"
- uses: actions/cache/restore@v4
name: toolchain cache
with:
path: ~/.rustup/toolchains/
key: ${{ runner.os }}-toolchain-build-${{ hashFiles('**/rust-toolchain.toml') }}
- uses: actions/cache@v4
name: cargo cache
with:
path: ~/.cargo/
key: ${{ runner.os }}-cargo-pkg-python-${{ hashFiles('**/Cargo.lock') }}
restore-keys: ${{ runner.os }}-cargo-pkg-build-${{ hashFiles('**/Cargo.lock') }}
- uses: actions/cache/restore@v4
name: build cache
with:
path: |
target/
!target/**/glaredb
!target/**/pgprototest
key: ${{ runner.os }}-cargo-glaredb-build-${{ github.ref_name }}
- run: just python venv
- run: just python build
- run: just python test
nodejs-bindings-tests:
name: Node.js Binding Tests
runs-on: ubuntu-latest-8-cores
needs: ["build"]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
- uses: extractions/setup-just@v2
with:
just-version: "1.23.0"
- uses: actions/cache/restore@v4
name: toolchain cache
with:
path: |
~/.rustup/toolchains/
key: ${{ runner.os }}-toolchain-build-${{ hashFiles('**/rust-toolchain.toml') }}
- uses: actions/cache@v4
name: cargo cache
with:
path: ~/.cargo/
key: ${{ runner.os }}-cargo-pkg-build-${{ hashFiles('**/Cargo.lock') }}
restore-keys: ${{ runner.os }}-cargo-pkg-build-
- uses: actions/cache/restore@v4
name: build cache
with:
path: |
target/
!target/**/glaredb
!target/**/pgprototest
key: ${{ runner.os }}-cargo-glaredb-build-${{ github.ref_name }}
- run: just js build-debug
- run: just js test
unit-tests:
name: Unit Tests
runs-on: ubuntu-latest-8-cores
needs: ["build"]
steps:
- name: checkout
uses: actions/checkout@v4
- uses: extractions/setup-just@v2
with:
just-version: "1.23.0"
- uses: actions/cache/restore@v4
name: toolchain cache
with:
path: ~/.rustup/toolchains/
key: ${{ runner.os }}-toolchain-build-${{ hashFiles('**/rust-toolchain.toml') }}
- uses: actions/cache@v4
name: cargo cache
with:
path: ~/.cargo/
key: ${{ runner.os }}-cargo-pkg-unit-${{ hashFiles('**/Cargo.lock') }}
- uses: actions/cache/restore@v4
name: build cache
with:
path: |
target/
!target/**/glaredb
!target/**/pgprototest
key: ${{ runner.os }}-cargo-glaredb-build-${{ github.ref_name }}
- uses: actions/cache/restore@v4
name: glaredb cache
with:
path: target/debug/glaredb
key: ${{ runner.os }}-glaredb-bin-${{ github.run_id }}
fail-on-cache-miss: true
- run: just unit-tests
cache:
# when we change the Cargo.lock we should copy the branch cache
# from this build to the expected cache location to better
# populate caches.
name: Cache Maintenance
runs-on: ubuntu-latest
needs: ["build"]
# parent branches can't use caches from child branches, so no need
# to update this cache _except_ on main.
if: github.ref == 'refs/heads/main'
steps:
- uses: actions/cache@v4
name: cache check
id: target-for-cargo-hash
with:
lookup-only: true
path: |
target/
!target/**/glaredb
!target/**/pgprototest
key: ${{ runner.os }}-cargo-pkg-build-${{ hashFiles('**/Cargo.lock') }}
- uses: actions/cache/restore@v4
name: build cache restore
if: ${{ steps.target-for-cargo-hash.cache-hit != 'true' }}
with:
path: |
target/
!target/**/glaredb
!target/**/pgprototest
key: ${{ runner.os }}-cargo-glaredb-build-${{ github.ref_name }}
- uses: actions/cache/save@v4
name: cache update
if: ${{ steps.target-for-cargo-hash.cache-hit != 'true' }}
with:
path: |
target/
!target/**/glaredb
!target/**/pgprototest
key: ${{ runner.os }}-cargo-pkg-build-${{ hashFiles('**/Cargo.lock') }}
pg-protocol:
name: PG Protocol Tests
runs-on: ubuntu-latest
needs: ["build"]
steps:
- name: checkout
uses: actions/checkout@v4
- uses: extractions/setup-just@v2
with:
just-version: "1.23.0"
- uses: actions/cache/restore@v4
name: glaredb cache
with:
path: target/debug/glaredb
key: ${{ runner.os }}-glaredb-bin-${{ github.run_id }}
fail-on-cache-miss: true
- uses: actions/cache/restore@v4
name: pgprototest cache
with:
path: target/debug/pgprototest
key: ${{ runner.os }}-pgprototest-bin-${{ github.run_id }}
fail-on-cache-miss: true
- run: ./scripts/protocol-test.sh
- run: just slt-bin-debug 'pgproto/*'
sql-logic-tests:
name: SQL Logic Tests
runs-on: ubuntu-latest
needs: ["build"]
strategy:
matrix:
protocol: ["postgres", "flightsql", "rpc"]
include:
- protocol: "postgres"
basic: just slt-bin --protocol=postgres 'sqllogictests/*'
debug: just slt-bin-debug --protocol=postgres 'sqllogictests/*'
- protocol: "flightsql"
basic: just slt-bin --protocol=flightsql 'sqllogictests/*'
debug: just slt-bin-debug --protocol=flightsql 'sqllogictests/*'
- protocol: "rpc"
basic: just rpc-tests
debug: just rpc-tests
steps:
- name: checkout
uses: actions/checkout@v4
- uses: extractions/setup-just@v2
with:
just-version: "1.23.0"
- uses: actions/cache/restore@v4
name: glaredb cache
with:
path: target/debug/glaredb
key: ${{ runner.os }}-glaredb-bin-${{ github.run_id }}
fail-on-cache-miss: true
- name: public sql logic tests DEBUG
if: ${{ env.ACTIONS_STEP_DEBUG == 'true' }}
run: ${{matrix.protocol.debug}}
- name: public sql logic tests
if: ${{ env.ACTIONS_STEP_DEBUG != 'true' }}
run: ${{matrix.protocol.basic}}
process-integration-tests:
name: Process Integration Tests (pytest)
runs-on: ubuntu-latest
needs: ["build"]
steps:
- name: checkout
uses: actions/checkout@v4
- uses: extractions/setup-just@v2
with:
just-version: "1.23.0"
- name: install poetry
uses: snok/install-poetry@v1
- name: install python
uses: actions/setup-python@v5
with:
python-version: "3.11"
cache: poetry
cache-dependency-path: tests/poetry.lock
- uses: actions/cache/restore@v4
name: glaredb cache
with:
path: target/debug/glaredb
key: ${{ runner.os }}-glaredb-bin-${{ github.run_id }}
fail-on-cache-miss: true
- run: just venv
- run: just pytest
service-integration-tests:
name: Object Store Service Integration Tests (SLT)
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == 'GlareDB'
runs-on: ubuntu-latest
needs: ["sql-logic-tests"]
steps:
- name: checkout
uses: actions/checkout@v4
- uses: extractions/setup-just@v2
with:
just-version: "1.23.0"
- uses: actions/cache/restore@v4
name: glaredb cache
with:
path: target/debug/glaredb
key: ${{ runner.os }}-glaredb-bin-${{ github.run_id }}
fail-on-cache-miss: true
- name: GCP authenticate
uses: google-github-actions/auth@v2
with:
credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}
- name: setup GCP
uses: google-github-actions/setup-gcloud@v2
- run: ./scripts/prepare-testdata.sh
- name: run tests (slt)
env:
GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}
GCP_PROJECT_ID: glaredb-artifacts
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AZURE_ACCESS_KEY: ${{ secrets.AZURE_ACCESS_KEY }}
AZURE_ACCOUNT: ${{ secrets.AZURE_ACCOUNT }}
TEST_BUCKET: glaredb-test-bucket
run: |
# Prepare SLT (Object store)
export GCS_BUCKET_NAME=glaredb-test
export AWS_S3_REGION=us-east-1
export AWS_S3_BUCKET_NAME=glaredb-test
# Unset application default credentials. We don't want to unknowingly
# depend on this.
unset GOOGLE_APPLICATION_CREDENTIALS
echo "-------------------------- Iceberg -------------------------- "
just slt-bin --protocol=rpc 'sqllogictests_iceberg/*'
echo "-------------------------- Native -------------------------- "
just slt-bin --protocol=rpc 'sqllogictests_native/*'
echo "-------------------------- Object Store -------------------------- "
just slt-bin --protocol=rpc 'sqllogictests_object_store/*'
echo "-------------------------- Fake GCS server with a subdirectory -------------------------- "
./scripts/create-test-gcs-store.sh # prepare fake GCS server
just slt-bin -l gs://$TEST_BUCKET/path/to/folder/1 -o service_account_path=/tmp/fake-gcs-creds.json 'sqllogictests_native/*'
just slt-bin -l gs://$TEST_BUCKET/path/to/folder/2 -o service_account_path=/tmp/fake-gcs-creds.json 'sqllogictests_native/*'
bigquery-integration-tests:
name: Big Query Integration Tests (SLT::MinIO)
runs-on: ubuntu-latest
needs: ["sql-logic-tests"]
steps:
- name: checkout
uses: actions/checkout@v4
- uses: extractions/setup-just@v2
with:
just-version: "1.23.0"
- uses: actions/cache/restore@v4
name: glaredb cache
with:
path: target/debug/glaredb
key: ${{ runner.os }}-glaredb-bin-${{ github.run_id }}
fail-on-cache-miss: true
- name: GCP authenticate
uses: google-github-actions/auth@v2
with:
credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}
- name: setup GCP
uses: google-github-actions/setup-gcloud@v2
- run: ./scripts/prepare-testdata.sh
- name: run tests (slt)
env:
GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}
GCP_PROJECT_ID: glaredb-artifacts
TEST_BUCKET: glaredb-test-bucket
run: |
# Prepare SLT (BigQuery)
if ./scripts/files-changed-in-branch.sh \
"scripts/prepare-testdata.sh" \
"scripts/create-test-bigquery-db.sh" \
"testdata/sqllogictests_datasources_common/data" \
"testdata/sqllogictests_bigquery/data"
then
export GCP_PROJECT_ID=glaredb-dev-playground
export BIGQUERY_DATASET_ID=$(./scripts/create-test-bigquery-db.sh)
else
export BIGQUERY_DATASET_ID=glaredb_test
fi
# Unset application default credentials. We don't want to unknowingly
# depend on this.
unset GOOGLE_APPLICATION_CREDENTIALS
just slt-bin --protocol=rpc 'sqllogictests_bigquery/*'
minio-integration-tests:
name: MinIO Integration Tests (SLT::MinIO)
runs-on: ubuntu-latest
needs: ["sql-logic-tests"]
env:
MINIO_ACCESS_KEY: glaredb
MINIO_SECRET_KEY: glaredb_test
TEST_BUCKET: glaredb-test-bucket
steps:
- name: checkout
uses: actions/checkout@v4
- uses: extractions/setup-just@v2
with:
just-version: "1.23.0"
- uses: actions/cache/restore@v4
name: glaredb cache
with:
path: target/debug/glaredb
key: ${{ runner.os }}-glaredb-bin-${{ github.run_id }}
fail-on-cache-miss: true
- run: ./scripts/prepare-testdata.sh
- run: ./scripts/create-test-minio-store.sh
- name: run tests pg (slt)
run: |
echo "-------------------------- MinIO (S3) PostgreSQL Protocol -------------------------- "
# PG protocol
just slt-bin --location http://localhost:9100 \
--option access_key_id=$MINIO_ACCESS_KEY \
--option secret_access_key=$MINIO_SECRET_KEY \
--option bucket=$TEST_BUCKET \
'sqllogictests/*' \
'sqllogictests_native/*'
echo "-------------------------- MinIO (S3) with a sub-directory -------------------------- "
# PG protocol
just slt-bin -l http://localhost:9100/$TEST_BUCKET/path/to/folder \
-o access_key_id=$MINIO_ACCESS_KEY \
-o secret_access_key=$MINIO_SECRET_KEY \
'sqllogictests/*'
- name: run tests FlightSQL (slt)
run: |
echo "-------------------------- MinIO (S3) FlightSQL -------------------------- "
just slt-bin --location http://localhost:9100 \
--option access_key_id=$MINIO_ACCESS_KEY \
--option secret_access_key=$MINIO_SECRET_KEY \
--option bucket=$TEST_BUCKET \
'sqllogictests/*' \
'sqllogictests_native/*' \
--protocol=flightsql
echo "-------------------------- MinIO (S3) with a sub-directory -------------------------- "
just slt-bin -l http://localhost:9100/$TEST_BUCKET/path/to/folder \
-o access_key_id=$MINIO_ACCESS_KEY \
-o secret_access_key=$MINIO_SECRET_KEY \
'sqllogictests/*' \
--protocol=flightsql
service-integration-tests-snowflake:
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == 'GlareDB'
name: Snowflake Integration Tests (SLT::Snowflake)
runs-on: ubuntu-latest
needs: ["sql-logic-tests"]
concurrency:
group: snowflake-integration-tests
cancel-in-progress: false
steps:
- name: checkout
uses: actions/checkout@v4
- uses: extractions/setup-just@v2
with:
just-version: "1.23.0"
- uses: actions/cache/restore@v4
name: glaredb cache
with:
path: target/debug/glaredb
key: ${{ runner.os }}-glaredb-bin-${{ github.run_id }}
fail-on-cache-miss: true
- name: snowflake setup (SnowSQL)
run: |
curl -o snowsql.bash \
https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.24-linux_x86_64.bash
mkdir -p ~/bin
SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql.bash
- run: ./scripts/prepare-testdata.sh
- name: run tests (slt)
env:
SNOWFLAKE_USERNAME: ${{ secrets.SNOWFLAKE_USERNAME }}
SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
run: |
# Prepare SLT (Snowflake)
export PATH="$HOME/bin:$PATH"
if ./scripts/files-changed-in-branch.sh \
"scripts/prepare-testdata.sh" \
"scripts/create-test-snowflake-db.sh" \
"testdata/sqllogictests_datasources_common/data" \
"testdata/sqllogictests_snowflake/data"
then
export SNOWFLAKE_DATABASE=$(./scripts/create-test-snowflake-db.sh)
else
export SNOWFLAKE_DATABASE=glaredb_test
fi
just slt-bin 'sqllogictests_snowflake/*'
just slt-bin --protocol=rpc 'sqllogictests_snowflake/*'
datasource-integration-tests:
name: Datasource Integration (${{matrix.settings.name}})
strategy:
matrix:
settings:
- name: Clickhouse
path: "sqllogictests_clickhouse/*"
prepare: |
./scripts/prepare-testdata.sh
source ./scripts/ci-install-clickhouse.sh
export CLICKHOUSE_CONN_STRING=$(./scripts/create-test-clickhouse-db.sh)
- name: SQLite
path: "sqllogictests_sqlite/*"
prepare: |
./scripts/prepare-testdata.sh
# If there's an old data-set in the cache, remove it so we can create a new one.
test -f testdata/sqllogictests_sqlite/data/db.sqlite3 && rm testdata/sqllogictests_sqlite/data/db.sqlite3
- name: Cassandra
path: "sqllogictests_cassandra/*"
prepare: |
export CASSANDRA_CONN_STRING=$(./scripts/create-test-cassandra-db.sh | tail -n 1)
- name: MySQL
path: "sqllogictests_mysql/*"
prepare: |
./scripts/prepare-testdata.sh
export MYSQL_CONN_STRING=$(./scripts/create-test-mysql-db.sh)
export MYSQL_TUNNEL_SSH_CONN_STRING=$(echo "$MYSQL_CONN_STRING" | sed -n 2p)
export MYSQL_CONN_STRING=$(echo "$MYSQL_CONN_STRING" | sed -n 1p)
- name: MongoDB
path: "sqllogictests_mongodb/*"
prepare: |
./scripts/prepare-testdata.sh
export MONGO_CONN_STRING=$(./scripts/create-test-mongodb.sh)
./scripts/create-fixture-mongodb.sh
- name: SQLServer
path: "sqllogictests_sqlserver/*"
prepare: |
./scripts/prepare-testdata.sh
export SQL_SERVER_CONN_STRING=$(./scripts/create-test-sqlserver-db.sh)
- name: PostgreSQL
path: "sqllogictests_postgres/*"
prepare: |
./scripts/prepare-testdata.sh
export POSTGRES_TEST_DB=$(./scripts/create-test-postgres-db.sh)
export POSTGRES_CONN_STRING=$(echo "$POSTGRES_TEST_DB" | sed -n 1p)
export POSTGRES_TUNNEL_SSH_CONN_STRING=$(echo "$POSTGRES_TEST_DB" | sed -n 2p)
runs-on: ubuntu-latest
needs: ["sql-logic-tests"]
steps:
- name: checkout
uses: actions/checkout@v4
- uses: extractions/setup-just@v2
with:
just-version: "1.23.0"
- uses: actions/cache/restore@v4
name: glaredb cache
with:
path: target/debug/glaredb
key: ${{ runner.os }}-glaredb-bin-${{ github.run_id }}
fail-on-cache-miss: true
- name: run tests (slt)
run: |
${{matrix.settings.prepare}}
just slt-bin ${{matrix.settings.path}}
[ "${{matrix.settings.name}}" = "MongoDB" ] && ./scripts/create-fixture-mongodb.sh
just slt-bin --protocol=rpc ${{matrix.settings.path}}
[ "${{matrix.settings.name}}" = "MongoDB" ] && ./scripts/create-fixture-mongodb.sh
# for sqlserver, skip flightsql because the suite takes 4-5
# minutes, and sqlserver is the longest/last task to finish
[ "${{matrix.settings.name}}" = "SQLServer" ] && exit 0
just slt-bin --protocol=flightsql ${{matrix.settings.path}}
docker-push:
name: Build and Push Docker Image
if: github.event_name == 'push' && github.repository == 'GlareDB/glaredb' && github.ref == 'refs/heads/main'
needs: ["sql-logic-tests"]
runs-on: ubuntu-latest-8-cores
permissions:
id-token: write # required to use OIDC auth
contents: write # required to checkout code
env:
GCP_PROJECT_ID: glaredb-artifacts
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Authenticate to Google Cloud
uses: google-github-actions/auth@v2
with:
credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}
- name: Setup gcloud
uses: google-github-actions/setup-gcloud@v2
- run: gcloud auth configure-docker -q
- run: ./scripts/build-and-push-images.sh