Skip to content

Commit

Permalink
Merge branch 'main' into tycho/python-deps
Browse files Browse the repository at this point in the history
  • Loading branch information
@tychoish
tychoish authored Feb 28, 2024
2 parents 4df441f + 0eabfad commit fdeecb1
Show file tree
Hide file tree
Showing 199 changed files with 8,406 additions and 2,084 deletions.
125 changes: 58 additions & 67 deletions .github/workflows/ci.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,14 @@ jobs:
key: ${{ runner.os }}-cargo-pkg-build-${{ hashFiles('**/Cargo.lock') }}
restore-keys: ${{ runner.os }}-cargo-pkg-build-
- uses: actions/cache@v4
name: build cache
name: build-cache
with:
path: |
target/
!target/**/glaredb
!target/**/pgprototest
key: ${{ runner.os }}-cargo-glaredb-build-${{ github.ref_name }}
restore-keys: |
${{ runner.os }}-cargo-glaredb-build-${{ hashFiles('**/Cargo.lock') }}
${{ runner.os }}-cargo-glaredb-build-
restore-keys: ${{ runner.os }}-cargo-glaredb-build-${{ hashFiles('**/Cargo.lock') }}
- run: just build
- run: cargo build --bin pgprototest
- uses: actions/cache/save@v4
Expand All @@ -57,7 +55,6 @@ jobs:
path: target/debug/pgprototest
key: ${{ runner.os }}-pgprototest-bin-${{ github.run_id }}


fmt:
name: Format (rustfmt +nightly)
needs: ["build"]
Expand All @@ -80,7 +77,6 @@ jobs:
if: ${{ steps.lint-toolchain.outputs.cache-hit != 'true' }}
- run: just fmt-check


lint:
name: Lint (clippy)
# emperically runtimes are the same for big/small hosts:
Expand All @@ -102,10 +98,7 @@ jobs:
with:
path: ~/.cargo/
key: ${{ runner.os }}-cargo-pkg-lint-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-pkg-build-${{ hashFiles('**/Cargo.lock') }}
${{ runner.os }}-cargo-pkg-lint-
${{ runner.os }}-cargo-pkg-build-
restore-keys: ${{ runner.os }}-cargo-pkg-build-${{ hashFiles('**/Cargo.lock') }}
- uses: actions/cache/restore@v4
name: build cache
with:
Expand All @@ -116,7 +109,6 @@ jobs:
key: ${{ runner.os }}-cargo-glaredb-build-${{ github.ref_name }}
- run: just clippy


python-binding-tests:
name: Python Binding Tests
runs-on: ubuntu-latest-4-cores
Expand All @@ -137,11 +129,8 @@ jobs:
with:
path: ~/.cargo/
key: ${{ runner.os }}-cargo-pkg-python-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-pkg-build-${{ hashFiles('**/Cargo.lock') }}
${{ runner.os }}-cargo-pkg-python-
${{ runner.os }}-cargo-pkg-build-
- uses: actions/cache@v4
restore-keys: ${{ runner.os }}-cargo-pkg-build-${{ hashFiles('**/Cargo.lock') }}
- uses: actions/cache/restore@v4
name: build cache
with:
path: |
Expand All @@ -152,7 +141,6 @@ jobs:
- run: just python build
- run: just python test


nodejs-bindings-tests:
name: Node.js Binding Tests
runs-on: ubuntu-latest-8-cores
Expand All @@ -169,7 +157,7 @@ jobs:
name: toolchain cache
with:
path: |
~/.rustup/toolchains/
~/.rustup/toolchains/
key: ${{ runner.os }}-toolchain-build-${{ hashFiles('**/rust-toolchain.toml') }}
- uses: actions/cache@v4
name: cargo cache
Expand All @@ -188,7 +176,6 @@ jobs:
- run: just js build-debug
- run: just js test


unit-tests:
name: Unit Tests
runs-on: ubuntu-latest-8-cores
Expand All @@ -209,10 +196,6 @@ jobs:
with:
path: ~/.cargo/
key: ${{ runner.os }}-cargo-pkg-unit-${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-pkg-build-${{ hashFiles('**/Cargo.lock') }}
${{ runner.os }}-cargo-pkg-unit-
${{ runner.os }}-cargo-pkg-build-
- uses: actions/cache/restore@v4
name: build cache
with:
Expand All @@ -229,7 +212,6 @@ jobs:
fail-on-cache-miss: true
- run: just unit-tests


cache:
# when we change the Cargo.lock we should copy the branch cache
# from this build to the expected cache location to better
Expand All @@ -239,7 +221,7 @@ jobs:
needs: ["build"]
# parent branches can't use caches from child branches, so no need
# to update this cache _except_ on main.
if: github.ref == 'refs/heads/master'
if: github.ref == 'refs/heads/main'
steps:
- uses: actions/cache@v4
name: cache check
Expand Down Expand Up @@ -270,7 +252,6 @@ jobs:
!target/**/pgprototest
key: ${{ runner.os }}-cargo-pkg-build-${{ hashFiles('**/Cargo.lock') }}


pg-protocol:
name: PG Protocol Tests
runs-on: ubuntu-latest
Expand All @@ -296,7 +277,6 @@ jobs:
- run: ./scripts/protocol-test.sh
- run: just slt-bin-debug 'pgproto/*'


sql-logic-tests:
name: SQL Logic Tests
runs-on: ubuntu-latest
Expand Down Expand Up @@ -333,7 +313,6 @@ jobs:
if: ${{ env.ACTIONS_STEP_DEBUG != 'true' }}
run: ${{matrix.protocol.basic}}


process-integration-tests:
name: Process Integration Tests (pytest)
runs-on: ubuntu-latest
Expand Down Expand Up @@ -361,7 +340,6 @@ jobs:
- run: just venv
- run: just pytest


service-integration-tests:
name: Service Integration Tests (SLT)
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == 'GlareDB'
Expand Down Expand Up @@ -424,38 +402,15 @@ jobs:
# depend on this.
unset GOOGLE_APPLICATION_CREDENTIALS
# Prepare SLT (Postgres)
POSTGRES_TEST_DB=$(./scripts/create-test-postgres-db.sh)
export POSTGRES_CONN_STRING=$(echo "$POSTGRES_TEST_DB" | sed -n 1p)
export POSTGRES_TUNNEL_SSH_CONN_STRING=$(echo "$POSTGRES_TEST_DB" | sed -n 2p)
# Prepare SLT (fake GCS server)
./scripts/create-test-gcs-store.sh
echo "-------------------------------- WITHOUT TUNNEL TEST --------------------------------"
# Run all data source tests without running tunnel tests or the basic
# SLT tests.
just slt-bin --exclude 'sqllogictests/*' \
--exclude '*/tunnels/ssh' \
--exclude 'sqllogictests_snowflake/*' \
--exclude 'sqllogictests_cassandra/*' \
--exclude 'sqllogictests_clickhouse/*' \
--exclude 'sqllogictests_sqlserver/*' \
--exclude 'sqllogictests_mongodb/*' \
--exclude 'sqllogictests_mysql/*' \
echo "-------------------------------- WITH TUNNEL TEST --------------------------------"
# SSH tests are prone to fail if we make a lot of connections at the
# same time. Hence, it makes sense to run all the SSH tests one-by-one
# in order to test the SSH tunnels (which is our aim).
just slt-bin --jobs=1 '*/tunnels/ssh' --exclude 'sqllogictests_mysql/*'
# TODO: move these into the datasource-integration-tests job
echo "-------------------------------- RPC TESTS --------------------------------"
# TODO: move these into the datasource-integration-tests job
echo "-------------------------- BigQuery -------------------------- "
just slt-bin --protocol=rpc 'sqllogictests_bigquery/*'
echo "-------------------------- Iceberg -------------------------- "
just slt-bin --protocol=rpc 'sqllogictests_iceberg/*'
Expand All @@ -465,14 +420,10 @@ jobs:
echo "-------------------------- Object Store -------------------------- "
just slt-bin --protocol=rpc 'sqllogictests_object_store/*'
echo "-------------------------- Postgres -------------------------- "
just slt-bin --protocol=rpc --exclude '*/tunnels/ssh' 'sqllogictests_postgres/*'
echo "-------------------------- Fake GCS server with a subdirectory -------------------------- "
just slt-bin -l gs://$TEST_BUCKET/path/to/folder/1 -o service_account_path=/tmp/fake-gcs-creds.json 'sqllogictests_native/*'
just slt-bin -l gs://$TEST_BUCKET/path/to/folder/2 -o service_account_path=/tmp/fake-gcs-creds.json 'sqllogictests_native/*'
minio-integration-tests:
name: MinIO Integration Tests (SLT::MinIO)
runs-on: ubuntu-latest
Expand Down Expand Up @@ -536,7 +487,6 @@ jobs:
'sqllogictests/*' \
--protocol=flightsql
service-integration-tests-snowflake:
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == 'GlareDB'
name: Snowflake Integration Tests (SLT::Snowflake)
Expand Down Expand Up @@ -588,7 +538,6 @@ jobs:
just slt-bin 'sqllogictests_snowflake/*'
just slt-bin --protocol=rpc 'sqllogictests_snowflake/*'
datasource-integration-tests:
name: Datasource Integration (${{matrix.settings.name}})
strategy:
Expand All @@ -600,11 +549,18 @@ jobs:
./scripts/prepare-testdata.sh
source ./scripts/ci-install-clickhouse.sh
export CLICKHOUSE_CONN_STRING=$(./scripts/create-test-clickhouse-db.sh)
- name: SQLite
path: "sqllogictests_sqlite/*"
prepare: |
./scripts/prepare-testdata.sh
# If there's an old data-set in the cache, remove it so we can create a new one.
test -f testdata/sqllogictests_sqlite/data/db.sqlite3 && rm testdata/sqllogictests_sqlite/data/db.sqlite3
- name: Cassandra
path: "sqllogictests_cassandra/*"
prepare: |
export CASSANDRA_CONN_STRING=$(./scripts/create-test-cassandra-db.sh | tail -n 1)
- name: Mysql
- name: MySQL
path: "sqllogictests_mysql/*"
prepare: |
./scripts/prepare-testdata.sh
Expand All @@ -615,12 +571,20 @@ jobs:
path: "sqllogictests_mongodb/*"
prepare: |
./scripts/prepare-testdata.sh
export MONGO_CONN_STRING=$(./scripts/create-test-mongo-db.sh)
- name: Sqlserver
export MONGO_CONN_STRING=$(./scripts/create-test-mongodb.sh)
./scripts/create-fixture-mongodb.sh
- name: SQLServer
path: "sqllogictests_sqlserver/*"
prepare: |
./scripts/prepare-testdata.sh
export SQL_SERVER_CONN_STRING=$(./scripts/create-test-sqlserver-db.sh)
- name: PostgreSQL
path: "sqllogictests_postgres/*"
prepare: |
./scripts/prepare-testdata.sh
export POSTGRES_TEST_DB=$(./scripts/create-test-postgres-db.sh)
export POSTGRES_CONN_STRING=$(echo "$POSTGRES_TEST_DB" | sed -n 1p)
export POSTGRES_TUNNEL_SSH_CONN_STRING=$(echo "$POSTGRES_TEST_DB" | sed -n 2p)
runs-on: ubuntu-latest
needs: ["sql-logic-tests"]
steps:
Expand All @@ -641,10 +605,37 @@ jobs:
just slt-bin ${{matrix.settings.path}}
just slt-bin --protocol=rpc --exclude '*/tunnels/ssh' ${{matrix.settings.path}}
[ "${{matrix.settings.name}}" = "MongoDB" ] && ./scripts/create-fixture-mongodb.sh
just slt-bin --protocol=rpc ${{matrix.settings.path}}
[ "${{matrix.settings.name}}" = "MongoDB" ] && ./scripts/create-fixture-mongodb.sh
# for sqlserver, skip flightsql because the suite takes 4-5
# minutes, and Sqlserver is the longest/last task to finish
[ "${{matrix.settings.name}}" = "Sqlserver" ] && exit 0
# minutes, and sqlserver is the longest/last task to finish
[ "${{matrix.settings.name}}" = "SQLServer" ] && exit 0
just slt-bin --protocol=flightsql ${{matrix.settings.path}}
just slt-bin --protocol=flightsql --exclude '*/tunnels/ssh' ${{matrix.settings.path}}
docker-push:
name: Build and Push Docker Image
if: github.event_name == 'push' && github.repository == 'GlareDB/glaredb' && github.ref == 'refs/heads/main'
needs: ["sql-logic-tests"]
runs-on: ubuntu-latest-8-cores
permissions:
id-token: write # required to use OIDC auth
contents: write # required to checkout code
env:
GCP_PROJECT_ID: glaredb-artifacts
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Authenticate to Google Cloud
uses: google-github-actions/auth@v2
with:
credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}
- name: Setup gcloud
uses: google-github-actions/setup-gcloud@v2
- run: gcloud auth configure-docker -q
- run: ./scripts/build-and-push-images.sh
44 changes: 0 additions & 44 deletions .github/workflows/image.yaml
View file

This file was deleted.

Loading

0 comments on commit fdeecb1

Please sign in to comment.