chore: isolate bigquery task in CI (#2813)

GlareDB · Mar 22, 2024 · c1afec7 · c1afec7
1 parent 56c79fa
commit c1afec7
Showing 1 changed file with 51 additions and 29 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -342,35 +342,29 @@ jobs:
       - run: just pytest
 
   service-integration-tests:
-    name: Service Integration Tests (SLT)
+    name: Object Store Service Integration Tests (SLT)
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login == 'GlareDB'
     runs-on: ubuntu-latest
     needs: ["sql-logic-tests"]
     steps:
       - name: checkout
         uses: actions/checkout@v4
-
       - uses: extractions/setup-just@v2
         with:
           just-version: "1.23.0"
-
       - uses: actions/cache/restore@v4
         name: glaredb cache
         with:
           path: target/debug/glaredb
           key: ${{ runner.os }}-glaredb-bin-${{ github.run_id }}
           fail-on-cache-miss: true
-
       - name: GCP authenticate
         uses: google-github-actions/auth@v2
         with:
           credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}
-
       - name: setup GCP
         uses: google-github-actions/setup-gcloud@v2
-
       - run: ./scripts/prepare-testdata.sh
-
       - name: run tests (slt)
         env:
           GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}
@@ -381,19 +375,6 @@ jobs:
           AZURE_ACCOUNT: ${{ secrets.AZURE_ACCOUNT }}
           TEST_BUCKET: glaredb-test-bucket
         run: |
-          # Prepare SLT (BigQuery)
-          if ./scripts/files-changed-in-branch.sh \
-            "scripts/prepare-testdata.sh" \
-            "scripts/create-test-bigquery-db.sh" \
-            "testdata/sqllogictests_datasources_common/data" \
-            "testdata/sqllogictests_bigquery/data"
-          then
-            export GCP_PROJECT_ID=glaredb-dev-playground
-            export BIGQUERY_DATASET_ID=$(./scripts/create-test-bigquery-db.sh)
-          else
-            export BIGQUERY_DATASET_ID=glaredb_test
-          fi
-
           # Prepare SLT (Object store)
           export GCS_BUCKET_NAME=glaredb-test
           export AWS_S3_REGION=us-east-1
@@ -403,15 +384,6 @@ jobs:
           # depend on this.
           unset GOOGLE_APPLICATION_CREDENTIALS
 
-          # Prepare SLT (fake GCS server)
-          ./scripts/create-test-gcs-store.sh
-
-          echo "-------------------------------- RPC TESTS --------------------------------"
-          # TODO: move these into the datasource-integration-tests job
-
-          echo "--------------------------  BigQuery -------------------------- "
-          just slt-bin --protocol=rpc 'sqllogictests_bigquery/*'
-
           echo "--------------------------  Iceberg -------------------------- "
           just slt-bin --protocol=rpc 'sqllogictests_iceberg/*'
 
@@ -422,9 +394,59 @@ jobs:
           just slt-bin --protocol=rpc 'sqllogictests_object_store/*'
 
           echo "--------------------------  Fake GCS server with a subdirectory -------------------------- "
+          ./scripts/create-test-gcs-store.sh # prepare fake GCS server
           just slt-bin -l gs://$TEST_BUCKET/path/to/folder/1 -o service_account_path=/tmp/fake-gcs-creds.json 'sqllogictests_native/*'
           just slt-bin -l gs://$TEST_BUCKET/path/to/folder/2 -o service_account_path=/tmp/fake-gcs-creds.json 'sqllogictests_native/*'
 
+
+  bigquery-integration-tests:
+    name: Big Query Integration Tests (SLT::MinIO)
+    runs-on: ubuntu-latest
+    needs: ["sql-logic-tests"]
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+      - uses: extractions/setup-just@v2
+        with:
+          just-version: "1.23.0"
+      - uses: actions/cache/restore@v4
+        name: glaredb cache
+        with:
+          path: target/debug/glaredb
+          key: ${{ runner.os }}-glaredb-bin-${{ github.run_id }}
+          fail-on-cache-miss: true
+      - name: GCP authenticate
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}
+      - name: setup GCP
+        uses: google-github-actions/setup-gcloud@v2
+      - run: ./scripts/prepare-testdata.sh
+      - name: run tests (slt)
+        env:
+          GCP_SERVICE_ACCOUNT_KEY: ${{ secrets.GCP_SERVICE_ACCOUNT_JSON }}
+          GCP_PROJECT_ID: glaredb-artifacts
+          TEST_BUCKET: glaredb-test-bucket
+        run: |
+          # Prepare SLT (BigQuery)
+          if ./scripts/files-changed-in-branch.sh \
+            "scripts/prepare-testdata.sh" \
+            "scripts/create-test-bigquery-db.sh" \
+            "testdata/sqllogictests_datasources_common/data" \
+            "testdata/sqllogictests_bigquery/data"
+          then
+            export GCP_PROJECT_ID=glaredb-dev-playground
+            export BIGQUERY_DATASET_ID=$(./scripts/create-test-bigquery-db.sh)
+          else
+            export BIGQUERY_DATASET_ID=glaredb_test
+          fi
+
+          # Unset application default credentials. We don't want to unknowingly
+          # depend on this.
+          unset GOOGLE_APPLICATION_CREDENTIALS
+
+          just slt-bin --protocol=rpc 'sqllogictests_bigquery/*'
+
   minio-integration-tests:
     name: MinIO Integration Tests (SLT::MinIO)
     runs-on: ubuntu-latest