Added BigQuery support (#12) (#13)

* added bigquery support * added bigquery support * added bigquery support * added bigquery support * added bigquery support * added bigquery support * added bigquery support --------- Co-authored-by: kevin <you@example.com>
kgmcquate · Jan 3, 2024 · 00fd9d9 · 00fd9d9
1 parent 6079d4b
commit 00fd9d9
Show file tree

Hide file tree

Showing 26 changed files with 150 additions and 94 deletions.
diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_test.yml
@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     env:
       DBT_TARGET: duckdb
@@ -34,7 +34,7 @@ jobs:
 
     - name: Install DBT deps
       run: dbt deps --target ${DBT_TARGET}
-      
+
     - name: load test data
       run: dbt seed --target ${DBT_TARGET}
 
@@ -67,7 +67,7 @@ jobs:
 
     - name: Install DBT deps
       run: dbt deps --target ${DBT_TARGET}
-      
+
     - name: load test data
       run: dbt seed --target ${DBT_TARGET}
 
@@ -95,7 +95,7 @@ jobs:
 
     - name: Install DBT deps
       run: dbt deps --target ${DBT_TARGET}
-      
+
     - name: load test data
       run: dbt seed --target ${DBT_TARGET}
 
@@ -123,12 +123,55 @@ jobs:
 
     - name: Install DBT deps
       run: dbt deps --target ${DBT_TARGET}
-      
+
     - name: load test data
       run: dbt seed --target ${DBT_TARGET}
 
     - name: Create seed tests
       run: bash tests/create_seeds_tests_yml.sh
 
     - name: dbt test
-      run: dbt test --target ${DBT_TARGET}
+      run: dbt test --target ${DBT_TARGET}
+
+  test-bigquery:
+    name: Test on BigQuery
+    runs-on: ubuntu-latest
+    container:
+      image: python:3.11
+
+    env:
+      DBT_TARGET: bigquery
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: create-json
+      id: create-json
+      uses: jsdaniell/create-json@1.1.2
+      with:
+        name: "gcp_keyfile.json"
+        dir: 'integration_tests/'
+        json: ${{ secrets.GCP_KEYFILE }}
+
+    #- run: echo '${{ secrets.GCP_KEYFILE }}' > gcp_keyfile.json
+
+    - name: pip install
+      run: pip install dbt-core dbt-${DBT_TARGET}
+
+    - name: Install DBT deps
+      run: dbt deps --target ${DBT_TARGET}
+
+    - name: load test data
+      run: dbt seed --target ${DBT_TARGET} -s users
+
+    - name: Create seed tests
+      run: |
+        dbt compile --target ${DBT_TARGET} -q \
+        --inline "{{ testgen.get_test_suggestions(ref('users'), resource_type='seeds', column_config={'quote': true, 'tags': ['dataset-users']} ) }}" \
+        > seeds/users_test_suggestions.yml
+
+    - name: dbt test
+      run: dbt test --target ${DBT_TARGET} -s tag:dataset-users
+
+
+
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,7 @@ dbt_packages/
 logs/
 dbt-env/
 integration_tests/seeds/*.yml
+*gcp_keyfile.json
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,7 @@
+FROM python:3.11
+
+RUN pip install dbt-core duckdb dbt-duckdb dbt-postgres dbt-redshift dbt-snowflake dbt-bigquery
+
+WORKDIR /run/dbt-testgen/
+
+ENTRYPOINT [ "bash" ]
diff --git a/README.md b/README.md
@@ -20,6 +20,7 @@ packages:
 The following databases are supported:
 - Snowflake
 - RedShift
+- BigQuery
 - Postgres
 - DuckDB
 
@@ -34,12 +35,48 @@ models:
 - name: mymodel
   columns:
   - name: user_id
+    description: Numeric range test generated by dbt-testgen
     tests:
     - unique
     - not_null
     - dbt_utils.accepted_range:
         min_value: 1
         max_value: 30
+  - name: username
+    description: String length test generated by dbt-testgen
+    tests:
+    - unique
+    - not_null
+    - dbt_expectations.expect_column_value_lengths_to_be_between:
+        min_value: 8
+        max_value: 15
+        row_condition: '"username" is not null'
+  - name: email
+    description: String length test generated by dbt-testgen
+    tests:
+    - unique
+    - not_null
+    - dbt_expectations.expect_column_value_lengths_to_be_between:
+        min_value: 18
+        max_value: 25
+        row_condition: '"email" is not null'
+  - name: user_status
+    description: String length test generated by dbt-testgen
+    tests:
+    - accepted_values:
+        values:
+        - active
+        - inactive
+    - dbt_expectations.expect_column_value_lengths_to_be_between:
+        min_value: 6
+        max_value: 8
+        row_condition: '"user_status" is not null'
+  - name: age
+    description: Numeric range test generated by dbt-testgen
+    tests:
+    - dbt_utils.accepted_range:
+        min_value: 22
+        max_value: 35
 ```
 
 ## Test types
@@ -49,5 +86,6 @@ dbt-testgen can generate these types of tests:
 - [string length](#string-length)
 - [range](#range)
 - [accepted_values](#accepted-values)
-- [freshness](#freshness)
+<!-- - [freshness](#freshness) -->
+<!-- - [std_dev] -->
 
diff --git a/integration_tests/Dockerfile b/integration_tests/Dockerfile
diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml
@@ -53,4 +53,11 @@ integration_tests:
       password: "{{ env_var('REDSHIFT_PASSWORD') }}"
       dbname: dbt_testgen
       schema: dbt_testgen
-      port: 5439
+      port: 5439
+    bigquery:
+      type: bigquery
+      method: service-account
+      project: dbt-testgen
+      dataset: dbt_testgen
+      threads: 1
+      keyfile: gcp_keyfile.json
diff --git a/integration_tests/run_docker.sh b/integration_tests/run_docker.sh
diff --git a/integration_tests/tests/create_seeds_tests_yml.sh b/integration_tests/tests/create_seeds_tests_yml.sh
@@ -1,7 +1,7 @@
 dbt compile --target ${DBT_TARGET} -q \
-    --inline "{{ testgen.get_test_suggestions(ref('users'), resource_type='seeds', column_config={'quote': true} ) }}" \
+    --inline "{{ testgen.get_test_suggestions(ref('users'), resource_type='seeds', column_config={'quote': true, 'tags': ['dataset-users']} ) }}" \
     > seeds/users_test_suggestions.yml
 
 dbt compile --target ${DBT_TARGET} -q \
-    --inline "{{ testgen.get_test_suggestions(ref('colnames_with_spaces'), resource_type='seeds', column_config={'quote': true}) }}"  \
+    --inline "{{ testgen.get_test_suggestions(ref('colnames_with_spaces'), resource_type='seeds', column_config={'quote': true, 'tags': ['dataset-colnames_with_spaces']}) }}"  \
     > seeds/colnames_with_spaces_test_suggestions.yml
diff --git a/...ation_tests/tests/generate_accepted_values_tests/accepted_values_colnames_with_spaces.sql b/...ation_tests/tests/generate_accepted_values_tests/accepted_values_colnames_with_spaces.sql
@@ -1,4 +1,6 @@
-
+{{ config(
+    tags="dataset-colnames_with_spaces"
+) }}
 
 {% set actual_yaml = testgen.to_yaml(
         testgen.get_accepted_values_test_suggestions(
@@ -12,23 +14,20 @@ models:
 - name: colnames_with_spaces
   columns:
   - name: first name
-    description: Accepted values test generated by dbt-testgen
     tests:
     - accepted_values:
         values:
         - Alice
         - Bob
         - John
   - name: age (years)
-    description: Accepted values test generated by dbt-testgen
     tests:
     - accepted_values:
         values:
         - '22'
         - '25'
         - '30'
   - name: current city
-    description: Accepted values test generated by dbt-testgen
     tests:
     - accepted_values:
         values:

diff --git a/integration_tests/tests/generate_accepted_values_tests/accepted_values_users.sql b/integration_tests/tests/generate_accepted_values_tests/accepted_values_users.sql
@@ -1,4 +1,6 @@
-
+{{ config(
+    tags="dataset-users"
+) }}
 
 {% set actual_yaml = testgen.to_yaml(
         testgen.get_accepted_values_test_suggestions(
@@ -12,7 +14,6 @@ models:
 - name: users
   columns:
   - name: user_status
-    description: Accepted values test generated by dbt-testgen
     tests:
     - accepted_values:
         values:

diff --git a/integration_tests/tests/generate_range_tests/range_colnames_with_spaces.sql b/integration_tests/tests/generate_range_tests/range_colnames_with_spaces.sql
@@ -1,3 +1,6 @@
+{{ config(
+    tags="dataset-colnames_with_spaces"
+) }}
 
 
 {% set actual_yaml = testgen.to_yaml(
@@ -12,7 +15,6 @@ models:
 - name: colnames_with_spaces
   columns:
   - name: age (years)
-    description: Numeric range test generated by dbt-testgen
     tests:
     - dbt_utils.accepted_range:
         min_value: 22

diff --git a/integration_tests/tests/generate_range_tests/range_users.sql b/integration_tests/tests/generate_range_tests/range_users.sql
@@ -1,4 +1,6 @@
-
+{{ config(
+    tags="dataset-users"
+) }}
 
 {% set actual_yaml = testgen.to_yaml(
         testgen.get_range_test_suggestions(
@@ -12,13 +14,11 @@ models:
 - name: users
   columns:
   - name: user_id
-    description: Numeric range test generated by dbt-testgen
     tests:
     - dbt_utils.accepted_range:
         min_value: 1
         max_value: 30
   - name: age
-    description: Numeric range test generated by dbt-testgen
     tests:
     - dbt_utils.accepted_range:
         min_value: 22

diff --git a/integration_tests/tests/generate_string_length_tests/string_length_colnames_with_spaces.sql b/integration_tests/tests/generate_string_length_tests/string_length_colnames_with_spaces.sql
@@ -1,3 +1,6 @@
+{{ config(
+    tags="dataset-colnames_with_spaces"
+) }}
 
 
 {% set actual_yaml = testgen.to_yaml(
@@ -14,19 +17,17 @@ models:
 - name: colnames_with_spaces
   columns:
   - name: first name
-    description: String length test generated by dbt-testgen
     tests:
     - dbt_expectations.expect_column_value_lengths_to_be_between:
         min_value: 3
         max_value: 5
-        row_condition: '"first name" is not null'
+        row_condition: '{{ adapter.quote('first name') }} is not null'
   - name: current city
-    description: String length test generated by dbt-testgen
     tests:
     - dbt_expectations.expect_column_value_lengths_to_be_between:
         min_value: 7
         max_value: 13
-        row_condition: '"current city" is not null'
+        row_condition: '{{ adapter.quote('current city') }} is not null'
 {% endset %}
 
 {{ assert_equal (actual_yaml | trim, expected_yaml | trim) }}
diff --git a/integration_tests/tests/generate_string_length_tests/string_length_users.sql b/integration_tests/tests/generate_string_length_tests/string_length_users.sql
@@ -1,5 +1,6 @@
-
-
+{{ config(
+    tags="dataset-users"
+) }}
 {% set actual_yaml = testgen.to_yaml(
         testgen.get_string_length_test_suggestions(
             ref('users'),
@@ -14,26 +15,23 @@ models:
 - name: users
   columns:
   - name: username
-    description: String length test generated by dbt-testgen
     tests:
     - dbt_expectations.expect_column_value_lengths_to_be_between:
         min_value: 8
         max_value: 15
-        row_condition: '"username" is not null'
+        row_condition: '{{ adapter.quote('username') }} is not null'
   - name: email
-    description: String length test generated by dbt-testgen
     tests:
     - dbt_expectations.expect_column_value_lengths_to_be_between:
         min_value: 18
         max_value: 25
-        row_condition: '"email" is not null'
+        row_condition: '{{ adapter.quote('email') }} is not null'
   - name: user_status
-    description: String length test generated by dbt-testgen
     tests:
     - dbt_expectations.expect_column_value_lengths_to_be_between:
         min_value: 6
         max_value: 8
-        row_condition: '"user_status" is not null'
+        row_condition: '{{ adapter.quote('user_status') }} is not null'
 {% endset %}
 
 {{ assert_equal (actual_yaml | trim, expected_yaml | trim) }}