Skip to content

Commit

Permalink
Release staging to main (#42)
Browse files Browse the repository at this point in the history
- Fix `data_tests` syntax
- Add support and CI/CD for Postgres
  • Loading branch information
gwenwindflower authored Apr 23, 2024
2 parents 375683a + 1a9ec13 commit 9a147c5
Show file tree
Hide file tree
Showing 13 changed files with 91 additions and 19 deletions.
24 changes: 24 additions & 0 deletions .github/workflows/cd_prod.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,3 +53,27 @@ jobs:
run: uv pip install -r requirements.txt --system
- name: Run dbt Cloud job
run: python3 .github/workflows/scripts/dbt_cloud_run_job.py

run_postgres:
name: dbt Cloud Deploy Prod Postgres
runs-on: macos-latest

env:
DBT_ACCOUNT_ID: 188483
DBT_PROJECT_ID: 288405
DBT_PR_JOB_ID: 577871
DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
DBT_JOB_CAUSE: "GitHub Actions Request"
DBT_JOB_BRANCH: main

steps:
- uses: "actions/checkout@v4"
- uses: "actions/setup-python@v5"
with:
python-version: "3.12"
- name: Install uv
run: python3 -m pip install uv
- name: Install deps
run: uv pip install -r requirements.txt --system
- name: Run dbt Cloud job
run: python3 .github/workflows/scripts/dbt_cloud_run_job.py
28 changes: 26 additions & 2 deletions .github/workflows/cd_staging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ jobs:
DBT_PR_JOB_ID: 565266
DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
DBT_JOB_CAUSE: "GitHub Actions Request"
DBT_JOB_BRANCH: main
DBT_JOB_BRANCH: staging

steps:
- uses: "actions/checkout@v4"
Expand All @@ -40,7 +40,31 @@ jobs:
DBT_PR_JOB_ID: 560539
DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
DBT_JOB_CAUSE: "GitHub Actions Request"
DBT_JOB_BRANCH: main
DBT_JOB_BRANCH: staging

steps:
- uses: "actions/checkout@v4"
- uses: "actions/setup-python@v5"
with:
python-version: "3.12"
- name: Install uv
run: python3 -m pip install uv
- name: Install deps
run: uv pip install -r requirements.txt --system
- name: Run dbt Cloud job
run: python3 .github/workflows/scripts/dbt_cloud_run_job.py

run_postgres:
name: dbt Cloud Deploy Staging Postgres
runs-on: macos-latest

env:
DBT_ACCOUNT_ID: 188483
DBT_PROJECT_ID: 288405
DBT_PR_JOB_ID: 577883
DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
DBT_JOB_CAUSE: "GitHub Actions Request"
DBT_JOB_BRANCH: staging

steps:
- uses: "actions/checkout@v4"
Expand Down
25 changes: 25 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,3 +54,28 @@ jobs:
run: uv pip install -r requirements.txt --system
- name: Run dbt Cloud job
run: python3 .github/workflows/scripts/dbt_cloud_run_job.py

run_postgres:
name: dbt Cloud PR CI Postgres
runs-on: macos-latest

env:
DBT_ACCOUNT_ID: 188483
DBT_PROJECT_ID: 288405
DBT_PR_JOB_ID: 577887
DBT_API_KEY: ${{ secrets.DBT_CLOUD_API_KEY }}
DBT_JOB_CAUSE: "GitHub Actions Request"
DBT_JOB_BRANCH: ${{ github.head_ref }}
DBT_JOB_SCHEMA_OVERRIDE: dbt_jsdx__pr_${{ github.head_ref}}

steps:
- uses: "actions/checkout@v4"
- uses: "actions/setup-python@v5"
with:
python-version: "3.12"
- name: Install uv
run: python3 -m pip install uv
- name: Install deps
run: uv pip install -r requirements.txt --system
- name: Run dbt Cloud job
run: python3 .github/workflows/scripts/dbt_cloud_run_job.py
1 change: 0 additions & 1 deletion dbt_project.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ clean-targets:
- "dbt_packages"

vars:
truncate_timespan_to: "{{ dbt.cast(dbt.current_timestamp(), 'datetime') }}"
"dbt_date:time_zone": "America/Los_Angeles"

seeds:
Expand Down
6 changes: 3 additions & 3 deletions models/marts/customers.yml
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
models:
- name: customers
description: Customer overview data mart, offering key details for each unique customer. One row per customer.
tests:
data_tests:
- dbt_utils.expression_is_true:
expression: "lifetime_spend_pretax + lifetime_tax_paid = lifetime_spend"
columns:
- name: customer_id
description: The unique key of the orders mart.
tests:
data_tests:
- not_null
- unique
- name: customer_name
Expand All @@ -26,7 +26,7 @@ models:
description: The sum of all the order totals (including tax) that a customer has ever placed.
- name: customer_type
description: Options are 'new' or 'returning', indicating if a customer has ordered more than once or has only placed their first order to date.
tests:
data_tests:
- accepted_values:
values: ["new", "returning"]

Expand Down
4 changes: 2 additions & 2 deletions models/marts/order_items.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ models:
- name: order_items
columns:
- name: order_item_id
tests:
data_tests:
- not_null
- unique
- name: order_id
tests:
data_tests:
- relationships:
to: ref('orders')
field: order_id
Expand Down
6 changes: 3 additions & 3 deletions models/marts/orders.yml
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
models:
- name: orders
description: Order overview data mart, offering key details for each order inlcluding if it's a customer's first order and a food vs. drink item breakdown. One row per order.
tests:
data_tests:
- dbt_utils.expression_is_true:
expression: "order_items_subtotal = subtotal"
- dbt_utils.expression_is_true:
expression: "order_total = subtotal + tax_paid"
columns:
- name: order_id
description: The unique key of the orders mart.
tests:
data_tests:
- not_null
- unique
- name: customer_id
description: The foreign key relating to the customer who placed the order.
tests:
data_tests:
- relationships:
to: ref('stg_customers')
field: customer_id
Expand Down
2 changes: 1 addition & 1 deletion models/staging/stg_customers.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@ models:
columns:
- name: customer_id
description: The unique key for each customer.
tests:
data_tests:
- not_null
- unique
2 changes: 1 addition & 1 deletion models/staging/stg_locations.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ models:
columns:
- name: location_id
description: The unique key for each location.
tests:
data_tests:
- not_null
- unique

Expand Down
4 changes: 2 additions & 2 deletions models/staging/stg_order_items.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@ models:
columns:
- name: order_item_id
description: The unique key for each order item.
tests:
data_tests:
- not_null
- unique
- name: order_id
description: The corresponding order each order item belongs to
tests:
data_tests:
- not_null
- relationships:
to: ref('stg_orders')
Expand Down
4 changes: 2 additions & 2 deletions models/staging/stg_orders.yml
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
models:
- name: stg_orders
description: Order data with basic cleaning and transformation applied, one row per order.
tests:
data_tests:
- dbt_utils.expression_is_true:
expression: "order_total - tax_paid = subtotal"
columns:
- name: order_id
description: The unique key for each order.
tests:
data_tests:
- not_null
- unique
2 changes: 1 addition & 1 deletion models/staging/stg_products.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@ models:
columns:
- name: product_id
description: The unique key for each product.
tests:
data_tests:
- not_null
- unique
2 changes: 1 addition & 1 deletion models/staging/stg_supplies.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,6 @@ models:
columns:
- name: supply_uuid
description: The unique key of our supplies per cost.
tests:
data_tests:
- not_null
- unique

0 comments on commit 9a147c5

Please sign in to comment.