Merge pull request #6 from raphaelvarieras/main

Use quotes for order table
fivetran · Aug 4, 2023 · 7a215f2 · 7a215f2
2 parents d97d81f + c183f11
commit 7a215f2
Show file tree

Hide file tree

Showing 48 changed files with 123 additions and 18 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,17 @@
+# dbt_zuora_source v0.1.2
+[PR #6](https://github.com/fivetran/dbt_zuora_source/pull/6) includes the following updates:
+## Bug Fixes
+- Included a where clause within **all** staging models to filter out `_fivetran_deleted` records.
+- Added a conditional config within the `src_zuora.yml` to properly handle the `order` (reserved word) source table when using Snowflake as a destination.
+
+## Documentation
+- Added a call out in the identifier configuration section of the README to provide instructions for Snowflake users to handle the `order` source table if their destination has case sensitivity enabled.
+
+## Under the Hood
+- Added the `_fivetran_deleted` field to **all** `get_*_columns` macros to ensure the field may be leveraged in the respective staging models.
+
+## Contributors
+- [@raphaelvarieras](https://github.com/raphaelvarieras) ([PR #6](https://github.com/fivetran/dbt_zuora_source/pull/6))
 # dbt_zuora_source v0.1.1
 ## Bug Fixes
 - Included model disable logic within the `stg_zuora__credit_balance_adjustment_tmp` and `stg_zuora__refund_invoice_payment_tmp` models in order to ensure they are not run if the appropriate variables are set to `false`.

diff --git a/README.md b/README.md
@@ -106,6 +106,15 @@ vars:
     zuora_<default_source_table_name>_identifier: your_table_name 
 ```
 
+### 🚨 Snowflake Users
+If you do **not** use the default all-caps naming conventions for Snowflake, you may need to provide the case-sensitive spelling of your source tables that are also Snowflake reserved words. 
+
+In this package, this would apply to the `ORDER` source. If you are receiving errors for this source, include the below identifier in your `dbt_project.yml` file:
+
+```yml
+vars:
+    zuora_order_identifier: "ORDER" # as an example, must include the double-quotes and correct case!
+```  
 
 </details>
 

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'zuora_source'
-version: '0.1.1'
+version: '0.1.2'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/index.html b/docs/index.html
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/docs/run_results.json b/docs/run_results.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'zuora_source_integration_tests'
-version: '0.1.1'
+version: '0.1.2'
 config-version: 2
 profile: 'integration_tests'
 

diff --git a/macros/get_account_columns.sql b/macros/get_account_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "account_number", "datatype": dbt.type_string()},
     {"name": "auto_pay", "datatype": "boolean"},
     {"name": "balance", "datatype": dbt.type_float()},

diff --git a/macros/get_amendment_columns.sql b/macros/get_amendment_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "auto_renew", "datatype": "boolean"},
     {"name": "booking_date", "datatype": "date"},
     {"name": "code", "datatype": dbt.type_string()},

diff --git a/macros/get_contact_columns.sql b/macros/get_contact_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "account_id", "datatype": dbt.type_string()},
     {"name": "address_1", "datatype": dbt.type_string()},
     {"name": "address_2", "datatype": dbt.type_string()},

diff --git a/macros/get_credit_balance_adjustment_columns.sql b/macros/get_credit_balance_adjustment_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "account_id", "datatype": dbt.type_string()},
     {"name": "account_receivable_accounting_code_id", "datatype": dbt.type_string()},
     {"name": "accounting_code", "datatype": dbt.type_string()},

diff --git a/macros/get_invoice_columns.sql b/macros/get_invoice_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "account_id", "datatype": dbt.type_string()},
     {"name": "adjustment_amount", "datatype": dbt.type_float()},
     {"name": "amount", "datatype": dbt.type_float()}, 

diff --git a/macros/get_invoice_item_columns.sql b/macros/get_invoice_item_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "account_id", "datatype": dbt.type_string()},
     {"name": "account_receivable_accounting_code_id", "datatype": dbt.type_string()},
     {"name": "accounting_code", "datatype": dbt.type_string()}, 

diff --git a/macros/get_invoice_payment_columns.sql b/macros/get_invoice_payment_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [ 
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "account_id", "datatype": dbt.type_string()},
     {"name": "account_receivable_accounting_code_id", "datatype": dbt.type_string()},
     {"name": "accounting_period_id", "datatype": dbt.type_string()},

diff --git a/macros/get_order_columns.sql b/macros/get_order_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "account_id", "datatype": dbt.type_string()},
     {"name": "bill_to_contact_id", "datatype": dbt.type_string()},
     {"name": "category", "datatype": dbt.type_string()},

diff --git a/macros/get_payment_columns.sql b/macros/get_payment_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "account_id", "datatype": dbt.type_string()},
     {"name": "accounting_code", "datatype": dbt.type_string()},
     {"name": "amount", "datatype": dbt.type_float()}, 

diff --git a/macros/get_payment_method_columns.sql b/macros/get_payment_method_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "account_id", "datatype": dbt.type_string()},
     {"name": "ach_account_type", "datatype": dbt.type_string()},
     {"name": "active", "datatype": "boolean"},

diff --git a/macros/get_product_columns.sql b/macros/get_product_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "category", "datatype": dbt.type_string()},
     {"name": "created_by_id", "datatype": dbt.type_string()},
     {"name": "created_date", "datatype": dbt.type_timestamp()},

diff --git a/macros/get_product_rate_plan_charge_columns.sql b/macros/get_product_rate_plan_charge_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "account_receivable_accounting_code_id", "datatype": dbt.type_string()},
     {"name": "accounting_code", "datatype": dbt.type_string()},
     {"name": "apply_discount_to", "datatype": dbt.type_string()},

diff --git a/macros/get_product_rate_plan_columns.sql b/macros/get_product_rate_plan_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "created_by_id", "datatype": dbt.type_string()},
     {"name": "created_date", "datatype": dbt.type_timestamp()},
     {"name": "description", "datatype": dbt.type_string()},

diff --git a/macros/get_rate_plan_charge_columns.sql b/macros/get_rate_plan_charge_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "account_id", "datatype": dbt.type_string()},
     {"name": "account_receivable_accounting_code_id", "datatype": dbt.type_string()},
     {"name": "accounting_code", "datatype": dbt.type_string()},

diff --git a/macros/get_rate_plan_columns.sql b/macros/get_rate_plan_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "account_id", "datatype": dbt.type_string()},
     {"name": "amendment_id", "datatype": dbt.type_string()}, 
     {"name": "bill_to_contact_id", "datatype": dbt.type_string()},

diff --git a/macros/get_refund_columns.sql b/macros/get_refund_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "accounting_code", "datatype": dbt.type_string()},
     {"name": "amount", "datatype": dbt.type_float()},
     {"name": "cancelled_on", "datatype": dbt.type_timestamp()},

diff --git a/macros/get_refund_invoice_payment_columns.sql b/macros/get_refund_invoice_payment_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "created_by_id", "datatype": dbt.type_string()},
     {"name": "created_date", "datatype": dbt.type_timestamp()},
     {"name": "id", "datatype": dbt.type_string()},

diff --git a/macros/get_subscription_columns.sql b/macros/get_subscription_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "account_id", "datatype": dbt.type_string()},
     {"name": "auto_renew", "datatype": "boolean"},
     {"name": "bill_to_contact_id", "datatype": dbt.type_string()},

diff --git a/macros/get_taxation_item_columns.sql b/macros/get_taxation_item_columns.sql
@@ -2,6 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
+    {"name": "_fivetran_deleted", "datatype": "boolean"},
     {"name": "account_id", "datatype": dbt.type_string()},
     {"name": "account_receivable_accounting_code_id", "datatype": dbt.type_string()},
     {"name": "accounting_code", "datatype": dbt.type_string()}, 

diff --git a/models/docs.md b/models/docs.md
@@ -118,6 +118,10 @@ Identifier of the default payment method for the account.
 Timestamp of when a record was last synced.
 {% enddocs %}
 
+{% docs _fivetran_deleted %}
+Boolean identifiying whether the record was deleted in the source.
+{% enddocs %}
+
 {% docs home_currency %}
 Home currency the customer is billed in.
 {% enddocs %}