Add compatibility for new Google Ads connector

.circleci/config.yml

@@ -33,6 +33,7 @@ jobs:
             dbt run --target postgres --full-refresh
             dbt test --target postgres
             dbt run --vars '{google_auto_tagging_enabled: true}' --target postgres --full-refresh
+            dbt run --vars '{api_source: google_ads}' --target postgres --full-refresh
             dbt test --target postgres
       - run:
           name: "Run Tests - Spark"
@@ -45,6 +46,7 @@ jobs:
             dbt run --target spark --full-refresh
             dbt test --target spark
             dbt run --vars '{google_auto_tagging_enabled: true}' --target spark --full-refresh
+            dbt run --vars '{api_source: google_ads}' --target spark --full-refresh
             dbt test --target spark
       - run:
           name: "Run Tests - Redshift"
@@ -57,6 +59,7 @@ jobs:
             dbt run --target redshift --full-refresh
             dbt test --target redshift
             dbt run --vars '{google_auto_tagging_enabled: true}' --target redshift --full-refresh
+            dbt run --vars '{api_source: google_ads}' --target redshift --full-refresh
             dbt test --target redshift
       - run:
           name: "Run Tests - Snowflake"
@@ -69,6 +72,7 @@ jobs:
             dbt run --target snowflake --full-refresh
             dbt test --target snowflake
             dbt run --vars '{google_auto_tagging_enabled: true}' --target snowflake --full-refresh
+            dbt run --vars '{api_source: google_ads}' --target snowflake --full-refresh
             dbt test --target snowflake
       - run:
           name: "Run Tests - BigQuery"
@@ -84,4 +88,5 @@ jobs:
             dbt run --target bigquery --full-refresh
             dbt test --target bigquery
             dbt run --vars '{google_auto_tagging_enabled: true}' --target bigquery --full-refresh
+            dbt run --vars '{api_source: google_ads}' --target bigquery --full-refresh
             dbt test --target bigquery

README.md

@@ -21,7 +21,24 @@ packages:
     version: [">=0.3.0", "<0.4.0"]
 ```
 ## Configuration
-To use this package, you will need to pull the following custom reports through Fivetran:
+
+This package allows users to leverage either the Adwords API or the Google Ads API. You will be able to determine which API your connector is using by navigating within your Fivetran UI to the `setup` tab -> `edit connection details` link -> and reference the `API configuration` used. You will want to refer to the respective configuration steps below based off the API used by your connector. 
+
+### Google Ads API Configuration
+If your connector is setup using the Google Ads API then you will need to configure your `dbt_project.yml` with the below variable:
+
+```yml
+# dbt_project.yml
+
+...
+config-version: 2
+
+vars:
+    api_source: google_ads  ## adwords by default
+```
+
+### Adwords API Configuration
+If your connector is setup using the Adwords API then you will need to pull the following custom reports through Fivetran:
 
 * Destination Table Name: `final_url_performance`
 * Report Type: `FINAL_URL_REPORT`
@@ -89,7 +106,9 @@ vars:
     google_ads__click_performance: adwords.click_performance_report
 ```
 
-By default, this package will look for your Google Ads data in the `adwords` schema of your [target database](https://docs.getdbt.com/docs/running-a-dbt-project/using-the-command-line-interface/configure-your-profile). If this is not where your Google Ads data is, please add the following configuration to your `dbt_project.yml` file:
+### Source Schema is Named Differently
+
+By default, this package will look for your Google Ads data in the `adwords` or `google_ads` schema of your [target database](https://docs.getdbt.com/docs/running-a-dbt-project/using-the-command-line-interface/configure-your-profile), depending on which API source you are using. If this is not where your Google Ads data is, please add the following configuration to your `dbt_project.yml` file:
 
 ```yml
 # dbt_project.yml
@@ -101,6 +120,23 @@ vars:
     google_ads_schema: your_schema_name
     google_ads_database: your_database_name
 ```
+## Optional Configurations
+### Passing Through Additional Metrics
+By default, this package will select `clicks`, `impressions`, and `cost` from the source reporting tables to store into the staging models. If you would like to pass through additional metrics to the staging models, add the following configuration to your `dbt_project.yml` file:
+
+```yml
+# dbt_project.yml
+
+...
+vars:
+    # If you're using the Adwords API source
+    google_ads__url_passthrough_metrics: ['the', 'list', 'of', 'metric', 'columns', 'to', 'include'] # from adwords.final_url_performance
+    google_ads__criteria_passthrough_metrics: ['the', 'list', 'of', 'metric', 'columns', 'to', 'include'] # from adwords.criteria_performance
+
+    # If you're using the Google Ads API source
+    google_ads__ad_stats_passthrough_metrics: ['the', 'list', 'of', 'metric', 'columns', 'to', 'include'] # from google_ads.ad_stats
+```
+
 ### UTM Auto Tagging Feature
 This package assumes you are manually adding UTM tags to the `EffectiveFinalUrl` field within the `FINAL_URL_REPORT` table. If you are leveraging the auto-tag feature within Google Ads then you will want to enable the `google_auto_tagging_enabled` variable to correctly populate the UTM fields within the `stg_google_ads__final_url_performance` model.
 ```yml

dbt_project.yml

@@ -1,16 +1,30 @@
 name: 'google_ads_source'
-version: '0.3.1'
+version: '0.4.0'
 config-version: 2
 require-dbt-version: [">=0.20.0"]
+
 vars:
-  google_ads_source:
-    google_ads__final_url_performance: "{{ source('adwords','final_url_performance') }}"
-    google_ads__click_performance: "{{ source('adwords','click_performance') }}"
-    google_ads__criteria_performance: "{{ source('adwords','criteria_performance') }}"
+  google_ads__final_url_performance: "{{ source('adwords','final_url_performance') }}"
+  google_ads__click_performance: "{{ source('adwords','click_performance') }}"
+  google_ads__criteria_performance: "{{ source('adwords','criteria_performance') }}"
+  google_ads__ad_final_url_history: "{{ source('adwords','ad_final_url_history') }}"
+  google_ads__ad_stats: "{{ source('adwords','ad_stats') }}"
+  google_ads__ad_history: "{{ source('adwords','ad_history') }}"
+  google_ads__ad_group_history: "{{ source('adwords','ad_group_history') }}"
+  google_ads__campaign_history: "{{ source('adwords','campaign_history') }}"
+  google_ads__account: "{{ source('adwords','account') }}"
+  api_source: adwords
+  google_ads__url_passthrough_metrics: []
+  google_ads__criteria_passthrough_metrics: []
+  google_ads__ad_stats_passthrough_metrics: []
 
 models:
   google_ads_source:
     +schema: stg_google_ads
     +materialized: table
-    tmp:
-      +materialized: view
+    adwords:
+      tmp:
+        +materialized: view
+    google_ads:
+      tmp:
+        +materialized: view

integration_tests/data/google_ads_account_data.csv

@@ -0,0 +1,2 @@
+id,sequence_id,_fivetran_synced,account_label_id,account_label_name,can_manage_clients,currency_code,date_timezone,manager_customer_id,name,test_account
+2121450128,0,2021-09-22T14:31:18.940,,,false,USD,America/Los_Angeles,,Fivetran,false

integration_tests/data/google_ads_ad_final_url_history_data.csv

@@ -0,0 +1,101 @@
+ad_group_id,ad_id,sequence_id,updated_at,_fivetran_synced,url
+104307716352,447458812703,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.878,c7J5Jqb7yaOep18fCNUHAQ==
+104307716352,447458812706,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.878,c7J5Jqb7yaOep18fCNUHAQ==
+104307716352,447458812709,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.878,c7J5Jqb7yaOep18fCNUHAQ==
+104307716352,447458812712,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.878,c7J5Jqb7yaOep18fCNUHAQ==
+104307716352,447458812715,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.878,c7J5Jqb7yaOep18fCNUHAQ==
+104307716352,447458812718,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.878,c7J5Jqb7yaOep18fCNUHAQ==
+104307716352,447458812721,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.878,IWA9w/1ERMOReNTiEwK7Sg==
+104307716352,447458812724,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.878,IWA9w/1ERMOReNTiEwK7Sg==
+104307716352,447458812727,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.878,IWA9w/1ERMOReNTiEwK7Sg==
+104307716352,447458812730,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.878,IWA9w/1ERMOReNTiEwK7Sg==
+104307716352,447458812733,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.878,IWA9w/1ERMOReNTiEwK7Sg==
+104307716352,447458812736,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.878,IWA9w/1ERMOReNTiEwK7Sg==
+104307716352,479045026681,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,IWA9w/1ERMOReNTiEwK7Sg==
+104307716352,480159717006,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,IWA9w/1ERMOReNTiEwK7Sg==
+104307716352,476148702155,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.878,hveE3NYLT4zI8YMKuhWImA==
+104307716352,476148702158,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,hveE3NYLT4zI8YMKuhWImA==
+104307716352,476148702161,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,hveE3NYLT4zI8YMKuhWImA==
+104307716352,476148702164,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,hveE3NYLT4zI8YMKuhWImA==
+104307716352,476148702167,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,hveE3NYLT4zI8YMKuhWImA==
+104307716352,476148702170,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,hveE3NYLT4zI8YMKuhWImA==
+104307716352,477248385764,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,hveE3NYLT4zI8YMKuhWImA==
+104307716352,477248385767,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,hveE3NYLT4zI8YMKuhWImA==
+104307716352,477248385770,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,hveE3NYLT4zI8YMKuhWImA==
+104307716352,477248385773,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,hveE3NYLT4zI8YMKuhWImA==
+104307716352,477248385776,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,hveE3NYLT4zI8YMKuhWImA==
+104307716352,479045026546,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,IWA9w/1ERMOReNTiEwK7Sg==
+104307716352,477248385779,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,hveE3NYLT4zI8YMKuhWImA==
+104307716352,479045026549,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,IWA9w/1ERMOReNTiEwK7Sg==
+104307716352,479045026552,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,IWA9w/1ERMOReNTiEwK7Sg==
+104307716352,479045026555,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,IWA9w/1ERMOReNTiEwK7Sg==
+104307716352,479045026558,0,2021-06-29T17:19:05.102,2021-06-29T17:19:48.879,IWA9w/1ERMOReNTiEwK7Sg==
+110519756800,461419190800,0,2021-06-29T17:19:05.102,2021-06-29T17:20:03.276,JbCxZf8flCfTxJgVEAjPwg==
+110519756800,461419190803,0,2021-06-29T17:19:05.102,2021-06-29T17:20:03.276,JbCxZf8flCfTxJgVEAjPwg==
+110519756800,461419190806,0,2021-06-29T17:19:05.102,2021-06-29T17:20:03.276,JbCxZf8flCfTxJgVEAjPwg==
+110519756800,461419190809,0,2021-06-29T17:19:05.102,2021-06-29T17:20:03.276,JbCxZf8flCfTxJgVEAjPwg==
+110519756800,460851988040,0,2021-06-29T17:19:05.102,2021-06-29T17:20:03.276,sMNzz0E6YO/M7af97xFwqw==
+110519756800,460851988043,0,2021-06-29T17:19:05.102,2021-06-29T17:20:03.276,sMNzz0E6YO/M7af97xFwqw==
+110519756800,460851988046,0,2021-06-29T17:19:05.102,2021-06-29T17:20:03.276,sMNzz0E6YO/M7af97xFwqw==
+110519756800,460851988049,0,2021-06-29T17:19:05.102,2021-06-29T17:20:03.276,sMNzz0E6YO/M7af97xFwqw==
+43937217536,247343893403,0,2021-06-29T17:19:05.102,2021-06-29T17:20:04.773,Evgei1NaJcZ2j/Wd08g9tQ==
+43937217536,247343893406,0,2021-06-29T17:19:05.102,2021-06-29T17:20:04.773,Evgei1NaJcZ2j/Wd08g9tQ==
+43937217536,247343893409,0,2021-06-29T17:19:05.102,2021-06-29T17:20:04.773,Evgei1NaJcZ2j/Wd08g9tQ==
+43937217536,247343893412,0,2021-06-29T17:19:05.102,2021-06-29T17:20:04.773,Evgei1NaJcZ2j/Wd08g9tQ==
+43937217536,201234919397,0,2021-06-29T17:19:05.102,2021-06-29T17:20:04.773,/9HAX+m1bXRx3kHl1cSGWQ==
+43937217536,201234919400,0,2021-06-29T17:19:05.102,2021-06-29T17:20:04.773,/9HAX+m1bXRx3kHl1cSGWQ==
+43937217536,201234919403,0,2021-06-29T17:19:05.102,2021-06-29T17:20:04.773,/9HAX+m1bXRx3kHl1cSGWQ==
+43937217536,201234919406,0,2021-06-29T17:19:05.102,2021-06-29T17:20:04.773,/9HAX+m1bXRx3kHl1cSGWQ==
+122377266944,487639204732,0,2021-06-29T17:19:05.102,2021-06-29T17:19:57.193,IWA9w/1ERMOReNTiEwK7Sg==
+122377266944,487639204735,0,2021-06-29T17:19:05.102,2021-06-29T17:19:57.193,ZEoF6SUxLazDpjaJPn7bfA==
+122377266944,487639204738,0,2021-06-29T17:19:05.102,2021-06-29T17:19:57.193,ZEoF6SUxLazDpjaJPn7bfA==
+122377266944,487639204741,0,2021-06-29T17:19:05.102,2021-06-29T17:19:57.193,dGgNLj9bWns5+yqCrN0XHg==
+122377266944,487639204744,0,2021-06-29T17:19:05.102,2021-06-29T17:19:57.193,IWA9w/1ERMOReNTiEwK7Sg==
+122377266944,487639204747,0,2021-06-29T17:19:05.102,2021-06-29T17:19:57.193,ZEoF6SUxLazDpjaJPn7bfA==
+122377266944,487639204750,0,2021-06-29T17:19:05.102,2021-06-29T17:19:57.193,Zumy6kXkbBzWgDM5jdAnUQ==
+122377266944,487639204753,0,2021-06-29T17:19:05.102,2021-06-29T17:19:57.193,Zumy6kXkbBzWgDM5jdAnUQ==
+122377266944,487639204756,0,2021-06-29T17:19:05.102,2021-06-29T17:19:57.193,ZEoF6SUxLazDpjaJPn7bfA==
+122377266944,487639204759,0,2021-06-29T17:19:05.102,2021-06-29T17:19:57.193,IWA9w/1ERMOReNTiEwK7Sg==
+122377266944,487639204762,0,2021-06-29T17:19:05.102,2021-06-29T17:19:57.193,dGgNLj9bWns5+yqCrN0XHg==
+110870855681,458635485556,0,2021-07-29T01:02:36.202,2021-07-29T08:25:35.663,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-07-27T22:37:17.916,2021-07-28T08:25:50.634,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-07-30T00:43:19.985,2021-07-30T08:22:19.712,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-07-31T01:16:49.321,2021-07-31T08:22:58.643,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-02T03:05:37.774,2021-08-02T14:28:08.408,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-01T02:42:20.589,2021-08-01T14:24:27.478,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-07-23T17:55:56.074,2021-07-24T02:22:52.440,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-07-24T18:37:20.763,2021-07-25T02:22:20.331,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-07-24T20:35:21.559,2021-07-25T08:22:08.109,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-07-25T20:38:34.366,2021-07-26T08:21:53.609,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-07-26T22:38:49.528,2021-07-27T08:22:40.541,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-03T04:54:24.757,2021-08-03T14:17:35.921,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-04T04:51:41.586,2021-08-04T14:26:01.407,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-06T07:26:52.742,2021-08-06T20:23:08.719,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-05T06:34:23.611,2021-08-05T14:24:17.579,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-06T06:37:37.835,2021-08-06T14:24:17.499,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-08T10:43:18.182,2021-08-08T20:22:42.812,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-07T08:44:05.452,2021-08-07T20:23:31.174,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-09T10:52:38.591,2021-08-09T20:24:14.571,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-12T12:38:09,2021-08-12T20:22:53.229,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-11T12:54:42,2021-08-11T20:23:48.745,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-10T11:31:19,2021-08-10T20:23:28.918,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-13T14:41:21,2021-08-14T02:21:30.747,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-12T14:53:08,2021-08-13T02:21:46.827,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-14T16:33:28,2021-08-15T02:23:00.028,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-15T16:34:01,2021-08-16T02:24:25.205,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-16T18:36:38,2021-08-17T02:23:18.379,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-18T18:37:44,2021-08-19T02:22:44.816,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-17T19:45:08,2021-08-18T08:22:40.567,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-17T17:37:57,2021-08-18T02:21:57.512,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-18T21:11:09,2021-08-19T08:23:09.666,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-08-19T20:35:30,2021-08-20T08:22:11.320,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-07-06T02:57:08.831,2021-07-06T14:23:23.537,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-07-04T00:54:18.173,2021-07-04T08:22:24.586,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-07-05T01:00:40.476,2021-07-05T08:23:13.528,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-07-01T22:38:14.746,2021-07-02T08:22:20.653,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-06-29T20:36:37.551,2021-06-30T08:23:21.849,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-07-08T03:56:33.647,2021-07-08T14:23:19.816,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-06-29T17:19:05.102,2021-06-29T17:20:09.040,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-07-03T00:48:18.030,2021-07-03T08:21:39.499,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-07-07T02:49:23.682,2021-07-07T14:24:45.268,10+yX9hVTGt70dM+wkDSKQ==
+110870855681,458635485556,0,2021-06-30T21:01:18.402,2021-07-01T08:23:36.542,10+yX9hVTGt70dM+wkDSKQ==