From e04368807ac6398aaadc9c2f9133de2e6e06cf39 Mon Sep 17 00:00:00 2001 From: Sam Wischnewsky Date: Fri, 26 May 2023 10:18:55 -0700 Subject: [PATCH 1/3] Add batching to sources and fix batch size for tests --- macros/upload_results.sql | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/macros/upload_results.sql b/macros/upload_results.sql index a1de2cb7..69cf99d9 100644 --- a/macros/upload_results.sql +++ b/macros/upload_results.sql @@ -82,8 +82,8 @@ {% for node in graph.nodes.values() | selectattr("resource_type", "equalto", "test") %} {% do tests_set.append(node) %} {% endfor %} - {# upload tests in chunks of 5000 tests (750 for BigQuery), or less #} - {% set upload_limit = 750 if target.type == 'bigquery' else 5000 %} + {# upload tests in chunks of 5000 tests (300 for BigQuery), or less #} + {% set upload_limit = 300 if target.type == 'bigquery' else 5000 %} {% for i in range(0, tests_set | length, upload_limit) -%} {% set content_tests = dbt_artifacts.upload_tests(tests_set[i: i + upload_limit]) %} {{ dbt_artifacts.insert_into_metadata_table( @@ -126,14 +126,22 @@ {% do log("Uploading sources", true) %} {% set sources = dbt_artifacts.get_relation('sources') %} - {% set content_sources = dbt_artifacts.upload_sources(graph) %} - {{ dbt_artifacts.insert_into_metadata_table( - database_name=sources.database, - schema_name=sources.schema, - table_name=sources.identifier, - content=content_sources - ) - }} + {% set sources_set = [] %} + {% for node in graph.nodes.values() | selectattr("resource_type", "equalto", "source") %} + {% do sources_set.append(node) %} + {% endfor %} + {# upload sources in chunks of 500 sources (300 for BigQuery), or less #} + {% set upload_limit = 300 if target.type == 'bigquery' else 500 %} + {% for i in range(0, sources_set | length, upload_limit) -%} + {% set content_sources = dbt_artifacts.upload_sources(sources_set[i: i + upload_limit]) %} + {{ dbt_artifacts.insert_into_metadata_table( + database_name=sources.database, + schema_name=sources.schema, + table_name=sources.identifier, + content=content_sources + ) + }} + {%- endfor %} {% do log("Uploading snapshots", true) %} {% set snapshots = dbt_artifacts.get_relation('snapshots') %} From 988b7760a8ce7a44862066eeb4414f52e1437905 Mon Sep 17 00:00:00 2001 From: Sam Wischnewsky Date: Tue, 30 May 2023 10:42:06 -0700 Subject: [PATCH 2/3] Address comments --- macros/upload_results.sql | 4 ++-- macros/upload_sources.sql | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/macros/upload_results.sql b/macros/upload_results.sql index 69cf99d9..50b195f8 100644 --- a/macros/upload_results.sql +++ b/macros/upload_results.sql @@ -130,8 +130,8 @@ {% for node in graph.nodes.values() | selectattr("resource_type", "equalto", "source") %} {% do sources_set.append(node) %} {% endfor %} - {# upload sources in chunks of 500 sources (300 for BigQuery), or less #} - {% set upload_limit = 300 if target.type == 'bigquery' else 500 %} + {# upload sources in chunks of 5000 sources (300 for BigQuery), or less #} + {% set upload_limit = 300 if target.type == 'bigquery' else 5000 %} {% for i in range(0, sources_set | length, upload_limit) -%} {% set content_sources = dbt_artifacts.upload_sources(sources_set[i: i + upload_limit]) %} {{ dbt_artifacts.insert_into_metadata_table( diff --git a/macros/upload_sources.sql b/macros/upload_sources.sql index 2d09e257..ffe0e2b3 100644 --- a/macros/upload_sources.sql +++ b/macros/upload_sources.sql @@ -1,8 +1,4 @@ -{% macro upload_sources(graph) -%} - {% set sources = [] %} - {% for node in graph.sources.values() %} - {% do sources.append(node) %} - {% endfor %} +{% macro upload_sources(sources) -%} {{ return(adapter.dispatch('get_sources_dml_sql', 'dbt_artifacts')(sources)) }} {%- endmacro %} From 0ea6b37b5b52d81a0f269a1671b3f531482008cd Mon Sep 17 00:00:00 2001 From: Sam Wischnewsky Date: Wed, 31 May 2023 09:15:45 -0700 Subject: [PATCH 3/3] Update macros/upload_results.sql Co-authored-by: Jared Rimmer <100997264+jared-rimmer@users.noreply.github.com> --- macros/upload_results.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/upload_results.sql b/macros/upload_results.sql index 50b195f8..dbfb8051 100644 --- a/macros/upload_results.sql +++ b/macros/upload_results.sql @@ -127,7 +127,7 @@ {% do log("Uploading sources", true) %} {% set sources = dbt_artifacts.get_relation('sources') %} {% set sources_set = [] %} - {% for node in graph.nodes.values() | selectattr("resource_type", "equalto", "source") %} + {% for node in graph.sources.values() %} {% do sources_set.append(node) %} {% endfor %} {# upload sources in chunks of 5000 sources (300 for BigQuery), or less #}