Add documentation for sources (#282)

* Add documentation for sources * Small change to fix nit and give new commit sha * Fix missing and extra fields * Remove field that doesn't exist everywhere * Change environment variables * Add back in TEST_ENV_VAR for old release
brooklyn-data · May 12, 2023 · 6d7625d · 6d7625d
1 parent 633c6bb
commit 6d7625d
Show file tree

Hide file tree

Showing 13 changed files with 364 additions and 3 deletions.
diff --git a/.github/workflows/publish_docs_on_release.yml b/.github/workflows/publish_docs_on_release.yml
@@ -20,7 +20,9 @@ env:
   DBT_ENV_SECRET_SNOWFLAKE_TEST_DATABASE: ${{ secrets.SNOWFLAKE_TEST_DATABASE }}
   DBT_ENV_SECRET_SNOWFLAKE_TEST_WAREHOUSE: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE }}
   # Env vars to test invocations model
-  TEST_ENV_VAR_1: TEST_VALUE
+  TEST_ENV_VAR_NUMBER: 3
+  TEST_ENV_VAR_EMPTY: ""
+  TEST_ENV_VAR_WITH_QUOTE: "Triggered via Apache Airflow by task 'trigger_dbt_cloud_job_run' in the airtable_ingest DAG."
 
 jobs:
   generate-dbt-docs:

diff --git a/integration_test_project/models/tests_and_exposures.yml b/integration_test_project/models/tests_and_exposures.yml
@@ -10,7 +10,7 @@ models:
           - unique
 
 exposures:
-  - name: "ceo's imaginary dashboard"
+  - name: ceo_imaginary_dashboard
     type: dashboard
     maturity: high
     description: "ceo's favourite dashboard"
@@ -24,7 +24,7 @@ exposures:
       name: Claire from Data
       email: data@jaffleshop.com
 
-  - name: "CIO dashboard"
+  - name: cio_dashboard
     type: dashboard
     maturity: high
     description: '{{ doc("clickstream") }}'

diff --git a/models/sources/exposures.yml b/models/sources/exposures.yml
@@ -0,0 +1,30 @@
+version: 2
+
+models:
+  - name: exposures
+    description: Base model for data about exposure exections. One row per node per run.
+    columns:
+      - name: command_invocation_id
+        description: '{{ doc("command_invocation_id") }}'
+      - name: node_id
+        description: '{{ doc("node_id") }}'
+      - name: run_started_at
+        description: '{{ doc("run_started_at") }}'
+      - name: name
+        description: '{{ doc("name") }}'
+      - name: type
+        description: '{{ doc("type") }}'
+      - name: owner
+        description: '{{ doc("owner") }}'
+      - name: maturity
+        description: '{{ doc("maturity") }}'
+      - name: path
+        description: '{{ doc("path") }}'
+      - name: description
+        description: '{{ doc("description") }}'
+      - name: url
+        description: '{{ doc("url") }}'
+      - name: package_name
+        description: '{{ doc("package_name") }}'
+      - name: depends_on_nodes
+        description: '{{ doc("depends_on_nodes") }}'
diff --git a/models/sources/invocations.yml b/models/sources/invocations.yml
@@ -0,0 +1,44 @@
+version: 2
+
+models:
+  - name: invocations
+    description: Base model for data about the invocations of dbt. One row per run.
+    columns:
+      - name: command_invocation_id
+        description: '{{ doc("command_invocation_id") }}'
+      - name: dbt_version
+        description: '{{ doc("dbt_version") }}'
+      - name: project_name
+        description: '{{ doc("project_name") }}'
+      - name: run_started_at
+        description: '{{ doc("run_started_at") }}'
+      - name: dbt_command
+        description: '{{ doc("dbt_command") }}'
+      - name: full_refresh_flag
+        description: '{{ doc("full_refresh_flag") }}'
+      - name: target_profile_name
+        description: '{{ doc("target_profile_name") }}'
+      - name: target_name
+        description: '{{ doc("target_name") }}'
+      - name: target_schema
+        description: '{{ doc("target_schema") }}'
+      - name: target_threads
+        description: '{{ doc("target_threads") }}'
+      - name: dbt_cloud_project_id
+        description: '{{ doc("dbt_cloud_project_id") }}'
+      - name: dbt_cloud_job_id
+        description: '{{ doc("dbt_cloud_job_id") }}'
+      - name: dbt_cloud_run_id
+        description: '{{ doc("dbt_cloud_run_id") }}'
+      - name: dbt_cloud_run_reason_category
+        description: '{{ doc("dbt_cloud_run_reason_category") }}'
+      - name: dbt_cloud_run_reason
+        description: '{{ doc("dbt_cloud_run_reason") }}'
+      - name: env_vars
+        description: '{{ doc("env_vars") }}'
+      - name: dbt_vars
+        description: '{{ doc("dbt_vars") }}'
+      - name: invocation_args
+        description: '{{ doc("invocation_args") }}'
+      - name: dbt_custom_envs
+        description: '{{ doc("dbt_custom_envs") }}'
diff --git a/models/sources/model_executions.yml b/models/sources/model_executions.yml
@@ -0,0 +1,39 @@
+version: 2
+
+models:
+  - name: model_executions
+    description: Base model for data about model executions. One row per model execution.
+    columns:
+      - name: command_invocation_id
+        description: '{{ doc("command_invocation_id") }}'
+      - name: node_id
+        description: '{{ doc("node_id") }}'
+      - name: run_started_at
+        description: '{{ doc("run_started_at") }}'
+      - name: was_full_refresh
+        description: '{{ doc("was_full_refresh") }}'
+      - name: thread_id
+        description: '{{ doc("thread_id") }}'
+      - name: status
+        description: '{{ doc("status") }}'
+      - name: compile_started_at
+        description: '{{ doc("compile_started_at") }}'
+      - name: query_completed_at
+        description: '{{ doc("query_completed_at") }}'
+      - name: total_node_runtime
+        description: '{{ doc("total_node_runtime") }}'
+      - name: rows_affected
+        description: '{{ doc("rows_affected") }}'
+      # BigQuery only - dbx requires all documented fields to exist in the model
+      # - name: bytes_processed
+      #   description: '{{ doc("bytes_processed") }}'
+      - name: materialization
+        description: '{{ doc("materialization") }}'
+      - name: schema
+        description: '{{ doc("schema") }}'
+      - name: name
+        description: '{{ doc("name") }}'
+      - name: alias
+        description: '{{ doc("alias") }}'
+      - name: message
+        description: '{{ doc("message") }}'
diff --git a/models/sources/models.yml b/models/sources/models.yml
@@ -0,0 +1,34 @@
+version: 2
+
+models:
+  - name: models
+    description: Staging model that contains metadata about model executions. One row per node per run.
+    columns:
+      - name: command_invocation_id
+        description: '{{ doc("command_invocation_id") }}'
+      - name: node_id
+        description: '{{ doc("node_id") }}'
+      - name: run_started_at
+        description: '{{ doc("run_started_at") }}'
+      - name: database
+        description: '{{ doc("database") }}'
+      - name: schema
+        description: '{{ doc("schema") }}'
+      - name: name
+        description: '{{ doc("name") }}'
+      - name: depends_on_nodes
+        description: '{{ doc("depends_on_nodes") }}'
+      - name: package_name
+        description: '{{ doc("package_name") }}'
+      - name: path
+        description: '{{ doc("path") }}'
+      - name: checksum
+        description: '{{ doc("checksum") }}'
+      - name: materialization
+        description: '{{ doc("materialization") }}'
+      - name: tags
+        description: '{{ doc("tags") }}'
+      - name: meta
+        description: '{{ doc("meta") }}'
+      - name: alias
+        description: '{{ doc("alias") }}'
diff --git a/models/sources/seed_executions.yml b/models/sources/seed_executions.yml
@@ -0,0 +1,36 @@
+version: 2
+
+models:
+  - name: seed_executions
+    description: Base model for data about seed executions. One row per seed execution.
+    columns:
+      - name: command_invocation_id
+        description: '{{ doc("command_invocation_id") }}'
+      - name: node_id
+        description: '{{ doc("node_id") }}'
+      - name: run_started_at
+        description: '{{ doc("run_started_at") }}'
+      - name: was_full_refresh
+        description: '{{ doc("was_full_refresh") }}'
+      - name: thread_id
+        description: '{{ doc("thread_id") }}'
+      - name: status
+        description: '{{ doc("status") }}'
+      - name: compile_started_at
+        description: '{{ doc("compile_started_at") }}'
+      - name: query_completed_at
+        description: '{{ doc("query_completed_at") }}'
+      - name: total_node_runtime
+        description: '{{ doc("total_node_runtime") }}'
+      - name: rows_affected
+        description: '{{ doc("rows_affected") }}'
+      - name: materialization
+        description: '{{ doc("materialization") }}'
+      - name: schema
+        description: '{{ doc("schema") }}'
+      - name: name
+        description: '{{ doc("name") }}'
+      - name: alias
+        description: '{{ doc("alias") }}'
+      - name: message
+        description: '{{ doc("message") }}'
diff --git a/models/sources/seeds.yml b/models/sources/seeds.yml
@@ -0,0 +1,28 @@
+version: 2
+
+models:
+  - name: seeds
+    description: Base model for metadata about seed executions. One row per node per run.
+    columns:
+      - name: command_invocation_id
+        description: '{{ doc("command_invocation_id") }}'
+      - name: node_id
+        description: '{{ doc("node_id") }}'
+      - name: run_started_at
+        description: '{{ doc("run_started_at") }}'
+      - name: database
+        description: '{{ doc("database") }}'
+      - name: schema
+        description: '{{ doc("schema") }}'
+      - name: name
+        description: '{{ doc("name") }}'
+      - name: package_name
+        description: '{{ doc("package_name") }}'
+      - name: path
+        description: '{{ doc("path") }}'
+      - name: checksum
+        description: '{{ doc("checksum") }}'
+      - name: meta
+        description: '{{ doc("meta") }}'
+      - name: alias
+        description: '{{ doc("alias") }}'
diff --git a/models/sources/snapshot_executions.yml b/models/sources/snapshot_executions.yml
@@ -0,0 +1,36 @@
+version: 2
+
+models:
+  - name: snapshot_executions
+    description: Base model for data about snapshot executions. One row per snapshot execution.
+    columns:
+      - name: command_invocation_id
+        description: '{{ doc("command_invocation_id") }}'
+      - name: node_id
+        description: '{{ doc("node_id") }}'
+      - name: run_started_at
+        description: '{{ doc("run_started_at") }}'
+      - name: was_full_refresh
+        description: '{{ doc("was_full_refresh") }}'
+      - name: thread_id
+        description: '{{ doc("thread_id") }}'
+      - name: status
+        description: '{{ doc("status") }}'
+      - name: compile_started_at
+        description: '{{ doc("compile_started_at") }}'
+      - name: query_completed_at
+        description: '{{ doc("query_completed_at") }}'
+      - name: total_node_runtime
+        description: '{{ doc("total_node_runtime") }}'
+      - name: rows_affected
+        description: '{{ doc("rows_affected") }}'
+      - name: materialization
+        description: '{{ doc("materialization") }}'
+      - name: schema
+        description: '{{ doc("schema") }}'
+      - name: name
+        description: '{{ doc("name") }}'
+      - name: alias
+        description: '{{ doc("alias") }}'
+      - name: message
+        description: '{{ doc("message") }}'
diff --git a/models/sources/snapshots.yml b/models/sources/snapshots.yml
@@ -0,0 +1,32 @@
+version: 2
+
+models:
+  - name: snapshots
+    description: Base model for metadata about seed executions. One row per node per run.
+    columns:
+      - name: command_invocation_id
+        description: '{{ doc("command_invocation_id") }}'
+      - name: node_id
+        description: '{{ doc("node_id") }}'
+      - name: run_started_at
+        description: '{{ doc("run_started_at") }}'
+      - name: database
+        description: '{{ doc("database") }}'
+      - name: schema
+        description: '{{ doc("schema") }}'
+      - name: name
+        description: '{{ doc("name") }}'
+      - name: depends_on_nodes
+        description: '{{ doc("depends_on_nodes") }}'
+      - name: package_name
+        description: '{{ doc("package_name") }}'
+      - name: path
+        description: '{{ doc("path") }}'
+      - name: checksum
+        description: '{{ doc("checksum") }}'
+      - name: strategy
+        description: '{{ doc("strategy") }}'
+      - name: meta
+        description: '{{ doc("meta") }}'
+      - name: alias
+        description: '{{ doc("alias") }}'
diff --git a/models/sources/sources.yml b/models/sources/sources.yml
@@ -0,0 +1,28 @@
+version: 2
+
+models:
+  - name: sources
+    description: Base model for data about sources. One row per node per execution.
+    columns:
+      - name: command_invocation_id
+        description: '{{ doc("command_invocation_id") }}'
+      - name: node_id
+        description: '{{ doc("node_id") }}'
+      - name: run_started_at
+        description: '{{ doc("run_started_at") }}'
+      - name: database
+        description: '{{ doc("database") }}'
+      - name: schema
+        description: '{{ doc("schema") }}'
+      - name: source_name
+        description: '{{ doc("source_name") }}'
+      - name: loader
+        description: '{{ doc("loader") }}'
+      - name: name
+        description: '{{ doc("name") }}'
+      - name: identifier
+        description: '{{ doc("identifier") }}'
+      - name: loaded_at_field
+        description: '{{ doc("loaded_at_field") }}'
+      - name: freshness
+        description: '{{ doc("freshness") }}'
diff --git a/models/sources/test_executions.yml b/models/sources/test_executions.yml
@@ -0,0 +1,30 @@
+version: 2
+
+models:
+  - name: test_executions
+    description: Base model for metadata about test executions. One row per test execution.
+    columns:
+      - name: command_invocation_id
+        description: '{{ doc("command_invocation_id") }}'
+      - name: node_id
+        description: '{{ doc("node_id") }}'
+      - name: run_started_at
+        description: '{{ doc("run_started_at") }}'
+      - name: was_full_refresh
+        description: '{{ doc("was_full_refresh") }}'
+      - name: thread_id
+        description: '{{ doc("thread_id") }}'
+      - name: status
+        description: '{{ doc("status") }}'
+      - name: compile_started_at
+        description: '{{ doc("compile_started_at") }}'
+      - name: query_completed_at
+        description: '{{ doc("query_completed_at") }}'
+      - name: total_node_runtime
+        description: '{{ doc("total_node_runtime") }}'
+      - name: rows_affected
+        description: '{{ doc("rows_affected") }}'
+      - name: failures
+        description: '{{ doc("failures") }}'
+      - name: message
+        description: '{{ doc("message") }}'
diff --git a/models/sources/tests.yml b/models/sources/tests.yml
@@ -0,0 +1,22 @@
+version: 2
+
+models:
+  - name: tests
+    description: Base model for metadata about test executions. One row per node per run.
+    columns:
+      - name: command_invocation_id
+        description: '{{ doc("command_invocation_id") }}'
+      - name: node_id
+        description: '{{ doc("node_id") }}'
+      - name: run_started_at
+        description: '{{ doc("run_started_at") }}'
+      - name: name
+        description: '{{ doc("name") }}'
+      - name: depends_on_nodes
+        description: '{{ doc("depends_on_nodes") }}'
+      - name: package_name
+        description: '{{ doc("package_name") }}'
+      - name: test_path
+        description: '{{ doc("test_path") }}'
+      - name: tags
+        description: '{{ doc("tags") }}'