Updated iOS onboarding funnel config to fix the submission_date error

.circleci/workflows.yml

@@ -743,6 +743,7 @@ jobs:
             -d "{\"dagrun_note\": \"${DAGRUN_NOTE}\", \"dag_id\": \"bqetl_artifact_deployment\"}"
   private-generate-sql:
     docker: *docker
+    resource_class: large
     steps:
       - when:
           condition: *deploy
@@ -772,6 +773,7 @@ jobs:
                   rsync --archive ~/private-bigquery-etl/dags.yaml dags.yaml
             - run:
                 name: Generate SQL content
+                no_output_timeout: 30m
                 command: |
                   mkdir -p /tmp/workspace/private-generated-sql
 

.github/pull_request_template.md

@@ -15,4 +15,4 @@ configured to automatically insert hyperlinks for DSRE and DENG tickets.
 See https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/managing-repository-settings/configuring-autolinks-to-reference-external-resources
 -->
 
-**Reviewer, please follow [this checklist](https://github.com/mozilla/bigquery-etl/.github/reviewer_checklist.md)**
+**Reviewer, please follow [this checklist](https://github.com/mozilla/bigquery-etl/blob/main/.github/reviewer_checklist.md)**

bigquery_etl/cli/monitoring.py

@@ -12,7 +12,15 @@
 from bigeye_sdk.client.enum import Method
 from bigeye_sdk.controller.metric_suite_controller import MetricSuiteController
 from bigeye_sdk.exceptions.exceptions import FileLoadException
-from bigeye_sdk.model.big_config import BigConfig, TableDeployment, TableDeploymentSuite
+from bigeye_sdk.model.big_config import (
+    BigConfig,
+    ColumnSelector,
+    RowCreationTimes,
+    TableDeployment,
+    TableDeploymentSuite,
+    TagDeployment,
+    TagDeploymentSuite,
+)
 from bigeye_sdk.model.protobuf_message_facade import (
     SimpleCollection,
     SimpleMetricDefinition,
@@ -107,14 +115,6 @@ def deploy(
                     sql_dir=sql_dir,
                     project_id=project_id,
                 )
-                mc.execute_bigconfig(
-                    input_path=[metadata_file.parent / BIGCONFIG_FILE],
-                    output_path=Path(sql_dir).parent if sql_dir else None,
-                    apply=True,
-                    recursive=False,
-                    strict_mode=True,
-                    auto_approve=True,
-                )
 
                 if (metadata_file.parent / VIEW_FILE).exists():
                     # monitoring to be deployed on a view
@@ -129,6 +129,136 @@ def deploy(
         except FileNotFoundError:
             print("No metadata file for: {}.{}.{}".format(project, dataset, table))
 
+    # Deploy BigConfig files at once.
+    # Deploying BigConfig files separately can lead to previously deployed metrics being removed.
+    mc.execute_bigconfig(
+        input_path=[
+            metadata_file.parent / BIGCONFIG_FILE
+            for metadata_file in list(set(metadata_files))
+        ],
+        output_path=Path(sql_dir).parent if sql_dir else None,
+        apply=True,
+        recursive=False,
+        strict_mode=True,
+        auto_approve=True,
+    )
+
+
+def _update_table_bigconfig(
+    bigconfig,
+    metadata,
+    project,
+    dataset,
+    table,
+):
+    """Update the BigConfig file to monitor a table."""
+    default_metrics = [
+        SimplePredefinedMetricName.FRESHNESS,
+        SimplePredefinedMetricName.VOLUME,
+    ]
+
+    for collection in bigconfig.table_deployments:
+        for deployment in collection.deployments:
+            for metric in deployment.table_metrics:
+                if metric.metric_type.predefined_metric in default_metrics:
+                    default_metrics.remove(metric.metric_type.predefined_metric)
+
+        if metadata.monitoring.collection and collection.collection is None:
+            collection.collection = SimpleCollection(
+                name=metadata.monitoring.collection
+            )
+
+    if len(default_metrics) > 0:
+        deployments = [
+            TableDeployment(
+                fq_table_name=f"{project}.{project}.{dataset}.{table}",
+                table_metrics=[
+                    SimpleMetricDefinition(
+                        metric_type=SimplePredefinedMetric(
+                            type="PREDEFINED", predefined_metric=metric
+                        ),
+                        metric_schedule=SimpleMetricSchedule(
+                            named_schedule=SimpleNamedSchedule(
+                                name="Default Schedule - 13:00 UTC"
+                            )
+                        ),
+                    )
+                    for metric in default_metrics
+                ],
+            )
+        ]
+
+        collection = None
+        if metadata.monitoring.collection:
+            collection = SimpleCollection(name=metadata.monitoring.collection)
+
+        bigconfig.table_deployments += [
+            TableDeploymentSuite(deployments=deployments, collection=collection)
+        ]
+
+
+def _update_view_bigconfig(
+    bigconfig,
+    metadata,
+    project,
+    dataset,
+    table,
+):
+    """Update the BigConfig file to monitor a view."""
+    default_metrics = [
+        SimplePredefinedMetricName.FRESHNESS_DATA,
+        SimplePredefinedMetricName.VOLUME_DATA,
+    ]
+
+    for collection in bigconfig.tag_deployments:
+        for deployment in collection.deployments:
+            for metric in deployment.metrics:
+                if metric.metric_type.predefined_metric in default_metrics:
+                    default_metrics.remove(metric.metric_type.predefined_metric)
+
+        if metadata.monitoring.collection and collection.collection is None:
+            collection.collection = SimpleCollection(
+                name=metadata.monitoring.collection
+            )
+
+    if len(default_metrics) > 0:
+        deployments = [
+            TagDeployment(
+                column_selectors=[
+                    ColumnSelector(name=f"{project}.{project}.{dataset}.{table}.*")
+                ],
+                metrics=[
+                    SimpleMetricDefinition(
+                        metric_type=SimplePredefinedMetric(
+                            type="PREDEFINED", predefined_metric=metric
+                        ),
+                        metric_schedule=SimpleMetricSchedule(
+                            named_schedule=SimpleNamedSchedule(
+                                name="Default Schedule - 13:00 UTC"
+                            )
+                        ),
+                    )
+                    for metric in default_metrics
+                ],
+            )
+        ]
+
+        collection = None
+        if metadata.monitoring.collection:
+            collection = SimpleCollection(name=metadata.monitoring.collection)
+
+        bigconfig.tag_deployments += [
+            TagDeploymentSuite(deployments=deployments, collection=collection)
+        ]
+
+        bigconfig.row_creation_times = RowCreationTimes(
+            column_selectors=[
+                ColumnSelector(
+                    name=f"{project}.{project}.{dataset}.{table}.{metadata.monitoring.partition_column}"
+                )
+            ]
+        )
+
 
 @monitoring.command(
     help="""
@@ -158,55 +288,22 @@ def update(name: str, sql_dir: Optional[str], project_id: Optional[str]) -> None
                 else:
                     bigconfig = BigConfig(type="BIGCONFIG_FILE")
 
-                default_metrics = [
-                    SimplePredefinedMetricName.FRESHNESS,
-                    SimplePredefinedMetricName.VOLUME,
-                ]
-
-                for collection in bigconfig.table_deployments:
-                    for deployment in collection.deployments:
-                        for metric in deployment.table_metrics:
-                            if metric.metric_type.predefined_metric in default_metrics:
-                                default_metrics.remove(
-                                    metric.metric_type.predefined_metric
-                                )
-
-                    if metadata.monitoring.collection and collection.collection is None:
-                        collection.collection = SimpleCollection(
-                            name=metadata.monitoring.collection
-                        )
-
-                if len(default_metrics) > 0:
-                    deployments = [
-                        TableDeployment(
-                            fq_table_name=f"{project}.{project}.{dataset}.{table}",
-                            table_metrics=[
-                                SimpleMetricDefinition(
-                                    metric_type=SimplePredefinedMetric(
-                                        type="PREDEFINED", predefined_metric=metric
-                                    ),
-                                    metric_schedule=SimpleMetricSchedule(
-                                        named_schedule=SimpleNamedSchedule(
-                                            name="Default Schedule - 17:00 UTC"
-                                        )
-                                    ),
-                                )
-                                for metric in default_metrics
-                            ],
-                        )
-                    ]
-
-                    collection = None
-                    if metadata.monitoring.collection:
-                        collection = SimpleCollection(
-                            name=metadata.monitoring.collection
-                        )
-
-                    bigconfig.table_deployments += [
-                        TableDeploymentSuite(
-                            deployments=deployments, collection=collection
-                        )
-                    ]
+                if (metadata_file.parent / VIEW_FILE).exists():
+                    _update_view_bigconfig(
+                        bigconfig=bigconfig,
+                        metadata=metadata,
+                        project=project,
+                        dataset=dataset,
+                        table=table,
+                    )
+                else:
+                    _update_table_bigconfig(
+                        bigconfig=bigconfig,
+                        metadata=metadata,
+                        project=project,
+                        dataset=dataset,
+                        table=table,
+                    )
 
                 bigconfig.save(
                     output_path=bigconfig_file.parent,

bigquery_etl/metadata/parse_metadata.py

@@ -478,6 +478,7 @@ class DatasetMetadata:
     user_facing: bool = attr.ib(False)
     labels: Dict = attr.ib({})
     default_table_workgroup_access: Optional[List[Dict[str, Any]]] = attr.ib(None)
+    default_table_expiration_ms: str = attr.ib(None)
     workgroup_access: list = attr.ib(DEFAULT_WORKGROUP_ACCESS)
     syndication: Dict = attr.ib({})