From 2650edb4d7e98c532dc6c45067324d763f8441e3 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Fri, 11 Oct 2024 10:53:09 -0400 Subject: [PATCH] Bug 1923976 - Change fxci_derived queries to run at 1800 These queries depend on data from the billing table, which often doesn't get finalized until the afternoon of the following day. It can sometimes populate records much later than that (one month I've seen in the worst case), but that's a problem for another time. This patch should at least fix the common case. --- dags.yaml | 26 +++++++++++++++++-- .../task_run_costs_v1/metadata.yaml | 4 +-- .../worker_costs_v1/metadata.yaml | 4 +-- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/dags.yaml b/dags.yaml index 77164377404..135488cd2e0 100644 --- a/dags.yaml +++ b/dags.yaml @@ -519,8 +519,8 @@ bqetl_desktop_platform: bqetl_internal_tooling: description: | - This DAG schedules queries for populating queries related to Mozilla's - internal developer tooling (e.g. mozregression and Firefox-CI). + This DAG schedules queries for populating tables related to Mozilla's + internal developer tooling (e.g. mozregression). default_args: depends_on_past: false email: @@ -1830,3 +1830,25 @@ bqetl_shredder_monitoring: - repo/bigquery-etl - impact/tier_3 - triage/no_triage + +bqetl_fxci: + description: | + This DAG schedules queries for populating tables related to the + Firefox-CI Taskcluster instance. + default_args: + depends_on_past: false + email: + - ahalberstadt@mozilla.com + - telemetry-alerts@mozilla.com + email_on_failure: true + email_on_retry: true + end_date: null + owner: ahalberstadt@mozilla.com + retries: 2 + retry_delay: 30m + start_date: "2020-10-11" + # This DAG needs to run late as it depends on the GCP billing export which + # often isn't finalized until the afternoon of the following day. + schedule_interval: 0 18 * * * + tags: + - impact/tier_3 diff --git a/sql/moz-fx-data-shared-prod/fxci_derived/task_run_costs_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/fxci_derived/task_run_costs_v1/metadata.yaml index 2febfd4c7e3..d1b905f2531 100644 --- a/sql/moz-fx-data-shared-prod/fxci_derived/task_run_costs_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/fxci_derived/task_run_costs_v1/metadata.yaml @@ -6,9 +6,9 @@ owners: labels: incremental: true owner1: ahalberstadt - dag: bqetl_internal_tooling + dag: bqetl_fxci scheduling: - dag_name: bqetl_internal_tooling + dag_name: bqetl_fxci bigquery: time_partitioning: type: day diff --git a/sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/metadata.yaml index 627a4f287cc..5383bf4b15a 100644 --- a/sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/fxci_derived/worker_costs_v1/metadata.yaml @@ -6,9 +6,9 @@ owners: labels: incremental: true owner1: ahalberstadt - dag: bqetl_internal_tooling + dag: bqetl_fxci scheduling: - dag_name: bqetl_internal_tooling + dag_name: bqetl_fxci task_name: fxci_worker_cost__v1 bigquery: time_partitioning: