Switch mobile_search_clients_daily derived table to pull from baseline ping from Aug 01st 2024 onwards

bigquery_etl/shredder/config.py

@@ -176,6 +176,9 @@ def fields(self) -> tuple[str, ...]:
     client_id_target(
         table="search_derived.mobile_search_clients_daily_v1"
     ): DESKTOP_SRC,
+    client_id_target(
+        table="search_derived.mobile_search_clients_daily_v2"
+    ): DESKTOP_SRC,
     client_id_target(table="search_derived.search_clients_daily_v8"): DESKTOP_SRC,
     client_id_target(
         table="telemetry_derived.desktop_engagement_clients_v1"

sql/moz-fx-data-shared-prod/fenix_derived/attributable_clients_v1/query.sql

@@ -36,7 +36,7 @@ searches AS (
     SUM(search_with_ads) AS searches_with_ads,
     SUM(ad_click) AS ad_clicks
   FROM
-    `moz-fx-data-shared-prod.search_derived.mobile_search_clients_daily_v1`
+    `moz-fx-data-shared-prod.search.mobile_search_clients_daily`
   WHERE
     {% if is_init() %}
       submission_date >= DATE("2021-01-01")

sql/moz-fx-data-shared-prod/fenix_derived/attributable_clients_v2/query.sql

@@ -37,7 +37,7 @@ searches AS (
     SUM(search_with_ads) AS searches_with_ads,
     SUM(ad_click) AS ad_clicks
   FROM
-    `moz-fx-data-shared-prod.search_derived.mobile_search_clients_daily_v1`
+    `moz-fx-data-shared-prod.search.mobile_search_clients_daily`
   WHERE
     {% if is_init() %}
       submission_date >= "2021-08-01"

sql/moz-fx-data-shared-prod/fenix_derived/new_profile_activation_v1/query.sql

@@ -29,7 +29,7 @@ client_search AS (
     client_id,
     SUM(search_count) AS search_count
   FROM
-    `moz-fx-data-shared-prod.search_derived.mobile_search_clients_daily_v1`
+    `moz-fx-data-shared-prod.search.mobile_search_clients_daily`
   JOIN
     client_first_seen
     USING (client_id)

sql/moz-fx-data-shared-prod/firefox_ios_derived/attributable_clients_v1/query.sql

@@ -37,7 +37,7 @@ metrics_searches AS (
     SUM(search_with_ads) AS searches_with_ads,
     SUM(ad_click) AS ad_clicks
   FROM
-    `moz-fx-data-shared-prod.search_derived.mobile_search_clients_daily_v1`
+    `moz-fx-data-shared-prod.search.mobile_search_clients_daily`
   WHERE
     {% if is_init() %}
       submission_date >= "2020-05-01"

sql/moz-fx-data-shared-prod/firefox_ios_derived/clients_activation_v1/query.sql

@@ -38,7 +38,7 @@ clients_search AS (
     channel,
     SUM(search_count) AS search_count
   FROM
-    `moz-fx-data-shared-prod.search_derived.mobile_search_clients_daily_v1`
+    `moz-fx-data-shared-prod.search.mobile_search_clients_daily`
   WHERE
     (submission_date BETWEEN DATE_SUB(@submission_date, INTERVAL 3 DAY) AND @submission_date)
     AND os = 'iOS'

sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profile_activation_v1/query.sql

@@ -29,7 +29,7 @@ client_search AS (
     client_id,
     SUM(search_count) AS search_count
   FROM
-    `moz-fx-data-shared-prod.search_derived.mobile_search_clients_daily_v1`
+    `moz-fx-data-shared-prod.search.mobile_search_clients_daily`
   JOIN
     client_first_seen
     USING (client_id)

sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profile_activation_v2/query.sql

@@ -24,7 +24,7 @@ client_search AS (
     client_id,
     SUM(search_count) AS search_count
   FROM
-    `moz-fx-data-shared-prod.search_derived.mobile_search_clients_daily_v1`
+    `moz-fx-data-shared-prod.search.mobile_search_clients_daily`
   WHERE
     (submission_date BETWEEN DATE_SUB(@submission_date, INTERVAL 3 DAY) AND @submission_date)
     AND os = 'iOS'

sql/moz-fx-data-shared-prod/search/mobile_search_clients_daily/metadata.yaml

@@ -0,0 +1,8 @@
+---
+friendly_name: Mobile Search Clients Daily
+description: |
+  A daily aggregate of baseline and metrics pings that have
+  searches from each (client_id, engine, source) combination,
+  partitioned by day.
+owners:
+  - akommasani@mozilla.com

sql/moz-fx-data-shared-prod/search/mobile_search_clients_daily/view.sql

@@ -0,0 +1,96 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.search.mobile_search_clients_daily`
+AS
+SELECT
+  submission_date,
+  client_id,
+  engine,
+  source,
+  app_name,
+  search_count,
+  organic,
+  tagged_sap,
+  tagged_follow_on,
+  ad_click,
+  search_with_ads,
+  unknown,
+  country,
+  locale,
+  app_version,
+  channel,
+  os,
+  os_version,
+  default_search_engine,
+  default_search_engine_submission_url,
+  distribution_id,
+  profile_creation_date,
+  profile_age_in_days,
+  sample_id,
+  experiments,
+  total_uri_count,
+  ad_click_organic,
+  search_with_ads_organic,
+  os_version_major,
+  os_version_minor,
+  `moz-fx-data-shared-prod`.udf.normalize_search_engine(engine) AS normalized_engine,
+  `mozfun.mobile_search.normalize_app_name`(
+    app_name,
+    os
+  ).normalized_app_name AS normalized_app_name,
+  `mozfun.norm.browser_version_info`(app_version) AS browser_version_info,
+  search_count AS sap,
+  `mozfun.mobile_search.normalize_app_name`(
+    app_name,
+    os
+  ).normalized_app_name AS normalized_app_name_os
+FROM
+  `moz-fx-data-shared-prod.search_derived.mobile_search_clients_daily_historical_pre202408`
+WHERE
+  submission_date <= '2024-07-31'
+UNION ALL
+SELECT
+  submission_date,
+  client_id,
+  engine,
+  source,
+  app_name,
+  search_count,
+  organic,
+  tagged_sap,
+  tagged_follow_on,
+  ad_click,
+  search_with_ads,
+  unknown,
+  country,
+  locale,
+  app_version,
+  channel,
+  os,
+  os_version,
+  default_search_engine,
+  default_search_engine_submission_url,
+  distribution_id,
+  profile_creation_date,
+  profile_age_in_days,
+  sample_id,
+  experiments,
+  total_uri_count,
+  ad_click_organic,
+  search_with_ads_organic,
+  os_version_major,
+  os_version_minor,
+  `moz-fx-data-shared-prod`.udf.normalize_search_engine(engine) AS normalized_engine,
+  `mozfun.mobile_search.normalize_app_name`(
+    app_name,
+    os
+  ).normalized_app_name AS normalized_app_name,
+  `mozfun.norm.browser_version_info`(app_version) AS browser_version_info,
+  search_count AS sap,
+  `mozfun.mobile_search.normalize_app_name`(
+    app_name,
+    os
+  ).normalized_app_name AS normalized_app_name_os
+FROM
+  `moz-fx-data-shared-prod.search_derived.mobile_search_clients_daily_v2`
+WHERE
+  submission_date > '2024-07-31'

sql/moz-fx-data-shared-prod/search/mobile_search_clients_engines_sources_daily/view.sql

@@ -2,37 +2,143 @@ CREATE OR REPLACE VIEW
   `moz-fx-data-shared-prod.search.mobile_search_clients_engines_sources_daily`
 AS
 SELECT
-  * EXCEPT (normalized_engine, normalized_app_name),
+  submission_date,
+  client_id,
+  engine,
+  source,
+  app_name,
+  search_count,
+  organic,
+  tagged_sap,
+  tagged_follow_on,
+  ad_click,
+  search_with_ads,
+  unknown,
+  country,
+  locale,
+  app_version,
+  channel,
+  os,
+  os_version,
+  default_search_engine,
+  default_search_engine_submission_url,
+  distribution_id,
+  profile_creation_date,
+  profile_age_in_days,
+  sample_id,
+  experiments,
+  total_uri_count,
+  ad_click_organic,
+  search_with_ads_organic,
+  os_version_major,
+  os_version_minor,
   `moz-fx-data-shared-prod`.udf.normalize_search_engine(engine) AS normalized_engine,
   `mozfun.mobile_search.normalize_app_name`(
     app_name,
     os
   ).normalized_app_name AS normalized_app_name,
   `mozfun.norm.browser_version_info`(app_version) AS browser_version_info,
   search_count AS sap,
-  `mozfun.mobile_search.normalize_app_name`(app_name, os).normalized_app_name_os
+  `mozfun.mobile_search.normalize_app_name`(
+    app_name,
+    os
+  ).normalized_app_name AS normalized_app_name_os
 FROM
-  `moz-fx-data-shared-prod.search_derived.mobile_search_clients_daily_v1`
+  `moz-fx-data-shared-prod.search_derived.mobile_search_clients_daily_historical_pre202408`
 WHERE
-  app_name NOT IN (
-    'Fennec',
-    'Focus Android Glean',
-    'Klar Android Glean',
-    'Focus iOS Glean',
-    'Klar iOS Glean',
-    'Focus',
-    'Klar'
-  )
-  OR (
-    app_name = 'Fennec'
-    AND (
-      os != 'iOS'
-      OR submission_date < '2023-01-01'
-      OR mozfun.norm.truncate_version(app_version, 'major') >= 28
+  submission_date <= '2024-07-31'
+  AND (
+    app_name NOT IN (
+      'Fennec',
+      'Focus Android Glean',
+      'Klar Android Glean',
+      'Focus iOS Glean',
+      'Klar iOS Glean',
+      'Focus',
+      'Klar'
     )
+    OR (
+      app_name = 'Fennec'
+      AND (
+        os != 'iOS'
+        OR submission_date < '2023-01-01'
+        OR mozfun.norm.truncate_version(app_version, 'major') >= 28
+      )
+    )
+    OR (
+      app_name IN ('Focus Android Glean', 'Klar Android Glean', 'Focus iOS Glean', 'Klar iOS Glean')
+      AND submission_date >= '2023-01-01'
+    )
+    OR (app_name IN ('Focus', 'Klar') AND submission_date < '2023-01-01')
   )
-  OR (
-    app_name IN ('Focus Android Glean', 'Klar Android Glean', 'Focus iOS Glean', 'Klar iOS Glean')
-    AND submission_date >= '2023-01-01'
-  )
-  OR (app_name IN ('Focus', 'Klar') AND submission_date < '2023-01-01')
+UNION ALL
+SELECT
+  submission_date,
+  client_id,
+  engine,
+  source,
+  app_name,
+  search_count,
+  organic,
+  tagged_sap,
+  tagged_follow_on,
+  ad_click,
+  search_with_ads,
+  unknown,
+  country,
+  locale,
+  app_version,
+  channel,
+  os,
+  os_version,
+  default_search_engine,
+  default_search_engine_submission_url,
+  distribution_id,
+  profile_creation_date,
+  profile_age_in_days,
+  sample_id,
+  experiments,
+  total_uri_count,
+  ad_click_organic,
+  search_with_ads_organic,
+  os_version_major,
+  os_version_minor,
+  `moz-fx-data-shared-prod`.udf.normalize_search_engine(engine) AS normalized_engine,
+  `mozfun.mobile_search.normalize_app_name`(
+    app_name,
+    os
+  ).normalized_app_name AS normalized_app_name,
+  `mozfun.norm.browser_version_info`(app_version) AS browser_version_info,
+  search_count AS sap,
+  `mozfun.mobile_search.normalize_app_name`(
+    app_name,
+    os
+  ).normalized_app_name AS normalized_app_name_os
+FROM
+  `moz-fx-data-shared-prod.search_derived.mobile_search_clients_daily_v2`
+WHERE
+  submission_date > '2024-07-31'
+  AND (
+    app_name NOT IN (
+      'Fennec',
+      'Focus Android Glean',
+      'Klar Android Glean',
+      'Focus iOS Glean',
+      'Klar iOS Glean',
+      'Focus',
+      'Klar'
+    )
+    OR (
+      app_name = 'Fennec'
+      AND (
+        os != 'iOS'
+        OR submission_date < '2023-01-01'
+        OR mozfun.norm.truncate_version(app_version, 'major') >= 28
+      )
+    )
+    OR (
+      app_name IN ('Focus Android Glean', 'Klar Android Glean', 'Focus iOS Glean', 'Klar iOS Glean')
+      AND submission_date >= '2023-01-01'
+    )
+    OR (app_name IN ('Focus', 'Klar') AND submission_date < '2023-01-01')
+  );