Merge branch 'main' into dependabot/pip/protobuf-4.25.8

Author: curtismorales

.circleci/config.yml

@@ -59,6 +59,7 @@ workflows:
             sql/mozfun/.* validate-routines true
             sql/moz-fx-data-shared-prod/udf/.* validate-routines true
             sql/moz-fx-data-shared-prod/udf_js/.* validate-routines true
+            bqetl_project.yaml trigger-sql-generation true
             bqetl_project.yaml validate-sql true
             bqetl_project.yaml validate-routines true
             dags.yaml validate-sql true

.circleci/workflows.yml

@@ -1134,8 +1134,8 @@ jobs:
           name: Manually trigger integration tests for fork
           # yamllint disable rule:line-length
           command: |
-            apt update
-            apt install jq -y
+            sudo apt update
+            sudo apt install jq -y
 
             CIRCLE_PR_BRANCH=`curl -s https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/pulls/${CIRCLE_PR_NUMBER} | jq -r '.head.label'`
 

bigquery_etl/format_sql/format.py

@@ -4,7 +4,7 @@
 import os
 import os.path
 import sys
-from functools import partial
+from functools import cache, partial
 from multiprocessing.pool import Pool
 from pathlib import Path
 from typing import Tuple
@@ -16,6 +16,7 @@
 from bigquery_etl.util.common import qualify_table_references_in_file
 
 
+@cache
 def skip_format():
     """Return a list of configured queries for which formatting should be skipped."""
     return [
@@ -25,6 +26,7 @@ def skip_format():
     ]
 
 
+@cache
 def skip_qualifying_references():
     """Return a list of configured queries where fully qualifying references should be skipped."""
     return [

bqetl_project.yaml

@@ -373,6 +373,7 @@ generate:
       - firefox_reality_pc
       - lockwise_android
       - lockwise_ios
+      - mozilla_vpn  # VPN events are in `main` pings rather than `events` pings (DENG-8596).
       - tiktokreporter_ios
       - tiktokreporter_android
       metrics_as_struct_apps:
@@ -563,6 +564,23 @@ generate:
 geo_deprecation:
   include_app_ids:
   - org_mozilla_ios_klar
+  - org_mozilla_klar
+  - org_mozilla_ios_focus
+  - org_mozilla_focus
+  - org_mozilla_focus_beta
+  - org_mozilla_focus_nightly
+  - org_mozilla_ios_firefox
+  - org_mozilla_ios_firefoxbeta
+  - org_mozilla_ios_fennec
+  - org_mozilla_firefox
+  - org_mozilla_firefox_beta
+  - org_mozilla_fenix
+  - org_mozilla_fenix_nightly
+  - org_mozilla_fennec_aurora
+  - firefox_desktop
+  - firefox_desktop_background_update
+  - firefox_desktop_background_defaultagent
+  - firefox_desktop_background_tasks
   skip_tables:
   - newtab
 

requirements.txt

@@ -654,9 +654,9 @@ google-api-core[grpc]==2.17.1 \
     #   google-cloud-core
     #   google-cloud-datacatalog-lineage
     #   google-cloud-storage
-google-auth==2.41.1 \
-    --hash=sha256:754843be95575b9a19c604a848a41be03f7f2afd8c019f716dc1f51ee41c639d \
-    --hash=sha256:b76b7b1f9e61f0cb7e88870d14f6a94aeef248959ef6992670efee37709cbfd2
+google-auth==2.42.0 \
+    --hash=sha256:9bbbeef3442586effb124d1ca032cfb8fb7acd8754ab79b55facd2b8f3ab2802 \
+    --hash=sha256:f8f944bcb9723339b0ef58a73840f3c61bc91b69bf7368464906120b55804473
     # via
     #   -r requirements.in
     #   gcsfs

sql/moz-fx-data-shared-prod/firefox_desktop/clients_first_seen/view.sql

@@ -52,7 +52,8 @@ SELECT
   b.architecture AS baseline_architecture,
   b.is_desktop AS baseline_is_desktop,
   b.windows_version AS baseline_windows_version,
-  b.normalized_app_name AS baseline_normalized_app_name
+  b.normalized_app_name AS baseline_normalized_app_name,
+  b.app_build_id AS baseline_app_build_id
 FROM
   `moz-fx-data-shared-prod.firefox_desktop.metrics_clients_first_seen` m
 FULL OUTER JOIN

sql/moz-fx-data-shared-prod/firefox_desktop/glean_baseline_clients_first_seen/view.sql

@@ -27,6 +27,7 @@ SELECT
   bcfs.locale,
   bcfs.normalized_os,
   bcfs.app_display_version,
+  bcfs.app_build_id,
   bcfs.normalized_channel,
   bcfs.normalized_os_version,
   bcfs.isp,

sql/moz-fx-data-shared-prod/firefox_desktop_derived/clients_first_seen_28_days_later_v1/query.sql

@@ -42,7 +42,8 @@ WITH clients_first_seen_28_days_ago AS (
     metric_installation_first_seen_version AS installation_first_seen_version,
     baseline_windows_build_number AS windows_build_number,
     baseline_windows_version AS windows_version,
-    baseline_legacy_telemetry_profile_group_id AS profile_group_id
+    baseline_legacy_telemetry_profile_group_id AS profile_group_id,
+    baseline_app_build_id AS app_build_id
   FROM
     `moz-fx-data-shared-prod.firefox_desktop.clients_first_seen`
   WHERE
@@ -63,24 +64,65 @@ clients_first_seen_28_days_ago_with_days_seen AS (
     AND cls.submission_date = @submission_date
 )
 SELECT
-  * REPLACE (
-    COALESCE(
-      days_seen_bits,
-      mozfun.bits28.from_string('0000000000000000000000000000')
-    ) AS days_seen_bits,
-    COALESCE(
-      days_active_bits,
-      mozfun.bits28.from_string('0000000000000000000000000000')
-    ) AS days_active_bits,
-    COALESCE(
-      days_visited_1_uri_bits,
-      mozfun.bits28.from_string('0000000000000000000000000000')
-    ) AS days_visited_1_uri_bits,
-    COALESCE(
-      days_interacted_bits,
-      mozfun.bits28.from_string('0000000000000000000000000000')
-    ) AS days_interacted_bits
-  ),
+  client_id,
+  sample_id,
+  first_seen_date,
+  architecture,
+  app_name,
+  locale,
+  app_version,
+  xpcom_abi,
+  distribution_id,
+  partner_distribution_version,
+  partner_distributor,
+  partner_distributor_channel,
+  partner_id,
+  attribution_campaign,
+  attribution_content,
+  attribution_dltoken,
+  attribution_dlsource,
+  attribution_experiment,
+  attribution_medium,
+  attribution_source,
+  attribution_ua,
+  attribution_variation,
+  apple_model_id,
+  isp_name,
+  normalized_channel,
+  country,
+  normalized_os,
+  normalized_os_version,
+  startup_profile_selection_reason,
+  installation_first_seen_admin_user,
+  installation_first_seen_default_path,
+  installation_first_seen_failure_reason,
+  installation_first_seen_from_msi,
+  installation_first_seen_install_existed,
+  installation_first_seen_installer_type,
+  installation_first_seen_other_inst,
+  installation_first_seen_other_msix_inst,
+  installation_first_seen_profdir_existed,
+  installation_first_seen_silent,
+  installation_first_seen_version,
+  windows_build_number,
+  windows_version,
+  profile_group_id,
+  COALESCE(
+    days_seen_bits,
+    mozfun.bits28.from_string('0000000000000000000000000000')
+  ) AS days_seen_bits,
+  COALESCE(
+    days_active_bits,
+    mozfun.bits28.from_string('0000000000000000000000000000')
+  ) AS days_active_bits,
+  COALESCE(
+    days_visited_1_uri_bits,
+    mozfun.bits28.from_string('0000000000000000000000000000')
+  ) AS days_visited_1_uri_bits,
+  COALESCE(
+    days_interacted_bits,
+    mozfun.bits28.from_string('0000000000000000000000000000')
+  ) AS days_interacted_bits,
   COALESCE(
     BIT_COUNT(mozfun.bits28.from_string('1111111000000000000000000000') & days_seen_bits) >= 5,
     FALSE
@@ -109,6 +151,7 @@ SELECT
     ) > 0,
     FALSE
   ) AS qualified_week4,
-  @submission_date AS submission_date
+  @submission_date AS submission_date,
+  app_build_id
 FROM
   clients_first_seen_28_days_ago_with_days_seen

sql/moz-fx-data-shared-prod/firefox_desktop_derived/clients_first_seen_28_days_later_v1/schema.yaml

@@ -205,3 +205,9 @@ fields:
   type: DATE
   mode: NULLABLE
   description: The date when the telemetry ping is received on the server side.
+- name: app_build_id
+  type: STRING
+  mode: NULLABLE
+  description: |-
+    The build identifier generated by the CI system (e.g. "1234/A").
+    If the value was not provided through configuration, this metric gets set to Unknown.

sql/moz-fx-data-shared-prod/monitoring_derived/looker_user_reported_issues_v1/metadata.yaml

@@ -0,0 +1,13 @@
+friendly_name: Looker User Reported Issues
+description: |-
+  Links to the following spreedsheet with user reported issues in Looker:
+  https://docs.google.com/spreadsheets/d/1joJXR9A8pjdpxEVKAXphXqPh-vy8yRZXs57UZwmUZVs/edit
+owners:
+- [email protected]
+external_data:
+  format: google_sheets
+  source_uris:
+  - https://docs.google.com/spreadsheets/d/1joJXR9A8pjdpxEVKAXphXqPh-vy8yRZXs57UZwmUZVs/edit
+  options:
+    skip_leading_rows: 1
+    range: Form Responses 1