Skip to content

Commit bb20c23

Browse files
kik-kikkik-kik
committed
dummy
1 parent 9739dc1 commit bb20c23

File tree

11 files changed

+792
-0
lines changed

11 files changed

+792
-0
lines changed

‎sql/moz-fx-data-shared-prod/amo_glean_derived/dataset_metadata.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
friendly_name: AMO Stats Prod
2+
description: |-
3+
Derived data used to power the AMO stats dashboards
4+
dataset_base_acl: derived
5+
user_facing: true
6+
labels: {}
7+
workgroup_access:
8+
- role: roles/bigquery.dataViewer
9+
members:
10+
- workgroup:mozilla-confidential
11+
- workgroup:amo/prod

‎sql/moz-fx-data-shared-prod/amo_glean_derived/fenix_addons_by_client_v1/metadata.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
---
2+
friendly_name: Fenix addons by client
3+
description: >-
4+
Clients_daily-like table on top of the various Firefox for Android channels
5+
that records only the dimensions and addon info necessary to power the daily
6+
amo_stats_dau_v2 query.
7+
owners:
8+
- [email protected]
9+
labels:
10+
application: amo
11+
incremental: true
12+
schedule: daily
13+
table_table: client_level
14+
scheduling:
15+
dag_name: bqetl_amo_stats
16+
bigquery:
17+
time_partitioning:
18+
type: day
19+
field: submission_date
20+
require_partition_filter: false
21+
expiration_days: 775
22+
clustering:
23+
fields:
24+
- sample_id

‎sql/moz-fx-data-shared-prod/amo_glean_derived/fenix_addons_by_client_v1/query.sql‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
CREATE TEMP FUNCTION get_fields(m ANY TYPE) AS (
2+
STRUCT(
3+
m.submission_timestamp,
4+
m.client_info.client_id,
5+
m.sample_id,
6+
m.metrics.string_list.addons_enabled_addons,
7+
m.normalized_country_code,
8+
m.client_info.locale,
9+
m.normalized_os
10+
)
11+
);
12+
13+
WITH unioned AS (
14+
SELECT
15+
get_fields(release).*,
16+
client_info.app_display_version AS app_version,
17+
FROM
18+
`moz-fx-data-shared-prod.org_mozilla_firefox.metrics` AS release
19+
UNION ALL
20+
SELECT
21+
get_fields(beta).*,
22+
-- Bug 1669516 We choose to show beta versions as 80.0.0b1, etc.
23+
REPLACE(client_info.app_display_version, '-beta.', 'b') AS app_version,
24+
FROM
25+
`moz-fx-data-shared-prod.org_mozilla_firefox_beta.metrics` AS beta
26+
UNION ALL
27+
SELECT
28+
get_fields(nightly).*,
29+
-- Bug 1669516 Nightly versions have app_display_version like "Nightly <timestamp>",
30+
-- so we take the geckoview version instead.
31+
nightly.metrics.string.geckoview_version AS app_version,
32+
FROM
33+
`moz-fx-data-shared-prod.org_mozilla_fenix.metrics` AS nightly
34+
UNION ALL
35+
SELECT
36+
get_fields(preview_nightly).*,
37+
preview_nightly.metrics.string.geckoview_version AS app_version,
38+
FROM
39+
`moz-fx-data-shared-prod.org_mozilla_fenix_nightly.metrics` AS preview_nightly
40+
UNION ALL
41+
SELECT
42+
get_fields(old_fenix_nightly).*,
43+
old_fenix_nightly.metrics.string.geckoview_version AS app_version,
44+
FROM
45+
`moz-fx-data-shared-prod.org_mozilla_fennec_aurora.metrics` AS old_fenix_nightly
46+
),
47+
cleaned AS (
48+
SELECT
49+
* REPLACE (
50+
IF(
51+
-- Accepts formats: 80.0 80.0.0 80.0.0a1 80.0.0b1
52+
REGEXP_CONTAINS(app_version, r'^(\d+\.\d+(\.\d+)?([ab]\d+)?)$'),
53+
app_version,
54+
NULL
55+
) AS app_version
56+
)
57+
FROM
58+
unioned
59+
),
60+
per_client AS (
61+
SELECT
62+
DATE(submission_timestamp) AS submission_date,
63+
client_id,
64+
sample_id,
65+
ARRAY_CONCAT_AGG(addons_enabled_addons ORDER BY submission_timestamp) AS addons,
66+
-- We always want to take the most recent seen version per
67+
-- https://bugzilla.mozilla.org/show_bug.cgi?id=1693308
68+
ARRAY_AGG(app_version ORDER BY mozfun.norm.truncate_version(app_version, "minor") DESC)[
69+
SAFE_OFFSET(0)
70+
] AS app_version,
71+
`moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(normalized_country_code)) AS country,
72+
`moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(locale)) AS locale,
73+
`moz-fx-data-shared-prod.udf.mode_last`(ARRAY_AGG(normalized_os)) AS app_os,
74+
FROM
75+
cleaned
76+
WHERE
77+
DATE(submission_timestamp) = @submission_date
78+
AND client_id IS NOT NULL
79+
GROUP BY
80+
submission_date,
81+
sample_id,
82+
client_id
83+
)
84+
SELECT
85+
* EXCEPT (addons),
86+
ARRAY(
87+
SELECT AS STRUCT
88+
TRIM(addon) AS addon,
89+
-- As of 2020-07-01, the metrics ping from Fenix contains no data about
90+
-- the version of installed addons, so we inject null and replace with
91+
-- an appropriate placeholder value when we get to the app-facing view.
92+
CAST(NULL AS STRING) AS version,
93+
FROM
94+
UNNEST(addons) AS addon
95+
GROUP BY
96+
TRIM(addon)
97+
) AS addons
98+
FROM
99+
per_client

‎sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_addons_by_client_v1/metadata.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
---
2+
friendly_name: Desktop addons by client
3+
description: >-
4+
Clients_daily-like table that records only the dimensions and addon info
5+
necessary to power daily the amo_stats_dau_v2 query.
6+
owners:
7+
- [email protected]
8+
labels:
9+
application: amo
10+
incremental: true
11+
schedule: daily
12+
table_table: client_level
13+
scheduling:
14+
dag_name: bqetl_amo_stats
15+
bigquery:
16+
time_partitioning:
17+
type: day
18+
field: submission_date
19+
require_partition_filter: false
20+
expiration_days: 775
21+
clustering:
22+
fields:
23+
- sample_id

‎sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_addons_by_client_v1/query.sql‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
WITH base AS (
2+
SELECT
3+
submission_timestamp,
4+
client_info.client_id,
5+
sample_id,
6+
normalized_channel,
7+
normalized_country_code,
8+
client_info.locale,
9+
normalized_os,
10+
client_info.app_display_version,
11+
JSON_QUERY_ARRAY(metrics.object.addons_active_addons, '$') AS active_addons,
12+
FROM
13+
`moz-fx-data-shared-prod.firefox_desktop.metrics`
14+
WHERE
15+
DATE(submission_timestamp) = @submission_date
16+
AND client_info.client_id IS NOT NULL
17+
AND metrics.object.addons_active_addons IS NOT NULL
18+
),
19+
per_clients_without_addons AS (
20+
SELECT
21+
DATE(submission_timestamp) AS submission_date,
22+
client_id,
23+
sample_id,
24+
ARRAY_AGG(
25+
app_display_version
26+
ORDER BY
27+
mozfun.norm.truncate_version(app_display_version, "minor") DESC
28+
)[SAFE_OFFSET(0)] AS app_version,
29+
mozfun.stats.mode_last(
30+
ARRAY_AGG(normalized_country_code ORDER BY submission_timestamp)
31+
) AS country,
32+
mozfun.stats.mode_last(ARRAY_AGG(locale ORDER BY submission_timestamp)) AS locale,
33+
mozfun.stats.mode_last(ARRAY_AGG(normalized_os ORDER BY submission_timestamp)) AS app_os,
34+
FROM
35+
base
36+
GROUP BY
37+
ALL
38+
),
39+
per_clients_just_addons AS (
40+
SELECT
41+
DATE(submission_timestamp) AS submission_date,
42+
client_id,
43+
sample_id,
44+
ARRAY_CONCAT_AGG(
45+
ARRAY(
46+
SELECT AS STRUCT
47+
JSON_VALUE(addon, '$.id') AS id,
48+
JSON_VALUE(addon, '$.version') AS `version`
49+
FROM
50+
UNNEST(active_addons) AS addon
51+
)
52+
) AS addons
53+
FROM
54+
base
55+
GROUP BY
56+
ALL
57+
),
58+
per_client AS (
59+
SELECT
60+
submission_date,
61+
client_id,
62+
sample_id,
63+
addons,
64+
app_version,
65+
country,
66+
locale,
67+
app_os,
68+
FROM
69+
per_clients_without_addons
70+
INNER JOIN
71+
per_clients_just_addons
72+
USING (submission_date, client_id, sample_id)
73+
)
74+
SELECT
75+
* EXCEPT (addons),
76+
ARRAY(
77+
SELECT AS STRUCT
78+
addon.id,
79+
-- Same methodology as for app_version above.
80+
ARRAY_AGG(addon.version ORDER BY mozfun.norm.truncate_version(addon.version, "minor") DESC)[
81+
SAFE_OFFSET(0)
82+
] AS `version`,
83+
FROM
84+
UNNEST(addons) AS addon
85+
GROUP BY
86+
addon.id
87+
) AS addons
88+
FROM
89+
per_client

‎sql/moz-fx-data-shared-prod/amo_glean_derived/firefox_desktop_stats_installs_v1/metadata.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
---
2+
friendly_name: AMO Stats DAU
3+
description: >
4+
Daily install statistics to power addons.mozilla.org stats pages.
5+
See bug 1654330. Note that this table uses a hashed_addon_id
6+
defined as `TO_HEX(SHA256(addon_id))` because the underlying event
7+
pings have limitations on length of properties attached to events
8+
and addon_id values are sometimes too long. The AMO stats application
9+
looks up records in this table based on the hashed_addon_id.
10+
11+
The hashed_addon_id is only valid for install_stats:
12+
https://searchfox.org/mozilla-central/rev/5b061cdc4d40d44988dc61aa941cfbd98e31791f/toolkit/components/telemetry/Events.yaml#233-234
13+
owners:
14+
- [email protected]
15+
labels:
16+
application: amo
17+
incremental: true
18+
schedule: daily
19+
table_type: aggregate
20+
shredder_mitigation: false
21+
scheduling:
22+
dag_name: bqetl_amo_stats
23+
bigquery:
24+
time_partitioning:
25+
type: day
26+
field: submission_date
27+
require_partition_filter: false
28+
clustering:
29+
fields:
30+
- hashed_addon_id

0 commit comments

Comments
 (0)