Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ele 3521 add collected by to metric properties #747

Merged
merged 4 commits into from
Aug 13, 2024
Merged

Ele 3521 add collected by to metric properties #747

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d635b34
add collected_by flag to collect_metrics
haritamar Aug 8, 2024
758f878
test_collect_metrics: add cloud_monitored parameter
haritamar Aug 8, 2024
6876c21
bugfixes
haritamar Aug 8, 2024
0d37cdb
validate_unique_metric_names - bugfix
haritamar Aug 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions macros/edr/metadata_collection/get_metric_properties.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,25 @@
time_bucket,
dimensions=none,
freshness_column=none,
event_timestamp_column=none
event_timestamp_column=none,
collected_by=none
) %}
{% set timestamp_column = elementary.get_test_argument('timestamp_column', timestamp_column, model_graph_node) %}
{% set where_expression = elementary.get_test_argument('where_expression', where_expression, model_graph_node) %}
{% set time_bucket = elementary.get_time_bucket(time_bucket, model_graph_node) %}
{% set freshness_column = elementary.get_test_argument('freshness_column', freshness_column, model_graph_node) %}
{% set event_timestamp_column = elementary.get_test_argument('event_timestamp_column', event_timestamp_column, model_graph_node) %}
{% set dimensions = elementary.get_test_argument('dimensions', dimensions, model_graph_node) %}
{% do return({
{% set metric_props = {
'timestamp_column': timestamp_column,
'where_expression': where_expression,
'time_bucket': time_bucket,
'freshness_column': freshness_column,
'event_timestamp_column': event_timestamp_column,
'dimensions': dimensions
}) %}
} %}
{% if collected_by %}
{% do metric_props.update({'collected_by': collected_by}) %}
{% endif %}
{% do return(metric_props) %}
{% endmacro %}
7 changes: 4 additions & 3 deletions macros/edr/tests/test_collect_metrics.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@
days_back=64,
backfill_days=none,
where_expression=none,
dimensions=none
dimensions=none,
cloud_monitored=false
) %}

{{ config(
Expand Down Expand Up @@ -73,11 +74,11 @@
{% endfor %}

{% if table_metrics %}
{% do elementary.collect_table_metrics(table_metrics, model, model_relation, timestamp_column, time_bucket, days_back, backfill_days, where_expression, dimensions) %}
{% do elementary.collect_table_metrics(table_metrics, model, model_relation, timestamp_column, time_bucket, days_back, backfill_days, where_expression, dimensions, collected_by="collect_metrics") %}
{% endif %}

{% for col_name, col_metrics in col_to_metrics.items() %}
{% do elementary.collect_column_metrics(col_metrics, model, model_relation, col_name, timestamp_column, time_bucket, days_back, backfill_days, where_expression, dimensions) %}
{% do elementary.collect_column_metrics(col_metrics, model, model_relation, col_name, timestamp_column, time_bucket, days_back, backfill_days, where_expression, dimensions, collected_by="collect_metrics") %}
{% endfor %}

{# This test always passes. #}
Expand Down
5 changes: 3 additions & 2 deletions macros/edr/tests/test_utils/collect_column_metrics.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@
days_back,
backfill_days,
where_expression,
dimensions
dimensions,
collected_by=none
) %}
{% set model_graph_node = elementary.get_model_graph_node(model_relation) %}
{% set metric_props = elementary.get_metric_properties(model_graph_node, timestamp_column, where_expression, time_bucket, dimensions) %}
{% set metric_props = elementary.get_metric_properties(model_graph_node, timestamp_column, where_expression, time_bucket, dimensions, collected_by=collected_by) %}
{% set days_back = elementary.get_test_argument('days_back', days_back, model_graph_node) %}
{% set backfill_days = elementary.get_test_argument('backfill_days', backfill_days, model_graph_node) %}

Expand Down
5 changes: 3 additions & 2 deletions macros/edr/tests/test_utils/collect_table_metrics.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@
days_back,
backfill_days,
where_expression,
dimensions
dimensions,
collected_by=none
) %}
{% set model_graph_node = elementary.get_model_graph_node(model_relation) %}
{% set metric_props = elementary.get_metric_properties(model_graph_node, timestamp_column, where_expression, time_bucket, dimensions) %}
{% set metric_props = elementary.get_metric_properties(model_graph_node, timestamp_column, where_expression, time_bucket, dimensions, collected_by=collected_by) %}
{% set days_back = elementary.get_test_argument('days_back', days_back, model_graph_node) %}
{% set backfill_days = elementary.get_test_argument('backfill_days', backfill_days, model_graph_node) %}

Expand Down
Loading