Ele 700 add default values to time bucket (#346)

* added default to time_bucket * default time_bucket and validation * default time_bucket and validation * fix * fix * added e2e * added e2e * remove debug * format * PR fixes
elementary-data · May 14, 2023 · 133ed73 · 133ed73
1 parent 15b3e2d
commit 133ed73
Show file tree

Hide file tree

Showing 18 changed files with 196 additions and 27 deletions.
diff --git a/integration_tests/macros/e2e_tests/test_config_levels.sql b/integration_tests/macros/e2e_tests/test_config_levels.sql
@@ -0,0 +1,20 @@
+{% test config_levels(model, expected_config, time_bucket) %}
+    {%- if execute and flags.WHICH in ['test', 'build'] %}
+        {%- set unexpected_config = [] %}
+        {%- set model_relation = dbt.load_relation(model) %}
+        {% set model_graph_node = elementary.get_model_graph_node(model_relation) %}
+
+        {%- set time_bucket = elementary.get_time_bucket(time_bucket, model_graph_node) %}
+        {%- if time_bucket != expected_config.get('time_bucket') %}
+            {%- set unexpected_message = ('got config: {0}, expected config: {1}').format(time_bucket, expected_config.get('time_bucket') ) %}
+            {%- do unexpected_config.append(unexpected_message) -%}
+        {%- endif %}
+
+        {%- if unexpected_config | length > 0 %}
+            {%- do exceptions.raise_compiler_error('Failure config_levels: ' ~ unexpected_config) -%}
+        {%- else %}
+            {#- test must run an sql query -#}
+            {{ elementary.no_results_query() }}
+        {%- endif %}
+    {%- endif %}
+{%- endtest %}
diff --git a/integration_tests/macros/e2e_tests/validate_config_levels.sql b/integration_tests/macros/e2e_tests/validate_config_levels.sql
@@ -0,0 +1,18 @@
+{% macro validate_config_levels() %}
+    {% set alerts_relation = ref('test_alerts_union') %}
+
+    {% set config_levels_validation_query %}
+    with error_tests as (
+        select
+            table_name, alert_description,
+            {{ elementary.contains('tags', 'config_levels') }} as is_config_levels_tag
+        from {{ alerts_relation }}
+        where status = 'error'
+        )
+    select table_name, alert_description
+    from error_tests
+    where is_config_levels_tag = true
+    {% endset %}
+    {% set results = elementary.agate_to_dicts(run_query(config_levels_validation_query)) | unique | list %}
+    {{ assert_lists_contain_same_items(results, []) }}
+{% endmacro %}
diff --git a/integration_tests/macros/unit_tests/test_get_time_bucket.sql b/integration_tests/macros/unit_tests/test_get_time_bucket.sql
@@ -0,0 +1,24 @@
+{% macro test_get_time_bucket() %}
+
+    {%- set default_config = elementary.get_default_time_bucket() %}
+
+    {%- set config = {'period': 'day', 'count': 5} %}
+    {%- set result = elementary.get_time_bucket(config) %}
+    {{ assert_value(result.period, "day") }}
+    {{ assert_value(result.count, 5) }}
+
+    {%- set config = {'period': 'week'} %}
+    {%- set result = elementary.get_time_bucket(config) %}
+    {{ assert_value(result.period, "week") }}
+    {{ assert_value(result.count, default_config.count) }}
+
+    {%- set config = {'count': 3} %}
+    {%- set result = elementary.get_time_bucket(config) %}
+    {{ assert_value(result.count, 3) }}
+    {{ assert_value(result.period, default_config.period) }}
+
+    {%- set result = elementary.get_time_bucket() %}
+    {{ assert_value(result.count, default_config.count) }}
+    {{ assert_value(result.period, default_config.period) }}
+
+{% endmacro %}
diff --git a/integration_tests/models/config_levels_project.sql b/integration_tests/models/config_levels_project.sql
@@ -0,0 +1 @@
+select 1 as one
diff --git a/integration_tests/models/config_levels_test_and_model.sql b/integration_tests/models/config_levels_test_and_model.sql
@@ -0,0 +1 @@
+select 1 as one
diff --git a/integration_tests/models/schema.yml b/integration_tests/models/schema.yml
@@ -393,6 +393,42 @@ models:
           tags: ["ephemeral_model", "error_test"]
 
 
+  - name: config_levels_test_and_model
+    tags: ["config_levels"]
+    config:
+      elementary:
+        time_bucket:
+          period: day
+          count: 3
+    tests:
+      - config_levels:
+          tags: ["config_levels"]
+          alias: 'test_level_config'
+          time_bucket:
+            period: hour
+            count: 4
+          expected_config:
+            time_bucket:
+              period: hour
+              count: 4
+      - config_levels:
+          tags: ["config_levels"]
+          alias: 'model_level_config'
+          expected_config:
+            time_bucket:
+              period: day
+              count: 3
+  - name: config_levels_project
+    tags: ["config_levels"]
+    tests:
+      - config_levels:
+          tags: ["config_levels"]
+          alias: 'project_level_config'
+          expected_config:
+            time_bucket:
+              period: day
+              count: 1
+
 sources:
   - name: training
     schema: test_seeds

diff --git a/integration_tests/run_e2e_tests.py b/integration_tests/run_e2e_tests.py
@@ -324,6 +324,16 @@ def e2e_tests(
         ]
         test_results.extend(results)
 
+    if "config_levels" in test_types:
+        dbt_runner.test(select="tag:config_levels")
+        results = [
+            TestResult(type="config_levels", message=msg)
+            for msg in dbt_runner.run_operation(
+                macro_name="validate_config_levels", should_log=False
+            )
+        ]
+        test_results.extend(results)
+
     if "artifacts" in test_types:
         results = [
             TestResult(type="artifacts", message=msg)
@@ -402,6 +412,7 @@ def main(target, e2e_type, generate_data, clear_tests):
             "backfill_days",
             "schema",
             "regular",
+            "config_levels",
             "artifacts",
             "error_test",
             "error_model",

diff --git a/macros/edr/data_monitoring/data_monitors_configuration/time_bucket.sql b/macros/edr/data_monitoring/data_monitors_configuration/time_bucket.sql
diff --git a/macros/edr/system/system_utils/get_test_argument.sql b/macros/edr/system/system_utils/get_test_argument.sql
@@ -1,7 +1,18 @@
-{% macro get_test_argument(argument_name, value=none) %}
+{% macro get_test_argument(argument_name, value, model_graph_node) %}
   {% if value %}
     {{ return(value) }}
-  {% else %}
+  {%- endif %}
+  {%- if model_graph_node %}
+    {% set elementary_config = elementary.get_elementary_config_from_node(model_graph_node) %}
+    {% if elementary_config and elementary_config is mapping %}
+        {%- set model_config_value = elementary_config.get(argument_name) %}
+        {%- if model_config_value %}
+            {{ return(model_config_value) }}
+        {%- endif %}
+    {% endif %}
+  {% endif %}
+  {%- if elementary.get_config_var(argument_name) %}
     {{ return(elementary.get_config_var(argument_name)) }}
   {% endif %}
-{% endmacro %}
+  {{ return(none) }}
+{% endmacro %}
diff --git a/macros/edr/tests/test_all_columns_anomalies.sql b/macros/edr/tests/test_all_columns_anomalies.sql
@@ -9,10 +9,6 @@
             {{ exceptions.raise_compiler_error("The test is not supported for ephemeral models, model name: {}".format(model.identifier)) }}
         {%- endif %}
 
-        {% if not time_bucket %}
-          {% set time_bucket = elementary.get_default_time_bucket() %}
-        {% endif %}
-
         {%- set test_table_name = elementary.get_elementary_test_table_name() %}
         {{- elementary.debug_log('collecting metrics for test: ' ~ test_table_name) }}
         {#- creates temp relation for test metrics -#}
@@ -30,6 +26,7 @@
 
         {% set model_graph_node = elementary.get_model_graph_node(model_relation) %}
         {% set timestamp_column = elementary.get_timestamp_column(timestamp_column, model_graph_node) %}
+        {%- set time_bucket = elementary.get_time_bucket(time_bucket, model_graph_node) %}
 
         {% do elementary.validate_seasonality_parameter(seasonality=seasonality, time_bucket=time_bucket, timestamp_column=timestamp_column) %}
         {% set days_back = elementary.get_days_back(seasonality=seasonality) %}

diff --git a/macros/edr/tests/test_column_anomalies.sql b/macros/edr/tests/test_column_anomalies.sql
@@ -9,10 +9,6 @@
             {{ exceptions.raise_compiler_error("The test is not supported for ephemeral models, model name: {}".format(model.identifier)) }}
         {%- endif %}
 
-        {% if not time_bucket %}
-          {% set time_bucket = elementary.get_default_time_bucket() %}
-        {% endif %}
-
         {% set test_table_name = elementary.get_elementary_test_table_name() %}
         {{ elementary.debug_log('collecting metrics for test: ' ~ test_table_name) }}
         {#- creates temp relation for test metrics -#}
@@ -28,6 +24,7 @@
 
         {% set model_graph_node = elementary.get_model_graph_node(model_relation) %}
         {% set timestamp_column = elementary.get_timestamp_column(timestamp_column, model_graph_node) %}
+        {%- set time_bucket = elementary.get_time_bucket(time_bucket, model_graph_node) %}
 
         {% do elementary.validate_seasonality_parameter(seasonality=seasonality, time_bucket=time_bucket, timestamp_column=timestamp_column) %}
         {% set days_back = elementary.get_days_back(seasonality=seasonality) %}

diff --git a/...os/edr/tests/test_utils/get_days_back.sql → ...ests/test_configuration/get_days_back.sql b/...os/edr/tests/test_utils/get_days_back.sql → ...ests/test_configuration/get_days_back.sql
diff --git a/...test_utils/get_model_baseline_columns.sql → ...figuration/get_model_baseline_columns.sql b/...test_utils/get_model_baseline_columns.sql → ...figuration/get_model_baseline_columns.sql
diff --git a/macros/edr/tests/test_configuration/get_time_bucket.sql b/macros/edr/tests/test_configuration/get_time_bucket.sql
@@ -0,0 +1,66 @@
+{% macro get_daily_time_bucket() %}
+  {% do return({"period": "day", "count": 1}) %}
+{% endmacro %}
+
+{% macro get_default_time_bucket() %}
+  {% do return(elementary.get_daily_time_bucket()) %}
+{% endmacro %}
+
+{% macro get_time_bucket(time_bucket, model_graph_node) %}
+    {%- set configured_time_bucket = elementary.get_test_argument('time_bucket', time_bucket, model_graph_node) %}
+    {%- do elementary.validate_time_bucket(configured_time_bucket) %}
+    {%- set default_time_bucket = elementary.get_default_time_bucket() %}
+
+    {%- if not configured_time_bucket %}
+        {{ return(default_time_bucket) }}
+    {%- else %}
+        {%- set time_bucket = default_time_bucket.copy() %}
+        {%- do time_bucket.update(configured_time_bucket) %}
+        {{ return(time_bucket) }}
+    {%- endif %}
+{% endmacro %}
+
+
+{% macro validate_time_bucket(time_bucket) %}
+    {% if time_bucket %}
+        {%- if time_bucket is not mapping %}
+            {% do exceptions.raise_compiler_error(
+            "
+            Invalid time_bucket format. Expected format:
+
+               time_bucket:
+                 count: int
+                 period: string
+            ") %}
+        {%- endif %}
+        {%- if time_bucket is mapping %}
+            {%- set invalid_keys = [] %}
+            {%- set valid_keys = ['period', 'count'] %}
+            {%- for key, value in time_bucket.items() %}
+                {%- if key not in valid_keys %}
+                    {%- do invalid_keys.append(key) -%}
+                {%- endif %}
+            {%- endfor %}
+            {%- if invalid_keys | length > 0 %}
+                {% do exceptions.raise_compiler_error(
+                ("
+                Found invalid keys in time_bucket: {0}.
+                Supported keys: {1}.
+                Expected format:
+
+                   time_bucket:
+                     count: int
+                     period: string
+                ").format(invalid_keys, valid_keys)) %}
+            {%- endif %}
+        {%- endif %}
+
+        {% if time_bucket.count and time_bucket.count is not integer %}
+            {% do exceptions.raise_compiler_error("time_bucket.count expects valid integer, got: {} (If it's an integer, try to remove quotes)".format(time_bucket.count)) %}
+        {% endif %}
+        {% set supported_periods = ['hour','day','week','month'] %}
+        {% if time_bucket.period and time_bucket.period not in supported_periods %}
+            {% do exceptions.raise_compiler_error("time_bucket.period value should be one of {0}, got: {1}".format(supported_periods, time_bucket.period)) %}
+        {% endif %}
+    {% endif %}
+{% endmacro %}
diff --git a/...tests/test_utils/get_timestamp_column.sql → ...st_configuration/get_timestamp_column.sql b/...tests/test_utils/get_timestamp_column.sql → ...st_configuration/get_timestamp_column.sql
diff --git a/...s/test_utils/validate_test_parameters.sql → ...onfiguration/validate_test_parameters.sql b/...s/test_utils/validate_test_parameters.sql → ...onfiguration/validate_test_parameters.sql
diff --git a/macros/edr/tests/test_dimension_anomalies.sql b/macros/edr/tests/test_dimension_anomalies.sql
@@ -11,9 +11,6 @@
         {% if not dimensions %}
             {{ exceptions.raise_compiler_error('Dimension anomalies test must get "dimensions" as a parameter!') }}
         {% endif %}
-        {% if not time_bucket %}
-          {% set time_bucket = elementary.get_default_time_bucket() %}
-        {% endif %}
 
         {% set test_table_name = elementary.get_elementary_test_table_name() %}
         {{ elementary.debug_log('collecting metrics for test: ' ~ test_table_name) }}
@@ -30,6 +27,7 @@
 
         {% set model_graph_node = elementary.get_model_graph_node(model_relation) %}
         {% set timestamp_column = elementary.get_timestamp_column(timestamp_column, model_graph_node) %}
+        {%- set time_bucket = elementary.get_time_bucket(time_bucket, model_graph_node) %}
 
         {% do elementary.validate_seasonality_parameter(seasonality=seasonality, time_bucket=time_bucket, timestamp_column=timestamp_column) %}
         {% set days_back = elementary.get_days_back(seasonality=seasonality) %}

diff --git a/macros/edr/tests/test_table_anomalies.sql b/macros/edr/tests/test_table_anomalies.sql
@@ -10,10 +10,6 @@
             {{ exceptions.raise_compiler_error("The test is not supported for ephemeral models, model name: {}".format(model.identifier)) }}
         {%- endif %}
 
-        {% if not time_bucket %}
-          {% set time_bucket = elementary.get_default_time_bucket() %}
-        {% endif %}
-
         {% set test_table_name = elementary.get_elementary_test_table_name() %}
         {{ elementary.debug_log('collecting metrics for test: ' ~ test_table_name) }}
         {#- creates temp relation for test metrics -#}
@@ -28,8 +24,8 @@
         {% endif %}
 
         {% set model_graph_node = elementary.get_model_graph_node(model_relation) %}
-        {% set timestamp_column = elementary.get_timestamp_column(timestamp_column, model_graph_node) %}
-
+        {%- set timestamp_column = elementary.get_timestamp_column(timestamp_column, model_graph_node) %}
+        {%- set time_bucket = elementary.get_time_bucket(time_bucket, model_graph_node) %}
 
         {% do elementary.validate_seasonality_parameter(seasonality=seasonality, time_bucket=time_bucket, timestamp_column=timestamp_column) %}
         {% set days_back = elementary.get_days_back(seasonality=seasonality) %}