From 25ac0ea8b7ebf9df8bb6bea16ef57fb986fc28e7 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Fri, 6 Dec 2024 15:15:49 -0700 Subject: [PATCH 1/4] Per product docs, apply the limit prior to saving the table/view --- .../macros/materializations/tests/test.sql | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/dbt/include/global_project/macros/materializations/tests/test.sql b/dbt/include/global_project/macros/materializations/tests/test.sql index ba205a9b2..92d05af11 100644 --- a/dbt/include/global_project/macros/materializations/tests/test.sql +++ b/dbt/include/global_project/macros/materializations/tests/test.sql @@ -1,6 +1,12 @@ {%- materialization test, default -%} {% set relations = [] %} + {% set limit = config.get('limit') %} + + {% set limited_sql %} + {{ sql }} + {{ "limit " ~ limit if limit != none }} + {% endset %} {% if should_store_failures() %} @@ -26,7 +32,7 @@ {% endif %} {% call statement(auto_begin=True) %} - {{ get_create_sql(target_relation, sql) }} + {{ get_create_sql(target_relation, limited_sql) }} {% endcall %} {% do relations.append(target_relation) %} @@ -40,18 +46,18 @@ {% else %} - {% set main_sql = sql %} + {% set main_sql = limited_sql %} {% endif %} - {% set limit = config.get('limit') %} {% set fail_calc = config.get('fail_calc') %} {% set warn_if = config.get('warn_if') %} {% set error_if = config.get('error_if') %} {% call statement('main', fetch_result=True) -%} - {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}} + {# Since the limit has already been applied above, no need to apply it again! #} + {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit=none)}} {%- endcall %} From 0ac068b2f66f6c924bcd2c02a4b5d25ab5d12c34 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Fri, 6 Dec 2024 15:24:00 -0700 Subject: [PATCH 2/4] Rename the variable holding the SQL for the data test --- .../global_project/macros/materializations/tests/test.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt/include/global_project/macros/materializations/tests/test.sql b/dbt/include/global_project/macros/materializations/tests/test.sql index 92d05af11..34ca2a81e 100644 --- a/dbt/include/global_project/macros/materializations/tests/test.sql +++ b/dbt/include/global_project/macros/materializations/tests/test.sql @@ -3,7 +3,7 @@ {% set relations = [] %} {% set limit = config.get('limit') %} - {% set limited_sql %} + {% set main_sql %} {{ sql }} {{ "limit " ~ limit if limit != none }} {% endset %} @@ -32,7 +32,7 @@ {% endif %} {% call statement(auto_begin=True) %} - {{ get_create_sql(target_relation, limited_sql) }} + {{ get_create_sql(target_relation, main_sql) }} {% endcall %} {% do relations.append(target_relation) %} @@ -46,7 +46,7 @@ {% else %} - {% set main_sql = limited_sql %} + {% set main_sql = main_sql %} {% endif %} From 4a647bb38c314307934d436a64d79c638ac56e90 Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Fri, 6 Dec 2024 15:24:49 -0700 Subject: [PATCH 3/4] Remove the redundant variable assignment that is a no-op --- .../global_project/macros/materializations/tests/test.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dbt/include/global_project/macros/materializations/tests/test.sql b/dbt/include/global_project/macros/materializations/tests/test.sql index 34ca2a81e..057ba30a9 100644 --- a/dbt/include/global_project/macros/materializations/tests/test.sql +++ b/dbt/include/global_project/macros/materializations/tests/test.sql @@ -44,10 +44,6 @@ {{ adapter.commit() }} - {% else %} - - {% set main_sql = main_sql %} - {% endif %} {% set fail_calc = config.get('fail_calc') %} From dc224be67afc6618f4741617e5aa73c7eeadb7ca Mon Sep 17 00:00:00 2001 From: Doug Beatty Date: Fri, 6 Dec 2024 15:26:20 -0700 Subject: [PATCH 4/4] Explain why the SQL query for the data test materialization is altered when storing failures -- to avoid re-computing --- .../global_project/macros/materializations/tests/test.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/include/global_project/macros/materializations/tests/test.sql b/dbt/include/global_project/macros/materializations/tests/test.sql index 057ba30a9..15f557489 100644 --- a/dbt/include/global_project/macros/materializations/tests/test.sql +++ b/dbt/include/global_project/macros/materializations/tests/test.sql @@ -37,6 +37,7 @@ {% do relations.append(target_relation) %} + {# Since the test failures have already been saved to the database, reuse that result rather than querying again #} {% set main_sql %} select * from {{ target_relation }}