From a1006c152f1245a7cf399fe099c9f5e5430bddce Mon Sep 17 00:00:00 2001 From: Adrian Burus Date: Tue, 17 Dec 2024 11:01:20 +0200 Subject: [PATCH 01/40] Added equals macro to handle null value comparison --- .../dbt/tests/adapter/utils/base_utils.py | 10 ---------- .../dbt/tests/adapter/utils/test_equals.py | 8 +------- .../materializations/models/incremental/merge.sql | 6 +++--- .../macros/materializations/snapshots/helpers.sql | 5 ++--- .../materializations/snapshots/snapshot_merge.sql | 4 ++-- dbt/include/global_project/macros/utils/equals.sql | 11 +++++++++++ 6 files changed, 19 insertions(+), 25 deletions(-) create mode 100644 dbt/include/global_project/macros/utils/equals.sql diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py b/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py index 23e1ca7fa..000b1cf90 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py @@ -1,16 +1,6 @@ import pytest from dbt.tests.util import run_dbt - -macros__equals_sql = """ -{% macro equals(expr1, expr2) -%} -case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) - then 0 - else 1 -end = 0 -{% endmacro %} -""" - macros__test_assert_equal_sql = """ {% test assert_equal(model, actual, expected) %} select * from {{ model }} diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py b/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py index c61f6fdff..d8596dc01 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py @@ -1,16 +1,10 @@ import pytest -from dbt.tests.adapter.utils import base_utils, fixture_equals +from dbt.tests.adapter.utils import fixture_equals from dbt.tests.util import relation_from_name, run_dbt class BaseEquals: - @pytest.fixture(scope="class") - def macros(self): - return { - "equals.sql": base_utils.macros__equals_sql, - } - @pytest.fixture(scope="class") def seeds(self): return { diff --git a/dbt/include/global_project/macros/materializations/models/incremental/merge.sql b/dbt/include/global_project/macros/materializations/models/incremental/merge.sql index ca972c9f2..2f9c2dee6 100644 --- a/dbt/include/global_project/macros/materializations/models/incremental/merge.sql +++ b/dbt/include/global_project/macros/materializations/models/incremental/merge.sql @@ -22,7 +22,7 @@ {% endfor %} {% else %} {% set unique_key_match %} - DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }} + {{ adapter.dispatch('equals', 'dbt')(DBT_INTERNAL_SOURCE.{{ unique_key }}, DBT_INTERNAL_DEST.{{ unique_key }}) }} {% endset %} {% do predicates.append(unique_key_match) %} {% endif %} @@ -62,11 +62,11 @@ {% if unique_key %} {% if unique_key is sequence and unique_key is not string %} - delete from {{target }} + delete from {{ target }} using {{ source }} where ( {% for key in unique_key %} - {{ source }}.{{ key }} = {{ target }}.{{ key }} + {{ adapter.dispatch('equals', 'dbt')({{ source }}.{{ key }}, {{ target }}.{{ key }}) }} {{ "and " if not loop.last}} {% endfor %} {% if incremental_predicates %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 33492cc95..f72e71e00 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -53,8 +53,7 @@ from {{ target_relation }} where {% if config.get('dbt_valid_to_current') %} - {# Check for either dbt_valid_to_current OR null, in order to correctly update records with nulls #} - ( {{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or {{ columns.dbt_valid_to }} is null) + {{ adapter.dispatch('equals', 'dbt')({{ columns.dbt_valid_to }}, {{ config.get('dbt_valid_to_current') }}) }} {% else %} {{ columns.dbt_valid_to }} is null {% endif %} @@ -276,7 +275,7 @@ {% macro unique_key_join_on(unique_key, identifier, from_identifier) %} {% if unique_key | is_list %} {% for key in unique_key %} - {{ identifier }}.dbt_unique_key_{{ loop.index }} = {{ from_identifier }}.dbt_unique_key_{{ loop.index }} + {{ adapter.dispatch('equals', 'dbt')({{ identifier }}.dbt_unique_key_{{ loop.index }}, {{ from_identifier }}.dbt_unique_key_{{ loop.index }}) }} {%- if not loop.last %} and {%- endif %} {% endfor %} {% else %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index cf787e4fb..2ecece096 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -15,8 +15,8 @@ when matched {% if config.get("dbt_valid_to_current") %} - and (DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or - DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null) + and {{ adapter.dispatch('equals', 'dbt')(DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }}, {{ config.get('dbt_valid_to_current') }}) }} + {% else %} and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null {% endif %} diff --git a/dbt/include/global_project/macros/utils/equals.sql b/dbt/include/global_project/macros/utils/equals.sql new file mode 100644 index 000000000..216cdcd86 --- /dev/null +++ b/dbt/include/global_project/macros/utils/equals.sql @@ -0,0 +1,11 @@ +{% macro equals(first_date, second_date, datepart) -%} + {{ return(adapter.dispatch('equals', 'dbt') (expression)) }} +{%- endmacro %} + +{% macro defaults__equals(expr1, expr2) -%} +case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) + then 0 + else 1 +end = 0 +{% endmacro %} + From 7e816bebc5965880ca88ca4b199f0b4afa918ef6 Mon Sep 17 00:00:00 2001 From: Adrian Burus Date: Tue, 17 Dec 2024 11:06:05 +0200 Subject: [PATCH 02/40] Changelog update --- .changes/unreleased/Under the Hood-20241217-110536.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20241217-110536.yaml diff --git a/.changes/unreleased/Under the Hood-20241217-110536.yaml b/.changes/unreleased/Under the Hood-20241217-110536.yaml new file mode 100644 index 000000000..5716da5e1 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241217-110536.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Added new equals macro that handles null value checks in sql +time: 2024-12-17T11:05:36.363421+02:00 +custom: + Author: adrianburusdbt + Issue: "159" From 70afd18a8e5f63652935914ef325a46471fcc8db Mon Sep 17 00:00:00 2001 From: Adrian Burus Date: Tue, 17 Dec 2024 11:28:00 +0200 Subject: [PATCH 03/40] Update equals.sql macro --- .../global_project/macros/utils/equals.sql | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/dbt/include/global_project/macros/utils/equals.sql b/dbt/include/global_project/macros/utils/equals.sql index 216cdcd86..adb75b927 100644 --- a/dbt/include/global_project/macros/utils/equals.sql +++ b/dbt/include/global_project/macros/utils/equals.sql @@ -1,11 +1,12 @@ -{% macro equals(first_date, second_date, datepart) -%} - {{ return(adapter.dispatch('equals', 'dbt') (expression)) }} +{% macro equals(first_date, second_date, datepart) %} + {{ return(adapter.dispatch('equals', 'dbt') (expr1, expr2)) }} {%- endmacro %} -{% macro defaults__equals(expr1, expr2) -%} -case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) - then 0 - else 1 -end = 0 -{% endmacro %} +{% macro default__equals(expr1, expr2) -%} + case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) + then 0 + else 1 + end = 0 + +{% endmacro %} \ No newline at end of file From b8312d6c02c9de2f842654bb07dad6437c3e6458 Mon Sep 17 00:00:00 2001 From: Adrian Burus Date: Tue, 17 Dec 2024 11:30:11 +0200 Subject: [PATCH 04/40] Remved unused test fixture --- dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py b/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py index 000b1cf90..943b2aa87 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py @@ -23,7 +23,6 @@ class BaseUtils: @pytest.fixture(scope="class") def macros(self): return { - "equals.sql": macros__equals_sql, "test_assert_equal.sql": macros__test_assert_equal_sql, "replace_empty.sql": macros__replace_empty_sql, } From 85c954f702ed3a0da1302e19300aef891e2a9a40 Mon Sep 17 00:00:00 2001 From: Adrian Burus Date: Tue, 17 Dec 2024 15:31:23 +0200 Subject: [PATCH 05/40] Attempt to fix end of file error --- dbt/include/global_project/macros/utils/equals.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/global_project/macros/utils/equals.sql b/dbt/include/global_project/macros/utils/equals.sql index adb75b927..2920b7b0c 100644 --- a/dbt/include/global_project/macros/utils/equals.sql +++ b/dbt/include/global_project/macros/utils/equals.sql @@ -9,4 +9,4 @@ else 1 end = 0 -{% endmacro %} \ No newline at end of file +{% endmacro %} From 7565deaf079898cbbb65328d69e70734e3b79ef1 Mon Sep 17 00:00:00 2001 From: Adrian Burus Date: Tue, 17 Dec 2024 15:39:41 +0200 Subject: [PATCH 06/40] Trim trailing whitespace --- .../macros/materializations/snapshots/snapshot_merge.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index 2ecece096..2d10d1536 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -16,7 +16,7 @@ when matched {% if config.get("dbt_valid_to_current") %} and {{ adapter.dispatch('equals', 'dbt')(DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }}, {{ config.get('dbt_valid_to_current') }}) }} - + {% else %} and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null {% endif %} From d582ab105f0d3db273071e4d46ea53a7a4e45669 Mon Sep 17 00:00:00 2001 From: Adrian Burus Date: Tue, 17 Dec 2024 20:14:37 +0200 Subject: [PATCH 07/40] PR comments --- .../macros/materializations/models/incremental/merge.sql | 4 ++-- .../macros/materializations/snapshots/helpers.sql | 4 ++-- .../macros/materializations/snapshots/snapshot_merge.sql | 2 +- dbt/include/global_project/macros/utils/equals.sql | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbt/include/global_project/macros/materializations/models/incremental/merge.sql b/dbt/include/global_project/macros/materializations/models/incremental/merge.sql index 2f9c2dee6..c59207162 100644 --- a/dbt/include/global_project/macros/materializations/models/incremental/merge.sql +++ b/dbt/include/global_project/macros/materializations/models/incremental/merge.sql @@ -22,7 +22,7 @@ {% endfor %} {% else %} {% set unique_key_match %} - {{ adapter.dispatch('equals', 'dbt')(DBT_INTERNAL_SOURCE.{{ unique_key }}, DBT_INTERNAL_DEST.{{ unique_key }}) }} + {{ equals(DBT_INTERNAL_SOURCE.{{ unique_key }}, DBT_INTERNAL_DEST.{{ unique_key }}) }} {% endset %} {% do predicates.append(unique_key_match) %} {% endif %} @@ -66,7 +66,7 @@ using {{ source }} where ( {% for key in unique_key %} - {{ adapter.dispatch('equals', 'dbt')({{ source }}.{{ key }}, {{ target }}.{{ key }}) }} + {{ equals({{ source }}.{{ key }}, {{ target }}.{{ key }}) }} {{ "and " if not loop.last}} {% endfor %} {% if incremental_predicates %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index f72e71e00..ff42fea49 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -53,7 +53,7 @@ from {{ target_relation }} where {% if config.get('dbt_valid_to_current') %} - {{ adapter.dispatch('equals', 'dbt')({{ columns.dbt_valid_to }}, {{ config.get('dbt_valid_to_current') }}) }} + {{ equals({{ columns.dbt_valid_to }}, {{ config.get('dbt_valid_to_current') }}) }} {% else %} {{ columns.dbt_valid_to }} is null {% endif %} @@ -275,7 +275,7 @@ {% macro unique_key_join_on(unique_key, identifier, from_identifier) %} {% if unique_key | is_list %} {% for key in unique_key %} - {{ adapter.dispatch('equals', 'dbt')({{ identifier }}.dbt_unique_key_{{ loop.index }}, {{ from_identifier }}.dbt_unique_key_{{ loop.index }}) }} + {{ equals({{ identifier }}.dbt_unique_key_{{ loop.index }}, {{ from_identifier }}.dbt_unique_key_{{ loop.index }}) }} {%- if not loop.last %} and {%- endif %} {% endfor %} {% else %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index 2d10d1536..cd2cdebd0 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -15,7 +15,7 @@ when matched {% if config.get("dbt_valid_to_current") %} - and {{ adapter.dispatch('equals', 'dbt')(DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }}, {{ config.get('dbt_valid_to_current') }}) }} + and {{ equals(DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }}, {{ config.get('dbt_valid_to_current') }}) }} {% else %} and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null diff --git a/dbt/include/global_project/macros/utils/equals.sql b/dbt/include/global_project/macros/utils/equals.sql index 2920b7b0c..d63b6cc17 100644 --- a/dbt/include/global_project/macros/utils/equals.sql +++ b/dbt/include/global_project/macros/utils/equals.sql @@ -1,4 +1,4 @@ -{% macro equals(first_date, second_date, datepart) %} +{% macro equals(expr1, expr2) %} {{ return(adapter.dispatch('equals', 'dbt') (expr1, expr2)) }} {%- endmacro %} From 8937201a1e81ec8b8776672c9290754aa80dbbec Mon Sep 17 00:00:00 2001 From: Adrian Burus Date: Tue, 17 Dec 2024 20:34:54 +0200 Subject: [PATCH 08/40] Avoid using nested {{ }} --- .../models/incremental/merge.sql | 17 +++++++++++++++-- .../materializations/snapshots/helpers.sql | 18 ++++++++++++++++-- .../snapshots/snapshot_merge.sql | 8 +++++++- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/dbt/include/global_project/macros/materializations/models/incremental/merge.sql b/dbt/include/global_project/macros/materializations/models/incremental/merge.sql index c59207162..d7e8af70c 100644 --- a/dbt/include/global_project/macros/materializations/models/incremental/merge.sql +++ b/dbt/include/global_project/macros/materializations/models/incremental/merge.sql @@ -21,8 +21,14 @@ {% do predicates.append(this_key_match) %} {% endfor %} {% else %} + {% set source_unique_key %} + DBT_INTERNAL_SOURCE.{{ unique_key }} + {% endset %} + {% set target_unique_key %} + DBT_INTERNAL_DEST.{{ unique_key }} + {% endset %} {% set unique_key_match %} - {{ equals(DBT_INTERNAL_SOURCE.{{ unique_key }}, DBT_INTERNAL_DEST.{{ unique_key }}) }} + {{ equals(source_unique_key, target_unique_key) }} {% endset %} {% do predicates.append(unique_key_match) %} {% endif %} @@ -66,7 +72,14 @@ using {{ source }} where ( {% for key in unique_key %} - {{ equals({{ source }}.{{ key }}, {{ target }}.{{ key }}) }} + {% set source_unique_key %} + {{ source }}.{{ key }} + {% endset %} + {% set target_unique_key %} + {{ target }}.{{ key }} + {% endset %} + + {{ equals(source_unique_key, target_unique_key) }} {{ "and " if not loop.last}} {% endfor %} {% if incremental_predicates %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index ff42fea49..905ab136e 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -53,7 +53,14 @@ from {{ target_relation }} where {% if config.get('dbt_valid_to_current') %} - {{ equals({{ columns.dbt_valid_to }}, {{ config.get('dbt_valid_to_current') }}) }} + {% set source_unique_key %} + columns.dbt_valid_to + {% endset %} + {% set target_unique_key %} + config.get('dbt_valid_to_current') + {% endset %} + + {{ equals(source_unique_key, target_unique_key) }} {% else %} {{ columns.dbt_valid_to }} is null {% endif %} @@ -275,7 +282,14 @@ {% macro unique_key_join_on(unique_key, identifier, from_identifier) %} {% if unique_key | is_list %} {% for key in unique_key %} - {{ equals({{ identifier }}.dbt_unique_key_{{ loop.index }}, {{ from_identifier }}.dbt_unique_key_{{ loop.index }}) }} + {% set source_unique_key %} + {{ identifier }}.dbt_unique_key_{{ loop.index }} + {% endset %} + {% set target_unique_key %} + {{ from_identifier }}.dbt_unique_key_{{ loop.index }} + {% endset %} + + {{ equals(source_unique_key, target_unique_key) }} {%- if not loop.last %} and {%- endif %} {% endfor %} {% else %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index cd2cdebd0..19a67f6b7 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -15,7 +15,13 @@ when matched {% if config.get("dbt_valid_to_current") %} - and {{ equals(DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }}, {{ config.get('dbt_valid_to_current') }}) }} + {% set source_unique_key %} + DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} + {% endset %} + {% set target_unique_key %} + {{ config.get('dbt_valid_to_current') }} + {% endset %} + and {{ equals(source_unique_key, target_unique_key) }} {% else %} and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null From 087e64de8eb1283ed0ae842a34169073db17ff7d Mon Sep 17 00:00:00 2001 From: Adrian Burus Date: Wed, 18 Dec 2024 14:18:50 +0200 Subject: [PATCH 09/40] Update test seeds / match counts --- .../incremental/test_incremental_unique_id.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_unique_id.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_unique_id.py index bddf407ed..34807062a 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_unique_id.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_unique_id.py @@ -240,6 +240,8 @@ select 'NY','New York','Manhattan','2021-04-01' union all select 'PA','Philadelphia','Philadelphia','2021-05-21' +union all +select 'CO','Denver',null,'2021-06-18' """ @@ -265,6 +267,8 @@ select 'NY','New York','Manhattan','2021-04-01' union all select 'PA','Philadelphia','Philadelphia','2021-05-21' +union all +select 'CO','Denver',null,'2021-06-18' """ @@ -288,6 +292,7 @@ NY,Kings,Brooklyn,2021-04-02 NY,New York,Manhattan,2021-04-01 PA,Philadelphia,Philadelphia,2021-05-21 +CO,Denver,,2021-06-18 """ seeds__add_new_rows_sql = """ @@ -439,7 +444,7 @@ def fail_to_build_inc_missing_unique_key_column(self, incremental_model_name): def test__no_unique_keys(self, project): """with no unique keys, seed and model should match""" - expected_fields = self.get_expected_fields(relation="seed", seed_rows=8) + expected_fields = self.get_expected_fields(relation="seed", seed_rows=9) test_case_fields = self.get_test_fields( project, seed="seed", incremental_model="no_unique_key", update_sql_file="add_new_rows" ) @@ -449,7 +454,7 @@ def test__no_unique_keys(self, project): def test__empty_str_unique_key(self, project): """with empty string for unique key, seed and model should match""" - expected_fields = self.get_expected_fields(relation="seed", seed_rows=8) + expected_fields = self.get_expected_fields(relation="seed", seed_rows=9) test_case_fields = self.get_test_fields( project, seed="seed", @@ -462,7 +467,7 @@ def test__one_unique_key(self, project): """with one unique key, model will overwrite existing row""" expected_fields = self.get_expected_fields( - relation="one_str__overwrite", seed_rows=7, opt_model_count=1 + relation="one_str__overwrite", seed_rows=8, opt_model_count=1 ) test_case_fields = self.get_test_fields( project, @@ -487,7 +492,7 @@ def test__bad_unique_key(self, project): def test__empty_unique_key_list(self, project): """with no unique keys, seed and model should match""" - expected_fields = self.get_expected_fields(relation="seed", seed_rows=8) + expected_fields = self.get_expected_fields(relation="seed", seed_rows=9) test_case_fields = self.get_test_fields( project, seed="seed", @@ -500,7 +505,7 @@ def test__unary_unique_key_list(self, project): """with one unique key, model will overwrite existing row""" expected_fields = self.get_expected_fields( - relation="unique_key_list__inplace_overwrite", seed_rows=7, opt_model_count=1 + relation="unique_key_list__inplace_overwrite", seed_rows=8, opt_model_count=1 ) test_case_fields = self.get_test_fields( project, @@ -515,7 +520,7 @@ def test__duplicated_unary_unique_key_list(self, project): """with two of the same unique key, model will overwrite existing row""" expected_fields = self.get_expected_fields( - relation="unique_key_list__inplace_overwrite", seed_rows=7, opt_model_count=1 + relation="unique_key_list__inplace_overwrite", seed_rows=8, opt_model_count=1 ) test_case_fields = self.get_test_fields( project, @@ -530,7 +535,7 @@ def test__trinary_unique_key_list(self, project): """with three unique keys, model will overwrite existing row""" expected_fields = self.get_expected_fields( - relation="unique_key_list__inplace_overwrite", seed_rows=7, opt_model_count=1 + relation="unique_key_list__inplace_overwrite", seed_rows=8, opt_model_count=1 ) test_case_fields = self.get_test_fields( project, @@ -545,7 +550,7 @@ def test__trinary_unique_key_list_no_update(self, project): """even with three unique keys, adding distinct rows to seed does not cause seed and model to diverge""" - expected_fields = self.get_expected_fields(relation="seed", seed_rows=8) + expected_fields = self.get_expected_fields(relation="seed", seed_rows=9) test_case_fields = self.get_test_fields( project, seed="seed", From e8843fba775704465097bb92b29a272f2a9d56d9 Mon Sep 17 00:00:00 2001 From: Adrian Burus Date: Thu, 19 Dec 2024 13:38:03 +0200 Subject: [PATCH 10/40] Add back macros__equals_sql test fixture --- .../dbt/tests/adapter/utils/base_utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py b/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py index 943b2aa87..15c042ec2 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py @@ -17,12 +17,22 @@ {% endmacro %} """ +macros__equals_sql = """ +{% macro equals(expr1, expr2) -%} +case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) + then 0 + else 1 +end = 0 +{% endmacro %} +""" + class BaseUtils: # setup @pytest.fixture(scope="class") def macros(self): return { + "equals.sql": macros__equals_sql, "test_assert_equal.sql": macros__test_assert_equal_sql, "replace_empty.sql": macros__replace_empty_sql, } From fc448bebacdd0883dc973b46cdef425ada1bd6b6 Mon Sep 17 00:00:00 2001 From: Adrian Burus Date: Wed, 8 Jan 2025 12:37:29 +0200 Subject: [PATCH 11/40] Add reference to test fixture for equals from adapter utils --- dbt-tests-adapter/dbt/tests/adapter/utils/fixture_equals.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/fixture_equals.py b/dbt-tests-adapter/dbt/tests/adapter/utils/fixture_equals.py index 67a6699c6..fc620becf 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/utils/fixture_equals.py +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/fixture_equals.py @@ -1,3 +1,5 @@ +from dbt.include.global_project.macros.utils import equals + # equals SEEDS__DATA_EQUALS_CSV = """key_name,x,y,expected From 11283d95698bdcae1c1256c358566b3a987cd86f Mon Sep 17 00:00:00 2001 From: Adrian Burus Date: Wed, 8 Jan 2025 12:43:32 +0200 Subject: [PATCH 12/40] Add equals as an include in test_equals.py --- dbt-tests-adapter/dbt/tests/adapter/utils/fixture_equals.py | 2 -- dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/fixture_equals.py b/dbt-tests-adapter/dbt/tests/adapter/utils/fixture_equals.py index fc620becf..67a6699c6 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/utils/fixture_equals.py +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/fixture_equals.py @@ -1,5 +1,3 @@ -from dbt.include.global_project.macros.utils import equals - # equals SEEDS__DATA_EQUALS_CSV = """key_name,x,y,expected diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py b/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py index d8596dc01..b127942e2 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py @@ -2,6 +2,7 @@ from dbt.tests.adapter.utils import fixture_equals from dbt.tests.util import relation_from_name, run_dbt +from dbt.include.global_project.macros.utils import equals class BaseEquals: From 9c2104cc3ee0888d3617d985ec1877f6b5d6d37b Mon Sep 17 00:00:00 2001 From: Adrian Burus Date: Wed, 8 Jan 2025 12:55:35 +0200 Subject: [PATCH 13/40] Added base_utils ref --- dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py b/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py index b127942e2..77a9c6f8c 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py @@ -1,11 +1,9 @@ import pytest -from dbt.tests.adapter.utils import fixture_equals +from dbt.tests.adapter.utils import base_utils, fixture_equals from dbt.tests.util import relation_from_name, run_dbt -from dbt.include.global_project.macros.utils import equals - -class BaseEquals: +class BaseEquals(base_utils.BaseUtils): @pytest.fixture(scope="class") def seeds(self): return { From 16169fc80877a7b2136fd587c8ed88c1987218c8 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 27 Jan 2025 00:13:40 -0800 Subject: [PATCH 14/40] TEMPORARY test refs to this branch. --- dbt-athena/hatch.toml | 5 ++++- dbt-bigquery/hatch.toml | 5 ++++- dbt-postgres/hatch.toml | 5 ++++- dbt-redshift/hatch.toml | 5 ++++- dbt-snowflake/hatch.toml | 5 ++++- dbt-spark/hatch.toml | 5 ++++- 6 files changed, 24 insertions(+), 6 deletions(-) diff --git a/dbt-athena/hatch.toml b/dbt-athena/hatch.toml index 8f0e1263b..e97b8a0ba 100644 --- a/dbt-athena/hatch.toml +++ b/dbt-athena/hatch.toml @@ -11,7 +11,7 @@ sources = ["src"] [envs.default] dependencies = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", @@ -55,3 +55,6 @@ check-sdist = [ "find ./dist/dbt_athena-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", "pip freeze | grep dbt-athena", ] + +[metadata] +allow-direct-references = true diff --git a/dbt-bigquery/hatch.toml b/dbt-bigquery/hatch.toml index eb972b66a..83e6389a2 100644 --- a/dbt-bigquery/hatch.toml +++ b/dbt-bigquery/hatch.toml @@ -12,7 +12,7 @@ sources = ["src"] [envs.default] python = "3.9" dependencies = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", @@ -62,3 +62,6 @@ check-sdist = [ "pip freeze | grep dbt-bigquery", ] docker-prod = "docker build -f docker/Dockerfile -t dbt-bigquery ." + +[metadata] +allow-ambiguous-features = true diff --git a/dbt-postgres/hatch.toml b/dbt-postgres/hatch.toml index 2e76633c9..352aa209c 100644 --- a/dbt-postgres/hatch.toml +++ b/dbt-postgres/hatch.toml @@ -11,7 +11,7 @@ sources = ["src"] [envs.default] dependencies = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", @@ -61,3 +61,6 @@ check-sdist = [ "find ./dist/dbt_postgres-*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/", "pip freeze | grep dbt-postgres", ] + +[metadata] +allow-direct-references = true diff --git a/dbt-redshift/hatch.toml b/dbt-redshift/hatch.toml index 0d78b8a1d..fc7691c57 100644 --- a/dbt-redshift/hatch.toml +++ b/dbt-redshift/hatch.toml @@ -11,7 +11,7 @@ sources = ["src"] [envs.default] dependencies = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", @@ -62,3 +62,6 @@ check-sdist = [ "pip freeze | grep dbt-redshift", ] docker-prod = "docker build -f docker/Dockerfile -t dbt-redshift ." + +[metadata] +allow-direct-references = true diff --git a/dbt-snowflake/hatch.toml b/dbt-snowflake/hatch.toml index 7a2f1e272..c83caa066 100644 --- a/dbt-snowflake/hatch.toml +++ b/dbt-snowflake/hatch.toml @@ -11,7 +11,7 @@ sources = ["src"] [envs.default] dependencies = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", @@ -60,3 +60,6 @@ check-sdist = [ "pip freeze | grep dbt-snowflake", ] docker-prod = "docker build -f docker/Dockerfile -t dbt-snowflake ." + +[metadata] +allow-direct-references = true diff --git a/dbt-spark/hatch.toml b/dbt-spark/hatch.toml index 638bba83c..010bafb58 100644 --- a/dbt-spark/hatch.toml +++ b/dbt-spark/hatch.toml @@ -11,7 +11,7 @@ sources = ["src"] [envs.default] dependencies = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", @@ -59,3 +59,6 @@ check-sdist = [ "pip freeze | grep dbt-spark", ] docker-prod = "docker build -f docker/Dockerfile -t dbt-spark ." + +[metadata] +allow-direct-references = true From 27602a3735bc01a1905dda1dc35f6e3268d065e6 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 27 Jan 2025 00:27:58 -0800 Subject: [PATCH 15/40] reformat --- dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py b/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py index 77a9c6f8c..09dfe934e 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/test_equals.py @@ -3,6 +3,7 @@ from dbt.tests.adapter.utils import base_utils, fixture_equals from dbt.tests.util import relation_from_name, run_dbt + class BaseEquals(base_utils.BaseUtils): @pytest.fixture(scope="class") def seeds(self): From e77f0a2e9a0216931face5383611dbae37c5b233 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 27 Jan 2025 02:10:32 -0800 Subject: [PATCH 16/40] TEMPORARY Move refs of branches. --- dbt-athena/pyproject.toml | 5 ++++- dbt-bigquery/pyproject.toml | 5 ++++- dbt-postgres/pyproject.toml | 5 ++++- dbt-redshift/pyproject.toml | 5 ++++- dbt-snowflake/pyproject.toml | 5 ++++- dbt-spark/pyproject.toml | 5 ++++- 6 files changed, 24 insertions(+), 6 deletions(-) diff --git a/dbt-athena/pyproject.toml b/dbt-athena/pyproject.toml index 69ec0018b..5ed2d6bca 100644 --- a/dbt-athena/pyproject.toml +++ b/dbt-athena/pyproject.toml @@ -27,7 +27,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies=[ - "dbt-adapters>=1.0.0,<2.0", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", "dbt-common>=1.0.0,<2.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", @@ -53,3 +53,6 @@ filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", ] + +[tool.hatch.metadata] +allow-direct-references = true diff --git a/dbt-bigquery/pyproject.toml b/dbt-bigquery/pyproject.toml index b2d55b25f..b41a6b5a4 100644 --- a/dbt-bigquery/pyproject.toml +++ b/dbt-bigquery/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ ] dependencies = [ "dbt-common>=1.10,<2.0", - "dbt-adapters>=1.7,<2.0", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", # 3.20 introduced pyarrow>=3.0 under the `pandas` extra "google-cloud-bigquery[pandas]>=3.0,<4.0", "google-cloud-storage~=2.4", @@ -55,3 +55,6 @@ filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", ] + +[tool.hatch.metadata] +allow-direct-references = true diff --git a/dbt-postgres/pyproject.toml b/dbt-postgres/pyproject.toml index 6f4b4604a..392fccc60 100644 --- a/dbt-postgres/pyproject.toml +++ b/dbt-postgres/pyproject.toml @@ -28,7 +28,7 @@ classifiers = [ ] dependencies = [ "psycopg2-binary>=2.9,<3.0", - "dbt-adapters>=1.7.0,<2.0", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", # installed via dbt-adapters but used directly @@ -47,3 +47,6 @@ testpaths = [ "tests/functional", "tests/unit", ] + +[tool.hatch.metadata] +allow-direct-references = true diff --git a/dbt-redshift/pyproject.toml b/dbt-redshift/pyproject.toml index 77f86fd33..4fc4b841d 100644 --- a/dbt-redshift/pyproject.toml +++ b/dbt-redshift/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ ] dependencies = [ "dbt-common>=1.10,<2.0", - "dbt-adapters>=1.11,<2.0", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", "dbt-postgres>=1.8,<1.10", # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. @@ -55,3 +55,6 @@ filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", ] + +[tool.hatch.metadata] +allow-direct-references = true diff --git a/dbt-snowflake/pyproject.toml b/dbt-snowflake/pyproject.toml index 568aa3533..a7c209c56 100644 --- a/dbt-snowflake/pyproject.toml +++ b/dbt-snowflake/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ ] dependencies = [ "dbt-common>=1.10,<2.0", - "dbt-adapters>=1.10.4,<2.0", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", "snowflake-connector-python[secure-local-storage]>=3.0.0,<3.12.4", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", @@ -46,3 +46,6 @@ addopts = "-v --color=yes -n auto" filterwarnings = [ "ignore:datetime.datetime.utcnow:DeprecationWarning", ] + +[tool.hatch.metadata] +allow-direct-references = true diff --git a/dbt-spark/pyproject.toml b/dbt-spark/pyproject.toml index de27f9cb2..9a38ddf15 100644 --- a/dbt-spark/pyproject.toml +++ b/dbt-spark/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ ] dependencies = [ "dbt-common>=1.10,<2.0", - "dbt-adapters>=1.7,<2.0", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", "sqlparams>=3.0.0", @@ -58,3 +58,6 @@ filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", ] + +[tool.hatch.metadata] +allow-direct-references = true From 5911260ce5510d558f9c790df465a1efb1c02442 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 27 Jan 2025 02:18:54 -0800 Subject: [PATCH 17/40] Fix git diff --- .../dbt/tests/adapter/utils/base_utils.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py b/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py index 15c042ec2..622b4ab42 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py @@ -1,6 +1,15 @@ import pytest from dbt.tests.util import run_dbt +macros__equals_sql = """ +{% macro equals(expr1, expr2) -%} +case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) + then 0 + else 1 +end = 0 +{% endmacro %} +""" + macros__test_assert_equal_sql = """ {% test assert_equal(model, actual, expected) %} select * from {{ model }} @@ -17,15 +26,6 @@ {% endmacro %} """ -macros__equals_sql = """ -{% macro equals(expr1, expr2) -%} -case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) - then 0 - else 1 -end = 0 -{% endmacro %} -""" - class BaseUtils: # setup From 3cb311733468f5a52c50043d6fea997f17a78fc5 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 27 Jan 2025 02:24:15 -0800 Subject: [PATCH 18/40] Fix ws --- dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py b/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py index 622b4ab42..23e1ca7fa 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py @@ -1,6 +1,7 @@ import pytest from dbt.tests.util import run_dbt + macros__equals_sql = """ {% macro equals(expr1, expr2) -%} case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) From e9e791ed01c3b8fbd748531b6864468a79063a69 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 27 Jan 2025 02:30:55 -0800 Subject: [PATCH 19/40] Fix bad metadata directive --- dbt-bigquery/hatch.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt-bigquery/hatch.toml b/dbt-bigquery/hatch.toml index 83e6389a2..7e15021b7 100644 --- a/dbt-bigquery/hatch.toml +++ b/dbt-bigquery/hatch.toml @@ -64,4 +64,4 @@ check-sdist = [ docker-prod = "docker build -f docker/Dockerfile -t dbt-bigquery ." [metadata] -allow-ambiguous-features = true +allow-direct-references = true From acf3b4db2f7bbde6d05c7ddb6b11beb641895f6d Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 27 Jan 2025 17:24:24 -0800 Subject: [PATCH 20/40] Add behavior flag for the null equality macro. --- .../incremental/test_incremental_unique_id.py | 119 +++++++++++------- .../dbt/tests/adapter/utils/base_utils.py | 14 ++- dbt/adapters/base/impl.py | 9 +- .../global_project/macros/utils/equals.sql | 12 +- 4 files changed, 99 insertions(+), 55 deletions(-) diff --git a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_unique_id.py b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_unique_id.py index 34807062a..f803ca36e 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_unique_id.py +++ b/dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_unique_id.py @@ -326,25 +326,6 @@ class BaseIncrementalUniqueKey: - @pytest.fixture(scope="class") - def models(self): - return { - "trinary_unique_key_list.sql": models__trinary_unique_key_list_sql, - "nontyped_trinary_unique_key_list.sql": models__nontyped_trinary_unique_key_list_sql, - "unary_unique_key_list.sql": models__unary_unique_key_list_sql, - "not_found_unique_key.sql": models__not_found_unique_key_sql, - "empty_unique_key_list.sql": models__empty_unique_key_list_sql, - "no_unique_key.sql": models__no_unique_key_sql, - "empty_str_unique_key.sql": models__empty_str_unique_key_sql, - "str_unique_key.sql": models__str_unique_key_sql, - "duplicated_unary_unique_key_list.sql": models__duplicated_unary_unique_key_list_sql, - "not_found_unique_key_list.sql": models__not_found_unique_key_list_sql, - "expected": { - "one_str__overwrite.sql": models__expected__one_str__overwrite_sql, - "unique_key_list__inplace_overwrite.sql": models__expected__unique_key_list__inplace_overwrite_sql, - }, - } - @pytest.fixture(scope="class") def seeds(self): return { @@ -440,25 +421,46 @@ def fail_to_build_inc_missing_unique_key_column(self, incremental_model_name): return run_result.status, run_result.message - # no unique_key test - def test__no_unique_keys(self, project): - """with no unique keys, seed and model should match""" - expected_fields = self.get_expected_fields(relation="seed", seed_rows=9) - test_case_fields = self.get_test_fields( - project, seed="seed", incremental_model="no_unique_key", update_sql_file="add_new_rows" +class IncrementalUniqueKeyFalseyNullsEquals(BaseIncrementalUniqueKey): + @pytest.fixture(scope="class") + def models(self): + return { + "trinary_unique_key_list.sql": models__trinary_unique_key_list_sql, + "nontyped_trinary_unique_key_list.sql": models__nontyped_trinary_unique_key_list_sql, + "unary_unique_key_list.sql": models__unary_unique_key_list_sql, + "not_found_unique_key.sql": models__not_found_unique_key_sql, + "empty_unique_key_list.sql": models__empty_unique_key_list_sql, + "no_unique_key.sql": models__no_unique_key_sql, + "empty_str_unique_key.sql": models__empty_str_unique_key_sql, + "str_unique_key.sql": models__str_unique_key_sql, + "duplicated_unary_unique_key_list.sql": models__duplicated_unary_unique_key_list_sql, + "not_found_unique_key_list.sql": models__not_found_unique_key_list_sql, + "expected": { + "one_str__overwrite.sql": models__expected__one_str__overwrite_sql, + "unique_key_list__inplace_overwrite.sql": models__expected__unique_key_list__inplace_overwrite_sql, + }, + } + + def test__bad_unique_key(self, project): + """expect compilation error from unique key not being a column""" + + (status, exc) = self.fail_to_build_inc_missing_unique_key_column( + incremental_model_name="not_found_unique_key" ) - self.check_scenario_correctness(expected_fields, test_case_fields, project) - # unique_key as str tests - def test__empty_str_unique_key(self, project): - """with empty string for unique key, seed and model should match""" + assert status == RunStatus.Error + assert "thisisnotacolumn" in exc.lower() + + # test unique_key as list + def test__empty_unique_key_list(self, project): + """with no unique keys, seed and model should match""" expected_fields = self.get_expected_fields(relation="seed", seed_rows=9) test_case_fields = self.get_test_fields( project, seed="seed", - incremental_model="empty_str_unique_key", + incremental_model="empty_unique_key_list", update_sql_file="add_new_rows", ) self.check_scenario_correctness(expected_fields, test_case_fields, project) @@ -478,25 +480,60 @@ def test__one_unique_key(self, project): ) self.check_scenario_correctness(expected_fields, test_case_fields, project) - def test__bad_unique_key(self, project): + def test__bad_unique_key_list(self, project): """expect compilation error from unique key not being a column""" (status, exc) = self.fail_to_build_inc_missing_unique_key_column( - incremental_model_name="not_found_unique_key" + incremental_model_name="not_found_unique_key_list" ) assert status == RunStatus.Error assert "thisisnotacolumn" in exc.lower() - # test unique_key as list - def test__empty_unique_key_list(self, project): + +class IncrementalUniqueKeyTruthyNullsEquals(BaseIncrementalUniqueKey): + @pytest.fixture(scope="class") + def models(self): + return { + "trinary_unique_key_list.sql": models__trinary_unique_key_list_sql, + "nontyped_trinary_unique_key_list.sql": models__nontyped_trinary_unique_key_list_sql, + "unary_unique_key_list.sql": models__unary_unique_key_list_sql, + "not_found_unique_key.sql": models__not_found_unique_key_sql, + "empty_unique_key_list.sql": models__empty_unique_key_list_sql, + "no_unique_key.sql": models__no_unique_key_sql, + "empty_str_unique_key.sql": models__empty_str_unique_key_sql, + "str_unique_key.sql": models__str_unique_key_sql, + "duplicated_unary_unique_key_list.sql": models__duplicated_unary_unique_key_list_sql, + "not_found_unique_key_list.sql": models__not_found_unique_key_list_sql, + "expected": { + "one_str__overwrite.sql": models__expected__one_str__overwrite_sql, + "unique_key_list__inplace_overwrite.sql": models__expected__unique_key_list__inplace_overwrite_sql, + }, + } + + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_truthy_nulls_equals_macro": True}} + + # no unique_key test + def test__no_unique_keys(self, project): """with no unique keys, seed and model should match""" + expected_fields = self.get_expected_fields(relation="seed", seed_rows=9) + test_case_fields = self.get_test_fields( + project, seed="seed", incremental_model="no_unique_key", update_sql_file="add_new_rows" + ) + self.check_scenario_correctness(expected_fields, test_case_fields, project) + + # unique_key as str tests + def test__empty_str_unique_key(self, project): + """with empty string for unique key, seed and model should match""" + expected_fields = self.get_expected_fields(relation="seed", seed_rows=9) test_case_fields = self.get_test_fields( project, seed="seed", - incremental_model="empty_unique_key_list", + incremental_model="empty_str_unique_key", update_sql_file="add_new_rows", ) self.check_scenario_correctness(expected_fields, test_case_fields, project) @@ -559,16 +596,10 @@ def test__trinary_unique_key_list_no_update(self, project): ) self.check_scenario_correctness(expected_fields, test_case_fields, project) - def test__bad_unique_key_list(self, project): - """expect compilation error from unique key not being a column""" - - (status, exc) = self.fail_to_build_inc_missing_unique_key_column( - incremental_model_name="not_found_unique_key_list" - ) - assert status == RunStatus.Error - assert "thisisnotacolumn" in exc.lower() +class TestIncrementalUniqueKeyFalseyNullsEquals(IncrementalUniqueKeyFalseyNullsEquals): + pass -class TestIncrementalUniqueKey(BaseIncrementalUniqueKey): +class TestIncrementalUniqueKeyTruthyNullsEquals(IncrementalUniqueKeyTruthyNullsEquals): pass diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py b/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py index 23e1ca7fa..bb41e7111 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py @@ -3,12 +3,14 @@ macros__equals_sql = """ -{% macro equals(expr1, expr2) -%} -case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) - then 0 - else 1 -end = 0 -{% endmacro %} +{%- if adapter.behavior.enable_truthy_nulls_equals_macro.no_warn %} + case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) +{%- else -%} + case when (({{ expr1 }} = {{ expr2 }}) +{%- endif %} + then 0 + else 1 + end = 0 """ macros__test_assert_equal_sql = """ diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index 8474b39d8..c325966d3 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -104,7 +104,14 @@ "name": "require_batched_execution_for_custom_microbatch_strategy", "default": False, "docs_url": "https://docs.getdbt.com/docs/build/incremental-microbatch", - } + }, + { + { + "name": "enable_truthy_nulls_equals_macro", + "default": False, + "docs_url": "", + }, + }, ] diff --git a/dbt/include/global_project/macros/utils/equals.sql b/dbt/include/global_project/macros/utils/equals.sql index d63b6cc17..171fc9082 100644 --- a/dbt/include/global_project/macros/utils/equals.sql +++ b/dbt/include/global_project/macros/utils/equals.sql @@ -4,9 +4,13 @@ {% macro default__equals(expr1, expr2) -%} - case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) - then 0 - else 1 - end = 0 + {%- if adapter.behavior.enable_truthy_nulls_equals_macro.no_warn %} + case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) + {%- else -%} + case when (({{ expr1 }} = {{ expr2 }}) + {%- endif %} + then 0 + else 1 + end = 0 {% endmacro %} From 548f0867640b6691c8fadba3bdb5cba94b547f8a Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 27 Jan 2025 17:34:28 -0800 Subject: [PATCH 21/40] Fix behavior flag. --- dbt/adapters/base/impl.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index c325966d3..9bd9358fd 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -106,11 +106,9 @@ "docs_url": "https://docs.getdbt.com/docs/build/incremental-microbatch", }, { - { - "name": "enable_truthy_nulls_equals_macro", - "default": False, - "docs_url": "", - }, + "name": "enable_truthy_nulls_equals_macro", + "default": False, + "docs_url": "", }, ] From f3e0620be7c4dd07ee8dcedf04fd313d007ae5f5 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 27 Jan 2025 18:36:10 -0800 Subject: [PATCH 22/40] Keep moving forward on fixing macros here. --- .../incremental/test_incremental_unique_id.py | 591 +++++++++++++++++- .../dbt/tests/adapter/utils/base_utils.py | 2 +- .../global_project/macros/utils/equals.sql | 2 +- 3 files changed, 591 insertions(+), 4 deletions(-) diff --git a/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_unique_id.py b/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_unique_id.py index a061adfb5..17b2a502d 100644 --- a/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_unique_id.py +++ b/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_unique_id.py @@ -1,5 +1,592 @@ -from dbt.tests.adapter.incremental.test_incremental_unique_id import BaseIncrementalUniqueKey +# from dbt.tests.adapter.incremental.test_incremental_unique_id import BaseIncrementalUniqueKey +# class TestUniqueKeyBigQuery(BaseIncrementalUniqueKey): +# pass -class TestUniqueKeyBigQuery(BaseIncrementalUniqueKey): +from collections import namedtuple +from pathlib import Path + +# TODO: repoint to dbt-artifacts when it's available +from dbt.artifacts.schemas.results import RunStatus +import pytest + +from dbt.tests.util import check_relations_equal, run_dbt + + +models__trinary_unique_key_list_sql = """ +-- a multi-argument unique key list should see overwriting on rows in the model +-- where all unique key fields apply + +{{ + config( + materialized='incremental', + unique_key=['state', 'county', 'city'] + ) +}} + +select + state as state, + county as county, + city as city, + last_visit_date as last_visit_date +from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} + +""" + +models__nontyped_trinary_unique_key_list_sql = """ +-- a multi-argument unique key list should see overwriting on rows in the model +-- where all unique key fields apply +-- N.B. needed for direct comparison with seed + +{{ + config( + materialized='incremental', + unique_key=['state', 'county', 'city'] + ) +}} + +select + state as state, + county as county, + city as city, + last_visit_date as last_visit_date +from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} + +""" + +models__unary_unique_key_list_sql = """ +-- a one argument unique key list should result in overwritting semantics for +-- that one matching field + +{{ + config( + materialized='incremental', + unique_key=['state'] + ) +}} + +select + state as state, + county as county, + city as city, + last_visit_date as last_visit_date +from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} + +""" + +models__not_found_unique_key_sql = """ +-- a model with a unique key not found in the table itself will error out + +{{ + config( + materialized='incremental', + unique_key='thisisnotacolumn' + ) +}} + +select + * +from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} + +""" + +models__empty_unique_key_list_sql = """ +-- model with empty list unique key should build normally + +{{ + config( + materialized='incremental', + unique_key=[] + ) +}} + +select * from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} + +""" + +models__no_unique_key_sql = """ +-- no specified unique key should cause no special build behavior + +{{ + config( + materialized='incremental' + ) +}} + +select + * +from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} + +""" + +models__empty_str_unique_key_sql = """ +-- ensure model with empty string unique key should build normally + +{{ + config( + materialized='incremental', + unique_key='' + ) +}} + +select + * +from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} + +""" + +models__str_unique_key_sql = """ +-- a unique key with a string should trigger to overwrite behavior when +-- the source has entries in conflict (i.e. more than one row per unique key +-- combination) + +{{ + config( + materialized='incremental', + unique_key='state' + ) +}} + +select + state as state, + county as county, + city as city, + last_visit_date as last_visit_date +from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} + +""" + +models__duplicated_unary_unique_key_list_sql = """ +{{ + config( + materialized='incremental', + unique_key=['state', 'state'] + ) +}} + +select + state as state, + county as county, + city as city, + last_visit_date as last_visit_date +from {{ ref('seed') }} + +{% if is_incremental() %} + where last_visit_date > (select max(last_visit_date) from {{ this }}) +{% endif %} + +""" + +models__not_found_unique_key_list_sql = """ +-- a unique key list with any element not in the model itself should error out + +{{ + config( + materialized='incremental', + unique_key=['state', 'thisisnotacolumn'] + ) +}} + +select * from {{ ref('seed') }} + +""" + +models__expected__one_str__overwrite_sql = """ +{{ + config( + materialized='table' + ) +}} + +select + 'CT' as state, + 'Hartford' as county, + 'Hartford' as city, + cast('2022-02-14' as date) as last_visit_date +union all +select 'MA','Suffolk','Boston','2020-02-12' +union all +select 'NJ','Mercer','Trenton','2022-01-01' +union all +select 'NY','Kings','Brooklyn','2021-04-02' +union all +select 'NY','New York','Manhattan','2021-04-01' +union all +select 'PA','Philadelphia','Philadelphia','2021-05-21' +union all +select 'CO','Denver',null,'2021-06-18' + +""" + +models__expected__unique_key_list__inplace_overwrite_sql = """ +{{ + config( + materialized='table' + ) +}} + +select + 'CT' as state, + 'Hartford' as county, + 'Hartford' as city, + cast('2022-02-14' as date) as last_visit_date +union all +select 'MA','Suffolk','Boston','2020-02-12' +union all +select 'NJ','Mercer','Trenton','2022-01-01' +union all +select 'NY','Kings','Brooklyn','2021-04-02' +union all +select 'NY','New York','Manhattan','2021-04-01' +union all +select 'PA','Philadelphia','Philadelphia','2021-05-21' +union all +select 'CO','Denver',null,'2021-06-18' + +""" + +seeds__duplicate_insert_sql = """ +-- Insert statement which when applied to seed.csv triggers the inplace +-- overwrite strategy of incremental models. Seed and incremental model +-- diverge. + +-- insert new row, which should not be in incremental model +-- with primary or first three columns unique +insert into {schema}.seed + (state, county, city, last_visit_date) +values ('CT','Hartford','Hartford','2022-02-14'); + +""" + +seeds__seed_csv = """state,county,city,last_visit_date +CT,Hartford,Hartford,2020-09-23 +MA,Suffolk,Boston,2020-02-12 +NJ,Mercer,Trenton,2022-01-01 +NY,Kings,Brooklyn,2021-04-02 +NY,New York,Manhattan,2021-04-01 +PA,Philadelphia,Philadelphia,2021-05-21 +CO,Denver,,2021-06-18 +""" + +seeds__add_new_rows_sql = """ +-- Insert statement which when applied to seed.csv sees incremental model +-- grow in size while not (necessarily) diverging from the seed itself. + +-- insert two new rows, both of which should be in incremental model +-- with any unique columns +insert into {schema}.seed + (state, county, city, last_visit_date) +values ('WA','King','Seattle','2022-02-01'); + +insert into {schema}.seed + (state, county, city, last_visit_date) +values ('CA','Los Angeles','Los Angeles','2022-02-01'); + +""" + + +ResultHolder = namedtuple( + "ResultHolder", + [ + "seed_count", + "model_count", + "seed_rows", + "inc_test_model_count", + "opt_model_count", + "relation", + ], +) + + +class BaseIncrementalUniqueKey: + @pytest.fixture(scope="class") + def seeds(self): + return { + "duplicate_insert.sql": seeds__duplicate_insert_sql, + "seed.csv": seeds__seed_csv, + "add_new_rows.sql": seeds__add_new_rows_sql, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "trinary_unique_key_list.sql": models__trinary_unique_key_list_sql, + "nontyped_trinary_unique_key_list.sql": models__nontyped_trinary_unique_key_list_sql, + "unary_unique_key_list.sql": models__unary_unique_key_list_sql, + "not_found_unique_key.sql": models__not_found_unique_key_sql, + "empty_unique_key_list.sql": models__empty_unique_key_list_sql, + "no_unique_key.sql": models__no_unique_key_sql, + "empty_str_unique_key.sql": models__empty_str_unique_key_sql, + "str_unique_key.sql": models__str_unique_key_sql, + "duplicated_unary_unique_key_list.sql": models__duplicated_unary_unique_key_list_sql, + "not_found_unique_key_list.sql": models__not_found_unique_key_list_sql, + "expected": { + "one_str__overwrite.sql": models__expected__one_str__overwrite_sql, + "unique_key_list__inplace_overwrite.sql": models__expected__unique_key_list__inplace_overwrite_sql, + }, + } + + @pytest.fixture(autouse=True) + def clean_up(self, project): + yield + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=project.database, schema=project.test_schema + ) + project.adapter.drop_schema(relation) + + pass + + def update_incremental_model(self, incremental_model): + """update incremental model after the seed table has been updated""" + model_result_set = run_dbt(["run", "--select", incremental_model]) + return len(model_result_set) + + def get_test_fields( + self, project, seed, incremental_model, update_sql_file, opt_model_count=None + ): + """build a test case and return values for assertions + [INFO] Models must be in place to test incremental model + construction and merge behavior. Database touches are side + effects to extract counts (which speak to health of unique keys).""" + # idempotently create some number of seeds and incremental models''' + + seed_count = len(run_dbt(["seed", "--select", seed, "--full-refresh"])) + + model_count = len(run_dbt(["run", "--select", incremental_model, "--full-refresh"])) + # pass on kwarg + relation = incremental_model + # update seed in anticipation of incremental model update + row_count_query = "select * from {}.{}".format(project.test_schema, seed) + project.run_sql_file(Path("seeds") / Path(update_sql_file + ".sql")) + seed_rows = len(project.run_sql(row_count_query, fetch="all")) + + # propagate seed state to incremental model according to unique keys + inc_test_model_count = self.update_incremental_model(incremental_model=incremental_model) + + return ResultHolder( + seed_count, model_count, seed_rows, inc_test_model_count, opt_model_count, relation + ) + + def check_scenario_correctness(self, expected_fields, test_case_fields, project): + """Invoke assertions to verify correct build functionality""" + # 1. test seed(s) should build afresh + assert expected_fields.seed_count == test_case_fields.seed_count + # 2. test model(s) should build afresh + assert expected_fields.model_count == test_case_fields.model_count + # 3. seeds should have intended row counts post update + assert expected_fields.seed_rows == test_case_fields.seed_rows + # 4. incremental test model(s) should be updated + assert expected_fields.inc_test_model_count == test_case_fields.inc_test_model_count + # 5. extra incremental model(s) should be built; optional since + # comparison may be between an incremental model and seed + if expected_fields.opt_model_count and test_case_fields.opt_model_count: + assert expected_fields.opt_model_count == test_case_fields.opt_model_count + # 6. result table should match intended result set (itself a relation) + check_relations_equal( + project.adapter, [expected_fields.relation, test_case_fields.relation] + ) + + def get_expected_fields(self, relation, seed_rows, opt_model_count=None): + return ResultHolder( + seed_count=1, + model_count=1, + seed_rows=seed_rows, + inc_test_model_count=1, + opt_model_count=opt_model_count, + relation=relation, + ) + + def fail_to_build_inc_missing_unique_key_column(self, incremental_model_name): + """should pass back error state when trying build an incremental + model whose unique key or keylist includes a column missing + from the incremental model""" + seed_count = len(run_dbt(["seed", "--select", "seed", "--full-refresh"])) # noqa:F841 + # unique keys are not applied on first run, so two are needed + run_dbt( + ["run", "--select", incremental_model_name, "--full-refresh"], + expect_pass=True, + ) + run_result = run_dbt( + ["run", "--select", incremental_model_name], expect_pass=False + ).results[0] + + return run_result.status, run_result.message + + +class IncrementalUniqueKeyFalseyNullsEquals(BaseIncrementalUniqueKey): + + def test__bad_unique_key(self, project): + """expect compilation error from unique key not being a column""" + + (status, exc) = self.fail_to_build_inc_missing_unique_key_column( + incremental_model_name="not_found_unique_key" + ) + + assert status == RunStatus.Error + assert "thisisnotacolumn" in exc.lower() + + # test unique_key as list + def test__empty_unique_key_list(self, project): + """with no unique keys, seed and model should match""" + + expected_fields = self.get_expected_fields(relation="seed", seed_rows=9) + test_case_fields = self.get_test_fields( + project, + seed="seed", + incremental_model="empty_unique_key_list", + update_sql_file="add_new_rows", + ) + self.check_scenario_correctness(expected_fields, test_case_fields, project) + + def test__one_unique_key(self, project): + """with one unique key, model will overwrite existing row""" + + expected_fields = self.get_expected_fields( + relation="one_str__overwrite", seed_rows=8, opt_model_count=1 + ) + test_case_fields = self.get_test_fields( + project, + seed="seed", + incremental_model="str_unique_key", + update_sql_file="duplicate_insert", + opt_model_count=self.update_incremental_model("one_str__overwrite"), + ) + self.check_scenario_correctness(expected_fields, test_case_fields, project) + + def test__bad_unique_key_list(self, project): + """expect compilation error from unique key not being a column""" + + (status, exc) = self.fail_to_build_inc_missing_unique_key_column( + incremental_model_name="not_found_unique_key_list" + ) + + assert status == RunStatus.Error + assert "thisisnotacolumn" in exc.lower() + + +class IncrementalUniqueKeyTruthyNullsEquals(BaseIncrementalUniqueKey): + @pytest.fixture(scope="class") + def project_config_update(self): + return {"flags": {"enable_truthy_nulls_equals_macro": True}} + + # no unique_key test + def test__no_unique_keys(self, project): + """with no unique keys, seed and model should match""" + + expected_fields = self.get_expected_fields(relation="seed", seed_rows=9) + test_case_fields = self.get_test_fields( + project, seed="seed", incremental_model="no_unique_key", update_sql_file="add_new_rows" + ) + self.check_scenario_correctness(expected_fields, test_case_fields, project) + + # unique_key as str tests + def test__empty_str_unique_key(self, project): + """with empty string for unique key, seed and model should match""" + + expected_fields = self.get_expected_fields(relation="seed", seed_rows=9) + test_case_fields = self.get_test_fields( + project, + seed="seed", + incremental_model="empty_str_unique_key", + update_sql_file="add_new_rows", + ) + self.check_scenario_correctness(expected_fields, test_case_fields, project) + + def test__unary_unique_key_list(self, project): + """with one unique key, model will overwrite existing row""" + + expected_fields = self.get_expected_fields( + relation="unique_key_list__inplace_overwrite", seed_rows=8, opt_model_count=1 + ) + test_case_fields = self.get_test_fields( + project, + seed="seed", + incremental_model="unary_unique_key_list", + update_sql_file="duplicate_insert", + opt_model_count=self.update_incremental_model("unique_key_list__inplace_overwrite"), + ) + self.check_scenario_correctness(expected_fields, test_case_fields, project) + + def test__duplicated_unary_unique_key_list(self, project): + """with two of the same unique key, model will overwrite existing row""" + + expected_fields = self.get_expected_fields( + relation="unique_key_list__inplace_overwrite", seed_rows=8, opt_model_count=1 + ) + test_case_fields = self.get_test_fields( + project, + seed="seed", + incremental_model="duplicated_unary_unique_key_list", + update_sql_file="duplicate_insert", + opt_model_count=self.update_incremental_model("unique_key_list__inplace_overwrite"), + ) + self.check_scenario_correctness(expected_fields, test_case_fields, project) + + def test__trinary_unique_key_list(self, project): + """with three unique keys, model will overwrite existing row""" + + expected_fields = self.get_expected_fields( + relation="unique_key_list__inplace_overwrite", seed_rows=8, opt_model_count=1 + ) + test_case_fields = self.get_test_fields( + project, + seed="seed", + incremental_model="trinary_unique_key_list", + update_sql_file="duplicate_insert", + opt_model_count=self.update_incremental_model("unique_key_list__inplace_overwrite"), + ) + self.check_scenario_correctness(expected_fields, test_case_fields, project) + + def test__trinary_unique_key_list_no_update(self, project): + """even with three unique keys, adding distinct rows to seed does not + cause seed and model to diverge""" + + expected_fields = self.get_expected_fields(relation="seed", seed_rows=9) + test_case_fields = self.get_test_fields( + project, + seed="seed", + incremental_model="nontyped_trinary_unique_key_list", + update_sql_file="add_new_rows", + ) + self.check_scenario_correctness(expected_fields, test_case_fields, project) + + +class TestIncrementalUniqueKeyFalseyNullsEquals(IncrementalUniqueKeyFalseyNullsEquals): + pass + + +class TestIncrementalUniqueKeyTruthyNullsEquals(IncrementalUniqueKeyTruthyNullsEquals): pass diff --git a/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py b/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py index bb41e7111..05ff08e2a 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py +++ b/dbt-tests-adapter/dbt/tests/adapter/utils/base_utils.py @@ -6,7 +6,7 @@ {%- if adapter.behavior.enable_truthy_nulls_equals_macro.no_warn %} case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) {%- else -%} - case when (({{ expr1 }} = {{ expr2 }}) + case when ({{ expr1 }} = {{ expr2 }}) {%- endif %} then 0 else 1 diff --git a/dbt/include/global_project/macros/utils/equals.sql b/dbt/include/global_project/macros/utils/equals.sql index 171fc9082..a1542cf70 100644 --- a/dbt/include/global_project/macros/utils/equals.sql +++ b/dbt/include/global_project/macros/utils/equals.sql @@ -7,7 +7,7 @@ {%- if adapter.behavior.enable_truthy_nulls_equals_macro.no_warn %} case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) {%- else -%} - case when (({{ expr1 }} = {{ expr2 }}) + case when ({{ expr1 }} = {{ expr2 }}) {%- endif %} then 0 else 1 From f52aed2f324d2028ce062f217dc0b47cbee0f3a3 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 27 Jan 2025 21:44:25 -0800 Subject: [PATCH 23/40] Disable stubborn test about partitioning. --- .../adapter/incremental/test_incremental_strategies.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_strategies.py b/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_strategies.py index 1a339d601..261be9d81 100644 --- a/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_strategies.py +++ b/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_strategies.py @@ -40,7 +40,8 @@ def models(self): return { "incremental_merge_range.sql": merge_range_sql, "incremental_merge_time.sql": merge_time_sql, - "incremental_merge_time_with_require_partition.sql": merge_time_with_require_partition_sql, + # TODO: restore this test after figuring out why moving to monorepo has broken this + # "incremental_merge_time_with_require_partition.sql": merge_time_with_require_partition_sql, "incremental_overwrite_date.sql": overwrite_date_sql, "incremental_overwrite_day.sql": overwrite_day_sql, "incremental_overwrite_day_with_copy_partitions.sql": overwrite_day_with_copy_partitions_sql, @@ -67,14 +68,14 @@ def seeds(self): def test__bigquery_assert_incremental_configurations_apply_the_right_strategy(self, project): run_dbt(["seed"]) results = run_dbt() - assert len(results) == 12 + assert len(results) == 11 results = run_dbt() - assert len(results) == 12 + assert len(results) == 11 incremental_strategies = [ ("incremental_merge_range", "merge_expected"), ("incremental_merge_time", "merge_expected"), - ("incremental_merge_time_with_require_partition_view", "merge_expected"), + # ("incremental_merge_time_with_require_partition_view", "merge_expected"), ("incremental_overwrite_time", "incremental_overwrite_time_expected"), ("incremental_overwrite_date", "incremental_overwrite_date_expected"), ("incremental_overwrite_partitions", "incremental_overwrite_date_expected"), From 886000ab253cc9f2546943a07b06d8736903c7f5 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 27 Jan 2025 21:56:52 -0800 Subject: [PATCH 24/40] remove unneeded directive. --- dbt-redshift/hatch.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/dbt-redshift/hatch.toml b/dbt-redshift/hatch.toml index e1c860840..ab7d9f46f 100644 --- a/dbt-redshift/hatch.toml +++ b/dbt-redshift/hatch.toml @@ -62,6 +62,3 @@ check-sdist = [ "pip freeze | grep dbt-redshift", ] docker-prod = "docker build -f docker/Dockerfile -t dbt-redshift ." - -[metadata] -allow-direct-references = true From ecc941ae649911ff360bdebf0fb119607c2fe345 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 27 Jan 2025 21:59:19 -0800 Subject: [PATCH 25/40] revert pyproject files. --- dbt-athena/pyproject.toml | 5 +---- dbt-bigquery/pyproject.toml | 5 +---- dbt-postgres/pyproject.toml | 5 +---- dbt-redshift/pyproject.toml | 5 +---- dbt-snowflake/pyproject.toml | 5 +---- dbt-spark/hatch.toml | 5 +---- dbt-spark/pyproject.toml | 5 +---- 7 files changed, 7 insertions(+), 28 deletions(-) diff --git a/dbt-athena/pyproject.toml b/dbt-athena/pyproject.toml index 5ed2d6bca..69ec0018b 100644 --- a/dbt-athena/pyproject.toml +++ b/dbt-athena/pyproject.toml @@ -27,7 +27,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies=[ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", + "dbt-adapters>=1.0.0,<2.0", "dbt-common>=1.0.0,<2.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", @@ -53,6 +53,3 @@ filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", ] - -[tool.hatch.metadata] -allow-direct-references = true diff --git a/dbt-bigquery/pyproject.toml b/dbt-bigquery/pyproject.toml index b41a6b5a4..b2d55b25f 100644 --- a/dbt-bigquery/pyproject.toml +++ b/dbt-bigquery/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ ] dependencies = [ "dbt-common>=1.10,<2.0", - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", + "dbt-adapters>=1.7,<2.0", # 3.20 introduced pyarrow>=3.0 under the `pandas` extra "google-cloud-bigquery[pandas]>=3.0,<4.0", "google-cloud-storage~=2.4", @@ -55,6 +55,3 @@ filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", ] - -[tool.hatch.metadata] -allow-direct-references = true diff --git a/dbt-postgres/pyproject.toml b/dbt-postgres/pyproject.toml index 392fccc60..6f4b4604a 100644 --- a/dbt-postgres/pyproject.toml +++ b/dbt-postgres/pyproject.toml @@ -28,7 +28,7 @@ classifiers = [ ] dependencies = [ "psycopg2-binary>=2.9,<3.0", - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", + "dbt-adapters>=1.7.0,<2.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", # installed via dbt-adapters but used directly @@ -47,6 +47,3 @@ testpaths = [ "tests/functional", "tests/unit", ] - -[tool.hatch.metadata] -allow-direct-references = true diff --git a/dbt-redshift/pyproject.toml b/dbt-redshift/pyproject.toml index 4fc4b841d..77f86fd33 100644 --- a/dbt-redshift/pyproject.toml +++ b/dbt-redshift/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ ] dependencies = [ "dbt-common>=1.10,<2.0", - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", + "dbt-adapters>=1.11,<2.0", "dbt-postgres>=1.8,<1.10", # dbt-redshift depends deeply on this package. it does not follow SemVer, therefore there have been breaking changes in previous patch releases # Pin to the patch or minor version, and bump in each new minor version of dbt-redshift. @@ -55,6 +55,3 @@ filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", ] - -[tool.hatch.metadata] -allow-direct-references = true diff --git a/dbt-snowflake/pyproject.toml b/dbt-snowflake/pyproject.toml index a7c209c56..568aa3533 100644 --- a/dbt-snowflake/pyproject.toml +++ b/dbt-snowflake/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ ] dependencies = [ "dbt-common>=1.10,<2.0", - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", + "dbt-adapters>=1.10.4,<2.0", "snowflake-connector-python[secure-local-storage]>=3.0.0,<3.12.4", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", @@ -46,6 +46,3 @@ addopts = "-v --color=yes -n auto" filterwarnings = [ "ignore:datetime.datetime.utcnow:DeprecationWarning", ] - -[tool.hatch.metadata] -allow-direct-references = true diff --git a/dbt-spark/hatch.toml b/dbt-spark/hatch.toml index 010bafb58..42a20d6d4 100644 --- a/dbt-spark/hatch.toml +++ b/dbt-spark/hatch.toml @@ -11,7 +11,7 @@ sources = ["src"] [envs.default] dependencies = [ - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-adapters", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", @@ -59,6 +59,3 @@ check-sdist = [ "pip freeze | grep dbt-spark", ] docker-prod = "docker build -f docker/Dockerfile -t dbt-spark ." - -[metadata] -allow-direct-references = true diff --git a/dbt-spark/pyproject.toml b/dbt-spark/pyproject.toml index 9a38ddf15..de27f9cb2 100644 --- a/dbt-spark/pyproject.toml +++ b/dbt-spark/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ ] dependencies = [ "dbt-common>=1.10,<2.0", - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git@add_null_equality_condition_to_equals_macro", + "dbt-adapters>=1.7,<2.0", # add dbt-core to ensure backwards compatibility of installation, this is not a functional dependency "dbt-core>=1.8.0", "sqlparams>=3.0.0", @@ -58,6 +58,3 @@ filterwarnings = [ "ignore:.*'soft_unicode' has been renamed to 'soft_str'*:DeprecationWarning", "ignore:unclosed file .*:ResourceWarning", ] - -[tool.hatch.metadata] -allow-direct-references = true From d16a938773a15c8c9c172cf0035d5c004a7c32c9 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 27 Jan 2025 22:15:45 -0800 Subject: [PATCH 26/40] Prevent undesired change of behavior. --- .../macros/materializations/snapshots/helpers.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 45d63ed1b..8c4e0392f 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -62,7 +62,7 @@ config.get('dbt_valid_to_current') {% endset %} - {{ equals(source_unique_key, target_unique_key) }} + {{ equals(source_unique_key, target_unique_key) or source_unique_key is none }} {% else %} {{ columns.dbt_valid_to }} is null {% endif %} From 9b05a8c6b68670b6b4b8718dc353ec09bf803c19 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 27 Jan 2025 22:52:31 -0800 Subject: [PATCH 27/40] Fix tests. --- .../incremental/test_incremental_unique_id.py | 452 +----------------- .../incremental/test_incremental_unique_id.py | 75 +-- 2 files changed, 25 insertions(+), 502 deletions(-) diff --git a/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_unique_id.py b/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_unique_id.py index 17b2a502d..6cdfbbbf5 100644 --- a/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_unique_id.py +++ b/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_unique_id.py @@ -1,453 +1,7 @@ -# from dbt.tests.adapter.incremental.test_incremental_unique_id import BaseIncrementalUniqueKey +from dbt.tests.adapter.incremental.test_incremental_unique_id import SubBaseIncrementalUniqueKey -# class TestUniqueKeyBigQuery(BaseIncrementalUniqueKey): -# pass - -from collections import namedtuple -from pathlib import Path - -# TODO: repoint to dbt-artifacts when it's available -from dbt.artifacts.schemas.results import RunStatus -import pytest - -from dbt.tests.util import check_relations_equal, run_dbt - - -models__trinary_unique_key_list_sql = """ --- a multi-argument unique key list should see overwriting on rows in the model --- where all unique key fields apply - -{{ - config( - materialized='incremental', - unique_key=['state', 'county', 'city'] - ) -}} - -select - state as state, - county as county, - city as city, - last_visit_date as last_visit_date -from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} - -""" - -models__nontyped_trinary_unique_key_list_sql = """ --- a multi-argument unique key list should see overwriting on rows in the model --- where all unique key fields apply --- N.B. needed for direct comparison with seed - -{{ - config( - materialized='incremental', - unique_key=['state', 'county', 'city'] - ) -}} - -select - state as state, - county as county, - city as city, - last_visit_date as last_visit_date -from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} - -""" - -models__unary_unique_key_list_sql = """ --- a one argument unique key list should result in overwritting semantics for --- that one matching field - -{{ - config( - materialized='incremental', - unique_key=['state'] - ) -}} - -select - state as state, - county as county, - city as city, - last_visit_date as last_visit_date -from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} - -""" - -models__not_found_unique_key_sql = """ --- a model with a unique key not found in the table itself will error out - -{{ - config( - materialized='incremental', - unique_key='thisisnotacolumn' - ) -}} - -select - * -from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} - -""" - -models__empty_unique_key_list_sql = """ --- model with empty list unique key should build normally - -{{ - config( - materialized='incremental', - unique_key=[] - ) -}} - -select * from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} - -""" - -models__no_unique_key_sql = """ --- no specified unique key should cause no special build behavior - -{{ - config( - materialized='incremental' - ) -}} - -select - * -from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} - -""" - -models__empty_str_unique_key_sql = """ --- ensure model with empty string unique key should build normally - -{{ - config( - materialized='incremental', - unique_key='' - ) -}} - -select - * -from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} - -""" - -models__str_unique_key_sql = """ --- a unique key with a string should trigger to overwrite behavior when --- the source has entries in conflict (i.e. more than one row per unique key --- combination) - -{{ - config( - materialized='incremental', - unique_key='state' - ) -}} - -select - state as state, - county as county, - city as city, - last_visit_date as last_visit_date -from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} - -""" - -models__duplicated_unary_unique_key_list_sql = """ -{{ - config( - materialized='incremental', - unique_key=['state', 'state'] - ) -}} - -select - state as state, - county as county, - city as city, - last_visit_date as last_visit_date -from {{ ref('seed') }} - -{% if is_incremental() %} - where last_visit_date > (select max(last_visit_date) from {{ this }}) -{% endif %} - -""" - -models__not_found_unique_key_list_sql = """ --- a unique key list with any element not in the model itself should error out - -{{ - config( - materialized='incremental', - unique_key=['state', 'thisisnotacolumn'] - ) -}} - -select * from {{ ref('seed') }} - -""" - -models__expected__one_str__overwrite_sql = """ -{{ - config( - materialized='table' - ) -}} - -select - 'CT' as state, - 'Hartford' as county, - 'Hartford' as city, - cast('2022-02-14' as date) as last_visit_date -union all -select 'MA','Suffolk','Boston','2020-02-12' -union all -select 'NJ','Mercer','Trenton','2022-01-01' -union all -select 'NY','Kings','Brooklyn','2021-04-02' -union all -select 'NY','New York','Manhattan','2021-04-01' -union all -select 'PA','Philadelphia','Philadelphia','2021-05-21' -union all -select 'CO','Denver',null,'2021-06-18' - -""" - -models__expected__unique_key_list__inplace_overwrite_sql = """ -{{ - config( - materialized='table' - ) -}} - -select - 'CT' as state, - 'Hartford' as county, - 'Hartford' as city, - cast('2022-02-14' as date) as last_visit_date -union all -select 'MA','Suffolk','Boston','2020-02-12' -union all -select 'NJ','Mercer','Trenton','2022-01-01' -union all -select 'NY','Kings','Brooklyn','2021-04-02' -union all -select 'NY','New York','Manhattan','2021-04-01' -union all -select 'PA','Philadelphia','Philadelphia','2021-05-21' -union all -select 'CO','Denver',null,'2021-06-18' - -""" - -seeds__duplicate_insert_sql = """ --- Insert statement which when applied to seed.csv triggers the inplace --- overwrite strategy of incremental models. Seed and incremental model --- diverge. - --- insert new row, which should not be in incremental model --- with primary or first three columns unique -insert into {schema}.seed - (state, county, city, last_visit_date) -values ('CT','Hartford','Hartford','2022-02-14'); - -""" - -seeds__seed_csv = """state,county,city,last_visit_date -CT,Hartford,Hartford,2020-09-23 -MA,Suffolk,Boston,2020-02-12 -NJ,Mercer,Trenton,2022-01-01 -NY,Kings,Brooklyn,2021-04-02 -NY,New York,Manhattan,2021-04-01 -PA,Philadelphia,Philadelphia,2021-05-21 -CO,Denver,,2021-06-18 -""" - -seeds__add_new_rows_sql = """ --- Insert statement which when applied to seed.csv sees incremental model --- grow in size while not (necessarily) diverging from the seed itself. - --- insert two new rows, both of which should be in incremental model --- with any unique columns -insert into {schema}.seed - (state, county, city, last_visit_date) -values ('WA','King','Seattle','2022-02-01'); - -insert into {schema}.seed - (state, county, city, last_visit_date) -values ('CA','Los Angeles','Los Angeles','2022-02-01'); - -""" - - -ResultHolder = namedtuple( - "ResultHolder", - [ - "seed_count", - "model_count", - "seed_rows", - "inc_test_model_count", - "opt_model_count", - "relation", - ], -) - - -class BaseIncrementalUniqueKey: - @pytest.fixture(scope="class") - def seeds(self): - return { - "duplicate_insert.sql": seeds__duplicate_insert_sql, - "seed.csv": seeds__seed_csv, - "add_new_rows.sql": seeds__add_new_rows_sql, - } - - @pytest.fixture(scope="class") - def models(self): - return { - "trinary_unique_key_list.sql": models__trinary_unique_key_list_sql, - "nontyped_trinary_unique_key_list.sql": models__nontyped_trinary_unique_key_list_sql, - "unary_unique_key_list.sql": models__unary_unique_key_list_sql, - "not_found_unique_key.sql": models__not_found_unique_key_sql, - "empty_unique_key_list.sql": models__empty_unique_key_list_sql, - "no_unique_key.sql": models__no_unique_key_sql, - "empty_str_unique_key.sql": models__empty_str_unique_key_sql, - "str_unique_key.sql": models__str_unique_key_sql, - "duplicated_unary_unique_key_list.sql": models__duplicated_unary_unique_key_list_sql, - "not_found_unique_key_list.sql": models__not_found_unique_key_list_sql, - "expected": { - "one_str__overwrite.sql": models__expected__one_str__overwrite_sql, - "unique_key_list__inplace_overwrite.sql": models__expected__unique_key_list__inplace_overwrite_sql, - }, - } - - @pytest.fixture(autouse=True) - def clean_up(self, project): - yield - with project.adapter.connection_named("__test"): - relation = project.adapter.Relation.create( - database=project.database, schema=project.test_schema - ) - project.adapter.drop_schema(relation) - - pass - - def update_incremental_model(self, incremental_model): - """update incremental model after the seed table has been updated""" - model_result_set = run_dbt(["run", "--select", incremental_model]) - return len(model_result_set) - - def get_test_fields( - self, project, seed, incremental_model, update_sql_file, opt_model_count=None - ): - """build a test case and return values for assertions - [INFO] Models must be in place to test incremental model - construction and merge behavior. Database touches are side - effects to extract counts (which speak to health of unique keys).""" - # idempotently create some number of seeds and incremental models''' - - seed_count = len(run_dbt(["seed", "--select", seed, "--full-refresh"])) - - model_count = len(run_dbt(["run", "--select", incremental_model, "--full-refresh"])) - # pass on kwarg - relation = incremental_model - # update seed in anticipation of incremental model update - row_count_query = "select * from {}.{}".format(project.test_schema, seed) - project.run_sql_file(Path("seeds") / Path(update_sql_file + ".sql")) - seed_rows = len(project.run_sql(row_count_query, fetch="all")) - - # propagate seed state to incremental model according to unique keys - inc_test_model_count = self.update_incremental_model(incremental_model=incremental_model) - - return ResultHolder( - seed_count, model_count, seed_rows, inc_test_model_count, opt_model_count, relation - ) - - def check_scenario_correctness(self, expected_fields, test_case_fields, project): - """Invoke assertions to verify correct build functionality""" - # 1. test seed(s) should build afresh - assert expected_fields.seed_count == test_case_fields.seed_count - # 2. test model(s) should build afresh - assert expected_fields.model_count == test_case_fields.model_count - # 3. seeds should have intended row counts post update - assert expected_fields.seed_rows == test_case_fields.seed_rows - # 4. incremental test model(s) should be updated - assert expected_fields.inc_test_model_count == test_case_fields.inc_test_model_count - # 5. extra incremental model(s) should be built; optional since - # comparison may be between an incremental model and seed - if expected_fields.opt_model_count and test_case_fields.opt_model_count: - assert expected_fields.opt_model_count == test_case_fields.opt_model_count - # 6. result table should match intended result set (itself a relation) - check_relations_equal( - project.adapter, [expected_fields.relation, test_case_fields.relation] - ) - - def get_expected_fields(self, relation, seed_rows, opt_model_count=None): - return ResultHolder( - seed_count=1, - model_count=1, - seed_rows=seed_rows, - inc_test_model_count=1, - opt_model_count=opt_model_count, - relation=relation, - ) - - def fail_to_build_inc_missing_unique_key_column(self, incremental_model_name): - """should pass back error state when trying build an incremental - model whose unique key or keylist includes a column missing - from the incremental model""" - seed_count = len(run_dbt(["seed", "--select", "seed", "--full-refresh"])) # noqa:F841 - # unique keys are not applied on first run, so two are needed - run_dbt( - ["run", "--select", incremental_model_name, "--full-refresh"], - expect_pass=True, - ) - run_result = run_dbt( - ["run", "--select", incremental_model_name], expect_pass=False - ).results[0] - - return run_result.status, run_result.message - - -class IncrementalUniqueKeyFalseyNullsEquals(BaseIncrementalUniqueKey): +class IncrementalUniqueKeyFalseyNullsEquals(SubBaseIncrementalUniqueKey): def test__bad_unique_key(self, project): """expect compilation error from unique key not being a column""" @@ -497,7 +51,7 @@ def test__bad_unique_key_list(self, project): assert "thisisnotacolumn" in exc.lower() -class IncrementalUniqueKeyTruthyNullsEquals(BaseIncrementalUniqueKey): +class IncrementalUniqueKeyTruthyNullsEquals(SubBaseIncrementalUniqueKey): @pytest.fixture(scope="class") def project_config_update(self): return {"flags": {"enable_truthy_nulls_equals_macro": True}} diff --git a/dbt-tests-adapter/src/dbt/tests/adapter/incremental/test_incremental_unique_id.py b/dbt-tests-adapter/src/dbt/tests/adapter/incremental/test_incremental_unique_id.py index f803ca36e..50f7f2f04 100644 --- a/dbt-tests-adapter/src/dbt/tests/adapter/incremental/test_incremental_unique_id.py +++ b/dbt-tests-adapter/src/dbt/tests/adapter/incremental/test_incremental_unique_id.py @@ -325,7 +325,7 @@ ) -class BaseIncrementalUniqueKey: +class SubBaseIncrementalUniqueKey: @pytest.fixture(scope="class") def seeds(self): return { @@ -334,6 +334,25 @@ def seeds(self): "add_new_rows.sql": seeds__add_new_rows_sql, } + @pytest.fixture(scope="class") + def models(self): + return { + "trinary_unique_key_list.sql": models__trinary_unique_key_list_sql, + "nontyped_trinary_unique_key_list.sql": models__nontyped_trinary_unique_key_list_sql, + "unary_unique_key_list.sql": models__unary_unique_key_list_sql, + "not_found_unique_key.sql": models__not_found_unique_key_sql, + "empty_unique_key_list.sql": models__empty_unique_key_list_sql, + "no_unique_key.sql": models__no_unique_key_sql, + "empty_str_unique_key.sql": models__empty_str_unique_key_sql, + "str_unique_key.sql": models__str_unique_key_sql, + "duplicated_unary_unique_key_list.sql": models__duplicated_unary_unique_key_list_sql, + "not_found_unique_key_list.sql": models__not_found_unique_key_list_sql, + "expected": { + "one_str__overwrite.sql": models__expected__one_str__overwrite_sql, + "unique_key_list__inplace_overwrite.sql": models__expected__unique_key_list__inplace_overwrite_sql, + }, + } + @pytest.fixture(autouse=True) def clean_up(self, project): yield @@ -343,8 +362,6 @@ def clean_up(self, project): ) project.adapter.drop_schema(relation) - pass - def update_incremental_model(self, incremental_model): """update incremental model after the seed table has been updated""" model_result_set = run_dbt(["run", "--select", incremental_model]) @@ -422,26 +439,7 @@ def fail_to_build_inc_missing_unique_key_column(self, incremental_model_name): return run_result.status, run_result.message -class IncrementalUniqueKeyFalseyNullsEquals(BaseIncrementalUniqueKey): - @pytest.fixture(scope="class") - def models(self): - return { - "trinary_unique_key_list.sql": models__trinary_unique_key_list_sql, - "nontyped_trinary_unique_key_list.sql": models__nontyped_trinary_unique_key_list_sql, - "unary_unique_key_list.sql": models__unary_unique_key_list_sql, - "not_found_unique_key.sql": models__not_found_unique_key_sql, - "empty_unique_key_list.sql": models__empty_unique_key_list_sql, - "no_unique_key.sql": models__no_unique_key_sql, - "empty_str_unique_key.sql": models__empty_str_unique_key_sql, - "str_unique_key.sql": models__str_unique_key_sql, - "duplicated_unary_unique_key_list.sql": models__duplicated_unary_unique_key_list_sql, - "not_found_unique_key_list.sql": models__not_found_unique_key_list_sql, - "expected": { - "one_str__overwrite.sql": models__expected__one_str__overwrite_sql, - "unique_key_list__inplace_overwrite.sql": models__expected__unique_key_list__inplace_overwrite_sql, - }, - } - +class BaseIncrementalUniqueKey(SubBaseIncrementalUniqueKey): def test__bad_unique_key(self, project): """expect compilation error from unique key not being a column""" @@ -490,31 +488,6 @@ def test__bad_unique_key_list(self, project): assert status == RunStatus.Error assert "thisisnotacolumn" in exc.lower() - -class IncrementalUniqueKeyTruthyNullsEquals(BaseIncrementalUniqueKey): - @pytest.fixture(scope="class") - def models(self): - return { - "trinary_unique_key_list.sql": models__trinary_unique_key_list_sql, - "nontyped_trinary_unique_key_list.sql": models__nontyped_trinary_unique_key_list_sql, - "unary_unique_key_list.sql": models__unary_unique_key_list_sql, - "not_found_unique_key.sql": models__not_found_unique_key_sql, - "empty_unique_key_list.sql": models__empty_unique_key_list_sql, - "no_unique_key.sql": models__no_unique_key_sql, - "empty_str_unique_key.sql": models__empty_str_unique_key_sql, - "str_unique_key.sql": models__str_unique_key_sql, - "duplicated_unary_unique_key_list.sql": models__duplicated_unary_unique_key_list_sql, - "not_found_unique_key_list.sql": models__not_found_unique_key_list_sql, - "expected": { - "one_str__overwrite.sql": models__expected__one_str__overwrite_sql, - "unique_key_list__inplace_overwrite.sql": models__expected__unique_key_list__inplace_overwrite_sql, - }, - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return {"flags": {"enable_truthy_nulls_equals_macro": True}} - # no unique_key test def test__no_unique_keys(self, project): """with no unique keys, seed and model should match""" @@ -597,9 +570,5 @@ def test__trinary_unique_key_list_no_update(self, project): self.check_scenario_correctness(expected_fields, test_case_fields, project) -class TestIncrementalUniqueKeyFalseyNullsEquals(IncrementalUniqueKeyFalseyNullsEquals): - pass - - -class TestIncrementalUniqueKeyTruthyNullsEquals(IncrementalUniqueKeyTruthyNullsEquals): +class TestIncrementalUniqueKey(BaseIncrementalUniqueKey): pass From 6abab94fad3d3a8b8682bf3b9e4d271457f48795 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 27 Jan 2025 22:56:38 -0800 Subject: [PATCH 28/40] restore macro definition headers. --- dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py b/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py index 05ff08e2a..75d8650b0 100644 --- a/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py +++ b/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py @@ -3,6 +3,7 @@ macros__equals_sql = """ +{% macro equals(expr1, expr2) -%} {%- if adapter.behavior.enable_truthy_nulls_equals_macro.no_warn %} case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) {%- else -%} @@ -11,6 +12,7 @@ then 0 else 1 end = 0 +{% endmacro %} """ macros__test_assert_equal_sql = """ From 0e8df0cbd43e2e529435adb6f8afb37d6f78c6f4 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 28 Jan 2025 00:15:16 -0800 Subject: [PATCH 29/40] Fix that stubborn bug by retracting ansi equals to plain equals except if flag is active --- .../global_project/macros/utils/equals.sql | 18 ++++++++---------- .../incremental/test_incremental_strategies.py | 9 ++++----- .../src/dbt/tests/adapter/utils/base_utils.py | 6 +++--- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/utils/equals.sql b/dbt-adapters/src/dbt/include/global_project/macros/utils/equals.sql index a1542cf70..defc3503a 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/utils/equals.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/utils/equals.sql @@ -3,14 +3,12 @@ {%- endmacro %} {% macro default__equals(expr1, expr2) -%} - - {%- if adapter.behavior.enable_truthy_nulls_equals_macro.no_warn %} - case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) - {%- else -%} - case when ({{ expr1 }} = {{ expr2 }}) - {%- endif %} - then 0 - else 1 - end = 0 - +{%- if adapter.behavior.enable_truthy_nulls_equals_macro.no_warn %} + case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) + then 0 + else 1 + end = 0 +{%- else -%} + ({{ expr1 }} = {{ expr2 }}) +{%- endif %} {% endmacro %} diff --git a/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_strategies.py b/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_strategies.py index 261be9d81..1a339d601 100644 --- a/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_strategies.py +++ b/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_strategies.py @@ -40,8 +40,7 @@ def models(self): return { "incremental_merge_range.sql": merge_range_sql, "incremental_merge_time.sql": merge_time_sql, - # TODO: restore this test after figuring out why moving to monorepo has broken this - # "incremental_merge_time_with_require_partition.sql": merge_time_with_require_partition_sql, + "incremental_merge_time_with_require_partition.sql": merge_time_with_require_partition_sql, "incremental_overwrite_date.sql": overwrite_date_sql, "incremental_overwrite_day.sql": overwrite_day_sql, "incremental_overwrite_day_with_copy_partitions.sql": overwrite_day_with_copy_partitions_sql, @@ -68,14 +67,14 @@ def seeds(self): def test__bigquery_assert_incremental_configurations_apply_the_right_strategy(self, project): run_dbt(["seed"]) results = run_dbt() - assert len(results) == 11 + assert len(results) == 12 results = run_dbt() - assert len(results) == 11 + assert len(results) == 12 incremental_strategies = [ ("incremental_merge_range", "merge_expected"), ("incremental_merge_time", "merge_expected"), - # ("incremental_merge_time_with_require_partition_view", "merge_expected"), + ("incremental_merge_time_with_require_partition_view", "merge_expected"), ("incremental_overwrite_time", "incremental_overwrite_time_expected"), ("incremental_overwrite_date", "incremental_overwrite_date_expected"), ("incremental_overwrite_partitions", "incremental_overwrite_date_expected"), diff --git a/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py b/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py index 75d8650b0..806322e45 100644 --- a/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py +++ b/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py @@ -6,12 +6,12 @@ {% macro equals(expr1, expr2) -%} {%- if adapter.behavior.enable_truthy_nulls_equals_macro.no_warn %} case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) -{%- else -%} - case when ({{ expr1 }} = {{ expr2 }}) -{%- endif %} then 0 else 1 end = 0 +{%- else -%} + ({{ expr1 }} = {{ expr2 }}) +{%- endif %} {% endmacro %} """ From 68c70673625be4968755f8709bc5709f6f28383b Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 28 Jan 2025 00:49:19 -0800 Subject: [PATCH 30/40] revert alteration to functional equality macro. --- .../src/dbt/tests/adapter/utils/base_utils.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py b/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py index 806322e45..23e1ca7fa 100644 --- a/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py +++ b/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py @@ -4,14 +4,10 @@ macros__equals_sql = """ {% macro equals(expr1, expr2) -%} -{%- if adapter.behavior.enable_truthy_nulls_equals_macro.no_warn %} - case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) - then 0 - else 1 - end = 0 -{%- else -%} - ({{ expr1 }} = {{ expr2 }}) -{%- endif %} +case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) + then 0 + else 1 +end = 0 {% endmacro %} """ From e70e95aee0fd5dfbec17b2c412d16184aaf1e99f Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 28 Jan 2025 00:53:19 -0800 Subject: [PATCH 31/40] Readd the change. --- .../src/dbt/tests/adapter/utils/base_utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py b/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py index 23e1ca7fa..806322e45 100644 --- a/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py +++ b/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py @@ -4,10 +4,14 @@ macros__equals_sql = """ {% macro equals(expr1, expr2) -%} -case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) - then 0 - else 1 -end = 0 +{%- if adapter.behavior.enable_truthy_nulls_equals_macro.no_warn %} + case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) + then 0 + else 1 + end = 0 +{%- else -%} + ({{ expr1 }} = {{ expr2 }}) +{%- endif %} {% endmacro %} """ From b9946bd4f57603619a24b50a370146d0b42e74bf Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Tue, 28 Jan 2025 01:05:43 -0800 Subject: [PATCH 32/40] Keep macro in place for now. Unpredictable to modify. --- .../src/dbt/tests/adapter/utils/base_utils.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py b/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py index 806322e45..23e1ca7fa 100644 --- a/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py +++ b/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py @@ -4,14 +4,10 @@ macros__equals_sql = """ {% macro equals(expr1, expr2) -%} -{%- if adapter.behavior.enable_truthy_nulls_equals_macro.no_warn %} - case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) - then 0 - else 1 - end = 0 -{%- else -%} - ({{ expr1 }} = {{ expr2 }}) -{%- endif %} +case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) + then 0 + else 1 +end = 0 {% endmacro %} """ From 4883cdc93243bcfcc7412fb9975e95b4e83663c2 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Sun, 2 Feb 2025 17:50:36 -0800 Subject: [PATCH 33/40] Push broken code to try proving a point. --- dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py b/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py index 23e1ca7fa..945397869 100644 --- a/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py +++ b/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py @@ -7,7 +7,8 @@ case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) then 0 else 1 -end = 0 + aaaa +end = 0aaaaaa {% endmacro %} """ From 88534dd70b992305d1768b94d5dcdf94af8ccdac Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Sun, 2 Feb 2025 19:13:48 -0800 Subject: [PATCH 34/40] FINALLY figured it out. Hatch even with recent adjustments will still install packages for main and you can override this with -e but you must check in python -m site and python freeze directly to ensure --- dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py b/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py index 945397869..23e1ca7fa 100644 --- a/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py +++ b/dbt-tests-adapter/src/dbt/tests/adapter/utils/base_utils.py @@ -7,8 +7,7 @@ case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null)) then 0 else 1 - aaaa -end = 0aaaaaa +end = 0 {% endmacro %} """ From 8e238e97c1792985616158015b427625a053498f Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Sun, 2 Feb 2025 19:34:19 -0800 Subject: [PATCH 35/40] Finish fixing tests.- --- .../adapter/incremental/test_incremental_unique_id.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_unique_id.py b/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_unique_id.py index 6cdfbbbf5..6c3e9ed86 100644 --- a/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_unique_id.py +++ b/dbt-bigquery/tests/functional/adapter/incremental/test_incremental_unique_id.py @@ -1,3 +1,7 @@ +import pytest + +from dbt.contracts.results import RunStatus + from dbt.tests.adapter.incremental.test_incremental_unique_id import SubBaseIncrementalUniqueKey From 7fc9289f11b223da67eb0f726ef1c8000b3566b3 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Sun, 2 Feb 2025 19:37:24 -0800 Subject: [PATCH 36/40] Add me author cus colin said so <(^-^<) --- .changes/unreleased/Under the Hood-20241217-110536.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/Under the Hood-20241217-110536.yaml b/.changes/unreleased/Under the Hood-20241217-110536.yaml index 5716da5e1..99710e4a5 100644 --- a/.changes/unreleased/Under the Hood-20241217-110536.yaml +++ b/.changes/unreleased/Under the Hood-20241217-110536.yaml @@ -2,5 +2,5 @@ kind: Under the Hood body: Added new equals macro that handles null value checks in sql time: 2024-12-17T11:05:36.363421+02:00 custom: - Author: adrianburusdbt + Author: adrianburusdbt,versusfacit Issue: "159" From b8843b1c4dd44f04240773ac83649a842de41407 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 3 Feb 2025 03:12:36 -0800 Subject: [PATCH 37/40] Adjust boolean logic of snapshot materializations --- .../macros/materializations/snapshots/helpers.sql | 8 +++++--- .../macros/materializations/snapshots/snapshot_merge.sql | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 8c4e0392f..3788bef81 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -56,13 +56,15 @@ where {% if config.get('dbt_valid_to_current') %} {% set source_unique_key %} - columns.dbt_valid_to + {{ columns.dbt_valid_to }} {% endset %} {% set target_unique_key %} - config.get('dbt_valid_to_current') + {{ config.get('dbt_valid_to_current') }} {% endset %} - {{ equals(source_unique_key, target_unique_key) or source_unique_key is none }} + {# The exact equals semantics between NULL values depends on the current behavior flag set. Also, update + records if the source field is null #} + {{ equals(source_unique_key, target_unique_key) or source_unique_key is none }} {% else %} {{ columns.dbt_valid_to }} is null {% endif %} diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index 19a67f6b7..ee2fb7a3e 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -21,7 +21,7 @@ {% set target_unique_key %} {{ config.get('dbt_valid_to_current') }} {% endset %} - and {{ equals(source_unique_key, target_unique_key) }} + and {{ equals(source_unique_key, target_unique_key) or source_unique_key is none }} {% else %} and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null From 5851e07c0f426cf560c667919be40984566c6a99 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 3 Feb 2025 03:17:39 -0800 Subject: [PATCH 38/40] More authentic mapping. --- .../macros/materializations/snapshots/helpers.sql | 5 ++--- .../macros/materializations/snapshots/snapshot_merge.sql | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 3788bef81..b6bb11cc6 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -62,9 +62,8 @@ {{ config.get('dbt_valid_to_current') }} {% endset %} - {# The exact equals semantics between NULL values depends on the current behavior flag set. Also, update - records if the source field is null #} - {{ equals(source_unique_key, target_unique_key) or source_unique_key is none }} + {# The exact equals semantics between NULL values depends on the current behavior flag set. Also, update records if the source field is null #} + ( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null ) {% else %} {{ columns.dbt_valid_to }} is null {% endif %} diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index ee2fb7a3e..017754e81 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -21,7 +21,7 @@ {% set target_unique_key %} {{ config.get('dbt_valid_to_current') }} {% endset %} - and {{ equals(source_unique_key, target_unique_key) or source_unique_key is none }} + and {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null {% else %} and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null From 5f7aa8a58fedc7c588eeadc9a7c7c255b35d8a79 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 3 Feb 2025 03:58:11 -0800 Subject: [PATCH 39/40] Still fixing core tests. --- .../macros/materializations/snapshots/helpers.sql | 8 ++------ .../materializations/snapshots/snapshot_merge.sql | 10 +++------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index b6bb11cc6..64db3eebe 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -55,12 +55,8 @@ from {{ target_relation }} where {% if config.get('dbt_valid_to_current') %} - {% set source_unique_key %} - {{ columns.dbt_valid_to }} - {% endset %} - {% set target_unique_key %} - {{ config.get('dbt_valid_to_current') }} - {% endset %} + {% set source_unique_key = columns.dbt_valid_to | trim %} + {% set target_unique_key = config.get('dbt_valid_to_current') | trim %} {# The exact equals semantics between NULL values depends on the current behavior flag set. Also, update records if the source field is null #} ( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null ) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql index 017754e81..75be104dc 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/snapshot_merge.sql @@ -15,13 +15,9 @@ when matched {% if config.get("dbt_valid_to_current") %} - {% set source_unique_key %} - DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} - {% endset %} - {% set target_unique_key %} - {{ config.get('dbt_valid_to_current') }} - {% endset %} - and {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null + {% set source_unique_key = ("DBT_INTERNAL_DEST." ~ columns.dbt_valid_to) | trim %} + {% set target_unique_key = config.get('dbt_valid_to_current') | trim %} + and ({{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null) {% else %} and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null From 4b349c64bcc120001ca170724b5b2b0f7b9a5049 Mon Sep 17 00:00:00 2001 From: VersusFacit <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 3 Feb 2025 04:03:16 -0800 Subject: [PATCH 40/40] One more adjustment for core. --- .../models/incremental/merge.sql | 21 +++++-------------- .../materializations/snapshots/helpers.sql | 11 +++------- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/models/incremental/merge.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/models/incremental/merge.sql index d7e8af70c..9df17def7 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/models/incremental/merge.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/models/incremental/merge.sql @@ -21,15 +21,9 @@ {% do predicates.append(this_key_match) %} {% endfor %} {% else %} - {% set source_unique_key %} - DBT_INTERNAL_SOURCE.{{ unique_key }} - {% endset %} - {% set target_unique_key %} - DBT_INTERNAL_DEST.{{ unique_key }} - {% endset %} - {% set unique_key_match %} - {{ equals(source_unique_key, target_unique_key) }} - {% endset %} + {% set source_unique_key = ("DBT_INTERNAL_SOURCE." ~ unique_key) | trim %} + {% set target_unique_key = ("DBT_INTERNAL_DEST." ~ unique_key) | trim %} + {% set unique_key_match = equals(source_unique_key, target_unique_key) | trim %} {% do predicates.append(unique_key_match) %} {% endif %} {% else %} @@ -72,13 +66,8 @@ using {{ source }} where ( {% for key in unique_key %} - {% set source_unique_key %} - {{ source }}.{{ key }} - {% endset %} - {% set target_unique_key %} - {{ target }}.{{ key }} - {% endset %} - + {% set source_unique_key = (source ~ "." ~ key) | trim %} + {% set target_unique_key = (target ~ "." ~ key) | trim %} {{ equals(source_unique_key, target_unique_key) }} {{ "and " if not loop.last}} {% endfor %} diff --git a/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 64db3eebe..c96e77d7b 100644 --- a/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt-adapters/src/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -282,14 +282,9 @@ {% macro unique_key_join_on(unique_key, identifier, from_identifier) %} {% if unique_key | is_list %} {% for key in unique_key %} - {% set source_unique_key %} - {{ identifier }}.dbt_unique_key_{{ loop.index }} - {% endset %} - {% set target_unique_key %} - {{ from_identifier }}.dbt_unique_key_{{ loop.index }} - {% endset %} - - {{ equals(source_unique_key, target_unique_key) }} + {% set source_unique_key = (identifier ~ ".dbt_unique_key_" ~ loop.index) | trim %} + {% set target_unique_key = (from_identifier ~ ".dbt_unique_key_" ~ loop.index) | trim %} + {{ equals(source_unique_key, target_unique_key) }} {%- if not loop.last %} and {%- endif %} {% endfor %} {% else %}