From 408e6792db2f906d55123c63b361396dc8624698 Mon Sep 17 00:00:00 2001 From: Noy Arie Date: Wed, 8 Jan 2025 12:32:45 +0200 Subject: [PATCH 1/4] Ele 3941 collect column info (#772) * replace old flags with new * change condition - meta & tags * pre-commit * fix upload columns --- integration_tests/tests/dbt_project.py | 2 +- .../tests/test_dbt_artifacts/test_columns.py | 14 ++++++-------- macros/edr/dbt_artifacts/upload_dbt_artifacts.sql | 2 +- macros/edr/dbt_artifacts/upload_dbt_columns.sql | 12 +++++++++++- macros/edr/system/system_utils/get_config_var.sql | 3 +-- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/integration_tests/tests/dbt_project.py b/integration_tests/tests/dbt_project.py index 5fd5168c2..bb2f026b3 100644 --- a/integration_tests/tests/dbt_project.py +++ b/integration_tests/tests/dbt_project.py @@ -18,7 +18,7 @@ _DEFAULT_VARS = { "disable_dbt_invocation_autoupload": True, "disable_dbt_artifacts_autoupload": True, - "disable_dbt_columns_autoupload": True, + "columns_upload_strategy": "none", "disable_run_results": True, "disable_freshness_results": True, "debug_logs": True, diff --git a/integration_tests/tests/test_dbt_artifacts/test_columns.py b/integration_tests/tests/test_dbt_artifacts/test_columns.py index 9c4ebeff7..41238bef4 100644 --- a/integration_tests/tests/test_dbt_artifacts/test_columns.py +++ b/integration_tests/tests/test_dbt_artifacts/test_columns.py @@ -28,17 +28,17 @@ @Parametrization.autodetect_parameters() @Parametrization.case( name="default", - only_with_description=None, + columns_upload_strategy=None, expected_columns=["with_description"], ) @Parametrization.case( name="only_with_description", - only_with_description=True, + columns_upload_strategy="enriched_only", expected_columns=["with_description"], ) @Parametrization.case( name="all", - only_with_description=False, + columns_upload_strategy="all", expected_columns=[ "with_description", "without_description", @@ -48,13 +48,11 @@ ) def test_flatten_table_columns( dbt_project: DbtProject, - only_with_description: Optional[bool], + columns_upload_strategy: Optional[str], expected_columns: List[str], ) -> None: - if only_with_description is not None: - dbt_project.dbt_runner.vars[ - "upload_only_columns_with_descriptions" - ] = only_with_description + if columns_upload_strategy is not None: + dbt_project.dbt_runner.vars["columns_upload_strategy"] = columns_upload_strategy flattened_columns = json.loads( dbt_project.dbt_runner.run_operation( "elementary.flatten_table_columns", macro_args={"table_node": TABLE_NODE} diff --git a/macros/edr/dbt_artifacts/upload_dbt_artifacts.sql b/macros/edr/dbt_artifacts/upload_dbt_artifacts.sql index 19fa0382f..a9b7070dc 100644 --- a/macros/edr/dbt_artifacts/upload_dbt_artifacts.sql +++ b/macros/edr/dbt_artifacts/upload_dbt_artifacts.sql @@ -11,7 +11,7 @@ } %} - {% if not elementary.get_config_var('disable_dbt_columns_autoupload') %} + {% if elementary.get_config_var('columns_upload_strategy') != 'none' %} {% do model_upload_func_map.update({"dbt_columns": elementary.upload_dbt_columns}) %} {% endif %} diff --git a/macros/edr/dbt_artifacts/upload_dbt_columns.sql b/macros/edr/dbt_artifacts/upload_dbt_columns.sql index 667fd6ebe..c87bb4c04 100644 --- a/macros/edr/dbt_artifacts/upload_dbt_columns.sql +++ b/macros/edr/dbt_artifacts/upload_dbt_columns.sql @@ -35,7 +35,17 @@ {% set flattened_columns = [] %} {% for column_node in column_nodes.values() %} - {% if not elementary.get_config_var('upload_only_columns_with_descriptions') or column_node.get('description') %} + {% set config_dict = elementary.safe_get_with_default(column_node, 'config', {}) %} + {% set config_meta_dict = elementary.safe_get_with_default(config_dict, 'meta') %} + {% set meta_dict = elementary.safe_get_with_default(column_node, 'meta', {}) %} + {% set has_meta = config_meta_dict or meta_dict | length > 0 %} + + {% set config_tags = elementary.safe_get_with_default(config_dict, 'tags') %} + {% set global_tags = elementary.safe_get_with_default(column_node, 'tags') %} + {% set meta_tags = elementary.safe_get_with_default(meta_dict, 'tags') %} + {% set has_tags = config_tags or global_tags or meta_tags %} + + {% if elementary.get_config_var('columns_upload_strategy') == 'all' or column_node.get('description') or has_meta or has_tags %} {% set flat_column = elementary.flatten_column(table_node, column_node) %} {% do flattened_columns.append(flat_column) %} {% endif %} diff --git a/macros/edr/system/system_utils/get_config_var.sql b/macros/edr/system/system_utils/get_config_var.sql index b90f580d8..9373319d6 100644 --- a/macros/edr/system/system_utils/get_config_var.sql +++ b/macros/edr/system/system_utils/get_config_var.sql @@ -32,8 +32,7 @@ 'disable_freshness_results': false, 'disable_tests_results': false, 'disable_dbt_artifacts_autoupload': false, - 'disable_dbt_columns_autoupload': false, - 'upload_only_columns_with_descriptions': true, + 'columns_upload_strategy': 'enriched_only', 'disable_dbt_invocation_autoupload': false, 'disable_skipped_model_alerts': true, 'disable_skipped_test_alerts': true, From a20f1c3aad1f2155183672a4290d41d043bea4e5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:31:39 +0200 Subject: [PATCH 2/4] release 0.16.3 (#774) Co-authored-by: GitHub Actions --- README.md | 2 +- dbt_project.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c16df275b..efafa501f 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Available as self-hosted or Cloud service with premium features. ```yml packages.yml packages: - package: elementary-data/elementary - version: 0.16.2 + version: 0.16.3 ## Docs: https://docs.elementary-data.com ``` diff --git a/dbt_project.yml b/dbt_project.yml index 8be053c0f..08df8952b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: "elementary" -version: "0.16.2" +version: "0.16.3" require-dbt-version: [">=1.0.0", "<2.0.0"] From dd84c68a9d49d75a8c70e6f7516a168e9e8588ba Mon Sep 17 00:00:00 2001 From: Itamar Hartstein Date: Mon, 13 Jan 2025 19:09:27 +0200 Subject: [PATCH 3/4] change tests to run on python 3.9 (#776) --- .github/workflows/test-warehouse.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index d278ef2c7..e3998e005 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -84,7 +84,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.9" cache: "pip" - name: Install Spark requirements From d13b0fe1173a208ed6d5fcc59c35fbda4d317f70 Mon Sep 17 00:00:00 2001 From: Noy Arie Date: Thu, 30 Jan 2025 10:06:22 +0200 Subject: [PATCH 4/4] Ele 4006 support test description (#778) * add support for test description from 1.9 * remove default from meta --- macros/edr/dbt_artifacts/upload_dbt_tests.sql | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/macros/edr/dbt_artifacts/upload_dbt_tests.sql b/macros/edr/dbt_artifacts/upload_dbt_tests.sql index 8327f90cf..5a4ee5a85 100644 --- a/macros/edr/dbt_artifacts/upload_dbt_tests.sql +++ b/macros/edr/dbt_artifacts/upload_dbt_tests.sql @@ -55,13 +55,18 @@ {% set default_description = elementary.get_default_description(test_original_name, test_namespace) %} {% set config_meta_dict = elementary.safe_get_with_default(config_dict, 'meta', {}) %} - {% set meta_dict = {} %} - {% if default_description %} - {% set meta_dict = {'description': default_description} %} - {% endif %} - {% do meta_dict.update(elementary.safe_get_with_default(node_dict, 'meta', {})) %} + {% set meta_dict = elementary.safe_get_with_default(node_dict, 'meta', {}) %} {% do meta_dict.update(config_meta_dict) %} + {% set description = none %} + {% if dbt_version >= '1.9.0' and node_dict.get('description') %} + {% set description = node_dict.get('description') %} + {% elif meta_dict.get('description') %} + {% set description = meta_dict.pop('description') %} + {% elif default_description %} + {% set description = default_description %} + {% endif %} + {% set config_tags = elementary.safe_get_with_default(config_dict, 'tags', []) %} {% set global_tags = elementary.safe_get_with_default(node_dict, 'tags', []) %} {% set meta_tags = elementary.safe_get_with_default(meta_dict, 'tags', []) %} @@ -160,7 +165,7 @@ 'depends_on_macros': elementary.filter_none_and_sort(depends_on_dict.get('macros', [])), 'depends_on_nodes': elementary.filter_none_and_sort(depends_on_dict.get('nodes', [])), 'parent_model_unique_id': primary_test_model_id.data, - 'description': meta_dict.get('description'), + 'description': description, 'name': node_dict.get('name'), 'package_name': node_dict.get('package_name'), 'type': elementary.get_test_sub_type(original_file_path, test_namespace),