Changed the columns models.

'dbt_columns' now only contains columns that are documented in the dbt project.
'information_schema_columns' contains all the columns from all the schemas that your dbt project addresses.

integration_tests/dbt_project/models/schema.yml

@@ -12,6 +12,7 @@ models:
         description: This is a unique identifier for a customer
 
       - name: name
+        data_type: string
         description: Customer's name.
 
   - name: orders

macros/edr/dbt_artifacts/dbt_columns/get_information_schema_columns_materialized.sql

@@ -0,0 +1,16 @@
+{% macro get_information_schema_columns_materialized() %}
+  {% if var("sync", false) %}
+    {% do return("table") %}
+  {% endif %}
+  {% do return(adapter.dispatch("get_information_schema_columns_materialized", "elementary")()) %}
+{% endmacro %}
+
+
+{% macro default__get_information_schema_columns_materialized() %}
+  {% do return("view") %}
+{% endmacro %}
+
+
+{% macro bigquery__get_information_schema_columns_materialized() %}
+  {% do return("incremental") %}
+{% endmacro %}

macros/edr/dbt_artifacts/dbt_columns/get_dbt_columns_query.sql → macros/edr/dbt_artifacts/dbt_columns/get_information_schema_columns_query.sql

@@ -1,4 +1,4 @@
-{% macro get_dbt_columns_query(is_model_build_context=true) %}
+{% macro get_information_schema_columns_query(is_model_build_context=true) %}
   {% if is_model_build_context %}
     {% set get_relation = ref %}
   {% else %}
@@ -51,7 +51,7 @@
       {{ elementary.get_columns_in_project() }}
   ),
 
-  dbt_columns as (
+  information_schema_columns as (
       select col_info.*
       from tables_information tbl_info
       join columns_information col_info
@@ -62,5 +62,5 @@
   )
 
   select *
-  from dbt_columns
+  from information_schema_columns
 {% endmacro %}

macros/edr/dbt_artifacts/dbt_columns/upload_information_schema_columns.sql

@@ -0,0 +1,7 @@
+{% macro upload_information_schema_columns() %}
+  {% set relation = elementary.get_elementary_relation("information_schema_columns") %}
+  {% if execute and relation %}
+    {% set information_schema_columns_query = elementary.get_information_schema_columns_query(is_model_build_context=false) %}
+    {% do elementary.create_or_replace(false, relation, information_schema_columns_query) %}
+  {% endif %}
+{% endmacro %}

macros/edr/dbt_artifacts/upload_artifacts_to_table.sql

@@ -2,9 +2,11 @@
     {% set flatten_artifact_dicts = [] %}
     {% do elementary.file_log("[{}] Flattening the artifacts.".format(table_relation.identifier)) %}
     {% for artifact in artifacts %}
-        {% set flatten_artifact_dict = flatten_artifact_callback(artifact) %}
-        {% if flatten_artifact_dict is not none %}
-            {% do flatten_artifact_dicts.append(flatten_artifact_dict) %}
+        {% set flatten_artifact = flatten_artifact_callback(artifact) %}
+        {% if flatten_artifact is mapping %}
+            {% do flatten_artifact_dicts.append(flatten_artifact) %}
+        {% elif flatten_artifact is iterable %}
+            {% do flatten_artifact_dicts.extend(flatten_artifact) %}
         {% endif %}
     {% endfor %}
     {% do elementary.file_log("[{}] Flattened {} artifacts.".format(table_relation.identifier, flatten_artifact_dicts | length)) %}

macros/edr/dbt_artifacts/upload_dbt_artifacts.sql

@@ -8,6 +8,7 @@
       "dbt_metrics": elementary.upload_dbt_metrics,
       "dbt_exposures": elementary.upload_dbt_exposures,
       "dbt_seeds": elementary.upload_dbt_seeds,
+      "dbt_columns": elementary.upload_dbt_columns,
       }
     %}
 
@@ -26,8 +27,8 @@
         {% do elementary.file_log('[{}] Artifacts already ran.'.format(artifacts_model)) %}
       {% endif %}
     {% endfor %}
-    {% if elementary.get_dbt_columns_materialized() != "view" and elementary.get_config_var("upload_dbt_columns") %}
-      {% do elementary.upload_dbt_columns() %}  {# dbt_columns upload must come after other artifacts, as it is dependant on them #}
+    {% if elementary.get_information_schema_columns_materialized() != "view" and elementary.get_config_var("upload_information_schema_columns") %}
+      {% do elementary.upload_information_schema_columns() %}  {# information_schema_columns upload must come after other artifacts, as it is dependant on them #}
     {% endif %}
     {% do elementary.file_log("Uploaded dbt artifacts.") %}
   {% endif %}

macros/edr/dbt_artifacts/upload_dbt_columns.sql

@@ -0,0 +1,67 @@
+{%- macro upload_dbt_columns(should_commit=false, metadata_hashes=none) -%}
+    {% set relation = elementary.get_elementary_relation('dbt_columns') %}
+    {% if execute and relation %}
+        {% set tables = graph.nodes.values() | list + graph.sources.values() | list %}
+        {% do elementary.upload_artifacts_to_table(relation, tables, elementary.flatten_table_columns, should_commit=should_commit, metadata_hashes=metadata_hashes) %}
+    {%- endif -%}
+    {{- return('') -}}
+{%- endmacro -%}
+
+{% macro get_dbt_columns_empty_table_query() %}
+    {% set columns = [
+        ('unique_id', 'string'),
+        ('parent_unique_id', 'string'),
+        ('name', 'string'),
+        ('data_type', 'string'),
+        ('tags', 'long_string'),
+        ('meta', 'long_string'),
+        ('database_name', 'string'),
+        ('schema_name', 'string'),
+        ('table_name', 'string'),
+        ('description', 'long_string'),
+        ('generated_at', 'string'),
+        ('metadata_hash', 'string'),
+    ] %}
+    {% set dbt_columns_empty_table_query = elementary.empty_table(columns) %}
+    {{ return(dbt_columns_empty_table_query) }}
+{% endmacro %}
+
+{% macro flatten_table_columns(table_node) %}
+    {% set column_nodes = table_node.get("columns") %}
+    {% if not column_nodes %}
+        {% do return(none) %}
+    {% endif %}
+
+    {% set flattened_columns = [] %}
+    {% for column_node in column_nodes.values() %}
+        {% do flattened_columns.append(elementary.flatten_column(table_node, column_node)) %}
+    {% endfor %}
+    {% do return(flattened_columns) %}
+{% endmacro %}
+
+{% macro flatten_column(table_node, column_node) %}
+    {% set config_dict = elementary.safe_get_with_default(column_node, 'config', {}) %}
+    {% set config_meta_dict = elementary.safe_get_with_default(config_dict, 'meta', {}) %}
+    {% set meta_dict = elementary.safe_get_with_default(column_node, 'meta', {}) %}
+    {% do meta_dict.update(config_meta_dict) %}
+    {% set config_tags = elementary.safe_get_with_default(config_dict, 'tags', []) %}
+    {% set global_tags = elementary.safe_get_with_default(column_node, 'tags', []) %}
+    {% set meta_tags = elementary.safe_get_with_default(meta_dict, 'tags', []) %}
+    {% set tags = elementary.union_lists(config_tags, global_tags) %}
+    {% set tags = elementary.union_lists(tags, meta_tags) %}
+    {% set flatten_column_metadata_dict = {
+        'parent_unique_id': table_node.get('unique_id'),
+        'unique_id': "column.{}.{}".format(table_node.get('unique_id'), column_node.get('name')),
+        'name': column_node.get('name'),
+        'data_type': column_node.get('data_type'),
+        'tags': elementary.filter_none_and_sort(tags),
+        'meta': meta_dict,
+        'description': column_node.get('description'),
+        'database_name': table_node.get('database'),
+        'schema_name': table_node.get('schema'),
+        'table_name': table_node.get('alias'),
+        'generated_at': elementary.datetime_now_utc_as_string(),
+    } %}
+    {% do flatten_column_metadata_dict.update({"metadata_hash": elementary.get_artifact_metadata_hash(flatten_column_metadata_dict)}) %}
+    {% do return(flatten_column_metadata_dict) %}
+{% endmacro %}

macros/edr/system/system_utils/get_config_var.sql

@@ -55,7 +55,7 @@
     'calculate_failed_count': true,
     'tests_use_temp_tables': false,
     'collect_metrics': true,
-    'upload_dbt_columns': false,
+    'upload_information_schema_columns': false,
     'clean_elementary_temp_tables': true,
     'force_metrics_backfill': false,
     'ignore_small_changes': {

models/edr/dbt_artifacts/dbt_columns.sql

@@ -1,10 +1,12 @@
 {{
   config(
-    materialized=elementary.get_dbt_columns_materialized(),
-  )
+    materialized='incremental',
+    transient=False,
+    post_hook='{{ elementary.upload_dbt_columns() }}',
+    unique_key='unique_id',
+    on_schema_change='sync_all_columns',
+    full_refresh=elementary.get_config_var('elementary_full_refresh')
+    )
 }}
-{% if elementary.get_dbt_columns_materialized() == "view" %}
-    {{ elementary.get_dbt_columns_query() }}
-{% else %}
-    {{ elementary.get_empty_columns_from_information_schema_table() }}
-{% endif %}
+
+{{ elementary.get_dbt_columns_empty_table_query() }}

models/edr/dbt_artifacts/information_schema_columns.sql

@@ -0,0 +1,10 @@
+{{
+  config(
+    materialized=elementary.get_information_schema_columns_materialized(),
+  )
+}}
+{% if elementary.get_information_schema_columns_materialized() == "view" %}
+    {{ elementary.get_information_schema_columns_query() }}
+{% else %}
+    {{ elementary.get_empty_columns_from_information_schema_table() }}
+{% endif %}