Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix code to work with mashumaro 3.15 #11051

Open
wants to merge 46 commits into
base: main
Choose a base branch
from
Open
Changes from 1 commit
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
261e124
Fix NodeVersion definition, make_semantic_model utility
gshank Nov 25, 2024
37be156
Bump up mashumaro version again
gshank Nov 25, 2024
52fa7c0
Try pinning to >=3.15,<4.0
gshank Nov 26, 2024
354af6e
specify dbt-adapters branch
gshank Nov 26, 2024
7876edb
Update v12.json schema
gshank Nov 26, 2024
c0ea26e
validate in "update_from"
gshank Nov 26, 2024
dfca4e5
error classes
gshank Nov 26, 2024
b81632e
Comments
gshank Nov 27, 2024
e47bed3
Put back test_disabled_model.py
gshank Nov 27, 2024
9d6daa1
Use older mashumaro
gshank Nov 27, 2024
baf42c4
Again
gshank Nov 27, 2024
9df05a8
Put back default for validate in _update_from_config
gshank Nov 27, 2024
524ce54
Fix NodeVersion in artifacts test
gshank Nov 27, 2024
df23c7d
Skip a couple of tests until mash 3.15
gshank Nov 30, 2024
42a1afb
Make specialized 'calculate_node_config' methods
gshank Nov 30, 2024
9c6883b
Change order of Union in source definitions partitions
gshank Dec 2, 2024
25899ca
Change order of external.partitions schema elements
gshank Dec 2, 2024
5b4af17
Changie
gshank Dec 2, 2024
86791e0
Remove unnecessary "base" parameter
gshank Dec 2, 2024
d5616de
passes unit tests
gshank Dec 2, 2024
4554eb3
comment
gshank Dec 2, 2024
b4f72e4
Use dbt-common branch
gshank Dec 3, 2024
54adabe
Fix a few tests, remove unnecessary finalize_and_validate calls
gshank Dec 3, 2024
bbfc6e6
rename config to context_config
gshank Dec 3, 2024
1afb2bb
fix hooks before validation
gshank Dec 3, 2024
7d78b47
Bump mashumaro again
gshank Dec 3, 2024
1c709d2
Remove skips from tests
gshank Dec 3, 2024
13105cd
fix expected_manifest version: "2"
gshank Dec 3, 2024
3f7ee0e
Remove unnecessary ConfigSource
gshank Dec 3, 2024
c16cbe0
Rename ContextConfig ConfigBuilder
gshank Dec 3, 2024
c0fd389
rename model_configs resource_configs
gshank Dec 3, 2024
049418e
More cleanup
gshank Dec 3, 2024
e6fc9b0
Remove unnecessary mangle_hooks, other comments and cleanup
gshank Dec 3, 2024
31ce086
More comments
gshank Dec 4, 2024
923b8b6
add test for serialization of ExternalPartitions in ExternalTable
gshank Dec 9, 2024
1a30f64
Update test_serialization to simplify for mashumaro, add test_graph_s…
gshank Dec 10, 2024
c137041
Merge branch 'main' into mashumaro_fixes
gshank Dec 10, 2024
f011789
Remove base parameter from get_config_for
gshank Dec 11, 2024
0fb57ee
Update return type of generate_node_config
gshank Dec 11, 2024
e919144
Merge branch 'main' into mashumaro_fixes
gshank Dec 12, 2024
ebc2359
Remove requirements for 3.15
gshank Dec 12, 2024
4585bd0
formatting
gshank Dec 16, 2024
715ad2f
dev-requirements
gshank Dec 19, 2024
d1274b1
Merge branch 'main' into mashumaro_fixes
gshank Dec 19, 2024
5f94de5
Put back Union order changes, use dbtClassMixin in test_serialization.py
gshank Dec 19, 2024
a0e32c8
Merge branch 'main' into mashumaro_fixes
gshank Jan 9, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Comments
gshank committed Nov 27, 2024
commit b81632e0ae66735abfd5fbe565f4bc50b37c46f2
43 changes: 25 additions & 18 deletions core/dbt/context/context_config.py
Original file line number Diff line number Diff line change
@@ -139,6 +139,7 @@ def _update_from_config(
@abstractmethod
def initial_result(self, resource_type: NodeType, base: bool) -> T: ...

# BaseContextConfigGenerator
def calculate_node_config(
self,
config_call_dict: Dict[str, Any],
@@ -150,29 +151,28 @@ def calculate_node_config(
) -> BaseConfig:
own_config = self.get_node_project(project_name)

result = self.initial_result(resource_type=resource_type, base=base)
# creates "default" config object ("cls.from_dict({})")
config_obj = self.initial_result(resource_type=resource_type, base=base)

project_configs = self._project_configs(own_config, fqn, resource_type)
for fqn_config in project_configs:
result = self._update_from_config(result, fqn_config)
config_obj = self._update_from_config(config_obj, fqn_config)

# When schema files patch config, it has lower precedence than
# config in the models (config_call_dict), so we add the patch_config_dict
# before the config_call_dict
if patch_config_dict:
result = self._update_from_config(result, patch_config_dict)
config_obj = self._update_from_config(config_obj, patch_config_dict)

# config_calls are created in the 'experimental' model parser and
# the ParseConfigObject (via add_config_call)
result = self._update_from_config(result, config_call_dict)
config_obj = self._update_from_config(config_obj, config_call_dict)

if own_config.project_name != self._active_project.project_name:
for fqn_config in self._active_project_configs(fqn, resource_type):
result = self._update_from_config(result, fqn_config)
config_obj = self._update_from_config(config_obj, fqn_config)

# this is mostly impactful in the snapshot config case
# TODO CT-211
return result # type: ignore[return-value]
return config_obj # type: ignore[return-value]

@abstractmethod
def calculate_node_config_dict(
@@ -223,6 +223,7 @@ def translate_hook_names(self, project_dict):
project_dict["post-hook"] = project_dict.pop("post_hook")
return project_dict

# ContextConfigGenerator
def calculate_node_config_dict(
self,
config_call_dict: Dict[str, Any],
@@ -232,7 +233,9 @@ def calculate_node_config_dict(
base: bool,
patch_config_dict: Optional[dict] = None,
) -> Dict[str, Any]:
config = self.calculate_node_config(

# calls BaseContextConfigGenerator.calculate_node_config
config_obj = self.calculate_node_config(
config_call_dict=config_call_dict,
fqn=fqn,
resource_type=resource_type,
@@ -241,17 +244,20 @@ def calculate_node_config_dict(
patch_config_dict=patch_config_dict,
)
try:
finalized = config.finalize_and_validate()
# Call "finalize_and_validate" on the config obj
finalized = config_obj.finalize_and_validate()
# THEN return a dictionary!!! Why!!
return finalized.to_dict(omit_none=True)
except ValidationError as exc:
# we got a ValidationError - probably bad types in config()
raise SchemaConfigError(exc, node=config) from exc
raise SchemaConfigError(exc, node=config_obj) from exc


class UnrenderedConfigGenerator(BaseContextConfigGenerator[Dict[str, Any]]):
def get_config_source(self, project: Project) -> ConfigSource:
return UnrenderedConfig(project)

# UnrenderedConfigGenerator
def calculate_node_config_dict(
self,
config_call_dict: Dict[str, Any],
@@ -261,7 +267,8 @@ def calculate_node_config_dict(
base: bool,
patch_config_dict: Optional[dict] = None,
) -> Dict[str, Any]:
# TODO CT-211

# calls BaseContextConfigGenerator.calculate_node_config
return self.calculate_node_config(
config_call_dict=config_call_dict,
fqn=fqn,
@@ -270,6 +277,7 @@ def calculate_node_config_dict(
base=base,
patch_config_dict=patch_config_dict,
) # type: ignore[return-value]
# Note: this returns a config_obj, NOT a dictionary

def initial_result(self, resource_type: NodeType, base: bool) -> Dict[str, Any]:
return {}
@@ -308,6 +316,7 @@ def add_unrendered_config_call(self, opts: Dict[str, Any]) -> None:
# Cannot perform complex merge behaviours on unrendered configs as they may not be appropriate types.
self._unrendered_config_call_dict.update(opts)

# ContextConfig
def build_config_dict(
self,
base: bool = False,
@@ -316,12 +325,10 @@ def build_config_dict(
patch_config_dict: Optional[dict] = None,
) -> Dict[str, Any]:
if rendered:
# TODO CT-211
src = ContextConfigGenerator(self._active_project) # type: ignore[var-annotated]
config_generator = ContextConfigGenerator(self._active_project) # type: ignore[var-annotated]
config_call_dict = self._config_call_dict
else:
# TODO CT-211
src = UnrenderedConfigGenerator(self._active_project) # type: ignore[assignment]
else: # unrendered
config_generator = UnrenderedConfigGenerator(self._active_project) # type: ignore[assignment]

# preserve legacy behaviour - using unreliable (potentially rendered) _config_call_dict
if get_flags().state_modified_compare_more_unrendered_values is False:
@@ -334,7 +341,7 @@ def build_config_dict(
else:
config_call_dict = self._unrendered_config_call_dict

return src.calculate_node_config_dict(
return config_generator.calculate_node_config_dict(
config_call_dict=config_call_dict,
fqn=self._fqn,
resource_type=self._resource_type,