diff --git a/.github/workflows/ossar-analysis.yml b/.github/workflows/ossar-analysis.yml index c33f318d..1d4435f4 100644 --- a/.github/workflows/ossar-analysis.yml +++ b/.github/workflows/ossar-analysis.yml @@ -26,16 +26,6 @@ jobs: - run: git checkout HEAD^2 if: ${{ github.event_name == 'pull_request' }} - # Ensure a compatible version of dotnet is installed. - # The [Microsoft Security Code Analysis CLI](https://aka.ms/mscadocs) is built with dotnet v3.1.201. - # A version greater than or equal to v3.1.201 of dotnet must be installed on the agent in order to run this action. - # Remote agents already have a compatible version of dotnet installed and this step may be skipped. - # For local agents, ensure dotnet version 3.1.201 or later is installed by including this action: - # - name: Install .NET - # uses: actions/setup-dotnet@v1 - # with: - # dotnet-version: '3.1.x' - # Run open source static analysis tools - name: Run OSSAR uses: github/ossar-action@v1 diff --git a/PARAMETER_DEPENDENCY_SERIALIZATION.md b/PARAMETER_DEPENDENCY_SERIALIZATION.md new file mode 100644 index 00000000..82bf0383 --- /dev/null +++ b/PARAMETER_DEPENDENCY_SERIALIZATION.md @@ -0,0 +1,216 @@ +# Parameter Dependency Serialization + +This document explains how to serialize and deserialize `Parameter` objects that have dependencies. + +## Overview + +Parameters with dependencies can now be serialized to dictionaries (and JSON) while preserving their dependency relationships. After deserialization, the dependencies are automatically reconstructed using the `serializer_id` attribute to match parameters, with `unique_name` attribute being used as a fallback. + +## Key Features + +- **Automatic dependency serialization**: Dependency expressions and maps are automatically saved during serialization +- **Reliable dependency resolution**: Dependencies are resolved using stable `serializer_id` attributes with `unique_name` as fallback after deserialization +- **Order-independent loading**: Parameters can be loaded in any order thanks to the reliable ID system +- **Bulk dependency resolution**: Utility functions help resolve all dependencies at once +- **JSON compatibility**: Full support for JSON serialization/deserialization +- **Backward compatibility**: Existing code using `unique_name` continues to work as fallback + +## Usage + +### Basic Serialization/Deserialization + +```python +import json +from easyscience import Parameter, global_object +from easyscience.variable.parameter_dependency_resolver import resolve_all_parameter_dependencies + +# Create parameters with dependencies +a = Parameter(name="a", value=2.0, unit="m", min=0, max=10) +b = Parameter.from_dependency( + name="b", + dependency_expression="2 * a", + dependency_map={"a": a}, + unit="m" +) + +# Serialize to dictionary and save to file +params_dict = {"a": a.as_dict(), "b": b.as_dict()} +with open("parameters.json", "w") as f: + json.dump(params_dict, f, indent=2, default=str) + +print("Parameters saved to parameters.json") +``` + +In a new Python session: + +```python +import json +from easyscience import Parameter, global_object +from easyscience.variable.parameter_dependency_resolver import resolve_all_parameter_dependencies + +# Load parameters from file +with open("parameters.json", "r") as f: + params_dict = json.load(f) + +# Clear global map (simulate new environment) +global_object.map._clear() + +# Deserialize parameters +new_a = Parameter.from_dict(params_dict["a"]) +new_b = Parameter.from_dict(params_dict["b"]) + +# Resolve dependencies +resolve_all_parameter_dependencies({"a": new_a, "b": new_b}) + +# Dependencies are now working +new_a.value = 5.0 +print(new_b.value) # Will be 10.0 (2 * 5.0) +``` + +### JSON Serialization + +```python +import json + +# Serialize to JSON +param_dict = parameter.as_dict() +json_str = json.dumps(param_dict, default=str) + +# Deserialize from JSON +loaded_dict = json.loads(json_str) +new_param = Parameter.from_dict(loaded_dict) + +# Resolve dependencies +resolve_all_parameter_dependencies(new_param) +``` + +### Bulk Operations + +```python +from easyscience.variable.parameter_dependency_resolver import get_parameters_with_pending_dependencies + +# Create multiple parameters with dependencies +params = create_parameter_hierarchy() # Your function + +# Serialize all +serialized = {name: param.as_dict() for name, param in params.items()} + +# Clear and deserialize +global_object.map._clear() +new_params = {name: Parameter.from_dict(d) for name, d in serialized.items()} + +# Check which parameters have pending dependencies +pending = get_parameters_with_pending_dependencies(new_params) +print(f"Found {len(pending)} parameters with pending dependencies") + +# Resolve all at once +resolve_all_parameter_dependencies(new_params) +``` + +## Implementation Details + +### Serialization + +During serialization, the following additional fields are added to dependent parameters: + +- `_dependency_string`: The original dependency expression +- `_dependency_map_serializer_ids`: A mapping of dependency keys to stable dependency IDs (preferred) +- `_dependency_map_unique_names`: A mapping of dependency keys to unique names (fallback) +- `__serializer_id`: The parameter's own unique dependency ID +- `_independent`: Boolean flag indicating if the parameter is dependent + +### Deserialization + +During deserialization: + +1. Parameters are created normally but marked as independent temporarily +2. Dependency information is stored in `_pending_dependency_string`, `_pending_dependency_map_serializer_ids`, and `_pending_dependency_map_unique_names` attributes +3. The parameter's own `__serializer_id` is restored from serialized data +4. After all parameters are loaded, `resolve_all_parameter_dependencies()` establishes the dependency relationships using dependency IDs first, then unique names as fallback + +### Dependency Resolution + +The dependency resolution process: + +1. Scans for parameters with pending dependencies +2. First attempts to look up dependency objects by their stable `serializer_id` +3. Falls back to `unique_name` lookup in the global map if serializer_id is not available +4. Calls `make_dependent_on()` to establish the dependency relationship +5. Cleans up temporary attributes + +This dual-strategy approach ensures reliable dependency resolution regardless of parameter loading order while maintaining backward compatibility. + +## Error Handling + +The system provides detailed error messages for common issues: + +- Missing dependencies (parameter with required unique_name not found) +- Invalid dependency expressions +- Circular dependency detection + +## Utility Functions + +### `resolve_all_parameter_dependencies(obj)` + +Recursively finds all Parameter objects with pending dependencies and resolves them. + +**Parameters:** +- `obj`: Object to search for Parameters (can be Parameter, list, dict, or complex object) + +**Returns:** +- None (modifies parameters in place) + +**Raises:** +- `ValueError`: If dependency resolution fails + +### `get_parameters_with_pending_dependencies(obj)` + +Finds all Parameter objects that have pending dependencies. + +**Parameters:** +- `obj`: Object to search for Parameters + +**Returns:** +- `List[Parameter]`: List of parameters with pending dependencies + +## Best Practices + +1. **Always resolve dependencies after deserialization**: Use `resolve_all_parameter_dependencies()` after loading serialized parameters + +2. **Handle the global map carefully**: The global map must contain all referenced parameters for dependency resolution to work + +3. **Use unique names for cross-references**: When creating dependency expressions that reference other parameters, consider using unique names with quotes: `'Parameter_0'` + +4. **Error handling**: Wrap dependency resolution in try-catch blocks for robust error handling + +5. **Bulk operations**: For complex object hierarchies, use the utility functions to handle all parameters at once + +6. **Reliable ordering**: With the new dependency ID system, parameters can be loaded in any order without affecting dependency resolution + +7. **Access dependency ID**: Use `parameter.serializer_id` to access the stable ID for debugging or manual cross-referencing + +## Example: Complex Hierarchy + +```python +def save_model(model): + \"\"\"Save a model with parameter dependencies to JSON.\"\"\" + model_dict = model.as_dict() + with open('model.json', 'w') as f: + json.dump(model_dict, f, indent=2, default=str) + +def load_model(filename): + \"\"\"Load a model from JSON and resolve dependencies.\"\"\" + global_object.map._clear() # Start fresh + + with open(filename) as f: + model_dict = json.load(f) + + model = Model.from_dict(model_dict) + + # Resolve all parameter dependencies + resolve_all_parameter_dependencies(model) + + return model +``` + +This system ensures that complex parameter hierarchies with dependencies can be reliably serialized and reconstructed while maintaining their behavioral relationships. \ No newline at end of file diff --git a/src/easyscience/base_classes/based_base.py b/src/easyscience/base_classes/based_base.py index de72286a..b575b486 100644 --- a/src/easyscience/base_classes/based_base.py +++ b/src/easyscience/base_classes/based_base.py @@ -5,6 +5,8 @@ # © 2021-2025 Contributors to the EasyScience project BasedBase: temp = self.as_dict(skip=['unique_name']) new_obj = self.__class__.from_dict(temp) return new_obj + + def as_dict(self, skip: Optional[List[str]] = None) -> Dict[str, Any]: + """ + Convert an object into a full dictionary using `SerializerDict`. + This is a shortcut for ```obj.encode(encoder=SerializerDict)``` + + :param skip: List of field names as strings to skip when forming the dictionary + :return: encoded object containing all information to reform an EasyScience object. + """ + # extend skip to include unique_name by default + if skip is None: + skip = [] + if 'unique_name' not in skip: + skip.append('unique_name') + return super().as_dict(skip=skip) diff --git a/src/easyscience/global_object/map.py b/src/easyscience/global_object/map.py index 53a3aac4..0f29c1bc 100644 --- a/src/easyscience/global_object/map.py +++ b/src/easyscience/global_object/map.py @@ -261,7 +261,7 @@ def is_connected(self, vertices_encountered=None, start_vertex=None) -> bool: return False def _clear(self): - """Reset the map to an empty state.""" + """Reset the map to an empty state. Only to be used for testing""" for vertex in self.vertices(): self.prune(vertex) gc.collect() diff --git a/src/easyscience/variable/descriptor_number.py b/src/easyscience/variable/descriptor_number.py index d798f0a7..7d35ca0b 100644 --- a/src/easyscience/variable/descriptor_number.py +++ b/src/easyscience/variable/descriptor_number.py @@ -1,6 +1,7 @@ from __future__ import annotations import numbers +import uuid from typing import Any from typing import Dict from typing import List @@ -52,6 +53,7 @@ def __init__( url: Optional[str] = None, display_name: Optional[str] = None, parent: Optional[Any] = None, + **kwargs: Any, # Additional keyword arguments (used for (de)serialization) ): """Constructor for the DescriptorNumber class @@ -67,6 +69,10 @@ def __init__( """ self._observers: List[DescriptorNumber] = [] + # Extract serializer_id if provided during deserialization + if '__serializer_id' in kwargs: + self.__serializer_id = kwargs.pop('__serializer_id') + if not isinstance(value, numbers.Number) or isinstance(value, bool): raise TypeError(f'{value=} must be a number') if variance is not None: @@ -113,10 +119,14 @@ def from_scipp(cls, name: str, full_value: Variable, **kwargs) -> DescriptorNumb def _attach_observer(self, observer: DescriptorNumber) -> None: """Attach an observer to the descriptor.""" self._observers.append(observer) + if not hasattr(self, '_DescriptorNumber__serializer_id'): + self.__serializer_id = str(uuid.uuid4()) def _detach_observer(self, observer: DescriptorNumber) -> None: """Detach an observer from the descriptor.""" self._observers.remove(observer) + if not self._observers: + del self.__serializer_id def _notify_observers(self) -> None: """Notify all observers of a change.""" @@ -326,6 +336,8 @@ def as_dict(self, skip: Optional[List[str]] = None) -> Dict[str, Any]: raw_dict['value'] = self._scalar.value raw_dict['unit'] = str(self._scalar.unit) raw_dict['variance'] = self._scalar.variance + if hasattr(self, '_DescriptorNumber__serializer_id'): + raw_dict['__serializer_id'] = self.__serializer_id return raw_dict def __add__(self, other: Union[DescriptorNumber, numbers.Number]) -> DescriptorNumber: diff --git a/src/easyscience/variable/parameter.py b/src/easyscience/variable/parameter.py index 94415aaa..da5dfe59 100644 --- a/src/easyscience/variable/parameter.py +++ b/src/easyscience/variable/parameter.py @@ -11,6 +11,7 @@ import weakref from typing import Any from typing import Dict +from typing import List from typing import Optional from typing import Union @@ -33,7 +34,8 @@ class Parameter(DescriptorNumber): """ # Used by serializer - _REDIRECT = DescriptorNumber._REDIRECT + # We copy the parent's _REDIRECT and modify it to avoid altering the parent's class dict + _REDIRECT = DescriptorNumber._REDIRECT.copy() _REDIRECT['callback'] = None def __init__( @@ -51,6 +53,7 @@ def __init__( display_name: Optional[str] = None, callback: property = property(), parent: Optional[Any] = None, + **kwargs: Any, # Additional keyword arguments (used for (de)serialization) ): """ This class is an extension of a `DescriptorNumber`. Where the descriptor was for static @@ -72,6 +75,11 @@ def __init__( .. note:: Undo/Redo functionality is implemented for the attributes `value`, `variance`, `error`, `min`, `max`, `bounds`, `fixed`, `unit` """ # noqa: E501 + # Extract and ignore serialization-specific fields from kwargs + kwargs.pop('_dependency_string', None) + kwargs.pop('_dependency_map_serializer_ids', None) + kwargs.pop('_independent', None) + if not isinstance(min, numbers.Number): raise TypeError('`min` must be a number') if not isinstance(max, numbers.Number): @@ -101,6 +109,7 @@ def __init__( url=url, display_name=display_name, parent=parent, + **kwargs, # Additional keyword arguments (used for (de)serialization) ) self._callback = callback # Callback is used by interface to link to model @@ -123,7 +132,11 @@ def from_dependency( :param kwargs: Additional keyword arguments to pass to the Parameter constructor. :return: A new dependent Parameter object. """ # noqa: E501 - parameter = cls(name=name, value=0.0, unit='', variance=0.0, min=-np.inf, max=np.inf, **kwargs) + # Set default values for required parameters for the constructor, they get overwritten by the dependency anyways + default_kwargs = {'value': 0.0, 'unit': '', 'variance': 0.0, 'min': -np.inf, 'max': np.inf} + # Update with user-provided kwargs, to avoid errors. + default_kwargs.update(kwargs) + parameter = cls(name=name, **default_kwargs) parameter.make_dependent_on(dependency_expression=dependency_expression, dependency_map=dependency_map) return parameter @@ -554,6 +567,25 @@ def free(self) -> bool: def free(self, value: bool) -> None: self.fixed = not value + def as_dict(self, skip: Optional[List[str]] = None) -> Dict[str, Any]: + """Overwrite the as_dict method to handle dependency information.""" + raw_dict = super().as_dict(skip=skip) + + # Add dependency information for dependent parameters + if not self._independent: + # Save the dependency expression + raw_dict['_dependency_string'] = self._clean_dependency_string + + # Mark that this parameter is dependent + raw_dict['_independent'] = self._independent + + # Convert dependency_map to use serializer_ids + raw_dict['_dependency_map_serializer_ids'] = {} + for key, obj in self._dependency_map.items(): + raw_dict['_dependency_map_serializer_ids'][key] = obj._DescriptorNumber__serializer_id + + return raw_dict + def _revert_dependency(self, skip_detach=False) -> None: """ Revert the dependency to the old dependency. This is used when an error is raised during setting the dependency. @@ -601,6 +633,31 @@ def _process_dependency_unique_names(self, dependency_expression: str): ) # noqa: E501 self._clean_dependency_string = clean_dependency_string + @classmethod + def from_dict(cls, obj_dict: dict) -> 'Parameter': + """ + Custom deserialization to handle parameter dependencies. + Override the parent method to handle dependency information. + """ + # Extract dependency information before creating the parameter + raw_dict = obj_dict.copy() # Don't modify the original dict + dependency_string = raw_dict.pop('_dependency_string', None) + dependency_map_serializer_ids = raw_dict.pop('_dependency_map_serializer_ids', None) + is_independent = raw_dict.pop('_independent', True) + # Note: Keep _serializer_id in the dict so it gets passed to __init__ + + # Create the parameter using the base class method (serializer_id is now handled in __init__) + param = super().from_dict(raw_dict) + + # Store dependency information for later resolution + if not is_independent: + param._pending_dependency_string = dependency_string + param._pending_dependency_map_serializer_ids = dependency_map_serializer_ids + # Keep parameter as independent initially - will be made dependent after all objects are loaded + param._independent = True + + return param + def __copy__(self) -> Parameter: new_obj = super().__copy__() new_obj._callback = property() @@ -928,9 +985,48 @@ def __abs__(self) -> Parameter: new_full_value = abs(self.full_value) combinations = [abs(self.min), abs(self.max)] if self.min < 0 and self.max > 0: - combinations.append(0) + combinations.append(0.0) min_value = min(combinations) max_value = max(combinations) parameter = Parameter.from_scipp(name=self.name, full_value=new_full_value, min=min_value, max=max_value) parameter.name = parameter.unique_name return parameter + + def resolve_pending_dependencies(self) -> None: + """Resolve pending dependencies after deserialization. + + This method should be called after all parameters have been deserialized + to establish dependency relationships using serializer_ids. + """ + if hasattr(self, '_pending_dependency_string'): + dependency_string = self._pending_dependency_string + dependency_map = {} + + if hasattr(self, '_pending_dependency_map_serializer_ids'): + dependency_map_serializer_ids = self._pending_dependency_map_serializer_ids + + # Build dependency_map by looking up objects by serializer_id + for key, serializer_id in dependency_map_serializer_ids.items(): + dep_obj = self._find_parameter_by_serializer_id(serializer_id) + if dep_obj is not None: + dependency_map[key] = dep_obj + else: + raise ValueError(f"Cannot find parameter with serializer_id '{serializer_id}'") + + # Establish the dependency relationship + try: + self.make_dependent_on(dependency_expression=dependency_string, dependency_map=dependency_map) + except Exception as e: + raise ValueError(f"Error establishing dependency '{dependency_string}': {e}") + + # Clean up temporary attributes + delattr(self, '_pending_dependency_string') + delattr(self, '_pending_dependency_map_serializer_ids') + + def _find_parameter_by_serializer_id(self, serializer_id: str) -> Optional['DescriptorNumber']: + """Find a parameter by its serializer_id from all parameters in the global map.""" + for obj in self._global_object.map._store.values(): + if isinstance(obj, DescriptorNumber) and hasattr(obj, '_DescriptorNumber__serializer_id'): + if obj._DescriptorNumber__serializer_id == serializer_id: + return obj + return None diff --git a/src/easyscience/variable/parameter_dependency_resolver.py b/src/easyscience/variable/parameter_dependency_resolver.py new file mode 100644 index 00000000..e6b7cbd6 --- /dev/null +++ b/src/easyscience/variable/parameter_dependency_resolver.py @@ -0,0 +1,147 @@ +# SPDX-FileCopyrightText: 2025 EasyScience contributors +# SPDX-License-Identifier: BSD-3-Clause +# © 2021-2025 Contributors to the EasyScience project None: + """ + Recursively find all Parameter objects in an object hierarchy and resolve their pending dependencies. + + This function should be called after deserializing a complex object that contains Parameters + with dependencies to ensure all dependency relationships are properly established. + + :param obj: The object to search for Parameters (can be a single Parameter, list, dict, or complex object) + """ + + def _collect_parameters(item: Any, parameters: List[Parameter]) -> None: + """Recursively collect all Parameter objects from an item.""" + if isinstance(item, Parameter): + parameters.append(item) + elif isinstance(item, dict): + for value in item.values(): + _collect_parameters(value, parameters) + elif isinstance(item, (list, tuple)): + for element in item: + _collect_parameters(element, parameters) + elif hasattr(item, '__dict__'): + # Check instance attributes + for attr_name, attr_value in item.__dict__.items(): + if not attr_name.startswith('_'): # Skip private attributes + _collect_parameters(attr_value, parameters) + + # Check class properties (descriptors like Parameter instances) + for attr_name in dir(type(item)): + if not attr_name.startswith('_'): # Skip private attributes + class_attr = getattr(type(item), attr_name, None) + if isinstance(class_attr, property): + try: + attr_value = getattr(item, attr_name) + _collect_parameters(attr_value, parameters) + except (AttributeError, Exception): + # log the exception + print(f"Error accessing property '{attr_name}' of {item}") + # Skip properties that can't be accessed + continue + + # Collect all parameters + all_parameters = [] + _collect_parameters(obj, all_parameters) + + # Resolve dependencies for all parameters that have pending dependencies + resolved_count = 0 + error_count = 0 + errors = [] + + for param in all_parameters: + if hasattr(param, '_pending_dependency_string'): + try: + param.resolve_pending_dependencies() + resolved_count += 1 + except Exception as e: + error_count += 1 + serializer_id = getattr(param, '_DescriptorNumber__serializer_id', 'unknown') + errors.append( + f"Failed to resolve dependencies for parameter '{param.name}'" + f" (unique_name: '{param.unique_name}', serializer_id: '{serializer_id}'): {e}" + ) + + # Report results + if resolved_count > 0: + print(f'Successfully resolved dependencies for {resolved_count} parameter(s).') + + if error_count > 0: + error_message = f'Failed to resolve dependencies for {error_count} parameter(s):\n' + '\n'.join(errors) + raise ValueError(error_message) + + +def get_parameters_with_pending_dependencies(obj: Any) -> List[Parameter]: + """ + Find all Parameter objects in an object hierarchy that have pending dependencies. + + :param obj: The object to search for Parameters + :return: List of Parameters with pending dependencies + """ + parameters_with_pending = [] + + def _collect_pending_parameters(item: Any) -> None: + """Recursively collect all Parameter objects with pending dependencies.""" + if isinstance(item, Parameter): + if hasattr(item, '_pending_dependency_string'): + parameters_with_pending.append(item) + elif isinstance(item, dict): + for value in item.values(): + _collect_pending_parameters(value) + elif isinstance(item, (list, tuple)): + for element in item: + _collect_pending_parameters(element) + elif hasattr(item, '__dict__'): + # Check instance attributes + for attr_name, attr_value in item.__dict__.items(): + if not attr_name.startswith('_'): # Skip private attributes + _collect_pending_parameters(attr_value) + + # Check class properties (descriptors like Parameter instances) + for attr_name in dir(type(item)): + if not attr_name.startswith('_'): # Skip private attributes + class_attr = getattr(type(item), attr_name, None) + if isinstance(class_attr, property): + try: + attr_value = getattr(item, attr_name) + _collect_pending_parameters(attr_value) + except (AttributeError, Exception): + # log the exception + print(f"Error accessing property '{attr_name}' of {item}") + # Skip properties that can't be accessed + continue + + _collect_pending_parameters(obj) + return parameters_with_pending + + +def deserialize_and_resolve_parameters(params_data: Dict[str, Dict[str, Any]]) -> Dict[str, Parameter]: + """ + Deserialize parameters from a dictionary and resolve their dependencies. + + This is a convenience function that combines Parameter.from_dict() deserialization + with dependency resolution in a single call. + + :param params_data: Dictionary mapping parameter names to their serialized data + :return: Dictionary mapping parameter names to deserialized Parameters with resolved dependencies + """ + # Deserialize all parameters first + new_params = {} + for name, data in params_data.items(): + new_params[name] = Parameter.from_dict(data) + + # Resolve all dependencies + resolve_all_parameter_dependencies(new_params) + + return new_params diff --git a/tests/unit_tests/base_classes/test_obj_base.py b/tests/unit_tests/base_classes/test_obj_base.py index 8a572eef..f004e9af 100644 --- a/tests/unit_tests/base_classes/test_obj_base.py +++ b/tests/unit_tests/base_classes/test_obj_base.py @@ -157,7 +157,6 @@ def test_ObjBase_as_dict(clear, setup_pars: dict): "@class": "ObjBase", "@version": easyscience.__version__, "name": "test", - "unique_name": "ObjBase_0", "par1": { "@module": Parameter.__module__, "@class": Parameter.__name__, diff --git a/tests/unit_tests/variable/test_parameter_dependency_serialization.py b/tests/unit_tests/variable/test_parameter_dependency_serialization.py new file mode 100644 index 00000000..8cf7abc0 --- /dev/null +++ b/tests/unit_tests/variable/test_parameter_dependency_serialization.py @@ -0,0 +1,500 @@ +import pytest +import json +from copy import deepcopy +from unittest.mock import Mock + +from easyscience import Parameter, global_object +from easyscience.variable.parameter_dependency_resolver import resolve_all_parameter_dependencies +from easyscience.variable.parameter_dependency_resolver import get_parameters_with_pending_dependencies +from easyscience.variable.parameter_dependency_resolver import deserialize_and_resolve_parameters + + +class TestParameterDependencySerialization: + + @pytest.fixture + def clear_global_map(self): + """This fixture pattern: + - Clears the map before each test (clean slate) + - Yields control to the test + - Clears the map after each test (cleanup) + + Dependency serialization tests require more robust + setup-yield-cleanup pattern because they involve complex + object lifecycles with serialization, deserialization, + and dependency resolution that are particularly sensitive + to global state contamination. + """ + # The global map uses weakref.WeakValueDictionary() for object storage, + # but also maintains strong references in __type_dict that need explicit cleanup. + global_object.map._clear() + yield + # final cleanup after test + global_object.map._clear() + + def test_independent_parameter_serialization(self, clear_global_map): + """Test that independent parameters serialize normally without dependency info.""" + param = Parameter(name="test", value=5.0, unit="m", min=0, max=10) + + # Serialize + serialized = param.as_dict() + + # Should not contain dependency fields + assert '_dependency_string' not in serialized + assert '_dependency_map_serializer_ids' not in serialized + assert '_independent' not in serialized + + # Deserialize + global_object.map._clear() + new_param = Parameter.from_dict(serialized) + + # Should be identical + assert new_param.name == param.name + assert new_param.value == param.value + assert new_param.unit == param.unit + assert new_param.independent is True + + def test_dependent_parameter_serialization(self, clear_global_map): + """Test serialization of parameters with dependencies.""" + # Create independent parameter + a = Parameter(name="a", value=2.0, unit="m", min=0, max=10) + + # Create dependent parameter + b = Parameter.from_dependency( + name="b", + dependency_expression="2 * a", + dependency_map={"a": a}, + unit="m" + ) + + # Serialize dependent parameter + serialized = b.as_dict() + + # Should contain dependency information + assert serialized['_dependency_string'] == "2 * a" + assert serialized['_dependency_map_serializer_ids'] == {"a": a._DescriptorNumber__serializer_id} + assert serialized['_independent'] is False + + # Deserialize + global_object.map._clear() + new_b = Parameter.from_dict(serialized) + + # Should have pending dependency info + assert hasattr(new_b, '_pending_dependency_string') + assert new_b._pending_dependency_string == "2 * a" + assert new_b._pending_dependency_map_serializer_ids == {"a": a._DescriptorNumber__serializer_id} + assert new_b.independent is True # Initially independent until dependencies resolved + + def test_dependency_resolution_after_deserialization(self, clear_global_map): + """Test that dependencies are properly resolved after deserialization.""" + # Create test parameters with dependencies + a = Parameter(name="a", value=2.0, unit="m", min=0, max=10) + b = Parameter(name="b", value=3.0, unit="m", min=0, max=10) + + c = Parameter.from_dependency( + name="c", + dependency_expression="a + b", + dependency_map={"a": a, "b": b}, + unit="m" + ) + + # Verify original dependency works + assert c.value == 5.0 # 2 + 3 + + # Serialize all parameters + params_data = { + "a": a.as_dict(), + "b": b.as_dict(), + "c": c.as_dict() + } + + # Clear and deserialize (manual approach) + global_object.map._clear() + new_params = {} + for name, data in params_data.items(): + new_params[name] = Parameter.from_dict(data) + + # Before resolution, c should be independent with pending dependency + assert new_params["c"].independent is True + assert hasattr(new_params["c"], '_pending_dependency_string') + + # Resolve dependencies + resolve_all_parameter_dependencies(new_params) + + # Alternative simplified approach using the helper function: + # global_object.map._clear() + # new_params = deserialize_and_resolve_parameters(params_data) + + # After resolution, c should be dependent and functional + assert new_params["c"].independent is False + assert new_params["c"].value == 5.0 # Still 2 + 3 + + # Test that dependency still works + new_params["a"].value = 10.0 + assert new_params["c"].value == 13.0 # 10 + 3 + + def test_unique_name_dependency_serialization(self, clear_global_map): + """Test serialization of dependencies using unique names.""" + a = Parameter(name="a", value=3.0, unit="m", min=0, max=10) + + # Create dependent parameter using unique name + b = Parameter.from_dependency( + name="b", + dependency_expression='2 * "Parameter_0"', # Using unique name + unit="m" + ) + + # Serialize both parameters + a_serialized = a.as_dict() + b_serialized = b.as_dict() + + # Should contain unique name mapping + assert b_serialized['_dependency_string'] == '2 * __Parameter_0__' + assert "__Parameter_0__" in b_serialized['_dependency_map_serializer_ids'] + assert b_serialized['_dependency_map_serializer_ids']["__Parameter_0__"] == a._DescriptorNumber__serializer_id + + # Deserialize both and resolve + global_object.map._clear() + c = Parameter(name='c', value=0.0) # Dummy to occupy unique name, to force new unique_names + + # Remove unique_name from serialized data to force generation of new unique names + a_serialized.pop('unique_name', None) + b_serialized.pop('unique_name', None) + + new_b = Parameter.from_dict(b_serialized) + new_a = Parameter.from_dict(a_serialized) + resolve_all_parameter_dependencies({"a": new_a, "b": new_b}) + + # Should work correctly + assert new_b.independent is False + new_a.value = 4.0 + assert new_b.value == 8.0 # 2 * 4 + + def test_json_serialization_roundtrip(self, clear_global_map): + """Test that parameter dependencies survive JSON serialization.""" + # Create parameters with dependencies + length = Parameter(name="length", value=10.0, unit="m", min=0, max=100) + width = Parameter(name="width", value=5.0, unit="m", min=0, max=50) + + area = Parameter.from_dependency( + name="area", + dependency_expression="length * width", + dependency_map={"length": length, "width": width}, + unit="m^2" + ) + + # Serialize to JSON + params_data = { + "length": length.as_dict(), + "width": width.as_dict(), + "area": area.as_dict() + } + json_str = json.dumps(params_data, default=str) + + # Deserialize from JSON + global_object.map._clear() + loaded_data = json.loads(json_str) + new_params = {} + for name, data in loaded_data.items(): + new_params[name] = Parameter.from_dict(data) + + # Resolve dependencies + resolve_all_parameter_dependencies(new_params) + + # Test functionality + assert new_params["area"].value == 50.0 # 10 * 5 + + # Test dependency updates + new_params["length"].value = 20.0 + assert new_params["area"].value == 100.0 # 20 * 5 + + def test_multiple_dependent_parameters(self, clear_global_map): + """Test serialization with multiple dependent parameters.""" + # Create a chain of dependencies + x = Parameter(name="x", value=2.0, unit="m", min=0, max=10) + + y = Parameter.from_dependency( + name="y", + dependency_expression="2 * x", + dependency_map={"x": x}, + unit="m" + ) + + z = Parameter.from_dependency( + name="z", + dependency_expression="y + x", + dependency_map={"y": y, "x": x}, + unit="m" + ) + + # Verify original chain works + assert y.value == 4.0 # 2 * 2 + assert z.value == 6.0 # 4 + 2 + + # Serialize all + params_data = { + "x": x.as_dict(), + "y": y.as_dict(), + "z": z.as_dict() + } + + # Deserialize and resolve + global_object.map._clear() + new_params = {} + for name, data in params_data.items(): + new_params[name] = Parameter.from_dict(data) + + resolve_all_parameter_dependencies(new_params) + + # Test chain still works + assert new_params["y"].value == 4.0 + assert new_params["z"].value == 6.0 + + # Test cascade updates + new_params["x"].value = 5.0 + assert new_params["y"].value == 10.0 # 2 * 5 + assert new_params["z"].value == 15.0 # 10 + 5 + + def test_dependency_with_descriptor_number(self, clear_global_map): + """Test that dependencies involving DescriptorNumber serialize correctly.""" + from easyscience.variable import DescriptorNumber + # When + + x = DescriptorNumber(name="x", value=3.0, unit="m") + y = Parameter(name="y", value=4.0, unit="m") + z = Parameter.from_dependency( + name="z", + dependency_expression="x + y", + dependency_map={"x": x, "y": y}, + ) + + # Verify original functionality + assert z.value == 7.0 # 3 + 4 + + # Then + # Serialize all + params_data = { + "x": x.as_dict(), + "y": y.as_dict(), + "z": z.as_dict() + } + # Deserialize and resolve + global_object.map._clear() + new_params = {} + for name, data in params_data.items(): + if name == "x": + new_params[name] = DescriptorNumber.from_dict(data) + else: + new_params[name] = Parameter.from_dict(data) + + resolve_all_parameter_dependencies(new_params) + + # Expect + # Test that functionality still works + assert new_params["z"].value == 7.0 # 3 + 4 + new_x = new_params["x"] + new_y = new_params["y"] + new_x.value = 4.0 + assert new_params["z"].value == 8.0 # 4 + 4 + new_y.value = 6.0 + assert new_params["z"].value == 10.0 # 4 + 6 + + def test_get_parameters_with_pending_dependencies(self, clear_global_map): + """Test utility function for finding parameters with pending dependencies.""" + # Create parameters + a = Parameter(name="a", value=1.0, unit="m") + b = Parameter.from_dependency( + name="b", + dependency_expression="2 * a", + dependency_map={"a": a}, + unit="m" + ) + + # Serialize and deserialize + params_data = {"a": a.as_dict(), "b": b.as_dict()} + global_object.map._clear() + new_params = {} + for name, data in params_data.items(): + new_params[name] = Parameter.from_dict(data) + + # Find pending dependencies + pending = get_parameters_with_pending_dependencies(new_params) + + assert len(pending) == 1 + assert pending[0].name == "b" + assert hasattr(pending[0], '_pending_dependency_string') + + # After resolution, should be empty + resolve_all_parameter_dependencies(new_params) + pending_after = get_parameters_with_pending_dependencies(new_params) + assert len(pending_after) == 0 + + def test_error_handling_missing_dependency(self, clear_global_map): + """Test error handling when dependency cannot be resolved.""" + a = Parameter(name="a", value=1.0, unit="m") + b = Parameter.from_dependency( + name="b", + dependency_expression="2 * a", + dependency_map={"a": a}, + unit="m" + ) + + # Serialize b but not a + b_data = b.as_dict() + + # Deserialize without a in the global map + global_object.map._clear() + new_b = Parameter.from_dict(b_data) + + # Should raise error when trying to resolve + with pytest.raises(ValueError, match="Cannot find parameter with serializer_id"): + new_b.resolve_pending_dependencies() + + def test_backward_compatibility_base_deserializer(self, clear_global_map): + """Test that the base deserializer path still works for dependent parameters.""" + from easyscience.io.serializer_dict import SerializerDict + + # Create dependent parameter + a = Parameter(name="a", value=2.0, unit="m") + b = Parameter.from_dependency( + name="b", + dependency_expression="3 * a", + dependency_map={"a": a}, + unit="m" + ) + + # Use base serializer path (SerializerDict.decode) + serialized = b.encode(encoder=SerializerDict) + global_object.map._clear() + + # This should not raise the "_independent" error anymore + deserialized = SerializerDict.decode(serialized) + + # Should be a valid Parameter (but without dependency resolution) + assert isinstance(deserialized, Parameter) + assert deserialized.name == "b" + assert deserialized.independent is True # Base path doesn't handle dependencies + + @pytest.mark.parametrize("order", [ + ["x", "y", "z"], + ["z", "x", "y"], + ["y", "z", "x"], + ["z", "y", "x"] + ], ids=['normal_order', 'dependent_first', 'mixed_order', 'dependent_first_reverse']) + def test_serializer_id_system_order_independence(self, clear_global_map, order): + """Test that dependency IDs allow parameters to be loaded in any order.""" + # WHEN + # Create parameters with dependencies + x = Parameter(name="x", value=5.0, unit="m", min=0, max=20) + y = Parameter(name="y", value=10.0, unit="m", min=0, max=30) + + z = Parameter.from_dependency( + name="z", + dependency_expression="x * y", + dependency_map={"x": x, "y": y}, + unit="m^2" + ) + + # Verify original functionality + assert z.value == 50.0 # 5 * 10 + + # Get dependency IDs + x_dep_id = x._DescriptorNumber__serializer_id + y_dep_id = y._DescriptorNumber__serializer_id + + # Serialize all parameters + params_data = { + "x": x.as_dict(), + "y": y.as_dict(), + "z": z.as_dict() + } + + # Verify dependency IDs are in serialized data + assert params_data["x"]["__serializer_id"] == x_dep_id + assert params_data["y"]["__serializer_id"] == y_dep_id + assert "__serializer_id" not in params_data["z"] + assert "_dependency_map_serializer_ids" in params_data["z"] + + # THEN + global_object.map._clear() + new_params = {} + + # Load in the specified order + for name in order: + new_params[name] = Parameter.from_dict(params_data[name]) + + # EXPECT + # Verify dependency IDs are preserved + assert new_params["x"]._DescriptorNumber__serializer_id == x_dep_id + assert new_params["y"]._DescriptorNumber__serializer_id == y_dep_id + + # Resolve dependencies + resolve_all_parameter_dependencies(new_params) + + # Verify functionality regardless of loading order + assert new_params["z"].independent is False + assert new_params["z"].value == 50.0 + + # Test dependency updates still work + new_params["x"].value = 6.0 + assert new_params["z"].value == 60.0 # 6 * 10 + + new_params["y"].value = 8.0 + assert new_params["z"].value == 48.0 # 6 * 8 + + def test_deserialize_and_resolve_parameters_helper(self, clear_global_map): + """Test the convenience helper function for deserialization and dependency resolution.""" + # Create test parameters with dependencies + a = Parameter(name="a", value=2.0, unit="m", min=0, max=10) + b = Parameter(name="b", value=3.0, unit="m", min=0, max=10) + + c = Parameter.from_dependency( + name="c", + dependency_expression="a + b", + dependency_map={"a": a, "b": b}, + unit="m" + ) + + # Verify original dependency works + assert c.value == 5.0 # 2 + 3 + + # Serialize all parameters + params_data = { + "a": a.as_dict(), + "b": b.as_dict(), + "c": c.as_dict() + } + + # Clear global map + global_object.map._clear() + + # Use the helper function instead of manual deserialization + resolution + new_params = deserialize_and_resolve_parameters(params_data) + + # Verify all parameters are correctly deserialized and dependencies resolved + assert len(new_params) == 3 + assert "a" in new_params + assert "b" in new_params + assert "c" in new_params + + # Check that independent parameters work + assert new_params["a"].name == "a" + assert new_params["a"].value == 2.0 + assert new_params["a"].independent is True + + assert new_params["b"].name == "b" + assert new_params["b"].value == 3.0 + assert new_params["b"].independent is True + + # Check that dependent parameter is properly resolved + assert new_params["c"].name == "c" + assert new_params["c"].value == 5.0 # 2 + 3 + assert new_params["c"].independent is False + + # Verify dependency still works after helper function + new_params["a"].value = 10.0 + assert new_params["c"].value == 13.0 # 10 + 3 + + # Verify no pending dependencies remain + pending = get_parameters_with_pending_dependencies(new_params) + assert len(pending) == 0 +