From 65dfac4d36f295520aeafa568f07db8723263e22 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Fri, 14 Feb 2025 13:06:36 +0000 Subject: [PATCH 1/2] Added deepmerge to dependencies and extra tests --- poetry.lock | 20 +++++++++-- pyproject.toml | 1 + simvue/api/objects/base.py | 6 ++-- tests/functional/test_run_class.py | 53 ++++++++++++++++++++++++++---- 4 files changed, 69 insertions(+), 11 deletions(-) diff --git a/poetry.lock b/poetry.lock index aa295ce7..4371e2de 100644 --- a/poetry.lock +++ b/poetry.lock @@ -505,7 +505,6 @@ files = [ {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:761817a3377ef15ac23cd7834715081791d4ec77f9297ee694ca1ee9c2c7e5eb"}, {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3c672a53c0fb4725a29c303be906d3c1fa99c32f58abe008a82705f9ee96f40b"}, {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:4ac4c9f37eba52cb6fbeaf5b59c152ea976726b865bd4cf87883a7e7006cc543"}, - {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:60eb32934076fa07e4316b7b2742fa52cbb190b42c2df2863dbc4230a0a9b385"}, {file = "cryptography-44.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ed3534eb1090483c96178fcb0f8893719d96d5274dfde98aa6add34614e97c8e"}, {file = "cryptography-44.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f3f6fdfa89ee2d9d496e2c087cebef9d4fcbb0ad63c40e821b39f74bf48d9c5e"}, {file = "cryptography-44.0.0-cp37-abi3-win32.whl", hash = "sha256:eb33480f1bad5b78233b0ad3e1b0be21e8ef1da745d8d2aecbb20671658b9053"}, @@ -516,7 +515,6 @@ files = [ {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c5eb858beed7835e5ad1faba59e865109f3e52b3783b9ac21e7e47dc5554e289"}, {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f53c2c87e0fb4b0c00fa9571082a057e37690a8f12233306161c8f4b819960b7"}, {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:9e6fc8a08e116fb7c7dd1f040074c9d7b51d74a8ea40d4df2fc7aa08b76b9e6c"}, - {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9abcc2e083cbe8dde89124a47e5e53ec38751f0d7dfd36801008f316a127d7ba"}, {file = "cryptography-44.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d2436114e46b36d00f8b72ff57e598978b37399d2786fd39793c36c6d5cb1c64"}, {file = "cryptography-44.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a01956ddfa0a6790d594f5b34fc1bfa6098aca434696a03cfdbe469b8ed79285"}, {file = "cryptography-44.0.0-cp39-abi3-win32.whl", hash = "sha256:eca27345e1214d1b9f9490d200f9db5a874479be914199194e746c893788d417"}, @@ -560,6 +558,22 @@ files = [ docs = ["ipython", "matplotlib", "numpydoc", "sphinx"] tests = ["pytest", "pytest-cov", "pytest-xdist"] +[[package]] +name = "deepmerge" +version = "2.0" +description = "A toolset for deeply merging Python dictionaries." +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version <= \"3.11\" or python_version >= \"3.12\"" +files = [ + {file = "deepmerge-2.0-py3-none-any.whl", hash = "sha256:6de9ce507115cff0bed95ff0ce9ecc31088ef50cbdf09bc90a09349a318b3d00"}, + {file = "deepmerge-2.0.tar.gz", hash = "sha256:5c3d86081fbebd04dd5de03626a0607b809a98fb6ccba5770b62466fe940ff20"}, +] + +[package.extras] +dev = ["black", "build", "mypy", "pytest", "pyupgrade", "twine", "validate-pyproject[all]"] + [[package]] name = "dnspython" version = "2.7.0" @@ -2636,4 +2650,4 @@ plot = ["matplotlib", "plotly"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "b87307deab6d125136242de2adc36049337970d6abea23392c9fdf57761230a6" +content-hash = "44571ffd2089db9c20200ad205318c25e0c419891b7bea5cef0e0fd4c918b7d3" diff --git a/pyproject.toml b/pyproject.toml index bbbfd633..194ba81f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ dependencies = [ "psutil (>=6.1.1,<7.0.0)", "tenacity (>=9.0.0,<10.0.0)", "typing-extensions (>=4.12.2,<5.0.0) ; python_version < \"3.11\"", + "deepmerge (>=2.0,<3.0)", ] [project.urls] diff --git a/simvue/api/objects/base.py b/simvue/api/objects/base.py index 7f43904f..cc683757 100644 --- a/simvue/api/objects/base.py +++ b/simvue/api/objects/base.py @@ -16,6 +16,7 @@ import msgpack import pydantic +from deepmerge import always_merger from simvue.config.user import SimvueConfiguration from simvue.exception import ObjectNotFoundError @@ -524,9 +525,10 @@ def _cache(self) -> None: with self._local_staging_file.open() as in_f: _local_data = json.load(in_f) - _local_data |= self._staging + _cache_data = always_merger.merge(_local_data, self._staging) + with self._local_staging_file.open("w", encoding="utf-8") as out_f: - json.dump(_local_data, out_f, indent=2) + json.dump(_cache_data, out_f, indent=2) def to_dict(self) -> dict[str, typing.Any]: return self._get() | self._staging diff --git a/tests/functional/test_run_class.py b/tests/functional/test_run_class.py index 5de51a8a..34381c0f 100644 --- a/tests/functional/test_run_class.py +++ b/tests/functional/test_run_class.py @@ -220,9 +220,14 @@ def test_offline_tags(create_plain_run_offline: tuple[sv_run.Run, dict]) -> None @pytest.mark.run def test_update_metadata_running(create_test_run: tuple[sv_run.Run, dict]) -> None: - METADATA = {"a": 10, "b": 1.2, "c": "word"} + METADATA = {"a": 1, "b": 1.2, "c": "word", "d": "new"} run, _ = create_test_run - run.update_metadata(METADATA) + # Add an initial set of metadata + run.update_metadata({"a": 10, "b": 1.2, "c": "word"}) + # Try updating a second time, check original dict isnt overwritten + run.update_metadata({"d": "new"}) + # Try updating an already defined piece of metadata + run.update_metadata({"a": 1}) run.close() time.sleep(1.0) client = sv_cl.Client() @@ -234,9 +239,14 @@ def test_update_metadata_running(create_test_run: tuple[sv_run.Run, dict]) -> No @pytest.mark.run def test_update_metadata_created(create_pending_run: tuple[sv_run.Run, dict]) -> None: - METADATA = {"a": 10, "b": 1.2, "c": "word"} + METADATA = {"a": 1, "b": 1.2, "c": "word", "d": "new"} run, _ = create_pending_run - run.update_metadata(METADATA) + # Add an initial set of metadata + run.update_metadata({"a": 10, "b": 1.2, "c": "word"}) + # Try updating a second time, check original dict isnt overwritten + run.update_metadata({"d": "new"}) + # Try updating an already defined piece of metadata + run.update_metadata({"a": 1}) time.sleep(1.0) client = sv_cl.Client() run_info = client.get_run(run.id) @@ -250,13 +260,21 @@ def test_update_metadata_created(create_pending_run: tuple[sv_run.Run, dict]) -> def test_update_metadata_offline( create_plain_run_offline: tuple[sv_run.Run, dict], ) -> None: - METADATA = {"a": 10, "b": 1.2, "c": "word"} + METADATA = {"a": 1, "b": 1.2, "c": "word", "d": "new"} run, _ = create_plain_run_offline run_name = run._name - run.update_metadata(METADATA) + # Add an initial set of metadata + run.update_metadata({"a": 10, "b": 1.2, "c": "word"}) + # Try updating a second time, check original dict isnt overwritten + run.update_metadata({"d": "new"}) + # Try updating an already defined piece of metadata + run.update_metadata({"a": 1}) + sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10) + import pdb; pdb.set_trace() run.close() time.sleep(1.0) + import pdb; pdb.set_trace() client = sv_cl.Client() run_info = client.get_run(client.get_run_id_from_name(run_name)) @@ -655,6 +673,29 @@ def test_update_tags_created( assert sorted(run_data.tags) == sorted(tags + ["additional"]) +@pytest.mark.offline +@pytest.mark.run +def test_update_tags_offline( + create_plain_run_offline: typing.Tuple[sv_run.Run, dict], +) -> None: + simvue_run, _ = create_plain_run_offline + run_name = simvue_run._name + + simvue_run.set_tags(["simvue_client_unit_tests",]) + + simvue_run.update_tags(["additional"]) + + sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10) + simvue_run.close() + time.sleep(1.0) + + client = sv_cl.Client() + run_data = client.get_run(client.get_run_id_from_name(run_name)) + + time.sleep(1) + run_data = client.get_run(simvue_run._id) + assert sorted(run_data.tags) == sorted(["simvue_client_unit_tests", "additional"]) + @pytest.mark.run @pytest.mark.parametrize("object_type", ("DataFrame", "ndarray")) def test_save_object( From 58a623b61621d6cfac4a25f529110c55dfaad8fd Mon Sep 17 00:00:00 2001 From: Matt Field Date: Fri, 14 Feb 2025 13:33:00 +0000 Subject: [PATCH 2/2] Added custom merge strategy to override lists --- simvue/api/objects/base.py | 6 +++--- simvue/utilities.py | 17 ++++++++++++++++- tests/functional/test_run_class.py | 5 ++--- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/simvue/api/objects/base.py b/simvue/api/objects/base.py index cc683757..b75c8e6e 100644 --- a/simvue/api/objects/base.py +++ b/simvue/api/objects/base.py @@ -16,8 +16,8 @@ import msgpack import pydantic -from deepmerge import always_merger +from simvue.utilities import staging_merger from simvue.config.user import SimvueConfiguration from simvue.exception import ObjectNotFoundError from simvue.version import __version__ @@ -525,10 +525,10 @@ def _cache(self) -> None: with self._local_staging_file.open() as in_f: _local_data = json.load(in_f) - _cache_data = always_merger.merge(_local_data, self._staging) + staging_merger.merge(_local_data, self._staging) with self._local_staging_file.open("w", encoding="utf-8") as out_f: - json.dump(_cache_data, out_f, indent=2) + json.dump(_local_data, out_f, indent=2) def to_dict(self) -> dict[str, typing.Any]: return self._get() | self._staging diff --git a/simvue/utilities.py b/simvue/utilities.py index 69a8ecd8..fe7746cb 100644 --- a/simvue/utilities.py +++ b/simvue/utilities.py @@ -11,8 +11,8 @@ import os import pathlib import typing - import jwt +from deepmerge import Merger from datetime import timezone from simvue.models import DATETIME_FORMAT @@ -395,3 +395,18 @@ def get_mimetype_for_file(file_path: pathlib.Path) -> str: """Return MIME type for the given file""" _guess, *_ = mimetypes.guess_type(file_path) return _guess or "application/octet-stream" + + +# Create a new Merge strategy for merging local file and staging attributes +staging_merger = Merger( + # pass in a list of tuple, with the + # strategies you are looking to apply + # to each type. + [(list, ["override"]), (dict, ["merge"]), (set, ["union"])], + # next, choose the fallback strategies, + # applied to all other types: + ["override"], + # finally, choose the strategies in + # the case where the types conflict: + ["override"], +) diff --git a/tests/functional/test_run_class.py b/tests/functional/test_run_class.py index 34381c0f..4352e97e 100644 --- a/tests/functional/test_run_class.py +++ b/tests/functional/test_run_class.py @@ -269,12 +269,11 @@ def test_update_metadata_offline( run.update_metadata({"d": "new"}) # Try updating an already defined piece of metadata run.update_metadata({"a": 1}) - + sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10) - import pdb; pdb.set_trace() run.close() time.sleep(1.0) - import pdb; pdb.set_trace() + client = sv_cl.Client() run_info = client.get_run(client.get_run_id_from_name(run_name))