Merge pull request #734 from simvue-io/dev

v2.0.0 Alpha 2 Release

Author: kzscisoft

.pre-commit-config.yaml

@@ -23,7 +23,7 @@ repos:
         args: [--branch, main, --branch, dev]
       - id: check-added-large-files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.6
+    rev: v0.9.7
     hooks:
       - id: ruff
         args: [ --fix, --exit-non-zero-on-fix, "--ignore=C901" ]

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Change log
 
+## [v2.0.0-alpha2](https://github.com/simvue-io/client/releases/tag/v2.0.0a2) - 2025-02-27
+* Removed 'no config file' and 'unstaged changes' warnings from Offline mode as they do not apply
+* Made `staging_check` not apply in Offline mode
+* Added heartbeat functionality to Offline mode
+* Moved away from `FlatDict` module for metadata collection, fixes Simvue in Jupyter notebooks
+* Fixed `reconnect()` by setting `read_only` to False and added tests
+* Fixed resource metrics collection to return measurement on startup and use short interval for more accurate measurements
+* Fixed `set_pid` so that resource metrics are also collected for child processes of it
+* Improved sender by having all cached files read at start and lock file so only one sender runs at once
+* Added `name` option in `log_alert` and added tests
+* Fixed client `get_alerts` and improved tests
+* Removed all server config checks in Offline mode
+
 ## [v2.0.0-alpha1](https://github.com/simvue-io/client/releases/tag/v2.0.0a1) - 2025-02-19
 * Fixed `add_alerts` so that it now works with both IDs and names
 * Improved alert and folder deduplication methods to rely on 409 responses from server upon creation

CITATION.cff

@@ -42,9 +42,9 @@ keywords:
   - alerting
   - simulation
 license: Apache-2.0
-commit: 124b2993d91dbff2e475aa972916b11e7bd02fa4
-version: 2.0.0a1
-date-released: '2025-02-19'
+commit: 83b9144abd2092d4be304bf742d72a249ad1d8ff
+version: 2.0.0a2
+date-released: '2025-02-27'
 references:
 - title: mlco2/codecarbon
   version: v2.8.2

poetry.lock

@@ -1,6 +1,6 @@
 [project]
 name = "simvue"
-version = "2.0.0a1"
+version = "2.0.0a2"
 description = "Simulation tracking and monitoring"
 authors = [
   {name = "Simvue Development Team", email = "[email protected]"}

pyproject.toml

@@ -50,6 +50,8 @@ def _wrapper(self) -> typing.Any:
             raise RuntimeError(
                 f"Cannot use 'staging_check' decorator on type '{type(self).__name__}'"
             )
+        if _sv_obj._offline:
+            return member_func(self)
         if not _sv_obj._read_only and member_func.__name__ in _sv_obj._staging:
             _sv_obj._logger.warning(
                 f"Uncommitted change found for attribute '{member_func.__name__}'"

simvue/api/objects/base.py

@@ -315,7 +315,14 @@ def events(
 
     @write_only
     def send_heartbeat(self) -> dict[str, typing.Any] | None:
-        if self._offline or not self._identifier:
+        if not self._identifier:
+            return None
+
+        if self._offline:
+            if not (_dir := self._local_staging_file.parent).exists():
+                _dir.mkdir(parents=True)
+            _heartbeat_file = self._local_staging_file.with_suffix(".heartbeat")
+            _heartbeat_file.touch()
             return None
 
         _url = self._base_url

simvue/api/objects/run.py

@@ -990,17 +990,23 @@ def get_alerts(
         RuntimeError
             if there was a failure retrieving data from the server
         """
-
         if not run_id:
+            if critical_only:
+                raise RuntimeError(
+                    "critical_only is ambiguous when returning alerts with no run ID specified."
+                )
             return [alert.name if names_only else alert for _, alert in Alert.get()]  # type: ignore
 
-        return [
-            alert.get("name")
-            if names_only
-            else Alert(identifier=alert.get("id"), **alert)
+        _alerts = [
+            Alert(identifier=alert.get("id"), **alert)
             for alert in Run(identifier=run_id).get_alert_details()
-            if not critical_only or alert["status"].get("current") == "critical"
-        ]  # type: ignore
+        ]
+
+        return [
+            alert.name if names_only else alert
+            for alert in _alerts
+            if not critical_only or alert.get_status(run_id) == "critical"
+        ]
 
     @prettify_pydantic
     @pydantic.validate_call

simvue/client.py

@@ -180,7 +180,7 @@ def fetch(
         except FileNotFoundError:
             if not server_token or not server_url:
                 _config_dict = {"server": {}}
-                logger.warning("No config file found, checking environment variables")
+                logger.debug("No config file found, checking environment variables")
 
         _config_dict["server"] = _config_dict.get("server", {})
 

simvue/config/user.py

@@ -348,9 +348,11 @@ def _update_alerts(self) -> None:
                 if self._runner._dispatcher:
                     self._runner._dispatcher.purge()
 
-                self._runner.log_alert(self._alert_ids[proc_id], "critical")
+                self._runner.log_alert(
+                    identifier=self._alert_ids[proc_id], state="critical"
+                )
             else:
-                self._runner.log_alert(self._alert_ids[proc_id], "ok")
+                self._runner.log_alert(identifier=self._alert_ids[proc_id], state="ok")
 
             _current_time: float = 0
             while (

simvue/executor.py

@@ -12,7 +12,6 @@
 import toml
 import logging
 import pathlib
-import flatdict
 
 from simvue.utilities import simvue_timestamp
 
@@ -64,7 +63,7 @@ def git_info(repository: str) -> dict[str, typing.Any]:
         )
         return {
             "git": {
-                "authors": json.dumps(list(author_list)),
+                "authors": list(author_list),
                 "ref": ref,
                 "msg": current_commit.message.strip(),
                 "time_stamp": simvue_timestamp(current_commit.committed_datetime),
@@ -84,34 +83,32 @@ def _python_env(repository: pathlib.Path) -> dict[str, typing.Any]:
     if (pyproject_file := pathlib.Path(repository).joinpath("pyproject.toml")).exists():
         content = toml.load(pyproject_file)
         if (poetry_content := content.get("tool", {}).get("poetry", {})).get("name"):
-            python_meta |= {
-                "python.project.name": poetry_content["name"],
-                "python.project.version": poetry_content["version"],
+            python_meta["project"] = {
+                "name": poetry_content["name"],
+                "version": poetry_content["version"],
             }
         elif other_content := content.get("project"):
-            python_meta |= {
-                "python.project.name": other_content["name"],
-                "python.project.version": other_content["version"],
+            python_meta["project"] = {
+                "name": other_content["name"],
+                "version": other_content["version"],
             }
 
     if (poetry_lock_file := pathlib.Path(repository).joinpath("poetry.lock")).exists():
         content = toml.load(poetry_lock_file).get("package", {})
-        python_meta |= {
-            f"python.environment.{package['name']}": package["version"]
-            for package in content
+        python_meta["environment"] = {
+            package["name"]: package["version"] for package in content
         }
     elif (uv_lock_file := pathlib.Path(repository).joinpath("uv.lock")).exists():
         content = toml.load(uv_lock_file).get("package", {})
-        python_meta |= {
-            f"python.environment.{package['name']}": package["version"]
-            for package in content
+        python_meta["environment"] = {
+            package["name"]: package["version"] for package in content
         }
     else:
         with contextlib.suppress((KeyError, ImportError)):
             from pip._internal.operations.freeze import freeze
 
-            python_meta |= {
-                f"python.environment.{entry[0]}": entry[-1]
+            python_meta["environment"] = {
+                entry[0]: entry[-1]
                 for line in freeze(local_only=True)
                 if (entry := line.split("=="))
             }
@@ -126,35 +123,33 @@ def _rust_env(repository: pathlib.Path) -> dict[str, typing.Any]:
     if (cargo_file := pathlib.Path(repository).joinpath("Cargo.toml")).exists():
         content = toml.load(cargo_file).get("package", {})
         if version := content.get("version"):
-            rust_meta |= {"rust.project.version": version}
+            rust_meta.setdefault("project", {})["version"] = version
 
         if name := content.get("name"):
-            rust_meta |= {"rust.project.name": name}
+            rust_meta.setdefault("project", {})["name"] = name
 
     if not (cargo_lock := pathlib.Path(repository).joinpath("Cargo.lock")).exists():
-        return {}
+        return rust_meta
 
     cargo_dat = toml.load(cargo_lock)
-
-    return rust_meta | {
-        f"rust.environment.{dependency['name']}": dependency["version"]
+    rust_meta["environment"] = {
+        dependency["name"]: dependency["version"]
         for dependency in cargo_dat.get("package")
     }
 
+    return rust_meta
+
 
 def _julia_env(repository: pathlib.Path) -> dict[str, typing.Any]:
     """Retrieve a dictionary of Julia dependencies if a project file is available"""
     julia_meta: dict[str, str] = {}
     if (project_file := pathlib.Path(repository).joinpath("Project.toml")).exists():
         content = toml.load(project_file)
-        julia_meta |= {
-            f"julia.project.{key}": value
-            for key, value in content.items()
-            if not isinstance(value, dict)
+        julia_meta["project"] = {
+            key: value for key, value in content.items() if not isinstance(value, dict)
         }
-        julia_meta |= {
-            f"julia.environment.{key}": value
-            for key, value in content.get("compat", {}).items()
+        julia_meta["environment"] = {
+            key: value for key, value in content.get("compat", {}).items()
         }
     return julia_meta
 
@@ -171,13 +166,11 @@ def _node_js_env(repository: pathlib.Path) -> dict[str, typing.Any]:
             )
             return {}
 
-        js_meta |= {
-            f"javascript.project.{key}": value
-            for key, value in content.items()
-            if key in ("name", "version")
+        js_meta["project"] = {
+            key: value for key, value in content.items() if key in ("name", "version")
         }
-        js_meta |= {
-            f"javascript.environment.{key.replace('@', '')}": value["version"]
+        js_meta["environment"] = {
+            key.replace("@", ""): value["version"]
             for key, value in content.get(
                 "packages" if lfv in (2, 3) else "dependencies", {}
             ).items()
@@ -188,16 +181,13 @@ def _node_js_env(repository: pathlib.Path) -> dict[str, typing.Any]:
 
 def environment(repository: pathlib.Path = pathlib.Path.cwd()) -> dict[str, typing.Any]:
     """Retrieve environment metadata"""
-    _environment_meta = flatdict.FlatDict(
-        _python_env(repository), delimiter="."
-    ).as_dict()
-    _environment_meta |= flatdict.FlatDict(
-        _rust_env(repository), delimiter="."
-    ).as_dict()
-    _environment_meta |= flatdict.FlatDict(
-        _julia_env(repository), delimiter="."
-    ).as_dict()
-    _environment_meta |= flatdict.FlatDict(
-        _node_js_env(repository), delimiter="."
-    ).as_dict()
+    _environment_meta = {}
+    if _python_meta := _python_env(repository):
+        _environment_meta["python"] = _python_meta
+    if _rust_meta := _rust_env(repository):
+        _environment_meta["rust"] = _rust_meta
+    if _julia_meta := _julia_env(repository):
+        _environment_meta["julia"] = _julia_meta
+    if _js_meta := _node_js_env(repository):
+        _environment_meta["javascript"] = _js_meta
     return _environment_meta

simvue/metadata.py

@@ -36,14 +36,18 @@ def get_process_memory(processes: list[psutil.Process]) -> int:
     return rss
 
 
-def get_process_cpu(processes: list[psutil.Process]) -> int:
+def get_process_cpu(
+    processes: list[psutil.Process], interval: float | None = None
+) -> int:
     """
     Get the CPU usage
+
+    If first time being called, use a small interval to collect initial CPU metrics.
     """
     cpu_percent: int = 0
     for process in processes:
         with contextlib.suppress(Exception):
-            cpu_percent += process.cpu_percent()
+            cpu_percent += process.cpu_percent(interval=interval)
 
     return cpu_percent