diff --git a/simvue/api/objects/base.py b/simvue/api/objects/base.py index f6f61762..c5930995 100644 --- a/simvue/api/objects/base.py +++ b/simvue/api/objects/base.py @@ -50,6 +50,8 @@ def _wrapper(self) -> typing.Any: raise RuntimeError( f"Cannot use 'staging_check' decorator on type '{type(self).__name__}'" ) + if _sv_obj._offline: + return member_func(self) if not _sv_obj._read_only and member_func.__name__ in _sv_obj._staging: _sv_obj._logger.warning( f"Uncommitted change found for attribute '{member_func.__name__}'" diff --git a/simvue/api/objects/run.py b/simvue/api/objects/run.py index 803681ae..674e0de4 100644 --- a/simvue/api/objects/run.py +++ b/simvue/api/objects/run.py @@ -315,7 +315,14 @@ def events( @write_only def send_heartbeat(self) -> dict[str, typing.Any] | None: - if self._offline or not self._identifier: + if not self._identifier: + return None + + if self._offline: + if not (_dir := self._local_staging_file.parent).exists(): + _dir.mkdir(parents=True) + _heartbeat_file = self._local_staging_file.with_suffix(".heartbeat") + _heartbeat_file.touch() return None _url = self._base_url diff --git a/simvue/config/user.py b/simvue/config/user.py index 22357ce5..f86bbcac 100644 --- a/simvue/config/user.py +++ b/simvue/config/user.py @@ -180,7 +180,7 @@ def fetch( except FileNotFoundError: if not server_token or not server_url: _config_dict = {"server": {}} - logger.warning("No config file found, checking environment variables") + logger.debug("No config file found, checking environment variables") _config_dict["server"] = _config_dict.get("server", {}) diff --git a/simvue/metadata.py b/simvue/metadata.py index abba595f..0f63981c 100644 --- a/simvue/metadata.py +++ b/simvue/metadata.py @@ -12,7 +12,6 @@ import toml import logging import pathlib -import flatdict from simvue.utilities import simvue_timestamp @@ -64,7 +63,7 @@ def git_info(repository: str) -> dict[str, typing.Any]: ) return { "git": { - "authors": json.dumps(list(author_list)), + "authors": list(author_list), "ref": ref, "msg": current_commit.message.strip(), "time_stamp": simvue_timestamp(current_commit.committed_datetime), @@ -84,34 +83,32 @@ def _python_env(repository: pathlib.Path) -> dict[str, typing.Any]: if (pyproject_file := pathlib.Path(repository).joinpath("pyproject.toml")).exists(): content = toml.load(pyproject_file) if (poetry_content := content.get("tool", {}).get("poetry", {})).get("name"): - python_meta |= { - "python.project.name": poetry_content["name"], - "python.project.version": poetry_content["version"], + python_meta["project"] = { + "name": poetry_content["name"], + "version": poetry_content["version"], } elif other_content := content.get("project"): - python_meta |= { - "python.project.name": other_content["name"], - "python.project.version": other_content["version"], + python_meta["project"] = { + "name": other_content["name"], + "version": other_content["version"], } if (poetry_lock_file := pathlib.Path(repository).joinpath("poetry.lock")).exists(): content = toml.load(poetry_lock_file).get("package", {}) - python_meta |= { - f"python.environment.{package['name']}": package["version"] - for package in content + python_meta["environment"] = { + package["name"]: package["version"] for package in content } elif (uv_lock_file := pathlib.Path(repository).joinpath("uv.lock")).exists(): content = toml.load(uv_lock_file).get("package", {}) - python_meta |= { - f"python.environment.{package['name']}": package["version"] - for package in content + python_meta["environment"] = { + package["name"]: package["version"] for package in content } else: with contextlib.suppress((KeyError, ImportError)): from pip._internal.operations.freeze import freeze - python_meta |= { - f"python.environment.{entry[0]}": entry[-1] + python_meta["environment"] = { + entry[0]: entry[-1] for line in freeze(local_only=True) if (entry := line.split("==")) } @@ -126,35 +123,33 @@ def _rust_env(repository: pathlib.Path) -> dict[str, typing.Any]: if (cargo_file := pathlib.Path(repository).joinpath("Cargo.toml")).exists(): content = toml.load(cargo_file).get("package", {}) if version := content.get("version"): - rust_meta |= {"rust.project.version": version} + rust_meta.setdefault("project", {})["version"] = version if name := content.get("name"): - rust_meta |= {"rust.project.name": name} + rust_meta.setdefault("project", {})["name"] = name if not (cargo_lock := pathlib.Path(repository).joinpath("Cargo.lock")).exists(): - return {} + return rust_meta cargo_dat = toml.load(cargo_lock) - - return rust_meta | { - f"rust.environment.{dependency['name']}": dependency["version"] + rust_meta["environment"] = { + dependency["name"]: dependency["version"] for dependency in cargo_dat.get("package") } + return rust_meta + def _julia_env(repository: pathlib.Path) -> dict[str, typing.Any]: """Retrieve a dictionary of Julia dependencies if a project file is available""" julia_meta: dict[str, str] = {} if (project_file := pathlib.Path(repository).joinpath("Project.toml")).exists(): content = toml.load(project_file) - julia_meta |= { - f"julia.project.{key}": value - for key, value in content.items() - if not isinstance(value, dict) + julia_meta["project"] = { + key: value for key, value in content.items() if not isinstance(value, dict) } - julia_meta |= { - f"julia.environment.{key}": value - for key, value in content.get("compat", {}).items() + julia_meta["environment"] = { + key: value for key, value in content.get("compat", {}).items() } return julia_meta @@ -171,13 +166,11 @@ def _node_js_env(repository: pathlib.Path) -> dict[str, typing.Any]: ) return {} - js_meta |= { - f"javascript.project.{key}": value - for key, value in content.items() - if key in ("name", "version") + js_meta["project"] = { + key: value for key, value in content.items() if key in ("name", "version") } - js_meta |= { - f"javascript.environment.{key.replace('@', '')}": value["version"] + js_meta["environment"] = { + key.replace("@", ""): value["version"] for key, value in content.get( "packages" if lfv in (2, 3) else "dependencies", {} ).items() @@ -188,16 +181,13 @@ def _node_js_env(repository: pathlib.Path) -> dict[str, typing.Any]: def environment(repository: pathlib.Path = pathlib.Path.cwd()) -> dict[str, typing.Any]: """Retrieve environment metadata""" - _environment_meta = flatdict.FlatDict( - _python_env(repository), delimiter="." - ).as_dict() - _environment_meta |= flatdict.FlatDict( - _rust_env(repository), delimiter="." - ).as_dict() - _environment_meta |= flatdict.FlatDict( - _julia_env(repository), delimiter="." - ).as_dict() - _environment_meta |= flatdict.FlatDict( - _node_js_env(repository), delimiter="." - ).as_dict() + _environment_meta = {} + if _python_meta := _python_env(repository): + _environment_meta["python"] = _python_meta + if _rust_meta := _rust_env(repository): + _environment_meta["rust"] = _rust_meta + if _julia_meta := _julia_env(repository): + _environment_meta["julia"] = _julia_meta + if _js_meta := _node_js_env(repository): + _environment_meta["javascript"] = _js_meta return _environment_meta diff --git a/simvue/sender.py b/simvue/sender.py index ca087d72..49751127 100644 --- a/simvue/sender.py +++ b/simvue/sender.py @@ -10,9 +10,11 @@ import logging from concurrent.futures import ThreadPoolExecutor import threading +import requests from simvue.config.user import SimvueConfiguration import simvue.api.objects +from simvue.version import __version__ UPLOAD_ORDER: list[str] = [ "tenants", @@ -134,7 +136,8 @@ def sender( objects_to_upload : list[str] Types of objects to upload, by default uploads all types of objects present in cache """ - cache_dir = cache_dir or SimvueConfiguration.fetch().offline.cache + _user_config = SimvueConfiguration.fetch() + cache_dir = cache_dir or _user_config.offline.cache cache_dir.joinpath("server_ids").mkdir(parents=True, exist_ok=True) _id_mapping: dict[str, str] = { file_path.name.split(".")[0]: file_path.read_text() @@ -160,4 +163,29 @@ def sender( ), _offline_files, ) + + # Send heartbeats + _headers: dict[str, str] = { + "Authorization": f"Bearer {_user_config.server.token.get_secret_value()}", + "User-Agent": f"Simvue Python client {__version__}", + } + + for _heartbeat_file in cache_dir.glob("runs/*.heartbeat"): + _offline_id = _heartbeat_file.name.split(".")[0] + _online_id = _id_mapping.get(_offline_id) + if not _online_id: + # Run has been closed - can just remove heartbeat and continue + _heartbeat_file.unlink() + continue + _logger.info(f"Sending heartbeat to run {_online_id}") + _response = requests.put( + f"{_user_config.server.url}/runs/{_online_id}/heartbeat", + headers=_headers, + ) + if _response.status_code == 200: + _heartbeat_file.unlink() + else: + _logger.warning( + f"Attempting to send heartbeat to run {_online_id} returned status code {_response.status_code}." + ) return _id_mapping diff --git a/tests/unit/test_metadata.py b/tests/unit/test_metadata.py index 4d01e14f..5c454e14 100644 --- a/tests/unit/test_metadata.py +++ b/tests/unit/test_metadata.py @@ -8,8 +8,8 @@ @pytest.mark.local def test_cargo_env() -> None: metadata = sv_meta._rust_env(pathlib.Path(__file__).parents[1].joinpath("example_data")) - assert metadata["rust.environment.serde"] == "1.0.123" - assert metadata["rust.project.name"] == "example_project" + assert metadata["environment"]["serde"] == "1.0.123" + assert metadata["project"]["name"] == "example_project" @pytest.mark.metadata @pytest.mark.local @@ -19,29 +19,36 @@ def test_cargo_env() -> None: def test_python_env(backend: str | None) -> None: if backend == "poetry": metadata = sv_meta._python_env(pathlib.Path(__file__).parents[1].joinpath("example_data", "python_poetry")) - assert metadata["python.project.name"] == "example-repo" + assert metadata["project"]["name"] == "example-repo" elif backend == "uv": metadata = sv_meta._python_env(pathlib.Path(__file__).parents[1].joinpath("example_data", "python_uv")) - assert metadata["python.project.name"] == "example-repo" + assert metadata["project"]["name"] == "example-repo" else: metadata = sv_meta._python_env(pathlib.Path(__file__).parents[1].joinpath("example_data")) - assert re.findall(r"\d+\.\d+\.\d+", metadata["python.environment.numpy"]) + assert re.findall(r"\d+\.\d+\.\d+", metadata["environment"]["numpy"]) @pytest.mark.metadata @pytest.mark.local def test_julia_env() -> None: metadata = sv_meta._julia_env(pathlib.Path(__file__).parents[1].joinpath("example_data")) - assert metadata["julia.project.name"] == "Julia Demo Project" - assert re.findall(r"\d+\.\d+\.\d+", metadata["julia.environment.AbstractDifferentiation"]) + assert metadata["project"]["name"] == "Julia Demo Project" + assert re.findall(r"\d+\.\d+\.\d+", metadata["environment"]["AbstractDifferentiation"]) @pytest.mark.metadata @pytest.mark.local def test_js_env() -> None: metadata = sv_meta._node_js_env(pathlib.Path(__file__).parents[1].joinpath("example_data")) - assert metadata["javascript.project.name"] == "my-awesome-project" - assert re.findall(r"\d+\.\d+\.\d+", metadata["javascript.environment.node_modules/dotenv"]) - + assert metadata["project"]["name"] == "my-awesome-project" + assert re.findall(r"\d+\.\d+\.\d+", metadata["environment"]["node_modules/dotenv"]) +@pytest.mark.metadata +@pytest.mark.local +def test_environment() -> None: + metadata = sv_meta.environment(pathlib.Path(__file__).parents[1].joinpath("example_data")) + assert metadata["python"]["project"]["name"] == "example-repo" + assert metadata["rust"]["project"]["name"] == "example_project" + assert metadata["julia"]["project"]["name"] == "Julia Demo Project" + assert metadata["javascript"]["project"]["name"] == "my-awesome-project" \ No newline at end of file