Skip to content

Commit cede0e6

Browse files
committed
Merge branch 'dev' into feature/env-var-metadata
2 parents bf42673 + 7efd676 commit cede0e6

File tree

7 files changed

+226
-14
lines changed

7 files changed

+226
-14
lines changed

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
## Unreleased
44

55
- Added ability to include environment variables within metadata for runs.
6+
- Improve handling of Conda based environments in metadata collection.
67

78
## [v2.1.2](https://github.com/simvue-io/client/releases/tag/v2.1.2) - 2025-06-25
89

poetry.lock

Lines changed: 64 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pyproject.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "simvue"
3-
version = "2.1.2"
3+
version = "2.1.3"
44
description = "Simulation tracking and monitoring"
55
authors = [
66
{name = "Simvue Development Team", email = "[email protected]"}
@@ -55,6 +55,7 @@ dependencies = [
5555
"deepmerge (>=2.0,<3.0)",
5656
"geocoder (>=1.38.1,<2.0.0)",
5757
"pydantic-extra-types (>=2.10.5,<3.0.0)",
58+
"pyyaml (>=6.0.2,<7.0.0)",
5859
]
5960

6061
[project.urls]

simvue/metadata.py

Lines changed: 88 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import os
1313
import fnmatch
1414
import toml
15+
import yaml
1516
import logging
1617
import pathlib
1718

@@ -78,9 +79,71 @@ def git_info(repository: str) -> dict[str, typing.Any]:
7879
return {}
7980

8081

82+
def _conda_dependency_parse(dependency: str) -> tuple[str, str] | None:
83+
"""Parse a dependency definition into module-version."""
84+
if dependency.startswith("::"):
85+
logger.warning(
86+
f"Skipping Conda specific channel definition '{dependency}' in Python environment metadata."
87+
)
88+
return None
89+
elif ">=" in dependency:
90+
module, version = dependency.split(">=")
91+
logger.warning(
92+
f"Ignoring '>=' constraint in Python package version, naively storing '{module}=={version}', "
93+
"for a more accurate record use 'conda env export > environment.yml'"
94+
)
95+
elif "~=" in dependency:
96+
module, version = dependency.split("~=")
97+
logger.warning(
98+
f"Ignoring '~=' constraint in Python package version, naively storing '{module}=={version}', "
99+
"for a more accurate record use 'conda env export > environment.yml'"
100+
)
101+
elif dependency.startswith("-e"):
102+
_, version = dependency.split("-e")
103+
version = version.strip()
104+
module = pathlib.Path(version).name
105+
elif dependency.startswith("file://"):
106+
_, version = dependency.split("file://")
107+
module = pathlib.Path(version).stem
108+
elif dependency.startswith("git+"):
109+
_, version = dependency.split("git+")
110+
if "#egg=" in version:
111+
repo, module = version.split("#egg=")
112+
module = repo.split("/")[-1].replace(".git", "")
113+
else:
114+
module = version.split("/")[-1].replace(".git", "")
115+
elif "==" not in dependency:
116+
logger.warning(
117+
f"Ignoring '{dependency}' in Python environment record as no version constraint specified."
118+
)
119+
return None
120+
else:
121+
module, version = dependency.split("==")
122+
123+
return module, version
124+
125+
126+
def _conda_env(environment_file: pathlib.Path) -> dict[str, str]:
127+
"""Parse/interpret a Conda environment file."""
128+
content = yaml.load(environment_file.open(), Loader=yaml.SafeLoader)
129+
python_environment: dict[str, str] = {}
130+
pip_dependencies: list[str] = []
131+
for dependency in content.get("dependencies", []):
132+
if isinstance(dependency, dict) and dependency.get("pip"):
133+
pip_dependencies = dependency["pip"]
134+
break
135+
136+
for dependency in pip_dependencies:
137+
if not (parsed := _conda_dependency_parse(dependency)):
138+
continue
139+
module, version = parsed
140+
python_environment[module.strip().replace("-", "_")] = version.strip()
141+
return python_environment
142+
143+
81144
def _python_env(repository: pathlib.Path) -> dict[str, typing.Any]:
82145
"""Retrieve a dictionary of Python dependencies if lock file is available"""
83-
python_meta: dict[str, str] = {}
146+
python_meta: dict[str, dict] = {}
84147

85148
if (pyproject_file := pathlib.Path(repository).joinpath("pyproject.toml")).exists():
86149
content = toml.load(pyproject_file)
@@ -105,22 +168,37 @@ def _python_env(repository: pathlib.Path) -> dict[str, typing.Any]:
105168
python_meta["environment"] = {
106169
package["name"]: package["version"] for package in content
107170
}
171+
# Handle Conda case, albeit naively given the user may or may not have used 'conda env'
172+
# to dump their exact dependency versions
173+
elif (
174+
environment_file := pathlib.Path(repository).joinpath("environment.yml")
175+
).exists():
176+
python_meta["environment"] = _conda_env(environment_file)
108177
else:
109178
with contextlib.suppress((KeyError, ImportError)):
110179
from pip._internal.operations.freeze import freeze
111180

112-
python_meta["environment"] = {
113-
entry[0]: entry[-1]
114-
for line in freeze(local_only=True)
115-
if (entry := line.split("=="))
116-
}
181+
# Conda supports having file names with @ as entries
182+
# in the requirements.txt file as opposed to ==
183+
python_meta["environment"] = {}
184+
185+
for line in freeze(local_only=True):
186+
if line.startswith("-e"):
187+
python_meta["environment"]["local_install"] = line.split(" ")[-1]
188+
continue
189+
if "@" in line:
190+
entry = line.split("@")
191+
python_meta["environment"][entry[0].strip()] = entry[-1].strip()
192+
elif "==" in line:
193+
entry = line.split("==")
194+
python_meta["environment"][entry[0].strip()] = entry[-1].strip()
117195

118196
return python_meta
119197

120198

121199
def _rust_env(repository: pathlib.Path) -> dict[str, typing.Any]:
122200
"""Retrieve a dictionary of Rust dependencies if lock file available"""
123-
rust_meta: dict[str, str] = {}
201+
rust_meta: dict[str, dict] = {}
124202

125203
if (cargo_file := pathlib.Path(repository).joinpath("Cargo.toml")).exists():
126204
content = toml.load(cargo_file).get("package", {})
@@ -136,15 +214,15 @@ def _rust_env(repository: pathlib.Path) -> dict[str, typing.Any]:
136214
cargo_dat = toml.load(cargo_lock)
137215
rust_meta["environment"] = {
138216
dependency["name"]: dependency["version"]
139-
for dependency in cargo_dat.get("package")
217+
for dependency in cargo_dat.get("package", [])
140218
}
141219

142220
return rust_meta
143221

144222

145223
def _julia_env(repository: pathlib.Path) -> dict[str, typing.Any]:
146224
"""Retrieve a dictionary of Julia dependencies if a project file is available"""
147-
julia_meta: dict[str, str] = {}
225+
julia_meta: dict[str, dict] = {}
148226
if (project_file := pathlib.Path(repository).joinpath("Project.toml")).exists():
149227
content = toml.load(project_file)
150228
julia_meta["project"] = {
@@ -157,7 +235,7 @@ def _julia_env(repository: pathlib.Path) -> dict[str, typing.Any]:
157235

158236

159237
def _node_js_env(repository: pathlib.Path) -> dict[str, typing.Any]:
160-
js_meta: dict[str, str] = {}
238+
js_meta: dict[str, dict] = {}
161239
if (
162240
project_file := pathlib.Path(repository).joinpath("package-lock.json")
163241
).exists():
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
name: advanced_env
2+
channels:
3+
- conda-forge
4+
- anaconda
5+
- defaults
6+
dependencies:
7+
# Basic Conda packages with different version specifiers
8+
- python=3.10.12
9+
- numpy>=1.23.5
10+
- pandas
11+
- scikit-learn<1.2
12+
- openjdk>=11,<12
13+
14+
# Platform-specific dependencies
15+
- libsass # Standard dependency
16+
- vsix-installer # Standard dependency
17+
- openblas # A package that may have platform-specific builds
18+
19+
# Using a sub-channel (also called a label)
20+
- ::my-package-from-subchannel
21+
22+
# A 'pip' section for installing packages from PyPI and other sources
23+
- pip
24+
- pip:
25+
# Public PyPI packages with different version specifiers
26+
- requests==2.31.0
27+
- black
28+
- jupyterlab~=4.0.0
29+
- numpy==2.32.2
30+
31+
# A local package from a path
32+
- -e ./path/to/my-local-package
33+
- file:///path/to/my-local-wheel.whl
34+
35+
# A package from a Git repository
36+
- git+https://github.com/myuser/myrepo.git#egg=myproject
37+
38+
variables:
39+
# Define environment variables
40+
MY_ENV_VAR: "some_value"

tests/functional/test_run_class.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import json
22
import logging
3+
import toml
34
import os
45
<<<<<<< HEAD
56
from os.path import basename
@@ -1338,3 +1339,28 @@ def test_reconnect_with_process() -> None:
13381339
remove_runs=True,
13391340
recursive=True
13401341
)
1342+
1343+
@pytest.mark.parametrize(
1344+
"environment", ("python_conda", "python_poetry", "python_uv", "julia", "rust", "nodejs")
1345+
)
1346+
def test_run_environment_metadata(environment: str, mocker: pytest_mock.MockerFixture) -> None:
1347+
"""Tests that the environment information is compatible with the server."""
1348+
from simvue.config.user import SimvueConfiguration
1349+
from simvue.metadata import environment as env_func
1350+
_data_dir = pathlib.Path(__file__).parents[1].joinpath("example_data")
1351+
_target_dir = _data_dir
1352+
if "python" in environment:
1353+
_target_dir = _data_dir.joinpath(environment)
1354+
_config = SimvueConfiguration.fetch()
1355+
1356+
with sv_run.Run(server_token=_config.server.token, server_url=_config.server.url) as run:
1357+
_uuid = f"{uuid.uuid4()}".split("-")[0]
1358+
run.init(
1359+
name=f"test_run_environment_metadata_{environment}",
1360+
folder=f"/simvue_unit_testing/{_uuid}",
1361+
retention_period=os.environ.get("SIMVUE_TESTING_RETENTION_PERIOD", "2 mins"),
1362+
running=False,
1363+
visibility="tenant" if os.environ.get("CI") else None,
1364+
)
1365+
run.update_metadata(env_func(_target_dir))
1366+

tests/unit/test_metadata.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ def test_cargo_env() -> None:
1515
@pytest.mark.metadata
1616
@pytest.mark.local
1717
@pytest.mark.parametrize(
18-
"backend", ("poetry", "uv", None)
18+
"backend", ("poetry", "uv", "conda", None)
1919
)
2020
def test_python_env(backend: str | None) -> None:
2121
if backend == "poetry":
@@ -24,6 +24,9 @@ def test_python_env(backend: str | None) -> None:
2424
elif backend == "uv":
2525
metadata = sv_meta._python_env(pathlib.Path(__file__).parents[1].joinpath("example_data", "python_uv"))
2626
assert metadata["project"]["name"] == "example-repo"
27+
elif backend == "conda":
28+
metadata = sv_meta._python_env(pathlib.Path(__file__).parents[1].joinpath("example_data", "python_conda"))
29+
assert metadata["environment"]["requests"]
2730
else:
2831
metadata = sv_meta._python_env(pathlib.Path(__file__).parents[1].joinpath("example_data"))
2932

@@ -54,7 +57,6 @@ def test_environment() -> None:
5457
assert metadata["julia"]["project"]["name"] == "Julia Demo Project"
5558
assert metadata["javascript"]["project"]["name"] == "my-awesome-project"
5659

57-
5860
@pytest.mark.metadata
5961
@pytest.mark.local
6062
def test_slurm_env_var_capture() -> None:
@@ -75,3 +77,4 @@ def test_slurm_env_var_capture() -> None:
7577

7678
sv_meta.metadata = sv_meta.environment(env_var_glob_exprs={"SLURM_*"})
7779
assert all((key, value) in sv_meta.metadata["shell"].items() for key, value in _slurm_env.items())
80+

0 commit comments

Comments
 (0)