Skip to content

Commit b0663ad

Browse files
Merge pull request #778 from simvue-io/wk9874/fix_eco_units
Fix Ecoclient Units and other things
2 parents 4e67a3f + a835d0f commit b0663ad

28 files changed

+160
-162
lines changed

simvue/api/objects/events.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ def get(
5151
_class_instance = cls(_read_only=True, _local=True)
5252
_count: int = 0
5353

54-
for response in cls._get_all_objects(offset, run=run_id, **kwargs):
54+
for response in cls._get_all_objects(offset, count=count, run=run_id, **kwargs):
5555
if (_data := response.get("data")) is None:
5656
raise RuntimeError(
5757
f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s"

simvue/api/request.py

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -303,23 +303,18 @@ def get_paginated(
303303
server response
304304
"""
305305
_offset: int = offset or 0
306-
307306
while (
308-
(
309-
_response := get(
310-
url=url,
311-
headers=headers,
312-
params=(params or {})
313-
| {"count": count or MAX_ENTRIES_PER_PAGE, "start": _offset},
314-
timeout=timeout,
315-
json=json,
316-
)
307+
_response := get(
308+
url=url,
309+
headers=headers,
310+
params=(params or {})
311+
| {"count": count or MAX_ENTRIES_PER_PAGE, "start": _offset},
312+
timeout=timeout,
313+
json=json,
317314
)
318-
.json()
319-
.get("data")
320-
):
315+
).json():
321316
yield _response
322317
_offset += MAX_ENTRIES_PER_PAGE
323318

324-
if count and _offset > count:
319+
if (count and _offset > count) or (_response.json().get("count", 0) < _offset):
325320
break

simvue/client.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -835,7 +835,8 @@ def get_metric_values(
835835

836836
_args = {"filters": json.dumps(run_filters)} if run_filters else {}
837837

838-
_run_data = dict(Run.get(**_args))
838+
if not run_ids:
839+
_run_data = dict(Run.get(**_args))
839840

840841
if not (
841842
_run_metrics := self._get_run_metrics_from_server(
@@ -853,7 +854,8 @@ def get_metric_values(
853854
)
854855
if use_run_names:
855856
_run_metrics = {
856-
_run_data[key].name: _run_metrics[key] for key in _run_metrics.keys()
857+
Run(identifier=key).name: _run_metrics[key]
858+
for key in _run_metrics.keys()
857859
}
858860
return parse_run_set_metrics(
859861
_run_metrics,

simvue/config/user.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,7 @@ def fetch(
200200
_default_dir = _config_dict["offline"].get(
201201
"cache", DEFAULT_OFFLINE_DIRECTORY
202202
)
203+
pathlib.Path(_default_dir).mkdir(parents=True, exist_ok=True)
203204

204205
_config_dict["offline"]["cache"] = _default_dir
205206

simvue/eco/api_client.py

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -77,18 +77,15 @@ def __init__(self, *args, **kwargs) -> None:
7777
co2_api_endpoint : str
7878
endpoint for CO2 signal API
7979
co2_api_token: str
80-
RECOMMENDED. The API token for the CO2 Signal API, default is None.
80+
The API token for the ElectricityMaps API, default is None.
8181
timeout : int
8282
timeout for API
8383
"""
8484
super().__init__(*args, **kwargs)
8585
self._logger = logging.getLogger(self.__class__.__name__)
8686

8787
if not self.co2_api_token:
88-
self._logger.warning(
89-
"⚠️ No API token provided for CO2 Signal, "
90-
"use of a token is strongly recommended."
91-
)
88+
raise ValueError("API token is required for ElectricityMaps API.")
9289

9390
self._get_user_location_info()
9491

simvue/eco/config.py

Lines changed: 1 addition & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,6 @@
88
__date__ = "2025-03-06"
99

1010
import pydantic
11-
import pathlib
12-
import os
13-
14-
from simvue.config.files import DEFAULT_OFFLINE_DIRECTORY
1511

1612

1713
class EcoConfig(pydantic.BaseModel):
@@ -25,30 +21,13 @@ class EcoConfig(pydantic.BaseModel):
2521
the TDP for the CPU
2622
gpu_thermal_design_power: int | None, optional
2723
the TDP for each GPU
28-
local_data_directory: str, optional
29-
the directory to store local data, default is Simvue offline directory
3024
"""
3125

3226
co2_signal_api_token: pydantic.SecretStr | None = None
3327
cpu_thermal_design_power: pydantic.PositiveInt | None = None
3428
cpu_n_cores: pydantic.PositiveInt | None = None
3529
gpu_thermal_design_power: pydantic.PositiveInt | None = None
36-
local_data_directory: pydantic.DirectoryPath | None = pydantic.Field(
37-
None, validate_default=True
38-
)
3930
intensity_refresh_interval: pydantic.PositiveInt | str | None = pydantic.Field(
40-
default="1 day", gt=2 * 60
31+
default="1 hour", gt=2 * 60
4132
)
4233
co2_intensity: float | None = None
43-
44-
@pydantic.field_validator("local_data_directory", mode="before", check_fields=True)
45-
@classmethod
46-
def check_local_data_env(
47-
cls, local_data_directory: pathlib.Path | None
48-
) -> pathlib.Path:
49-
if _data_directory := os.environ.get("SIMVUE_ECO_DATA_DIRECTORY"):
50-
return pathlib.Path(_data_directory)
51-
if not local_data_directory:
52-
local_data_directory = pathlib.Path(DEFAULT_OFFLINE_DIRECTORY)
53-
local_data_directory.mkdir(exist_ok=True, parents=True)
54-
return local_data_directory

simvue/eco/emissions_monitor.py

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,15 @@ def __init__(self, *args, **kwargs) -> None:
108108
"""
109109
_logger = logging.getLogger(self.__class__.__name__)
110110

111+
if not (
112+
kwargs.get("co2_intensity")
113+
or kwargs.get("co2_signal_api_token")
114+
or kwargs.get("offline")
115+
):
116+
raise ValueError(
117+
"ElectricityMaps API token or hardcoeded CO2 intensity value is required for emissions tracking."
118+
)
119+
111120
if not isinstance(kwargs.get("thermal_design_power_per_cpu"), float):
112121
kwargs["thermal_design_power_per_cpu"] = 80.0
113122
_logger.warning(
@@ -229,7 +238,6 @@ def estimate_co2_emissions(
229238

230239
if self.co2_intensity:
231240
_current_co2_intensity = self.co2_intensity
232-
_co2_units = "kgCO2/kWh"
233241
else:
234242
self.check_refresh()
235243
# If no local data yet then return
@@ -238,7 +246,7 @@ def estimate_co2_emissions(
238246
"No CO2 emission data recorded as no CO2 intensity value "
239247
"has been provided and there is no local intensity data available."
240248
)
241-
return
249+
return False
242250

243251
if self._client:
244252
_country_code = self._client.country_code
@@ -251,10 +259,8 @@ def estimate_co2_emissions(
251259
**self._local_data[_country_code]
252260
)
253261
_current_co2_intensity = self._current_co2_data.data.carbon_intensity
254-
_co2_units = self._current_co2_data.carbon_intensity_units
255262
_process.gpu_percentage = gpu_percent
256263
_process.cpu_percentage = cpu_percent
257-
_previous_energy: float = _process.total_energy
258264
_process.power_usage = (_process.cpu_percentage / 100.0) * (
259265
self.thermal_design_power_per_cpu / self.n_cores_per_cpu
260266
)
@@ -263,23 +269,23 @@ def estimate_co2_emissions(
263269
_process.power_usage += (
264270
_process.gpu_percentage / 100.0
265271
) * self.thermal_design_power_per_gpu
272+
# Convert W to kW
273+
_process.power_usage /= 1000
274+
# Measure energy in kWh
275+
_process.energy_delta = _process.power_usage * measure_interval / 3600
276+
_process.total_energy += _process.energy_delta
266277

267-
_process.total_energy += _process.power_usage * measure_interval
268-
_process.energy_delta = _process.total_energy - _previous_energy
269-
270-
# Measured value is in g/kWh, convert to kg/kWs
271-
_carbon_intensity_kgpws: float = _current_co2_intensity / (60 * 60 * 1e3)
272-
273-
_process.co2_delta = (
274-
_process.power_usage * _carbon_intensity_kgpws * measure_interval
275-
)
278+
# Measured value is in g/kWh, convert to kg/kWh
279+
_carbon_intensity: float = _current_co2_intensity / 1000
276280

281+
_process.co2_delta = _process.energy_delta * _carbon_intensity
277282
_process.co2_emission += _process.co2_delta
278283

279284
self._logger.debug(
280-
f"📝 For process '{process_id}', recorded: CPU={_process.cpu_percentage:.2f}%, "
281-
f"Power={_process.power_usage:.2f}W, CO2={_process.co2_emission:.2e}{_co2_units}"
285+
f"📝 For process '{process_id}', in interval {measure_interval}, recorded: CPU={_process.cpu_percentage:.2f}%, "
286+
f"Power={_process.power_usage:.2f}kW, Energy = {_process.energy_delta}kWh, CO2={_process.co2_delta:.2e}kg"
282287
)
288+
return True
283289

284290
def simvue_metrics(self) -> dict[str, float]:
285291
"""Retrieve metrics to send to Simvue server."""

simvue/run.py

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -374,19 +374,20 @@ def _get_internal_metrics(
374374
# For the first emissions metrics reading, the time interval to use
375375
# Is the time since the run started, otherwise just use the time between readings
376376
if self._emissions_monitor:
377-
self._emissions_monitor.estimate_co2_emissions(
377+
_estimated = self._emissions_monitor.estimate_co2_emissions(
378378
process_id=f"{self._name}",
379379
cpu_percent=_current_system_measure.cpu_percent,
380380
measure_interval=(time.time() - self._start_time)
381381
if system_metrics_step == 0
382382
else self._system_metrics_interval,
383383
gpu_percent=_current_system_measure.gpu_percent,
384384
)
385-
self._add_metrics_to_dispatch(
386-
self._emissions_monitor.simvue_metrics(),
387-
join_on_fail=False,
388-
step=system_metrics_step,
389-
)
385+
if _estimated:
386+
self._add_metrics_to_dispatch(
387+
self._emissions_monitor.simvue_metrics(),
388+
join_on_fail=False,
389+
step=system_metrics_step,
390+
)
390391

391392
def _create_heartbeat_callback(
392393
self,
@@ -1083,7 +1084,7 @@ def config(
10831084
self._emissions_monitor = CO2Monitor(
10841085
intensity_refresh_interval=None,
10851086
co2_intensity=self._user_config.eco.co2_intensity,
1086-
local_data_directory=self._user_config.eco.local_data_directory,
1087+
local_data_directory=self._user_config.offline.cache,
10871088
co2_signal_api_token=None,
10881089
thermal_design_power_per_cpu=self._user_config.eco.cpu_thermal_design_power,
10891090
thermal_design_power_per_gpu=self._user_config.eco.gpu_thermal_design_power,
@@ -1092,7 +1093,7 @@ def config(
10921093
else:
10931094
self._emissions_monitor = CO2Monitor(
10941095
intensity_refresh_interval=self._user_config.eco.intensity_refresh_interval,
1095-
local_data_directory=self._user_config.eco.local_data_directory,
1096+
local_data_directory=self._user_config.offline.cache,
10961097
co2_signal_api_token=self._user_config.eco.co2_signal_api_token,
10971098
co2_intensity=self._user_config.eco.co2_intensity,
10981099
thermal_design_power_per_cpu=self._user_config.eco.cpu_thermal_design_power,

simvue/sender.py

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,6 @@ def sender(
151151
max_workers: int = 5,
152152
threading_threshold: int = 10,
153153
objects_to_upload: list[str] = UPLOAD_ORDER,
154-
co2_intensity_refresh: int | None | str = None,
155154
) -> dict[str, str]:
156155
"""Send data from a local cache directory to the Simvue server.
157156
@@ -165,9 +164,6 @@ def sender(
165164
The number of cached files above which threading will be used
166165
objects_to_upload : list[str]
167166
Types of objects to upload, by default uploads all types of objects present in cache
168-
co2_intensity_refresh: int | None | str
169-
the refresh interval for the CO2 intensity value, if None use config value if available,
170-
else do not refresh.
171167
172168
Returns
173169
-------
@@ -249,17 +245,13 @@ def sender(
249245
# refreshes the CO2 intensity value if required. No emission metrics
250246
# will be taken by the sender itself, values are assumed to be recorded
251247
# by any offline runs being sent.
252-
253-
if (
254-
_refresh_interval := co2_intensity_refresh
255-
or _user_config.eco.intensity_refresh_interval
256-
):
248+
if _user_config.metrics.enable_emission_metrics:
257249
CO2Monitor(
258250
thermal_design_power_per_gpu=None,
259251
thermal_design_power_per_cpu=None,
260252
local_data_directory=cache_dir,
261-
intensity_refresh_interval=_refresh_interval,
262-
co2_intensity=co2_intensity_refresh or _user_config.eco.co2_intensity,
253+
intensity_refresh_interval=_user_config.eco.intensity_refresh_interval,
254+
co2_intensity=_user_config.eco.co2_intensity,
263255
co2_signal_api_token=_user_config.eco.co2_signal_api_token,
264256
).check_refresh()
265257

tests/conftest.py

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,17 @@ def clear_out_files() -> None:
4848

4949
for file_obj in out_files:
5050
file_obj.unlink()
51-
52-
51+
52+
@pytest.fixture()
53+
def offline_cache_setup(monkeypatch: monkeypatch.MonkeyPatch):
54+
# Will be executed before test
55+
cache_dir = tempfile.TemporaryDirectory()
56+
monkeypatch.setenv("SIMVUE_OFFLINE_DIRECTORY", cache_dir.name)
57+
yield cache_dir
58+
# Will be executed after test
59+
cache_dir.cleanup()
60+
monkeypatch.setenv("SIMVUE_OFFLINE_DIRECTORY", None)
61+
5362
@pytest.fixture
5463
def mock_co2_signal(monkeypatch: monkeypatch.MonkeyPatch) -> dict[str, dict | str]:
5564
_mock_data = {
@@ -86,13 +95,21 @@ def _mock_location_info(self) -> None:
8695

8796
monkeypatch.setattr(requests, "get", _mock_get)
8897
monkeypatch.setattr(sv_eco.APIClient, "_get_user_location_info", _mock_location_info)
89-
98+
99+
_fetch = sv_cfg.SimvueConfiguration.fetch
100+
@classmethod
101+
def _mock_fetch(cls, *args, **kwargs) -> sv_cfg.SimvueConfiguration:
102+
_conf = _fetch(*args, **kwargs)
103+
_conf.eco.co2_signal_api_token = "test_token"
104+
_conf.metrics.enable_emission_metrics = True
105+
return _conf
106+
monkeypatch.setattr(sv_cfg.SimvueConfiguration, "fetch", _mock_fetch)
90107
return _mock_data
91108

92109

93110
@pytest.fixture
94111
def speedy_heartbeat(monkeypatch: monkeypatch.MonkeyPatch) -> None:
95-
monkeypatch.setattr(sv_run, "HEARTBEAT_INTERVAL", 0.1)
112+
monkeypatch.setattr(sv_run, "HEARTBEAT_INTERVAL", 1)
96113

97114

98115
@pytest.fixture(autouse=True)
@@ -286,15 +303,3 @@ def setup_test_run(run: sv_run.Run, create_objects: bool, request: pytest.Fixtur
286303
TEST_DATA["alert_ids"] = _alert_ids
287304

288305
return TEST_DATA
289-
290-
291-
@pytest.fixture
292-
def offline_test() -> pathlib.Path:
293-
with tempfile.TemporaryDirectory() as tempd:
294-
_tempdir = pathlib.Path(tempd)
295-
_cache_dir = _tempdir.joinpath(".simvue")
296-
_cache_dir.mkdir(exist_ok=True)
297-
os.environ["SIMVUE_OFFLINE_DIRECTORY"] = f"{_cache_dir}"
298-
assert sv_cfg.SimvueConfiguration.fetch().offline.cache == _cache_dir
299-
yield _tempdir
300-

0 commit comments

Comments
 (0)