From 85e6f9fca9f1c860a715c9afcae0a6911fdcec58 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Fri, 4 Apr 2025 11:33:33 +0100 Subject: [PATCH 01/23] Fix suppress errors test --- simvue/metrics.py | 7 +- simvue/run.py | 132 +++++++++++------------------ tests/unit/test_suppress_errors.py | 6 +- 3 files changed, 55 insertions(+), 90 deletions(-) diff --git a/simvue/metrics.py b/simvue/metrics.py index bf5b209d..2914b351 100644 --- a/simvue/metrics.py +++ b/simvue/metrics.py @@ -139,7 +139,6 @@ def __init__( self, processes: list[psutil.Process], interval: float | None, - cpu_only: bool = False, ) -> None: """Perform a measurement of system resource consumption. @@ -149,14 +148,10 @@ def __init__( processes to measure across. interval: float | None interval to measure, if None previous measure time used for interval. - cpu_only: bool, optional - only record CPU information, default False """ self.cpu_percent: float | None = get_process_cpu(processes, interval=interval) self.cpu_memory: float | None = get_process_memory(processes) - self.gpus: list[dict[str, float]] = ( - None if cpu_only else get_gpu_metrics(processes) - ) + self.gpus: list[dict[str, float]] = get_gpu_metrics(processes) def to_dict(self) -> dict[str, float]: """Create metrics dictionary for sending to a Simvue server.""" diff --git a/simvue/run.py b/simvue/run.py index dc5ef56d..f95a792f 100644 --- a/simvue/run.py +++ b/simvue/run.py @@ -334,10 +334,7 @@ def _terminate_run( def _get_internal_metrics( self, - system_metrics_step: int | None, - emission_metrics_step: int | None, - res_measure_interval: int | None = None, - ems_measure_interval: int | None = None, + system_metrics_step: int, ) -> None: """Refresh resource and emissions metrics. @@ -346,16 +343,8 @@ def _get_internal_metrics( Parameters ---------- - system_metrics_step: int | None - the current step for this resource metric record, - None if skipping resource metrics. - emission_metrics_step: int | None - the current step for this emission metrics record, - None if skipping emission metrics. - res_measure_interval: int | None, optional - the interval for resource metric gathering, default is None - ems_measure_interval: int | None, optional - the interval for emission metric gathering, default is None + system_metrics_step: int + The current step for this system metric record Return ------ @@ -363,38 +352,40 @@ def _get_internal_metrics( new resource metric measure time new emissions metric measure time """ + + # In order to get a resource metric reading at t=0 + # because there is no previous CPU reading yet we cannot + # use the default of None for the interval here, so we measure + # at an interval of 1s. _current_system_measure = SystemResourceMeasurement( self.processes, - interval=res_measure_interval, - cpu_only=not system_metrics_step, + interval=1 if system_metrics_step == 0 else None, ) - if system_metrics_step is not None: - # Set join on fail to false as if an error is thrown - # join would be called on this thread and a thread cannot - # join itself! - self._add_metrics_to_dispatch( - _current_system_measure.to_dict(), - join_on_fail=False, - step=system_metrics_step, - ) + # Set join on fail to false as if an error is thrown + # join would be called on this thread and a thread cannot + # join itself! + self._add_metrics_to_dispatch( + _current_system_measure.to_dict(), + join_on_fail=False, + step=system_metrics_step, + ) - if ( - self._emissions_monitor - and emission_metrics_step is not None - and ems_measure_interval is not None - and _current_system_measure.cpu_percent is not None - ): + # For the first emissions metrics reading, the time interval to use + # Is the time since the run started, otherwise just use the time between readings + if self._emissions_monitor: self._emissions_monitor.estimate_co2_emissions( process_id=f"{self._name}", cpu_percent=_current_system_measure.cpu_percent, - measure_interval=ems_measure_interval, + measure_interval=(time.time() - self._start_time) + if system_metrics_step == 0 + else self._system_metrics_interval, gpu_percent=_current_system_measure.gpu_percent, ) self._add_metrics_to_dispatch( self._emissions_monitor.simvue_metrics(), join_on_fail=False, - step=emission_metrics_step, + step=system_metrics_step, ) def _create_heartbeat_callback( @@ -416,61 +407,30 @@ def _heartbeat( raise RuntimeError("Expected initialisation of heartbeat") last_heartbeat: float = 0 - last_res_metric_call: float = 0 - last_co2_metric_call: float = 0 - - co2_step: int = 0 - res_step: int = 0 + last_sys_metric_call: float = 0 - initial_ems_metrics_interval: float = time.time() - self._start_time + sys_step: int = 0 while not heartbeat_trigger.is_set(): with self._configuration_lock: _current_time: float = time.time() + _update_system_metrics: bool = ( self._system_metrics_interval is not None - and _current_time - last_res_metric_call - > self._system_metrics_interval - and self._status == "running" - ) - _update_emissions_metrics: bool = ( - self._system_metrics_interval is not None - and self._emissions_monitor - and _current_time - last_co2_metric_call + and _current_time - last_sys_metric_call > self._system_metrics_interval and self._status == "running" ) - # In order to get a resource metric reading at t=0 - # because there is no previous CPU reading yet we cannot - # use the default of None for the interval here, so we measure - # at an interval of 1s. For emissions metrics the first step - # is time since run start - self._get_internal_metrics( - emission_metrics_step=co2_step - if _update_emissions_metrics - else None, - system_metrics_step=res_step - if _update_system_metrics - else None, - res_measure_interval=1 if res_step == 0 else None, - ems_measure_interval=initial_ems_metrics_interval - if co2_step == 0 - else self._system_metrics_interval, - ) + if _update_system_metrics: + self._get_internal_metrics(system_metrics_step=sys_step) - res_step += 1 - co2_step += 1 + sys_step += 1 - last_res_metric_call = ( + last_sys_metric_call = ( _current_time if _update_system_metrics - else last_res_metric_call - ) - last_co2_metric_call = ( - _current_time - if _update_emissions_metrics - else last_co2_metric_call + else last_sys_metric_call ) if time.time() - last_heartbeat < self._heartbeat_interval: @@ -1055,7 +1015,7 @@ def config( queue_blocking: bool | None = None, system_metrics_interval: pydantic.PositiveInt | None = None, enable_emission_metrics: bool | None = None, - disable_system_metrics: bool | None = None, + disable_resources_metrics: bool | None = None, storage_id: str | None = None, abort_on_alert: typing.Literal["run", "all", "ignore"] | bool | None = None, ) -> bool: @@ -1069,10 +1029,10 @@ def config( queue_blocking : bool, optional block thread queues during metric/event recording system_metrics_interval : int, optional - frequency at which to collect resource metrics + frequency at which to collect resource and emissions metrics, if enabled enable_emission_metrics : bool, optional enable monitoring of emission metrics - disable_system_metrics : bool, optional + disable_resources_metrics : bool, optional disable monitoring of resource metrics storage_id : str, optional identifier of storage to use, by default None @@ -1095,17 +1055,30 @@ def config( if queue_blocking is not None: self._queue_blocking = queue_blocking - if system_metrics_interval and disable_system_metrics: + if system_metrics_interval and disable_resources_metrics: self._error( "Setting of resource metric interval and disabling resource metrics is ambiguous" ) return False - if disable_system_metrics: + if system_metrics_interval: + self._system_metrics_interval = system_metrics_interval + + if disable_resources_metrics: + if self._emissions_monitor: + self._error( + "Emissions metrics require resource metrics collection." + ) + return False self._pid = None self._system_metrics_interval = None if enable_emission_metrics: + if not self._system_metrics_interval: + self._error( + "Emissions metrics require resource metrics collection - make sure resource metrics are enabled!" + ) + return False if self._user_config.run.mode == "offline": # Create an emissions monitor with no API calls self._emissions_monitor = CO2Monitor( @@ -1130,9 +1103,6 @@ def config( elif enable_emission_metrics is False and self._emissions_monitor: self._error("Cannot disable emissions monitor once it has been started") - if system_metrics_interval: - self._system_metrics_interval = system_metrics_interval - if abort_on_alert is not None: if isinstance(abort_on_alert, bool): warnings.warn( diff --git a/tests/unit/test_suppress_errors.py b/tests/unit/test_suppress_errors.py index 0ba7d022..73c114dc 100644 --- a/tests/unit/test_suppress_errors.py +++ b/tests/unit/test_suppress_errors.py @@ -12,7 +12,7 @@ def test_suppress_errors_false() -> None: with pytest.raises(RuntimeError) as e: run.config( suppress_errors=False, - disable_system_metrics=123, + disable_resources_metrics=123, ) assert "Input should be a valid boolean, unable to interpret input" in f"{e.value}" @@ -25,7 +25,7 @@ def test_suppress_errors_true(caplog) -> None: run.config(suppress_errors=True) run.config( - disable_system_metrics=123, + disable_resources_metrics=123, ) caplog.set_level(logging.ERROR) @@ -41,7 +41,7 @@ def test_suppress_errors_default(caplog) -> None: run.config(suppress_errors=True) run.config( - disable_system_metrics=123, + disable_resources_metrics=123, ) caplog.set_level(logging.ERROR) From 4ef09d9b8cc654d8d8d1d4b05ab7d36686ebc583 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Fri, 4 Apr 2025 13:13:54 +0100 Subject: [PATCH 02/23] Improved ecoclient tests, changed token to Secretstr --- simvue/eco/emissions_monitor.py | 5 +---- tests/unit/test_ecoclient.py | 33 +++++++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/simvue/eco/emissions_monitor.py b/simvue/eco/emissions_monitor.py index b1798e96..931ff0e4 100644 --- a/simvue/eco/emissions_monitor.py +++ b/simvue/eco/emissions_monitor.py @@ -47,7 +47,7 @@ class CO2Monitor(pydantic.BaseModel): local_data_directory: pydantic.DirectoryPath intensity_refresh_interval: int | None | str co2_intensity: float | None - co2_signal_api_token: str | None + co2_signal_api_token: pydantic.SecretStr | None offline: bool = False def now(self) -> str: @@ -252,7 +252,6 @@ def estimate_co2_emissions( ) _current_co2_intensity = self._current_co2_data.data.carbon_intensity _co2_units = self._current_co2_data.carbon_intensity_units - _process.gpu_percentage = gpu_percent _process.cpu_percentage = cpu_percent _previous_energy: float = _process.total_energy @@ -271,8 +270,6 @@ def estimate_co2_emissions( # Measured value is in g/kWh, convert to kg/kWs _carbon_intensity_kgpws: float = _current_co2_intensity / (60 * 60 * 1e3) - _previous_emission: float = _process.co2_emission - _process.co2_delta = ( _process.power_usage * _carbon_intensity_kgpws * measure_interval ) diff --git a/tests/unit/test_ecoclient.py b/tests/unit/test_ecoclient.py index 691a66fa..2ccc91f0 100644 --- a/tests/unit/test_ecoclient.py +++ b/tests/unit/test_ecoclient.py @@ -55,7 +55,7 @@ def test_outdated_data_check( time.sleep(3) _ems_monitor.estimate_co2_emissions(**_measure_params) - assert _spy.call_count == 2 if refresh else 1, f"{_spy.call_count} != {2 if refresh else 1}" + assert _spy.call_count == (2 if refresh else 1), f"{_spy.call_count} != {2 if refresh else 1}" def test_co2_monitor_properties(mock_co2_signal) -> None: @@ -65,20 +65,37 @@ def test_co2_monitor_properties(mock_co2_signal) -> None: thermal_design_power_per_gpu=130, local_data_directory=tempd, intensity_refresh_interval=None, - co2_intensity=None, + co2_intensity=40, co2_signal_api_token=None ) _measure_params = { "process_id": "test_co2_monitor_properties", "cpu_percent": 20, "gpu_percent": 40, - "measure_interval": 1 + "measure_interval": 2 } + _ems_monitor.estimate_co2_emissions(**_measure_params) + assert _ems_monitor.current_carbon_intensity + assert _ems_monitor.n_cores_per_cpu == 4 assert _ems_monitor.process_data["test_co2_monitor_properties"] - assert _ems_monitor.total_power_usage - assert _ems_monitor.total_co2_emission - assert _ems_monitor.total_co2_delta - assert _ems_monitor.total_energy - assert _ems_monitor.total_energy_delta + + # Will use this equation + # Power used = (TDP_cpu * cpu_percent / num_cores) + (TDP_gpu * gpu_percent) + assert _ems_monitor.total_power_usage == (80 * 0.2 * 1 / 4) + (130 * 0.4 * 1) + + # Energy used = power used * measure interval + assert _ems_monitor.total_energy == _ems_monitor.total_power_usage * 2 + + # CO2 emission = energy * CO2 intensity + # Need to convert CO2 intensity from g/kWh to kg/kWs + # So divide by 1000 (g -> kg) + # And divide by 60*60 (kWh -> kws) + assert _ems_monitor.total_co2_emission == _ems_monitor.total_energy * 40 / (60*60*1000) + + _ems_monitor.estimate_co2_emissions(**_measure_params) + # Check delta is half of total, since we've now called this twice + assert _ems_monitor.total_co2_delta == _ems_monitor.total_co2_emission / 2 + assert _ems_monitor.total_energy_delta == _ems_monitor.total_energy / 2 + From 2890d43d072a59c943e0552314f57fc0d5d6dcf1 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Fri, 4 Apr 2025 13:55:06 +0100 Subject: [PATCH 03/23] Refactor to use new electricitymaps API endpoints --- simvue/eco/api_client.py | 25 +++++++++---------------- simvue/eco/emissions_monitor.py | 4 ++-- tests/conftest.py | 17 ++++++++--------- tests/unit/test_ecoclient.py | 9 ++++----- 4 files changed, 23 insertions(+), 32 deletions(-) diff --git a/simvue/eco/api_client.py b/simvue/eco/api_client.py index e4597ee6..dc03c8b7 100644 --- a/simvue/eco/api_client.py +++ b/simvue/eco/api_client.py @@ -19,38 +19,33 @@ import geocoder.location import typing -CO2_SIGNAL_API_ENDPOINT: str = "https://api.co2signal.com/v1/latest" +CO2_SIGNAL_API_ENDPOINT: str = ( + "https://api.electricitymap.org/v3/carbon-intensity/latest" +) class CO2SignalData(pydantic.BaseModel): datetime: datetime.datetime carbon_intensity: float - fossil_fuel_percentage: float class CO2SignalResponse(pydantic.BaseModel): - disclaimer: str country_code: str - status: str data: CO2SignalData carbon_intensity_units: str @classmethod def from_json_response(cls, json_response: dict) -> "CO2SignalResponse": - _data: dict[str, typing.Any] = json_response["data"] _co2_signal_data = CO2SignalData( datetime=datetime.datetime.fromisoformat( - _data["datetime"].replace("Z", "+00:00") + json_response["datetime"].replace("Z", "+00:00") ), - carbon_intensity=_data["carbonIntensity"], - fossil_fuel_percentage=_data["fossilFuelPercentage"], + carbon_intensity=json_response["carbonIntensity"], ) return cls( - disclaimer=json_response["_disclaimer"], - country_code=json_response["countryCode"], - status=json_response["status"], + country_code=json_response["zone"], data=_co2_signal_data, - carbon_intensity_units=json_response["units"]["carbonIntensity"], + carbon_intensity_units="gCO2e/kWh", ) @@ -109,16 +104,14 @@ def _get_user_location_info(self) -> None: def get(self) -> CO2SignalResponse: """Get the current data""" _params: dict[str, float | str] = { - "lat": self._latitude, - "lon": self._longitude, - "countryCode": self._two_letter_country_code, + "zone": self._two_letter_country_code, } if self.co2_api_token: _params["auth-token"] = self.co2_api_token.get_secret_value() self._logger.debug(f"🍃 Retrieving carbon intensity data for: {_params}") - _response = requests.get(f"{self.co2_api_endpoint}", params=_params) + _response = requests.get(f"{self.co2_api_endpoint}", headers=_params) if _response.status_code != http.HTTPStatus.OK: try: diff --git a/simvue/eco/emissions_monitor.py b/simvue/eco/emissions_monitor.py index 931ff0e4..02b32b34 100644 --- a/simvue/eco/emissions_monitor.py +++ b/simvue/eco/emissions_monitor.py @@ -52,7 +52,7 @@ class CO2Monitor(pydantic.BaseModel): def now(self) -> str: """Return data file timestamp for the current time""" - _now: datetime.datetime = datetime.datetime.now(datetime.UTC) + _now: datetime.datetime = datetime.datetime.now(datetime.timezone.utc) return _now.strftime(TIME_FORMAT) @property @@ -187,7 +187,7 @@ def check_refresh(self) -> bool: with self._data_file_path.open("r") as in_f: self._local_data = json.load(in_f) - if not self._client or not self._local_data: + if not self._client: return False if ( diff --git a/tests/conftest.py b/tests/conftest.py index 0ac0d563..c09f84da 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -53,15 +53,14 @@ def clear_out_files() -> None: @pytest.fixture def mock_co2_signal(monkeypatch: monkeypatch.MonkeyPatch) -> dict[str, dict | str]: _mock_data = { - "data": { - "datetime": datetime.datetime.now().isoformat(), - "carbonIntensity": 40, - "fossilFuelPercentage": 39, - }, - "_disclaimer": "test disclaimer", - "countryCode": "GB", - "status": "unknown", - "units": {"carbonIntensity": "eqCO2kg/kwh"} + "zone":"GB", + "carbonIntensity":85, + "datetime":"2025-04-04T12:00:00.000Z", + "updatedAt":"2025-04-04T11:41:12.947Z", + "createdAt":"2025-04-01T12:43:58.056Z", + "emissionFactorType":"lifecycle", + "isEstimated":True, + "estimationMethod":"TIME_SLICER_AVERAGE" } class MockCo2SignalAPIResponse: def json(*_, **__) -> dict: diff --git a/tests/unit/test_ecoclient.py b/tests/unit/test_ecoclient.py index 2ccc91f0..aa2bedce 100644 --- a/tests/unit/test_ecoclient.py +++ b/tests/unit/test_ecoclient.py @@ -18,10 +18,9 @@ def test_api_client_get_loc_info(mock_co2_signal) -> None: def test_api_client_query(mock_co2_signal: dict[str, dict | str]) -> None: _client = sv_eco_api.APIClient() _response: sv_eco_api.CO2SignalResponse = _client.get() - assert _response.carbon_intensity_units == mock_co2_signal["units"]["carbonIntensity"] - assert _response.country_code == mock_co2_signal["countryCode"] - assert _response.data.carbon_intensity == mock_co2_signal["data"]["carbonIntensity"] - assert _response.data.fossil_fuel_percentage == mock_co2_signal["data"]["fossilFuelPercentage"] + assert _response.carbon_intensity_units == "gCO2e/kWh" + assert _response.country_code == mock_co2_signal["zone"] + assert _response.data.carbon_intensity == mock_co2_signal["carbonIntensity"] @pytest.mark.eco @@ -41,7 +40,7 @@ def test_outdated_data_check( thermal_design_power_per_cpu=80, thermal_design_power_per_gpu=130, local_data_directory=tempd, - intensity_refresh_interval=1 if refresh else 2, + intensity_refresh_interval=1 if refresh else None, co2_intensity=None, co2_signal_api_token=None ) From 8d04feabfc2bf1ba4723d4002053e26f751fe683 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Fri, 4 Apr 2025 13:58:54 +0100 Subject: [PATCH 04/23] Move sys step to be under update sys metrics if --- simvue/run.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/simvue/run.py b/simvue/run.py index f95a792f..ef5dd8a8 100644 --- a/simvue/run.py +++ b/simvue/run.py @@ -425,13 +425,13 @@ def _heartbeat( if _update_system_metrics: self._get_internal_metrics(system_metrics_step=sys_step) - sys_step += 1 + sys_step += 1 - last_sys_metric_call = ( - _current_time - if _update_system_metrics - else last_sys_metric_call - ) + last_sys_metric_call = ( + _current_time + if _update_system_metrics + else last_sys_metric_call + ) if time.time() - last_heartbeat < self._heartbeat_interval: time.sleep(1) From 520d583154c7dc491138d9b877097eff4a6fcb05 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Fri, 4 Apr 2025 13:59:09 +0100 Subject: [PATCH 05/23] Move sys step to be under update sys metrics if --- simvue/run.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/simvue/run.py b/simvue/run.py index ef5dd8a8..099016f1 100644 --- a/simvue/run.py +++ b/simvue/run.py @@ -424,14 +424,13 @@ def _heartbeat( if _update_system_metrics: self._get_internal_metrics(system_metrics_step=sys_step) - sys_step += 1 - last_sys_metric_call = ( - _current_time - if _update_system_metrics - else last_sys_metric_call - ) + last_sys_metric_call = ( + _current_time + if _update_system_metrics + else last_sys_metric_call + ) if time.time() - last_heartbeat < self._heartbeat_interval: time.sleep(1) From 5f047debf28b49b40e03b3fc02cc12dfbcbba385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Mon, 7 Apr 2025 14:59:02 +0100 Subject: [PATCH 06/23] Fix count limit when retrieving objects --- simvue/api/objects/base.py | 8 ++++---- simvue/api/request.py | 6 ++++-- tests/unit/test_folder.py | 34 ++++++++++++++++++++++------------ tests/unit/test_run.py | 35 +++++++++++++++++++++++------------ 4 files changed, 53 insertions(+), 30 deletions(-) diff --git a/simvue/api/objects/base.py b/simvue/api/objects/base.py index bc3227f1..d99db66d 100644 --- a/simvue/api/objects/base.py +++ b/simvue/api/objects/base.py @@ -365,7 +365,7 @@ def ids( """ _class_instance = cls(_read_only=True, _local=True) _count: int = 0 - for response in cls._get_all_objects(offset): + for response in cls._get_all_objects(offset, count=count): if (_data := response.get("data")) is None: raise RuntimeError( f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s" @@ -404,7 +404,7 @@ def get( """ _class_instance = cls(_read_only=True, _local=True) _count: int = 0 - for _response in cls._get_all_objects(offset, **kwargs): + for _response in cls._get_all_objects(offset, count=count, **kwargs): if count and _count > count: return if (_data := _response.get("data")) is None: @@ -438,7 +438,7 @@ def count(cls, **kwargs) -> int: @classmethod def _get_all_objects( - cls, offset: int | None, **kwargs + cls, offset: int | None, count: int | None, **kwargs ) -> typing.Generator[dict, None, None]: _class_instance = cls(_read_only=True) _url = f"{_class_instance._base_url}" @@ -448,7 +448,7 @@ def _get_all_objects( _label = _label[:-1] for response in get_paginated( - _url, headers=_class_instance._headers, offset=offset, **kwargs + _url, headers=_class_instance._headers, offset=offset, count=count, **kwargs ): yield get_json_from_response( response=response, diff --git a/simvue/api/request.py b/simvue/api/request.py index fe61f7bd..6a0c05a3 100644 --- a/simvue/api/request.py +++ b/simvue/api/request.py @@ -281,6 +281,7 @@ def get_paginated( timeout: int = DEFAULT_API_TIMEOUT, json: dict[str, typing.Any] | None = None, offset: int | None = None, + count: int | None = None, **params, ) -> typing.Generator[requests.Response, None, None]: """Paginate results of a server query. @@ -309,7 +310,7 @@ def get_paginated( url=url, headers=headers, params=(params or {}) - | {"count": MAX_ENTRIES_PER_PAGE, "start": _offset}, + | {"count": count or MAX_ENTRIES_PER_PAGE, "start": _offset}, timeout=timeout, json=json, ) @@ -320,4 +321,5 @@ def get_paginated( yield _response _offset += MAX_ENTRIES_PER_PAGE - yield _response + if count and _offset > count: + break diff --git a/tests/unit/test_folder.py b/tests/unit/test_folder.py index a029563c..0cfc68c5 100644 --- a/tests/unit/test_folder.py +++ b/tests/unit/test_folder.py @@ -38,22 +38,32 @@ def test_folder_creation_offline() -> None: with _folder._local_staging_file.open() as in_f: _local_data = json.load(in_f) - + assert _folder._local_staging_file.name.split(".")[0] == _folder.id assert _local_data.get("path", None) == _path - + sender(_folder._local_staging_file.parents[1], 2, 10, ["folders"]) time.sleep(1) client = Client() - + _folder_new = client.get_folder(_path) assert _folder_new.path == _path - + _folder_new.delete() - + assert not _folder._local_staging_file.exists() +@pytest.mark.api +@pytest.mark.online +def test_get_folder() -> None: + _uuid: str = f"{uuid.uuid4()}".split("-")[0] + _folder_name = f"/simvue_unit_testing/{_uuid}" + _folder_1 = Folder.new(path=f"{_folder_name}/dir_1") + _folder_2 = Folder.new(path=f"{_folder_name}/dir_2") + assert len(list(Folder._get_all_objects(count=2, offset=None))) == 2 + + @pytest.mark.api @pytest.mark.online def test_folder_modification_online() -> None: @@ -85,34 +95,34 @@ def test_folder_modification_offline() -> None: _tags = ["testing", "api"] _folder = Folder.new(path=_path, offline=True) _folder.commit() - + sender(_folder._local_staging_file.parents[1], 2, 10, ["folders"]) time.sleep(1) - + client = Client() _folder_online = client.get_folder(_path) assert _folder_online.path == _path - + _folder_new = Folder(identifier=_folder.id) _folder_new.read_only(False) _folder_new.tags = _tags _folder_new.description = _description _folder_new.commit() - + with _folder._local_staging_file.open() as in_f: _local_data = json.load(in_f) assert _folder._local_staging_file.name.split(".")[0] == _folder.id assert _local_data.get("description", None) == _description assert _local_data.get("tags", None) == _tags - + sender(_folder._local_staging_file.parents[1], 2, 10, ["folders"]) time.sleep(1) - + _folder_online.refresh() assert _folder_online.path == _path assert _folder_online.description == _description assert _folder_online.tags == _tags - + _folder_online.read_only(False) _folder_online.delete() diff --git a/tests/unit/test_run.py b/tests/unit/test_run.py index a35b8ef6..5d9b12b3 100644 --- a/tests/unit/test_run.py +++ b/tests/unit/test_run.py @@ -39,17 +39,17 @@ def test_run_creation_offline() -> None: _local_data = json.load(in_f) assert _local_data.get("name") == f"simvue_offline_run_{_uuid}" assert _local_data.get("folder") == _folder_name - + sender(_run._local_staging_file.parents[1], 1, 10, ["folders", "runs"]) time.sleep(1) - + # Get online ID and retrieve run _online_id = _run._local_staging_file.parents[1].joinpath("server_ids", f"{_run._local_staging_file.name.split('.')[0]}.txt").read_text() _online_run = Run(_online_id) - + assert _online_run.name == _run_name assert _online_run.folder == _folder_name - + _run.delete() _run._local_staging_file.parents[1].joinpath("server_ids", f"{_run._local_staging_file.name.split('.')[0]}.txt").unlink() client = Client() @@ -117,40 +117,51 @@ def test_run_modification_offline() -> None: assert _new_run.ttl == 120 assert _new_run.description == "Simvue test run" assert _new_run.name == "simvue_test_run" - + sender(_run._local_staging_file.parents[1], 1, 10, ["folders", "runs"]) time.sleep(1) - + # Get online ID and retrieve run _online_id = _run._local_staging_file.parents[1].joinpath("server_ids", f"{_run._local_staging_file.name.split('.')[0]}.txt").read_text() _online_run = Run(_online_id) - + assert _online_run.ttl == 120 assert _online_run.description == "Simvue test run" assert _online_run.name == "simvue_test_run" assert _online_run.folder == _folder_name - + # Now add a new set of tags in offline mode and send _new_run.tags = ["simvue", "test", "tag"] _new_run.commit() - + # Shouldn't yet be available in the online run since it hasnt been sent _online_run.refresh() assert _online_run.tags == [] - + sender(_run._local_staging_file.parents[1], 1, 10, ["folders", "runs"]) time.sleep(1) - + _online_run.refresh() assert sorted(_new_run.tags) == sorted(["simvue", "test", "tag"]) assert sorted(_online_run.tags) == sorted(["simvue", "test", "tag"]) - + _run.delete() _run._local_staging_file.parents[1].joinpath("server_ids", f"{_run._local_staging_file.name.split('.')[0]}.txt").unlink() client = Client() client.delete_folder(_folder_name, recursive=True, remove_runs=True) +@pytest.mark.api +@pytest.mark.online +def test_get_run() -> None: + _uuid: str = f"{uuid.uuid4()}".split("-")[0] + _folder_name = f"/simvue_unit_testing/{_uuid}" + _folder = Folder.new(path=_folder_name) + _run_1 = Run.new(folder=_folder_name) + _run_2 = Run.new(folder=_folder_name) + assert len(list(Run._get_all_objects(count=2, offset=None))) == 2 + + @pytest.mark.api @pytest.mark.online def test_run_get_properties() -> None: From a333e9ea375e994607d27a352c0f74b8149caae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Mon, 7 Apr 2025 16:12:31 +0100 Subject: [PATCH 07/23] Fix tests --- tests/unit/test_folder.py | 4 ++-- tests/unit/test_run.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_folder.py b/tests/unit/test_folder.py index 0cfc68c5..ae47651c 100644 --- a/tests/unit/test_folder.py +++ b/tests/unit/test_folder.py @@ -56,12 +56,12 @@ def test_folder_creation_offline() -> None: @pytest.mark.api @pytest.mark.online -def test_get_folder() -> None: +def test_get_folder_count() -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _folder_name = f"/simvue_unit_testing/{_uuid}" _folder_1 = Folder.new(path=f"{_folder_name}/dir_1") _folder_2 = Folder.new(path=f"{_folder_name}/dir_2") - assert len(list(Folder._get_all_objects(count=2, offset=None))) == 2 + assert len(list(Folder.get(count=2, offset=None))) == 2 @pytest.mark.api diff --git a/tests/unit/test_run.py b/tests/unit/test_run.py index 5d9b12b3..0db9c09f 100644 --- a/tests/unit/test_run.py +++ b/tests/unit/test_run.py @@ -153,13 +153,13 @@ def test_run_modification_offline() -> None: @pytest.mark.api @pytest.mark.online -def test_get_run() -> None: +def test_get_run_count() -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _folder_name = f"/simvue_unit_testing/{_uuid}" _folder = Folder.new(path=_folder_name) _run_1 = Run.new(folder=_folder_name) _run_2 = Run.new(folder=_folder_name) - assert len(list(Run._get_all_objects(count=2, offset=None))) == 2 + assert len(list(Run.get(count=2, offset=None))) == 2 @pytest.mark.api From 148d2c6bab7fe904b3fb4a44109531b7ea8995d2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 7 Apr 2025 16:40:28 +0000 Subject: [PATCH 08/23] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.11.2 → v0.11.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.11.2...v0.11.4) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d3e1a49..e1dc3af9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,7 +23,7 @@ repos: args: [--branch, main, --branch, dev] - id: check-added-large-files - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.2 + rev: v0.11.4 hooks: - id: ruff args: [ --fix, --exit-non-zero-on-fix, "--ignore=C901" ] From d167935d34abb90b6c0ff5dd9e1bae3cf154454b Mon Sep 17 00:00:00 2001 From: Matt Field Date: Thu, 10 Apr 2025 10:35:58 +0100 Subject: [PATCH 09/23] Fixed units --- simvue/eco/emissions_monitor.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/simvue/eco/emissions_monitor.py b/simvue/eco/emissions_monitor.py index 02b32b34..06459203 100644 --- a/simvue/eco/emissions_monitor.py +++ b/simvue/eco/emissions_monitor.py @@ -229,7 +229,6 @@ def estimate_co2_emissions( if self.co2_intensity: _current_co2_intensity = self.co2_intensity - _co2_units = "kgCO2/kWh" else: self.check_refresh() # If no local data yet then return @@ -251,10 +250,8 @@ def estimate_co2_emissions( **self._local_data[_country_code] ) _current_co2_intensity = self._current_co2_data.data.carbon_intensity - _co2_units = self._current_co2_data.carbon_intensity_units _process.gpu_percentage = gpu_percent _process.cpu_percentage = cpu_percent - _previous_energy: float = _process.total_energy _process.power_usage = (_process.cpu_percentage / 100.0) * ( self.thermal_design_power_per_cpu / self.n_cores_per_cpu ) @@ -263,22 +260,21 @@ def estimate_co2_emissions( _process.power_usage += ( _process.gpu_percentage / 100.0 ) * self.thermal_design_power_per_gpu + # Convert W to kW + _process.power_usage /= 1000 + # Measure energy in kWh + _process.energy_delta = _process.power_usage * measure_interval / 3600 + _process.total_energy += _process.energy_delta - _process.total_energy += _process.power_usage * measure_interval - _process.energy_delta = _process.total_energy - _previous_energy - - # Measured value is in g/kWh, convert to kg/kWs - _carbon_intensity_kgpws: float = _current_co2_intensity / (60 * 60 * 1e3) - - _process.co2_delta = ( - _process.power_usage * _carbon_intensity_kgpws * measure_interval - ) + # Measured value is in g/kWh, convert to kg/kWh + _carbon_intensity: float = _current_co2_intensity / 1000 + _process.co2_delta = _process.energy_delta * _carbon_intensity _process.co2_emission += _process.co2_delta self._logger.debug( - f"📝 For process '{process_id}', recorded: CPU={_process.cpu_percentage:.2f}%, " - f"Power={_process.power_usage:.2f}W, CO2={_process.co2_emission:.2e}{_co2_units}" + f"📝 For process '{process_id}', in interval {measure_interval}, recorded: CPU={_process.cpu_percentage:.2f}%, " + f"Power={_process.power_usage:.2f}kW, Energy = {_process.energy_delta}kWh, CO2={_process.co2_delta:.2e}kg" ) def simvue_metrics(self) -> dict[str, float]: From 1aada8a5961272fafd2c230b1f76b931db7d1d76 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Thu, 10 Apr 2025 10:41:37 +0100 Subject: [PATCH 10/23] Fix tests --- tests/unit/test_ecoclient.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/unit/test_ecoclient.py b/tests/unit/test_ecoclient.py index aa2bedce..3e2d3f98 100644 --- a/tests/unit/test_ecoclient.py +++ b/tests/unit/test_ecoclient.py @@ -2,7 +2,7 @@ import pytest import time import pytest_mock - +import pytest import simvue.eco.api_client as sv_eco_api import simvue.eco.emissions_monitor as sv_eco_ems @@ -81,17 +81,15 @@ def test_co2_monitor_properties(mock_co2_signal) -> None: assert _ems_monitor.process_data["test_co2_monitor_properties"] # Will use this equation - # Power used = (TDP_cpu * cpu_percent / num_cores) + (TDP_gpu * gpu_percent) - assert _ems_monitor.total_power_usage == (80 * 0.2 * 1 / 4) + (130 * 0.4 * 1) + # Power used = (TDP_cpu * cpu_percent / num_cores) + (TDP_gpu * gpu_percent) / 1000 (for kW) + assert _ems_monitor.total_power_usage == pytest.approx(((80 * 0.2 * 1 / 4) + (130 * 0.4 * 1)) / 1000) - # Energy used = power used * measure interval - assert _ems_monitor.total_energy == _ems_monitor.total_power_usage * 2 + # Energy used = power used * measure interval / 3600 (for kWh) + assert _ems_monitor.total_energy == pytest.approx(_ems_monitor.total_power_usage * 2 / 3600) # CO2 emission = energy * CO2 intensity - # Need to convert CO2 intensity from g/kWh to kg/kWs - # So divide by 1000 (g -> kg) - # And divide by 60*60 (kWh -> kws) - assert _ems_monitor.total_co2_emission == _ems_monitor.total_energy * 40 / (60*60*1000) + # Need to convert CO2 intensity from g/kWh to kg/kWh, so divide by 1000 + assert _ems_monitor.total_co2_emission == pytest.approx(_ems_monitor.total_energy * 40 / 1000) _ems_monitor.estimate_co2_emissions(**_measure_params) # Check delta is half of total, since we've now called this twice From d9295d956209b7c38c30d0db8b25d2e7688ef568 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Tue, 15 Apr 2025 11:51:22 +0100 Subject: [PATCH 11/23] Change default interval to 1hr --- simvue/eco/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simvue/eco/config.py b/simvue/eco/config.py index 06e57761..3d924afa 100644 --- a/simvue/eco/config.py +++ b/simvue/eco/config.py @@ -37,7 +37,7 @@ class EcoConfig(pydantic.BaseModel): None, validate_default=True ) intensity_refresh_interval: pydantic.PositiveInt | str | None = pydantic.Field( - default="1 day", gt=2 * 60 + default="1 hour", gt=2 * 60 ) co2_intensity: float | None = None From f226f04819e79d22bdb374481b8f2ae988e3955d Mon Sep 17 00:00:00 2001 From: Matt Field Date: Tue, 15 Apr 2025 11:56:07 +0100 Subject: [PATCH 12/23] Fix sender usage of eco --- simvue/sender.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/simvue/sender.py b/simvue/sender.py index 47e8f7fc..a9599e73 100644 --- a/simvue/sender.py +++ b/simvue/sender.py @@ -151,7 +151,6 @@ def sender( max_workers: int = 5, threading_threshold: int = 10, objects_to_upload: list[str] = UPLOAD_ORDER, - co2_intensity_refresh: int | None | str = None, ) -> dict[str, str]: """Send data from a local cache directory to the Simvue server. @@ -165,9 +164,6 @@ def sender( The number of cached files above which threading will be used objects_to_upload : list[str] Types of objects to upload, by default uploads all types of objects present in cache - co2_intensity_refresh: int | None | str - the refresh interval for the CO2 intensity value, if None use config value if available, - else do not refresh. Returns ------- @@ -250,16 +246,13 @@ def sender( # will be taken by the sender itself, values are assumed to be recorded # by any offline runs being sent. - if ( - _refresh_interval := co2_intensity_refresh - or _user_config.eco.intensity_refresh_interval - ): + if _user_config.metrics.enable_emission_metrics: CO2Monitor( thermal_design_power_per_gpu=None, thermal_design_power_per_cpu=None, local_data_directory=cache_dir, - intensity_refresh_interval=_refresh_interval, - co2_intensity=co2_intensity_refresh or _user_config.eco.co2_intensity, + intensity_refresh_interval=_user_config.eco.intensity_refresh_interval, + co2_intensity=_user_config.eco.co2_intensity, co2_signal_api_token=_user_config.eco.co2_signal_api_token, ).check_refresh() From a17f42245c02486ddf5903bf779dc9ce2cd50ada Mon Sep 17 00:00:00 2001 From: Matt Field Date: Wed, 16 Apr 2025 11:45:43 +0100 Subject: [PATCH 13/23] Fix events get --- simvue/api/objects/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simvue/api/objects/events.py b/simvue/api/objects/events.py index b32a7445..ff5b9e50 100644 --- a/simvue/api/objects/events.py +++ b/simvue/api/objects/events.py @@ -51,7 +51,7 @@ def get( _class_instance = cls(_read_only=True, _local=True) _count: int = 0 - for response in cls._get_all_objects(offset, run=run_id, **kwargs): + for response in cls._get_all_objects(offset, count=count, run=run_id, **kwargs): if (_data := response.get("data")) is None: raise RuntimeError( f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s" From 0f2b275169a32cc014743da56893e0218ad91938 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Thu, 17 Apr 2025 11:37:35 +0100 Subject: [PATCH 14/23] Fix get_paginated for metrics and set offline dir to tmp for all unit tests --- simvue/api/request.py | 20 ++++++++------------ tests/conftest.py | 12 ++++++++++-- tests/unit/test_event_alert.py | 4 ++-- tests/unit/test_events.py | 2 +- tests/unit/test_file_artifact.py | 2 +- tests/unit/test_file_storage.py | 2 +- tests/unit/test_folder.py | 4 ++-- tests/unit/test_metric_range_alert.py | 4 ++-- tests/unit/test_metric_threshold_alert.py | 4 ++-- tests/unit/test_metrics.py | 2 +- tests/unit/test_object_artifact.py | 2 +- tests/unit/test_run.py | 4 ++-- tests/unit/test_s3_storage.py | 2 +- tests/unit/test_tag.py | 4 ++-- tests/unit/test_tenant.py | 2 +- tests/unit/test_user.py | 2 +- tests/unit/test_user_alert.py | 6 +++--- 17 files changed, 41 insertions(+), 37 deletions(-) diff --git a/simvue/api/request.py b/simvue/api/request.py index 6a0c05a3..3d3ac6f7 100644 --- a/simvue/api/request.py +++ b/simvue/api/request.py @@ -305,19 +305,15 @@ def get_paginated( _offset: int = offset or 0 while ( - ( - _response := get( - url=url, - headers=headers, - params=(params or {}) - | {"count": count or MAX_ENTRIES_PER_PAGE, "start": _offset}, - timeout=timeout, - json=json, - ) + _response := get( + url=url, + headers=headers, + params=(params or {}) + | {"count": count or MAX_ENTRIES_PER_PAGE, "start": _offset}, + timeout=timeout, + json=json, ) - .json() - .get("data") - ): + ).json(): yield _response _offset += MAX_ENTRIES_PER_PAGE diff --git a/tests/conftest.py b/tests/conftest.py index c09f84da..fd7ee293 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -48,8 +48,16 @@ def clear_out_files() -> None: for file_obj in out_files: file_obj.unlink() - - + +@pytest.fixture(autouse=True) +def offline_cache_setup(monkeypatch: monkeypatch.MonkeyPatch): + # Will be executed before the test + cache_dir = tempfile.TemporaryDirectory() + monkeypatch.setenv("SIMVUE_OFFLINE_DIRECTORY", cache_dir.name) + yield cache_dir + # Will be executed after the test + cache_dir.cleanup() + @pytest.fixture def mock_co2_signal(monkeypatch: monkeypatch.MonkeyPatch) -> dict[str, dict | str]: _mock_data = { diff --git a/tests/unit/test_event_alert.py b/tests/unit/test_event_alert.py index d55fd08c..254a44d7 100644 --- a/tests/unit/test_event_alert.py +++ b/tests/unit/test_event_alert.py @@ -30,7 +30,7 @@ def test_event_alert_creation_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_event_alert_creation_offline() -> None: +def test_event_alert_creation_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _alert = EventsAlert.new( name=f"events_alert_{_uuid}", @@ -95,7 +95,7 @@ def test_event_alert_modification_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_event_alert_modification_offline() -> None: +def test_event_alert_modification_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _alert = EventsAlert.new( name=f"events_alert_{_uuid}", diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index fb5d0587..41c2d2c1 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -33,7 +33,7 @@ def test_events_creation_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_events_creation_offline() -> None: +def test_events_creation_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _folder_name = f"/simvue_unit_testing/{_uuid}" _folder = Folder.new(path=_folder_name, offline=True) diff --git a/tests/unit/test_file_artifact.py b/tests/unit/test_file_artifact.py index f21b8ca1..89cac771 100644 --- a/tests/unit/test_file_artifact.py +++ b/tests/unit/test_file_artifact.py @@ -51,7 +51,7 @@ def test_file_artifact_creation_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_file_artifact_creation_offline(offline_test: pathlib.Path) -> None: +def test_file_artifact_creation_offline(offline_test: pathlib.Path, offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _folder_name = f"/simvue_unit_testing/{_uuid}" _folder = Folder.new(path=_folder_name, offline=True) diff --git a/tests/unit/test_file_storage.py b/tests/unit/test_file_storage.py index 46f053cd..57eb0d24 100644 --- a/tests/unit/test_file_storage.py +++ b/tests/unit/test_file_storage.py @@ -23,7 +23,7 @@ def test_create_file_storage_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_create_file_storage_offline() -> None: +def test_create_file_storage_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _storage = FileStorage.new(name=_uuid, disable_check=True, is_tenant_useable=False, is_default=False, offline=True, is_enabled=False) diff --git a/tests/unit/test_folder.py b/tests/unit/test_folder.py index ae47651c..c3b0e6aa 100644 --- a/tests/unit/test_folder.py +++ b/tests/unit/test_folder.py @@ -30,7 +30,7 @@ def test_folder_creation_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_folder_creation_offline() -> None: +def test_folder_creation_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _path = f"/simvue_unit_testing/objects/folder/{_uuid}" _folder = Folder.new(path=_path, offline=True) @@ -88,7 +88,7 @@ def test_folder_modification_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_folder_modification_offline() -> None: +def test_folder_modification_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _path = f"/simvue_unit_testing/objects/folder/{_uuid}" _description = "Test study" diff --git a/tests/unit/test_metric_range_alert.py b/tests/unit/test_metric_range_alert.py index 4079acb7..3707e0da 100644 --- a/tests/unit/test_metric_range_alert.py +++ b/tests/unit/test_metric_range_alert.py @@ -34,7 +34,7 @@ def test_metric_range_alert_creation_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_metric_range_alert_creation_offline() -> None: +def test_metric_range_alert_creation_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _alert = MetricsRangeAlert.new( name=f"metrics_range_alert_{_uuid}", @@ -108,7 +108,7 @@ def test_metric_range_alert_modification_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_metric_range_alert_modification_offline() -> None: +def test_metric_range_alert_modification_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _alert = MetricsRangeAlert.new( name=f"metrics_range_alert_{_uuid}", diff --git a/tests/unit/test_metric_threshold_alert.py b/tests/unit/test_metric_threshold_alert.py index 6d737945..014cd8d9 100644 --- a/tests/unit/test_metric_threshold_alert.py +++ b/tests/unit/test_metric_threshold_alert.py @@ -32,7 +32,7 @@ def test_metric_threshold_alert_creation_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_metric_threshold_alert_creation_offline() -> None: +def test_metric_threshold_alert_creation_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _alert = MetricsThresholdAlert.new( name=f"metrics_threshold_alert_{_uuid}", @@ -107,7 +107,7 @@ def test_metric_threshold_alert_modification_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_metric_threshold_alert_modification_offline() -> None: +def test_metric_threshold_alert_modification_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _alert = MetricsThresholdAlert.new( name=f"metrics_threshold_alert_{_uuid}", diff --git a/tests/unit/test_metrics.py b/tests/unit/test_metrics.py index f3bcb947..bef9ea23 100644 --- a/tests/unit/test_metrics.py +++ b/tests/unit/test_metrics.py @@ -50,7 +50,7 @@ def test_metrics_creation_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_metrics_creation_offline() -> None: +def test_metrics_creation_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _folder_name = f"/simvue_unit_testing/{_uuid}" _folder = Folder.new(path=_folder_name, offline=True) diff --git a/tests/unit/test_object_artifact.py b/tests/unit/test_object_artifact.py index ae1bd464..47b3342f 100644 --- a/tests/unit/test_object_artifact.py +++ b/tests/unit/test_object_artifact.py @@ -38,7 +38,7 @@ def test_object_artifact_creation_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_object_artifact_creation_offline(offline_test: pathlib.Path) -> None: +def test_object_artifact_creation_offline(offline_test: pathlib.Path, offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _folder_name = f"/simvue_unit_testing/{_uuid}" _folder = Folder.new(path=_folder_name, offline=True) diff --git a/tests/unit/test_run.py b/tests/unit/test_run.py index 0db9c09f..b560da43 100644 --- a/tests/unit/test_run.py +++ b/tests/unit/test_run.py @@ -24,7 +24,7 @@ def test_run_creation_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_run_creation_offline() -> None: +def test_run_creation_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _run_name = f"simvue_offline_run_{_uuid}" _folder_name = f"/simvue_unit_testing/{_uuid}" @@ -91,7 +91,7 @@ def test_run_modification_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_run_modification_offline() -> None: +def test_run_modification_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _run_name = f"simvue_offline_run_{_uuid}" _folder_name = f"/simvue_unit_testing/{_uuid}" diff --git a/tests/unit/test_s3_storage.py b/tests/unit/test_s3_storage.py index 8c80a75f..8fc96b45 100644 --- a/tests/unit/test_s3_storage.py +++ b/tests/unit/test_s3_storage.py @@ -44,7 +44,7 @@ def test_create_s3_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_create_s3_offline() -> None: +def test_create_s3_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _storage = S3Storage.new( name=_uuid, diff --git a/tests/unit/test_tag.py b/tests/unit/test_tag.py index 1ddac7b1..4959f3f7 100644 --- a/tests/unit/test_tag.py +++ b/tests/unit/test_tag.py @@ -21,7 +21,7 @@ def test_tag_creation_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_tag_creation_offline() -> None: +def test_tag_creation_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _tag = Tag.new(name=f"test_tag_{_uuid}", offline=True) _tag.commit() @@ -68,7 +68,7 @@ def test_tag_modification_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_tag_modification_offline() -> None: +def test_tag_modification_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _tag = Tag.new(name=f"test_tag_{_uuid}", offline=True) _tag.commit() diff --git a/tests/unit/test_tenant.py b/tests/unit/test_tenant.py index 82cf001d..38e3f9e3 100644 --- a/tests/unit/test_tenant.py +++ b/tests/unit/test_tenant.py @@ -26,7 +26,7 @@ def test_create_tenant_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_create_tenant_offline() -> None: +def test_create_tenant_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _tenant = Tenant.new(name=_uuid, offline=True) _tenant.commit() diff --git a/tests/unit/test_user.py b/tests/unit/test_user.py index 97e503c3..5a23349a 100644 --- a/tests/unit/test_user.py +++ b/tests/unit/test_user.py @@ -38,7 +38,7 @@ def test_create_user_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_create_user_offline() -> None: +def test_create_user_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _user = User.new( username="jbloggs", diff --git a/tests/unit/test_user_alert.py b/tests/unit/test_user_alert.py index b1625e37..7984ecce 100644 --- a/tests/unit/test_user_alert.py +++ b/tests/unit/test_user_alert.py @@ -26,7 +26,7 @@ def test_user_alert_creation_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_user_alert_creation_offline() -> None: +def test_user_alert_creation_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _alert = UserAlert.new( name=f"users_alert_{_uuid}", @@ -84,7 +84,7 @@ def test_user_alert_modification_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_user_alert_modification_offline() -> None: +def test_user_alert_modification_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _alert = UserAlert.new( name=f"users_alert_{_uuid}", @@ -176,7 +176,7 @@ def test_user_alert_status() -> None: @pytest.mark.api @pytest.mark.offline -def test_user_alert_status_offline() -> None: +def test_user_alert_status_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _alert = UserAlert.new( name=f"users_alert_{_uuid}", From 3c6faa6d8a25a7385e5bef442ea0fcfb0d81cadc Mon Sep 17 00:00:00 2001 From: Matt Field Date: Thu, 17 Apr 2025 17:40:47 +0100 Subject: [PATCH 15/23] Lots of fixes --- simvue/api/request.py | 5 +++-- simvue/client.py | 6 ++++-- simvue/eco/api_client.py | 7 ++----- simvue/eco/emissions_monitor.py | 9 +++++++++ tests/functional/test_run_class.py | 1 + 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/simvue/api/request.py b/simvue/api/request.py index 3d3ac6f7..eabb24ae 100644 --- a/simvue/api/request.py +++ b/simvue/api/request.py @@ -303,7 +303,6 @@ def get_paginated( server response """ _offset: int = offset or 0 - while ( _response := get( url=url, @@ -317,5 +316,7 @@ def get_paginated( yield _response _offset += MAX_ENTRIES_PER_PAGE - if count and _offset > count: + if (count and _offset > count) or ( + _response.json().get("count", 0) < (count or MAX_ENTRIES_PER_PAGE) + ): break diff --git a/simvue/client.py b/simvue/client.py index 82b4fc75..685e1c12 100644 --- a/simvue/client.py +++ b/simvue/client.py @@ -835,7 +835,8 @@ def get_metric_values( _args = {"filters": json.dumps(run_filters)} if run_filters else {} - _run_data = dict(Run.get(**_args)) + if not run_ids: + _run_data = dict(Run.get(**_args)) if not ( _run_metrics := self._get_run_metrics_from_server( @@ -853,7 +854,8 @@ def get_metric_values( ) if use_run_names: _run_metrics = { - _run_data[key].name: _run_metrics[key] for key in _run_metrics.keys() + Run(identifier=key).name: _run_metrics[key] + for key in _run_metrics.keys() } return parse_run_set_metrics( _run_metrics, diff --git a/simvue/eco/api_client.py b/simvue/eco/api_client.py index dc03c8b7..9d3b094f 100644 --- a/simvue/eco/api_client.py +++ b/simvue/eco/api_client.py @@ -77,7 +77,7 @@ def __init__(self, *args, **kwargs) -> None: co2_api_endpoint : str endpoint for CO2 signal API co2_api_token: str - RECOMMENDED. The API token for the CO2 Signal API, default is None. + The API token for the ElectricityMaps API, default is None. timeout : int timeout for API """ @@ -85,10 +85,7 @@ def __init__(self, *args, **kwargs) -> None: self._logger = logging.getLogger(self.__class__.__name__) if not self.co2_api_token: - self._logger.warning( - "⚠️ No API token provided for CO2 Signal, " - "use of a token is strongly recommended." - ) + raise ValueError("API token is required for ElectricityMaps API.") self._get_user_location_info() diff --git a/simvue/eco/emissions_monitor.py b/simvue/eco/emissions_monitor.py index 06459203..de2bbe9d 100644 --- a/simvue/eco/emissions_monitor.py +++ b/simvue/eco/emissions_monitor.py @@ -108,6 +108,15 @@ def __init__(self, *args, **kwargs) -> None: """ _logger = logging.getLogger(self.__class__.__name__) + if not ( + kwargs.get("co2_intensity") + or kwargs.get("co2_signal_api_token") + or kwargs.get("offline") + ): + raise ValueError( + "ElectricityMaps API token or hardcoeded CO2 intensity value is required for emissions tracking." + ) + if not isinstance(kwargs.get("thermal_design_power_per_cpu"), float): kwargs["thermal_design_power_per_cpu"] = 80.0 _logger.warning( diff --git a/tests/functional/test_run_class.py b/tests/functional/test_run_class.py index 527d63da..cdd1b4e3 100644 --- a/tests/functional/test_run_class.py +++ b/tests/functional/test_run_class.py @@ -79,6 +79,7 @@ def test_run_with_emissions_offline(speedy_heartbeat, mock_co2_signal, create_pl run_created, _ = create_plain_run_offline run_created.config(enable_emission_metrics=True) time.sleep(2) + # Run should continue, but fail to log metrics until sender runs and creates file id_mapping = sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"]) _run = RunObject(identifier=id_mapping[run_created.id]) _metric_names = [item[0] for item in _run.metrics] From d4119b0e2b0ea80a02b661d1c4f2e1184ab611e7 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Tue, 22 Apr 2025 22:45:56 +0100 Subject: [PATCH 16/23] more fixes --- simvue/api/request.py | 4 +-- simvue/eco/emissions_monitor.py | 3 +- simvue/run.py | 13 +++++---- simvue/sender.py | 1 - tests/conftest.py | 19 +++++++++---- tests/functional/test_config.py | 44 +++++++++++++++--------------- tests/functional/test_run_class.py | 35 +++++++++++++++++++----- tests/unit/test_ecoclient.py | 8 +++--- 8 files changed, 78 insertions(+), 49 deletions(-) diff --git a/simvue/api/request.py b/simvue/api/request.py index eabb24ae..2ae11a8d 100644 --- a/simvue/api/request.py +++ b/simvue/api/request.py @@ -316,7 +316,5 @@ def get_paginated( yield _response _offset += MAX_ENTRIES_PER_PAGE - if (count and _offset > count) or ( - _response.json().get("count", 0) < (count or MAX_ENTRIES_PER_PAGE) - ): + if (count and _offset > count) or (_response.json().get("count", 0) < _offset): break diff --git a/simvue/eco/emissions_monitor.py b/simvue/eco/emissions_monitor.py index de2bbe9d..0dc32d5c 100644 --- a/simvue/eco/emissions_monitor.py +++ b/simvue/eco/emissions_monitor.py @@ -246,7 +246,7 @@ def estimate_co2_emissions( "No CO2 emission data recorded as no CO2 intensity value " "has been provided and there is no local intensity data available." ) - return + return False if self._client: _country_code = self._client.country_code @@ -285,6 +285,7 @@ def estimate_co2_emissions( f"📝 For process '{process_id}', in interval {measure_interval}, recorded: CPU={_process.cpu_percentage:.2f}%, " f"Power={_process.power_usage:.2f}kW, Energy = {_process.energy_delta}kWh, CO2={_process.co2_delta:.2e}kg" ) + return True def simvue_metrics(self) -> dict[str, float]: """Retrieve metrics to send to Simvue server.""" diff --git a/simvue/run.py b/simvue/run.py index 099016f1..d06dc44d 100644 --- a/simvue/run.py +++ b/simvue/run.py @@ -374,7 +374,7 @@ def _get_internal_metrics( # For the first emissions metrics reading, the time interval to use # Is the time since the run started, otherwise just use the time between readings if self._emissions_monitor: - self._emissions_monitor.estimate_co2_emissions( + _estimated = self._emissions_monitor.estimate_co2_emissions( process_id=f"{self._name}", cpu_percent=_current_system_measure.cpu_percent, measure_interval=(time.time() - self._start_time) @@ -382,11 +382,12 @@ def _get_internal_metrics( else self._system_metrics_interval, gpu_percent=_current_system_measure.gpu_percent, ) - self._add_metrics_to_dispatch( - self._emissions_monitor.simvue_metrics(), - join_on_fail=False, - step=system_metrics_step, - ) + if _estimated: + self._add_metrics_to_dispatch( + self._emissions_monitor.simvue_metrics(), + join_on_fail=False, + step=system_metrics_step, + ) def _create_heartbeat_callback( self, diff --git a/simvue/sender.py b/simvue/sender.py index a9599e73..ea5e6e68 100644 --- a/simvue/sender.py +++ b/simvue/sender.py @@ -245,7 +245,6 @@ def sender( # refreshes the CO2 intensity value if required. No emission metrics # will be taken by the sender itself, values are assumed to be recorded # by any offline runs being sent. - if _user_config.metrics.enable_emission_metrics: CO2Monitor( thermal_design_power_per_gpu=None, diff --git a/tests/conftest.py b/tests/conftest.py index fd7ee293..8178fdcd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,14 +49,15 @@ def clear_out_files() -> None: for file_obj in out_files: file_obj.unlink() -@pytest.fixture(autouse=True) +@pytest.fixture() def offline_cache_setup(monkeypatch: monkeypatch.MonkeyPatch): - # Will be executed before the test + # Will be executed before test cache_dir = tempfile.TemporaryDirectory() monkeypatch.setenv("SIMVUE_OFFLINE_DIRECTORY", cache_dir.name) yield cache_dir - # Will be executed after the test + # Will be executed after test cache_dir.cleanup() + monkeypatch.setenv("SIMVUE_OFFLINE_DIRECTORY", None) @pytest.fixture def mock_co2_signal(monkeypatch: monkeypatch.MonkeyPatch) -> dict[str, dict | str]: @@ -94,13 +95,21 @@ def _mock_location_info(self) -> None: monkeypatch.setattr(requests, "get", _mock_get) monkeypatch.setattr(sv_eco.APIClient, "_get_user_location_info", _mock_location_info) - + + _fetch = sv_cfg.SimvueConfiguration.fetch + @classmethod + def _mock_fetch(cls, *args, **kwargs) -> sv_cfg.SimvueConfiguration: + _conf = _fetch(*args, **kwargs) + _conf.eco.co2_signal_api_token = "test_token" + _conf.metrics.enable_emission_metrics = True + return _conf + monkeypatch.setattr(sv_cfg.SimvueConfiguration, "fetch", _mock_fetch) return _mock_data @pytest.fixture def speedy_heartbeat(monkeypatch: monkeypatch.MonkeyPatch) -> None: - monkeypatch.setattr(sv_run, "HEARTBEAT_INTERVAL", 0.1) + monkeypatch.setattr(sv_run, "HEARTBEAT_INTERVAL", 1) @pytest.fixture(autouse=True) diff --git a/tests/functional/test_config.py b/tests/functional/test_config.py index 29534e5e..be34d7a8 100644 --- a/tests/functional/test_config.py +++ b/tests/functional/test_config.py @@ -56,35 +56,35 @@ def test_config_setup( if use_file: if use_file == "pyproject.toml": _lines_ppt: str = f""" -[tool.poetry] -name = "simvue_testing" -version = "0.1.0" -description = "A dummy test project" - -[tool.simvue.run] -description = "{_description_ppt}" -folder = "{_folder_ppt}" -tags = {_tags_ppt} -""" + [tool.poetry] + name = "simvue_testing" + version = "0.1.0" + description = "A dummy test project" + + [tool.simvue.run] + description = "{_description_ppt}" + folder = "{_folder_ppt}" + tags = {_tags_ppt} + """ with open((_ppt_file := pathlib.Path(temp_d).joinpath("pyproject.toml")), "w") as out_f: out_f.write(_lines_ppt) with open(_config_file := pathlib.Path(temp_d).joinpath("simvue.toml"), "w") as out_f: _lines: str = f""" -[server] -url = "{_url}" -token = "{_token}" + [server] + url = "{_url}" + token = "{_token}" -[offline] -cache = "{temp_d}" -""" + [offline] + cache = "{temp_d}" + """ if use_file == "extended": _lines += f""" -[run] -description = "{_description}" -folder = "{_folder}" -tags = {_tags} -""" + [run] + description = "{_description}" + folder = "{_folder}" + tags = {_tags} + """ out_f.write(_lines) SimvueConfiguration.config_file.cache_clear() @@ -109,7 +109,7 @@ def _mocked_find(file_names: list[str], *_, ppt_file=_ppt_file, conf_file=_confi ) else: _config = simvue.config.user.SimvueConfiguration.fetch() - + if use_file and use_file != "pyproject.toml": assert _config.config_file() == _config_file diff --git a/tests/functional/test_run_class.py b/tests/functional/test_run_class.py index cdd1b4e3..58a17b7b 100644 --- a/tests/functional/test_run_class.py +++ b/tests/functional/test_run_class.py @@ -22,6 +22,7 @@ import simvue.run as sv_run import simvue.client as sv_cl import simvue.sender as sv_send +import simvue.config.user as sv_cfg from simvue.api.objects import Run as RunObject @@ -53,8 +54,9 @@ def test_check_run_initialised_decorator() -> None: @pytest.mark.online def test_run_with_emissions_online(speedy_heartbeat, mock_co2_signal, create_plain_run) -> None: run_created, _ = create_plain_run + run_created._user_config.eco.co2_signal_api_token = "test_token" run_created.config(enable_emission_metrics=True) - time.sleep(3) + time.sleep(5) _run = RunObject(identifier=run_created.id) _metric_names = [item[0] for item in _run.metrics] client = sv_cl.Client() @@ -69,20 +71,34 @@ def test_run_with_emissions_online(speedy_heartbeat, mock_co2_signal, create_pla output_format="dataframe", run_ids=[run_created.id], ) - assert _total_metric_name in _metric_values - + # Check that total = previous total + latest delta + _total_values = _metric_values[_total_metric_name].tolist() + _delta_values = _metric_values[_delta_metric_name].tolist() + assert len(_total_values) > 1 + for i in range(1, len(_total_values)): + assert _total_values[i] == _total_values[i - 1] + _delta_values[i] @pytest.mark.run @pytest.mark.eco @pytest.mark.offline -def test_run_with_emissions_offline(speedy_heartbeat, mock_co2_signal, create_plain_run_offline) -> None: +def test_run_with_emissions_offline(speedy_heartbeat, mock_co2_signal, create_plain_run_offline, monkeypatch) -> None: run_created, _ = create_plain_run_offline run_created.config(enable_emission_metrics=True) - time.sleep(2) + time.sleep(5) # Run should continue, but fail to log metrics until sender runs and creates file id_mapping = sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"]) _run = RunObject(identifier=id_mapping[run_created.id]) _metric_names = [item[0] for item in _run.metrics] + for _metric in ["emissions", "energy_consumed"]: + _total_metric_name = f"sustainability.{_metric}.total" + _delta_metric_name = f"sustainability.{_metric}.delta" + assert _total_metric_name not in _metric_names + assert _delta_metric_name not in _metric_names + # Sender should now have made a local file, and the run should be able to use it to create emissions metrics + time.sleep(5) + id_mapping = sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"]) + _run.refresh() + _metric_names = [item[0] for item in _run.metrics] client = sv_cl.Client() for _metric in ["emissions", "energy_consumed"]: _total_metric_name = f"sustainability.{_metric}.total" @@ -95,8 +111,13 @@ def test_run_with_emissions_offline(speedy_heartbeat, mock_co2_signal, create_pl output_format="dataframe", run_ids=[id_mapping[run_created.id]], ) - assert _total_metric_name in _metric_values - + # Check that total = previous total + latest delta + _total_values = _metric_values[_total_metric_name].tolist() + _delta_values = _metric_values[_delta_metric_name].tolist() + assert len(_total_values) > 1 + for i in range(1, len(_total_values)): + assert _total_values[i] == _total_values[i - 1] + _delta_values[i] + @pytest.mark.run @pytest.mark.parametrize( "timestamp", diff --git a/tests/unit/test_ecoclient.py b/tests/unit/test_ecoclient.py index 3e2d3f98..f7b922ef 100644 --- a/tests/unit/test_ecoclient.py +++ b/tests/unit/test_ecoclient.py @@ -8,7 +8,7 @@ @pytest.mark.eco def test_api_client_get_loc_info(mock_co2_signal) -> None: - _client = sv_eco_api.APIClient() + _client = sv_eco_api.APIClient(co2_api_token="test_token") assert _client.latitude assert _client.longitude assert _client.country_code @@ -16,7 +16,7 @@ def test_api_client_get_loc_info(mock_co2_signal) -> None: @pytest.mark.eco def test_api_client_query(mock_co2_signal: dict[str, dict | str]) -> None: - _client = sv_eco_api.APIClient() + _client = sv_eco_api.APIClient(co2_api_token="test_token") _response: sv_eco_api.CO2SignalResponse = _client.get() assert _response.carbon_intensity_units == "gCO2e/kWh" assert _response.country_code == mock_co2_signal["zone"] @@ -42,7 +42,7 @@ def test_outdated_data_check( local_data_directory=tempd, intensity_refresh_interval=1 if refresh else None, co2_intensity=None, - co2_signal_api_token=None + co2_signal_api_token="test_token" ) _measure_params = { "process_id": "test_outdated_data_check", @@ -65,7 +65,7 @@ def test_co2_monitor_properties(mock_co2_signal) -> None: local_data_directory=tempd, intensity_refresh_interval=None, co2_intensity=40, - co2_signal_api_token=None + co2_signal_api_token="test_token" ) _measure_params = { "process_id": "test_co2_monitor_properties", From 2429cdd4280d028a9b2371f56a9c13d03e199077 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Tue, 22 Apr 2025 22:48:30 +0100 Subject: [PATCH 17/23] Got rid of ecoclient specific local data dir in favour of just using offline cache --- simvue/eco/config.py | 21 --------------------- simvue/run.py | 4 ++-- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/simvue/eco/config.py b/simvue/eco/config.py index 3d924afa..7e855b7c 100644 --- a/simvue/eco/config.py +++ b/simvue/eco/config.py @@ -8,10 +8,6 @@ __date__ = "2025-03-06" import pydantic -import pathlib -import os - -from simvue.config.files import DEFAULT_OFFLINE_DIRECTORY class EcoConfig(pydantic.BaseModel): @@ -25,30 +21,13 @@ class EcoConfig(pydantic.BaseModel): the TDP for the CPU gpu_thermal_design_power: int | None, optional the TDP for each GPU - local_data_directory: str, optional - the directory to store local data, default is Simvue offline directory """ co2_signal_api_token: pydantic.SecretStr | None = None cpu_thermal_design_power: pydantic.PositiveInt | None = None cpu_n_cores: pydantic.PositiveInt | None = None gpu_thermal_design_power: pydantic.PositiveInt | None = None - local_data_directory: pydantic.DirectoryPath | None = pydantic.Field( - None, validate_default=True - ) intensity_refresh_interval: pydantic.PositiveInt | str | None = pydantic.Field( default="1 hour", gt=2 * 60 ) co2_intensity: float | None = None - - @pydantic.field_validator("local_data_directory", mode="before", check_fields=True) - @classmethod - def check_local_data_env( - cls, local_data_directory: pathlib.Path | None - ) -> pathlib.Path: - if _data_directory := os.environ.get("SIMVUE_ECO_DATA_DIRECTORY"): - return pathlib.Path(_data_directory) - if not local_data_directory: - local_data_directory = pathlib.Path(DEFAULT_OFFLINE_DIRECTORY) - local_data_directory.mkdir(exist_ok=True, parents=True) - return local_data_directory diff --git a/simvue/run.py b/simvue/run.py index d06dc44d..e950d6cd 100644 --- a/simvue/run.py +++ b/simvue/run.py @@ -1084,7 +1084,7 @@ def config( self._emissions_monitor = CO2Monitor( intensity_refresh_interval=None, co2_intensity=self._user_config.eco.co2_intensity, - local_data_directory=self._user_config.eco.local_data_directory, + local_data_directory=self._user_config.offline.cache, co2_signal_api_token=None, thermal_design_power_per_cpu=self._user_config.eco.cpu_thermal_design_power, thermal_design_power_per_gpu=self._user_config.eco.gpu_thermal_design_power, @@ -1093,7 +1093,7 @@ def config( else: self._emissions_monitor = CO2Monitor( intensity_refresh_interval=self._user_config.eco.intensity_refresh_interval, - local_data_directory=self._user_config.eco.local_data_directory, + local_data_directory=self._user_config.offline.cache, co2_signal_api_token=self._user_config.eco.co2_signal_api_token, co2_intensity=self._user_config.eco.co2_intensity, thermal_design_power_per_cpu=self._user_config.eco.cpu_thermal_design_power, From fd5a2240819d54982f9808db5c939975b5280725 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Tue, 22 Apr 2025 23:07:33 +0100 Subject: [PATCH 18/23] Fixed offline artifact tests --- tests/conftest.py | 12 ------------ tests/unit/test_file_artifact.py | 6 +++--- tests/unit/test_object_artifact.py | 4 ++-- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 8178fdcd..ed54f791 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -303,15 +303,3 @@ def setup_test_run(run: sv_run.Run, create_objects: bool, request: pytest.Fixtur TEST_DATA["alert_ids"] = _alert_ids return TEST_DATA - - -@pytest.fixture -def offline_test() -> pathlib.Path: - with tempfile.TemporaryDirectory() as tempd: - _tempdir = pathlib.Path(tempd) - _cache_dir = _tempdir.joinpath(".simvue") - _cache_dir.mkdir(exist_ok=True) - os.environ["SIMVUE_OFFLINE_DIRECTORY"] = f"{_cache_dir}" - assert sv_cfg.SimvueConfiguration.fetch().offline.cache == _cache_dir - yield _tempdir - diff --git a/tests/unit/test_file_artifact.py b/tests/unit/test_file_artifact.py index 89cac771..22a97ae2 100644 --- a/tests/unit/test_file_artifact.py +++ b/tests/unit/test_file_artifact.py @@ -51,13 +51,13 @@ def test_file_artifact_creation_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_file_artifact_creation_offline(offline_test: pathlib.Path, offline_cache_setup) -> None: +def test_file_artifact_creation_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _folder_name = f"/simvue_unit_testing/{_uuid}" _folder = Folder.new(path=_folder_name, offline=True) _run = Run.new(name=f"test_file_artifact_creation_offline_{_uuid}",folder=_folder_name, offline=True) - _path = offline_test.joinpath("hello_world.txt") + _path = pathlib.Path(offline_cache_setup.name).joinpath("hello_world.txt") with _path.open("w") as out_f: out_f.write(f"Hello World! {_uuid}") @@ -80,7 +80,7 @@ def test_file_artifact_creation_offline(offline_test: pathlib.Path, offline_cach assert _local_data.get("name") == f"test_file_artifact_{_uuid}" assert _local_data.get("runs") == {_run._identifier: "input"} - _id_mapping = sender(offline_test.joinpath(".simvue"), 1, 10) + _id_mapping = sender(pathlib.Path(offline_cache_setup.name), 1, 10) time.sleep(1) _online_artifact = Artifact(_id_mapping[_artifact.id]) diff --git a/tests/unit/test_object_artifact.py b/tests/unit/test_object_artifact.py index 47b3342f..0dfb5af1 100644 --- a/tests/unit/test_object_artifact.py +++ b/tests/unit/test_object_artifact.py @@ -38,7 +38,7 @@ def test_object_artifact_creation_online() -> None: @pytest.mark.api @pytest.mark.offline -def test_object_artifact_creation_offline(offline_test: pathlib.Path, offline_cache_setup) -> None: +def test_object_artifact_creation_offline(offline_cache_setup) -> None: _uuid: str = f"{uuid.uuid4()}".split("-")[0] _folder_name = f"/simvue_unit_testing/{_uuid}" _folder = Folder.new(path=_folder_name, offline=True) @@ -63,7 +63,7 @@ def test_object_artifact_creation_offline(offline_test: pathlib.Path, offline_ca assert _local_data.get("mime_type") == "application/vnd.simvue.numpy.v1" assert _local_data.get("runs") == {_run.id: "input"} - _id_mapping = sender(offline_test.joinpath(".simvue"), 1, 10) + _id_mapping = sender(pathlib.Path(offline_cache_setup.name), 1, 10) time.sleep(1) _online_artifact = Artifact(_id_mapping.get(_artifact.id)) From a835d0f7bd5cb8bf32c74ea25cd9a6b9f65094c2 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Tue, 22 Apr 2025 23:15:16 +0100 Subject: [PATCH 19/23] Make offline cache dir in configuration --- simvue/config/user.py | 1 + 1 file changed, 1 insertion(+) diff --git a/simvue/config/user.py b/simvue/config/user.py index 7b0c0631..f631fa01 100644 --- a/simvue/config/user.py +++ b/simvue/config/user.py @@ -200,6 +200,7 @@ def fetch( _default_dir = _config_dict["offline"].get( "cache", DEFAULT_OFFLINE_DIRECTORY ) + pathlib.Path(_default_dir).mkdir(parents=True, exist_ok=True) _config_dict["offline"]["cache"] = _default_dir From 86c35c55d5d9d3a70f3045296ca709a3e275c1eb Mon Sep 17 00:00:00 2001 From: Matt Field Date: Wed, 23 Apr 2025 17:22:24 +0100 Subject: [PATCH 20/23] Fix visibility and change show_shared to true by default --- simvue/client.py | 4 +- simvue/run.py | 10 +-- tests/functional/test_run_class.py | 103 +++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 7 deletions(-) diff --git a/simvue/client.py b/simvue/client.py index 82b4fc75..2ff5c458 100644 --- a/simvue/client.py +++ b/simvue/client.py @@ -181,7 +181,7 @@ def get_runs( output_format: typing.Literal["dict", "objects", "dataframe"] = "objects", count_limit: pydantic.PositiveInt | None = 100, start_index: pydantic.NonNegativeInt = 0, - show_shared: bool = False, + show_shared: bool = True, sort_by_columns: list[tuple[str, bool]] | None = None, ) -> DataFrame | typing.Generator[tuple[str, Run], None, None] | None: """Retrieve all runs matching filters. @@ -210,7 +210,7 @@ def get_runs( start_index : int, optional the index from which to count entries. Default is 0. show_shared : bool, optional - whether to include runs shared with the current user. Default is False. + whether to include runs shared with the current user. Default is True. sort_by_columns : list[tuple[str, bool]], optional sort by columns in the order given, list of tuples in the form (column_name: str, sort_descending: bool), diff --git a/simvue/run.py b/simvue/run.py index 099016f1..cc7ddd8b 100644 --- a/simvue/run.py +++ b/simvue/run.py @@ -729,11 +729,11 @@ def init( if name: self._sv_obj.name = name - self._sv_obj.visibility = { - "users": visibility if isinstance(visibility, list) else [], - "tenant": visibility == "tenant", - "public": visibility == "public", - } + self._sv_obj.visibility.tenant = visibility == "tenant" + self._sv_obj.visibility.public = visibility == "public" + self._sv_obj.visibility.users = ( + visibility if isinstance(visibility, list) else [] + ) self._sv_obj.ttl = self._retention self._sv_obj.status = self._status self._sv_obj.tags = tags diff --git a/tests/functional/test_run_class.py b/tests/functional/test_run_class.py index 527d63da..b8208fc5 100644 --- a/tests/functional/test_run_class.py +++ b/tests/functional/test_run_class.py @@ -217,6 +217,109 @@ def test_log_metrics_offline(create_plain_run_offline: tuple[sv_run.Run, dict]) _steps = set(_steps) assert len(_steps) == 1 +@pytest.mark.run +@pytest.mark.parametrize( + "visibility", ("bad_option", "tenant", "public", ["ciuser01"], None) +) +def test_visibility( + request: pytest.FixtureRequest, + visibility: typing.Literal["public", "tenant"] | list[str] | None, +) -> None: + + run = sv_run.Run() + run.config(suppress_errors=False) + + if visibility == "bad_option": + with pytest.raises(SimvueRunError, match="visibility") as e: + run.init( + name=f"test_visibility_{str(uuid.uuid4()).split('-', 1)[0]}", + tags=[ + "simvue_client_unit_tests", + request.node.name.replace("[", "_").replace("]", "_"), + ], + folder="/simvue_unit_testing", + retention_period="1 hour", + visibility=visibility, + ) + return + + run.init( + name=f"test_visibility_{str(uuid.uuid4()).split('-', 1)[0]}", + tags=[ + "simvue_client_unit_tests", + request.node.name.replace("[", "_").replace("]", "_"), + ], + folder="/simvue_unit_testing", + visibility=visibility, + retention_period="1 hour", + ) + time.sleep(1) + _id = run._id + run.close() + _retrieved_run = RunObject(identifier=_id) + + if visibility == "tenant": + assert _retrieved_run.visibility.tenant + elif visibility == "public": + assert _retrieved_run.visibility.public + elif not visibility: + assert not _retrieved_run.visibility.tenant and not _retrieved_run.visibility.public + else: + assert _retrieved_run.visibility.users == visibility + +@pytest.mark.run +@pytest.mark.offline +@pytest.mark.parametrize( + "visibility", ("bad_option", "tenant", "public", ["ciuser01"], None) +) +def test_visibility_offline( + request: pytest.FixtureRequest, + monkeypatch, + visibility: typing.Literal["public", "tenant"] | list[str] | None, +) -> None: + with tempfile.TemporaryDirectory() as tempd: + os.environ["SIMVUE_OFFLINE_DIRECTORY"] = tempd + run = sv_run.Run(mode="offline") + run.config(suppress_errors=False) + + if visibility == "bad_option": + with pytest.raises(SimvueRunError, match="visibility") as e: + run.init( + name=f"test_visibility_{str(uuid.uuid4()).split('-', 1)[0]}", + tags=[ + "simvue_client_unit_tests", + request.node.name.replace("[", "_").replace("]", "_"), + ], + folder="/simvue_unit_testing", + retention_period="1 hour", + visibility=visibility, + ) + return + + run.init( + name=f"test_visibility_{str(uuid.uuid4()).split('-', 1)[0]}", + tags=[ + "simvue_client_unit_tests", + request.node.name.replace("[", "_").replace("]", "_"), + ], + folder="/simvue_unit_testing", + visibility=visibility, + retention_period="1 hour", + ) + time.sleep(1) + _id = run._id + _id_mapping = sv_send.sender(os.environ["SIMVUE_OFFLINE_DIRECTORY"], 2, 10) + run.close() + _retrieved_run = RunObject(identifier=_id_mapping.get(_id)) + + if visibility == "tenant": + assert _retrieved_run.visibility.tenant + elif visibility == "public": + assert _retrieved_run.visibility.public + elif not visibility: + assert not _retrieved_run.visibility.tenant and not _retrieved_run.visibility.public + else: + assert _retrieved_run.visibility.users == visibility @pytest.mark.run def test_log_events_online(create_test_run: tuple[sv_run.Run, dict]) -> None: From 125776421ee0da264de5b2d1148b1e98f7609df9 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Fri, 25 Apr 2025 13:05:35 +0100 Subject: [PATCH 21/23] change username for visibility tests --- tests/functional/test_run_class.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/functional/test_run_class.py b/tests/functional/test_run_class.py index b8208fc5..ffe361e8 100644 --- a/tests/functional/test_run_class.py +++ b/tests/functional/test_run_class.py @@ -104,7 +104,7 @@ def test_run_with_emissions_offline(speedy_heartbeat, mock_co2_signal, create_pl ) @pytest.mark.parametrize("overload_buffer", (True, False), ids=("overload", "normal")) @pytest.mark.parametrize( - "visibility", ("bad_option", "tenant", "public", ["ciuser01"], None) + "visibility", ("bad_option", "tenant", "public", ["user01"], None) ) def test_log_metrics( overload_buffer: bool, @@ -219,9 +219,9 @@ def test_log_metrics_offline(create_plain_run_offline: tuple[sv_run.Run, dict]) @pytest.mark.run @pytest.mark.parametrize( - "visibility", ("bad_option", "tenant", "public", ["ciuser01"], None) + "visibility", ("bad_option", "tenant", "public", ["user01"], None) ) -def test_visibility( +def test_visibility_online( request: pytest.FixtureRequest, visibility: typing.Literal["public", "tenant"] | list[str] | None, ) -> None: @@ -270,7 +270,7 @@ def test_visibility( @pytest.mark.run @pytest.mark.offline @pytest.mark.parametrize( - "visibility", ("bad_option", "tenant", "public", ["ciuser01"], None) + "visibility", ("bad_option", "tenant", "public", ["user01"], None) ) def test_visibility_offline( request: pytest.FixtureRequest, From f4340984dc2f11cd09107e7399aae978121b4311 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Fri, 25 Apr 2025 15:22:06 +0100 Subject: [PATCH 22/23] Fix get_runs filters --- simvue/client.py | 3 ++- tests/functional/test_client.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/simvue/client.py b/simvue/client.py index 246e4b65..915e9db0 100644 --- a/simvue/client.py +++ b/simvue/client.py @@ -234,8 +234,9 @@ def get_runs( RuntimeError if there was a failure in data retrieval from the server """ + filters = filters or [] if not show_shared: - filters = (filters or []) + ["user == self"] + filters += ["user == self"] _runs = Run.get( count=count_limit, diff --git a/tests/functional/test_client.py b/tests/functional/test_client.py index c262de00..3c4b46d7 100644 --- a/tests/functional/test_client.py +++ b/tests/functional/test_client.py @@ -227,7 +227,7 @@ def test_get_artifacts_as_files( def test_get_runs(create_test_run: tuple[sv_run.Run, dict], output_format: str, sorting: list[tuple[str, bool]] | None) -> None: client = svc.Client() - _result = client.get_runs(filters=None, output_format=output_format, count_limit=10, sort_by_columns=sorting) + _result = client.get_runs(filters=[], output_format=output_format, count_limit=10, sort_by_columns=sorting) if output_format == "dataframe": assert not _result.empty From 0d387bf6c208c82b768ffa01939afc81d354d2c8 Mon Sep 17 00:00:00 2001 From: Matt Field Date: Fri, 25 Apr 2025 15:58:09 +0100 Subject: [PATCH 23/23] Update files for 2.1.1 release --- CHANGELOG.md | 7 +++++++ CITATION.cff | 6 +++--- pyproject.toml | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be034b0b..72545e26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,11 @@ # Change log +## [v2.1.1](https://github.com/simvue-io/client/releases/tag/v2.1.1) - 2025-04-25 +* Changed from CO2 Signal to ElectricityMaps +* Fixed a number of bugs in how offline mode is handled with emissions +* Streamlined EmissionsMonitor class and handling +* Fixed bugs in client getting results from Simvue server arising from pagination +* Fixed bug in setting visibility in `run.init` method +* Default setting in `Client.get_runs` is now `show_shared=True` ## [v2.1.0](https://github.com/simvue-io/client/releases/tag/v2.1.0) - 2025-03-28 * Removed CodeCarbon dependence in favour of a slimmer solution using the CO2 Signal API. * Added sorting to server queries, users can now specify to sort by columns during data retrieval from the database. diff --git a/CITATION.cff b/CITATION.cff index faebcb65..0e406546 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -42,6 +42,6 @@ keywords: - alerting - simulation license: Apache-2.0 -commit: 8f13a7adb2ad0ec53f0a4949e44e1c5676ae342d -version: 2.1.0 -date-released: '2025-03-28' +commit: f1bde5646b33f01ec15ef72a0c5843c1fe181ac1 +version: 2.1.1 +date-released: '2025-04-25' diff --git a/pyproject.toml b/pyproject.toml index 3bfcb226..77212d32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "simvue" -version = "2.1.0" +version = "2.1.1" description = "Simulation tracking and monitoring" authors = [ {name = "Simvue Development Team", email = "info@simvue.io"}