From e310b9718552cf549b19ab249eab119492bd0dfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Sun, 9 Mar 2025 12:12:44 +0000 Subject: [PATCH 1/6] Started adding pagination --- pyproject.toml | 2 +- simvue/api/objects/base.py | 76 +++++++++++++++++---------------- simvue/api/request.py | 40 +++++++++++++++++ tests/functional/test_client.py | 6 +-- 4 files changed, 84 insertions(+), 40 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 37da217a..38af1d38 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "simvue" -version = "2.0.0" +version = "2.1.0" description = "Simulation tracking and monitoring" authors = [ {name = "Simvue Development Team", email = "info@simvue.io"} diff --git a/simvue/api/objects/base.py b/simvue/api/objects/base.py index f9098d2d..69969068 100644 --- a/simvue/api/objects/base.py +++ b/simvue/api/objects/base.py @@ -23,6 +23,7 @@ from simvue.version import __version__ from simvue.api.request import ( get as sv_get, + get_paginated, post as sv_post, put as sv_put, delete as sv_delete, @@ -305,14 +306,20 @@ def new(cls, **_) -> Self: @classmethod def ids( cls, count: int | None = None, offset: int | None = None, **kwargs - ) -> list[str]: + ) -> typing.Generator[str, None, None]: """Retrieve a list of all object identifiers""" _class_instance = cls(_read_only=True, _local=True) - if (_data := cls._get_all_objects(count, offset, **kwargs).get("data")) is None: - raise RuntimeError( - f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s" - ) - return [_entry["id"] for _entry in _data] + _count: int = 0 + for _data in cls._get_all_objects(offset): + if count and _count > count: + return + if _data.get("data") is None: + raise RuntimeError( + f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s" + ) + for entry in _data: + yield entry["id"] + _count += 1 @classmethod @pydantic.validate_call @@ -323,52 +330,49 @@ def get( **kwargs, ) -> typing.Generator[tuple[str, T | None], None, None]: _class_instance = cls(_read_only=True, _local=True) - if (_data := cls._get_all_objects(count, offset, **kwargs).get("data")) is None: - raise RuntimeError( - f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s" - ) - - for _entry in _data: - if not (_id := _entry.pop("id", None)): + _count: int = 0 + for _response in cls._get_all_objects(offset, **kwargs): + if count and _count > count: + return + if (_data := _response.get("data")) is None: raise RuntimeError( - f"Expected key 'id' for {_class_instance.__class__.__name__.lower()}" + f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s" ) - yield _id, cls(_read_only=True, identifier=_id, _local=True, **_entry) + + for entry in _data: + _id = entry["id"] + yield _id, cls(_read_only=True, identifier=_id, _local=True, **entry) + _count += 1 @classmethod def count(cls, **kwargs) -> int: _class_instance = cls(_read_only=True) - if ( - _count := cls._get_all_objects(count=None, offset=None, **kwargs).get( - "count" - ) - ) is None: - raise RuntimeError( - f"Expected key 'count' for retrieval of {_class_instance.__class__.__name__.lower()}s" - ) - return _count + _count_total: int = 0 + for _data in cls._get_all_objects(**kwargs): + if not (_count := _data.get("count")): + raise RuntimeError( + f"Expected key 'count' for retrieval of {_class_instance.__class__.__name__.lower()}s" + ) + _count_total += _count + return _count_total @classmethod def _get_all_objects( - cls, count: int | None, offset: int | None, **kwargs - ) -> dict[str, typing.Any]: + cls, offset: int | None, **kwargs + ) -> typing.Generator[dict, None, None]: _class_instance = cls(_read_only=True) _url = f"{_class_instance._base_url}" - _response = sv_get( - _url, - headers=_class_instance._headers, - params={"start": offset, "count": count} | kwargs, - ) _label = _class_instance.__class__.__name__.lower() if _label.endswith("s"): _label = _label[:-1] - return get_json_from_response( - response=_response, - expected_status=[http.HTTPStatus.OK], - scenario=f"Retrieval of {_label}s", - ) + for response in get_paginated(_url, headers=_class_instance._headers, offset=offset, **kwargs): + yield get_json_from_response( + response=response, + expected_status=[http.HTTPStatus.OK], + scenario=f"Retrieval of {_label}s", + ) # type: ignore def read_only(self, is_read_only: bool) -> None: self._read_only = is_read_only diff --git a/simvue/api/request.py b/simvue/api/request.py index 8dd6a8bd..1bf78173 100644 --- a/simvue/api/request.py +++ b/simvue/api/request.py @@ -27,6 +27,7 @@ RETRY_MIN = 4 RETRY_MAX = 10 RETRY_STOP = 5 +MAX_ENTRIES_PER_PAGE: int = 100 RETRY_STATUS_CODES = ( http.HTTPStatus.BAD_REQUEST, http.HTTPStatus.SERVICE_UNAVAILABLE, @@ -273,3 +274,42 @@ def get_json_from_response( error_str += f": {txt_response}" raise RuntimeError(error_str) + + +def get_paginated( + url: str, + headers: dict[str, str] | None = None, + timeout: int = DEFAULT_API_TIMEOUT, + json: dict[str, typing.Any] | None = None, + offset: int | None = None, + **params +) -> typing.Generator[requests.Response, None, None]: + """Paginate results of a server query. + + Parameters + ---------- + url : str + URL to put to + headers : dict[str, str] + headers for the post request + timeout : int, optional + timeout of request, by default DEFAULT_API_TIMEOUT + json : dict[str, Any] | None, optional + any json to send in request + + Yield + ----- + requests.Response + server response + """ + _offset: int = offset or 0 + + while (_response := get( + url=url, + headers=headers, + params=(params or {}) | {"count": MAX_ENTRIES_PER_PAGE, "start": _offset}, + timeout=timeout, + json=json, + )).json().get("data"): + yield _response + _offset += 1 diff --git a/tests/functional/test_client.py b/tests/functional/test_client.py index 85407b6c..a7c70b49 100644 --- a/tests/functional/test_client.py +++ b/tests/functional/test_client.py @@ -34,16 +34,16 @@ def test_get_events(create_test_run: tuple[sv_run.Run, dict]) -> None: "critical_only", (True, False), ids=("critical_only", "all_states") ) def test_get_alerts(create_plain_run: tuple[sv_run.Run, dict], from_run: bool, names_only: bool, critical_only: bool) -> None: - run, run_data = create_plain_run + run, _ = create_plain_run run_id = run.id unique_id = f"{uuid.uuid4()}".split("-")[0] _id_1 = run.create_user_alert( name=f"user_alert_1_{unique_id}", ) - _id_2 = run.create_user_alert( + run.create_user_alert( name=f"user_alert_2_{unique_id}", ) - _id_3 = run.create_user_alert( + run.create_user_alert( name=f"user_alert_3_{unique_id}", attach_to_run=False ) From 9f201174bcb5c0b0588a1c3fb3f9760cb75ae629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Sun, 9 Mar 2025 12:23:10 +0000 Subject: [PATCH 2/6] Fixed pagination offset increment --- simvue/api/request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/simvue/api/request.py b/simvue/api/request.py index 1bf78173..f7639cfa 100644 --- a/simvue/api/request.py +++ b/simvue/api/request.py @@ -27,7 +27,7 @@ RETRY_MIN = 4 RETRY_MAX = 10 RETRY_STOP = 5 -MAX_ENTRIES_PER_PAGE: int = 100 +MAX_ENTRIES_PER_PAGE: int = 5 RETRY_STATUS_CODES = ( http.HTTPStatus.BAD_REQUEST, http.HTTPStatus.SERVICE_UNAVAILABLE, @@ -312,4 +312,4 @@ def get_paginated( json=json, )).json().get("data"): yield _response - _offset += 1 + _offset += MAX_ENTRIES_PER_PAGE From 130c83e243e257ef7b9057df43d0760e487fa9c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Sun, 9 Mar 2025 12:26:41 +0000 Subject: [PATCH 3/6] Set page size to 100 --- simvue/api/request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simvue/api/request.py b/simvue/api/request.py index f7639cfa..bd82c463 100644 --- a/simvue/api/request.py +++ b/simvue/api/request.py @@ -27,7 +27,7 @@ RETRY_MIN = 4 RETRY_MAX = 10 RETRY_STOP = 5 -MAX_ENTRIES_PER_PAGE: int = 5 +MAX_ENTRIES_PER_PAGE: int = 100 RETRY_STATUS_CODES = ( http.HTTPStatus.BAD_REQUEST, http.HTTPStatus.SERVICE_UNAVAILABLE, From 1d29eb7f77813233d5d3ff2109164c8342734739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Sun, 9 Mar 2025 12:28:17 +0000 Subject: [PATCH 4/6] Run linting --- simvue/api/objects/base.py | 6 ++++-- simvue/api/request.py | 25 ++++++++++++++++--------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/simvue/api/objects/base.py b/simvue/api/objects/base.py index 69969068..99b21965 100644 --- a/simvue/api/objects/base.py +++ b/simvue/api/objects/base.py @@ -367,12 +367,14 @@ def _get_all_objects( if _label.endswith("s"): _label = _label[:-1] - for response in get_paginated(_url, headers=_class_instance._headers, offset=offset, **kwargs): + for response in get_paginated( + _url, headers=_class_instance._headers, offset=offset, **kwargs + ): yield get_json_from_response( response=response, expected_status=[http.HTTPStatus.OK], scenario=f"Retrieval of {_label}s", - ) # type: ignore + ) # type: ignore def read_only(self, is_read_only: bool) -> None: self._read_only = is_read_only diff --git a/simvue/api/request.py b/simvue/api/request.py index bd82c463..b776ae58 100644 --- a/simvue/api/request.py +++ b/simvue/api/request.py @@ -282,10 +282,10 @@ def get_paginated( timeout: int = DEFAULT_API_TIMEOUT, json: dict[str, typing.Any] | None = None, offset: int | None = None, - **params + **params, ) -> typing.Generator[requests.Response, None, None]: """Paginate results of a server query. - + Parameters ---------- url : str @@ -304,12 +304,19 @@ def get_paginated( """ _offset: int = offset or 0 - while (_response := get( - url=url, - headers=headers, - params=(params or {}) | {"count": MAX_ENTRIES_PER_PAGE, "start": _offset}, - timeout=timeout, - json=json, - )).json().get("data"): + while ( + ( + _response := get( + url=url, + headers=headers, + params=(params or {}) + | {"count": MAX_ENTRIES_PER_PAGE, "start": _offset}, + timeout=timeout, + json=json, + ) + ) + .json() + .get("data") + ): yield _response _offset += MAX_ENTRIES_PER_PAGE From c6b1f9466af6abb093987efcfea4fe2008c5450b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Mon, 10 Mar 2025 08:28:01 +0000 Subject: [PATCH 5/6] Fix ID retrieval --- simvue/api/objects/base.py | 4 ++-- simvue/client.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/simvue/api/objects/base.py b/simvue/api/objects/base.py index 99b21965..c9b1b46e 100644 --- a/simvue/api/objects/base.py +++ b/simvue/api/objects/base.py @@ -310,10 +310,10 @@ def ids( """Retrieve a list of all object identifiers""" _class_instance = cls(_read_only=True, _local=True) _count: int = 0 - for _data in cls._get_all_objects(offset): + for response in cls._get_all_objects(offset): if count and _count > count: return - if _data.get("data") is None: + if (_data := response.get("data")) is None: raise RuntimeError( f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s" ) diff --git a/simvue/client.py b/simvue/client.py index 453c1f75..bd49b1fd 100644 --- a/simvue/client.py +++ b/simvue/client.py @@ -333,7 +333,10 @@ def _get_folder_id_from_path(self, path: str) -> str | None: """ _ids = Folder.ids(filters=json.dumps([f"path == {path}"])) - return _ids[0] if _ids else None + try: + return next(_ids) + except StopIteration: + return None @prettify_pydantic @pydantic.validate_call From 598c9fbf8b39deccb682952f32cf79f53b727093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Mon, 10 Mar 2025 09:47:42 +0000 Subject: [PATCH 6/6] Updated metric and event retrieval --- simvue/api/objects/base.py | 4 ++-- simvue/api/objects/events.py | 24 +++++++++++++----------- simvue/api/objects/metrics.py | 9 ++++----- simvue/api/request.py | 2 ++ 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/simvue/api/objects/base.py b/simvue/api/objects/base.py index c9b1b46e..6d777c7b 100644 --- a/simvue/api/objects/base.py +++ b/simvue/api/objects/base.py @@ -311,8 +311,6 @@ def ids( _class_instance = cls(_read_only=True, _local=True) _count: int = 0 for response in cls._get_all_objects(offset): - if count and _count > count: - return if (_data := response.get("data")) is None: raise RuntimeError( f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s" @@ -320,6 +318,8 @@ def ids( for entry in _data: yield entry["id"] _count += 1 + if count and _count > count: + return @classmethod @pydantic.validate_call diff --git a/simvue/api/objects/events.py b/simvue/api/objects/events.py index b330501b..a997aba7 100644 --- a/simvue/api/objects/events.py +++ b/simvue/api/objects/events.py @@ -44,17 +44,19 @@ def get( **kwargs, ) -> typing.Generator[EventSet, None, None]: _class_instance = cls(_read_only=True, _local=True) - if ( - _data := cls._get_all_objects(count, offset, run=run_id, **kwargs).get( - "data" - ) - ) is None: - raise RuntimeError( - f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s" - ) - - for _entry in _data: - yield EventSet(**_entry) + _count: int = 0 + + for response in cls._get_all_objects(offset, run=run_id, **kwargs): + if (_data := response.get("data")) is None: + raise RuntimeError( + f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s" + ) + + for _entry in _data: + yield EventSet(**_entry) + _count += 1 + if _count > count: + return @classmethod @pydantic.validate_call diff --git a/simvue/api/objects/metrics.py b/simvue/api/objects/metrics.py index 43b75561..642dc09e 100644 --- a/simvue/api/objects/metrics.py +++ b/simvue/api/objects/metrics.py @@ -56,18 +56,17 @@ def get( count: pydantic.PositiveInt | None = None, offset: pydantic.PositiveInt | None = None, **kwargs, - ) -> typing.Generator[MetricSet, None, None]: + ) -> typing.Generator[dict[str, dict[str, list[dict[str, float]]]], None, None]: + """Yields the values for the given metrics for each run.""" _class_instance = cls(_read_only=True, _local=True) - _data = cls._get_all_objects( - count, + yield from cls._get_all_objects( offset, metrics=json.dumps(metrics), runs=json.dumps(runs), xaxis=xaxis, + count=count, **kwargs, ) - # TODO: Temp fix, just return the dictionary. Not sure what format we really want this in... - return _data @pydantic.validate_call def span(self, run_ids: list[str]) -> dict[str, int | float]: diff --git a/simvue/api/request.py b/simvue/api/request.py index b776ae58..4a376749 100644 --- a/simvue/api/request.py +++ b/simvue/api/request.py @@ -320,3 +320,5 @@ def get_paginated( ): yield _response _offset += MAX_ENTRIES_PER_PAGE + + yield _response