Skip to content

Commit 02bbc23

Browse files
authored
Merge pull request #762 from simvue-io/feature/pagination
Paginate results from server
2 parents 07d47b8 + ff0864b commit 02bbc23

File tree

7 files changed

+117
-72
lines changed

7 files changed

+117
-72
lines changed

pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "simvue"
3-
version = "2.0.1"
3+
version = "2.1.0"
44
description = "Simulation tracking and monitoring"
55
authors = [
66
{name = "Simvue Development Team", email = "[email protected]"}

simvue/api/objects/base.py

Lines changed: 44 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
from simvue.version import __version__
2424
from simvue.api.request import (
2525
get as sv_get,
26+
get_paginated,
2627
post as sv_post,
2728
put as sv_put,
2829
delete as sv_delete,
@@ -347,7 +348,7 @@ def new(cls, **_) -> Self:
347348
@classmethod
348349
def ids(
349350
cls, count: int | None = None, offset: int | None = None, **kwargs
350-
) -> list[str]:
351+
) -> typing.Generator[str, None, None]:
351352
"""Retrieve a list of all object identifiers.
352353
353354
Parameters
@@ -357,17 +358,23 @@ def ids(
357358
offset : int | None, optional
358359
set start index for objects list
359360
360-
Returns
361+
Yields
361362
-------
362-
list[str]
363+
str
363364
identifiers for all objects of this type.
364365
"""
365366
_class_instance = cls(_read_only=True, _local=True)
366-
if (_data := cls._get_all_objects(count, offset, **kwargs).get("data")) is None:
367-
raise RuntimeError(
368-
f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s"
369-
)
370-
return [_entry["id"] for _entry in _data]
367+
_count: int = 0
368+
for response in cls._get_all_objects(offset):
369+
if (_data := response.get("data")) is None:
370+
raise RuntimeError(
371+
f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s"
372+
)
373+
for entry in _data:
374+
yield entry["id"]
375+
_count += 1
376+
if count and _count > count:
377+
return
371378

372379
@classmethod
373380
@pydantic.validate_call
@@ -396,23 +403,19 @@ def get(
396403
Generator[tuple[str, SimvueObject | None], None, None]
397404
"""
398405
_class_instance = cls(_read_only=True, _local=True)
399-
if (
400-
_data := cls._get_all_objects(
401-
count=count,
402-
offset=offset,
403-
**kwargs,
404-
).get("data")
405-
) is None:
406-
raise RuntimeError(
407-
f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s"
408-
)
409-
410-
for _entry in _data:
411-
if not (_id := _entry.pop("id", None)):
406+
_count: int = 0
407+
for _response in cls._get_all_objects(offset, **kwargs):
408+
if count and _count > count:
409+
return
410+
if (_data := _response.get("data")) is None:
412411
raise RuntimeError(
413-
f"Expected key 'id' for {_class_instance.__class__.__name__.lower()}"
412+
f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s"
414413
)
415-
yield _id, cls(_read_only=True, identifier=_id, _local=True, **_entry)
414+
415+
for entry in _data:
416+
_id = entry["id"]
417+
yield _id, cls(_read_only=True, identifier=_id, _local=True, **entry)
418+
_count += 1
416419

417420
@classmethod
418421
def count(cls, **kwargs) -> int:
@@ -424,42 +427,34 @@ def count(cls, **kwargs) -> int:
424427
total from server database for current user.
425428
"""
426429
_class_instance = cls(_read_only=True)
427-
if (
428-
_count := cls._get_all_objects(count=None, offset=None, **kwargs).get(
429-
"count"
430-
)
431-
) is None:
432-
raise RuntimeError(
433-
f"Expected key 'count' for retrieval of {_class_instance.__class__.__name__.lower()}s"
434-
)
435-
return _count
430+
_count_total: int = 0
431+
for _data in cls._get_all_objects(**kwargs):
432+
if not (_count := _data.get("count")):
433+
raise RuntimeError(
434+
f"Expected key 'count' for retrieval of {_class_instance.__class__.__name__.lower()}s"
435+
)
436+
_count_total += _count
437+
return _count_total
436438

437439
@classmethod
438440
def _get_all_objects(
439-
cls,
440-
count: int | None,
441-
offset: int | None,
442-
**kwargs,
443-
) -> dict[str, typing.Any]:
441+
cls, offset: int | None, **kwargs
442+
) -> typing.Generator[dict, None, None]:
444443
_class_instance = cls(_read_only=True)
445444
_url = f"{_class_instance._base_url}"
446-
_params: dict[str, int | str] = {"start": offset, "count": count}
447-
448-
_response = sv_get(
449-
_url,
450-
headers=_class_instance._headers,
451-
params=_params | kwargs,
452-
)
453445

454446
_label = _class_instance.__class__.__name__.lower()
455447
if _label.endswith("s"):
456448
_label = _label[:-1]
457449

458-
return get_json_from_response(
459-
response=_response,
460-
expected_status=[http.HTTPStatus.OK],
461-
scenario=f"Retrieval of {_label}s",
462-
)
450+
for response in get_paginated(
451+
_url, headers=_class_instance._headers, offset=offset, **kwargs
452+
):
453+
yield get_json_from_response(
454+
response=response,
455+
expected_status=[http.HTTPStatus.OK],
456+
scenario=f"Retrieval of {_label}s",
457+
) # type: ignore
463458

464459
def read_only(self, is_read_only: bool) -> None:
465460
"""Set whether this object is in read only state.

simvue/api/objects/events.py

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -49,18 +49,19 @@ def get(
4949
**kwargs,
5050
) -> typing.Generator[EventSet, None, None]:
5151
_class_instance = cls(_read_only=True, _local=True)
52-
53-
if (
54-
_data := cls._get_all_objects(count, offset, run=run_id, **kwargs).get(
55-
"data"
56-
)
57-
) is None:
58-
raise RuntimeError(
59-
f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s"
60-
)
61-
62-
for _entry in _data:
63-
yield EventSet(**_entry)
52+
_count: int = 0
53+
54+
for response in cls._get_all_objects(offset, run=run_id, **kwargs):
55+
if (_data := response.get("data")) is None:
56+
raise RuntimeError(
57+
f"Expected key 'data' for retrieval of {_class_instance.__class__.__name__.lower()}s"
58+
)
59+
60+
for _entry in _data:
61+
yield EventSet(**_entry)
62+
_count += 1
63+
if _count > count:
64+
return
6465

6566
@classmethod
6667
@pydantic.validate_call

simvue/api/objects/metrics.py

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ def get(
7979
count: pydantic.PositiveInt | None = None,
8080
offset: pydantic.PositiveInt | None = None,
8181
**kwargs,
82-
) -> typing.Generator[MetricSet, None, None]:
82+
) -> typing.Generator[dict[str, dict[str, list[dict[str, float]]]], None, None]:
8383
"""Retrieve metrics from the server for a given set of runs.
8484
8585
Parameters
@@ -100,20 +100,17 @@ def get(
100100
101101
Yields
102102
------
103-
MetricSet
103+
dict[str, dict[str, list[dict[str, float]]]
104104
metric set object containing metrics for run.
105105
"""
106-
_class_instance = cls(_read_only=True, _local=True)
107-
_data = cls._get_all_objects(
108-
count,
106+
yield from cls._get_all_objects(
109107
offset,
110108
metrics=json.dumps(metrics),
111109
runs=json.dumps(runs),
112110
xaxis=xaxis,
111+
count=count,
113112
**kwargs,
114113
)
115-
# TODO: Temp fix, just return the dictionary. Not sure what format we really want this in...
116-
return _data
117114

118115
@pydantic.validate_call
119116
def span(self, run_ids: list[str]) -> dict[str, int | float]:

simvue/api/request.py

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
RETRY_MIN = 4
2828
RETRY_MAX = 10
2929
RETRY_STOP = 5
30+
MAX_ENTRIES_PER_PAGE: int = 100
3031
RETRY_STATUS_CODES = (
3132
http.HTTPStatus.BAD_REQUEST,
3233
http.HTTPStatus.SERVICE_UNAVAILABLE,
@@ -273,3 +274,51 @@ def get_json_from_response(
273274
error_str += f": {txt_response}"
274275

275276
raise RuntimeError(error_str)
277+
278+
279+
def get_paginated(
280+
url: str,
281+
headers: dict[str, str] | None = None,
282+
timeout: int = DEFAULT_API_TIMEOUT,
283+
json: dict[str, typing.Any] | None = None,
284+
offset: int | None = None,
285+
**params,
286+
) -> typing.Generator[requests.Response, None, None]:
287+
"""Paginate results of a server query.
288+
289+
Parameters
290+
----------
291+
url : str
292+
URL to put to
293+
headers : dict[str, str]
294+
headers for the post request
295+
timeout : int, optional
296+
timeout of request, by default DEFAULT_API_TIMEOUT
297+
json : dict[str, Any] | None, optional
298+
any json to send in request
299+
300+
Yield
301+
-----
302+
requests.Response
303+
server response
304+
"""
305+
_offset: int = offset or 0
306+
307+
while (
308+
(
309+
_response := get(
310+
url=url,
311+
headers=headers,
312+
params=(params or {})
313+
| {"count": MAX_ENTRIES_PER_PAGE, "start": _offset},
314+
timeout=timeout,
315+
json=json,
316+
)
317+
)
318+
.json()
319+
.get("data")
320+
):
321+
yield _response
322+
_offset += MAX_ENTRIES_PER_PAGE
323+
324+
yield _response

simvue/client.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -346,7 +346,10 @@ def _get_folder_id_from_path(self, path: str) -> str | None:
346346
"""
347347
_ids = Folder.ids(filters=json.dumps([f"path == {path}"]))
348348

349-
return _ids[0] if _ids else None
349+
try:
350+
return next(_ids)
351+
except StopIteration:
352+
return None
350353

351354
@prettify_pydantic
352355
@pydantic.validate_call

tests/functional/test_client.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,10 @@ def test_get_alerts(
4545
_id_1 = run.create_user_alert(
4646
name=f"user_alert_1_{unique_id}",
4747
)
48-
_id_2 = run.create_user_alert(
48+
run.create_user_alert(
4949
name=f"user_alert_2_{unique_id}",
5050
)
51-
_id_3 = run.create_user_alert(
51+
run.create_user_alert(
5252
name=f"user_alert_3_{unique_id}",
5353
attach_to_run=False
5454
)

0 commit comments

Comments
 (0)