From a50ed2e41559b83f3aa18db8087ac936d038d63c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Fri, 23 Feb 2024 09:52:36 +0000 Subject: [PATCH 01/10] Added filter class --- simvue/filter.py | 230 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 simvue/filter.py diff --git a/simvue/filter.py b/simvue/filter.py new file mode 100644 index 00000000..d8fba8a1 --- /dev/null +++ b/simvue/filter.py @@ -0,0 +1,230 @@ +import abc +import enum +import sys + +if sys.version_info < (3, 11): + from typing_extensions import Self +else: + from typing import Self + +class Status(str, enum.Enum): + Created = "created" + Running = "running" + Completed = "completed" + Lost = "lost" + Terminated = "terminated" + Failed = "failed" + + +class Time(str, enum.Enum): + Created = "created" + Started = "started" + Modified = "modified" + Ended = "ended" + +class System(str, enum.Enum): + Working_Directory = "cwd" + Hostname = "hostname" + Python_Version = "pythonversion" + Platform_System = "platform.system" + Platform_Release = "platform.release" + Platform_Version = "platform.version" + CPU_Architecture = "cpu.arch" + CPU_Processor = "cpu.processor" + GPU_Name = "gpu.name" + GPU_Driver = "gpu.driver" + + +class RestAPIFilter(abc.ABC): + def __init__(self) -> None: + self._filters: list[str] = [] + self._generate_members() + + def _time_within(self, time_type: Time, *, hours: int=0, days: int=0, years: int=0) -> Self: + if len(_non_zero := list(i for i in (hours, days, years) if i != 0 )) > 1: + raise AssertionError("Only one duration type may be provided: hours, days or years") + if len(_non_zero) < 1: + raise AssertionError(f"No duration provided for filter '{time_type.value}_within'") + + if hours: + self._filters.append(f"{time_type.value} < {hours}h") + elif days: + self._filters.append(f"{time_type.value} < {days}d") + else: + self._filters.append(f"{time_type.value} < {years}y") + return self + + @abc.abstractmethod + def _generate_members(self) -> None: + pass + + def has_name(self, name: str) -> Self: + self._filters.append(f"name == {name}") + return self + + def has_name_containing(self, name: str) -> Self: + self._filters.append(f"name contains {name}") + return self + + def created_within(self, *, hours: int=0, days: int=0, years: int=0) -> Self: + return self._time_within(Time.Created, hours=hours, days=days, years=years) + + def has_description_containing(self, search_str: str) -> Self: + self._filters.append(f"description contains {search_str}") + return self + + def exclude_description_containing(self, search_str: str) -> Self: + self._filters.append(f"description not contains {search_str}") + return self + + def has_tag(self, tag: str) -> Self: + self._filters.append(f"has tag.{tag}") + return self + + def as_list(self) -> list[str]: + return self._filters + + def clear(self) -> None: + self._filters = [] + + +class FoldersFilter(RestAPIFilter): + def has_path(self, name: str) -> "FoldersFilter": + self._filters.append(f"path == {name}") + return self + + def has_path_containing(self, name: str) -> "FoldersFilter": + self._filters.append(f"path contains {name}") + return self + + def _generate_members(self) -> None: + return super()._generate_members() + + +class RunsFilter(RestAPIFilter): + def _generate_members(self) -> None: + _global_comparators = [ + self._value_contains, + self._value_eq, + self._value_neq + ] + + _numeric_comparators = [ + self._value_geq, + self._value_leq, + self._value_lt, + self._value_gt + ] + + for label, system_spec in System.__members__.items(): + for function in _global_comparators: + _label: str = label.lower() + _func_name: str = function.__name__.replace("_value", _label) + _out_func = lambda value, func=function: func("system", system_spec.value, value) + _out_func.__name__ = _func_name + setattr(self, _func_name, _out_func) + + for function in _global_comparators + _numeric_comparators: + _func_name: str = function.__name__.replace("_value", "metadata") + _out_func = lambda attribute, value, func=function: func("metadata", attribute, value) + _out_func.__name__ = _func_name + setattr(self, _func_name, _out_func) + + def author(self, username: str="self") -> "RunsFilter": + self._filters.append(f"user == {username}") + return self + + def exclude_author(self, username: str="self") -> "RunsFilter": + self._filters.append(f"user != {username}") + return self + + def starred(self) -> "RunsFilter": + self._filters.append("starred") + return self + + def has_name(self, name: str) -> "RunsFilter": + self._filters.append(f"name == {name}") + return self + + def has_name_containing(self, name: str) -> "RunsFilter": + self._filters.append(f"name contains {name}") + return self + + def has_status(self, status: Status) -> "RunsFilter": + self._filters.append(f"status == {status.value}") + return self + + def is_running(self) -> "RunsFilter": + return self.has_status(Status.Running) + + def is_lost(self) -> "RunsFilter": + return self.has_status(Status.Lost) + + def has_completed(self) -> "RunsFilter": + return self.has_status(Status.Completed) + + def has_failed(self) -> "RunsFilter": + return self.has_status(Status.Failed) + + def has_alert(self, alert_name: str, is_critical: bool | None=None) -> "RunsFilter": + self._filters.append(f"alert.name == {alert_name}") + if is_critical is True: + self._filters.append("alert.status == critical") + elif is_critical is False: + self._filters.append("alert.status == ok") + return self + + def started_within(self, *, hours: int=0, days: int=0, years: int=0) -> "RunsFilter": + return self._time_within(Time.Started, hours=hours, days=days, years=years) + + def modified_within(self, *, hours: int=0, days: int=0, years: int=0) -> "RunsFilter": + return self._time_within(Time.Modified, hours=hours, days=days, years=years) + + def ended_within(self, *, hours: int=0, days: int=0, years: int=0) -> "RunsFilter": + return self._time_within(Time.Ended, hours=hours, days=days, years=years) + + def in_folder(self, folder_name: str) -> "RunsFilter": + self._filters.append(f"folder.path == {folder_name}") + return self + + def has_metadata_attribute(self, attribute: str) -> "RunsFilter": + self._filters.append(f"metadata.{attribute} exists") + return self + + def exclude_metadata_attribute(self, attribute: str) -> "RunsFilter": + self._filters.append(f"metadata.{attribute} not exists") + return self + + def _value_eq(self, category: str, attribute: str, value: str | int | float) -> "RunsFilter": + self._filters.append(f"{category}.{attribute} == {value}") + return self + + def _value_neq(self, category: str, attribute: str, value: str | int | float) -> "RunsFilter": + self._filters.append(f"{category}.{attribute} != {value}") + return self + + def _value_contains(self, category: str, attribute: str, value: str | int | float) -> "RunsFilter": + self._filters.append(f"{category}.{attribute} contains {value}") + return self + + def _value_leq(self, category: str, attribute: str, value: int | float) -> "RunsFilter": + self._filters.append(f"{category}.{attribute} <= {value}") + return self + + def _value_geq(self, category: str, attribute: str, value: int | float) -> "RunsFilter": + self._filters.append(f"{category}.{attribute} >= {value}") + return self + + def _value_lt(self, category: str, attribute: str, value: int | float) -> "RunsFilter": + self._filters.append(f"{category}.{attribute} < {value}") + return self + + def _value_gt(self, category: str, attribute: str, value: int | float) -> "RunsFilter": + self._filters.append(f"{category}.{attribute} > {value}") + return self + + def __str__(self) -> str: + return " && ".join(self._filters) if self._filters else "None" + + def __repr__(self) -> str: + return f"{super().__repr__()[:-1]}, filters={self._filters}>" From 5fb15b2911dc92c8df195f1da3c47c270601734c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Fri, 23 Feb 2024 10:02:49 +0000 Subject: [PATCH 02/10] Add option between filters list and filter object --- simvue/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/simvue/client.py b/simvue/client.py index 583cd1a4..20711dab 100644 --- a/simvue/client.py +++ b/simvue/client.py @@ -1,9 +1,9 @@ from concurrent.futures import ProcessPoolExecutor import json import os -import pickle import requests +from .filter import RunsFilter from .serialization import Deserializer from .utilities import get_auth, check_extra from .converters import to_dataframe, metrics_to_dataframe @@ -76,6 +76,9 @@ def get_runs(self, filters, system=False, tags=False, metadata=False, format='di """ Get runs """ + if isinstance(filters, RunsFilter): + filters = filters.as_list() + params = {'name': None, 'filters': json.dumps(filters), 'return_basic': True, From 73174516b8203f146160cb72583a937b91fdb9cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Fri, 23 Feb 2024 10:06:09 +0000 Subject: [PATCH 03/10] Revert filter to legacy typing --- simvue/filter.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/simvue/filter.py b/simvue/filter.py index d8fba8a1..ea16a042 100644 --- a/simvue/filter.py +++ b/simvue/filter.py @@ -1,6 +1,7 @@ import abc import enum import sys +import typing if sys.version_info < (3, 11): from typing_extensions import Self @@ -166,7 +167,7 @@ def has_completed(self) -> "RunsFilter": def has_failed(self) -> "RunsFilter": return self.has_status(Status.Failed) - def has_alert(self, alert_name: str, is_critical: bool | None=None) -> "RunsFilter": + def has_alert(self, alert_name: str, is_critical: typing.Optional[bool]=None) -> "RunsFilter": self._filters.append(f"alert.name == {alert_name}") if is_critical is True: self._filters.append("alert.status == critical") @@ -195,31 +196,31 @@ def exclude_metadata_attribute(self, attribute: str) -> "RunsFilter": self._filters.append(f"metadata.{attribute} not exists") return self - def _value_eq(self, category: str, attribute: str, value: str | int | float) -> "RunsFilter": + def _value_eq(self, category: str, attribute: str, value: typing.Union[str, int, float]) -> "RunsFilter": self._filters.append(f"{category}.{attribute} == {value}") return self - def _value_neq(self, category: str, attribute: str, value: str | int | float) -> "RunsFilter": + def _value_neq(self, category: str, attribute: str, value: typing.Union[str, int, float]) -> "RunsFilter": self._filters.append(f"{category}.{attribute} != {value}") return self - def _value_contains(self, category: str, attribute: str, value: str | int | float) -> "RunsFilter": + def _value_contains(self, category: str, attribute: str, value: typing.Union[str, int, float]) -> "RunsFilter": self._filters.append(f"{category}.{attribute} contains {value}") return self - def _value_leq(self, category: str, attribute: str, value: int | float) -> "RunsFilter": + def _value_leq(self, category: str, attribute: str, value: typing.Union[int, float]) -> "RunsFilter": self._filters.append(f"{category}.{attribute} <= {value}") return self - def _value_geq(self, category: str, attribute: str, value: int | float) -> "RunsFilter": + def _value_geq(self, category: str, attribute: str, value: typing.Union[int, float]) -> "RunsFilter": self._filters.append(f"{category}.{attribute} >= {value}") return self - def _value_lt(self, category: str, attribute: str, value: int | float) -> "RunsFilter": + def _value_lt(self, category: str, attribute: str, value: typing.Union[int, float]) -> "RunsFilter": self._filters.append(f"{category}.{attribute} < {value}") return self - def _value_gt(self, category: str, attribute: str, value: int | float) -> "RunsFilter": + def _value_gt(self, category: str, attribute: str, value: typing.Union[int, float]) -> "RunsFilter": self._filters.append(f"{category}.{attribute} > {value}") return self From 201155464b8e9d60d1ed9c437f7c3cde76733f49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Fri, 23 Feb 2024 10:39:25 +0000 Subject: [PATCH 04/10] Added filter test --- tests/unit/test_filter.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 tests/unit/test_filter.py diff --git a/tests/unit/test_filter.py b/tests/unit/test_filter.py new file mode 100644 index 00000000..d66d1af7 --- /dev/null +++ b/tests/unit/test_filter.py @@ -0,0 +1,31 @@ +import pytest +import uuid +import simvue.filter as sv_filter + + +@pytest.mark.unit +def test_generate_meta_filter() -> None: + _filter = sv_filter.RunsFilter() + _attribute: str = f"{uuid.uuid4()}" + _filter.has_metadata_attribute(_attribute) + assert f"{_filter}" == f"metadata.{_attribute} exists" + _filter.clear() + + _expected_members: dict[str, str] = { + "eq": "==", + "leq": "<=", + "geq": ">=", + "lt": "<", + "gt": ">", + "neq": "!=", + "contains": "contains" + } + + assert all(hasattr(_filter, f"metadata_{i}") for i in _expected_members.keys()) + + _comparison: str = "0234" + + for func, symbol in _expected_members.items(): + getattr(_filter, f"metadata_{func}")(_attribute, _comparison) + assert f"{_filter}" == f"metadata.{_attribute} {symbol} {_comparison}" + _filter.clear() \ No newline at end of file From a5b3c0df268ec63fa7659f36f0b296ef937f3491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Wed, 22 May 2024 08:34:48 +0100 Subject: [PATCH 05/10] Added missing test for filters with runs retrieval --- simvue/client.py | 2 +- tests/refactor/test_client.py | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/simvue/client.py b/simvue/client.py index cda9199d..1449696e 100644 --- a/simvue/client.py +++ b/simvue/client.py @@ -253,7 +253,7 @@ def get_run_name_from_id(self, run_id: str) -> str: def get_runs( self, - filters: typing.Optional[list[str]], + filters: typing.Optional[typing.Union[list[str], RunsFilter]], system: bool = False, metrics: bool = False, alerts: bool = False, diff --git a/tests/refactor/test_client.py b/tests/refactor/test_client.py index 9d68d2e5..2aac41ab 100644 --- a/tests/refactor/test_client.py +++ b/tests/refactor/test_client.py @@ -6,6 +6,7 @@ import tempfile import simvue.client as svc import simvue.run as sv_run +import simvue.filter as sv_filter @pytest.mark.dependency @@ -118,9 +119,21 @@ def test_get_artifacts_as_files(create_test_run: tuple[sv_run.Run, dict]) -> Non @pytest.mark.dependency @pytest.mark.client -def test_get_runs(create_test_run: tuple[sv_run.Run, dict]) -> None: +@pytest.mark.parametrize( + "filters", (None, "list", "object"), ids=("no_filters", "filter_list", "filter_object") +) +def test_get_runs(create_test_run: tuple[sv_run.Run, dict], filters: typing.Optional[str]) -> None: client = svc.Client() - assert client.get_runs(filters=None) + + if filters == "list": + filter_obj = ["has tag.simvue_client_unit_tests"] + elif filters == "object": + filter_obj = sv_filter.RunsFilter() + filter_obj.has_tag("simvue_client_unit_tests") + else: + filter_obj = None + + assert client.get_runs(filters=filter_obj) @pytest.mark.dependency From 793cc34b25d9f1da4d8361b2fa313b0706687109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Wed, 22 May 2024 09:02:47 +0100 Subject: [PATCH 06/10] Add FoldersFilter tests --- simvue/__init__.py | 2 + simvue/client.py | 11 ++- simvue/filter.py | 175 +++++++++++++++++++++++----------- tests/refactor/test_client.py | 29 +++++- 4 files changed, 154 insertions(+), 63 deletions(-) diff --git a/simvue/__init__.py b/simvue/__init__.py index 267add6e..ff1046bc 100644 --- a/simvue/__init__.py +++ b/simvue/__init__.py @@ -2,3 +2,5 @@ from simvue.handler import Handler as Handler from simvue.models import RunInput as RunInput from simvue.run import Run as Run +from simvue.filter import RunsFilter as RunsFilter +from simvue.filter import FoldersFilter as FoldersFilter diff --git a/simvue/client.py b/simvue/client.py index 1449696e..8460243e 100644 --- a/simvue/client.py +++ b/simvue/client.py @@ -14,7 +14,7 @@ import requests -from .filter import RunsFilter +from .filter import RunsFilter, FoldersFilter from .converters import ( aggregated_metrics_to_dataframe, to_dataframe, @@ -268,7 +268,7 @@ def get_runs( Parameters ---------- - filters: list[str] | None + filters: list[str] | RunsFilter | None set of filters to apply to query results. If None is specified return all results without filtering. metadata : bool, optional @@ -791,7 +791,7 @@ def get_folder(self, folder_id: str) -> typing.Optional[dict[str, typing.Any]]: def get_folders( self, - filters: typing.Optional[list[str]] = None, + filters: typing.Optional[typing.Union[list[str], FoldersFilter]] = None, count: int = 100, start_index: int = 0, ) -> list[dict[str, typing.Any]]: @@ -799,7 +799,7 @@ def get_folders( Parameters ---------- - filters : list[str] | None + filters : list[str] | FoldersFilter | None set of filters to apply to the search count : int, optional maximum number of entries to return. Default is 100. @@ -816,6 +816,9 @@ def get_folders( RuntimeError if there was a failure retrieving data from the server """ + if isinstance(filters, FoldersFilter): + filters = filters.as_list() + params: dict[str, typing.Union[str, int]] = { "filters": json.dumps(filters or []), "count": count, diff --git a/simvue/filter.py b/simvue/filter.py index ea16a042..1aa7a434 100644 --- a/simvue/filter.py +++ b/simvue/filter.py @@ -2,12 +2,14 @@ import enum import sys import typing +import pydantic if sys.version_info < (3, 11): from typing_extensions import Self else: from typing import Self + class Status(str, enum.Enum): Created = "created" Running = "running" @@ -23,6 +25,7 @@ class Time(str, enum.Enum): Modified = "modified" Ended = "ended" + class System(str, enum.Enum): Working_Directory = "cwd" Hostname = "hostname" @@ -41,12 +44,18 @@ def __init__(self) -> None: self._filters: list[str] = [] self._generate_members() - def _time_within(self, time_type: Time, *, hours: int=0, days: int=0, years: int=0) -> Self: - if len(_non_zero := list(i for i in (hours, days, years) if i != 0 )) > 1: - raise AssertionError("Only one duration type may be provided: hours, days or years") + def _time_within( + self, time_type: Time, *, hours: int = 0, days: int = 0, years: int = 0 + ) -> Self: + if len(_non_zero := list(i for i in (hours, days, years) if i != 0)) > 1: + raise AssertionError( + "Only one duration type may be provided: hours, days or years" + ) if len(_non_zero) < 1: - raise AssertionError(f"No duration provided for filter '{time_type.value}_within'") - + raise AssertionError( + f"No duration provided for filter '{time_type.value}_within'" + ) + if hours: self._filters.append(f"{time_type.value} < {hours}h") elif days: @@ -59,25 +68,37 @@ def _time_within(self, time_type: Time, *, hours: int=0, days: int=0, years: int def _generate_members(self) -> None: pass + @pydantic.validate_call def has_name(self, name: str) -> Self: self._filters.append(f"name == {name}") return self - + + @pydantic.validate_call def has_name_containing(self, name: str) -> Self: self._filters.append(f"name contains {name}") return self - - def created_within(self, *, hours: int=0, days: int=0, years: int=0) -> Self: + + @pydantic.validate_call + def created_within( + self, + *, + hours: pydantic.NonNegativeInt = 0, + days: pydantic.NonNegativeInt = 0, + years: pydantic.NonNegativeInt = 0, + ) -> Self: return self._time_within(Time.Created, hours=hours, days=days, years=years) - + + @pydantic.validate_call def has_description_containing(self, search_str: str) -> Self: self._filters.append(f"description contains {search_str}") return self - + + @pydantic.validate_call def exclude_description_containing(self, search_str: str) -> Self: self._filters.append(f"description not contains {search_str}") return self - + + @pydantic.validate_call def has_tag(self, tag: str) -> Self: self._filters.append(f"has tag.{tag}") return self @@ -93,81 +114,89 @@ class FoldersFilter(RestAPIFilter): def has_path(self, name: str) -> "FoldersFilter": self._filters.append(f"path == {name}") return self - + def has_path_containing(self, name: str) -> "FoldersFilter": self._filters.append(f"path contains {name}") return self - + def _generate_members(self) -> None: return super()._generate_members() class RunsFilter(RestAPIFilter): def _generate_members(self) -> None: - _global_comparators = [ - self._value_contains, - self._value_eq, - self._value_neq - ] + _global_comparators = [self._value_contains, self._value_eq, self._value_neq] _numeric_comparators = [ self._value_geq, self._value_leq, self._value_lt, - self._value_gt + self._value_gt, ] for label, system_spec in System.__members__.items(): for function in _global_comparators: _label: str = label.lower() _func_name: str = function.__name__.replace("_value", _label) - _out_func = lambda value, func=function: func("system", system_spec.value, value) + _out_func = lambda value, func=function: func( + "system", system_spec.value, value + ) _out_func.__name__ = _func_name setattr(self, _func_name, _out_func) for function in _global_comparators + _numeric_comparators: _func_name: str = function.__name__.replace("_value", "metadata") - _out_func = lambda attribute, value, func=function: func("metadata", attribute, value) + _out_func = lambda attribute, value, func=function: func( + "metadata", attribute, value + ) _out_func.__name__ = _func_name setattr(self, _func_name, _out_func) - def author(self, username: str="self") -> "RunsFilter": + @pydantic.validate_call + def author(self, username: str = "self") -> "RunsFilter": self._filters.append(f"user == {username}") return self - def exclude_author(self, username: str="self") -> "RunsFilter": + @pydantic.validate_call + def exclude_author(self, username: str = "self") -> "RunsFilter": self._filters.append(f"user != {username}") return self - + def starred(self) -> "RunsFilter": self._filters.append("starred") return self - + + @pydantic.validate_call def has_name(self, name: str) -> "RunsFilter": self._filters.append(f"name == {name}") return self - + + @pydantic.validate_call def has_name_containing(self, name: str) -> "RunsFilter": self._filters.append(f"name contains {name}") return self + @pydantic.validate_call def has_status(self, status: Status) -> "RunsFilter": self._filters.append(f"status == {status.value}") return self - + def is_running(self) -> "RunsFilter": return self.has_status(Status.Running) - + def is_lost(self) -> "RunsFilter": return self.has_status(Status.Lost) - + def has_completed(self) -> "RunsFilter": return self.has_status(Status.Completed) - + def has_failed(self) -> "RunsFilter": return self.has_status(Status.Failed) - - def has_alert(self, alert_name: str, is_critical: typing.Optional[bool]=None) -> "RunsFilter": + + @pydantic.validate_call + def has_alert( + self, alert_name: str, is_critical: typing.Optional[bool] = None + ) -> "RunsFilter": self._filters.append(f"alert.name == {alert_name}") if is_critical is True: self._filters.append("alert.status == critical") @@ -175,57 +204,95 @@ def has_alert(self, alert_name: str, is_critical: typing.Optional[bool]=None) -> self._filters.append("alert.status == ok") return self - def started_within(self, *, hours: int=0, days: int=0, years: int=0) -> "RunsFilter": + @pydantic.validate_call + def started_within( + self, + *, + hours: pydantic.PositiveInt = 0, + days: pydantic.PositiveInt = 0, + years: pydantic.PositiveInt = 0, + ) -> "RunsFilter": return self._time_within(Time.Started, hours=hours, days=days, years=years) - - def modified_within(self, *, hours: int=0, days: int=0, years: int=0) -> "RunsFilter": + + @pydantic.validate_call + def modified_within( + self, + *, + hours: pydantic.PositiveInt = 0, + days: pydantic.PositiveInt = 0, + years: pydantic.PositiveInt = 0, + ) -> "RunsFilter": return self._time_within(Time.Modified, hours=hours, days=days, years=years) - - def ended_within(self, *, hours: int=0, days: int=0, years: int=0) -> "RunsFilter": + + @pydantic.validate_call + def ended_within( + self, + *, + hours: pydantic.PositiveInt = 0, + days: pydantic.PositiveInt = 0, + years: pydantic.PositiveInt = 0, + ) -> "RunsFilter": return self._time_within(Time.Ended, hours=hours, days=days, years=years) - + + @pydantic.validate_call def in_folder(self, folder_name: str) -> "RunsFilter": self._filters.append(f"folder.path == {folder_name}") return self + @pydantic.validate_call def has_metadata_attribute(self, attribute: str) -> "RunsFilter": self._filters.append(f"metadata.{attribute} exists") return self + @pydantic.validate_call def exclude_metadata_attribute(self, attribute: str) -> "RunsFilter": self._filters.append(f"metadata.{attribute} not exists") return self - - def _value_eq(self, category: str, attribute: str, value: typing.Union[str, int, float]) -> "RunsFilter": + + def _value_eq( + self, category: str, attribute: str, value: typing.Union[str, int, float] + ) -> "RunsFilter": self._filters.append(f"{category}.{attribute} == {value}") return self - - def _value_neq(self, category: str, attribute: str, value: typing.Union[str, int, float]) -> "RunsFilter": + + def _value_neq( + self, category: str, attribute: str, value: typing.Union[str, int, float] + ) -> "RunsFilter": self._filters.append(f"{category}.{attribute} != {value}") return self - - def _value_contains(self, category: str, attribute: str, value: typing.Union[str, int, float]) -> "RunsFilter": + + def _value_contains( + self, category: str, attribute: str, value: typing.Union[str, int, float] + ) -> "RunsFilter": self._filters.append(f"{category}.{attribute} contains {value}") return self - - def _value_leq(self, category: str, attribute: str, value: typing.Union[int, float]) -> "RunsFilter": + + def _value_leq( + self, category: str, attribute: str, value: typing.Union[int, float] + ) -> "RunsFilter": self._filters.append(f"{category}.{attribute} <= {value}") return self - - def _value_geq(self, category: str, attribute: str, value: typing.Union[int, float]) -> "RunsFilter": + + def _value_geq( + self, category: str, attribute: str, value: typing.Union[int, float] + ) -> "RunsFilter": self._filters.append(f"{category}.{attribute} >= {value}") return self - - def _value_lt(self, category: str, attribute: str, value: typing.Union[int, float]) -> "RunsFilter": + + def _value_lt( + self, category: str, attribute: str, value: typing.Union[int, float] + ) -> "RunsFilter": self._filters.append(f"{category}.{attribute} < {value}") return self - - def _value_gt(self, category: str, attribute: str, value: typing.Union[int, float]) -> "RunsFilter": + + def _value_gt( + self, category: str, attribute: str, value: typing.Union[int, float] + ) -> "RunsFilter": self._filters.append(f"{category}.{attribute} > {value}") return self - + def __str__(self) -> str: return " && ".join(self._filters) if self._filters else "None" - + def __repr__(self) -> str: return f"{super().__repr__()[:-1]}, filters={self._filters}>" diff --git a/tests/refactor/test_client.py b/tests/refactor/test_client.py index 2aac41ab..9e09cdcd 100644 --- a/tests/refactor/test_client.py +++ b/tests/refactor/test_client.py @@ -120,9 +120,13 @@ def test_get_artifacts_as_files(create_test_run: tuple[sv_run.Run, dict]) -> Non @pytest.mark.dependency @pytest.mark.client @pytest.mark.parametrize( - "filters", (None, "list", "object"), ids=("no_filters", "filter_list", "filter_object") + "filters", + (None, "list", "object"), + ids=("no_filters", "filter_list", "filter_object"), ) -def test_get_runs(create_test_run: tuple[sv_run.Run, dict], filters: typing.Optional[str]) -> None: +def test_get_runs( + create_test_run: tuple[sv_run.Run, dict], filters: typing.Optional[str] +) -> None: client = svc.Client() if filters == "list": @@ -145,15 +149,30 @@ def test_get_run(create_test_run: tuple[sv_run.Run, dict]) -> None: @pytest.mark.dependency @pytest.mark.client -def test_get_folder(create_test_run: tuple[sv_run.Run, dict]) -> None: +@pytest.mark.parametrize( + "filters", + (None, "list", "object"), + ids=("no_filters", "filters_list", "filters_object"), +) +def test_get_folder( + create_test_run: tuple[sv_run.Run, dict], + filters: typing.Optional[str] +) -> None: + if filters == "list": + filter_object = ["path == /simvue_unit_test_folder"] + elif filters == "object": + filter_object = sv_filter.FoldersFilter() + filter_object.has_path("/simvue_unit_test_folder") + else: + filter_object = None client = svc.Client() - assert (folders := client.get_folders()) + assert (folders := client.get_folders(filters=filter_object)) assert (folder_id := folders[0].get("id")) assert client.get_folder(folder_id) @pytest.mark.dependency -@pytest.mark.client +@pytest.mark.client def test_get_metrics_names(create_test_run: tuple[sv_run.Run, dict]) -> None: client = svc.Client() time.sleep(1) From 412c9407a7376734ce77ad94f60962a72257d468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Fri, 2 Aug 2024 08:58:52 +0100 Subject: [PATCH 07/10] Fix typing of filters --- simvue/client.py | 4 ++-- simvue/filter.py | 48 ++++++++++++++++++++++++------------------------ 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/simvue/client.py b/simvue/client.py index 289f238e..aa5cd473 100644 --- a/simvue/client.py +++ b/simvue/client.py @@ -262,7 +262,7 @@ def get_run_name_from_id(self, run_id: str) -> str: return _name @prettify_pydantic - @pydantic.validate_call + @pydantic.validate_call(config={"arbitrary_types_allowed": True}) def get_runs( self, filters: typing.Optional[typing.Union[list[str], RunsFilter]], @@ -901,7 +901,7 @@ def get_folder( return None return _folders[0] - @pydantic.validate_call + @pydantic.validate_call(config={"arbitrary_types_allowed": True}) def get_folders( self, filters: typing.Optional[typing.Union[list[str], FoldersFilter]] = None, diff --git a/simvue/filter.py b/simvue/filter.py index 1aa7a434..30a10e76 100644 --- a/simvue/filter.py +++ b/simvue/filter.py @@ -153,50 +153,50 @@ def _generate_members(self) -> None: setattr(self, _func_name, _out_func) @pydantic.validate_call - def author(self, username: str = "self") -> "RunsFilter": + def author(self, username: str = "self") -> Self: self._filters.append(f"user == {username}") return self @pydantic.validate_call - def exclude_author(self, username: str = "self") -> "RunsFilter": + def exclude_author(self, username: str = "self") -> Self: self._filters.append(f"user != {username}") return self - def starred(self) -> "RunsFilter": + def starred(self) -> Self: self._filters.append("starred") return self @pydantic.validate_call - def has_name(self, name: str) -> "RunsFilter": + def has_name(self, name: str) -> Self: self._filters.append(f"name == {name}") return self @pydantic.validate_call - def has_name_containing(self, name: str) -> "RunsFilter": + def has_name_containing(self, name: str) -> Self: self._filters.append(f"name contains {name}") return self @pydantic.validate_call - def has_status(self, status: Status) -> "RunsFilter": + def has_status(self, status: Status) -> Self: self._filters.append(f"status == {status.value}") return self - def is_running(self) -> "RunsFilter": + def is_running(self) -> Self: return self.has_status(Status.Running) - def is_lost(self) -> "RunsFilter": + def is_lost(self) -> Self: return self.has_status(Status.Lost) - def has_completed(self) -> "RunsFilter": + def has_completed(self) -> Self: return self.has_status(Status.Completed) - def has_failed(self) -> "RunsFilter": + def has_failed(self) -> Self: return self.has_status(Status.Failed) @pydantic.validate_call def has_alert( self, alert_name: str, is_critical: typing.Optional[bool] = None - ) -> "RunsFilter": + ) -> Self: self._filters.append(f"alert.name == {alert_name}") if is_critical is True: self._filters.append("alert.status == critical") @@ -211,7 +211,7 @@ def started_within( hours: pydantic.PositiveInt = 0, days: pydantic.PositiveInt = 0, years: pydantic.PositiveInt = 0, - ) -> "RunsFilter": + ) -> Self: return self._time_within(Time.Started, hours=hours, days=days, years=years) @pydantic.validate_call @@ -221,7 +221,7 @@ def modified_within( hours: pydantic.PositiveInt = 0, days: pydantic.PositiveInt = 0, years: pydantic.PositiveInt = 0, - ) -> "RunsFilter": + ) -> Self: return self._time_within(Time.Modified, hours=hours, days=days, years=years) @pydantic.validate_call @@ -231,63 +231,63 @@ def ended_within( hours: pydantic.PositiveInt = 0, days: pydantic.PositiveInt = 0, years: pydantic.PositiveInt = 0, - ) -> "RunsFilter": + ) -> Self: return self._time_within(Time.Ended, hours=hours, days=days, years=years) @pydantic.validate_call - def in_folder(self, folder_name: str) -> "RunsFilter": + def in_folder(self, folder_name: str) -> Self: self._filters.append(f"folder.path == {folder_name}") return self @pydantic.validate_call - def has_metadata_attribute(self, attribute: str) -> "RunsFilter": + def has_metadata_attribute(self, attribute: str) -> Self: self._filters.append(f"metadata.{attribute} exists") return self @pydantic.validate_call - def exclude_metadata_attribute(self, attribute: str) -> "RunsFilter": + def exclude_metadata_attribute(self, attribute: str) -> Self: self._filters.append(f"metadata.{attribute} not exists") return self def _value_eq( self, category: str, attribute: str, value: typing.Union[str, int, float] - ) -> "RunsFilter": + ) -> Self: self._filters.append(f"{category}.{attribute} == {value}") return self def _value_neq( self, category: str, attribute: str, value: typing.Union[str, int, float] - ) -> "RunsFilter": + ) -> Self: self._filters.append(f"{category}.{attribute} != {value}") return self def _value_contains( self, category: str, attribute: str, value: typing.Union[str, int, float] - ) -> "RunsFilter": + ) -> Self: self._filters.append(f"{category}.{attribute} contains {value}") return self def _value_leq( self, category: str, attribute: str, value: typing.Union[int, float] - ) -> "RunsFilter": + ) -> Self: self._filters.append(f"{category}.{attribute} <= {value}") return self def _value_geq( self, category: str, attribute: str, value: typing.Union[int, float] - ) -> "RunsFilter": + ) -> Self: self._filters.append(f"{category}.{attribute} >= {value}") return self def _value_lt( self, category: str, attribute: str, value: typing.Union[int, float] - ) -> "RunsFilter": + ) -> Self: self._filters.append(f"{category}.{attribute} < {value}") return self def _value_gt( self, category: str, attribute: str, value: typing.Union[int, float] - ) -> "RunsFilter": + ) -> Self: self._filters.append(f"{category}.{attribute} > {value}") return self From b48e9933f9eb38c8be497059d35e630b1df7fd5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Tue, 10 Sep 2024 08:20:07 +0100 Subject: [PATCH 08/10] Fix folder retrieval test --- tests/refactor/test_client.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/refactor/test_client.py b/tests/refactor/test_client.py index 3856f8f0..22b18942 100644 --- a/tests/refactor/test_client.py +++ b/tests/refactor/test_client.py @@ -176,14 +176,8 @@ def test_get_folder( else: filter_object = None client = svc.Client() -<<<<<<< HEAD - assert (folders := client.get_folders(filters=filter_object)) - assert (folder_id := folders[0].get("id")) -======= - assert (folders := client.get_folders()) - assert (folder_id := folders[1].get("path")) ->>>>>>> dev - assert client.get_folder(folder_id) + assert client.get_folders(filters=filter_object) + assert client.get_folder(folder_path="/simvue_unit_test_folder") @pytest.mark.dependency From 10686a0e111ce49af3003dace2295dc4dae0d356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Tue, 10 Sep 2024 15:13:32 +0100 Subject: [PATCH 09/10] Retrieve correct folder during testing --- tests/refactor/test_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/refactor/test_client.py b/tests/refactor/test_client.py index 22b18942..7ebe7ef3 100644 --- a/tests/refactor/test_client.py +++ b/tests/refactor/test_client.py @@ -169,15 +169,15 @@ def test_get_folder( filters: typing.Optional[str] ) -> None: if filters == "list": - filter_object = ["path == /simvue_unit_test_folder"] + filter_object = ["path == /simvue_unit_testing"] elif filters == "object": filter_object = sv_filter.FoldersFilter() - filter_object.has_path("/simvue_unit_test_folder") + filter_object.has_path("/simvue_unit_testing") else: filter_object = None client = svc.Client() assert client.get_folders(filters=filter_object) - assert client.get_folder(folder_path="/simvue_unit_test_folder") + assert client.get_folder(folder_path="/simvue_unit_testing") @pytest.mark.dependency From 83327d161710bb2676e5b77e7a53c74e550a9eac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Zar=C4=99bski?= Date: Wed, 25 Sep 2024 13:54:13 +0100 Subject: [PATCH 10/10] Fix linting issues --- simvue/filter.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/simvue/filter.py b/simvue/filter.py index 30a10e76..775afc19 100644 --- a/simvue/filter.py +++ b/simvue/filter.py @@ -125,9 +125,13 @@ def _generate_members(self) -> None: class RunsFilter(RestAPIFilter): def _generate_members(self) -> None: - _global_comparators = [self._value_contains, self._value_eq, self._value_neq] + _global_comparators: list[typing.Callable] = [ + self._value_contains, + self._value_eq, + self._value_neq, + ] - _numeric_comparators = [ + _numeric_comparators: list[typing.Callable] = [ self._value_geq, self._value_leq, self._value_lt, @@ -138,17 +142,19 @@ def _generate_members(self) -> None: for function in _global_comparators: _label: str = label.lower() _func_name: str = function.__name__.replace("_value", _label) - _out_func = lambda value, func=function: func( - "system", system_spec.value, value - ) + + def _out_func(value, func=function): + return func("system", system_spec.value, value) + _out_func.__name__ = _func_name setattr(self, _func_name, _out_func) for function in _global_comparators + _numeric_comparators: - _func_name: str = function.__name__.replace("_value", "metadata") - _out_func = lambda attribute, value, func=function: func( - "metadata", attribute, value - ) + _func_name = function.__name__.replace("_value", "metadata") + + def _out_func(attribute, value, func=function): + return func("metadata", attribute, value) + _out_func.__name__ = _func_name setattr(self, _func_name, _out_func)