sdk/python: Add PyYaml requirement and update dependency versions

Signed-off-by: Aaron Wilson <[email protected]>

python/.pylintrc

@@ -358,13 +358,13 @@ ignore-comments=yes
 ignore-docstrings=yes
 
 # Imports are removed from the similarity computation
-ignore-imports=no
+ignore-imports=yes
 
 # Signatures are removed from the similarity computation
-ignore-signatures=no
+ignore-signatures=yes
 
 # Minimum lines number of a similarity.
-min-similarity-lines=4
+min-similarity-lines=6
 
 
 [VARIABLES]

python/aistore/common_requirements

@@ -1,12 +1,13 @@
-pytest==7.1.1
+boto3==1.35.51
+braceexpand==0.1.7
+cloudpickle==3.1.0
+humanize==4.10.0
+msgspec>=0.15.1
+overrides==7.7.0
 pydantic==1.10.14
-cloudpickle==2.2.0
-pylint==3.2.6
-typing_extensions==4.2.0
+pylint==3.2.7
+pytest==7.4.4
+PyYAML==6.0.2
 requests==2.32.3
-humanize>=4.6.0
-braceexpand>=0.1.7
-msgspec>=0.15.1
-overrides>=7.7.0
-boto3==1.34.15
-webdataset==0.2.86
+typing_extensions==4.12.2
+webdataset==0.2.100

python/aistore/sdk/authn/authn_client.py

@@ -21,7 +21,6 @@
 logger = get_logger(__name__)
 
 
-# pylint: disable=too-many-arguments, too-few-public-methods
 class AuthNClient:
     """
     AuthN client for managing authentication.
@@ -40,6 +39,7 @@ class AuthNClient:
         token (str, optional): Authorization token.
     """
 
+    # pylint: disable=too-many-arguments
     def __init__(
         self,
         endpoint: str,

python/aistore/sdk/authn/role_manager.py

@@ -1,9 +1,6 @@
 #
 # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
 #
-
-# pylint: disable=too-many-arguments, duplicate-code
-
 from typing import List
 
 from aistore.sdk.provider import Provider
@@ -42,6 +39,7 @@ class RoleManager:
         client (RequestClient): The RequestClient used to make HTTP requests.
     """
 
+    # pylint: disable=too-many-arguments
     def __init__(self, client: RequestClient):
         self._client = client
 

python/aistore/sdk/client.py

@@ -1,8 +1,6 @@
 #
 # Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
 #
-
-from __future__ import annotations  # pylint: disable=unused-variable
 from typing import Optional, Tuple, Union
 import os
 
@@ -23,7 +21,6 @@
 from aistore.sdk.errors import InvalidURLException
 
 
-# pylint: disable=unused-variable, duplicate-code, too-many-arguments
 class Client:
     """
     AIStore client for managing buckets, objects, ETL jobs
@@ -41,6 +38,7 @@ class Client:
             will be used. Defaults to None.
     """
 
+    # pylint: disable=too-many-arguments
     def __init__(
         self,
         endpoint: str,

python/aistore/sdk/job.py

@@ -2,10 +2,7 @@
 # Copyright (c) 2022-2023, NVIDIA CORPORATION. All rights reserved.
 #
 
-from __future__ import annotations  # pylint: disable=unused-variable
-
 import itertools
-import logging
 from datetime import datetime, timezone
 from typing import List, Dict
 import time
@@ -25,10 +22,11 @@
 from aistore.sdk.errors import Timeout, JobInfoNotFound
 from aistore.sdk.request_client import RequestClient
 from aistore.sdk.types import JobStatus, JobArgs, ActionMsg, JobSnapshot, BucketModel
-from aistore.sdk.utils import probing_frequency
+from aistore.sdk.utils import probing_frequency, get_logger
+
+logger = get_logger(__name__)
 
 
-# pylint: disable=unused-variable
 class Job:
     """
     A class containing job-related functions.
@@ -39,7 +37,6 @@ class Job:
         job_kind (str, optional): Specific kind of job, empty for all kinds
     """
 
-    # pylint: disable=duplicate-code
     def __init__(self, client: RequestClient, job_id: str = "", job_kind: str = ""):
         self._client = client
         self._job_id = job_id
@@ -106,7 +103,6 @@ def wait(
             requests.ReadTimeout: Timed out waiting response from AIStore
             errors.Timeout: Timeout while waiting for the job to finish
         """
-        logger = logging.getLogger(f"{__name__}.wait")
         logger.disabled = not verbose
         passed = 0
         sleep_time = probing_frequency(timeout)
@@ -157,10 +153,8 @@ def wait_for_idle(
             errors.JobInfoNotFound: Raised when information on a job's status could not be found on the AIS cluster
         """
         action = f"job '{self._job_id}' to reach idle state"
-        logger_name = "wait_for_idle"
-        self._wait_for_condition(
-            self._check_job_idle, action, logger_name, verbose, timeout
-        )
+        logger.disabled = not verbose
+        self._wait_for_condition(self._check_job_idle, action, timeout)
 
     def wait_single_node(
         self,
@@ -185,11 +179,9 @@ def wait_single_node(
             errors.Timeout: Timeout while waiting for the job to finish
             errors.JobInfoNotFound: Raised when information on a job's status could not be found on the AIS cluster
         """
-        logger_name = "wait_single_node"
         action = f"job '{self._job_id}' to finish"
-        self._wait_for_condition(
-            self._check_snapshot_finished, action, logger_name, verbose, timeout
-        )
+        logger.disabled = not verbose
+        self._wait_for_condition(self._check_snapshot_finished, action, timeout)
 
     def start(
         self,
@@ -294,7 +286,7 @@ def _query_job_snapshots(self) -> List[JobSnapshot]:
         snapshots = list(itertools.chain.from_iterable(snapshot_lists))
         return snapshots
 
-    def _check_snapshot_finished(self, snapshots, logger):
+    def _check_snapshot_finished(self, snapshots):
         job_found = False
         snapshot = None
         for snap in snapshots:
@@ -316,7 +308,7 @@ def _check_snapshot_finished(self, snapshots, logger):
             return True
         return False
 
-    def _check_job_idle(self, snapshots, logger):
+    def _check_job_idle(self, snapshots):
         job_found = False
         for snap in snapshots:
             if snap.id != self._job_id:
@@ -330,18 +322,14 @@ def _check_job_idle(self, snapshots, logger):
         logger.info("Job '%s' reached idle state", self._job_id)
         return True
 
-    # pylint: disable=too-many-arguments
-    def _wait_for_condition(self, condition_fn, action, logger_name, verbose, timeout):
-        logger = logging.getLogger(f"{__name__}.{logger_name}")
-        logger.disabled = not verbose
+    def _wait_for_condition(self, condition_fn, action, timeout):
         passed = 0
         sleep_time = probing_frequency(timeout)
 
         while True:
-            snapshots = []
             snapshots = self._query_job_snapshots()
             try:
-                if condition_fn(snapshots, logger):
+                if condition_fn(snapshots):
                     return
             except JobInfoNotFound:
                 logger.info("No information found for job %s, retrying", self._job_id)

python/aistore/sdk/multiobj/object_group.py

@@ -32,7 +32,6 @@
 from aistore.sdk.request_client import RequestClient
 
 
-# pylint: disable=unused-variable
 class ObjectGroup(AISSource):
     """
     A class representing multiple objects within the same bucket. Only one of obj_names, obj_range, or obj_template
@@ -350,6 +349,7 @@ def transform(
             HTTP_METHOD_POST, ACT_TRANSFORM_OBJECTS, value=value
         ).text
 
+    # pylint: disable=too-many-arguments
     def archive(
         self,
         archive_name: str,

python/aistore/sdk/multiobj/object_range.py

@@ -10,7 +10,7 @@
 from aistore.sdk.multiobj.object_collection import ObjectCollection
 
 
-# pylint: disable=unused-variable,too-few-public-methods
+# pylint: disable=too-few-public-methods
 class ObjectRange(ObjectCollection):
     """
     Class representing a range of object names

python/aistore/sdk/request_client.py

@@ -23,7 +23,6 @@
 T = TypeVar("T")
 
 
-# pylint: disable=unused-variable, duplicate-code, too-many-arguments
 class RequestClient:
     """
     Internal client for making requests to an AIS cluster or a specific proxy (e.g. AuthN).
@@ -39,6 +38,7 @@ class RequestClient:
             raise_ais_error.
     """
 
+    # pylint: disable=too-many-arguments
     def __init__(
         self,
         endpoint: str,

python/pyproject.toml

@@ -25,7 +25,7 @@ authors = [
 ]
 description = "A (growing) set of client-side APIs to access and utilize clusters, buckets, and objects on AIStore."
 readme = "README.md"
-requires-python = ">=3.7"
+requires-python = ">=3.8"
 license = {text = "MIT License"}
 classifiers = [
     "Development Status :: 4 - Beta",
@@ -41,8 +41,9 @@ dependencies = [
     "requests",
     "packaging",
     "overrides",
-    "pydantic==1.10.14",
-    "cloudpickle==2.2.0",
+    "PyYAML",
+    "pydantic>=1.10.14",
+    "cloudpickle>=2.2.0",
     "humanize>=4.6.0",
     "braceexpand>=0.1.7",
     "msgspec>=0.15.1",