From 0709b3ccba96fd38e388151069075ada5c71471a Mon Sep 17 00:00:00 2001 From: William Shen Date: Wed, 23 Nov 2022 17:18:22 -0500 Subject: [PATCH 1/2] Support custom S3 endpoint --- ml_logger/ml_logger.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ml_logger/ml_logger.py b/ml_logger/ml_logger.py index 6504414..3c1ccbc 100644 --- a/ml_logger/ml_logger.py +++ b/ml_logger/ml_logger.py @@ -37,6 +37,7 @@ # ML_Logger defaults ROOT = os.environ.get("ML_LOGGER_ROOT", CWD) or CWD S3_ROOT = os.environ.get("ML_LOGGER_S3_ROOT", None) +S3_ENDPOINT = os.environ.get("ML_LOGGER_S3_ENDPOINT", None) LOGGER_USER = os.environ.get("ML_LOGGER_USER", USER) ACCESS_TOKEN = os.environ.get("ML_LOGGER_ACCESS_TOKEN", None) @@ -939,7 +940,7 @@ def glob_s3(self, query="*", wd=None, max_keys=1000, **KWargs): query_prefix = '/'.join(query_prefix) truncate = len(work_prefix) + 1 if work_prefix else 0 - s3_client = boto3.client('s3') + s3_client = boto3.client('s3', endpoint_url=S3_ENDPOINT) # list_objects_v2 supports pagination. -- Ge response = s3_client.list_objects(Bucket=bucket, Prefix=s3_prefix, MaxKeys=max_keys, **KWargs) @@ -1416,8 +1417,8 @@ def download_dir(self, source_path, to, unpack='tar'): @staticmethod def remove_s3(bucket, *keys): import boto3 - client = boto3.client('s3') - return client.delete_object(Bucket=bucket, Key=pJoin(*keys)) + s3_client = boto3.client('s3', endpoint_url=S3_ENDPOINT) + return s3_client.delete_object(Bucket=bucket, Key=pJoin(*keys)) @staticmethod def remove_gs(*keys, path=None): @@ -1487,7 +1488,7 @@ def upload_s3(source_path, *keys, path=None): # todo: consider adding exception handling -- good or bad? # Upload the file - s3_client = boto3.client('s3') + s3_client = boto3.client('s3', endpoint_url=S3_ENDPOINT) # from botocore.exceptions import ClientError # try: response = s3_client.upload_file(source_path, bucket, object_name) @@ -1506,8 +1507,8 @@ def download_s3(*keys, path=None, to): bucket, *object_name = path.split('/') object_name = '/'.join(object_name) - s3 = boto3.client('s3') - return s3.download_file(bucket, object_name, to) + s3_client = boto3.client('s3', endpoint_url=S3_ENDPOINT) + return s3_client.download_file(bucket, object_name, to) def save_images(self, stack, key, n_rows=None, n_cols=None, cmap=None, normalize=None, value_range=(0, 1), background=1, dtype=np.uint8): From f89a71c31466edc743ba987c70c99d79ec18c15f Mon Sep 17 00:00:00 2001 From: William Shen Date: Wed, 23 Nov 2022 17:18:46 -0500 Subject: [PATCH 2/2] Increment version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 8a56a06..c514dac 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.8.109 +0.8.110