From f0222614a5cb42b494df2eac83ccb7a9c0f22e31 Mon Sep 17 00:00:00 2001 From: Rehan Ullah Date: Tue, 23 Dec 2025 17:13:52 +0000 Subject: [PATCH 1/2] add iam auth --- .../dfp_external_storage.js | 13 +++++ .../dfp_external_storage.json | 42 ++++++++++---- .../dfp_external_storage.py | 57 +++++++++++++++---- pyproject.toml | 1 + 4 files changed, 92 insertions(+), 21 deletions(-) diff --git a/dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.js b/dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.js index ec59391..5d91c2a 100644 --- a/dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.js +++ b/dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.js @@ -5,7 +5,20 @@ frappe.ui.form.on('DFP External Storage', { frm.button_remote_files_list = null }, + auth_type: function(frm) { + // Toggle visibility of credential fields based on auth type + frm.toggle_reqd('access_key', frm.doc.auth_type === 'Key based') + frm.toggle_reqd('secret_key', frm.doc.auth_type === 'Key based') + frm.refresh_field('access_key') + frm.refresh_field('secret_key') + }, + refresh: function(frm) { + // Set initial required state based on auth_type + if (frm.doc.auth_type) { + frm.toggle_reqd('access_key', frm.doc.auth_type === 'Key based') + frm.toggle_reqd('secret_key', frm.doc.auth_type === 'Key based') + } if (frm.is_new() && !frm.doc.doctypes_ignored.length) { frm.doc.doctypes_ignored.push({doctype_to_ignore: 'Data Import'}) frm.doc.doctypes_ignored.push({doctype_to_ignore: 'Prepared Report'}) diff --git a/dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.json b/dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.json index 7df880b..cbbd2c6 100644 --- a/dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.json +++ b/dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.json @@ -16,8 +16,10 @@ "secure", "bucket_name", "region", + "auth_type", "access_key", "secret_key", + "session_token", "folders", "files_within", "advanced_settings_section", @@ -44,31 +46,51 @@ "reqd": 1 }, { + "default": "auto", + "description": "Region name of buckets in S3 service. Default value: \"auto\".", + "documentation_url": "https://min.io/docs/minio/linux/developers/python/API.html", + "fieldname": "region", + "fieldtype": "Data", + "label": "Region", + "reqd": 1 + }, + { + "default": "Key based", + "description": "Authentication method to use. 'Key based' requires Access Key and Secret Key. You can also optionally provide session token if you have short-lived credentials. 'Aws Iam' to use AWS IAM credentials (e.g., from AWS credentials file or IAM role).", + "fieldname": "auth_type", + "fieldtype": "Select", + "label": "Authentication Type", + "options": "Key based\nAws Iam", + "reqd": 1 + }, + { + "depends_on": "eval:doc.auth_type == 'Key based'", "description": "Access key (aka user ID) of your account in S3 service.", "documentation_url": "https://min.io/docs/minio/linux/developers/python/API.html", "fieldname": "access_key", "fieldtype": "Data", "in_list_view": 1, - "label": "Access Key", - "reqd": 1 + "label": "Access Key" }, { + "depends_on": "eval:doc.auth_type == 'Key based'", "description": "Secret Key (aka password) of your account in S3 service.", "documentation_url": "https://min.io/docs/minio/linux/developers/python/API.html", "fieldname": "secret_key", "fieldtype": "Password", "label": "Secret Key", - "no_copy": 1, - "reqd": 1 + "no_copy": 1 }, { - "default": "auto", - "description": "Region name of buckets in S3 service. Default value: \"auto\".", + "depends_on": "eval:doc.auth_type == 'Key based'", + "description": "Session token of your account in S3 service.", "documentation_url": "https://min.io/docs/minio/linux/developers/python/API.html", - "fieldname": "region", - "fieldtype": "Data", - "label": "Region", - "reqd": 1 + "fieldname": "session_token", + "fieldtype": "Password", + "label": "Session Token", + "length": 2000, + "reqd": 0, + "no_copy": 1 }, { "description": "Be careful! Please set up your bucket as private in your provider and/or remove public access to objects. Some files you upload to Frappe will be for sure private!", diff --git a/dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.py b/dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.py index f982662..cad24f9 100644 --- a/dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.py +++ b/dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.py @@ -15,7 +15,8 @@ from frappe.core.doctype.file.file import URL_PREFIXES from frappe.model.document import Document from frappe.utils.password import get_decrypted_password - +from minio.credentials.providers import IamAwsProvider +from minio.credentials.providers import StaticProvider DFP_EXTERNAL_STORAGE_PUBLIC_CACHE_PREFIX = "external_storage_public_file:" @@ -25,9 +26,9 @@ DFP_EXTERNAL_STORAGE_CONNECTION_FIELDS = [ - "type", "endpoint", "secure", "bucket_name", "region", "access_key", "secret_key"] + "type", "endpoint", "secure", "bucket_name", "region", "auth_type", "access_key", "secret_key"] DFP_EXTERNAL_STORAGE_CRITICAL_FIELDS = [ - "type", "endpoint", "secure", "bucket_name", "region", "access_key", "secret_key", "folders"] + "type", "endpoint", "secure", "bucket_name", "region", "auth_type", "access_key", "secret_key", "folders"] class S3FileProxy: @@ -124,35 +125,69 @@ def validate_bucket(self): @cached_property def client(self): - if self.endpoint and self.access_key and self.secret_key and self.region: + if not self.endpoint or not self.region: + return None + + # Handle IAM authentication + if getattr(self, 'auth_type', 'Key based') == 'Aws Iam': + try: + credentials = IamAwsProvider( + region=self.region, + ) + if credentials: + return MinioConnection( + endpoint=self.endpoint, + credentials=credentials, + region=self.region, + ) + except: + pass + + # Handle Key based authentication + if self.access_key and self.secret_key: try: if self.is_new() and self.secret_key: key_secret = self.secret_key else: key_secret = get_decrypted_password("DFP External Storage", self.name, "secret_key") + + # session_token is optional, so handle the case where it might not exist + session_token = None + if self.is_new() and self.session_token: + session_token = self.session_token + else: + try: + session_token = get_decrypted_password("DFP External Storage", self.name, "session_token") + except: + # session_token is optional, so it's okay if it doesn't exist + session_token = None + if key_secret: - return MinioConnection( - endpoint=self.endpoint, + credentials = StaticProvider( access_key=self.access_key, secret_key=key_secret, + session_token=session_token, + ) + return MinioConnection( + endpoint=self.endpoint, + credentials=credentials, region=self.region, - secure=self.secure, ) except: pass + + return None def remote_files_list(self): return self.client.list_objects(self.bucket_name, recursive=True) class MinioConnection: - def __init__(self, endpoint:str, access_key:str, secret_key:str, region:str, secure:bool): + def __init__(self, endpoint:str, region:str, credentials:StaticProvider | IamAwsProvider): self.client = Minio( endpoint=endpoint, - access_key=access_key, - secret_key=secret_key, region=region, - secure=secure, + credentials=credentials ) def validate_bucket(self, bucket_name:str): diff --git a/pyproject.toml b/pyproject.toml index d8869a0..10d1f13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ readme = "README.md" dynamic = ["version"] dependencies = [ # "frappe~=15.0.0" # Installed and managed by bench. + "minio", ] [build-system] From 5cefdf6eba7f5cd5c639ab08eb5d662a31439f8c Mon Sep 17 00:00:00 2001 From: Rehan Ullah Date: Mon, 29 Dec 2025 18:49:50 +0000 Subject: [PATCH 2/2] pass secure flag --- .../doctype/dfp_external_storage/dfp_external_storage.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.py b/dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.py index 8a18287..bf1f069 100644 --- a/dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.py +++ b/dfp_external_storage/dfp_external_storage/doctype/dfp_external_storage/dfp_external_storage.py @@ -139,6 +139,7 @@ def client(self): endpoint=self.endpoint, credentials=credentials, region=self.region, + secure=self.secure, ) except: pass @@ -174,6 +175,7 @@ def client(self): endpoint=self.endpoint, credentials=credentials, region=self.region, + secure=self.secure, ) except Exception: pass @@ -185,11 +187,12 @@ def remote_files_list(self): class MinioConnection: - def __init__(self, endpoint:str, region:str, credentials:StaticProvider | IamAwsProvider): + def __init__(self, endpoint:str, region:str, credentials:StaticProvider | IamAwsProvider, secure:bool): self.client = Minio( endpoint=endpoint, region=region, - credentials=credentials + credentials=credentials, + secure=secure, ) def validate_bucket(self, bucket_name:str):