Patch code up to fd9e0b7

Signed-off-by: Github Actions <[email protected]>

deployment/istio/authz/authz-chatqa.yaml

@@ -88,6 +88,7 @@ spec:
     - source:
         notPrincipals:
         - cluster.local/ns/chatqa/sa/reranking-usvc
+        - cluster.local/ns/edp/sa/edp-chart
         - cluster.local/ns/monitoring/sa/telemetry-kube-prometheus-prometheus
 ---
 apiVersion: security.istio.io/v1
@@ -281,6 +282,7 @@ spec:
     - source:
         notPrincipals:
         - cluster.local/ns/chatqa/sa/default
+        - cluster.local/ns/edp/sa/edp-chart
         - cluster.local/ns/monitoring/sa/telemetry-kube-prometheus-prometheus
 ---
 # llm-svc

deployment/istio/authz/authz-edp.yaml

@@ -61,6 +61,7 @@ spec:
   - from:
     - source:
         notPrincipals:
+        - cluster.local/ns/edp/sa/edp-chart
         - cluster.local/ns/edp/sa/edp-minio
         - cluster.local/ns/auth-apisix/sa/auth-apisix
         - cluster.local/ns/monitoring/sa/telemetry-kube-prometheus-prometheus

src/edp/app/tasks.py

@@ -99,9 +99,11 @@ def process_file_task(self, file_id: Any, *args, **kwargs):
     file_db.job_message = 'Data preparation in progress.'
     self.db.commit()
 
+    minio_response = None
+    file_base64= None
     try:
-        response = self.minio.get_object(bucket_name=file_db.bucket_name, object_name=file_db.object_name)
-        file_data = response.read()
+        minio_response = self.minio.get_object(bucket_name=file_db.bucket_name, object_name=file_db.object_name)
+        file_data = minio_response.read()
         file_base64 = base64.b64encode(file_data).decode('ascii')
         logger.debug(f"[{file_db.id}] Retrievied file from S3 storage.")
     except S3Error as e:
@@ -111,8 +113,9 @@ def process_file_task(self, file_id: Any, *args, **kwargs):
         self.db.commit()
         raise Exception(f"Error downloading file. {e}")
     finally:
-        response.close()
-        response.release_conn()
+        if minio_response is not None:
+            minio_response.close()
+            minio_response.release_conn()
 
     # Step 2 - Call the data preparation service
     filename = file_db.object_name.split('/')[-1]
@@ -244,6 +247,7 @@ def delete_file_task(self, file_id: Any, *args, **kwargs):
     # Step 2 - Delete the file from database
     id = file_db.id
     self.db.delete(file_db)
+    self.db.commit()
     logger.debug(f"[{id}] File deleted successfully from database.")
     return True
 
@@ -401,5 +405,6 @@ def delete_link_task(self, link_id: Any, *args, **kwargs):
     # Step 2 - Delete the file from database
     id = link_db.id
     self.db.delete(link_db)
+    self.db.commit()
     logger.debug(f"[{id}] File deleted successfully from database.")
     return True

src/edp/terraform/README.md

@@ -26,7 +26,7 @@ export edp_storage_type="s3"
 export s3_access_key=$(terraform output -raw access_key)
 export s3_secret_key=$(terraform output -raw secret_key)
 export s3_sqs_queue=$(terraform output -raw queue_url)
-export s3_region="us-east-1" # optional
+export s3_region=$(terraform output -raw region)
 
 cd deployment
 ./install_chatqna.sh --auth --deploy xeon_torch --ui --upgrade --kind

src/edp/terraform/main.tf

@@ -8,6 +8,7 @@ resource "aws_s3_bucket" "edp_buckets" {
 
 resource "aws_sqs_queue" "edp_queue_1" {
   name = var.queue_name
+  sqs_managed_sse_enabled = true
 }
 
 resource "aws_iam_user" "edp_user" {
@@ -28,6 +29,16 @@ resource "aws_iam_policy" "list_all_buckets" {
         Action   = ["s3:ListAllMyBuckets"]
         Effect   = "Allow"
         Resource = "*"
+      },
+      {
+        Action = ["s3:*"],
+        Effect = "Deny",
+        Resource = "*",
+        Condition = {
+          Bool = {
+            "aws:SecureTransport" = "false"
+          }
+        }
       }
     ]
   })
@@ -68,6 +79,16 @@ resource "aws_iam_policy" "read_only_bucket" {
         Action   = ["s3:GetObject"]
         Effect   = "Allow"
         Resource = "${aws_s3_bucket.edp_buckets[0].arn}/*"
+      },
+      {
+        Action = ["s3:*"],
+        Effect = "Deny",
+        Resource = ["${aws_s3_bucket.edp_buckets[0].arn}/*"],
+        Condition = {
+          Bool = {
+            "aws:SecureTransport" = "false"
+          }
+        }
       }
     ]
   })
@@ -83,6 +104,16 @@ resource "aws_iam_policy" "read_write_bucket_1" {
         Action   = ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"]
         Effect   = "Allow"
         Resource = "${aws_s3_bucket.edp_buckets[1].arn}/*"
+      },
+      {
+        Action = ["s3:*"],
+        Effect = "Deny",
+        Resource = ["${aws_s3_bucket.edp_buckets[1].arn}/*"],
+        Condition = {
+          Bool = {
+            "aws:SecureTransport" = "false"
+          }
+        }
       }
     ]
   })
@@ -98,6 +129,16 @@ resource "aws_iam_policy" "read_write_bucket_2" {
         Action   = ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"]
         Effect   = "Allow"
         Resource = "${aws_s3_bucket.edp_buckets[2].arn}/*"
+      },
+      {
+        Action = ["s3:*"],
+        Effect = "Deny",
+        Resource = ["${aws_s3_bucket.edp_buckets[1].arn}/*"],
+        Condition = {
+          Bool = {
+            "aws:SecureTransport" = "false"
+          }
+        }
       }
     ]
   })
@@ -178,3 +219,7 @@ output "secret_key" {
   value = aws_iam_access_key.edp_access_key.secret
   sensitive = true
 }
+
+output "region" {
+  value = var.region
+}