From f08b96b47206847dd02ef80c44ec2eb50c0739eb Mon Sep 17 00:00:00 2001 From: Laurent TOURREAU Date: Tue, 1 Jul 2025 09:01:00 +0200 Subject: [PATCH] Externalize image registry URL configuration --- 3-prod_datascience/evaluate_model.py | 17 ++++++----------- .../prod_train_save_pipeline.py | 15 +++++++-------- 3-prod_datascience/save_model.py | 19 +++++-------------- 3 files changed, 18 insertions(+), 33 deletions(-) diff --git a/3-prod_datascience/evaluate_model.py b/3-prod_datascience/evaluate_model.py index 1d16cd2..81aad85 100644 --- a/3-prod_datascience/evaluate_model.py +++ b/3-prod_datascience/evaluate_model.py @@ -17,9 +17,9 @@ def evaluate_keras_model_performance( scaler: Input[Artifact], label_encoder: Input[Artifact], model_name: str, - cluster_domain: str, version: str, - prod_flag: bool, + model_registry_server_address: str, + model_registry_port: int, metrics: Output[Metrics], classification_metrics: Output[ClassificationMetrics] ): @@ -47,15 +47,10 @@ def evaluate_keras_model_performance( y_test_argmax = np.argmax(y_test, axis=1) accuracy = np.sum(y_pred_argmax == y_test_argmax) / len(y_pred_argmax) - - # Get the previous models properties from the Model Registry - if prod_flag: - namespace = environ.get("NAMESPACE").split("-")[0]+"-prod" - else: - namespace = environ.get("NAMESPACE").split("-")[0] - environ["KF_PIPELINES_SA_TOKEN_PATH"] = "/var/run/secrets/kubernetes.io/serviceaccount/token" # Hotfix to access the endpoint - registry = ModelRegistry(server_address=f"https://{namespace}-registry-rest.{cluster_domain}", port=443, author="someone", is_secure=False) + environ["KF_PIPELINES_SA_TOKEN_PATH"] = "/var/run/secrets/kubernetes.io/serviceaccount/token" + registry = ModelRegistry(server_address=model_registry_server_address, port=model_registry_port, author="someone",is_secure=False) + previous_model_properties = {} #Wrap with try except to see if the model exists in the registry @@ -119,4 +114,4 @@ def validate_onnx_model( for rt_res, keras_res in zip(onnx_pred[0], keras_pred): np.testing.assert_allclose(rt_res, keras_res, rtol=1e-5, atol=1e-5) - print("Results match") + print("Results match") \ No newline at end of file diff --git a/3-prod_datascience/prod_train_save_pipeline.py b/3-prod_datascience/prod_train_save_pipeline.py index a5b7d5f..a8a8c02 100644 --- a/3-prod_datascience/prod_train_save_pipeline.py +++ b/3-prod_datascience/prod_train_save_pipeline.py @@ -1,4 +1,3 @@ -# kfp imports import kfp import kfp.dsl as dsl from kfp.dsl import ( @@ -30,7 +29,7 @@ name='kfp-training-pipeline', description='We train an amazing model 🚂' ) -def training_pipeline(hyperparameters: dict, model_name: str, version: str, cluster_domain: str, model_storage_pvc: str, prod_flag: bool): +def training_pipeline(hyperparameters: dict, model_name: str, version: str, model_storage_pvc: str, prod_flag: bool, model_registry_server_address: str, model_registry_port: int): ### 🐶 Fetch Data from GitHub fetch_task = fetch_data() @@ -63,9 +62,9 @@ def training_pipeline(hyperparameters: dict, model_name: str, version: str, clus scaler = pre_processing_task.outputs["scaler"], label_encoder = pre_processing_task.outputs["label_encoder"], model_name = model_name, - cluster_domain = cluster_domain, - version = version, # Add version to force a rerun of this step every new version - prod_flag = prod_flag, + version = version, + model_registry_server_address = model_registry_server_address, + model_registry_port = model_registry_port, ) kubernetes.use_field_path_as_env( model_evaluation_task, @@ -85,8 +84,9 @@ def training_pipeline(hyperparameters: dict, model_name: str, version: str, clus register_model_task = push_to_model_registry( model_name = model_name, version = version, - cluster_domain = cluster_domain, prod_flag = prod_flag, + model_registry_server_address = model_registry_server_address, + model_registry_port = model_registry_port, keras_model = training_task.outputs["trained_model"], model = convert_task.outputs["onnx_model"], metrics = model_evaluation_task.outputs["metrics"], @@ -122,7 +122,6 @@ def training_pipeline(hyperparameters: dict, model_name: str, version: str, clus }, "model_name": "jukebox", "version": "0.0.2", - "cluster_domain": "", # 👈 add your cluster domain here "model_storage_pvc": "jukebox-model-pvc", "prod_flag": False } @@ -154,4 +153,4 @@ def training_pipeline(hyperparameters: dict, model_name: str, version: str, clus arguments=metadata, experiment_name="kfp-training-pipeline", enable_caching=True - ) + ) \ No newline at end of file diff --git a/3-prod_datascience/save_model.py b/3-prod_datascience/save_model.py index fc0988b..e38c374 100644 --- a/3-prod_datascience/save_model.py +++ b/3-prod_datascience/save_model.py @@ -14,8 +14,9 @@ def push_to_model_registry( model_name: str, version: str, - cluster_domain: str, prod_flag: bool, + model_registry_server_address: str, + model_registry_port: int, keras_model: Input[Artifact], model: Input[Artifact], metrics: Input[Metrics], @@ -82,24 +83,14 @@ def _do_upload(s3_client, model_path, object_name, s3_bucket_name): environ["KF_PIPELINES_SA_TOKEN_PATH"] = "/var/run/secrets/kubernetes.io/serviceaccount/token" ############ Register to Model Registry ############ - namespace_file_path =\ - '/var/run/secrets/kubernetes.io/serviceaccount/namespace' - with open(namespace_file_path, 'r') as namespace_file: - namespace = namespace_file.read() - - if prod_flag: - namespace = namespace.split("-")[0]+"-prod" - else: - namespace = namespace.split("-")[0] model_object_prefix = model_name if model_name else "model" version = version if version else datetime.now().strftime('%y%m%d%H%M') - server_address = f"https://{namespace}-registry-rest.{cluster_domain}" registry = ModelRegistry( - server_address=server_address, - port=443, - author=namespace, + server_address=model_registry_server_address, + port=model_registry_port, + author="someone", is_secure=False ) registered_model_name = model_object_prefix