From 56bf4a0f03f90dbeeb8592aff4db4e375554fc96 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Tue, 18 Mar 2025 15:14:01 +0700 Subject: [PATCH 01/78] feat: adjust kaniko job flags --- api/turing/imagebuilder/imagebuilder.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/turing/imagebuilder/imagebuilder.go b/api/turing/imagebuilder/imagebuilder.go index 73d42e83c..bf69c3ebb 100644 --- a/api/turing/imagebuilder/imagebuilder.go +++ b/api/turing/imagebuilder/imagebuilder.go @@ -297,8 +297,6 @@ func (ib *imageBuilder) createKanikoJob( fmt.Sprintf("--build-arg=MLFLOW_ARTIFACT_STORAGE_TYPE=%s", ib.artifactServiceType), fmt.Sprintf("--build-arg=FOLDER_NAME=%s", folderName), fmt.Sprintf("--destination=%s", imageRef), - "--cache=true", - "--single-snapshot", } kanikoArgs = append(kanikoArgs, ib.imageBuildingConfig.KanikoConfig.AdditionalArgs...) From b4035ef73c2d1d3a100e257ddedb085ac0ed5390 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 19 Mar 2025 14:29:04 +0700 Subject: [PATCH 02/78] feat: implement getHashedModelDependenciesUrl --- api/turing/imagebuilder/imagebuilder.go | 43 +++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/api/turing/imagebuilder/imagebuilder.go b/api/turing/imagebuilder/imagebuilder.go index bf69c3ebb..6b5979b6a 100644 --- a/api/turing/imagebuilder/imagebuilder.go +++ b/api/turing/imagebuilder/imagebuilder.go @@ -2,6 +2,7 @@ package imagebuilder import ( "context" + "crypto/sha256" "fmt" "net/http" "os" @@ -22,6 +23,7 @@ import ( "github.com/caraml-dev/merlin/utils" mlp "github.com/caraml-dev/mlp/api/client" + "github.com/caraml-dev/mlp/api/pkg/artifact" "github.com/caraml-dev/turing/api/turing/cluster" "github.com/caraml-dev/turing/api/turing/config" "github.com/caraml-dev/turing/api/turing/log" @@ -44,6 +46,7 @@ const ( kanikoSecretFileName = "kaniko-secret.json" kanikoSecretMountpath = "/secret" kanikoDockerCredentialConfigPath = "/kaniko/.docker" + modelDependenciesPath = "/turing/model_dependencies" ) // JobStatus is the current status of the image building job. @@ -131,6 +134,7 @@ type imageBuilder struct { nameGenerator nameGenerator runnerType models.EnsemblerRunnerType artifactServiceType string + artifactService artifact.Service } // NewImageBuilder creates a new ImageBuilder @@ -659,3 +663,42 @@ func parseJobConditions(jobConditions []apibatchv1.JobCondition) (string, error) jobTable := utils.LogTable(jobConditionHeaders, jobConditionRows) return jobTable, err } + +// getHashedModelDependenciesUrl stores dependency to storage using it's content hashed name as filename. +// if the artifact is recreated with different id but has the same dependency content the URL for the +// stored dependency will still be the same. This ensure we can use Docker layer caching mechanism for +// the built dependency even for different artifact id given the dependency content is not changed. +func (ib *imageBuilder) getHashedModelDependenciesUrl(ctx context.Context, artifactURI string) (string, error) { + artifactURL, err := ib.artifactService.ParseURL(artifactURI) + if err != nil { + return "", err + } + + condaEnvUrl := fmt.Sprintf("%s://%s/%s/model/conda.yaml", ib.artifactService.GetURLScheme(), artifactURL.Bucket, artifactURL.Object) + + condaEnv, err := ib.artifactService.ReadArtifact(ctx, condaEnvUrl) + if err != nil { + return "", err + } + + hash := sha256.New() + hash.Write(condaEnv) + hashEnv := hash.Sum(nil) + + hashedDependenciesUrl := fmt.Sprintf("%s://%s%s/%x", ib.artifactService.GetURLScheme(), artifactURL.Bucket, modelDependenciesPath, hashEnv) + + _, err = ib.artifactService.ReadArtifact(ctx, hashedDependenciesUrl) + if err == nil { + return hashedDependenciesUrl, nil + } + + if !errors.Is(err, artifact.ErrObjectNotExist) { + return "", err + } + + if err := ib.artifactService.WriteArtifact(ctx, hashedDependenciesUrl, condaEnv); err != nil { + return "", err + } + + return hashedDependenciesUrl, nil +} From 0a41d34071fe484b2f6f4cf031dafbd6ac0b02c1 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 19 Mar 2025 14:29:21 +0700 Subject: [PATCH 03/78] feat: create unit test for getHashedModelDependenciesUrl --- api/turing/imagebuilder/imagebuilder_test.go | 89 ++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/api/turing/imagebuilder/imagebuilder_test.go b/api/turing/imagebuilder/imagebuilder_test.go index 556165e0d..f5b7634c7 100644 --- a/api/turing/imagebuilder/imagebuilder_test.go +++ b/api/turing/imagebuilder/imagebuilder_test.go @@ -1,6 +1,8 @@ package imagebuilder import ( + "context" + "crypto/sha256" "fmt" "testing" "time" @@ -13,6 +15,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "github.com/caraml-dev/mlp/api/pkg/artifact" + "github.com/caraml-dev/mlp/api/pkg/artifact/mocks" "github.com/caraml-dev/turing/api/turing/cluster" clustermock "github.com/caraml-dev/turing/api/turing/cluster/mocks" "github.com/caraml-dev/turing/api/turing/config" @@ -1266,3 +1270,88 @@ func TestDeleteEnsemblerServiceImageBuildingJob(t *testing.T) { }) } } + +func Test_imageBuilder_getHashedModelDependenciesUrl(t *testing.T) { + testArtifactURISuffix := "://bucket-name/mlflow/3069/e130c40703ee424da97b9ecee7b874b7/artifacts" + testArtifactURI := "gs://bucket-name/mlflow/3069/e130c40703ee424da97b9ecee7b874b7/artifacts" + testArtifactGsutilURL := &artifact.URL{ + Bucket: "bucket-name", + Object: "mlflow/3069/e130c40703ee424da97b9ecee7b874b7/artifacts", + } + testCondaEnvContent := `dependencies: +- python=3.9.* +- pip: + - mlflow` + testCondaEnvUrlSuffix := testArtifactURISuffix + "/model/conda.yaml" + + hash := sha256.New() + hash.Write([]byte(testCondaEnvContent)) + hashEnv := hash.Sum(nil) + + modelDependenciesURL := fmt.Sprintf("gs://%s/turing/model_dependencies/%x", testArtifactGsutilURL.Bucket, hashEnv) + + type args struct { + ctx context.Context + artifactURI string + } + tests := []struct { + name string + args args + artifactServiceMock func(*mocks.Service) + want string + wantErr bool + }{ + { + name: "hash dependencies is already exist", + args: args{ + ctx: context.Background(), + artifactURI: testArtifactURI, + }, + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, + want: modelDependenciesURL, + wantErr: false, + }, + { + name: "hash dependencies is not exist yet", + args: args{ + ctx: context.Background(), + artifactURI: testArtifactURI, + }, + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return(nil, artifact.ErrObjectNotExist) + artifactServiceMock.On("WriteArtifact", mock.Anything, modelDependenciesURL, []byte(testCondaEnvContent)).Return(nil) + }, + want: modelDependenciesURL, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + artifactServiceMock := &mocks.Service{} + tt.artifactServiceMock(artifactServiceMock) + + c := &imageBuilder{ + artifactService: artifactServiceMock, + } + + got, err := c.getHashedModelDependenciesUrl(tt.args.ctx, tt.args.artifactURI) + if (err != nil) != tt.wantErr { + t.Errorf("imageBuilder.getHashedModelDependenciesUrl() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("imageBuilder.getHashedModelDependenciesUrl() = %v, want %v", got, tt.want) + } + }) + } +} From 538e74a361289748cefd602c76974e76c5e2260a Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 19 Mar 2025 14:55:07 +0700 Subject: [PATCH 04/78] feat: refactor app.Dockerfile to use hashed dependency --- engines/pyfunc-ensembler-service/app.Dockerfile | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/engines/pyfunc-ensembler-service/app.Dockerfile b/engines/pyfunc-ensembler-service/app.Dockerfile index 29776cc45..2d8cb76a9 100644 --- a/engines/pyfunc-ensembler-service/app.Dockerfile +++ b/engines/pyfunc-ensembler-service/app.Dockerfile @@ -7,6 +7,7 @@ ARG MLFLOW_ARTIFACT_STORAGE_TYPE ARG MODEL_URL ARG FOLDER_NAME ARG GOOGLE_APPLICATION_CREDENTIALS +ARG MODEL_DEPENDENCIES_URL ARG AWS_ACCESS_KEY_ID ARG AWS_SECRET_ACCESS_KEY @@ -23,6 +24,20 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ echo "No credentials are used"; \ fi +# Download and install user model dependencies +ARG MODEL_DEPENDENCIES_URL +RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ + gsutil cp ${MODEL_DEPENDENCIES_URL} conda.yaml; \ + elif [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "s3" ]; then \ + S3_KEY=${MODEL_DEPENDENCIES_URL##*s3://}; \ + aws s3api get-object --bucket ${S3_KEY%%/*} --key ${S3_KEY#*/} conda.yaml; \ + else \ + echo "No credentials are used"; \ + fi + +RUN /bin/bash -c "conda env update --name ${CONDA_ENV_NAME} --file ./conda.yaml" + + RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ gsutil -m cp -r ${MODEL_URL} .; \ elif [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "s3" ]; then \ @@ -31,8 +46,6 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ echo "No credentials are used"; \ fi -RUN /bin/bash -c "conda env update --name ${CONDA_ENV_NAME} --file ./${FOLDER_NAME}/conda.yaml" - ENV FOLDER_NAME=$FOLDER_NAME SHELL ["/bin/bash", "-c"] ENTRYPOINT . activate ${CONDA_ENV_NAME} && \ From 26a6ad3bb0f1785fc97532d0424454a4d4c35189 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 19 Mar 2025 14:55:49 +0700 Subject: [PATCH 05/78] feat: call getHashedModelDependenciesUrl and pass to kanikojob --- api/turing/imagebuilder/imagebuilder.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/api/turing/imagebuilder/imagebuilder.go b/api/turing/imagebuilder/imagebuilder.go index 6b5979b6a..71b6e9268 100644 --- a/api/turing/imagebuilder/imagebuilder.go +++ b/api/turing/imagebuilder/imagebuilder.go @@ -173,6 +173,12 @@ func (ib *imageBuilder) BuildImage(request BuildImageRequest) (string, error) { return imageRef, nil } + hashedModelDependenciesUrl, err := ib.getHashedModelDependenciesUrl(context.Background(), request.ArtifactURI) + if err != nil { + log.Errorf("unable to get model dependencies url: %v", err) + return "", err + } + // Check if there is an existing build job kanikoJobName := ib.nameGenerator.generateBuilderName( request.ProjectName, @@ -193,7 +199,7 @@ func (ib *imageBuilder) BuildImage(request BuildImageRequest) (string, error) { } job, err = ib.createKanikoJob(kanikoJobName, imageRef, request.ArtifactURI, request.BuildLabels, - request.EnsemblerFolder, request.BaseImageRefTag) + request.EnsemblerFolder, request.BaseImageRefTag, hashedModelDependenciesUrl) if err != nil { log.Errorf("unable to build image %s, error: %v", imageRef, err) return "", ErrUnableToBuildImage @@ -214,7 +220,7 @@ func (ib *imageBuilder) BuildImage(request BuildImageRequest) (string, error) { } job, err = ib.createKanikoJob(kanikoJobName, imageRef, request.ArtifactURI, request.BuildLabels, - request.EnsemblerFolder, request.BaseImageRefTag) + request.EnsemblerFolder, request.BaseImageRefTag, hashedModelDependenciesUrl) if err != nil { log.Errorf("unable to build image %s, error: %v", imageRef, err) return "", ErrUnableToBuildImage @@ -284,6 +290,7 @@ func (ib *imageBuilder) createKanikoJob( buildLabels map[string]string, ensemblerFolder string, baseImageRefTag string, + hashedModelDependenciesUrl string, ) (*apibatchv1.Job, error) { splitURI := strings.Split(artifactURI, "/") folderName := fmt.Sprintf("%s/%s", splitURI[len(splitURI)-1], ensemblerFolder) @@ -300,6 +307,7 @@ func (ib *imageBuilder) createKanikoJob( fmt.Sprintf("--build-arg=BASE_IMAGE=%s", baseImage), fmt.Sprintf("--build-arg=MLFLOW_ARTIFACT_STORAGE_TYPE=%s", ib.artifactServiceType), fmt.Sprintf("--build-arg=FOLDER_NAME=%s", folderName), + fmt.Sprintf("--build-arg=MODEL_DEPENDENCIES_URL=%s", hashedModelDependenciesUrl), fmt.Sprintf("--destination=%s", imageRef), } From 928a33cc9b789d8c8e338d8b531598b557aecc17 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 20 Mar 2025 12:01:53 +0700 Subject: [PATCH 06/78] chore: artifactService dependency injection --- api/turing/api/appcontext.go | 18 ++++++++++++++++++ api/turing/imagebuilder/ensembler.go | 5 +++++ api/turing/imagebuilder/imagebuilder.go | 4 +++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/api/turing/api/appcontext.go b/api/turing/api/appcontext.go index ba35de433..1dc83f30e 100644 --- a/api/turing/api/appcontext.go +++ b/api/turing/api/appcontext.go @@ -10,6 +10,7 @@ import ( mlpcluster "github.com/caraml-dev/mlp/api/pkg/cluster" + "github.com/caraml-dev/mlp/api/pkg/artifact" batchensembling "github.com/caraml-dev/turing/api/turing/batch/ensembling" batchrunner "github.com/caraml-dev/turing/api/turing/batch/runner" "github.com/caraml-dev/turing/api/turing/cluster" @@ -94,6 +95,8 @@ func NewAppContext( // Init ensemblers service ensemblersService := service.NewEnsemblersService(db) + artifactService, err := initArtifactService(cfg) + if cfg.BatchEnsemblingConfig.Enabled { if cfg.BatchEnsemblingConfig.JobConfig == nil { return nil, errors.Wrapf(err, "BatchEnsemblingConfig.JobConfig was not set") @@ -125,6 +128,7 @@ func NewAppContext( imageBuildingController, *cfg.BatchEnsemblingConfig.ImageBuildingConfig, cfg.MlflowConfig.ArtifactServiceType, + artifactService, ) if err != nil { return nil, errors.Wrapf(err, "Failed initializing ensembling image builder") @@ -160,6 +164,7 @@ func NewAppContext( clusterControllers[cfg.EnsemblerServiceBuilderConfig.ClusterName], *cfg.EnsemblerServiceBuilderConfig.ImageBuildingConfig, cfg.MlflowConfig.ArtifactServiceType, + artifactService, ) if err != nil { return nil, errors.Wrapf(err, "Failed initializing ensembler service builder") @@ -236,3 +241,16 @@ func buildKubeconfigStore(mlpSvc service.MLPService, cfg *config.Config) (map[st } return k8sConfigStore, nil } + +func initArtifactService(cfg *config.Config) (artifact.Service, error) { + if cfg.MlflowConfig.ArtifactServiceType == "gcs" { + return artifact.NewGcsArtifactClient() + } + if cfg.MlflowConfig.ArtifactServiceType == "s3" { + return artifact.NewS3ArtifactClient() + } + if cfg.MlflowConfig.ArtifactServiceType == "nop" { + return artifact.NewNopArtifactClient(), nil + } + return nil, fmt.Errorf("invalid artifact service type %s", cfg.MlflowConfig.ArtifactServiceType) +} diff --git a/api/turing/imagebuilder/ensembler.go b/api/turing/imagebuilder/ensembler.go index 617f7505d..76858eda3 100644 --- a/api/turing/imagebuilder/ensembler.go +++ b/api/turing/imagebuilder/ensembler.go @@ -3,6 +3,7 @@ package imagebuilder import ( "fmt" + "github.com/caraml-dev/mlp/api/pkg/artifact" "github.com/caraml-dev/turing/api/turing/cluster" "github.com/caraml-dev/turing/api/turing/config" "github.com/caraml-dev/turing/api/turing/models" @@ -13,6 +14,7 @@ func NewEnsemblerJobImageBuilder( clusterController cluster.Controller, imageBuildingConfig config.ImageBuildingConfig, artifactServiceType string, + artifactService artifact.Service, ) (ImageBuilder, error) { return newImageBuilder( clusterController, @@ -20,6 +22,7 @@ func NewEnsemblerJobImageBuilder( &ensemblerJobNameGenerator{registry: imageBuildingConfig.DestinationRegistry}, models.EnsemblerRunnerTypeJob, artifactServiceType, + artifactService, ) } @@ -51,6 +54,7 @@ func NewEnsemblerServiceImageBuilder( clusterController cluster.Controller, imageBuildingConfig config.ImageBuildingConfig, artifactServiceType string, + artifactService artifact.Service, ) (ImageBuilder, error) { return newImageBuilder( clusterController, @@ -58,6 +62,7 @@ func NewEnsemblerServiceImageBuilder( &ensemblerServiceNameGenerator{registry: imageBuildingConfig.DestinationRegistry}, models.EnsemblerRunnerTypeService, artifactServiceType, + artifactService, ) } diff --git a/api/turing/imagebuilder/imagebuilder.go b/api/turing/imagebuilder/imagebuilder.go index 71b6e9268..32b0baff8 100644 --- a/api/turing/imagebuilder/imagebuilder.go +++ b/api/turing/imagebuilder/imagebuilder.go @@ -144,6 +144,7 @@ func newImageBuilder( nameGenerator nameGenerator, runnerType models.EnsemblerRunnerType, artifactServiceType string, + artifactService artifact.Service, ) (ImageBuilder, error) { err := checkParseResources(imageBuildingConfig.KanikoConfig.ResourceRequestsLimits) if err != nil { @@ -156,6 +157,7 @@ func newImageBuilder( nameGenerator: nameGenerator, runnerType: runnerType, artifactServiceType: artifactServiceType, + artifactService: artifactService, }, nil } @@ -682,7 +684,7 @@ func (ib *imageBuilder) getHashedModelDependenciesUrl(ctx context.Context, artif return "", err } - condaEnvUrl := fmt.Sprintf("%s://%s/%s/model/conda.yaml", ib.artifactService.GetURLScheme(), artifactURL.Bucket, artifactURL.Object) + condaEnvUrl := fmt.Sprintf("%s://%s/%s/ensembler/conda.yaml", ib.artifactService.GetURLScheme(), artifactURL.Bucket, artifactURL.Object) condaEnv, err := ib.artifactService.ReadArtifact(ctx, condaEnvUrl) if err != nil { From 58e50f57b673136af3ea55d048ddaf26ecf4d632 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 20 Mar 2025 12:30:05 +0700 Subject: [PATCH 07/78] chore: adjust imagebuilder_test --- api/turing/imagebuilder/imagebuilder_test.go | 220 +++++++++++++++++-- 1 file changed, 200 insertions(+), 20 deletions(-) diff --git a/api/turing/imagebuilder/imagebuilder_test.go b/api/turing/imagebuilder/imagebuilder_test.go index f5b7634c7..169d1d096 100644 --- a/api/turing/imagebuilder/imagebuilder_test.go +++ b/api/turing/imagebuilder/imagebuilder_test.go @@ -23,7 +23,28 @@ import ( "github.com/caraml-dev/turing/api/turing/models" ) -var timeout, _ = time.ParseDuration("10s") +var ( + timeout, _ = time.ParseDuration("10s") + testArtifactURISuffix = "://bucket-name/mlflow/3069/e130c40703ee424da97b9ecee7b874b7/artifacts" + testArtifactURI = "gs://bucket-name/mlflow/3069/e130c40703ee424da97b9ecee7b874b7/artifacts" + testArtifactGsutilURL = &artifact.URL{ + Bucket: "bucket-name", + Object: "mlflow/3069/e130c40703ee424da97b9ecee7b874b7/artifacts", + } + testCondaEnvContent = `dependencies: +- python=3.9.* +- pip: + - mlflow` + testCondaEnvUrlSuffix = testArtifactURISuffix + "/ensembler/conda.yaml" +) + +func getHashedModelDependenciesUrl() string { + hash := sha256.New() + hash.Write([]byte(testCondaEnvContent)) + hashEnv := hash.Sum(nil) + + return fmt.Sprintf("gs://%s/turing/model_dependencies/%x", testArtifactGsutilURL.Bucket, hashEnv) +} const ( projectName = "test-project" @@ -42,6 +63,8 @@ const ( ) func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { + modelDependenciesURL := getHashedModelDependenciesUrl() + imageBuildingConfig := config.ImageBuildingConfig{ BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, @@ -81,6 +104,7 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { clusterController func() cluster.Controller ensemblerFolder string imageTag string + artifactServiceMock func(*mocks.Service) }{ "success | no existing job": { expected: fmt.Sprintf("%s/%s/ensembler-jobs/%s:%s", dockerRegistry, projectName, modelName, runID), @@ -142,6 +166,13 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { imageBuildingConfig: imageBuildingConfig, ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, "success: existing job is running": { expected: fmt.Sprintf("%s/%s/ensembler-jobs/%s:%s", dockerRegistry, projectName, modelName, runID), @@ -204,6 +235,13 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { imageBuildingConfig: imageBuildingConfig, ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, "success: existing job failed": { expected: fmt.Sprintf("%s/%s/ensembler-jobs/%s:%s", dockerRegistry, projectName, modelName, runID), @@ -290,17 +328,28 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { imageBuildingConfig: imageBuildingConfig, ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { + artifactServiceMock := &mocks.Service{} + tt.artifactServiceMock(artifactServiceMock) + clusterController := tt.clusterController() ib, err := NewEnsemblerJobImageBuilder( clusterController, tt.imageBuildingConfig, googleCloudStorageArtifactServiceType, + artifactServiceMock, ) assert.Nil(t, err) @@ -309,7 +358,7 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { tt.modelName, tt.modelID, tt.versionID, - tt.artifactURI, + testArtifactURI, tt.buildLabels, tt.ensemblerFolder, tt.imageTag, @@ -322,6 +371,8 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { } func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { + modelDependenciesURL := getHashedModelDependenciesUrl() + imageBuildingConfig := config.ImageBuildingConfig{ BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, @@ -362,6 +413,7 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { clusterController func() cluster.Controller ensemblerFolder string imageTag string + artifactServiceMock func(*mocks.Service) }{ "success | no existing job": { expectedImage: fmt.Sprintf("%s/%s/ensembler-services/%s:%s", dockerRegistry, projectName, modelName, runID), @@ -423,6 +475,13 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { imageBuildingConfig: imageBuildingConfig, ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, "success: existing job is running": { expectedImage: fmt.Sprintf("%s/%s/ensembler-services/%s:%s", dockerRegistry, projectName, modelName, runID), @@ -485,6 +544,13 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { imageBuildingConfig: imageBuildingConfig, ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, "success: existing job failed": { expectedImage: fmt.Sprintf("%s/%s/ensembler-services/%s:%s", dockerRegistry, projectName, modelName, runID), @@ -571,6 +637,13 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { imageBuildingConfig: imageBuildingConfig, ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, "failure: image tag not matched": { expectedImageBuildingError: "error building OCI image", @@ -602,17 +675,28 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { imageBuildingConfig: imageBuildingConfig, ensemblerFolder: ensemblerFolder, imageTag: "3.8.*", + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { + artifactServiceMock := &mocks.Service{} + tt.artifactServiceMock(artifactServiceMock) + clusterController := tt.clusterController() ib, err := NewEnsemblerServiceImageBuilder( clusterController, tt.imageBuildingConfig, googleCloudStorageArtifactServiceType, + artifactServiceMock, ) assert.Nil(t, err) @@ -621,7 +705,7 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { tt.modelName, tt.modelID, tt.versionID, - tt.artifactURI, + testArtifactURI, tt.buildLabels, tt.ensemblerFolder, tt.imageTag, @@ -723,6 +807,8 @@ func TestParseResources(t *testing.T) { } func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { + modelDependenciesURL := getHashedModelDependenciesUrl() + imageBuildingConfig := config.ImageBuildingConfig{ BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, @@ -752,6 +838,7 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { imageBuildingConfig config.ImageBuildingConfig hasErr bool expected JobStatus + artifactServiceMock func(*mocks.Service) }{ "success | active": { imageBuildingConfig: imageBuildingConfig, @@ -771,6 +858,13 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { expected: JobStatus{ State: JobStateActive, }, + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, "success | succeeded": { imageBuildingConfig: imageBuildingConfig, @@ -790,6 +884,13 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { expected: JobStatus{ State: JobStateSucceeded, }, + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, "success | Failed": { imageBuildingConfig: imageBuildingConfig, @@ -870,6 +971,13 @@ Pod container status: Pod last termination message: CondaEnvException: Pip failed`, }, + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, "success | Unknown": { imageBuildingConfig: imageBuildingConfig, @@ -891,6 +999,13 @@ CondaEnvException: Pip failed`, expected: JobStatus{ State: JobStateUnknown, }, + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, "failure | Unknown": { imageBuildingConfig: imageBuildingConfig, @@ -907,15 +1022,26 @@ CondaEnvException: Pip failed`, State: JobStateUnknown, Message: "hello", }, + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { + artifactServiceMock := &mocks.Service{} + tt.artifactServiceMock(artifactServiceMock) + clusterController := tt.clusterController() ib, _ := NewEnsemblerJobImageBuilder( clusterController, tt.imageBuildingConfig, googleCloudStorageArtifactServiceType, + artifactServiceMock, ) status := ib.GetImageBuildingJobStatus(projectName, modelName, models.ID(1), runID) assert.Equal(t, tt.expected, status) @@ -924,6 +1050,8 @@ CondaEnvException: Pip failed`, } func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { + modelDependenciesURL := getHashedModelDependenciesUrl() + imageBuildingConfig := config.ImageBuildingConfig{ BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, @@ -953,6 +1081,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { imageBuildingConfig config.ImageBuildingConfig hasErr bool expected JobStatus + artifactServiceMock func(*mocks.Service) }{ "success | active": { imageBuildingConfig: imageBuildingConfig, @@ -972,6 +1101,13 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { expected: JobStatus{ State: JobStateActive, }, + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, "success | succeeded": { imageBuildingConfig: imageBuildingConfig, @@ -991,6 +1127,13 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { expected: JobStatus{ State: JobStateSucceeded, }, + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, "success | Failed": { imageBuildingConfig: config.ImageBuildingConfig{ @@ -1039,6 +1182,13 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { expected: JobStatus{ State: JobStateFailed, }, + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, "success | Unknown": { imageBuildingConfig: config.ImageBuildingConfig{ @@ -1083,6 +1233,13 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { expected: JobStatus{ State: JobStateUnknown, }, + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, "failure | Unknown": { imageBuildingConfig: config.ImageBuildingConfig{ @@ -1122,15 +1279,26 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { State: JobStateUnknown, Message: "hello", }, + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { + artifactServiceMock := &mocks.Service{} + tt.artifactServiceMock(artifactServiceMock) + clusterController := tt.clusterController() ib, _ := NewEnsemblerServiceImageBuilder( clusterController, tt.imageBuildingConfig, googleCloudStorageArtifactServiceType, + artifactServiceMock, ) status := ib.GetImageBuildingJobStatus("", "", models.ID(1), runID) assert.Equal(t, tt.expected, status) @@ -1139,6 +1307,8 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { } func TestDeleteEnsemblerJobImageBuildingJob(t *testing.T) { + modelDependenciesURL := getHashedModelDependenciesUrl() + imageBuildingConfig := config.ImageBuildingConfig{ BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, @@ -1167,6 +1337,7 @@ func TestDeleteEnsemblerJobImageBuildingJob(t *testing.T) { clusterController func() cluster.Controller imageBuildingConfig config.ImageBuildingConfig hasErr bool + artifactServiceMock func(*mocks.Service) }{ "success | no error": { imageBuildingConfig: imageBuildingConfig, @@ -1184,15 +1355,26 @@ func TestDeleteEnsemblerJobImageBuildingJob(t *testing.T) { return ctlr }, hasErr: false, + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { + artifactServiceMock := &mocks.Service{} + tt.artifactServiceMock(artifactServiceMock) + clusterController := tt.clusterController() ib, _ := NewEnsemblerJobImageBuilder( clusterController, tt.imageBuildingConfig, googleCloudStorageArtifactServiceType, + artifactServiceMock, ) err := ib.DeleteImageBuildingJob("", "", models.ID(1), runID) @@ -1206,10 +1388,13 @@ func TestDeleteEnsemblerJobImageBuildingJob(t *testing.T) { } func TestDeleteEnsemblerServiceImageBuildingJob(t *testing.T) { + modelDependenciesURL := getHashedModelDependenciesUrl() + tests := map[string]struct { clusterController func() cluster.Controller imageBuildingConfig config.ImageBuildingConfig hasErr bool + artifactServiceMock func(*mocks.Service) }{ "success | no error": { imageBuildingConfig: config.ImageBuildingConfig{ @@ -1250,15 +1435,26 @@ func TestDeleteEnsemblerServiceImageBuildingJob(t *testing.T) { return ctlr }, hasErr: false, + artifactServiceMock: func(artifactServiceMock *mocks.Service) { + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + }, }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { + artifactServiceMock := &mocks.Service{} + tt.artifactServiceMock(artifactServiceMock) + clusterController := tt.clusterController() ib, _ := NewEnsemblerJobImageBuilder( clusterController, tt.imageBuildingConfig, googleCloudStorageArtifactServiceType, + artifactServiceMock, ) err := ib.DeleteImageBuildingJob("", "", models.ID(1), runID) @@ -1272,23 +1468,7 @@ func TestDeleteEnsemblerServiceImageBuildingJob(t *testing.T) { } func Test_imageBuilder_getHashedModelDependenciesUrl(t *testing.T) { - testArtifactURISuffix := "://bucket-name/mlflow/3069/e130c40703ee424da97b9ecee7b874b7/artifacts" - testArtifactURI := "gs://bucket-name/mlflow/3069/e130c40703ee424da97b9ecee7b874b7/artifacts" - testArtifactGsutilURL := &artifact.URL{ - Bucket: "bucket-name", - Object: "mlflow/3069/e130c40703ee424da97b9ecee7b874b7/artifacts", - } - testCondaEnvContent := `dependencies: -- python=3.9.* -- pip: - - mlflow` - testCondaEnvUrlSuffix := testArtifactURISuffix + "/model/conda.yaml" - - hash := sha256.New() - hash.Write([]byte(testCondaEnvContent)) - hashEnv := hash.Sum(nil) - - modelDependenciesURL := fmt.Sprintf("gs://%s/turing/model_dependencies/%x", testArtifactGsutilURL.Bucket, hashEnv) + modelDependenciesURL := getHashedModelDependenciesUrl() type args struct { ctx context.Context From 37b4d8eaa3c9e2255e661fe88a7fa552b47d0072 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 20 Mar 2025 12:38:08 +0700 Subject: [PATCH 08/78] chore: delete unused attribute and reorder code --- api/turing/imagebuilder/imagebuilder_test.go | 42 +++++++++----------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/api/turing/imagebuilder/imagebuilder_test.go b/api/turing/imagebuilder/imagebuilder_test.go index 169d1d096..4e9421dc9 100644 --- a/api/turing/imagebuilder/imagebuilder_test.go +++ b/api/turing/imagebuilder/imagebuilder_test.go @@ -52,7 +52,6 @@ const ( modelVersion = models.ID(1) runID = "abc123" dockerRegistry = "ghcr.io" - artifactURI = "gs://bucket/ensembler" pyFuncEnsemblerJobDockerfilePath = "engines/pyfunc-ensembler-job/app.Dockerfile" pyFuncEnsemblerServiceDockerfilePath = "engines/pyfunc-ensembler-service/app.Dockerfile" buildContext = "git://github.com/caraml-dev/turing.git#refs/heads/master" @@ -63,8 +62,6 @@ const ( ) func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { - modelDependenciesURL := getHashedModelDependenciesUrl() - imageBuildingConfig := config.ImageBuildingConfig{ BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, @@ -94,7 +91,6 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { expected string projectName string modelName string - artifactURI string modelID models.ID versionID string inputDependencies []string @@ -112,7 +108,6 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { modelName: modelName, modelID: modelVersion, versionID: runID, - artifactURI: artifactURI, clusterController: func() cluster.Controller { ctlr := &clustermock.Controller{} // First time it's called @@ -167,6 +162,7 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -180,7 +176,6 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { modelName: modelName, modelID: modelVersion, versionID: runID, - artifactURI: artifactURI, buildLabels: map[string]string{ "gojek.io/team": "dsp", }, @@ -236,6 +231,7 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -249,7 +245,6 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { modelName: modelName, modelID: modelVersion, versionID: runID, - artifactURI: artifactURI, clusterController: func() cluster.Controller { ctlr := &clustermock.Controller{} // First time it's called @@ -329,6 +324,7 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -371,8 +367,6 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { } func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { - modelDependenciesURL := getHashedModelDependenciesUrl() - imageBuildingConfig := config.ImageBuildingConfig{ BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, @@ -403,7 +397,6 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { expectedImageBuildingError string projectName string modelName string - artifactURI string modelID models.ID versionID string inputDependencies []string @@ -421,7 +414,6 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { modelName: modelName, modelID: modelVersion, versionID: runID, - artifactURI: artifactURI, clusterController: func() cluster.Controller { ctlr := &clustermock.Controller{} // First time it's called @@ -476,6 +468,7 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -489,7 +482,6 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { modelName: modelName, modelID: modelVersion, versionID: runID, - artifactURI: artifactURI, buildLabels: map[string]string{ "gojek.io/team": "dsp", }, @@ -545,6 +537,7 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -558,7 +551,6 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { modelName: modelName, modelID: modelVersion, versionID: runID, - artifactURI: artifactURI, clusterController: func() cluster.Controller { ctlr := &clustermock.Controller{} // First time it's called @@ -638,6 +630,7 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -651,7 +644,6 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { modelName: modelName, modelID: modelVersion, versionID: runID, - artifactURI: artifactURI, clusterController: func() cluster.Controller { ctlr := &clustermock.Controller{} // First time it's called @@ -676,6 +668,7 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.8.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -807,8 +800,6 @@ func TestParseResources(t *testing.T) { } func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { - modelDependenciesURL := getHashedModelDependenciesUrl() - imageBuildingConfig := config.ImageBuildingConfig{ BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, @@ -859,6 +850,7 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { State: JobStateActive, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -885,6 +877,7 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { State: JobStateSucceeded, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -972,6 +965,7 @@ Pod last termination message: CondaEnvException: Pip failed`, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1000,6 +994,7 @@ CondaEnvException: Pip failed`, State: JobStateUnknown, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1023,6 +1018,7 @@ CondaEnvException: Pip failed`, Message: "hello", }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1050,8 +1046,6 @@ CondaEnvException: Pip failed`, } func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { - modelDependenciesURL := getHashedModelDependenciesUrl() - imageBuildingConfig := config.ImageBuildingConfig{ BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, @@ -1102,6 +1096,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { State: JobStateActive, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1128,6 +1123,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { State: JobStateSucceeded, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1183,6 +1179,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { State: JobStateFailed, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1234,6 +1231,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { State: JobStateUnknown, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1280,6 +1278,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { Message: "hello", }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1307,8 +1306,6 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { } func TestDeleteEnsemblerJobImageBuildingJob(t *testing.T) { - modelDependenciesURL := getHashedModelDependenciesUrl() - imageBuildingConfig := config.ImageBuildingConfig{ BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, @@ -1356,6 +1353,7 @@ func TestDeleteEnsemblerJobImageBuildingJob(t *testing.T) { }, hasErr: false, artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1388,8 +1386,6 @@ func TestDeleteEnsemblerJobImageBuildingJob(t *testing.T) { } func TestDeleteEnsemblerServiceImageBuildingJob(t *testing.T) { - modelDependenciesURL := getHashedModelDependenciesUrl() - tests := map[string]struct { clusterController func() cluster.Controller imageBuildingConfig config.ImageBuildingConfig @@ -1436,6 +1432,7 @@ func TestDeleteEnsemblerServiceImageBuildingJob(t *testing.T) { }, hasErr: false, artifactServiceMock: func(artifactServiceMock *mocks.Service) { + modelDependenciesURL := getHashedModelDependenciesUrl() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1469,7 +1466,6 @@ func TestDeleteEnsemblerServiceImageBuildingJob(t *testing.T) { func Test_imageBuilder_getHashedModelDependenciesUrl(t *testing.T) { modelDependenciesURL := getHashedModelDependenciesUrl() - type args struct { ctx context.Context artifactURI string From 0d64d893b99c41705d170ef8b428deac2727be51 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 20 Mar 2025 12:45:27 +0700 Subject: [PATCH 09/78] chore: fix appcontext_test --- api/turing/api/appcontext_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/api/turing/api/appcontext_test.go b/api/turing/api/appcontext_test.go index d50a260f0..16b2dd148 100644 --- a/api/turing/api/appcontext_test.go +++ b/api/turing/api/appcontext_test.go @@ -1,6 +1,8 @@ package api import ( + "crypto/sha256" + "fmt" "net/http" "testing" "time" @@ -12,14 +14,17 @@ import ( //nolint:all "bou.ke/monkey" merlin "github.com/caraml-dev/merlin/client" + "github.com/caraml-dev/mlp/api/pkg/artifact" mlpcluster "github.com/caraml-dev/mlp/api/pkg/cluster" "github.com/caraml-dev/mlp/api/pkg/instrumentation/sentry" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "gorm.io/gorm" "k8s.io/apimachinery/pkg/api/resource" clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1" + mlpmocks "github.com/caraml-dev/mlp/api/pkg/artifact/mocks" batchensembling "github.com/caraml-dev/turing/api/turing/batch/ensembling" batchrunner "github.com/caraml-dev/turing/api/turing/batch/runner" "github.com/caraml-dev/turing/api/turing/cluster" @@ -242,6 +247,29 @@ func TestNewAppContext(t *testing.T) { }, }, nil) + // create mock artifact service + testArtifactURISuffix := "://bucket-name/mlflow/3069/e130c40703ee424da97b9ecee7b874b7/artifacts" + testArtifactGsutilURL := &artifact.URL{ + Bucket: "bucket-name", + Object: "mlflow/3069/e130c40703ee424da97b9ecee7b874b7/artifacts", + } + testCondaEnvContent := `dependencies: +- python=3.9.* +- pip: + - mlflow` + hash := sha256.New() + hash.Write([]byte(testCondaEnvContent)) + hashEnv := hash.Sum(nil) + modelDependenciesURL := fmt.Sprintf("gs://%s/turing/model_dependencies/%x", testArtifactGsutilURL.Bucket, hashEnv) + + testCondaEnvUrlSuffix := testArtifactURISuffix + "/ensembler/conda.yaml" + artifactServiceMock := &mlpmocks.Service{} + artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("GetURLScheme").Return("gs") + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + // Patch the functions from other packages defer monkey.UnpatchAll() monkey.Patch(service.NewExperimentsService, @@ -310,6 +338,7 @@ func TestNewAppContext(t *testing.T) { nil, *testCfg.BatchEnsemblingConfig.ImageBuildingConfig, testCfg.MlflowConfig.ArtifactServiceType, + artifactServiceMock, ) assert.Nil(t, err) @@ -344,6 +373,7 @@ func TestNewAppContext(t *testing.T) { nil, *testCfg.EnsemblerServiceBuilderConfig.ImageBuildingConfig, testCfg.MlflowConfig.ArtifactServiceType, + artifactServiceMock, ) assert.NoError(t, err) From b30d4ff283a0a614c7873ace21fbcc074291e62d Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 21 Mar 2025 13:38:28 +0700 Subject: [PATCH 10/78] feat: refactor ensembler job app.Dockerfile --- engines/pyfunc-ensembler-job/app.Dockerfile | 17 ++++++++++++++++- engines/pyfunc-ensembler-service/app.Dockerfile | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/engines/pyfunc-ensembler-job/app.Dockerfile b/engines/pyfunc-ensembler-job/app.Dockerfile index 57e5b1109..7c354b44e 100644 --- a/engines/pyfunc-ensembler-job/app.Dockerfile +++ b/engines/pyfunc-ensembler-job/app.Dockerfile @@ -6,6 +6,7 @@ ARG MLFLOW_ARTIFACT_STORAGE_TYPE ARG MODEL_URL ARG GOOGLE_APPLICATION_CREDENTIALS +ARG MODEL_DEPENDENCIES_URL ARG AWS_ACCESS_KEY_ID ARG AWS_SECRET_ACCESS_KEY @@ -23,6 +24,20 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ echo "No credentials are used"; \ fi +# Download and install user model dependencies +ARG MODEL_DEPENDENCIES_URL +RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ + gsutil cp ${MODEL_DEPENDENCIES_URL} conda.yaml; \ + elif [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "s3" ]; then \ + S3_KEY=${MODEL_DEPENDENCIES_URL##*s3://}; \ + aws s3api get-object --bucket ${S3_KEY%%/*} --key ${S3_KEY#*/} conda.yaml; \ + else \ + echo "No credentials are used"; \ + fi + +RUN /bin/bash -c "conda env update --name ${CONDA_ENV_NAME} --file ./conda.yaml" + +# Download model artifact RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ gsutil -m cp -r ${MODEL_URL} .; \ elif [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "s3" ]; then \ @@ -32,4 +47,4 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ fi ARG FOLDER_NAME -RUN /bin/bash -c ". activate ${CONDA_ENVIRONMENT} && conda env update --name ${CONDA_ENVIRONMENT} --file /${HOME}/${FOLDER_NAME}/conda.yaml" +RUN /bin/bash -c ". activate ${CONDA_ENVIRONMENT}" diff --git a/engines/pyfunc-ensembler-service/app.Dockerfile b/engines/pyfunc-ensembler-service/app.Dockerfile index 2d8cb76a9..766f59e46 100644 --- a/engines/pyfunc-ensembler-service/app.Dockerfile +++ b/engines/pyfunc-ensembler-service/app.Dockerfile @@ -37,7 +37,7 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ RUN /bin/bash -c "conda env update --name ${CONDA_ENV_NAME} --file ./conda.yaml" - +# Download model artifact RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ gsutil -m cp -r ${MODEL_URL} .; \ elif [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "s3" ]; then \ From 76d470603995c852b51d7bb958914e0119aa2e7b Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 21 Mar 2025 17:24:31 +0700 Subject: [PATCH 11/78] feat: remove python installation process on base Dockerfile, install python on the app.Dockerfile instead --- engines/pyfunc-ensembler-service/Dockerfile | 2 +- engines/pyfunc-ensembler-service/env-3.10.yaml | 8 -------- engines/pyfunc-ensembler-service/env-3.9.yaml | 8 -------- .../pyfunc-ensembler-service/{env-3.8.yaml => env.yaml} | 1 - 4 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 engines/pyfunc-ensembler-service/env-3.10.yaml delete mode 100644 engines/pyfunc-ensembler-service/env-3.9.yaml rename engines/pyfunc-ensembler-service/{env-3.8.yaml => env.yaml} (91%) diff --git a/engines/pyfunc-ensembler-service/Dockerfile b/engines/pyfunc-ensembler-service/Dockerfile index 2a6b4d646..ff135dac1 100644 --- a/engines/pyfunc-ensembler-service/Dockerfile +++ b/engines/pyfunc-ensembler-service/Dockerfile @@ -18,5 +18,5 @@ RUN wget -q https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip && unzip aw COPY . . COPY ./temp-deps/sdk ./../../sdk -RUN conda env create -f ./env-${PYTHON_VERSION}.yaml -n $CONDA_ENV_NAME && \ +RUN conda env create -f ./env.yaml -n $CONDA_ENV_NAME && \ rm -rf /root/.cache diff --git a/engines/pyfunc-ensembler-service/env-3.10.yaml b/engines/pyfunc-ensembler-service/env-3.10.yaml deleted file mode 100644 index 3d0a32549..000000000 --- a/engines/pyfunc-ensembler-service/env-3.10.yaml +++ /dev/null @@ -1,8 +0,0 @@ -name: pyfunc-ensembler-service -dependencies: - - python=3.10.* - - pip=22.2.2 - - pip: - - -r requirements.txt - - --extra-index-url=https://test.pypi.org/simple - - --trusted-host=test.pypi.org \ No newline at end of file diff --git a/engines/pyfunc-ensembler-service/env-3.9.yaml b/engines/pyfunc-ensembler-service/env-3.9.yaml deleted file mode 100644 index 7c679a515..000000000 --- a/engines/pyfunc-ensembler-service/env-3.9.yaml +++ /dev/null @@ -1,8 +0,0 @@ -name: pyfunc-ensembler-service -dependencies: - - python=3.9.* - - pip=21.2.2 - - pip: - - -r requirements.txt - - --extra-index-url=https://test.pypi.org/simple - - --trusted-host=test.pypi.org \ No newline at end of file diff --git a/engines/pyfunc-ensembler-service/env-3.8.yaml b/engines/pyfunc-ensembler-service/env.yaml similarity index 91% rename from engines/pyfunc-ensembler-service/env-3.8.yaml rename to engines/pyfunc-ensembler-service/env.yaml index 34b6ba9ae..6f079e0d4 100644 --- a/engines/pyfunc-ensembler-service/env-3.8.yaml +++ b/engines/pyfunc-ensembler-service/env.yaml @@ -1,6 +1,5 @@ name: pyfunc-ensembler-service dependencies: - - python=3.8.* - pip=21.2.2 - pip: - -r requirements.txt From fa7c302c8c82cc9900087e87c9a919a81a4b83ed Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Mon, 24 Mar 2025 13:52:14 +0700 Subject: [PATCH 12/78] feat: remove python installation from ensembling job base image --- engines/pyfunc-ensembler-job/Dockerfile | 2 +- engines/pyfunc-ensembler-job/env-3.10.yaml | 8 -------- engines/pyfunc-ensembler-job/env-3.9.yaml | 8 -------- engines/pyfunc-ensembler-job/{env-3.8.yaml => env.yaml} | 1 - 4 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 engines/pyfunc-ensembler-job/env-3.10.yaml delete mode 100644 engines/pyfunc-ensembler-job/env-3.9.yaml rename engines/pyfunc-ensembler-job/{env-3.8.yaml => env.yaml} (91%) diff --git a/engines/pyfunc-ensembler-job/Dockerfile b/engines/pyfunc-ensembler-job/Dockerfile index d920ed997..d162010a7 100644 --- a/engines/pyfunc-ensembler-job/Dockerfile +++ b/engines/pyfunc-ensembler-job/Dockerfile @@ -91,7 +91,7 @@ COPY --chown=$UID:$GID ./temp-deps/sdk $PROJECT_DIR/../../sdk # Setup base conda environment ARG PYTHON_VERSION ENV CONDA_ENVIRONMENT turing-batch-ensembler -RUN conda env create -f $PROJECT_DIR/env-${PYTHON_VERSION}.yaml -n $CONDA_ENVIRONMENT && \ +RUN conda env create -f env.yaml -n $CONDA_ENVIRONMENT && \ rm -rf $HOME/.cache RUN echo "conda activate $CONDA_ENVIRONMENT" >> $HOME/.bashrc diff --git a/engines/pyfunc-ensembler-job/env-3.10.yaml b/engines/pyfunc-ensembler-job/env-3.10.yaml deleted file mode 100644 index 8e86ce424..000000000 --- a/engines/pyfunc-ensembler-job/env-3.10.yaml +++ /dev/null @@ -1,8 +0,0 @@ -name: pyfunc-ensembler-job -dependencies: - - python=3.10.* - - pip=22.2.2 - - pip: - - -r requirements.txt - - --extra-index-url=https://test.pypi.org/simple - - --trusted-host=test.pypi.org diff --git a/engines/pyfunc-ensembler-job/env-3.9.yaml b/engines/pyfunc-ensembler-job/env-3.9.yaml deleted file mode 100644 index d1fea63f4..000000000 --- a/engines/pyfunc-ensembler-job/env-3.9.yaml +++ /dev/null @@ -1,8 +0,0 @@ -name: pyfunc-ensembler-job -dependencies: - - python=3.9.* - - pip=22.2.2 - - pip: - - -r requirements.txt - - --extra-index-url=https://test.pypi.org/simple - - --trusted-host=test.pypi.org diff --git a/engines/pyfunc-ensembler-job/env-3.8.yaml b/engines/pyfunc-ensembler-job/env.yaml similarity index 91% rename from engines/pyfunc-ensembler-job/env-3.8.yaml rename to engines/pyfunc-ensembler-job/env.yaml index d2f1b94b7..5fb79e339 100644 --- a/engines/pyfunc-ensembler-job/env-3.8.yaml +++ b/engines/pyfunc-ensembler-job/env.yaml @@ -1,6 +1,5 @@ name: pyfunc-ensembler-job dependencies: - - python=3.8.* - pip=22.2.2 - pip: - -r requirements.txt From bf46ceaa603615845ca04e3d3e9fb24b4f2f35b3 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Tue, 25 Mar 2025 13:17:04 +0700 Subject: [PATCH 13/78] feat: update Makefile to publish turing pyfunc-ensembler-service to pypi --- engines/pyfunc-ensembler-service/Makefile | 10 ++++++++++ engines/pyfunc-ensembler-service/requirements.txt | 1 - engines/pyfunc-ensembler-service/version.py | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 engines/pyfunc-ensembler-service/version.py diff --git a/engines/pyfunc-ensembler-service/Makefile b/engines/pyfunc-ensembler-service/Makefile index 21f5885d8..67a1d9f19 100644 --- a/engines/pyfunc-ensembler-service/Makefile +++ b/engines/pyfunc-ensembler-service/Makefile @@ -41,3 +41,13 @@ build-image: version version: $(eval VERSION=$(if $(OVERWRITE_VERSION),$(OVERWRITE_VERSION),v$(shell ../../scripts/vertagen/vertagen.sh -p ${APP_NAME}/))) @echo "turing-pyfunc-ensembler-service version:" $(VERSION) + +.PHONY: build +build: version + @rm -rf build dist + @pip install "setuptools>=64,<75" "setuptools_scm>=8" "twine" "wheel" + @tag=$$(python -m setuptools_scm -r ../.. | sed 's/\+.*//'); \ + sed -i -e "s|turing-sdk.*|turing-sdk==$$tag|g" ./requirements.txt; \ + sed -i -e "s|VERSION = \".*\"|VERSION = \"$$tag\"|g" ./version.py + @python setup.py sdist bdist_wheel + @TWINE_USERNAME=$(PYPI_USERNAME) TWINE_PASSWORD=$(PYPI_PASSWORD) twine upload dist/* diff --git a/engines/pyfunc-ensembler-service/requirements.txt b/engines/pyfunc-ensembler-service/requirements.txt index 0cd140da6..811d1a47f 100644 --- a/engines/pyfunc-ensembler-service/requirements.txt +++ b/engines/pyfunc-ensembler-service/requirements.txt @@ -3,4 +3,3 @@ cloudpickle==2.0.0 orjson==3.6.8 setuptools<75 tornado==6.1 -file:../../sdk diff --git a/engines/pyfunc-ensembler-service/version.py b/engines/pyfunc-ensembler-service/version.py new file mode 100644 index 000000000..4203e4bef --- /dev/null +++ b/engines/pyfunc-ensembler-service/version.py @@ -0,0 +1 @@ +VERSION = "1.21.2.dev18" From ae204cfede813f05f195c17eca90ce3032b34038 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Tue, 25 Mar 2025 13:39:58 +0700 Subject: [PATCH 14/78] chore: add version when publishing pyfunc-ensembler-service package --- engines/pyfunc-ensembler-service/setup.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/engines/pyfunc-ensembler-service/setup.py b/engines/pyfunc-ensembler-service/setup.py index 101c117a7..af9714b46 100644 --- a/engines/pyfunc-ensembler-service/setup.py +++ b/engines/pyfunc-ensembler-service/setup.py @@ -1,7 +1,12 @@ import setuptools import pathlib import pkg_resources +import imp +import os +version = imp.load_source( + "pyfuncserver.version", os.path.join("version.py") +).VERSION with pathlib.Path("requirements.txt").open() as requirements_txt: requirements = [ @@ -17,8 +22,10 @@ setuptools.setup( name="pyfunc-ensembler-service", + version=version, packages=setuptools.find_packages(), install_requires=requirements, dev_requirements=dev_requirements, python_requires=">=3.8,<3.11", + setup_requires=["setuptools_scm"], ) From 6bb664a5fd39b4ad487701073bfacabc286fdff3 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Tue, 25 Mar 2025 15:51:23 +0700 Subject: [PATCH 15/78] chore: test app.Dockerfile changes --- engines/pyfunc-ensembler-service/app.Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/engines/pyfunc-ensembler-service/app.Dockerfile b/engines/pyfunc-ensembler-service/app.Dockerfile index 766f59e46..986f03915 100644 --- a/engines/pyfunc-ensembler-service/app.Dockerfile +++ b/engines/pyfunc-ensembler-service/app.Dockerfile @@ -35,6 +35,9 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ echo "No credentials are used"; \ fi +# Update conda.yaml to add turing-sdk +ARG TURING_DEP_CONSTRAINT +RUN process_conda_env.sh conda.yaml "nops-pyfunc-ensembler-service" "${TURING_DEP_CONSTRAINT}" RUN /bin/bash -c "conda env update --name ${CONDA_ENV_NAME} --file ./conda.yaml" # Download model artifact From f52d53b8b093d2ae32b160718026442cf4689bcb Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 26 Mar 2025 13:14:37 +0700 Subject: [PATCH 16/78] chore: try to create new conda env instead of updating the conda env on app.Dockerfile --- engines/pyfunc-ensembler-service/app.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/pyfunc-ensembler-service/app.Dockerfile b/engines/pyfunc-ensembler-service/app.Dockerfile index 986f03915..fc397c29d 100644 --- a/engines/pyfunc-ensembler-service/app.Dockerfile +++ b/engines/pyfunc-ensembler-service/app.Dockerfile @@ -38,7 +38,7 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ # Update conda.yaml to add turing-sdk ARG TURING_DEP_CONSTRAINT RUN process_conda_env.sh conda.yaml "nops-pyfunc-ensembler-service" "${TURING_DEP_CONSTRAINT}" -RUN /bin/bash -c "conda env update --name ${CONDA_ENV_NAME} --file ./conda.yaml" +RUN /bin/bash -c "conda env create --name ${CONDA_ENV_NAME} --file ./conda.yaml" # Download model artifact RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ From e74f95c78b8387d1b2df87f58fd119377f616e2d Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 26 Mar 2025 14:04:59 +0700 Subject: [PATCH 17/78] chore: change turing_dep value for testing --- engines/pyfunc-ensembler-service/app.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/pyfunc-ensembler-service/app.Dockerfile b/engines/pyfunc-ensembler-service/app.Dockerfile index fc397c29d..862035256 100644 --- a/engines/pyfunc-ensembler-service/app.Dockerfile +++ b/engines/pyfunc-ensembler-service/app.Dockerfile @@ -37,7 +37,7 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ # Update conda.yaml to add turing-sdk ARG TURING_DEP_CONSTRAINT -RUN process_conda_env.sh conda.yaml "nops-pyfunc-ensembler-service" "${TURING_DEP_CONSTRAINT}" +RUN process_conda_env.sh conda.yaml "naufal-pyfunc-ensembler-service-2" "${TURING_DEP_CONSTRAINT}" RUN /bin/bash -c "conda env create --name ${CONDA_ENV_NAME} --file ./conda.yaml" # Download model artifact From 55b01af1034ef46376211779c1f415366a39d92e Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 26 Mar 2025 14:55:26 +0700 Subject: [PATCH 18/78] chore: change turing_dep value for testing --- engines/pyfunc-ensembler-service/app.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/pyfunc-ensembler-service/app.Dockerfile b/engines/pyfunc-ensembler-service/app.Dockerfile index 862035256..ea52d85cb 100644 --- a/engines/pyfunc-ensembler-service/app.Dockerfile +++ b/engines/pyfunc-ensembler-service/app.Dockerfile @@ -37,7 +37,7 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ # Update conda.yaml to add turing-sdk ARG TURING_DEP_CONSTRAINT -RUN process_conda_env.sh conda.yaml "naufal-pyfunc-ensembler-service-2" "${TURING_DEP_CONSTRAINT}" +RUN process_conda_env.sh conda.yaml "naufal-pyfunc-ensembler-service-3" "${TURING_DEP_CONSTRAINT}" RUN /bin/bash -c "conda env create --name ${CONDA_ENV_NAME} --file ./conda.yaml" # Download model artifact From 77cc5a1172a4fb577873be925549850ef62e81cd Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 26 Mar 2025 15:28:10 +0700 Subject: [PATCH 19/78] feat: install turing-sdk via process_conda_env.sh --- engines/pyfunc-ensembler-service/Dockerfile | 10 ++++-- engines/pyfunc-ensembler-service/env.yaml | 7 ---- .../process_conda_env.sh | 33 +++++++++++++++++++ .../pyfunc-ensembler-service/requirements.txt | 1 + engines/pyfunc-ensembler-service/version.py | 2 +- 5 files changed, 42 insertions(+), 11 deletions(-) delete mode 100644 engines/pyfunc-ensembler-service/env.yaml create mode 100755 engines/pyfunc-ensembler-service/process_conda_env.sh diff --git a/engines/pyfunc-ensembler-service/Dockerfile b/engines/pyfunc-ensembler-service/Dockerfile index ff135dac1..0d9a0d14f 100644 --- a/engines/pyfunc-ensembler-service/Dockerfile +++ b/engines/pyfunc-ensembler-service/Dockerfile @@ -12,11 +12,15 @@ ENV CONDA_ENV_NAME=$CONDA_ENV_NAME # Install gcloud SDK RUN wget -qO- https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-367.0.0-linux-x86_64.tar.gz | tar xzf - ENV PATH=$PATH:/google-cloud-sdk/bin + # Install aws CLI RUN wget -q https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip && unzip awscli-exe-linux-x86_64.zip && ./aws/install +# Install yq +ENV YQ_VERSION=v4.42.1 +RUN wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq && \ + chmod +x /usr/bin/yq + COPY . . COPY ./temp-deps/sdk ./../../sdk - -RUN conda env create -f ./env.yaml -n $CONDA_ENV_NAME && \ - rm -rf /root/.cache +COPY process_conda_env.sh /bin/process_conda_env.sh diff --git a/engines/pyfunc-ensembler-service/env.yaml b/engines/pyfunc-ensembler-service/env.yaml deleted file mode 100644 index 6f079e0d4..000000000 --- a/engines/pyfunc-ensembler-service/env.yaml +++ /dev/null @@ -1,7 +0,0 @@ -name: pyfunc-ensembler-service -dependencies: - - pip=21.2.2 - - pip: - - -r requirements.txt - - --extra-index-url=https://test.pypi.org/simple - - --trusted-host=test.pypi.org \ No newline at end of file diff --git a/engines/pyfunc-ensembler-service/process_conda_env.sh b/engines/pyfunc-ensembler-service/process_conda_env.sh new file mode 100755 index 000000000..442cb9782 --- /dev/null +++ b/engines/pyfunc-ensembler-service/process_conda_env.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# This script is designed to manipulate a Conda environment file to add turing dependencies. +# +# Usage: +# ./process_conda_env.sh CONDA_ENV_PATH TURING_DEP TURING_DEP_CONSTRAINT +# +# Input: +# CONDA_ENV_PATH: Path to the Conda environment YAML file. +# TURING_DEP: The dependency to be added or removed. +# TURING_DEP_CONSTRAINT: The constraint for the dependency, such as a version specification. + + +CONDA_ENV_PATH="$1" +TURING_DEP="$2" +TURING_DEP_CONSTRAINT="$3" + +echo "Processing conda environment file: ${CONDA_ENV_PATH}" +echo "Current conda environment file content:" +cat "${CONDA_ENV_PATH}" + +# Remove `mlflow` constraint +yq --inplace 'del(.dependencies[].pip[] | select(. == "mlflow*"))' "${CONDA_ENV_PATH}" +yq --inplace ".dependencies[].pip += [\"mlflow\"]" "${CONDA_ENV_PATH}" + +# Remove `turing-sdk` from conda's pip dependencies +yq --inplace 'del(.dependencies[].pip[] | select(. == "turing-sdk*"))' "${CONDA_ENV_PATH}" + +# Add `${TURING_DEP}` with its constaint (`${TURING_DEP_CONSTRAINT}`) to conda's pip dependencies, if not exist +yq --inplace "with(.dependencies[].pip; select(all_c(. != \"*${TURING_DEP}*\")) | . += [\"${TURING_DEP}${TURING_DEP_CONSTRAINT}\"] )" "${CONDA_ENV_PATH}" + +echo "Processed conda environment file content:" +cat "${CONDA_ENV_PATH}" diff --git a/engines/pyfunc-ensembler-service/requirements.txt b/engines/pyfunc-ensembler-service/requirements.txt index 811d1a47f..2f55e1bac 100644 --- a/engines/pyfunc-ensembler-service/requirements.txt +++ b/engines/pyfunc-ensembler-service/requirements.txt @@ -3,3 +3,4 @@ cloudpickle==2.0.0 orjson==3.6.8 setuptools<75 tornado==6.1 +turing-sdk==0.0.0 diff --git a/engines/pyfunc-ensembler-service/version.py b/engines/pyfunc-ensembler-service/version.py index 4203e4bef..7723ca46a 100644 --- a/engines/pyfunc-ensembler-service/version.py +++ b/engines/pyfunc-ensembler-service/version.py @@ -1 +1 @@ -VERSION = "1.21.2.dev18" +VERSION = "0.0.0" From 66b73c238ca66a043b56d9990182b2452c15eae4 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 26 Mar 2025 15:29:00 +0700 Subject: [PATCH 20/78] chore: change TURING_DEP value --- engines/pyfunc-ensembler-service/app.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/pyfunc-ensembler-service/app.Dockerfile b/engines/pyfunc-ensembler-service/app.Dockerfile index ea52d85cb..e05956c60 100644 --- a/engines/pyfunc-ensembler-service/app.Dockerfile +++ b/engines/pyfunc-ensembler-service/app.Dockerfile @@ -37,7 +37,7 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ # Update conda.yaml to add turing-sdk ARG TURING_DEP_CONSTRAINT -RUN process_conda_env.sh conda.yaml "naufal-pyfunc-ensembler-service-3" "${TURING_DEP_CONSTRAINT}" +RUN process_conda_env.sh conda.yaml "pyfunc-ensembler-service" "${TURING_DEP_CONSTRAINT}" RUN /bin/bash -c "conda env create --name ${CONDA_ENV_NAME} --file ./conda.yaml" # Download model artifact From 64dc081572e26c3cd158c14e3310ca1a62ca242c Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 26 Mar 2025 17:55:16 +0700 Subject: [PATCH 21/78] feat: use 1 base image for every python version --- api/turing/config/config.go | 2 ++ api/turing/imagebuilder/imagebuilder.go | 12 +++--------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/api/turing/config/config.go b/api/turing/config/config.go index 06820e769..d1a31d131 100644 --- a/api/turing/config/config.go +++ b/api/turing/config/config.go @@ -181,6 +181,8 @@ type ImageBuildingConfig struct { NodeSelector map[string]string // Value for cluster-autoscaler.kubernetes.io/safe-to-evict annotation SafeToEvict bool + // BaseImageRef is the image name of the base ensembler image built from the engines/pyfunc-ensembler-*/Dockerfile + BaseImage string } // Resource contains the Kubernetes resource request and limits diff --git a/api/turing/imagebuilder/imagebuilder.go b/api/turing/imagebuilder/imagebuilder.go index 32b0baff8..7b44826c9 100644 --- a/api/turing/imagebuilder/imagebuilder.go +++ b/api/turing/imagebuilder/imagebuilder.go @@ -201,7 +201,7 @@ func (ib *imageBuilder) BuildImage(request BuildImageRequest) (string, error) { } job, err = ib.createKanikoJob(kanikoJobName, imageRef, request.ArtifactURI, request.BuildLabels, - request.EnsemblerFolder, request.BaseImageRefTag, hashedModelDependenciesUrl) + request.EnsemblerFolder, hashedModelDependenciesUrl) if err != nil { log.Errorf("unable to build image %s, error: %v", imageRef, err) return "", ErrUnableToBuildImage @@ -222,7 +222,7 @@ func (ib *imageBuilder) BuildImage(request BuildImageRequest) (string, error) { } job, err = ib.createKanikoJob(kanikoJobName, imageRef, request.ArtifactURI, request.BuildLabels, - request.EnsemblerFolder, request.BaseImageRefTag, hashedModelDependenciesUrl) + request.EnsemblerFolder, hashedModelDependenciesUrl) if err != nil { log.Errorf("unable to build image %s, error: %v", imageRef, err) return "", ErrUnableToBuildImage @@ -291,22 +291,16 @@ func (ib *imageBuilder) createKanikoJob( artifactURI string, buildLabels map[string]string, ensemblerFolder string, - baseImageRefTag string, hashedModelDependenciesUrl string, ) (*apibatchv1.Job, error) { splitURI := strings.Split(artifactURI, "/") folderName := fmt.Sprintf("%s/%s", splitURI[len(splitURI)-1], ensemblerFolder) - baseImage, ok := ib.imageBuildingConfig.BaseImageRef[baseImageRefTag] - if !ok { - return nil, fmt.Errorf("No matching base image for tag %s", baseImageRefTag) - } - kanikoArgs := []string{ fmt.Sprintf("--dockerfile=%s", ib.imageBuildingConfig.KanikoConfig.DockerfileFilePath), fmt.Sprintf("--context=%s", ib.imageBuildingConfig.KanikoConfig.BuildContextURI), fmt.Sprintf("--build-arg=MODEL_URL=%s", artifactURI), - fmt.Sprintf("--build-arg=BASE_IMAGE=%s", baseImage), + fmt.Sprintf("--build-arg=BASE_IMAGE=%s", ib.imageBuildingConfig.BaseImage), fmt.Sprintf("--build-arg=MLFLOW_ARTIFACT_STORAGE_TYPE=%s", ib.artifactServiceType), fmt.Sprintf("--build-arg=FOLDER_NAME=%s", folderName), fmt.Sprintf("--build-arg=MODEL_DEPENDENCIES_URL=%s", hashedModelDependenciesUrl), From 5135e3bde4a359284daf20be45d9a0906d206385 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 26 Mar 2025 17:57:22 +0700 Subject: [PATCH 22/78] chore: remove BaseImageRefTag --- api/turing/batch/ensembling/runner.go | 1 - api/turing/imagebuilder/imagebuilder.go | 1 - api/turing/imagebuilder/imagebuilder_test.go | 2 -- api/turing/service/ensembler_image_service.go | 1 - api/turing/service/router_deployment_service.go | 1 - 5 files changed, 6 deletions(-) diff --git a/api/turing/batch/ensembling/runner.go b/api/turing/batch/ensembling/runner.go index 99103d78d..918e20b54 100644 --- a/api/turing/batch/ensembling/runner.go +++ b/api/turing/batch/ensembling/runner.go @@ -443,7 +443,6 @@ func (r *ensemblingJobRunner) buildImage( ArtifactURI: *ensemblingJob.InfraConfig.ArtifactUri, BuildLabels: buildLabels, EnsemblerFolder: service.EnsemblerFolder, - BaseImageRefTag: baseImageTag, } return r.imageBuilder.BuildImage(request) } diff --git a/api/turing/imagebuilder/imagebuilder.go b/api/turing/imagebuilder/imagebuilder.go index 7b44826c9..581815376 100644 --- a/api/turing/imagebuilder/imagebuilder.go +++ b/api/turing/imagebuilder/imagebuilder.go @@ -90,7 +90,6 @@ type BuildImageRequest struct { ArtifactURI string BuildLabels map[string]string EnsemblerFolder string - BaseImageRefTag string } type EnsemblerImage struct { diff --git a/api/turing/imagebuilder/imagebuilder_test.go b/api/turing/imagebuilder/imagebuilder_test.go index 4e9421dc9..dc50bdbe1 100644 --- a/api/turing/imagebuilder/imagebuilder_test.go +++ b/api/turing/imagebuilder/imagebuilder_test.go @@ -357,7 +357,6 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { testArtifactURI, tt.buildLabels, tt.ensemblerFolder, - tt.imageTag, } actual, err := ib.BuildImage(buildImageRequest) assert.Nil(t, err) @@ -701,7 +700,6 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { testArtifactURI, tt.buildLabels, tt.ensemblerFolder, - tt.imageTag, } actual, err := ib.BuildImage(buildImageRequest) if tt.expectedImageBuildingError == "" { diff --git a/api/turing/service/ensembler_image_service.go b/api/turing/service/ensembler_image_service.go index 398087dd7..38dd6cac9 100644 --- a/api/turing/service/ensembler_image_service.go +++ b/api/turing/service/ensembler_image_service.go @@ -98,7 +98,6 @@ func (s *ensemblerImagesService) BuildImage( }, ), EnsemblerFolder: EnsemblerFolder, - BaseImageRefTag: ensembler.PythonVersion, } if _, err := ib.BuildImage(request); err != nil { diff --git a/api/turing/service/router_deployment_service.go b/api/turing/service/router_deployment_service.go index 635e390b2..53d196e71 100644 --- a/api/turing/service/router_deployment_service.go +++ b/api/turing/service/router_deployment_service.go @@ -440,7 +440,6 @@ func (ds *deploymentService) buildEnsemblerServiceImage( }, ), EnsemblerFolder: EnsemblerFolder, - BaseImageRefTag: ensembler.PythonVersion, } eventsCh.Write( models.NewInfoEvent( From b678b8db5bffdeefcc9729fc6714fe950913a5be Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 27 Mar 2025 15:03:32 +0700 Subject: [PATCH 23/78] feat: create rule to build pyfunc-ensembler-job setup.py --- engines/pyfunc-ensembler-job/Makefile | 10 ++++++++++ engines/pyfunc-ensembler-job/env.yaml | 7 ------- engines/pyfunc-ensembler-job/requirements.txt | 2 +- engines/pyfunc-ensembler-job/setup.py | 7 +++++++ engines/pyfunc-ensembler-job/version.py | 1 + 5 files changed, 19 insertions(+), 8 deletions(-) delete mode 100644 engines/pyfunc-ensembler-job/env.yaml create mode 100644 engines/pyfunc-ensembler-job/version.py diff --git a/engines/pyfunc-ensembler-job/Makefile b/engines/pyfunc-ensembler-job/Makefile index 8cb9e445f..2553dd0f4 100644 --- a/engines/pyfunc-ensembler-job/Makefile +++ b/engines/pyfunc-ensembler-job/Makefile @@ -47,3 +47,13 @@ build-image: version version: $(eval VERSION=$(if $(OVERWRITE_VERSION),$(OVERWRITE_VERSION),v$(shell ../../scripts/vertagen/vertagen.sh -p ${APP_NAME}/))) @echo "turing-pyfunc-ensembler-job version:" $(VERSION) + +.PHONY: build +build: version + @rm -rf build dist + @pip install "setuptools>=64,<75" "setuptools_scm>=8" "twine" "wheel" + @tag=$$(python -m setuptools_scm -r ../.. | sed 's/\+.*//'); \ + sed -i -e "s|turing-sdk.*|turing-sdk==$$tag|g" ./requirements.txt; \ + sed -i -e "s|VERSION = \".*\"|VERSION = \"$$tag\"|g" ./version.py + @python setup.py sdist bdist_wheel + @TWINE_USERNAME=$(PYPI_USERNAME) TWINE_PASSWORD=$(PYPI_PASSWORD) twine upload dist/* diff --git a/engines/pyfunc-ensembler-job/env.yaml b/engines/pyfunc-ensembler-job/env.yaml deleted file mode 100644 index 5fb79e339..000000000 --- a/engines/pyfunc-ensembler-job/env.yaml +++ /dev/null @@ -1,7 +0,0 @@ -name: pyfunc-ensembler-job -dependencies: - - pip=22.2.2 - - pip: - - -r requirements.txt - - --extra-index-url=https://test.pypi.org/simple - - --trusted-host=test.pypi.org diff --git a/engines/pyfunc-ensembler-job/requirements.txt b/engines/pyfunc-ensembler-job/requirements.txt index f1ef1f972..9656848d7 100644 --- a/engines/pyfunc-ensembler-job/requirements.txt +++ b/engines/pyfunc-ensembler-job/requirements.txt @@ -9,4 +9,4 @@ pyarrow>=0.14.1,<=9.0.0 pyspark==3.0.1 pyyaml==6.0 setuptools<75 -file:../../sdk +turing-sdk==0.0.0 diff --git a/engines/pyfunc-ensembler-job/setup.py b/engines/pyfunc-ensembler-job/setup.py index 2ee7aca07..d24210dc9 100644 --- a/engines/pyfunc-ensembler-job/setup.py +++ b/engines/pyfunc-ensembler-job/setup.py @@ -1,7 +1,12 @@ import setuptools import pathlib import pkg_resources +import imp +import os +version = imp.load_source( + "pyfuncserver.version", os.path.join("version.py") +).VERSION with pathlib.Path("requirements.txt").open() as requirements_txt: requirements = [ @@ -17,8 +22,10 @@ setuptools.setup( name="pyfunc-ensembler-job", + version=version, packages=setuptools.find_packages(), install_requires=requirements, dev_requirements=dev_requirements, python_requires=">=3.8,<3.11", + setup_requires=["setuptools_scm"], ) diff --git a/engines/pyfunc-ensembler-job/version.py b/engines/pyfunc-ensembler-job/version.py new file mode 100644 index 000000000..7723ca46a --- /dev/null +++ b/engines/pyfunc-ensembler-job/version.py @@ -0,0 +1 @@ +VERSION = "0.0.0" From b695c849050c91807f2d3136a71fb9e8ab23d30a Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 27 Mar 2025 15:50:09 +0700 Subject: [PATCH 24/78] feat: adjust app.Dockerfile and base Dockerfile for pyfunc-ensembler-job --- engines/pyfunc-ensembler-job/Dockerfile | 27 ++++++--------- engines/pyfunc-ensembler-job/app.Dockerfile | 7 ++-- .../pyfunc-ensembler-job/process_conda_env.sh | 33 +++++++++++++++++++ .../pyfunc-ensembler-service/app.Dockerfile | 2 +- 4 files changed, 49 insertions(+), 20 deletions(-) create mode 100755 engines/pyfunc-ensembler-job/process_conda_env.sh diff --git a/engines/pyfunc-ensembler-job/Dockerfile b/engines/pyfunc-ensembler-job/Dockerfile index d162010a7..7bc432680 100644 --- a/engines/pyfunc-ensembler-job/Dockerfile +++ b/engines/pyfunc-ensembler-job/Dockerfile @@ -1,4 +1,4 @@ -FROM apache/spark-py:v3.1.3 +FROM --platform=linux/amd64 apache/spark-py:v3.1.3 # Switch to user root so we can add additional jars and configuration files. USER root @@ -60,10 +60,10 @@ COPY ./entrypoint.sh /opt/entrypoint.sh ARG username=spark ARG spark_uid=185 ARG spark_gid=100 -ENV USER $username -ENV UID $spark_uid -ENV GID $spark_gid -ENV HOME /home/$USER +ENV USER=$username +ENV UID=$spark_uid +ENV GID=$spark_gid +ENV HOME=/home/$USER RUN adduser --disabled-password --uid $UID --gid $GID --home $HOME $USER # Switch to Spark user @@ -71,8 +71,8 @@ USER ${USER} WORKDIR $HOME # Install miniconda -ENV CONDA_DIR ${HOME}/miniconda3 -ENV PATH ${CONDA_DIR}/bin:$PATH +ENV CONDA_DIR=${HOME}/miniconda3 +ENV PATH=${CONDA_DIR}/bin:$PATH ENV MINIFORGE_VERSION=23.3.1-1 RUN wget --quiet https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh -O miniconda.sh && \ @@ -82,20 +82,13 @@ RUN wget --quiet https://github.com/conda-forge/miniforge/releases/download/${MI echo "source $CONDA_DIR/etc/profile.d/conda.sh" >> $HOME/.bashrc # Copy PySpark application -ENV PROJECT_DIR $HOME/batch-ensembler +ENV PROJECT_DIR=$HOME/batch-ensembler RUN mkdir -p $PROJECT_DIR COPY --chown=$UID:$GID . $PROJECT_DIR/ # Hack to ensure that it's compatible with the ../../sdk stated in requirements.txt COPY --chown=$UID:$GID ./temp-deps/sdk $PROJECT_DIR/../../sdk -# Setup base conda environment -ARG PYTHON_VERSION -ENV CONDA_ENVIRONMENT turing-batch-ensembler -RUN conda env create -f env.yaml -n $CONDA_ENVIRONMENT && \ - rm -rf $HOME/.cache - -RUN echo "conda activate $CONDA_ENVIRONMENT" >> $HOME/.bashrc - -ENV PATH $CONDA_DIR/envs/$CONDA_ENVIRONMENT/bin:$PATH +# Copy conda env processor script +COPY process_conda_env.sh /bin/process_conda_env.sh ENTRYPOINT ["/opt/entrypoint.sh"] diff --git a/engines/pyfunc-ensembler-job/app.Dockerfile b/engines/pyfunc-ensembler-job/app.Dockerfile index 7c354b44e..58c35ea28 100644 --- a/engines/pyfunc-ensembler-job/app.Dockerfile +++ b/engines/pyfunc-ensembler-job/app.Dockerfile @@ -24,7 +24,7 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ echo "No credentials are used"; \ fi -# Download and install user model dependencies +# Download model dependencies ARG MODEL_DEPENDENCIES_URL RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ gsutil cp ${MODEL_DEPENDENCIES_URL} conda.yaml; \ @@ -35,7 +35,10 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ echo "No credentials are used"; \ fi -RUN /bin/bash -c "conda env update --name ${CONDA_ENV_NAME} --file ./conda.yaml" +# Update conda.yaml to add turing-sdk +ARG TURING_DEP_CONSTRAINT +RUN process_conda_env.sh conda.yaml "naufal-pyfunc-ensembler-job" "${TURING_DEP_CONSTRAINT}" +RUN /bin/bash -c "conda env create --name ${CONDA_ENV_NAME} --file ./conda.yaml" # Download model artifact RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ diff --git a/engines/pyfunc-ensembler-job/process_conda_env.sh b/engines/pyfunc-ensembler-job/process_conda_env.sh new file mode 100755 index 000000000..442cb9782 --- /dev/null +++ b/engines/pyfunc-ensembler-job/process_conda_env.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# This script is designed to manipulate a Conda environment file to add turing dependencies. +# +# Usage: +# ./process_conda_env.sh CONDA_ENV_PATH TURING_DEP TURING_DEP_CONSTRAINT +# +# Input: +# CONDA_ENV_PATH: Path to the Conda environment YAML file. +# TURING_DEP: The dependency to be added or removed. +# TURING_DEP_CONSTRAINT: The constraint for the dependency, such as a version specification. + + +CONDA_ENV_PATH="$1" +TURING_DEP="$2" +TURING_DEP_CONSTRAINT="$3" + +echo "Processing conda environment file: ${CONDA_ENV_PATH}" +echo "Current conda environment file content:" +cat "${CONDA_ENV_PATH}" + +# Remove `mlflow` constraint +yq --inplace 'del(.dependencies[].pip[] | select(. == "mlflow*"))' "${CONDA_ENV_PATH}" +yq --inplace ".dependencies[].pip += [\"mlflow\"]" "${CONDA_ENV_PATH}" + +# Remove `turing-sdk` from conda's pip dependencies +yq --inplace 'del(.dependencies[].pip[] | select(. == "turing-sdk*"))' "${CONDA_ENV_PATH}" + +# Add `${TURING_DEP}` with its constaint (`${TURING_DEP_CONSTRAINT}`) to conda's pip dependencies, if not exist +yq --inplace "with(.dependencies[].pip; select(all_c(. != \"*${TURING_DEP}*\")) | . += [\"${TURING_DEP}${TURING_DEP_CONSTRAINT}\"] )" "${CONDA_ENV_PATH}" + +echo "Processed conda environment file content:" +cat "${CONDA_ENV_PATH}" diff --git a/engines/pyfunc-ensembler-service/app.Dockerfile b/engines/pyfunc-ensembler-service/app.Dockerfile index e05956c60..008e9451b 100644 --- a/engines/pyfunc-ensembler-service/app.Dockerfile +++ b/engines/pyfunc-ensembler-service/app.Dockerfile @@ -24,7 +24,7 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ echo "No credentials are used"; \ fi -# Download and install user model dependencies +# Download user model dependencies ARG MODEL_DEPENDENCIES_URL RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ gsutil cp ${MODEL_DEPENDENCIES_URL} conda.yaml; \ From ca23625fb5df367e28bbb7c85faece250bcd98ab Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Tue, 8 Apr 2025 10:17:22 +0700 Subject: [PATCH 25/78] chore: add yq installation step on pyfunc-ensembler-job base Dockerfile --- engines/pyfunc-ensembler-job/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/engines/pyfunc-ensembler-job/Dockerfile b/engines/pyfunc-ensembler-job/Dockerfile index 7bc432680..947e7ba78 100644 --- a/engines/pyfunc-ensembler-job/Dockerfile +++ b/engines/pyfunc-ensembler-job/Dockerfile @@ -54,6 +54,11 @@ RUN wget -qO- \ # Install aws CLI RUN wget -q https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip && unzip awscli-exe-linux-x86_64.zip && ./aws/install +# Install yq +ENV YQ_VERSION=v4.42.1 +RUN wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq && \ + chmod +x /usr/bin/yq + COPY ./entrypoint.sh /opt/entrypoint.sh # Configure non-root user From 864a033cf0f4f4a617f016770ec276a9bd5550a9 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Tue, 8 Apr 2025 11:19:51 +0700 Subject: [PATCH 26/78] chore: fix breaking appcontext_test.go --- api/turing/api/appcontext_test.go | 35 +++++-------------------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/api/turing/api/appcontext_test.go b/api/turing/api/appcontext_test.go index 16b2dd148..55b77ea52 100644 --- a/api/turing/api/appcontext_test.go +++ b/api/turing/api/appcontext_test.go @@ -1,8 +1,6 @@ package api import ( - "crypto/sha256" - "fmt" "net/http" "testing" "time" @@ -14,17 +12,14 @@ import ( //nolint:all "bou.ke/monkey" merlin "github.com/caraml-dev/merlin/client" - "github.com/caraml-dev/mlp/api/pkg/artifact" mlpcluster "github.com/caraml-dev/mlp/api/pkg/cluster" "github.com/caraml-dev/mlp/api/pkg/instrumentation/sentry" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "gorm.io/gorm" "k8s.io/apimachinery/pkg/api/resource" clientcmdapiv1 "k8s.io/client-go/tools/clientcmd/api/v1" - mlpmocks "github.com/caraml-dev/mlp/api/pkg/artifact/mocks" batchensembling "github.com/caraml-dev/turing/api/turing/batch/ensembling" batchrunner "github.com/caraml-dev/turing/api/turing/batch/runner" "github.com/caraml-dev/turing/api/turing/cluster" @@ -247,29 +242,6 @@ func TestNewAppContext(t *testing.T) { }, }, nil) - // create mock artifact service - testArtifactURISuffix := "://bucket-name/mlflow/3069/e130c40703ee424da97b9ecee7b874b7/artifacts" - testArtifactGsutilURL := &artifact.URL{ - Bucket: "bucket-name", - Object: "mlflow/3069/e130c40703ee424da97b9ecee7b874b7/artifacts", - } - testCondaEnvContent := `dependencies: -- python=3.9.* -- pip: - - mlflow` - hash := sha256.New() - hash.Write([]byte(testCondaEnvContent)) - hashEnv := hash.Sum(nil) - modelDependenciesURL := fmt.Sprintf("gs://%s/turing/model_dependencies/%x", testArtifactGsutilURL.Bucket, hashEnv) - - testCondaEnvUrlSuffix := testArtifactURISuffix + "/ensembler/conda.yaml" - artifactServiceMock := &mlpmocks.Service{} - artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) - artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) - artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) - // Patch the functions from other packages defer monkey.UnpatchAll() monkey.Patch(service.NewExperimentsService, @@ -334,11 +306,14 @@ func TestNewAppContext(t *testing.T) { alertService, err := service.NewGitlabOpsAlertService(nil, *testCfg.AlertConfig) assert.NoError(t, err) + artifactService, err := initArtifactService(testCfg) + assert.NoError(t, err) + ensemblingImageBuilder, err := imagebuilder.NewEnsemblerJobImageBuilder( nil, *testCfg.BatchEnsemblingConfig.ImageBuildingConfig, testCfg.MlflowConfig.ArtifactServiceType, - artifactServiceMock, + artifactService, ) assert.Nil(t, err) @@ -373,7 +348,7 @@ func TestNewAppContext(t *testing.T) { nil, *testCfg.EnsemblerServiceBuilderConfig.ImageBuildingConfig, testCfg.MlflowConfig.ArtifactServiceType, - artifactServiceMock, + artifactService, ) assert.NoError(t, err) From 65f9d3ea4a4f7e24474ec10fbd7a9109b98cd6fa Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Tue, 8 Apr 2025 13:48:32 +0700 Subject: [PATCH 27/78] chore: refactor line with char > 120 --- api/turing/imagebuilder/imagebuilder.go | 5 +- api/turing/imagebuilder/imagebuilder_test.go | 66 +++++++++++++------- 2 files changed, 47 insertions(+), 24 deletions(-) diff --git a/api/turing/imagebuilder/imagebuilder.go b/api/turing/imagebuilder/imagebuilder.go index 581815376..42ed98185 100644 --- a/api/turing/imagebuilder/imagebuilder.go +++ b/api/turing/imagebuilder/imagebuilder.go @@ -677,7 +677,8 @@ func (ib *imageBuilder) getHashedModelDependenciesUrl(ctx context.Context, artif return "", err } - condaEnvUrl := fmt.Sprintf("%s://%s/%s/ensembler/conda.yaml", ib.artifactService.GetURLScheme(), artifactURL.Bucket, artifactURL.Object) + urlSchema := ib.artifactService.GetURLScheme() + condaEnvUrl := fmt.Sprintf("%s://%s/%s/ensembler/conda.yaml", urlSchema, artifactURL.Bucket, artifactURL.Object) condaEnv, err := ib.artifactService.ReadArtifact(ctx, condaEnvUrl) if err != nil { @@ -688,7 +689,7 @@ func (ib *imageBuilder) getHashedModelDependenciesUrl(ctx context.Context, artif hash.Write(condaEnv) hashEnv := hash.Sum(nil) - hashedDependenciesUrl := fmt.Sprintf("%s://%s%s/%x", ib.artifactService.GetURLScheme(), artifactURL.Bucket, modelDependenciesPath, hashEnv) + hashedDependenciesUrl := fmt.Sprintf("%s://%s%s/%x", urlSchema, artifactURL.Bucket, modelDependenciesPath, hashEnv) _, err = ib.artifactService.ReadArtifact(ctx, hashedDependenciesUrl) if err == nil { diff --git a/api/turing/imagebuilder/imagebuilder_test.go b/api/turing/imagebuilder/imagebuilder_test.go index dc50bdbe1..8072da8d9 100644 --- a/api/turing/imagebuilder/imagebuilder_test.go +++ b/api/turing/imagebuilder/imagebuilder_test.go @@ -166,7 +166,8 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -235,7 +236,8 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -328,7 +330,8 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -471,7 +474,8 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -540,7 +544,8 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -633,7 +638,8 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -671,7 +677,8 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -852,7 +859,8 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -879,7 +887,8 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -967,7 +976,8 @@ CondaEnvException: Pip failed`, artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -996,7 +1006,8 @@ CondaEnvException: Pip failed`, artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -1020,7 +1031,8 @@ CondaEnvException: Pip failed`, artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -1098,7 +1110,8 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -1125,7 +1138,8 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -1181,7 +1195,8 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -1233,7 +1248,8 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -1280,7 +1296,8 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -1355,7 +1372,8 @@ func TestDeleteEnsemblerJobImageBuildingJob(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -1434,7 +1452,8 @@ func TestDeleteEnsemblerServiceImageBuildingJob(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, @@ -1485,7 +1504,8 @@ func Test_imageBuilder_getHashedModelDependenciesUrl(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, want: modelDependenciesURL, @@ -1501,9 +1521,11 @@ func Test_imageBuilder_getHashedModelDependenciesUrl(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return(nil, artifact.ErrObjectNotExist) - artifactServiceMock.On("WriteArtifact", mock.Anything, modelDependenciesURL, []byte(testCondaEnvContent)).Return(nil) + artifactServiceMock.On("WriteArtifact", mock.Anything, modelDependenciesURL, []byte(testCondaEnvContent)). + Return(nil) }, want: modelDependenciesURL, wantErr: false, From ddd1b85e2001cd7dde06abb649de5cd58c83eab1 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Tue, 8 Apr 2025 13:57:56 +0700 Subject: [PATCH 28/78] chore: remove unittest related to image tag --- api/turing/imagebuilder/imagebuilder_test.go | 39 -------------------- 1 file changed, 39 deletions(-) diff --git a/api/turing/imagebuilder/imagebuilder_test.go b/api/turing/imagebuilder/imagebuilder_test.go index 8072da8d9..85beba0c1 100644 --- a/api/turing/imagebuilder/imagebuilder_test.go +++ b/api/turing/imagebuilder/imagebuilder_test.go @@ -643,45 +643,6 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, }, - "failure: image tag not matched": { - expectedImageBuildingError: "error building OCI image", - projectName: projectName, - modelName: modelName, - modelID: modelVersion, - versionID: runID, - clusterController: func() cluster.Controller { - ctlr := &clustermock.Controller{} - // First time it's called - ctlr.On( - "GetJob", - mock.Anything, - mock.Anything, - mock.Anything, - ).Return( - nil, - k8serrors.NewNotFound( - schema.GroupResource{}, - fmt.Sprintf("service-builder-%s-%s-%d-%s", projectName, modelName, modelVersion, runID[:5]), - ), - ).Once() - return ctlr - }, - buildLabels: map[string]string{ - "gojek.io/team": "dsp", - }, - imageBuildingConfig: imageBuildingConfig, - ensemblerFolder: ensemblerFolder, - imageTag: "3.8.*", - artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() - artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) - artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). - Return([]byte(testCondaEnvContent), nil) - artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) - }, - }, } for name, tt := range tests { From 2d620f753a0fab0f9f691150ab716fd17bb2df04 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Tue, 8 Apr 2025 14:24:27 +0700 Subject: [PATCH 29/78] chore: fix golangci-lint related issues --- api/turing/batch/ensembling/runner.go | 22 +---- api/turing/imagebuilder/imagebuilder.go | 18 ++-- api/turing/imagebuilder/imagebuilder_test.go | 90 ++++++++++---------- 3 files changed, 55 insertions(+), 75 deletions(-) diff --git a/api/turing/batch/ensembling/runner.go b/api/turing/batch/ensembling/runner.go index 918e20b54..f151d67f0 100644 --- a/api/turing/batch/ensembling/runner.go +++ b/api/turing/batch/ensembling/runner.go @@ -287,28 +287,9 @@ func (r *ensemblingJobRunner) processOneEnsemblingJob(ensemblingJob *models.Ense return } - // Get ensembler - ensembler, err := r.ensemblersService.FindByID( - ensemblingJob.EnsemblerID, - service.EnsemblersFindByIDOptions{ProjectID: &ensemblingJob.ProjectID}, - ) - if err != nil { - r.saveStatusOrTerminate(ensemblingJob, mlpProject, models.JobPending, err.Error(), true) - return - } - - // Get base image tag - var baseImageTag string - if pyfuncEnsembler, ok := ensembler.(*models.PyFuncEnsembler); ok { - baseImageTag = pyfuncEnsembler.PythonVersion - } else { - r.saveStatusOrTerminate(ensemblingJob, mlpProject, models.JobPending, err.Error(), true) - return - } - // Build Image labels := r.buildLabels(ensemblingJob, mlpProject) - imageRef, imageBuildErr := r.buildImage(ensemblingJob, mlpProject, labels, baseImageTag) + imageRef, imageBuildErr := r.buildImage(ensemblingJob, mlpProject, labels) if imageBuildErr != nil { // Here unfortunately we have to wait till the image building process has @@ -433,7 +414,6 @@ func (r *ensemblingJobRunner) buildImage( ensemblingJob *models.EnsemblingJob, mlpProject *mlp.Project, buildLabels map[string]string, - baseImageTag string, ) (string, error) { request := imagebuilder.BuildImageRequest{ ProjectName: mlpProject.Name, diff --git a/api/turing/imagebuilder/imagebuilder.go b/api/turing/imagebuilder/imagebuilder.go index 42ed98185..378b8ae83 100644 --- a/api/turing/imagebuilder/imagebuilder.go +++ b/api/turing/imagebuilder/imagebuilder.go @@ -174,7 +174,7 @@ func (ib *imageBuilder) BuildImage(request BuildImageRequest) (string, error) { return imageRef, nil } - hashedModelDependenciesUrl, err := ib.getHashedModelDependenciesUrl(context.Background(), request.ArtifactURI) + hashedModelDependenciesURL, err := ib.gethashedModelDependenciesURL(context.Background(), request.ArtifactURI) if err != nil { log.Errorf("unable to get model dependencies url: %v", err) return "", err @@ -200,7 +200,7 @@ func (ib *imageBuilder) BuildImage(request BuildImageRequest) (string, error) { } job, err = ib.createKanikoJob(kanikoJobName, imageRef, request.ArtifactURI, request.BuildLabels, - request.EnsemblerFolder, hashedModelDependenciesUrl) + request.EnsemblerFolder, hashedModelDependenciesURL) if err != nil { log.Errorf("unable to build image %s, error: %v", imageRef, err) return "", ErrUnableToBuildImage @@ -221,7 +221,7 @@ func (ib *imageBuilder) BuildImage(request BuildImageRequest) (string, error) { } job, err = ib.createKanikoJob(kanikoJobName, imageRef, request.ArtifactURI, request.BuildLabels, - request.EnsemblerFolder, hashedModelDependenciesUrl) + request.EnsemblerFolder, hashedModelDependenciesURL) if err != nil { log.Errorf("unable to build image %s, error: %v", imageRef, err) return "", ErrUnableToBuildImage @@ -290,7 +290,7 @@ func (ib *imageBuilder) createKanikoJob( artifactURI string, buildLabels map[string]string, ensemblerFolder string, - hashedModelDependenciesUrl string, + hashedModelDependenciesURL string, ) (*apibatchv1.Job, error) { splitURI := strings.Split(artifactURI, "/") folderName := fmt.Sprintf("%s/%s", splitURI[len(splitURI)-1], ensemblerFolder) @@ -302,7 +302,7 @@ func (ib *imageBuilder) createKanikoJob( fmt.Sprintf("--build-arg=BASE_IMAGE=%s", ib.imageBuildingConfig.BaseImage), fmt.Sprintf("--build-arg=MLFLOW_ARTIFACT_STORAGE_TYPE=%s", ib.artifactServiceType), fmt.Sprintf("--build-arg=FOLDER_NAME=%s", folderName), - fmt.Sprintf("--build-arg=MODEL_DEPENDENCIES_URL=%s", hashedModelDependenciesUrl), + fmt.Sprintf("--build-arg=MODEL_DEPENDENCIES_URL=%s", hashedModelDependenciesURL), fmt.Sprintf("--destination=%s", imageRef), } @@ -667,20 +667,20 @@ func parseJobConditions(jobConditions []apibatchv1.JobCondition) (string, error) return jobTable, err } -// getHashedModelDependenciesUrl stores dependency to storage using it's content hashed name as filename. +// gethashedModelDependenciesURL stores dependency to storage using it's content hashed name as filename. // if the artifact is recreated with different id but has the same dependency content the URL for the // stored dependency will still be the same. This ensure we can use Docker layer caching mechanism for // the built dependency even for different artifact id given the dependency content is not changed. -func (ib *imageBuilder) getHashedModelDependenciesUrl(ctx context.Context, artifactURI string) (string, error) { +func (ib *imageBuilder) gethashedModelDependenciesURL(ctx context.Context, artifactURI string) (string, error) { artifactURL, err := ib.artifactService.ParseURL(artifactURI) if err != nil { return "", err } urlSchema := ib.artifactService.GetURLScheme() - condaEnvUrl := fmt.Sprintf("%s://%s/%s/ensembler/conda.yaml", urlSchema, artifactURL.Bucket, artifactURL.Object) + condaEnvURL := fmt.Sprintf("%s://%s/%s/ensembler/conda.yaml", urlSchema, artifactURL.Bucket, artifactURL.Object) - condaEnv, err := ib.artifactService.ReadArtifact(ctx, condaEnvUrl) + condaEnv, err := ib.artifactService.ReadArtifact(ctx, condaEnvURL) if err != nil { return "", err } diff --git a/api/turing/imagebuilder/imagebuilder_test.go b/api/turing/imagebuilder/imagebuilder_test.go index 85beba0c1..7d7259009 100644 --- a/api/turing/imagebuilder/imagebuilder_test.go +++ b/api/turing/imagebuilder/imagebuilder_test.go @@ -35,10 +35,10 @@ var ( - python=3.9.* - pip: - mlflow` - testCondaEnvUrlSuffix = testArtifactURISuffix + "/ensembler/conda.yaml" + testCondaEnvURLSuffix = testArtifactURISuffix + "/ensembler/conda.yaml" ) -func getHashedModelDependenciesUrl() string { +func gethashedModelDependenciesURL() string { hash := sha256.New() hash.Write([]byte(testCondaEnvContent)) hashEnv := hash.Sum(nil) @@ -162,11 +162,11 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -232,11 +232,11 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -326,11 +326,11 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -470,11 +470,11 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -540,11 +540,11 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -634,11 +634,11 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -816,11 +816,11 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { State: JobStateActive, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -844,11 +844,11 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { State: JobStateSucceeded, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -933,11 +933,11 @@ Pod last termination message: CondaEnvException: Pip failed`, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -963,11 +963,11 @@ CondaEnvException: Pip failed`, State: JobStateUnknown, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -988,11 +988,11 @@ CondaEnvException: Pip failed`, Message: "hello", }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -1067,11 +1067,11 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { State: JobStateActive, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -1095,11 +1095,11 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { State: JobStateSucceeded, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -1152,11 +1152,11 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { State: JobStateFailed, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -1205,11 +1205,11 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { State: JobStateUnknown, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -1253,11 +1253,11 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { Message: "hello", }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -1329,11 +1329,11 @@ func TestDeleteEnsemblerJobImageBuildingJob(t *testing.T) { }, hasErr: false, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -1409,11 +1409,11 @@ func TestDeleteEnsemblerServiceImageBuildingJob(t *testing.T) { }, hasErr: false, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := getHashedModelDependenciesUrl() + modelDependenciesURL := gethashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -1442,8 +1442,8 @@ func TestDeleteEnsemblerServiceImageBuildingJob(t *testing.T) { } } -func Test_imageBuilder_getHashedModelDependenciesUrl(t *testing.T) { - modelDependenciesURL := getHashedModelDependenciesUrl() +func Test_imageBuilder_gethashedModelDependenciesURL(t *testing.T) { + modelDependenciesURL := gethashedModelDependenciesURL() type args struct { ctx context.Context artifactURI string @@ -1465,7 +1465,7 @@ func Test_imageBuilder_getHashedModelDependenciesUrl(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) }, @@ -1482,7 +1482,7 @@ func Test_imageBuilder_getHashedModelDependenciesUrl(t *testing.T) { artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") - artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvUrlSuffix)). + artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return(nil, artifact.ErrObjectNotExist) artifactServiceMock.On("WriteArtifact", mock.Anything, modelDependenciesURL, []byte(testCondaEnvContent)). @@ -1501,13 +1501,13 @@ func Test_imageBuilder_getHashedModelDependenciesUrl(t *testing.T) { artifactService: artifactServiceMock, } - got, err := c.getHashedModelDependenciesUrl(tt.args.ctx, tt.args.artifactURI) + got, err := c.gethashedModelDependenciesURL(tt.args.ctx, tt.args.artifactURI) if (err != nil) != tt.wantErr { - t.Errorf("imageBuilder.getHashedModelDependenciesUrl() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("imageBuilder.gethashedModelDependenciesURL() error = %v, wantErr %v", err, tt.wantErr) return } if got != tt.want { - t.Errorf("imageBuilder.getHashedModelDependenciesUrl() = %v, want %v", got, tt.want) + t.Errorf("imageBuilder.gethashedModelDependenciesURL() = %v, want %v", got, tt.want) } }) } From 7e0ba5fcec948ce21067c62564ba323634185395 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Tue, 8 Apr 2025 14:32:57 +0700 Subject: [PATCH 30/78] chore: fix golangci-lint related issues (2) --- api/turing/imagebuilder/imagebuilder.go | 20 ++++---- api/turing/imagebuilder/imagebuilder_test.go | 48 ++++++++++---------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/api/turing/imagebuilder/imagebuilder.go b/api/turing/imagebuilder/imagebuilder.go index 378b8ae83..ecf400711 100644 --- a/api/turing/imagebuilder/imagebuilder.go +++ b/api/turing/imagebuilder/imagebuilder.go @@ -174,7 +174,7 @@ func (ib *imageBuilder) BuildImage(request BuildImageRequest) (string, error) { return imageRef, nil } - hashedModelDependenciesURL, err := ib.gethashedModelDependenciesURL(context.Background(), request.ArtifactURI) + hashedModelDependenciesURL, err := ib.getHashedModelDependenciesURL(context.Background(), request.ArtifactURI) if err != nil { log.Errorf("unable to get model dependencies url: %v", err) return "", err @@ -667,20 +667,20 @@ func parseJobConditions(jobConditions []apibatchv1.JobCondition) (string, error) return jobTable, err } -// gethashedModelDependenciesURL stores dependency to storage using it's content hashed name as filename. +// getHashedModelDependenciesURL stores dependency to storage using it's content hashed name as filename. // if the artifact is recreated with different id but has the same dependency content the URL for the // stored dependency will still be the same. This ensure we can use Docker layer caching mechanism for // the built dependency even for different artifact id given the dependency content is not changed. -func (ib *imageBuilder) gethashedModelDependenciesURL(ctx context.Context, artifactURI string) (string, error) { +func (ib *imageBuilder) getHashedModelDependenciesURL(ctx context.Context, artifactURI string) (string, error) { artifactURL, err := ib.artifactService.ParseURL(artifactURI) if err != nil { return "", err } urlSchema := ib.artifactService.GetURLScheme() - condaEnvURL := fmt.Sprintf("%s://%s/%s/ensembler/conda.yaml", urlSchema, artifactURL.Bucket, artifactURL.Object) + condaEnvUrl := fmt.Sprintf("%s://%s/%s/ensembler/conda.yaml", urlSchema, artifactURL.Bucket, artifactURL.Object) - condaEnv, err := ib.artifactService.ReadArtifact(ctx, condaEnvURL) + condaEnv, err := ib.artifactService.ReadArtifact(ctx, condaEnvUrl) if err != nil { return "", err } @@ -689,20 +689,20 @@ func (ib *imageBuilder) gethashedModelDependenciesURL(ctx context.Context, artif hash.Write(condaEnv) hashEnv := hash.Sum(nil) - hashedDependenciesUrl := fmt.Sprintf("%s://%s%s/%x", urlSchema, artifactURL.Bucket, modelDependenciesPath, hashEnv) + hashedDependenciesURL := fmt.Sprintf("%s://%s%s/%x", urlSchema, artifactURL.Bucket, modelDependenciesPath, hashEnv) - _, err = ib.artifactService.ReadArtifact(ctx, hashedDependenciesUrl) + _, err = ib.artifactService.ReadArtifact(ctx, hashedDependenciesURL) if err == nil { - return hashedDependenciesUrl, nil + return hashedDependenciesURL, nil } if !errors.Is(err, artifact.ErrObjectNotExist) { return "", err } - if err := ib.artifactService.WriteArtifact(ctx, hashedDependenciesUrl, condaEnv); err != nil { + if err := ib.artifactService.WriteArtifact(ctx, hashedDependenciesURL, condaEnv); err != nil { return "", err } - return hashedDependenciesUrl, nil + return hashedDependenciesURL, nil } diff --git a/api/turing/imagebuilder/imagebuilder_test.go b/api/turing/imagebuilder/imagebuilder_test.go index 7d7259009..bc53d9bae 100644 --- a/api/turing/imagebuilder/imagebuilder_test.go +++ b/api/turing/imagebuilder/imagebuilder_test.go @@ -38,7 +38,7 @@ var ( testCondaEnvURLSuffix = testArtifactURISuffix + "/ensembler/conda.yaml" ) -func gethashedModelDependenciesURL() string { +func getHashedModelDependenciesURL() string { hash := sha256.New() hash.Write([]byte(testCondaEnvContent)) hashEnv := hash.Sum(nil) @@ -162,7 +162,7 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -232,7 +232,7 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -326,7 +326,7 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -470,7 +470,7 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -540,7 +540,7 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -634,7 +634,7 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { ensemblerFolder: ensemblerFolder, imageTag: "3.7.*", artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -816,7 +816,7 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { State: JobStateActive, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -844,7 +844,7 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { State: JobStateSucceeded, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -933,7 +933,7 @@ Pod last termination message: CondaEnvException: Pip failed`, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -963,7 +963,7 @@ CondaEnvException: Pip failed`, State: JobStateUnknown, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -988,7 +988,7 @@ CondaEnvException: Pip failed`, Message: "hello", }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1067,7 +1067,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { State: JobStateActive, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1095,7 +1095,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { State: JobStateSucceeded, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1152,7 +1152,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { State: JobStateFailed, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1205,7 +1205,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { State: JobStateUnknown, }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1253,7 +1253,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { Message: "hello", }, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1329,7 +1329,7 @@ func TestDeleteEnsemblerJobImageBuildingJob(t *testing.T) { }, hasErr: false, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1409,7 +1409,7 @@ func TestDeleteEnsemblerServiceImageBuildingJob(t *testing.T) { }, hasErr: false, artifactServiceMock: func(artifactServiceMock *mocks.Service) { - modelDependenciesURL := gethashedModelDependenciesURL() + modelDependenciesURL := getHashedModelDependenciesURL() artifactServiceMock.On("ParseURL", fmt.Sprintf("gs%s", testArtifactURISuffix)).Return(testArtifactGsutilURL, nil) artifactServiceMock.On("GetURLScheme").Return("gs") artifactServiceMock.On("GetURLScheme").Return("gs") @@ -1442,8 +1442,8 @@ func TestDeleteEnsemblerServiceImageBuildingJob(t *testing.T) { } } -func Test_imageBuilder_gethashedModelDependenciesURL(t *testing.T) { - modelDependenciesURL := gethashedModelDependenciesURL() +func Test_imageBuilder_getHashedModelDependenciesURL(t *testing.T) { + modelDependenciesURL := getHashedModelDependenciesURL() type args struct { ctx context.Context artifactURI string @@ -1501,13 +1501,13 @@ func Test_imageBuilder_gethashedModelDependenciesURL(t *testing.T) { artifactService: artifactServiceMock, } - got, err := c.gethashedModelDependenciesURL(tt.args.ctx, tt.args.artifactURI) + got, err := c.getHashedModelDependenciesURL(tt.args.ctx, tt.args.artifactURI) if (err != nil) != tt.wantErr { - t.Errorf("imageBuilder.gethashedModelDependenciesURL() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("imageBuilder.getHashedModelDependenciesURL() error = %v, wantErr %v", err, tt.wantErr) return } if got != tt.want { - t.Errorf("imageBuilder.gethashedModelDependenciesURL() = %v, want %v", got, tt.want) + t.Errorf("imageBuilder.getHashedModelDependenciesURL() = %v, want %v", got, tt.want) } }) } From 7480267699c74a2c931472f8a1379a0170881e4c Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Tue, 8 Apr 2025 14:40:08 +0700 Subject: [PATCH 31/78] chore: fix golangci-lint related issue (3) --- api/turing/imagebuilder/imagebuilder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/turing/imagebuilder/imagebuilder.go b/api/turing/imagebuilder/imagebuilder.go index ecf400711..abc03f24e 100644 --- a/api/turing/imagebuilder/imagebuilder.go +++ b/api/turing/imagebuilder/imagebuilder.go @@ -678,9 +678,9 @@ func (ib *imageBuilder) getHashedModelDependenciesURL(ctx context.Context, artif } urlSchema := ib.artifactService.GetURLScheme() - condaEnvUrl := fmt.Sprintf("%s://%s/%s/ensembler/conda.yaml", urlSchema, artifactURL.Bucket, artifactURL.Object) + condaEnvURL := fmt.Sprintf("%s://%s/%s/ensembler/conda.yaml", urlSchema, artifactURL.Bucket, artifactURL.Object) - condaEnv, err := ib.artifactService.ReadArtifact(ctx, condaEnvUrl) + condaEnv, err := ib.artifactService.ReadArtifact(ctx, condaEnvURL) if err != nil { return "", err } From 56ec8827175190d846f9a500f5a8eb57966a957f Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Tue, 8 Apr 2025 16:22:02 +0700 Subject: [PATCH 32/78] fix: fix wrong arg name in ensembler job app.Dockerfile --- engines/pyfunc-ensembler-job/app.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/pyfunc-ensembler-job/app.Dockerfile b/engines/pyfunc-ensembler-job/app.Dockerfile index 58c35ea28..beb817d28 100644 --- a/engines/pyfunc-ensembler-job/app.Dockerfile +++ b/engines/pyfunc-ensembler-job/app.Dockerfile @@ -50,4 +50,4 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ fi ARG FOLDER_NAME -RUN /bin/bash -c ". activate ${CONDA_ENVIRONMENT}" +RUN /bin/bash -c ". activate ${CONDA_ENV_NAME}" From d1cc242a9a0cf09e107d774220b761e20c74ffa8 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 9 Apr 2025 14:34:26 +0700 Subject: [PATCH 33/78] chore: use pyfunc-ensembler-job package instead of dummy package --- engines/pyfunc-ensembler-job/app.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/pyfunc-ensembler-job/app.Dockerfile b/engines/pyfunc-ensembler-job/app.Dockerfile index beb817d28..52adf1f93 100644 --- a/engines/pyfunc-ensembler-job/app.Dockerfile +++ b/engines/pyfunc-ensembler-job/app.Dockerfile @@ -37,7 +37,7 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ # Update conda.yaml to add turing-sdk ARG TURING_DEP_CONSTRAINT -RUN process_conda_env.sh conda.yaml "naufal-pyfunc-ensembler-job" "${TURING_DEP_CONSTRAINT}" +RUN process_conda_env.sh conda.yaml "pyfunc-ensembler-job" "${TURING_DEP_CONSTRAINT}" RUN /bin/bash -c "conda env create --name ${CONDA_ENV_NAME} --file ./conda.yaml" # Download model artifact From 93dae7e3f6e5142b692cf66ae6f1335391103af8 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 9 Apr 2025 15:58:30 +0700 Subject: [PATCH 34/78] feat: add Publish pyfunc-ensembler-job package step on github CI --- .github/workflows/pyfunc-ensembler-job.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/pyfunc-ensembler-job.yaml b/.github/workflows/pyfunc-ensembler-job.yaml index b04aa9c8d..866883646 100644 --- a/.github/workflows/pyfunc-ensembler-job.yaml +++ b/.github/workflows/pyfunc-ensembler-job.yaml @@ -24,6 +24,12 @@ on: # To make it possible to trigger e2e CI workflow for any arbitrary git ref workflow_dispatch: + workflow_call: + secrets: + pypi_username: + required: true + pypi_password: + required: true jobs: test: @@ -117,3 +123,13 @@ jobs: - name: Publish Pyfunc Ensembler Job Docker Image run: docker push ${{ steps.build.outputs.pyfunc-ensembler-job }} + + - name: Publish pyfunc-ensembler-job package + env: + TWINE_USERNAME: ${{ secrets.pypi_username }} + TWINE_PASSWORD: ${{ secrets.pypi_password }} + working-directory: engines/pyfunc-ensembler-job + run: | + set -o pipefail + make build | tee output.log + echo "pyfunc-ensembler-job=$(sed -n 's%Building docker image: \(.*\)%\1%p' output.log)" >> $GITHUB_OUTPUT From 438f646cae8398312a443c1e889cfe377e3d0c00 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 9 Apr 2025 16:02:34 +0700 Subject: [PATCH 35/78] chore: adjust github ci for testing --- .github/workflows/pyfunc-ensembler-job.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pyfunc-ensembler-job.yaml b/.github/workflows/pyfunc-ensembler-job.yaml index 866883646..8fd4a3a9f 100644 --- a/.github/workflows/pyfunc-ensembler-job.yaml +++ b/.github/workflows/pyfunc-ensembler-job.yaml @@ -87,9 +87,8 @@ jobs: # Dev build can be released either from the 'main' branch or # by running this workflow manually with `workflow_dispatch` event. if: >- - contains('release,pre-release', needs.release-rules.outputs.release-type) - || ( github.event_name != 'pull_request' ) - || ( github.event.pull_request.head.repo.full_name == github.repository ) + contains('release,pre-release', needs.release-rules.outputs.release-type) || + ( github.event.pull_request.head.repo.full_name == github.repository ) environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }} runs-on: ubuntu-latest strategy: @@ -97,7 +96,6 @@ jobs: python-version: ["3.8", "3.9", "3.10"] needs: - release-rules - - test steps: - uses: actions/checkout@v4 with: From f1b96e7ca88235008371163bf1be06327facd787 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 9 Apr 2025 16:15:24 +0700 Subject: [PATCH 36/78] fix: use importlib.util instead of imp (imp was removed in python 3.12) --- engines/pyfunc-ensembler-job/setup.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/engines/pyfunc-ensembler-job/setup.py b/engines/pyfunc-ensembler-job/setup.py index d24210dc9..54895b9b1 100644 --- a/engines/pyfunc-ensembler-job/setup.py +++ b/engines/pyfunc-ensembler-job/setup.py @@ -1,12 +1,18 @@ import setuptools import pathlib import pkg_resources -import imp +import importlib.util import os -version = imp.load_source( +# get version from version.py +spec = importlib.util.spec_from_file_location( "pyfuncserver.version", os.path.join("version.py") -).VERSION +) + +v_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(v_module) + +version = v_module.VERSION with pathlib.Path("requirements.txt").open() as requirements_txt: requirements = [ From 671b7e89d336ce769903f507345c4bdc95c36f72 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 9 Apr 2025 16:23:47 +0700 Subject: [PATCH 37/78] chore: remove TWINE auth from makefile command, use env instead --- engines/pyfunc-ensembler-job/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/pyfunc-ensembler-job/Makefile b/engines/pyfunc-ensembler-job/Makefile index 2553dd0f4..101af9c23 100644 --- a/engines/pyfunc-ensembler-job/Makefile +++ b/engines/pyfunc-ensembler-job/Makefile @@ -56,4 +56,4 @@ build: version sed -i -e "s|turing-sdk.*|turing-sdk==$$tag|g" ./requirements.txt; \ sed -i -e "s|VERSION = \".*\"|VERSION = \"$$tag\"|g" ./version.py @python setup.py sdist bdist_wheel - @TWINE_USERNAME=$(PYPI_USERNAME) TWINE_PASSWORD=$(PYPI_PASSWORD) twine upload dist/* + @twine upload dist/* From 60fdba10d4757c8aa441c9d34edfb0ac99337f57 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 9 Apr 2025 16:51:17 +0700 Subject: [PATCH 38/78] chore: adjust pypi related env --- .github/workflows/pyfunc-ensembler-job.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pyfunc-ensembler-job.yaml b/.github/workflows/pyfunc-ensembler-job.yaml index 8fd4a3a9f..da5da6381 100644 --- a/.github/workflows/pyfunc-ensembler-job.yaml +++ b/.github/workflows/pyfunc-ensembler-job.yaml @@ -26,9 +26,9 @@ on: workflow_dispatch: workflow_call: secrets: - pypi_username: + PYPI_USERNAME: required: true - pypi_password: + PYPI_API_TOKEN: required: true jobs: @@ -124,8 +124,8 @@ jobs: - name: Publish pyfunc-ensembler-job package env: - TWINE_USERNAME: ${{ secrets.pypi_username }} - TWINE_PASSWORD: ${{ secrets.pypi_password }} + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} working-directory: engines/pyfunc-ensembler-job run: | set -o pipefail From 99ba8900a93d72fe158a86d096615ecf3f84e4a7 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 9 Apr 2025 17:14:03 +0700 Subject: [PATCH 39/78] feat: add publish pyfunc-ensembler-service package to github CI --- .github/workflows/pyfunc-ensembler-service.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/pyfunc-ensembler-service.yaml b/.github/workflows/pyfunc-ensembler-service.yaml index 5ad1e4b01..44fb37482 100644 --- a/.github/workflows/pyfunc-ensembler-service.yaml +++ b/.github/workflows/pyfunc-ensembler-service.yaml @@ -111,3 +111,13 @@ jobs: - name: Publish Pyfunc Ensembler Service Docker Image run: docker push ${{ steps.build.outputs.pyfunc-ensembler-service-image }} + + - name: Publish pyfunc-ensembler-service package + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + working-directory: engines/pyfunc-ensembler-service + run: | + set -o pipefail + make build | tee output.log + echo "pyfunc-ensembler-service=$(sed -n 's%Building docker image: \(.*\)%\1%p' output.log)" >> $GITHUB_OUTPUT From e971d3c95704d9b86d4738480adb672b551794c5 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 9 Apr 2025 17:15:36 +0700 Subject: [PATCH 40/78] chore: adjust ci rule for testing --- .github/workflows/pyfunc-ensembler-service.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pyfunc-ensembler-service.yaml b/.github/workflows/pyfunc-ensembler-service.yaml index 44fb37482..9c656e8d3 100644 --- a/.github/workflows/pyfunc-ensembler-service.yaml +++ b/.github/workflows/pyfunc-ensembler-service.yaml @@ -75,9 +75,8 @@ jobs: # Dev build can be released either from the 'main' branch or # by running this workflow manually with `workflow_dispatch` event. if: >- - contains('release,pre-release', needs.release-rules.outputs.release-type) - || ( github.event_name != 'pull_request' ) - || ( github.event.pull_request.head.repo.full_name == github.repository ) + contains('release,pre-release', needs.release-rules.outputs.release-type) || + ( github.event.pull_request.head.repo.full_name == github.repository ) environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }} runs-on: ubuntu-latest strategy: @@ -85,7 +84,6 @@ jobs: python-version: ["3.8", "3.9", "3.10"] needs: - release-rules - - test steps: - uses: actions/checkout@v4 with: From b5cc22dcb5d50d94423a3f912f049aff0c72f50c Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 9 Apr 2025 17:24:11 +0700 Subject: [PATCH 41/78] feat: use importlib.util instead of imp for pyfunc-ensembler-service --- engines/pyfunc-ensembler-service/setup.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/engines/pyfunc-ensembler-service/setup.py b/engines/pyfunc-ensembler-service/setup.py index af9714b46..653e99112 100644 --- a/engines/pyfunc-ensembler-service/setup.py +++ b/engines/pyfunc-ensembler-service/setup.py @@ -1,12 +1,18 @@ import setuptools import pathlib import pkg_resources -import imp +import importlib.util import os -version = imp.load_source( +# get version from version.py +spec = importlib.util.spec_from_file_location( "pyfuncserver.version", os.path.join("version.py") -).VERSION +) + +v_module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(v_module) + +version = v_module.VERSION with pathlib.Path("requirements.txt").open() as requirements_txt: requirements = [ From 2c0c6139d2a98c8a52b817c34808f10081a250f1 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 9 Apr 2025 17:32:23 +0700 Subject: [PATCH 42/78] fix: add secret to pyfunc-ensembler-service CI --- .github/workflows/pyfunc-ensembler-service.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pyfunc-ensembler-service.yaml b/.github/workflows/pyfunc-ensembler-service.yaml index 9c656e8d3..75eb3fdc7 100644 --- a/.github/workflows/pyfunc-ensembler-service.yaml +++ b/.github/workflows/pyfunc-ensembler-service.yaml @@ -24,6 +24,12 @@ on: # To make it possible to trigger e2e CI workflow for any arbitrary git ref workflow_dispatch: + workflow_call: + secrets: + PYPI_USERNAME: + required: true + PYPI_API_TOKEN: + required: true jobs: test: From 6d580ccd931be7380f66f9cac4b622b7f80670a0 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 9 Apr 2025 17:36:52 +0700 Subject: [PATCH 43/78] chore: remove arg in make build --- engines/pyfunc-ensembler-service/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/pyfunc-ensembler-service/Makefile b/engines/pyfunc-ensembler-service/Makefile index 67a1d9f19..affe657d8 100644 --- a/engines/pyfunc-ensembler-service/Makefile +++ b/engines/pyfunc-ensembler-service/Makefile @@ -50,4 +50,4 @@ build: version sed -i -e "s|turing-sdk.*|turing-sdk==$$tag|g" ./requirements.txt; \ sed -i -e "s|VERSION = \".*\"|VERSION = \"$$tag\"|g" ./version.py @python setup.py sdist bdist_wheel - @TWINE_USERNAME=$(PYPI_USERNAME) TWINE_PASSWORD=$(PYPI_PASSWORD) twine upload dist/* + @twine upload dist/* From fbcedf1764a391d9ed7fae41afbf298ff8421285 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 9 Apr 2025 17:46:34 +0700 Subject: [PATCH 44/78] feat: add turing prefix to package name --- engines/pyfunc-ensembler-job/Makefile | 2 +- engines/pyfunc-ensembler-job/app.Dockerfile | 2 +- engines/pyfunc-ensembler-job/setup.py | 2 +- engines/pyfunc-ensembler-service/Makefile | 2 +- engines/pyfunc-ensembler-service/app.Dockerfile | 2 +- engines/pyfunc-ensembler-service/setup.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/engines/pyfunc-ensembler-job/Makefile b/engines/pyfunc-ensembler-job/Makefile index 101af9c23..85453a282 100644 --- a/engines/pyfunc-ensembler-job/Makefile +++ b/engines/pyfunc-ensembler-job/Makefile @@ -56,4 +56,4 @@ build: version sed -i -e "s|turing-sdk.*|turing-sdk==$$tag|g" ./requirements.txt; \ sed -i -e "s|VERSION = \".*\"|VERSION = \"$$tag\"|g" ./version.py @python setup.py sdist bdist_wheel - @twine upload dist/* + @twine upload dist/* --verbose diff --git a/engines/pyfunc-ensembler-job/app.Dockerfile b/engines/pyfunc-ensembler-job/app.Dockerfile index 52adf1f93..75a5248a5 100644 --- a/engines/pyfunc-ensembler-job/app.Dockerfile +++ b/engines/pyfunc-ensembler-job/app.Dockerfile @@ -37,7 +37,7 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ # Update conda.yaml to add turing-sdk ARG TURING_DEP_CONSTRAINT -RUN process_conda_env.sh conda.yaml "pyfunc-ensembler-job" "${TURING_DEP_CONSTRAINT}" +RUN process_conda_env.sh conda.yaml "turing-pyfunc-ensembler-job" "${TURING_DEP_CONSTRAINT}" RUN /bin/bash -c "conda env create --name ${CONDA_ENV_NAME} --file ./conda.yaml" # Download model artifact diff --git a/engines/pyfunc-ensembler-job/setup.py b/engines/pyfunc-ensembler-job/setup.py index 54895b9b1..bc3116502 100644 --- a/engines/pyfunc-ensembler-job/setup.py +++ b/engines/pyfunc-ensembler-job/setup.py @@ -27,7 +27,7 @@ ] setuptools.setup( - name="pyfunc-ensembler-job", + name="turing-pyfunc-ensembler-job", version=version, packages=setuptools.find_packages(), install_requires=requirements, diff --git a/engines/pyfunc-ensembler-service/Makefile b/engines/pyfunc-ensembler-service/Makefile index affe657d8..bfa082887 100644 --- a/engines/pyfunc-ensembler-service/Makefile +++ b/engines/pyfunc-ensembler-service/Makefile @@ -50,4 +50,4 @@ build: version sed -i -e "s|turing-sdk.*|turing-sdk==$$tag|g" ./requirements.txt; \ sed -i -e "s|VERSION = \".*\"|VERSION = \"$$tag\"|g" ./version.py @python setup.py sdist bdist_wheel - @twine upload dist/* + @twine upload dist/* --verbose diff --git a/engines/pyfunc-ensembler-service/app.Dockerfile b/engines/pyfunc-ensembler-service/app.Dockerfile index 008e9451b..61e4867c7 100644 --- a/engines/pyfunc-ensembler-service/app.Dockerfile +++ b/engines/pyfunc-ensembler-service/app.Dockerfile @@ -37,7 +37,7 @@ RUN if [ "${MLFLOW_ARTIFACT_STORAGE_TYPE}" = "gcs" ]; then \ # Update conda.yaml to add turing-sdk ARG TURING_DEP_CONSTRAINT -RUN process_conda_env.sh conda.yaml "pyfunc-ensembler-service" "${TURING_DEP_CONSTRAINT}" +RUN process_conda_env.sh conda.yaml "turing-pyfunc-ensembler-service" "${TURING_DEP_CONSTRAINT}" RUN /bin/bash -c "conda env create --name ${CONDA_ENV_NAME} --file ./conda.yaml" # Download model artifact diff --git a/engines/pyfunc-ensembler-service/setup.py b/engines/pyfunc-ensembler-service/setup.py index 653e99112..fea67313f 100644 --- a/engines/pyfunc-ensembler-service/setup.py +++ b/engines/pyfunc-ensembler-service/setup.py @@ -27,7 +27,7 @@ ] setuptools.setup( - name="pyfunc-ensembler-service", + name="turing-pyfunc-ensembler-service", version=version, packages=setuptools.find_packages(), install_requires=requirements, From 19d42d0d9c1149a59297e3f4541fab39c8a6c7f3 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Wed, 9 Apr 2025 17:52:32 +0700 Subject: [PATCH 45/78] chore: revert changes related to CI testing --- .github/workflows/pyfunc-ensembler-job.yaml | 6 ++++-- .github/workflows/pyfunc-ensembler-service.yaml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pyfunc-ensembler-job.yaml b/.github/workflows/pyfunc-ensembler-job.yaml index da5da6381..820425364 100644 --- a/.github/workflows/pyfunc-ensembler-job.yaml +++ b/.github/workflows/pyfunc-ensembler-job.yaml @@ -87,8 +87,9 @@ jobs: # Dev build can be released either from the 'main' branch or # by running this workflow manually with `workflow_dispatch` event. if: >- - contains('release,pre-release', needs.release-rules.outputs.release-type) || - ( github.event.pull_request.head.repo.full_name == github.repository ) + contains('release,pre-release', needs.release-rules.outputs.release-type) + || ( github.event_name != 'pull_request' ) + || ( github.event.pull_request.head.repo.full_name == github.repository ) environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }} runs-on: ubuntu-latest strategy: @@ -96,6 +97,7 @@ jobs: python-version: ["3.8", "3.9", "3.10"] needs: - release-rules + - test steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/pyfunc-ensembler-service.yaml b/.github/workflows/pyfunc-ensembler-service.yaml index 75eb3fdc7..eb1dba941 100644 --- a/.github/workflows/pyfunc-ensembler-service.yaml +++ b/.github/workflows/pyfunc-ensembler-service.yaml @@ -81,8 +81,9 @@ jobs: # Dev build can be released either from the 'main' branch or # by running this workflow manually with `workflow_dispatch` event. if: >- - contains('release,pre-release', needs.release-rules.outputs.release-type) || - ( github.event.pull_request.head.repo.full_name == github.repository ) + contains('release,pre-release', needs.release-rules.outputs.release-type) + || ( github.event_name != 'pull_request' ) + || ( github.event.pull_request.head.repo.full_name == github.repository ) environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }} runs-on: ubuntu-latest strategy: @@ -90,6 +91,7 @@ jobs: python-version: ["3.8", "3.9", "3.10"] needs: - release-rules + - test steps: - uses: actions/checkout@v4 with: From 4dabb224445206ba831a047ef6067e5df42a4c7d Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 10:29:28 +0700 Subject: [PATCH 46/78] feat: remove python-version matrix from CI --- .github/workflows/pyfunc-ensembler-job.yaml | 4 ---- .github/workflows/pyfunc-ensembler-service.yaml | 4 ---- engines/pyfunc-ensembler-job/Makefile | 2 +- engines/pyfunc-ensembler-service/Dockerfile | 1 - engines/pyfunc-ensembler-service/Makefile | 3 +-- 5 files changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pyfunc-ensembler-job.yaml b/.github/workflows/pyfunc-ensembler-job.yaml index 820425364..2aee27858 100644 --- a/.github/workflows/pyfunc-ensembler-job.yaml +++ b/.github/workflows/pyfunc-ensembler-job.yaml @@ -92,9 +92,6 @@ jobs: || ( github.event.pull_request.head.repo.full_name == github.repository ) environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }} runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10"] needs: - release-rules - test @@ -115,7 +112,6 @@ jobs: working-directory: engines/pyfunc-ensembler-job env: DOCKER_REGISTRY: ghcr.io/${{ github.repository }} - PYTHON_VERSION: ${{ matrix.python-version }} run: | set -o pipefail make build-image | tee output.log diff --git a/.github/workflows/pyfunc-ensembler-service.yaml b/.github/workflows/pyfunc-ensembler-service.yaml index eb1dba941..d067b0cdb 100644 --- a/.github/workflows/pyfunc-ensembler-service.yaml +++ b/.github/workflows/pyfunc-ensembler-service.yaml @@ -86,9 +86,6 @@ jobs: || ( github.event.pull_request.head.repo.full_name == github.repository ) environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }} runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10"] needs: - release-rules - test @@ -109,7 +106,6 @@ jobs: working-directory: engines/pyfunc-ensembler-service env: DOCKER_REGISTRY: ghcr.io/${{ github.repository }} - PYTHON_VERSION: ${{ matrix.python-version }} run: | set -o pipefail make build-image | tee output.log diff --git a/engines/pyfunc-ensembler-job/Makefile b/engines/pyfunc-ensembler-job/Makefile index 85453a282..49ffffe93 100644 --- a/engines/pyfunc-ensembler-job/Makefile +++ b/engines/pyfunc-ensembler-job/Makefile @@ -40,7 +40,7 @@ build-image: version @cp -r ../../sdk temp-deps/ @$(eval IMAGE_TAG = $(if $(DOCKER_REGISTRY),$(DOCKER_REGISTRY)/,)${APP_NAME}:${VERSION}) @echo "Building docker image: ${IMAGE_TAG}" - @docker build . --tag ${IMAGE_TAG} --build-arg PYTHON_VERSION=${PYTHON_VERSION} + @docker build . --tag ${IMAGE_TAG} @rm -rf temp-deps .PHONY: version diff --git a/engines/pyfunc-ensembler-service/Dockerfile b/engines/pyfunc-ensembler-service/Dockerfile index 0d9a0d14f..45ac1c6fd 100644 --- a/engines/pyfunc-ensembler-service/Dockerfile +++ b/engines/pyfunc-ensembler-service/Dockerfile @@ -4,7 +4,6 @@ RUN apt-get update && apt-get install unzip ARG APP_NAME ARG CONDA_ENV_NAME -ARG PYTHON_VERSION ENV APP_NAME=$APP_NAME ENV CONDA_ENV_NAME=$CONDA_ENV_NAME diff --git a/engines/pyfunc-ensembler-service/Makefile b/engines/pyfunc-ensembler-service/Makefile index bfa082887..220f4e9f5 100644 --- a/engines/pyfunc-ensembler-service/Makefile +++ b/engines/pyfunc-ensembler-service/Makefile @@ -33,8 +33,7 @@ build-image: version @echo "Building docker image: ${IMAGE_TAG}" @docker build . --tag ${IMAGE_TAG} \ --build-arg APP_NAME=${APP_NAME} \ - --build-arg CONDA_ENV_NAME=${CONDA_ENV_NAME} \ - --build-arg PYTHON_VERSION=${PYTHON_VERSION} + --build-arg CONDA_ENV_NAME=${CONDA_ENV_NAME} @rm -rf temp-deps .PHONY: version From 690b5897ee68f514c7c76ab0ee4d1e565d0e3d3b Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 10:37:14 +0700 Subject: [PATCH 47/78] chore: separate build and publish rule in Makefile --- engines/pyfunc-ensembler-job/Makefile | 6 +++++- engines/pyfunc-ensembler-service/Makefile | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/engines/pyfunc-ensembler-job/Makefile b/engines/pyfunc-ensembler-job/Makefile index 49ffffe93..e14436b9b 100644 --- a/engines/pyfunc-ensembler-job/Makefile +++ b/engines/pyfunc-ensembler-job/Makefile @@ -49,7 +49,7 @@ version: @echo "turing-pyfunc-ensembler-job version:" $(VERSION) .PHONY: build -build: version +build: @rm -rf build dist @pip install "setuptools>=64,<75" "setuptools_scm>=8" "twine" "wheel" @tag=$$(python -m setuptools_scm -r ../.. | sed 's/\+.*//'); \ @@ -57,3 +57,7 @@ build: version sed -i -e "s|VERSION = \".*\"|VERSION = \"$$tag\"|g" ./version.py @python setup.py sdist bdist_wheel @twine upload dist/* --verbose + +.PHONY: build-and-publish +build-and-publish: build + @twine upload dist/* --verbose diff --git a/engines/pyfunc-ensembler-service/Makefile b/engines/pyfunc-ensembler-service/Makefile index 220f4e9f5..e7a395208 100644 --- a/engines/pyfunc-ensembler-service/Makefile +++ b/engines/pyfunc-ensembler-service/Makefile @@ -42,11 +42,14 @@ version: @echo "turing-pyfunc-ensembler-service version:" $(VERSION) .PHONY: build -build: version +build: @rm -rf build dist @pip install "setuptools>=64,<75" "setuptools_scm>=8" "twine" "wheel" @tag=$$(python -m setuptools_scm -r ../.. | sed 's/\+.*//'); \ sed -i -e "s|turing-sdk.*|turing-sdk==$$tag|g" ./requirements.txt; \ sed -i -e "s|VERSION = \".*\"|VERSION = \"$$tag\"|g" ./version.py @python setup.py sdist bdist_wheel + +.PHONY: build-and-publish +build-and-publish: build @twine upload dist/* --verbose From 74fcb5a3df66f26072a94ca2382356ea6787e7e9 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 16:09:44 +0700 Subject: [PATCH 48/78] feat: use python/ tag to determine pyfunc-ensembler-service and pyfunc-ensembler-job version --- engines/pyfunc-ensembler-job/Makefile | 12 +++++------- engines/pyfunc-ensembler-job/Pipfile | 22 ++++++++++++++++++++++ engines/pyfunc-ensembler-job/setup.py | 4 +++- engines/pyfunc-ensembler-service/Makefile | 11 +++++------ engines/pyfunc-ensembler-service/Pipfile | 16 ++++++++++++++++ engines/pyfunc-ensembler-service/setup.py | 4 +++- 6 files changed, 54 insertions(+), 15 deletions(-) create mode 100644 engines/pyfunc-ensembler-job/Pipfile create mode 100644 engines/pyfunc-ensembler-service/Pipfile diff --git a/engines/pyfunc-ensembler-job/Makefile b/engines/pyfunc-ensembler-job/Makefile index e14436b9b..b5ed075f7 100644 --- a/engines/pyfunc-ensembler-job/Makefile +++ b/engines/pyfunc-ensembler-job/Makefile @@ -1,7 +1,7 @@ SHELL := /bin/bash PYTHON_VERSION ?= 3.8 -APP_NAME := pyfunc-ensembler-job-py$(PYTHON_VERSION) +APP_NAME := turing-pyfunc-ensembler-job CONDA_ENV_NAME ?= $(APP_NAME) ACTIVATE_ENV = source $$(conda info --base)/etc/profile.d/conda.sh ; conda activate $(CONDA_ENV_NAME) @@ -45,18 +45,16 @@ build-image: version .PHONY: version version: - $(eval VERSION=$(if $(OVERWRITE_VERSION),$(OVERWRITE_VERSION),v$(shell ../../scripts/vertagen/vertagen.sh -p ${APP_NAME}/))) + $(eval VERSION=$(if $(OVERWRITE_VERSION),$(OVERWRITE_VERSION),v$(shell ../../scripts/vertagen/vertagen.sh -p python/ -y))) @echo "turing-pyfunc-ensembler-job version:" $(VERSION) .PHONY: build -build: +build: version @rm -rf build dist @pip install "setuptools>=64,<75" "setuptools_scm>=8" "twine" "wheel" - @tag=$$(python -m setuptools_scm -r ../.. | sed 's/\+.*//'); \ - sed -i -e "s|turing-sdk.*|turing-sdk==$$tag|g" ./requirements.txt; \ - sed -i -e "s|VERSION = \".*\"|VERSION = \"$$tag\"|g" ./version.py + @sed -i -e "s|turing-sdk.*|turing-sdk==$(VERSION)|g" ./requirements.txt + @sed -i -e "s|VERSION = \".*\"|VERSION = \"$(VERSION)\"|g" ./version.py @python setup.py sdist bdist_wheel - @twine upload dist/* --verbose .PHONY: build-and-publish build-and-publish: build diff --git a/engines/pyfunc-ensembler-job/Pipfile b/engines/pyfunc-ensembler-job/Pipfile new file mode 100644 index 000000000..8452c6b89 --- /dev/null +++ b/engines/pyfunc-ensembler-job/Pipfile @@ -0,0 +1,22 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +cloudpickle = "==2.0.0" +google-cloud-storage = ">=1.37.0" +jinjasql = "==0.1.8" +jinja2 = "==3.0.3" +numpy = "==1.21.6" +pandas = "==1.3.5" +py4j = "==0.10.9" +pyarrow = "<=9.0.0,>=0.14.1" +pyspark = "==3.0.1" +pyyaml = "==6.0" +turing-sdk = "==1.22.1.dev47" + +[dev-packages] + +[requires] +python_version = "3.10" diff --git a/engines/pyfunc-ensembler-job/setup.py b/engines/pyfunc-ensembler-job/setup.py index bc3116502..1068384c7 100644 --- a/engines/pyfunc-ensembler-job/setup.py +++ b/engines/pyfunc-ensembler-job/setup.py @@ -31,7 +31,9 @@ version=version, packages=setuptools.find_packages(), install_requires=requirements, - dev_requirements=dev_requirements, + extras_require={ + "dev": dev_requirements, + }, python_requires=">=3.8,<3.11", setup_requires=["setuptools_scm"], ) diff --git a/engines/pyfunc-ensembler-service/Makefile b/engines/pyfunc-ensembler-service/Makefile index e7a395208..e853768e2 100644 --- a/engines/pyfunc-ensembler-service/Makefile +++ b/engines/pyfunc-ensembler-service/Makefile @@ -1,7 +1,7 @@ SHELL := /bin/bash PYTHON_VERSION ?= 3.8 -APP_NAME := pyfunc-ensembler-service-py$(PYTHON_VERSION) +APP_NAME := turing-pyfunc-ensembler-service CONDA_ENV_NAME ?= $(APP_NAME) ACTIVATE_ENV = source $$(conda info --base)/etc/profile.d/conda.sh ; conda activate $(CONDA_ENV_NAME) @@ -38,16 +38,15 @@ build-image: version .PHONY: version version: - $(eval VERSION=$(if $(OVERWRITE_VERSION),$(OVERWRITE_VERSION),v$(shell ../../scripts/vertagen/vertagen.sh -p ${APP_NAME}/))) + $(eval VERSION=$(if $(OVERWRITE_VERSION),$(OVERWRITE_VERSION),v$(shell ../../scripts/vertagen/vertagen.sh -p python/ -y))) @echo "turing-pyfunc-ensembler-service version:" $(VERSION) .PHONY: build -build: +build: version @rm -rf build dist @pip install "setuptools>=64,<75" "setuptools_scm>=8" "twine" "wheel" - @tag=$$(python -m setuptools_scm -r ../.. | sed 's/\+.*//'); \ - sed -i -e "s|turing-sdk.*|turing-sdk==$$tag|g" ./requirements.txt; \ - sed -i -e "s|VERSION = \".*\"|VERSION = \"$$tag\"|g" ./version.py + @sed -i -e "s|turing-sdk.*|turing-sdk==$(VERSION)|g" ./requirements.txt + @sed -i -e "s|VERSION = \".*\"|VERSION = \"$(VERSION)\"|g" ./version.py @python setup.py sdist bdist_wheel .PHONY: build-and-publish diff --git a/engines/pyfunc-ensembler-service/Pipfile b/engines/pyfunc-ensembler-service/Pipfile new file mode 100644 index 000000000..0473a9b93 --- /dev/null +++ b/engines/pyfunc-ensembler-service/Pipfile @@ -0,0 +1,16 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +argparse = ">=1.4.0" +cloudpickle = "==2.0.0" +orjson = "==3.6.8" +tornado = "==6.1" +turing-sdk = "==v0.15.1" + +[dev-packages] + +[requires] +python_version = "3.10" diff --git a/engines/pyfunc-ensembler-service/setup.py b/engines/pyfunc-ensembler-service/setup.py index fea67313f..05d604942 100644 --- a/engines/pyfunc-ensembler-service/setup.py +++ b/engines/pyfunc-ensembler-service/setup.py @@ -31,7 +31,9 @@ version=version, packages=setuptools.find_packages(), install_requires=requirements, - dev_requirements=dev_requirements, + extras_require={ + "dev": dev_requirements, + }, python_requires=">=3.8,<3.11", setup_requires=["setuptools_scm"], ) From 4b66da3465be604648fd0121d425e7d8f1e176f9 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 16:14:05 +0700 Subject: [PATCH 49/78] feat: adjust make setup and make test --- engines/pyfunc-ensembler-job/Makefile | 14 +++++--------- engines/pyfunc-ensembler-service/Makefile | 14 +++++--------- 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/engines/pyfunc-ensembler-job/Makefile b/engines/pyfunc-ensembler-job/Makefile index b5ed075f7..a37c6e938 100644 --- a/engines/pyfunc-ensembler-job/Makefile +++ b/engines/pyfunc-ensembler-job/Makefile @@ -6,10 +6,10 @@ CONDA_ENV_NAME ?= $(APP_NAME) ACTIVATE_ENV = source $$(conda info --base)/etc/profile.d/conda.sh ; conda activate $(CONDA_ENV_NAME) .PHONY: setup -setup: $(CONDA_ENV_NAME) -$(CONDA_ENV_NAME): - @conda env update -f env-$(PYTHON_VERSION).yaml -n $(CONDA_ENV_NAME) --prune - $(ACTIVATE_ENV) && pip install -r requirements.dev.txt +setup: + @pip install pipenv + $(MAKE) build + @pipenv run pip install 'dist/turing_pyfunc_ensembler_job-$(VERSION)-py3-none-any.whl[dev]' .PHONY: type-check type-check: @@ -28,11 +28,7 @@ lint: .PHONY: test test: type-check - @$(ACTIVATE_ENV) && \ - python -m pytest \ - --cov=ensembler \ - --ignore=env \ - -W ignore + @pipenv run pytest -W ignore .PHONY: build-image build-image: version diff --git a/engines/pyfunc-ensembler-service/Makefile b/engines/pyfunc-ensembler-service/Makefile index e853768e2..aecdef047 100644 --- a/engines/pyfunc-ensembler-service/Makefile +++ b/engines/pyfunc-ensembler-service/Makefile @@ -6,10 +6,10 @@ CONDA_ENV_NAME ?= $(APP_NAME) ACTIVATE_ENV = source $$(conda info --base)/etc/profile.d/conda.sh ; conda activate $(CONDA_ENV_NAME) .PHONY: setup -setup: $(CONDA_ENV_NAME) -$(CONDA_ENV_NAME): - @conda env update -f env-$(PYTHON_VERSION).yaml -n $(CONDA_ENV_NAME) --prune - $(ACTIVATE_ENV) && pip install -r requirements.dev.txt +setup: + @pip install pipenv + $(MAKE) build + @pipenv run pip install 'dist/turing_pyfunc_ensembler_service-$(VERSION)-py3-none-any.whl[dev]' .PHONY: lint lint: @@ -19,11 +19,7 @@ lint: .PHONY: test test: - @$(ACTIVATE_ENV) && \ - python -m pytest \ - --cov=pyfunc_ensembler_runner \ - --cov-report term-missing \ - -W ignore + @pipenv run pytest -W ignore .PHONY: build-image build-image: version From 8cd9f07371f778b190db6b642a774aa8a6caea13 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 16:16:42 +0700 Subject: [PATCH 50/78] chore: adjust github workflow for testing --- .github/workflows/pyfunc-ensembler-job.yaml | 12 +++--------- .github/workflows/pyfunc-ensembler-service.yaml | 12 +++--------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pyfunc-ensembler-job.yaml b/.github/workflows/pyfunc-ensembler-job.yaml index 2aee27858..926f8f2db 100644 --- a/.github/workflows/pyfunc-ensembler-job.yaml +++ b/.github/workflows/pyfunc-ensembler-job.yaml @@ -34,19 +34,14 @@ on: jobs: test: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v4 - - name: Setup Python ${{ matrix.python-version }} + - name: Setup Python uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} cache-dependency-path: | - engines/pyfunc-ensembler-job/env-${{ matrix.python-version }}.yaml engines/pyfunc-ensembler-job/requirements.txt engines/pyfunc-ensembler-job/requirements.dev.txt @@ -87,9 +82,8 @@ jobs: # Dev build can be released either from the 'main' branch or # by running this workflow manually with `workflow_dispatch` event. if: >- - contains('release,pre-release', needs.release-rules.outputs.release-type) - || ( github.event_name != 'pull_request' ) - || ( github.event.pull_request.head.repo.full_name == github.repository ) + contains('release,pre-release', needs.release-rules.outputs.release-type) || + ( github.event.pull_request.head.repo.full_name == github.repository ) environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }} runs-on: ubuntu-latest needs: diff --git a/.github/workflows/pyfunc-ensembler-service.yaml b/.github/workflows/pyfunc-ensembler-service.yaml index d067b0cdb..cfac1c79c 100644 --- a/.github/workflows/pyfunc-ensembler-service.yaml +++ b/.github/workflows/pyfunc-ensembler-service.yaml @@ -34,19 +34,14 @@ on: jobs: test: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v4 - - name: Setup Python ${{ matrix.python-version }} + - name: Setup Python uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} cache-dependency-path: | - engines/pyfunc-ensembler-service/env-${{ matrix.python-version }}.yaml engines/pyfunc-ensembler-service/requirements.txt engines/pyfunc-ensembler-service/requirements.dev.txt @@ -81,9 +76,8 @@ jobs: # Dev build can be released either from the 'main' branch or # by running this workflow manually with `workflow_dispatch` event. if: >- - contains('release,pre-release', needs.release-rules.outputs.release-type) - || ( github.event_name != 'pull_request' ) - || ( github.event.pull_request.head.repo.full_name == github.repository ) + contains('release,pre-release', needs.release-rules.outputs.release-type) || + ( github.event.pull_request.head.repo.full_name == github.repository ) environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }} runs-on: ubuntu-latest needs: From db6bcad415ccf0c1262aee7aa2ce69dc6e1eeada Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 16:26:37 +0700 Subject: [PATCH 51/78] chore: remove python version requirement --- engines/pyfunc-ensembler-job/Pipfile | 3 --- engines/pyfunc-ensembler-service/Pipfile | 3 --- 2 files changed, 6 deletions(-) diff --git a/engines/pyfunc-ensembler-job/Pipfile b/engines/pyfunc-ensembler-job/Pipfile index 8452c6b89..d23f1f57e 100644 --- a/engines/pyfunc-ensembler-job/Pipfile +++ b/engines/pyfunc-ensembler-job/Pipfile @@ -17,6 +17,3 @@ pyyaml = "==6.0" turing-sdk = "==1.22.1.dev47" [dev-packages] - -[requires] -python_version = "3.10" diff --git a/engines/pyfunc-ensembler-service/Pipfile b/engines/pyfunc-ensembler-service/Pipfile index 0473a9b93..cb50d59a7 100644 --- a/engines/pyfunc-ensembler-service/Pipfile +++ b/engines/pyfunc-ensembler-service/Pipfile @@ -11,6 +11,3 @@ tornado = "==6.1" turing-sdk = "==v0.15.1" [dev-packages] - -[requires] -python_version = "3.10" From 557c0a3e56714e615787630cfda9a3844c6b9f51 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 16:41:40 +0700 Subject: [PATCH 52/78] fix: fix wrong make setup command --- engines/pyfunc-ensembler-job/Makefile | 5 ++--- engines/pyfunc-ensembler-service/Makefile | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/engines/pyfunc-ensembler-job/Makefile b/engines/pyfunc-ensembler-job/Makefile index a37c6e938..903b22474 100644 --- a/engines/pyfunc-ensembler-job/Makefile +++ b/engines/pyfunc-ensembler-job/Makefile @@ -6,10 +6,9 @@ CONDA_ENV_NAME ?= $(APP_NAME) ACTIVATE_ENV = source $$(conda info --base)/etc/profile.d/conda.sh ; conda activate $(CONDA_ENV_NAME) .PHONY: setup -setup: +setup: build @pip install pipenv - $(MAKE) build - @pipenv run pip install 'dist/turing_pyfunc_ensembler_job-$(VERSION)-py3-none-any.whl[dev]' + @pipenv run pip install 'dist/turing_pyfunc_ensembler_job-$(shell echo $(VERSION) | sed 's/^v//')-py3-none-any.whl[dev]' .PHONY: type-check type-check: diff --git a/engines/pyfunc-ensembler-service/Makefile b/engines/pyfunc-ensembler-service/Makefile index aecdef047..b0bcf6b95 100644 --- a/engines/pyfunc-ensembler-service/Makefile +++ b/engines/pyfunc-ensembler-service/Makefile @@ -6,10 +6,9 @@ CONDA_ENV_NAME ?= $(APP_NAME) ACTIVATE_ENV = source $$(conda info --base)/etc/profile.d/conda.sh ; conda activate $(CONDA_ENV_NAME) .PHONY: setup -setup: +setup: build @pip install pipenv - $(MAKE) build - @pipenv run pip install 'dist/turing_pyfunc_ensembler_service-$(VERSION)-py3-none-any.whl[dev]' + @pipenv run pip install 'dist/turing_pyfunc_ensembler_service-$(shell echo $(VERSION) | sed 's/^v//')-py3-none-any.whl[dev]' .PHONY: lint lint: From f50fad9aa4af4a150a62026aed4babe0a3204a2c Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 16:57:20 +0700 Subject: [PATCH 53/78] chore: debug git tag --- .github/workflows/pyfunc-ensembler-job.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pyfunc-ensembler-job.yaml b/.github/workflows/pyfunc-ensembler-job.yaml index 926f8f2db..8657ed22a 100644 --- a/.github/workflows/pyfunc-ensembler-job.yaml +++ b/.github/workflows/pyfunc-ensembler-job.yaml @@ -37,6 +37,12 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Show tags + run: git tag -l - name: Setup Python uses: actions/setup-python@v5 From 703bffdd076bb064aa95dbfed52396db5fe78678 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 17:01:50 +0700 Subject: [PATCH 54/78] chore: remove -y from vertagen --- engines/pyfunc-ensembler-job/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/pyfunc-ensembler-job/Makefile b/engines/pyfunc-ensembler-job/Makefile index 903b22474..88c41c675 100644 --- a/engines/pyfunc-ensembler-job/Makefile +++ b/engines/pyfunc-ensembler-job/Makefile @@ -40,7 +40,7 @@ build-image: version .PHONY: version version: - $(eval VERSION=$(if $(OVERWRITE_VERSION),$(OVERWRITE_VERSION),v$(shell ../../scripts/vertagen/vertagen.sh -p python/ -y))) + $(eval VERSION=$(if $(OVERWRITE_VERSION),$(OVERWRITE_VERSION),v$(shell ../../scripts/vertagen/vertagen.sh -p python/))) @echo "turing-pyfunc-ensembler-job version:" $(VERSION) .PHONY: build From 65bfde23c82049acc1b0aabc7b68fbcc4bc7f76d Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 17:12:11 +0700 Subject: [PATCH 55/78] chore: trigger ci from python/ tag push --- .github/workflows/pyfunc-ensembler-job.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pyfunc-ensembler-job.yaml b/.github/workflows/pyfunc-ensembler-job.yaml index 8657ed22a..3488a6780 100644 --- a/.github/workflows/pyfunc-ensembler-job.yaml +++ b/.github/workflows/pyfunc-ensembler-job.yaml @@ -6,6 +6,7 @@ on: push: tags: - "pyfunc-ensembler-job/v[0-9]+.[0-9]+.[0-9]+*" + - "python/v[0-9]+.[0-9]+.[0-9]+*" branches: - main paths: From 6c2d3b19121d0e233c68570a68d0a0091f317005 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 17:15:31 +0700 Subject: [PATCH 56/78] chore: delete Pipfile --- engines/pyfunc-ensembler-job/Pipfile | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 engines/pyfunc-ensembler-job/Pipfile diff --git a/engines/pyfunc-ensembler-job/Pipfile b/engines/pyfunc-ensembler-job/Pipfile deleted file mode 100644 index d23f1f57e..000000000 --- a/engines/pyfunc-ensembler-job/Pipfile +++ /dev/null @@ -1,19 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[packages] -cloudpickle = "==2.0.0" -google-cloud-storage = ">=1.37.0" -jinjasql = "==0.1.8" -jinja2 = "==3.0.3" -numpy = "==1.21.6" -pandas = "==1.3.5" -py4j = "==0.10.9" -pyarrow = "<=9.0.0,>=0.14.1" -pyspark = "==3.0.1" -pyyaml = "==6.0" -turing-sdk = "==1.22.1.dev47" - -[dev-packages] From 92702c23b781b33f0c8ec1720cc018b1cda937b8 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 17:18:46 +0700 Subject: [PATCH 57/78] chore: bump up numpy version --- engines/pyfunc-ensembler-job/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/pyfunc-ensembler-job/requirements.txt b/engines/pyfunc-ensembler-job/requirements.txt index 9656848d7..203c2e4c1 100644 --- a/engines/pyfunc-ensembler-job/requirements.txt +++ b/engines/pyfunc-ensembler-job/requirements.txt @@ -2,7 +2,7 @@ cloudpickle==2.0.0 google-cloud-storage>=1.37.0 jinjasql==0.1.8 jinja2==3.0.3 -numpy==1.21.6 +numpy==1.22.0 pandas==1.3.5 py4j==0.10.9 pyarrow>=0.14.1,<=9.0.0 From da5f434de6f8f001d97366b891584b1ec24fb37e Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 17:27:52 +0700 Subject: [PATCH 58/78] chore: try to bump python version --- .github/workflows/pyfunc-ensembler-job.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/pyfunc-ensembler-job.yaml b/.github/workflows/pyfunc-ensembler-job.yaml index 3488a6780..26630ddac 100644 --- a/.github/workflows/pyfunc-ensembler-job.yaml +++ b/.github/workflows/pyfunc-ensembler-job.yaml @@ -38,16 +38,11 @@ jobs: steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 - fetch-tags: true - - - name: Show tags - run: git tag -l - name: Setup Python uses: actions/setup-python@v5 with: + python-version: 3.10 cache-dependency-path: | engines/pyfunc-ensembler-job/requirements.txt engines/pyfunc-ensembler-job/requirements.dev.txt From a9c1b97f169798cf5827f366443ccd7867def758 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 17:30:11 +0700 Subject: [PATCH 59/78] chore: fix invalid python version --- .github/workflows/pyfunc-ensembler-job.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pyfunc-ensembler-job.yaml b/.github/workflows/pyfunc-ensembler-job.yaml index 26630ddac..846641bb9 100644 --- a/.github/workflows/pyfunc-ensembler-job.yaml +++ b/.github/workflows/pyfunc-ensembler-job.yaml @@ -42,7 +42,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: "3.10" cache-dependency-path: | engines/pyfunc-ensembler-job/requirements.txt engines/pyfunc-ensembler-job/requirements.dev.txt From 50dc701f1e5f3371108daf1f06867ae048f116a1 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 17:33:40 +0700 Subject: [PATCH 60/78] feat: use pipenv for make type-check --- engines/pyfunc-ensembler-job/Makefile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/engines/pyfunc-ensembler-job/Makefile b/engines/pyfunc-ensembler-job/Makefile index 88c41c675..79809bbf2 100644 --- a/engines/pyfunc-ensembler-job/Makefile +++ b/engines/pyfunc-ensembler-job/Makefile @@ -12,12 +12,7 @@ setup: build .PHONY: type-check type-check: - @$(ACTIVATE_ENV) && mypy \ - --install-types \ - --non-interactive \ - --ignore-missing-imports \ - --allow-untyped-globals \ - ensembler + @pipenv run mypy --ignore-missing-imports --allow-untyped-globals --implicit-optional merlin --follow-imports silent .PHONY: lint lint: From daeb4b5a013f81345dfb42c36e5dd033c73a046a Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 17:36:44 +0700 Subject: [PATCH 61/78] chore: fix wrong folder name in make test-type --- engines/pyfunc-ensembler-job/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/pyfunc-ensembler-job/Makefile b/engines/pyfunc-ensembler-job/Makefile index 79809bbf2..9bd228c54 100644 --- a/engines/pyfunc-ensembler-job/Makefile +++ b/engines/pyfunc-ensembler-job/Makefile @@ -12,7 +12,7 @@ setup: build .PHONY: type-check type-check: - @pipenv run mypy --ignore-missing-imports --allow-untyped-globals --implicit-optional merlin --follow-imports silent + @pipenv run mypy --ignore-missing-imports --allow-untyped-globals --implicit-optional ensembler --follow-imports silent .PHONY: lint lint: From f62cceabf4a10935992d95901af258ead235d12a Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Thu, 10 Apr 2025 17:47:02 +0700 Subject: [PATCH 62/78] chore: add types-PyYAML to dev req --- engines/pyfunc-ensembler-job/requirements.dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/engines/pyfunc-ensembler-job/requirements.dev.txt b/engines/pyfunc-ensembler-job/requirements.dev.txt index 101a896ef..dfc2cde42 100644 --- a/engines/pyfunc-ensembler-job/requirements.dev.txt +++ b/engines/pyfunc-ensembler-job/requirements.dev.txt @@ -5,3 +5,4 @@ mypy>=0.910 pytest<=8.1.2 pytest-cov pylint +types-PyYAML From c590d4bcef7497b8c16d2f89bef532d6d9e29fc0 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 11 Apr 2025 13:43:56 +0700 Subject: [PATCH 63/78] feat: adjust github CI for pyfunc-ensembler-service --- .github/workflows/pyfunc-ensembler-service.yaml | 2 ++ engines/pyfunc-ensembler-service/Pipfile | 13 ------------- .../pyfunc-ensembler-service/requirements.dev.txt | 1 + 3 files changed, 3 insertions(+), 13 deletions(-) delete mode 100644 engines/pyfunc-ensembler-service/Pipfile diff --git a/.github/workflows/pyfunc-ensembler-service.yaml b/.github/workflows/pyfunc-ensembler-service.yaml index cfac1c79c..6559ff144 100644 --- a/.github/workflows/pyfunc-ensembler-service.yaml +++ b/.github/workflows/pyfunc-ensembler-service.yaml @@ -6,6 +6,7 @@ on: push: tags: - "pyfunc-ensembler-service/v[0-9]+.[0-9]+.[0-9]+*" + - "python/v[0-9]+.[0-9]+.[0-9]+*" branches: - main paths: @@ -41,6 +42,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: + python-version: "3.10" cache-dependency-path: | engines/pyfunc-ensembler-service/requirements.txt engines/pyfunc-ensembler-service/requirements.dev.txt diff --git a/engines/pyfunc-ensembler-service/Pipfile b/engines/pyfunc-ensembler-service/Pipfile deleted file mode 100644 index cb50d59a7..000000000 --- a/engines/pyfunc-ensembler-service/Pipfile +++ /dev/null @@ -1,13 +0,0 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - -[packages] -argparse = ">=1.4.0" -cloudpickle = "==2.0.0" -orjson = "==3.6.8" -tornado = "==6.1" -turing-sdk = "==v0.15.1" - -[dev-packages] diff --git a/engines/pyfunc-ensembler-service/requirements.dev.txt b/engines/pyfunc-ensembler-service/requirements.dev.txt index 54192dc07..1c2d013bb 100644 --- a/engines/pyfunc-ensembler-service/requirements.dev.txt +++ b/engines/pyfunc-ensembler-service/requirements.dev.txt @@ -3,3 +3,4 @@ black==22.6.0 pytest<=8.1.2 pytest-cov pylint +types-PyYAML From 953ec56f25eb63a1aecde2c1c4090c14b7809a8f Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 11 Apr 2025 13:52:55 +0700 Subject: [PATCH 64/78] feat: only run pyfunc-ensembler-* CI for python/ tag release --- .github/workflows/pyfunc-ensembler-job.yaml | 7 ++++--- .github/workflows/pyfunc-ensembler-service.yaml | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pyfunc-ensembler-job.yaml b/.github/workflows/pyfunc-ensembler-job.yaml index 846641bb9..e09c6e7d4 100644 --- a/.github/workflows/pyfunc-ensembler-job.yaml +++ b/.github/workflows/pyfunc-ensembler-job.yaml @@ -73,7 +73,7 @@ jobs: - id: release-rules uses: ./.github/actions/release-rules with: - prefix: pyfunc-ensembler-job/ + prefix: python/ publish: # Automatically publish release and pre-release artifacts. @@ -84,8 +84,9 @@ jobs: # Dev build can be released either from the 'main' branch or # by running this workflow manually with `workflow_dispatch` event. if: >- - contains('release,pre-release', needs.release-rules.outputs.release-type) || - ( github.event.pull_request.head.repo.full_name == github.repository ) + contains('release,pre-release', needs.release-rules.outputs.release-type) + || ( github.event_name != 'pull_request' ) + || ( github.event.pull_request.head.repo.full_name == github.repository ) environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }} runs-on: ubuntu-latest needs: diff --git a/.github/workflows/pyfunc-ensembler-service.yaml b/.github/workflows/pyfunc-ensembler-service.yaml index 6559ff144..b276f66f9 100644 --- a/.github/workflows/pyfunc-ensembler-service.yaml +++ b/.github/workflows/pyfunc-ensembler-service.yaml @@ -67,7 +67,7 @@ jobs: - id: release-rules uses: ./.github/actions/release-rules with: - prefix: pyfunc-ensembler-service/ + prefix: python/ publish: # Automatically publish release and pre-release artifacts. @@ -78,8 +78,9 @@ jobs: # Dev build can be released either from the 'main' branch or # by running this workflow manually with `workflow_dispatch` event. if: >- - contains('release,pre-release', needs.release-rules.outputs.release-type) || - ( github.event.pull_request.head.repo.full_name == github.repository ) + contains('release,pre-release', needs.release-rules.outputs.release-type) + || ( github.event_name != 'pull_request' ) + || ( github.event.pull_request.head.repo.full_name == github.repository ) environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }} runs-on: ubuntu-latest needs: From 166b0b14b247fa21b2c9f2aa2cb451d89c96b95e Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 11 Apr 2025 13:53:43 +0700 Subject: [PATCH 65/78] feat: change release-rules for sdk, release if new python/ tag is released --- .github/workflows/sdk.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sdk.yaml b/.github/workflows/sdk.yaml index 5d83e9595..e7c5f4d71 100644 --- a/.github/workflows/sdk.yaml +++ b/.github/workflows/sdk.yaml @@ -66,7 +66,7 @@ jobs: - id: release-rules uses: ./.github/actions/release-rules with: - prefix: sdk/ + prefix: python/ publish: # Automatically publish release and pre-release artifacts. From 5ac74dc5edb9e76cafe05bb523e2b00660fa3c00 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 11 Apr 2025 14:05:32 +0700 Subject: [PATCH 66/78] chore: fix make rule called by pyfunc-ensembler-* workflow, add tags trigger for sdk workflow --- .github/workflows/pyfunc-ensembler-job.yaml | 2 +- .github/workflows/pyfunc-ensembler-service.yaml | 2 +- .github/workflows/sdk.yaml | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pyfunc-ensembler-job.yaml b/.github/workflows/pyfunc-ensembler-job.yaml index e09c6e7d4..d52588b8e 100644 --- a/.github/workflows/pyfunc-ensembler-job.yaml +++ b/.github/workflows/pyfunc-ensembler-job.yaml @@ -124,5 +124,5 @@ jobs: working-directory: engines/pyfunc-ensembler-job run: | set -o pipefail - make build | tee output.log + make build-and-publish | tee output.log echo "pyfunc-ensembler-job=$(sed -n 's%Building docker image: \(.*\)%\1%p' output.log)" >> $GITHUB_OUTPUT diff --git a/.github/workflows/pyfunc-ensembler-service.yaml b/.github/workflows/pyfunc-ensembler-service.yaml index b276f66f9..83b8f8f4b 100644 --- a/.github/workflows/pyfunc-ensembler-service.yaml +++ b/.github/workflows/pyfunc-ensembler-service.yaml @@ -118,5 +118,5 @@ jobs: working-directory: engines/pyfunc-ensembler-service run: | set -o pipefail - make build | tee output.log + make build-and-publish | tee output.log echo "pyfunc-ensembler-service=$(sed -n 's%Building docker image: \(.*\)%\1%p' output.log)" >> $GITHUB_OUTPUT diff --git a/.github/workflows/sdk.yaml b/.github/workflows/sdk.yaml index e7c5f4d71..2664028d4 100644 --- a/.github/workflows/sdk.yaml +++ b/.github/workflows/sdk.yaml @@ -6,6 +6,7 @@ on: push: tags: - "sdk/v[0-9]+.[0-9]+.[0-9]+*" + - "python/v[0-9]+.[0-9]+.[0-9]+*" branches: - main paths: From 6a81433dfed90a26e6dc46eb95a756a14417ef3b Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 11 Apr 2025 14:17:45 +0700 Subject: [PATCH 67/78] fix: change regex to parse tag to dist version --- engines/pyfunc-ensembler-job/Makefile | 4 +++- engines/pyfunc-ensembler-service/Makefile | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/engines/pyfunc-ensembler-job/Makefile b/engines/pyfunc-ensembler-job/Makefile index 9bd228c54..dbf5cd819 100644 --- a/engines/pyfunc-ensembler-job/Makefile +++ b/engines/pyfunc-ensembler-job/Makefile @@ -8,7 +8,9 @@ ACTIVATE_ENV = source $$(conda info --base)/etc/profile.d/conda.sh ; conda activ .PHONY: setup setup: build @pip install pipenv - @pipenv run pip install 'dist/turing_pyfunc_ensembler_job-$(shell echo $(VERSION) | sed 's/^v//')-py3-none-any.whl[dev]' + @DIST_VERSION=$$(echo $(VERSION) | \ + sed -E 's/^v([0-9]+\.[0-9]+\.[0-9]+)(-rc([0-9]+))?/\1rc\3/'); \ + pipenv run pip install "dist/turing_pyfunc_ensembler_job-$${DIST_VERSION}-py3-none-any.whl[dev]" .PHONY: type-check type-check: diff --git a/engines/pyfunc-ensembler-service/Makefile b/engines/pyfunc-ensembler-service/Makefile index b0bcf6b95..1d0c5db04 100644 --- a/engines/pyfunc-ensembler-service/Makefile +++ b/engines/pyfunc-ensembler-service/Makefile @@ -8,7 +8,9 @@ ACTIVATE_ENV = source $$(conda info --base)/etc/profile.d/conda.sh ; conda activ .PHONY: setup setup: build @pip install pipenv - @pipenv run pip install 'dist/turing_pyfunc_ensembler_service-$(shell echo $(VERSION) | sed 's/^v//')-py3-none-any.whl[dev]' + @DIST_VERSION=$$(echo $(VERSION) | \ + sed -E 's/^v([0-9]+\.[0-9]+\.[0-9]+)(-rc([0-9]+))?/\1rc\3/'); \ + pipenv run pip install "dist/turing_pyfunc_ensembler_service-$${DIST_VERSION}-py3-none-any.whl[dev]" .PHONY: lint lint: From acb00b601af3092060ef05ff361844537b69edcf Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 11 Apr 2025 14:23:09 +0700 Subject: [PATCH 68/78] feat: only run pyfunc-ensembler-* workflow after sdk workflow is successfully run --- .github/workflows/pyfunc-ensembler-job.yaml | 12 ++++++++++-- .github/workflows/pyfunc-ensembler-service.yaml | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pyfunc-ensembler-job.yaml b/.github/workflows/pyfunc-ensembler-job.yaml index d52588b8e..8a7984745 100644 --- a/.github/workflows/pyfunc-ensembler-job.yaml +++ b/.github/workflows/pyfunc-ensembler-job.yaml @@ -31,6 +31,13 @@ on: required: true PYPI_API_TOKEN: required: true + + # The package build by this job requires latest version of sdk, + # so we can only run this workflow after sdk workflow is successfully run + workflow_run: + workflows: ["sdk"] + types: + - completed jobs: test: @@ -84,9 +91,10 @@ jobs: # Dev build can be released either from the 'main' branch or # by running this workflow manually with `workflow_dispatch` event. if: >- - contains('release,pre-release', needs.release-rules.outputs.release-type) + (contains('release,pre-release', needs.release-rules.outputs.release-type) || ( github.event_name != 'pull_request' ) - || ( github.event.pull_request.head.repo.full_name == github.repository ) + || ( github.event.pull_request.head.repo.full_name == github.repository )) && + ${{ github.event.workflow_run.conclusion == 'success' }} environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }} runs-on: ubuntu-latest needs: diff --git a/.github/workflows/pyfunc-ensembler-service.yaml b/.github/workflows/pyfunc-ensembler-service.yaml index 83b8f8f4b..163d975ec 100644 --- a/.github/workflows/pyfunc-ensembler-service.yaml +++ b/.github/workflows/pyfunc-ensembler-service.yaml @@ -32,6 +32,13 @@ on: PYPI_API_TOKEN: required: true + # The package build by this job requires latest version of sdk, + # so we can only run this workflow after sdk workflow is successfully run + workflow_run: + workflows: ["sdk"] + types: + - completed + jobs: test: runs-on: ubuntu-latest @@ -78,9 +85,10 @@ jobs: # Dev build can be released either from the 'main' branch or # by running this workflow manually with `workflow_dispatch` event. if: >- - contains('release,pre-release', needs.release-rules.outputs.release-type) + (contains('release,pre-release', needs.release-rules.outputs.release-type) || ( github.event_name != 'pull_request' ) - || ( github.event.pull_request.head.repo.full_name == github.repository ) + || ( github.event.pull_request.head.repo.full_name == github.repository )) && + ${{ github.event.workflow_run.conclusion == 'success' }} environment: ${{ needs.release-rules.outputs.release-type == 'dev' && 'manual' || '' }} runs-on: ubuntu-latest needs: From 6480f682e899e1fed5b68e843f88c480b6885fcf Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 11 Apr 2025 14:45:33 +0700 Subject: [PATCH 69/78] feat: remove other trigger for pyfunc-ensembler-* workflow --- .github/workflows/pyfunc-ensembler-job.yaml | 31 ------------------- .../workflows/pyfunc-ensembler-service.yaml | 31 ------------------- 2 files changed, 62 deletions(-) diff --git a/.github/workflows/pyfunc-ensembler-job.yaml b/.github/workflows/pyfunc-ensembler-job.yaml index 8a7984745..cf5dd8049 100644 --- a/.github/workflows/pyfunc-ensembler-job.yaml +++ b/.github/workflows/pyfunc-ensembler-job.yaml @@ -1,37 +1,6 @@ name: engines/pyfunc-ensembler-job on: - # Automatically run CI on Release and Pre-Release tags and main branch - # (only if there are changes to relevant paths) - push: - tags: - - "pyfunc-ensembler-job/v[0-9]+.[0-9]+.[0-9]+*" - - "python/v[0-9]+.[0-9]+.[0-9]+*" - branches: - - main - paths: - - ".github/workflows/pyfunc-ensembler-job.yaml" - - "engines/pyfunc-ensembler-job/**" - - "sdk/**" - - # Automatically run CI on branches, that have active PR opened - pull_request: - branches: - - main - paths: - - ".github/workflows/pyfunc-ensembler-job.yaml" - - "engines/pyfunc-ensembler-job/**" - - "sdk/**" - - # To make it possible to trigger e2e CI workflow for any arbitrary git ref - workflow_dispatch: - workflow_call: - secrets: - PYPI_USERNAME: - required: true - PYPI_API_TOKEN: - required: true - # The package build by this job requires latest version of sdk, # so we can only run this workflow after sdk workflow is successfully run workflow_run: diff --git a/.github/workflows/pyfunc-ensembler-service.yaml b/.github/workflows/pyfunc-ensembler-service.yaml index 163d975ec..47eeda289 100644 --- a/.github/workflows/pyfunc-ensembler-service.yaml +++ b/.github/workflows/pyfunc-ensembler-service.yaml @@ -1,37 +1,6 @@ name: engines/pyfunc-ensembler-service on: - # Automatically run CI on Release and Pre-Release tags and main branch - # (only if there are changes to relevant paths) - push: - tags: - - "pyfunc-ensembler-service/v[0-9]+.[0-9]+.[0-9]+*" - - "python/v[0-9]+.[0-9]+.[0-9]+*" - branches: - - main - paths: - - ".github/workflows/pyfunc-ensembler-service.yaml" - - "engines/pyfunc-ensembler-service/**" - - "sdk/**" - - # Automatically run CI on branches, that have active PR opened - pull_request: - branches: - - main - paths: - - ".github/workflows/pyfunc-ensembler-service.yaml" - - "engines/pyfunc-ensembler-service/**" - - "sdk/**" - - # To make it possible to trigger e2e CI workflow for any arbitrary git ref - workflow_dispatch: - workflow_call: - secrets: - PYPI_USERNAME: - required: true - PYPI_API_TOKEN: - required: true - # The package build by this job requires latest version of sdk, # so we can only run this workflow after sdk workflow is successfully run workflow_run: From 3649ed9b7a828131e931fe4a1efd948a207e29a2 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 11 Apr 2025 16:00:22 +0700 Subject: [PATCH 70/78] fix: change sdk app name param for vertagen --- engines/pyfunc-ensembler-job/Makefile | 2 +- sdk/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/engines/pyfunc-ensembler-job/Makefile b/engines/pyfunc-ensembler-job/Makefile index dbf5cd819..194e1620e 100644 --- a/engines/pyfunc-ensembler-job/Makefile +++ b/engines/pyfunc-ensembler-job/Makefile @@ -37,7 +37,7 @@ build-image: version .PHONY: version version: - $(eval VERSION=$(if $(OVERWRITE_VERSION),$(OVERWRITE_VERSION),v$(shell ../../scripts/vertagen/vertagen.sh -p python/))) + $(eval VERSION=$(if $(OVERWRITE_VERSION),$(OVERWRITE_VERSION),v$(shell ../../scripts/vertagen/vertagen.sh -p python/ -y))) @echo "turing-pyfunc-ensembler-job version:" $(VERSION) .PHONY: build diff --git a/sdk/Makefile b/sdk/Makefile index c184c02ed..f18b2e44b 100644 --- a/sdk/Makefile +++ b/sdk/Makefile @@ -13,7 +13,7 @@ gen-client: .PHONY: version version: - $(eval VERSION=$(if $(OVERWRITE_VERSION),$(OVERWRITE_VERSION),$(shell ../scripts/vertagen/vertagen.sh -p sdk/ -y))) + $(eval VERSION=$(if $(OVERWRITE_VERSION),$(OVERWRITE_VERSION),$(shell ../scripts/vertagen/vertagen.sh -p python/ -y))) @echo 'VERSION = "$(VERSION)"' > turing/version.py @echo "turing-sdk version:" $(VERSION) From 69419b6ebed4ebbd1c8c1ab0167b30d2395fe352 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 11 Apr 2025 17:18:09 +0700 Subject: [PATCH 71/78] chore: remove --platform=linux/amd64 from pyfunc-ensembler-job base Dockerfile --- engines/pyfunc-ensembler-job/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/pyfunc-ensembler-job/Dockerfile b/engines/pyfunc-ensembler-job/Dockerfile index 947e7ba78..2c4471cf0 100644 --- a/engines/pyfunc-ensembler-job/Dockerfile +++ b/engines/pyfunc-ensembler-job/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=linux/amd64 apache/spark-py:v3.1.3 +FROM apache/spark-py:v3.1.3 # Switch to user root so we can add additional jars and configuration files. USER root From 17712a3dfe403d0c76fbefd537495bb2ab9ea899 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 25 Apr 2025 14:25:50 +0700 Subject: [PATCH 72/78] feat: do pyfunc-ensembler-job unit testing for python 3.8 - 3.10 --- .github/workflows/pyfunc-ensembler-job.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pyfunc-ensembler-job.yaml b/.github/workflows/pyfunc-ensembler-job.yaml index cf5dd8049..0663883b8 100644 --- a/.github/workflows/pyfunc-ensembler-job.yaml +++ b/.github/workflows/pyfunc-ensembler-job.yaml @@ -11,14 +11,16 @@ on: jobs: test: runs-on: ubuntu-latest - + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v4 - - name: Setup Python + - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: ${{ matrix.python-version }} cache-dependency-path: | engines/pyfunc-ensembler-job/requirements.txt engines/pyfunc-ensembler-job/requirements.dev.txt From 44f594a8ca3df571701145c888e42aba4de2ff7f Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 25 Apr 2025 14:27:56 +0700 Subject: [PATCH 73/78] feat: check error from initArtifactService --- api/turing/api/appcontext.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/turing/api/appcontext.go b/api/turing/api/appcontext.go index 1dc83f30e..c76a997fc 100644 --- a/api/turing/api/appcontext.go +++ b/api/turing/api/appcontext.go @@ -96,6 +96,9 @@ func NewAppContext( ensemblersService := service.NewEnsemblersService(db) artifactService, err := initArtifactService(cfg) + if err != nil { + return nil, errors.Wrapf(err, "Failed initializing artifact service") + } if cfg.BatchEnsemblingConfig.Enabled { if cfg.BatchEnsemblingConfig.JobConfig == nil { From b520dd51f7e56d2c13b694903d2ed5589cf30875 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 25 Apr 2025 14:54:53 +0700 Subject: [PATCH 74/78] chore: remove BaseImageRef and adjust the unittest --- api/turing/api/appcontext_test.go | 12 +++---- api/turing/config/config.go | 4 --- api/turing/config/config_test.go | 12 +++---- api/turing/imagebuilder/imagebuilder_test.go | 36 +++++--------------- 4 files changed, 17 insertions(+), 47 deletions(-) diff --git a/api/turing/api/appcontext_test.go b/api/turing/api/appcontext_test.go index 55b77ea52..64cbf3935 100644 --- a/api/turing/api/appcontext_test.go +++ b/api/turing/api/appcontext_test.go @@ -70,10 +70,8 @@ func TestNewAppContext(t *testing.T) { MaxRetryCount: 3, }, ImageBuildingConfig: &config.ImageBuildingConfig{ - DestinationRegistry: "ghcr.io", - BaseImageRef: map[string]string{ - "3.7.*": "ghcr.io/caraml-dev/turing/pyfunc-ensembler-job:0.0.0-build.1-98b071d", - }, + DestinationRegistry: "ghcr.io", + BaseImage: "ghcr.io/caraml-dev/turing/pyfunc-ensembler-job:0.0.0-build.1-98b071d", BuildNamespace: "default", BuildTimeoutDuration: 10 * time.Minute, KanikoConfig: config.KanikoConfig{ @@ -97,10 +95,8 @@ func TestNewAppContext(t *testing.T) { EnsemblerServiceBuilderConfig: config.EnsemblerServiceBuilderConfig{ ClusterName: defaultEnvironment, ImageBuildingConfig: &config.ImageBuildingConfig{ - DestinationRegistry: "ghcr.io", - BaseImageRef: map[string]string{ - "3.7.*": "ghcr.io/caraml-dev/turing/pyfunc-ensembler-service:0.0.0-build.1-98b071d", - }, + DestinationRegistry: "ghcr.io", + BaseImage: "ghcr.io/caraml-dev/turing/pyfunc-ensembler-service:0.0.0-build.1-98b071d", BuildNamespace: "default", BuildTimeoutDuration: 10 * time.Minute, KanikoConfig: config.KanikoConfig{ diff --git a/api/turing/config/config.go b/api/turing/config/config.go index d1a31d131..d7a84a1f3 100644 --- a/api/turing/config/config.go +++ b/api/turing/config/config.go @@ -169,10 +169,6 @@ type ImageBuildingConfig struct { BuildTimeoutDuration time.Duration `validate:"required"` // DestinationRegistry is the registry of the newly built ensembler image. DestinationRegistry string `validate:"required"` - // BaseImageRef is the image name of the base ensembler image built from the - // engines/pyfunc-ensembler-*/Dockerfile. It's a map of image names, per - // minor python version supported by the SDK. - BaseImageRef map[string]string `validate:"required"` // KanikoConfig contains the configuration related to the kaniko executor image builder. KanikoConfig KanikoConfig `validate:"required"` // TolerationName allow the scheduler to schedule image building jobs with the matching name diff --git a/api/turing/config/config_test.go b/api/turing/config/config_test.go index 3fea7e264..114ced468 100644 --- a/api/turing/config/config_test.go +++ b/api/turing/config/config_test.go @@ -822,10 +822,8 @@ func TestConfigValidate(t *testing.T) { MaxRetryCount: 3, }, ImageBuildingConfig: &config.ImageBuildingConfig{ - DestinationRegistry: "ghcr.io", - BaseImageRef: map[string]string{ - "3.7.*": "ghcr.io/caraml-dev/turing/pyfunc-ensembler-job:0.0.0-build.1-98b071d", - }, + DestinationRegistry: "ghcr.io", + BaseImage: "ghcr.io/caraml-dev/turing/pyfunc-ensembler-job:0.0.0-build.1-98b071d", BuildNamespace: "default", BuildTimeoutDuration: 10 * time.Minute, KanikoConfig: config.KanikoConfig{ @@ -850,10 +848,8 @@ func TestConfigValidate(t *testing.T) { EnsemblerServiceBuilderConfig: config.EnsemblerServiceBuilderConfig{ ClusterName: "dev", ImageBuildingConfig: &config.ImageBuildingConfig{ - DestinationRegistry: "ghcr.io", - BaseImageRef: map[string]string{ - "3.7.*": "ghcr.io/caraml-dev/turing/pyfunc-ensembler-service:0.0.0-build.1-98b071d", - }, + DestinationRegistry: "ghcr.io", + BaseImage: "ghcr.io/caraml-dev/turing/pyfunc-ensembler-service:0.0.0-build.1-98b071d", BuildNamespace: "default", BuildTimeoutDuration: 10 * time.Minute, KanikoConfig: config.KanikoConfig{ diff --git a/api/turing/imagebuilder/imagebuilder_test.go b/api/turing/imagebuilder/imagebuilder_test.go index bc53d9bae..289de9487 100644 --- a/api/turing/imagebuilder/imagebuilder_test.go +++ b/api/turing/imagebuilder/imagebuilder_test.go @@ -66,9 +66,7 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, DestinationRegistry: dockerRegistry, - BaseImageRef: map[string]string{ - "3.7.*": pyFuncEnsemblerJobBaseImageRef, - }, + BaseImage: pyFuncEnsemblerJobBaseImageRef, KanikoConfig: config.KanikoConfig{ BuildContextURI: buildContext, DockerfileFilePath: pyFuncEnsemblerJobDockerfilePath, @@ -373,9 +371,7 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, DestinationRegistry: dockerRegistry, - BaseImageRef: map[string]string{ - "3.7.*": pyFuncEnsemblerServiceBaseImageRef, - }, + BaseImage: pyFuncEnsemblerServiceBaseImageRef, KanikoConfig: config.KanikoConfig{ BuildContextURI: buildContext, DockerfileFilePath: pyFuncEnsemblerServiceDockerfilePath, @@ -770,9 +766,7 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, DestinationRegistry: dockerRegistry, - BaseImageRef: map[string]string{ - "3.7.*": pyFuncEnsemblerJobBaseImageRef, - }, + BaseImage: pyFuncEnsemblerJobBaseImageRef, KanikoConfig: config.KanikoConfig{ BuildContextURI: buildContext, DockerfileFilePath: pyFuncEnsemblerJobDockerfilePath, @@ -1021,9 +1015,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, DestinationRegistry: dockerRegistry, - BaseImageRef: map[string]string{ - "3.7.*": pyFuncEnsemblerJobBaseImageRef, - }, + BaseImage: pyFuncEnsemblerJobBaseImageRef, KanikoConfig: config.KanikoConfig{ BuildContextURI: buildContext, DockerfileFilePath: pyFuncEnsemblerServiceDockerfilePath, @@ -1109,9 +1101,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, DestinationRegistry: dockerRegistry, - BaseImageRef: map[string]string{ - "3.7.*": pyFuncEnsemblerJobBaseImageRef, - }, + BaseImage: pyFuncEnsemblerJobBaseImageRef, KanikoConfig: config.KanikoConfig{ BuildContextURI: buildContext, DockerfileFilePath: pyFuncEnsemblerServiceDockerfilePath, @@ -1166,9 +1156,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, DestinationRegistry: dockerRegistry, - BaseImageRef: map[string]string{ - "3.7.*": pyFuncEnsemblerJobBaseImageRef, - }, + BaseImage: pyFuncEnsemblerJobBaseImageRef, KanikoConfig: config.KanikoConfig{ BuildContextURI: buildContext, DockerfileFilePath: pyFuncEnsemblerServiceDockerfilePath, @@ -1219,9 +1207,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, DestinationRegistry: dockerRegistry, - BaseImageRef: map[string]string{ - "3.7.*": pyFuncEnsemblerJobBaseImageRef, - }, + BaseImage: pyFuncEnsemblerJobBaseImageRef, KanikoConfig: config.KanikoConfig{ BuildContextURI: buildContext, DockerfileFilePath: pyFuncEnsemblerServiceDockerfilePath, @@ -1286,9 +1272,7 @@ func TestDeleteEnsemblerJobImageBuildingJob(t *testing.T) { BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, DestinationRegistry: dockerRegistry, - BaseImageRef: map[string]string{ - "3.7.*": pyFuncEnsemblerJobBaseImageRef, - }, + BaseImage: pyFuncEnsemblerJobBaseImageRef, KanikoConfig: config.KanikoConfig{ BuildContextURI: buildContext, DockerfileFilePath: pyFuncEnsemblerServiceDockerfilePath, @@ -1374,9 +1358,7 @@ func TestDeleteEnsemblerServiceImageBuildingJob(t *testing.T) { BuildNamespace: buildNamespace, BuildTimeoutDuration: timeout, DestinationRegistry: dockerRegistry, - BaseImageRef: map[string]string{ - "3.7.*": pyFuncEnsemblerJobBaseImageRef, - }, + BaseImage: pyFuncEnsemblerJobBaseImageRef, KanikoConfig: config.KanikoConfig{ BuildContextURI: buildContext, DockerfileFilePath: pyFuncEnsemblerServiceDockerfilePath, From 3df13eb66d45e2baea25dd78c950ff22160a2e28 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 25 Apr 2025 15:05:40 +0700 Subject: [PATCH 75/78] feat: use artifactService.GetType() instead of artifactServiceType to get artifact service type --- api/turing/api/appcontext.go | 2 -- api/turing/api/appcontext_test.go | 2 -- api/turing/imagebuilder/ensembler.go | 4 --- api/turing/imagebuilder/imagebuilder.go | 11 +++------ api/turing/imagebuilder/imagebuilder_test.go | 26 +++++++++++++++----- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/api/turing/api/appcontext.go b/api/turing/api/appcontext.go index c76a997fc..662db7cbc 100644 --- a/api/turing/api/appcontext.go +++ b/api/turing/api/appcontext.go @@ -130,7 +130,6 @@ func NewAppContext( ensemblingImageBuilder, err = imagebuilder.NewEnsemblerJobImageBuilder( imageBuildingController, *cfg.BatchEnsemblingConfig.ImageBuildingConfig, - cfg.MlflowConfig.ArtifactServiceType, artifactService, ) if err != nil { @@ -166,7 +165,6 @@ func NewAppContext( ensemblerServiceImageBuilder, err := imagebuilder.NewEnsemblerServiceImageBuilder( clusterControllers[cfg.EnsemblerServiceBuilderConfig.ClusterName], *cfg.EnsemblerServiceBuilderConfig.ImageBuildingConfig, - cfg.MlflowConfig.ArtifactServiceType, artifactService, ) if err != nil { diff --git a/api/turing/api/appcontext_test.go b/api/turing/api/appcontext_test.go index 64cbf3935..e14aabd3f 100644 --- a/api/turing/api/appcontext_test.go +++ b/api/turing/api/appcontext_test.go @@ -308,7 +308,6 @@ func TestNewAppContext(t *testing.T) { ensemblingImageBuilder, err := imagebuilder.NewEnsemblerJobImageBuilder( nil, *testCfg.BatchEnsemblingConfig.ImageBuildingConfig, - testCfg.MlflowConfig.ArtifactServiceType, artifactService, ) assert.Nil(t, err) @@ -343,7 +342,6 @@ func TestNewAppContext(t *testing.T) { ensemblerImageBuilder, err := imagebuilder.NewEnsemblerServiceImageBuilder( nil, *testCfg.EnsemblerServiceBuilderConfig.ImageBuildingConfig, - testCfg.MlflowConfig.ArtifactServiceType, artifactService, ) diff --git a/api/turing/imagebuilder/ensembler.go b/api/turing/imagebuilder/ensembler.go index 76858eda3..7432161d7 100644 --- a/api/turing/imagebuilder/ensembler.go +++ b/api/turing/imagebuilder/ensembler.go @@ -13,7 +13,6 @@ import ( func NewEnsemblerJobImageBuilder( clusterController cluster.Controller, imageBuildingConfig config.ImageBuildingConfig, - artifactServiceType string, artifactService artifact.Service, ) (ImageBuilder, error) { return newImageBuilder( @@ -21,7 +20,6 @@ func NewEnsemblerJobImageBuilder( imageBuildingConfig, &ensemblerJobNameGenerator{registry: imageBuildingConfig.DestinationRegistry}, models.EnsemblerRunnerTypeJob, - artifactServiceType, artifactService, ) } @@ -53,7 +51,6 @@ func (n *ensemblerJobNameGenerator) generateDockerImageName(projectName string, func NewEnsemblerServiceImageBuilder( clusterController cluster.Controller, imageBuildingConfig config.ImageBuildingConfig, - artifactServiceType string, artifactService artifact.Service, ) (ImageBuilder, error) { return newImageBuilder( @@ -61,7 +58,6 @@ func NewEnsemblerServiceImageBuilder( imageBuildingConfig, &ensemblerServiceNameGenerator{registry: imageBuildingConfig.DestinationRegistry}, models.EnsemblerRunnerTypeService, - artifactServiceType, artifactService, ) } diff --git a/api/turing/imagebuilder/imagebuilder.go b/api/turing/imagebuilder/imagebuilder.go index abc03f24e..4cf8e5f74 100644 --- a/api/turing/imagebuilder/imagebuilder.go +++ b/api/turing/imagebuilder/imagebuilder.go @@ -132,7 +132,6 @@ type imageBuilder struct { imageBuildingConfig config.ImageBuildingConfig nameGenerator nameGenerator runnerType models.EnsemblerRunnerType - artifactServiceType string artifactService artifact.Service } @@ -142,7 +141,6 @@ func newImageBuilder( imageBuildingConfig config.ImageBuildingConfig, nameGenerator nameGenerator, runnerType models.EnsemblerRunnerType, - artifactServiceType string, artifactService artifact.Service, ) (ImageBuilder, error) { err := checkParseResources(imageBuildingConfig.KanikoConfig.ResourceRequestsLimits) @@ -155,7 +153,6 @@ func newImageBuilder( imageBuildingConfig: imageBuildingConfig, nameGenerator: nameGenerator, runnerType: runnerType, - artifactServiceType: artifactServiceType, artifactService: artifactService, }, nil } @@ -300,7 +297,7 @@ func (ib *imageBuilder) createKanikoJob( fmt.Sprintf("--context=%s", ib.imageBuildingConfig.KanikoConfig.BuildContextURI), fmt.Sprintf("--build-arg=MODEL_URL=%s", artifactURI), fmt.Sprintf("--build-arg=BASE_IMAGE=%s", ib.imageBuildingConfig.BaseImage), - fmt.Sprintf("--build-arg=MLFLOW_ARTIFACT_STORAGE_TYPE=%s", ib.artifactServiceType), + fmt.Sprintf("--build-arg=MLFLOW_ARTIFACT_STORAGE_TYPE=%s", ib.artifactService.GetType()), fmt.Sprintf("--build-arg=FOLDER_NAME=%s", folderName), fmt.Sprintf("--build-arg=MODEL_DEPENDENCIES_URL=%s", hashedModelDependenciesURL), fmt.Sprintf("--destination=%s", imageRef), @@ -386,7 +383,7 @@ func (ib *imageBuilder) createKanikoJob( func (ib *imageBuilder) configureKanikoArgsToAddCredentials(kanikoArgs []string) []string { if ib.imageBuildingConfig.KanikoConfig.PushRegistryType == googleCloudRegistryPushRegistryType || - ib.artifactServiceType == googleCloudStorageArtifactServiceType { + ib.artifactService.GetType() == googleCloudStorageArtifactServiceType { if ib.imageBuildingConfig.KanikoConfig.ServiceAccount == "" { kanikoArgs = append(kanikoArgs, fmt.Sprintf("--build-arg=GOOGLE_APPLICATION_CREDENTIALS=%s/%s", @@ -401,7 +398,7 @@ func (ib *imageBuilder) configureVolumesAndVolumeMountsToAddCredentials( volumeMounts []cluster.VolumeMount, ) ([]cluster.SecretVolume, []cluster.VolumeMount) { if ib.imageBuildingConfig.KanikoConfig.PushRegistryType == googleCloudRegistryPushRegistryType || - ib.artifactServiceType == googleCloudStorageArtifactServiceType { + ib.artifactService.GetType() == googleCloudStorageArtifactServiceType { // If kaniko service account is not set, use kaniko secret if ib.imageBuildingConfig.KanikoConfig.ServiceAccount == "" { volumes = append(volumes, cluster.SecretVolume{ @@ -429,7 +426,7 @@ func (ib *imageBuilder) configureVolumesAndVolumeMountsToAddCredentials( func (ib *imageBuilder) configureEnvVarsToAddCredentials(envVar []cluster.Env) []cluster.Env { if ib.imageBuildingConfig.KanikoConfig.PushRegistryType == googleCloudRegistryPushRegistryType || - ib.artifactServiceType == googleCloudStorageArtifactServiceType { + ib.artifactService.GetType() == googleCloudStorageArtifactServiceType { if ib.imageBuildingConfig.KanikoConfig.ServiceAccount == "" { envVar = append(envVar, cluster.Env{ Name: googleApplicationEnvVarName, diff --git a/api/turing/imagebuilder/imagebuilder_test.go b/api/turing/imagebuilder/imagebuilder_test.go index 289de9487..037548bf8 100644 --- a/api/turing/imagebuilder/imagebuilder_test.go +++ b/api/turing/imagebuilder/imagebuilder_test.go @@ -167,6 +167,7 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, "success: existing job is running": { @@ -237,6 +238,7 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, "success: existing job failed": { @@ -331,6 +333,7 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, } @@ -345,7 +348,6 @@ func TestBuildPyFuncEnsemblerJobImage(t *testing.T) { ib, err := NewEnsemblerJobImageBuilder( clusterController, tt.imageBuildingConfig, - googleCloudStorageArtifactServiceType, artifactServiceMock, ) assert.Nil(t, err) @@ -473,6 +475,7 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, "success: existing job is running": { @@ -543,6 +546,7 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, "success: existing job failed": { @@ -637,6 +641,7 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, } @@ -651,7 +656,6 @@ func TestBuildPyFuncEnsemblerServiceImage(t *testing.T) { ib, err := NewEnsemblerServiceImageBuilder( clusterController, tt.imageBuildingConfig, - googleCloudStorageArtifactServiceType, artifactServiceMock, ) assert.Nil(t, err) @@ -817,6 +821,7 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, "success | succeeded": { @@ -845,6 +850,7 @@ func TestGetEnsemblerJobImageBuildingJobStatus(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, "success | Failed": { @@ -934,6 +940,7 @@ CondaEnvException: Pip failed`, artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, "success | Unknown": { @@ -964,6 +971,7 @@ CondaEnvException: Pip failed`, artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, "failure | Unknown": { @@ -989,6 +997,7 @@ CondaEnvException: Pip failed`, artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, } @@ -1001,7 +1010,6 @@ CondaEnvException: Pip failed`, ib, _ := NewEnsemblerJobImageBuilder( clusterController, tt.imageBuildingConfig, - googleCloudStorageArtifactServiceType, artifactServiceMock, ) status := ib.GetImageBuildingJobStatus(projectName, modelName, models.ID(1), runID) @@ -1066,6 +1074,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, "success | succeeded": { @@ -1094,6 +1103,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, "success | Failed": { @@ -1149,6 +1159,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, "success | Unknown": { @@ -1200,6 +1211,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, "failure | Unknown": { @@ -1246,6 +1258,7 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, } @@ -1258,7 +1271,6 @@ func TestGetEnsemblerServiceImageBuildingJobStatus(t *testing.T) { ib, _ := NewEnsemblerServiceImageBuilder( clusterController, tt.imageBuildingConfig, - googleCloudStorageArtifactServiceType, artifactServiceMock, ) status := ib.GetImageBuildingJobStatus("", "", models.ID(1), runID) @@ -1320,6 +1332,7 @@ func TestDeleteEnsemblerJobImageBuildingJob(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, } @@ -1332,7 +1345,6 @@ func TestDeleteEnsemblerJobImageBuildingJob(t *testing.T) { ib, _ := NewEnsemblerJobImageBuilder( clusterController, tt.imageBuildingConfig, - googleCloudStorageArtifactServiceType, artifactServiceMock, ) err := ib.DeleteImageBuildingJob("", "", models.ID(1), runID) @@ -1398,6 +1410,7 @@ func TestDeleteEnsemblerServiceImageBuildingJob(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, }, } @@ -1410,7 +1423,6 @@ func TestDeleteEnsemblerServiceImageBuildingJob(t *testing.T) { ib, _ := NewEnsemblerJobImageBuilder( clusterController, tt.imageBuildingConfig, - googleCloudStorageArtifactServiceType, artifactServiceMock, ) err := ib.DeleteImageBuildingJob("", "", models.ID(1), runID) @@ -1450,6 +1462,7 @@ func Test_imageBuilder_getHashedModelDependenciesURL(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, fmt.Sprintf("gs%s", testCondaEnvURLSuffix)). Return([]byte(testCondaEnvContent), nil) artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return([]byte(testCondaEnvContent), nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, want: modelDependenciesURL, wantErr: false, @@ -1469,6 +1482,7 @@ func Test_imageBuilder_getHashedModelDependenciesURL(t *testing.T) { artifactServiceMock.On("ReadArtifact", mock.Anything, modelDependenciesURL).Return(nil, artifact.ErrObjectNotExist) artifactServiceMock.On("WriteArtifact", mock.Anything, modelDependenciesURL, []byte(testCondaEnvContent)). Return(nil) + artifactServiceMock.On("GetType").Return(googleCloudStorageArtifactServiceType) }, want: modelDependenciesURL, wantErr: false, From a0f11e60f07ea03fdd185136668d942ebb5e568a Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 25 Apr 2025 15:07:44 +0700 Subject: [PATCH 76/78] chore: reorder install yq command in Dockerfile --- engines/pyfunc-ensembler-job/Dockerfile | 10 +++++----- engines/pyfunc-ensembler-service/Dockerfile | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/engines/pyfunc-ensembler-job/Dockerfile b/engines/pyfunc-ensembler-job/Dockerfile index 2c4471cf0..ae849aae4 100644 --- a/engines/pyfunc-ensembler-job/Dockerfile +++ b/engines/pyfunc-ensembler-job/Dockerfile @@ -44,6 +44,11 @@ RUN apt-get update --allow-releaseinfo-change-suite -q && \ wget \ && apt-get clean +# Install yq +ENV YQ_VERSION=v4.42.1 +RUN wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq && \ + chmod +x /usr/bin/yq + # Install gcloud SDK ENV PATH=$PATH:/google-cloud-sdk/bin ARG GCLOUD_VERSION=410.0.0 @@ -54,11 +59,6 @@ RUN wget -qO- \ # Install aws CLI RUN wget -q https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip && unzip awscli-exe-linux-x86_64.zip && ./aws/install -# Install yq -ENV YQ_VERSION=v4.42.1 -RUN wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq && \ - chmod +x /usr/bin/yq - COPY ./entrypoint.sh /opt/entrypoint.sh # Configure non-root user diff --git a/engines/pyfunc-ensembler-service/Dockerfile b/engines/pyfunc-ensembler-service/Dockerfile index 45ac1c6fd..cbaaad79f 100644 --- a/engines/pyfunc-ensembler-service/Dockerfile +++ b/engines/pyfunc-ensembler-service/Dockerfile @@ -8,6 +8,11 @@ ARG CONDA_ENV_NAME ENV APP_NAME=$APP_NAME ENV CONDA_ENV_NAME=$CONDA_ENV_NAME +# Install yq +ENV YQ_VERSION=v4.42.1 +RUN wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq && \ + chmod +x /usr/bin/yq + # Install gcloud SDK RUN wget -qO- https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-367.0.0-linux-x86_64.tar.gz | tar xzf - ENV PATH=$PATH:/google-cloud-sdk/bin @@ -15,11 +20,6 @@ ENV PATH=$PATH:/google-cloud-sdk/bin # Install aws CLI RUN wget -q https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip && unzip awscli-exe-linux-x86_64.zip && ./aws/install -# Install yq -ENV YQ_VERSION=v4.42.1 -RUN wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq && \ - chmod +x /usr/bin/yq - COPY . . COPY ./temp-deps/sdk ./../../sdk COPY process_conda_env.sh /bin/process_conda_env.sh From 515ce0b8a37954a36211e469e8166b11325becff Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 25 Apr 2025 15:44:29 +0700 Subject: [PATCH 77/78] feat: adjust how to get version from version.py --- engines/pyfunc-ensembler-job/setup.py | 14 ++++---------- engines/pyfunc-ensembler-service/setup.py | 14 ++++---------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/engines/pyfunc-ensembler-job/setup.py b/engines/pyfunc-ensembler-job/setup.py index 1068384c7..8d028798f 100644 --- a/engines/pyfunc-ensembler-job/setup.py +++ b/engines/pyfunc-ensembler-job/setup.py @@ -1,18 +1,12 @@ import setuptools import pathlib import pkg_resources -import importlib.util -import os # get version from version.py -spec = importlib.util.spec_from_file_location( - "pyfuncserver.version", os.path.join("version.py") -) - -v_module = importlib.util.module_from_spec(spec) -spec.loader.exec_module(v_module) - -version = v_module.VERSION +with pathlib.Path("version.py").open() as version_py: + _locals = locals() + exec(version_py.read(), globals(), _locals) + version = _locals["VERSION"] with pathlib.Path("requirements.txt").open() as requirements_txt: requirements = [ diff --git a/engines/pyfunc-ensembler-service/setup.py b/engines/pyfunc-ensembler-service/setup.py index 05d604942..f3e7fc2e4 100644 --- a/engines/pyfunc-ensembler-service/setup.py +++ b/engines/pyfunc-ensembler-service/setup.py @@ -1,18 +1,12 @@ import setuptools import pathlib import pkg_resources -import importlib.util -import os # get version from version.py -spec = importlib.util.spec_from_file_location( - "pyfuncserver.version", os.path.join("version.py") -) - -v_module = importlib.util.module_from_spec(spec) -spec.loader.exec_module(v_module) - -version = v_module.VERSION +with pathlib.Path("version.py").open() as version_py: + _locals = locals() + exec(version_py.read(), globals(), _locals) + version = _locals["VERSION"] with pathlib.Path("requirements.txt").open() as requirements_txt: requirements = [ From 2ea96b33a16456a6f6dd56a0d6bdeae0097bc465 Mon Sep 17 00:00:00 2001 From: Muhammad Naufal Andika Natsir Putra Date: Fri, 25 Apr 2025 15:57:00 +0700 Subject: [PATCH 78/78] feat: remove from requirements.txt --- engines/pyfunc-ensembler-job/requirements.dev.txt | 1 - engines/pyfunc-ensembler-service/requirements.dev.txt | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/engines/pyfunc-ensembler-job/requirements.dev.txt b/engines/pyfunc-ensembler-job/requirements.dev.txt index dfc2cde42..101a896ef 100644 --- a/engines/pyfunc-ensembler-job/requirements.dev.txt +++ b/engines/pyfunc-ensembler-job/requirements.dev.txt @@ -5,4 +5,3 @@ mypy>=0.910 pytest<=8.1.2 pytest-cov pylint -types-PyYAML diff --git a/engines/pyfunc-ensembler-service/requirements.dev.txt b/engines/pyfunc-ensembler-service/requirements.dev.txt index 1c2d013bb..dffdea880 100644 --- a/engines/pyfunc-ensembler-service/requirements.dev.txt +++ b/engines/pyfunc-ensembler-service/requirements.dev.txt @@ -2,5 +2,4 @@ black==22.6.0 # The next release 8.2.0 of pytest breaks the unit tests pytest<=8.1.2 pytest-cov -pylint -types-PyYAML +pylint \ No newline at end of file