diff --git a/.wordlist-txt b/.wordlist-txt index b54331752..b3a42a811 100644 --- a/.wordlist-txt +++ b/.wordlist-txt @@ -23,6 +23,7 @@ CICD CNI CRD CRDs +CRs CUDA CheCluster CloudEvents @@ -74,6 +75,7 @@ ITMS IgnoreExtraneous ImageSetConfiguration ImageStreams +InferenceServices Infiniband Istio Jaeger @@ -140,6 +142,7 @@ README RHACM RHOAI RawDeployments +RayJobs Rebasing RoleBinding Rollout @@ -341,6 +344,7 @@ rawdeployments rbac rebase redhat +reinstalls redistributions repo repo's @@ -393,6 +397,7 @@ vSphere vllm vscode vsphere +webhooks wordlist workspaces xlarge diff --git a/components/operators/openshift-ai/instance-2.x/components/cleanup-on-delete/README.md b/components/operators/openshift-ai/instance-2.x/components/cleanup-on-delete/README.md new file mode 100644 index 000000000..c2a2dd96b --- /dev/null +++ b/components/operators/openshift-ai/instance-2.x/components/cleanup-on-delete/README.md @@ -0,0 +1,44 @@ +# cleanup-on-delete + +Runs the [official Red Hat RHOAI uninstall procedure](https://docs.redhat.com/en/documentation/red_hat_openshift_ai_self-managed) automatically as an ArgoCD `PreDelete` hook when the RHOAI Application is deleted. + +Without this component, ArgoCD deletes RHOAI manifests in an arbitrary order, which leaves namespaces stuck in `Terminating` and webhooks that block future reinstalls. + +## What it does + +Executes a 5-step cleanup Job before ArgoCD removes any resources: + +1. Delete all user workload CRs across all namespaces (InferenceServices, Notebooks, RayJobs, etc.) — prevents finalizer deadlocks +2. Delete the `DataScienceCluster` and `DSCInitialization` CRs +3. Delete OLM resources: `Subscription`, `ClusterServiceVersion`, `OperatorGroup`, `InstallPlan` +4. Delete RHOAI `ValidatingWebhookConfiguration` and `MutatingWebhookConfiguration` resources +5. Delete RHOAI namespaces and CRDs (explicit list per Red Hat documentation) + +## How to enable + +Add this component to your instance overlay's `components:` list: + +```yaml +# components/operators/openshift-ai/instance-2.x/overlays/stable-2.25/kustomization.yaml +components: + - ../../components/cleanup-on-delete + - ../../components/components-kserve + # ... other components +``` + +## Resources created + +All resources are ArgoCD `PreDelete` hooks and are automatically cleaned up after the Job completes: + +- `ConfigMap/rhoai-cleanup-script` in `openshift-gitops` — mounts the cleanup script +- `ServiceAccount/rhoai-cleanup` in `openshift-gitops` +- `ClusterRole/rhoai-cleanup` +- `ClusterRoleBinding/rhoai-cleanup` +- `Job/delete-rhoai-resources` in `openshift-gitops` + +## Notes + +- The cleanup script is maintained in `delete-rhoai-resources.sh` and mounted into the Job via a ConfigMap +- `backoffLimit: 0` — the Job does not retry +- The Job runs in `openshift-gitops` (not `redhat-ods-operator`) to avoid being killed when RHOAI namespaces are deleted +- The Job exits with code `0` even when resources are not found, so ArgoCD always proceeds with Application deletion diff --git a/components/operators/openshift-ai/instance-2.x/components/cleanup-on-delete/delete-rhoai-resources.sh b/components/operators/openshift-ai/instance-2.x/components/cleanup-on-delete/delete-rhoai-resources.sh new file mode 100644 index 000000000..44d6eb246 --- /dev/null +++ b/components/operators/openshift-ai/instance-2.x/components/cleanup-on-delete/delete-rhoai-resources.sh @@ -0,0 +1,305 @@ +#!/usr/bin/env bash + +set -e + +echo "================================================" +echo "Red Hat OpenShift AI Cleanup" +echo "================================================" +echo "" +echo "Following: docs.redhat.com/en/documentation/red_hat_openshift_ai_self-managed" +echo "" + +# Step 0: Delete user workload CRs across all namespaces before removing the DSC. +# Deleting CRs first prevents finalizer deadlocks during operator cleanup. +echo "Step 0: Deleting user workload CRs..." + +# Data Science Pipelines +oc delete datasciencepipelinesapplications.datasciencepipelinesapplications.opendatahub.io \ + -A --all --ignore-not-found 2>/dev/null || true +oc delete datasciencepipelines.components.platform.opendatahub.io \ + -A --all --ignore-not-found 2>/dev/null || true + +# Argo Workflows +oc delete clusterworkflowtemplates.argoproj.io --all --ignore-not-found 2>/dev/null || true +oc delete cronworkflows.argoproj.io -A --all --ignore-not-found 2>/dev/null || true +oc delete workflowartifactgctasks.argoproj.io -A --all --ignore-not-found 2>/dev/null || true +oc delete workfloweventbindings.argoproj.io -A --all --ignore-not-found 2>/dev/null || true +oc delete workflows.argoproj.io -A --all --ignore-not-found 2>/dev/null || true +oc delete workflowtaskresults.argoproj.io -A --all --ignore-not-found 2>/dev/null || true +oc delete workflowtasksets.argoproj.io -A --all --ignore-not-found 2>/dev/null || true +oc delete workflowtemplates.argoproj.io -A --all --ignore-not-found 2>/dev/null || true + +# KServe +oc delete clusterstoragecontainers.serving.kserve.io --all --ignore-not-found 2>/dev/null || true +oc delete inferencegraphs.serving.kserve.io -A --all --ignore-not-found 2>/dev/null || true +oc delete inferenceservices.serving.kserve.io -A --all --ignore-not-found 2>/dev/null || true +oc delete llminferenceserviceconfigs.serving.kserve.io -A --all --ignore-not-found 2>/dev/null || true +oc delete llminferenceservices.serving.kserve.io -A --all --ignore-not-found 2>/dev/null || true +oc delete predictors.serving.kserve.io -A --all --ignore-not-found 2>/dev/null || true +oc delete servingruntimes.serving.kserve.io -A --all --ignore-not-found 2>/dev/null || true +oc delete trainedmodels.serving.kserve.io -A --all --ignore-not-found 2>/dev/null || true + +# Inference (networking) +oc delete inferencemodels.inference.networking.x-k8s.io -A --all --ignore-not-found 2>/dev/null || true +oc delete inferencepools.inference.networking.x-k8s.io -A --all --ignore-not-found 2>/dev/null || true + +# Kueue +oc delete admissionchecks.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete clusterqueues.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete cohorts.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete localqueues.kueue.x-k8s.io -A --all --ignore-not-found 2>/dev/null || true +oc delete multikueueclusters.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete multikueueconfigs.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete provisioningrequestconfigs.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete resourceflavors.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete topologies.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete workloadpriorityclasses.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete workloads.kueue.x-k8s.io -A --all --ignore-not-found 2>/dev/null || true + +# Workbenches +oc delete notebooks.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true + +# Kubeflow Training Operator +oc delete jaxjobs.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete mpijobs.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete mxjobs.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete paddlejobs.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete pipelines.pipelines.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete pipelineversions.pipelines.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete pytorchjobs.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete scheduledworkflows.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete tfjobs.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete viewers.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete xgboostjobs.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true + +# TrustyAI +oc delete guardrailsorchestrators.trustyai.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete lmevaljobs.trustyai.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete nemoguardrails.trustyai.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete trustyaiservices.trustyai.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true + +# Ray +oc delete rayclusters.ray.io -A --all --ignore-not-found 2>/dev/null || true +oc delete rayjobs.ray.io -A --all --ignore-not-found 2>/dev/null || true +oc delete rayservices.ray.io -A --all --ignore-not-found 2>/dev/null || true + +# CodeFlare / AppWrappers +oc delete appwrappers.workload.codeflare.dev -A --all --ignore-not-found 2>/dev/null || true + +# Feast +oc delete featurestores.feast.dev -A --all --ignore-not-found 2>/dev/null || true + +# LlamaStack +oc delete llamastackdistributions.llamastack.io -A --all --ignore-not-found 2>/dev/null || true + +# Model Registry +oc delete modelregistries.modelregistry.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true + +# RHOAI / ODH platform CRs +oc delete accounts.nim.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete applications.app.k8s.io -A --all --ignore-not-found 2>/dev/null || true +oc delete auths.services.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete dashboards.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete feastoperators.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete kserves.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete kueues.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete llamastackoperators.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete modelcontrollers.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete modelmeshservings.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete modelregistries.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete monitorings.services.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete rays.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete servicemeshes.services.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete sparkoperators.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete trainingoperators.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete trustyais.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete workbenches.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true + +echo " Done" +echo "" + +# Step 1: Delete DataScienceCluster and DSCInitialization +echo "Step 1: Deleting DataScienceCluster and DSCInitialization..." +oc delete datascienceclusters.datasciencecluster.opendatahub.io --all --ignore-not-found 2>/dev/null || true +oc delete dscinitializations.dscinitialization.opendatahub.io --all --ignore-not-found 2>/dev/null || true +oc delete featuretrackers.features.opendatahub.io --all --ignore-not-found 2>/dev/null || true +echo " Done" +echo "" + +# Step 2: Delete OLM resources +echo "Step 2: Deleting Subscription, CSV, OperatorGroup, InstallPlan..." +oc delete subscription --all -n redhat-ods-operator --ignore-not-found +oc delete clusterserviceversion --all -n redhat-ods-operator --ignore-not-found +oc delete operatorgroup --all -n redhat-ods-operator --ignore-not-found +oc delete installplan --all -n redhat-ods-operator --ignore-not-found +echo " Done" +echo "" + +# Step 3: Delete webhooks +echo "Step 3: Deleting RHOAI webhooks..." +oc get validatingwebhookconfiguration -o name 2>/dev/null \ + | grep -E "opendatahub|kserve|rhoai" \ + | xargs -r oc delete --ignore-not-found 2>/dev/null || true +oc get mutatingwebhookconfiguration -o name 2>/dev/null \ + | grep -E "opendatahub|kserve|rhoai" \ + | xargs -r oc delete --ignore-not-found 2>/dev/null || true +echo " Done" +echo "" + +# Step 4: Delete RHOAI namespaces +echo "Step 4: Deleting RHOAI namespaces..." +oc delete namespace redhat-ods-applications --ignore-not-found 2>/dev/null || true +oc delete namespace redhat-ods-monitoring --ignore-not-found 2>/dev/null || true +oc delete namespace redhat-ods-operator --ignore-not-found 2>/dev/null || true +oc delete namespace rhods-notebooks --ignore-not-found 2>/dev/null || true +echo " Done" +echo "" + +# Step 5: Delete RHOAI CRDs +echo "Step 5: Deleting RHOAI CRDs..." + +# Data Science Pipelines +oc delete crd \ + datasciencepipelinesapplications.datasciencepipelinesapplications.opendatahub.io \ + datasciencepipelines.components.platform.opendatahub.io \ + --ignore-not-found 2>/dev/null || true + +# Argo Workflows +oc delete crd \ + clusterworkflowtemplates.argoproj.io \ + cronworkflows.argoproj.io \ + workflowartifactgctasks.argoproj.io \ + workfloweventbindings.argoproj.io \ + workflows.argoproj.io \ + workflowtaskresults.argoproj.io \ + workflowtasksets.argoproj.io \ + workflowtemplates.argoproj.io \ + --ignore-not-found 2>/dev/null || true + +# KServe +oc delete crd \ + clusterstoragecontainers.serving.kserve.io \ + inferencegraphs.serving.kserve.io \ + inferenceservices.serving.kserve.io \ + llminferenceserviceconfigs.serving.kserve.io \ + llminferenceservices.serving.kserve.io \ + predictors.serving.kserve.io \ + servingruntimes.serving.kserve.io \ + trainedmodels.serving.kserve.io \ + --ignore-not-found 2>/dev/null || true + +# Inference (networking) +oc delete crd \ + inferencemodels.inference.networking.x-k8s.io \ + inferencepools.inference.networking.x-k8s.io \ + --ignore-not-found 2>/dev/null || true + +# Kueue +oc delete crd \ + admissionchecks.kueue.x-k8s.io \ + clusterqueues.kueue.x-k8s.io \ + cohorts.kueue.x-k8s.io \ + localqueues.kueue.x-k8s.io \ + multikueueclusters.kueue.x-k8s.io \ + multikueueconfigs.kueue.x-k8s.io \ + provisioningrequestconfigs.kueue.x-k8s.io \ + resourceflavors.kueue.x-k8s.io \ + topologies.kueue.x-k8s.io \ + workloadpriorityclasses.kueue.x-k8s.io \ + workloads.kueue.x-k8s.io \ + --ignore-not-found 2>/dev/null || true + +# Workbenches / Kubeflow +oc delete crd \ + notebooks.kubeflow.org \ + viewers.kubeflow.org \ + --ignore-not-found 2>/dev/null || true + +# Kubeflow Training Operator +oc delete crd \ + jaxjobs.kubeflow.org \ + mpijobs.kubeflow.org \ + mxjobs.kubeflow.org \ + paddlejobs.kubeflow.org \ + pipelines.pipelines.kubeflow.org \ + pipelineversions.pipelines.kubeflow.org \ + pytorchjobs.kubeflow.org \ + scheduledworkflows.kubeflow.org \ + tfjobs.kubeflow.org \ + xgboostjobs.kubeflow.org \ + --ignore-not-found 2>/dev/null || true + +# TrustyAI +oc delete crd \ + guardrailsorchestrators.trustyai.opendatahub.io \ + lmevaljobs.trustyai.opendatahub.io \ + nemoguardrails.trustyai.opendatahub.io \ + trustyaiservices.trustyai.opendatahub.io \ + --ignore-not-found 2>/dev/null || true + +# Ray +oc delete crd \ + rayclusters.ray.io \ + rayjobs.ray.io \ + rayservices.ray.io \ + --ignore-not-found 2>/dev/null || true + +# CodeFlare +oc delete crd appwrappers.workload.codeflare.dev --ignore-not-found 2>/dev/null || true + +# Feast +oc delete crd \ + feastoperators.components.platform.opendatahub.io \ + featurestores.feast.dev \ + --ignore-not-found 2>/dev/null || true + +# LlamaStack +oc delete crd llamastackdistributions.llamastack.io --ignore-not-found 2>/dev/null || true + +# RHOAI / ODH platform components +oc delete crd \ + accounts.nim.opendatahub.io \ + applications.app.k8s.io \ + auths.services.platform.opendatahub.io \ + codeflares.components.platform.opendatahub.io \ + dashboards.components.platform.opendatahub.io \ + kserves.components.platform.opendatahub.io \ + kueues.components.platform.opendatahub.io \ + llamastackoperators.components.platform.opendatahub.io \ + mlflowoperators.components.platform.opendatahub.io \ + modelcontrollers.components.platform.opendatahub.io \ + modelmeshservings.components.platform.opendatahub.io \ + modelregistries.components.platform.opendatahub.io \ + modelregistries.modelregistry.opendatahub.io \ + monitorings.services.platform.opendatahub.io \ + rays.components.platform.opendatahub.io \ + servicemeshes.services.platform.opendatahub.io \ + sparkoperators.components.platform.opendatahub.io \ + trainingoperators.components.platform.opendatahub.io \ + trainers.components.platform.opendatahub.io \ + trustyais.components.platform.opendatahub.io \ + workbenches.components.platform.opendatahub.io \ + --ignore-not-found 2>/dev/null || true + +# RHOAI core +oc delete crd \ + acceleratorprofiles.dashboard.opendatahub.io \ + datascienceclusters.datasciencecluster.opendatahub.io \ + dscinitializations.dscinitialization.opendatahub.io \ + featuretrackers.features.opendatahub.io \ + gatewayconfigs.services.platform.opendatahub.io \ + hardwareprofiles.dashboard.opendatahub.io \ + hardwareprofiles.infrastructure.opendatahub.io \ + modelsasservices.components.platform.opendatahub.io \ + odhapplications.dashboard.opendatahub.io \ + odhdashboardconfigs.opendatahub.io \ + odhdocuments.dashboard.opendatahub.io \ + odhquickstarts.console.openshift.io \ + --ignore-not-found 2>/dev/null || true + +echo " Done" +echo "" + +echo "================================================" +echo "RHOAI cleanup completed" +echo "================================================" diff --git a/components/operators/openshift-ai/instance-2.x/components/cleanup-on-delete/delete-rhoai-resources.yaml b/components/operators/openshift-ai/instance-2.x/components/cleanup-on-delete/delete-rhoai-resources.yaml new file mode 100644 index 000000000..a533f0ce5 --- /dev/null +++ b/components/operators/openshift-ai/instance-2.x/components/cleanup-on-delete/delete-rhoai-resources.yaml @@ -0,0 +1,141 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + argocd.argoproj.io/hook: PreDelete + argocd.argoproj.io/sync-wave: "-1" + name: rhoai-cleanup + namespace: openshift-gitops +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + argocd.argoproj.io/hook: PreDelete + argocd.argoproj.io/sync-wave: "-1" + name: rhoai-cleanup +rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - delete + - get + - list + - apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - delete + - get + - list + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - delete + - get + - list + - apiGroups: + - operators.coreos.com + resources: + - clusterserviceversions + - installplans + - operatorgroups + - subscriptions + verbs: + - delete + - get + - list + - apiGroups: + - components.platform.opendatahub.io + - dashboard.opendatahub.io + - datasciencecluster.opendatahub.io + - datasciencepipelinesapplications.opendatahub.io + - dscinitialization.opendatahub.io + - features.opendatahub.io + - infrastructure.opendatahub.io + - modelregistry.opendatahub.io + - nim.opendatahub.io + - opendatahub.io + - services.platform.opendatahub.io + - trustyai.opendatahub.io + resources: + - "*" + verbs: + - delete + - get + - list + - apiGroups: + - app.k8s.io + - argoproj.io + - console.openshift.io + - feast.dev + - inference.networking.x-k8s.io + - kubeflow.org + - kueue.x-k8s.io + - llamastack.io + - ray.io + - serving.kserve.io + - workload.codeflare.dev + resources: + - "*" + verbs: + - delete + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + argocd.argoproj.io/hook: PreDelete + argocd.argoproj.io/sync-wave: "-1" + name: rhoai-cleanup +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rhoai-cleanup +subjects: + - kind: ServiceAccount + name: rhoai-cleanup + namespace: openshift-gitops +--- +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + argocd.argoproj.io/hook: PreDelete + argocd.argoproj.io/hook-delete-policy: HookSucceeded,HookFailed + argocd.argoproj.io/sync-options: Force=true + argocd.argoproj.io/sync-wave: "0" + name: delete-rhoai-resources + namespace: openshift-gitops +spec: + backoffLimit: 0 + template: + metadata: + name: delete-rhoai-resources + spec: + containers: + - command: + - /bin/bash + - /scripts/delete-rhoai-resources.sh + image: registry.redhat.io/openshift4/ose-cli:latest + name: delete-rhoai-resources + volumeMounts: + - mountPath: /scripts + name: scripts + restartPolicy: Never + serviceAccountName: rhoai-cleanup + terminationGracePeriodSeconds: 30 + volumes: + - configMap: + defaultMode: 0755 + name: rhoai-cleanup-script + name: scripts diff --git a/components/operators/openshift-ai/instance-2.x/components/cleanup-on-delete/kustomization.yaml b/components/operators/openshift-ai/instance-2.x/components/cleanup-on-delete/kustomization.yaml new file mode 100644 index 000000000..cb0b406c7 --- /dev/null +++ b/components/operators/openshift-ai/instance-2.x/components/cleanup-on-delete/kustomization.yaml @@ -0,0 +1,26 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: + - files: + - delete-rhoai-resources.sh + name: rhoai-cleanup-script + namespace: openshift-gitops + +patches: + - patch: |- + - op: add + path: /metadata/annotations + value: + argocd.argoproj.io/hook: PreDelete + argocd.argoproj.io/hook-delete-policy: HookSucceeded,HookFailed + argocd.argoproj.io/sync-wave: "-1" + target: + kind: ConfigMap + name: rhoai-cleanup-script + +resources: + - delete-rhoai-resources.yaml diff --git a/components/operators/openshift-ai/instance-2.x/overlays/eus-2.16-modelregistry-only/kustomization.yaml b/components/operators/openshift-ai/instance-2.x/overlays/eus-2.16-modelregistry-only/kustomization.yaml index 15d4350e3..ae59a52b1 100644 --- a/components/operators/openshift-ai/instance-2.x/overlays/eus-2.16-modelregistry-only/kustomization.yaml +++ b/components/operators/openshift-ai/instance-2.x/overlays/eus-2.16-modelregistry-only/kustomization.yaml @@ -5,6 +5,7 @@ resources: - ../../base components: + - ../../components/cleanup-on-delete - ../../components/components-modelregistry - ../../components/rhoai-dashboard-access - ../../components/wait-for-servicemesh diff --git a/components/operators/openshift-ai/instance-2.x/overlays/eus-2.16-nvidia-gpu/kustomization.yaml b/components/operators/openshift-ai/instance-2.x/overlays/eus-2.16-nvidia-gpu/kustomization.yaml index 08d0e8fba..9fc8ccdd5 100644 --- a/components/operators/openshift-ai/instance-2.x/overlays/eus-2.16-nvidia-gpu/kustomization.yaml +++ b/components/operators/openshift-ai/instance-2.x/overlays/eus-2.16-nvidia-gpu/kustomization.yaml @@ -9,6 +9,7 @@ resources: components: - ../../components/anythingllm-image - ../../components/auth-with-authorino + - ../../components/cleanup-on-delete - ../../components/components-kserve - ../../components/components-modelmesh - ../../components/components-training diff --git a/components/operators/openshift-ai/instance-2.x/overlays/eus-2.16/kustomization.yaml b/components/operators/openshift-ai/instance-2.x/overlays/eus-2.16/kustomization.yaml index ab5fe1576..a5ab87098 100644 --- a/components/operators/openshift-ai/instance-2.x/overlays/eus-2.16/kustomization.yaml +++ b/components/operators/openshift-ai/instance-2.x/overlays/eus-2.16/kustomization.yaml @@ -8,6 +8,7 @@ resources: components: - ../../components/auth-with-authorino + - ../../components/cleanup-on-delete - ../../components/components-kserve - ../../components/components-modelmesh - ../../components/components-training diff --git a/components/operators/openshift-ai/instance-2.x/overlays/eus-2.25-nvidia-gpu/kustomization.yaml b/components/operators/openshift-ai/instance-2.x/overlays/eus-2.25-nvidia-gpu/kustomization.yaml index 300d75ecb..35a2419a2 100644 --- a/components/operators/openshift-ai/instance-2.x/overlays/eus-2.25-nvidia-gpu/kustomization.yaml +++ b/components/operators/openshift-ai/instance-2.x/overlays/eus-2.25-nvidia-gpu/kustomization.yaml @@ -6,6 +6,7 @@ resources: components: - ../../components/anythingllm-image + - ../../components/cleanup-on-delete - ../../components/components-kserve-rawdeployments-only - ../../components/components-llamastack - ../../components/components-modelregistry diff --git a/components/operators/openshift-ai/instance-2.x/overlays/eus-2.25/kustomization.yaml b/components/operators/openshift-ai/instance-2.x/overlays/eus-2.25/kustomization.yaml index d4993c8b8..b0f299307 100644 --- a/components/operators/openshift-ai/instance-2.x/overlays/eus-2.25/kustomization.yaml +++ b/components/operators/openshift-ai/instance-2.x/overlays/eus-2.25/kustomization.yaml @@ -5,6 +5,7 @@ resources: - ../../base components: + - ../../components/cleanup-on-delete - ../../components/components-kserve-rawdeployments-only - ../../components/components-llamastack - ../../components/components-modelregistry diff --git a/components/operators/openshift-ai/instance-2.x/overlays/fast-nvidia-gpu/kustomization.yaml b/components/operators/openshift-ai/instance-2.x/overlays/fast-nvidia-gpu/kustomization.yaml index 300d75ecb..35a2419a2 100644 --- a/components/operators/openshift-ai/instance-2.x/overlays/fast-nvidia-gpu/kustomization.yaml +++ b/components/operators/openshift-ai/instance-2.x/overlays/fast-nvidia-gpu/kustomization.yaml @@ -6,6 +6,7 @@ resources: components: - ../../components/anythingllm-image + - ../../components/cleanup-on-delete - ../../components/components-kserve-rawdeployments-only - ../../components/components-llamastack - ../../components/components-modelregistry diff --git a/components/operators/openshift-ai/instance-2.x/overlays/fast/kustomization.yaml b/components/operators/openshift-ai/instance-2.x/overlays/fast/kustomization.yaml index d4993c8b8..b0f299307 100644 --- a/components/operators/openshift-ai/instance-2.x/overlays/fast/kustomization.yaml +++ b/components/operators/openshift-ai/instance-2.x/overlays/fast/kustomization.yaml @@ -5,6 +5,7 @@ resources: - ../../base components: + - ../../components/cleanup-on-delete - ../../components/components-kserve-rawdeployments-only - ../../components/components-llamastack - ../../components/components-modelregistry diff --git a/components/operators/openshift-ai/instance-3.x/components/cleanup-on-delete/README.md b/components/operators/openshift-ai/instance-3.x/components/cleanup-on-delete/README.md new file mode 100644 index 000000000..c2a2dd96b --- /dev/null +++ b/components/operators/openshift-ai/instance-3.x/components/cleanup-on-delete/README.md @@ -0,0 +1,44 @@ +# cleanup-on-delete + +Runs the [official Red Hat RHOAI uninstall procedure](https://docs.redhat.com/en/documentation/red_hat_openshift_ai_self-managed) automatically as an ArgoCD `PreDelete` hook when the RHOAI Application is deleted. + +Without this component, ArgoCD deletes RHOAI manifests in an arbitrary order, which leaves namespaces stuck in `Terminating` and webhooks that block future reinstalls. + +## What it does + +Executes a 5-step cleanup Job before ArgoCD removes any resources: + +1. Delete all user workload CRs across all namespaces (InferenceServices, Notebooks, RayJobs, etc.) — prevents finalizer deadlocks +2. Delete the `DataScienceCluster` and `DSCInitialization` CRs +3. Delete OLM resources: `Subscription`, `ClusterServiceVersion`, `OperatorGroup`, `InstallPlan` +4. Delete RHOAI `ValidatingWebhookConfiguration` and `MutatingWebhookConfiguration` resources +5. Delete RHOAI namespaces and CRDs (explicit list per Red Hat documentation) + +## How to enable + +Add this component to your instance overlay's `components:` list: + +```yaml +# components/operators/openshift-ai/instance-2.x/overlays/stable-2.25/kustomization.yaml +components: + - ../../components/cleanup-on-delete + - ../../components/components-kserve + # ... other components +``` + +## Resources created + +All resources are ArgoCD `PreDelete` hooks and are automatically cleaned up after the Job completes: + +- `ConfigMap/rhoai-cleanup-script` in `openshift-gitops` — mounts the cleanup script +- `ServiceAccount/rhoai-cleanup` in `openshift-gitops` +- `ClusterRole/rhoai-cleanup` +- `ClusterRoleBinding/rhoai-cleanup` +- `Job/delete-rhoai-resources` in `openshift-gitops` + +## Notes + +- The cleanup script is maintained in `delete-rhoai-resources.sh` and mounted into the Job via a ConfigMap +- `backoffLimit: 0` — the Job does not retry +- The Job runs in `openshift-gitops` (not `redhat-ods-operator`) to avoid being killed when RHOAI namespaces are deleted +- The Job exits with code `0` even when resources are not found, so ArgoCD always proceeds with Application deletion diff --git a/components/operators/openshift-ai/instance-3.x/components/cleanup-on-delete/delete-rhoai-resources.sh b/components/operators/openshift-ai/instance-3.x/components/cleanup-on-delete/delete-rhoai-resources.sh new file mode 100644 index 000000000..44d6eb246 --- /dev/null +++ b/components/operators/openshift-ai/instance-3.x/components/cleanup-on-delete/delete-rhoai-resources.sh @@ -0,0 +1,305 @@ +#!/usr/bin/env bash + +set -e + +echo "================================================" +echo "Red Hat OpenShift AI Cleanup" +echo "================================================" +echo "" +echo "Following: docs.redhat.com/en/documentation/red_hat_openshift_ai_self-managed" +echo "" + +# Step 0: Delete user workload CRs across all namespaces before removing the DSC. +# Deleting CRs first prevents finalizer deadlocks during operator cleanup. +echo "Step 0: Deleting user workload CRs..." + +# Data Science Pipelines +oc delete datasciencepipelinesapplications.datasciencepipelinesapplications.opendatahub.io \ + -A --all --ignore-not-found 2>/dev/null || true +oc delete datasciencepipelines.components.platform.opendatahub.io \ + -A --all --ignore-not-found 2>/dev/null || true + +# Argo Workflows +oc delete clusterworkflowtemplates.argoproj.io --all --ignore-not-found 2>/dev/null || true +oc delete cronworkflows.argoproj.io -A --all --ignore-not-found 2>/dev/null || true +oc delete workflowartifactgctasks.argoproj.io -A --all --ignore-not-found 2>/dev/null || true +oc delete workfloweventbindings.argoproj.io -A --all --ignore-not-found 2>/dev/null || true +oc delete workflows.argoproj.io -A --all --ignore-not-found 2>/dev/null || true +oc delete workflowtaskresults.argoproj.io -A --all --ignore-not-found 2>/dev/null || true +oc delete workflowtasksets.argoproj.io -A --all --ignore-not-found 2>/dev/null || true +oc delete workflowtemplates.argoproj.io -A --all --ignore-not-found 2>/dev/null || true + +# KServe +oc delete clusterstoragecontainers.serving.kserve.io --all --ignore-not-found 2>/dev/null || true +oc delete inferencegraphs.serving.kserve.io -A --all --ignore-not-found 2>/dev/null || true +oc delete inferenceservices.serving.kserve.io -A --all --ignore-not-found 2>/dev/null || true +oc delete llminferenceserviceconfigs.serving.kserve.io -A --all --ignore-not-found 2>/dev/null || true +oc delete llminferenceservices.serving.kserve.io -A --all --ignore-not-found 2>/dev/null || true +oc delete predictors.serving.kserve.io -A --all --ignore-not-found 2>/dev/null || true +oc delete servingruntimes.serving.kserve.io -A --all --ignore-not-found 2>/dev/null || true +oc delete trainedmodels.serving.kserve.io -A --all --ignore-not-found 2>/dev/null || true + +# Inference (networking) +oc delete inferencemodels.inference.networking.x-k8s.io -A --all --ignore-not-found 2>/dev/null || true +oc delete inferencepools.inference.networking.x-k8s.io -A --all --ignore-not-found 2>/dev/null || true + +# Kueue +oc delete admissionchecks.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete clusterqueues.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete cohorts.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete localqueues.kueue.x-k8s.io -A --all --ignore-not-found 2>/dev/null || true +oc delete multikueueclusters.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete multikueueconfigs.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete provisioningrequestconfigs.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete resourceflavors.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete topologies.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete workloadpriorityclasses.kueue.x-k8s.io --all --ignore-not-found 2>/dev/null || true +oc delete workloads.kueue.x-k8s.io -A --all --ignore-not-found 2>/dev/null || true + +# Workbenches +oc delete notebooks.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true + +# Kubeflow Training Operator +oc delete jaxjobs.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete mpijobs.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete mxjobs.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete paddlejobs.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete pipelines.pipelines.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete pipelineversions.pipelines.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete pytorchjobs.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete scheduledworkflows.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete tfjobs.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete viewers.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true +oc delete xgboostjobs.kubeflow.org -A --all --ignore-not-found 2>/dev/null || true + +# TrustyAI +oc delete guardrailsorchestrators.trustyai.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete lmevaljobs.trustyai.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete nemoguardrails.trustyai.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete trustyaiservices.trustyai.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true + +# Ray +oc delete rayclusters.ray.io -A --all --ignore-not-found 2>/dev/null || true +oc delete rayjobs.ray.io -A --all --ignore-not-found 2>/dev/null || true +oc delete rayservices.ray.io -A --all --ignore-not-found 2>/dev/null || true + +# CodeFlare / AppWrappers +oc delete appwrappers.workload.codeflare.dev -A --all --ignore-not-found 2>/dev/null || true + +# Feast +oc delete featurestores.feast.dev -A --all --ignore-not-found 2>/dev/null || true + +# LlamaStack +oc delete llamastackdistributions.llamastack.io -A --all --ignore-not-found 2>/dev/null || true + +# Model Registry +oc delete modelregistries.modelregistry.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true + +# RHOAI / ODH platform CRs +oc delete accounts.nim.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete applications.app.k8s.io -A --all --ignore-not-found 2>/dev/null || true +oc delete auths.services.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete dashboards.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete feastoperators.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete kserves.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete kueues.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete llamastackoperators.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete modelcontrollers.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete modelmeshservings.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete modelregistries.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete monitorings.services.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete rays.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete servicemeshes.services.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete sparkoperators.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete trainingoperators.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete trustyais.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true +oc delete workbenches.components.platform.opendatahub.io -A --all --ignore-not-found 2>/dev/null || true + +echo " Done" +echo "" + +# Step 1: Delete DataScienceCluster and DSCInitialization +echo "Step 1: Deleting DataScienceCluster and DSCInitialization..." +oc delete datascienceclusters.datasciencecluster.opendatahub.io --all --ignore-not-found 2>/dev/null || true +oc delete dscinitializations.dscinitialization.opendatahub.io --all --ignore-not-found 2>/dev/null || true +oc delete featuretrackers.features.opendatahub.io --all --ignore-not-found 2>/dev/null || true +echo " Done" +echo "" + +# Step 2: Delete OLM resources +echo "Step 2: Deleting Subscription, CSV, OperatorGroup, InstallPlan..." +oc delete subscription --all -n redhat-ods-operator --ignore-not-found +oc delete clusterserviceversion --all -n redhat-ods-operator --ignore-not-found +oc delete operatorgroup --all -n redhat-ods-operator --ignore-not-found +oc delete installplan --all -n redhat-ods-operator --ignore-not-found +echo " Done" +echo "" + +# Step 3: Delete webhooks +echo "Step 3: Deleting RHOAI webhooks..." +oc get validatingwebhookconfiguration -o name 2>/dev/null \ + | grep -E "opendatahub|kserve|rhoai" \ + | xargs -r oc delete --ignore-not-found 2>/dev/null || true +oc get mutatingwebhookconfiguration -o name 2>/dev/null \ + | grep -E "opendatahub|kserve|rhoai" \ + | xargs -r oc delete --ignore-not-found 2>/dev/null || true +echo " Done" +echo "" + +# Step 4: Delete RHOAI namespaces +echo "Step 4: Deleting RHOAI namespaces..." +oc delete namespace redhat-ods-applications --ignore-not-found 2>/dev/null || true +oc delete namespace redhat-ods-monitoring --ignore-not-found 2>/dev/null || true +oc delete namespace redhat-ods-operator --ignore-not-found 2>/dev/null || true +oc delete namespace rhods-notebooks --ignore-not-found 2>/dev/null || true +echo " Done" +echo "" + +# Step 5: Delete RHOAI CRDs +echo "Step 5: Deleting RHOAI CRDs..." + +# Data Science Pipelines +oc delete crd \ + datasciencepipelinesapplications.datasciencepipelinesapplications.opendatahub.io \ + datasciencepipelines.components.platform.opendatahub.io \ + --ignore-not-found 2>/dev/null || true + +# Argo Workflows +oc delete crd \ + clusterworkflowtemplates.argoproj.io \ + cronworkflows.argoproj.io \ + workflowartifactgctasks.argoproj.io \ + workfloweventbindings.argoproj.io \ + workflows.argoproj.io \ + workflowtaskresults.argoproj.io \ + workflowtasksets.argoproj.io \ + workflowtemplates.argoproj.io \ + --ignore-not-found 2>/dev/null || true + +# KServe +oc delete crd \ + clusterstoragecontainers.serving.kserve.io \ + inferencegraphs.serving.kserve.io \ + inferenceservices.serving.kserve.io \ + llminferenceserviceconfigs.serving.kserve.io \ + llminferenceservices.serving.kserve.io \ + predictors.serving.kserve.io \ + servingruntimes.serving.kserve.io \ + trainedmodels.serving.kserve.io \ + --ignore-not-found 2>/dev/null || true + +# Inference (networking) +oc delete crd \ + inferencemodels.inference.networking.x-k8s.io \ + inferencepools.inference.networking.x-k8s.io \ + --ignore-not-found 2>/dev/null || true + +# Kueue +oc delete crd \ + admissionchecks.kueue.x-k8s.io \ + clusterqueues.kueue.x-k8s.io \ + cohorts.kueue.x-k8s.io \ + localqueues.kueue.x-k8s.io \ + multikueueclusters.kueue.x-k8s.io \ + multikueueconfigs.kueue.x-k8s.io \ + provisioningrequestconfigs.kueue.x-k8s.io \ + resourceflavors.kueue.x-k8s.io \ + topologies.kueue.x-k8s.io \ + workloadpriorityclasses.kueue.x-k8s.io \ + workloads.kueue.x-k8s.io \ + --ignore-not-found 2>/dev/null || true + +# Workbenches / Kubeflow +oc delete crd \ + notebooks.kubeflow.org \ + viewers.kubeflow.org \ + --ignore-not-found 2>/dev/null || true + +# Kubeflow Training Operator +oc delete crd \ + jaxjobs.kubeflow.org \ + mpijobs.kubeflow.org \ + mxjobs.kubeflow.org \ + paddlejobs.kubeflow.org \ + pipelines.pipelines.kubeflow.org \ + pipelineversions.pipelines.kubeflow.org \ + pytorchjobs.kubeflow.org \ + scheduledworkflows.kubeflow.org \ + tfjobs.kubeflow.org \ + xgboostjobs.kubeflow.org \ + --ignore-not-found 2>/dev/null || true + +# TrustyAI +oc delete crd \ + guardrailsorchestrators.trustyai.opendatahub.io \ + lmevaljobs.trustyai.opendatahub.io \ + nemoguardrails.trustyai.opendatahub.io \ + trustyaiservices.trustyai.opendatahub.io \ + --ignore-not-found 2>/dev/null || true + +# Ray +oc delete crd \ + rayclusters.ray.io \ + rayjobs.ray.io \ + rayservices.ray.io \ + --ignore-not-found 2>/dev/null || true + +# CodeFlare +oc delete crd appwrappers.workload.codeflare.dev --ignore-not-found 2>/dev/null || true + +# Feast +oc delete crd \ + feastoperators.components.platform.opendatahub.io \ + featurestores.feast.dev \ + --ignore-not-found 2>/dev/null || true + +# LlamaStack +oc delete crd llamastackdistributions.llamastack.io --ignore-not-found 2>/dev/null || true + +# RHOAI / ODH platform components +oc delete crd \ + accounts.nim.opendatahub.io \ + applications.app.k8s.io \ + auths.services.platform.opendatahub.io \ + codeflares.components.platform.opendatahub.io \ + dashboards.components.platform.opendatahub.io \ + kserves.components.platform.opendatahub.io \ + kueues.components.platform.opendatahub.io \ + llamastackoperators.components.platform.opendatahub.io \ + mlflowoperators.components.platform.opendatahub.io \ + modelcontrollers.components.platform.opendatahub.io \ + modelmeshservings.components.platform.opendatahub.io \ + modelregistries.components.platform.opendatahub.io \ + modelregistries.modelregistry.opendatahub.io \ + monitorings.services.platform.opendatahub.io \ + rays.components.platform.opendatahub.io \ + servicemeshes.services.platform.opendatahub.io \ + sparkoperators.components.platform.opendatahub.io \ + trainingoperators.components.platform.opendatahub.io \ + trainers.components.platform.opendatahub.io \ + trustyais.components.platform.opendatahub.io \ + workbenches.components.platform.opendatahub.io \ + --ignore-not-found 2>/dev/null || true + +# RHOAI core +oc delete crd \ + acceleratorprofiles.dashboard.opendatahub.io \ + datascienceclusters.datasciencecluster.opendatahub.io \ + dscinitializations.dscinitialization.opendatahub.io \ + featuretrackers.features.opendatahub.io \ + gatewayconfigs.services.platform.opendatahub.io \ + hardwareprofiles.dashboard.opendatahub.io \ + hardwareprofiles.infrastructure.opendatahub.io \ + modelsasservices.components.platform.opendatahub.io \ + odhapplications.dashboard.opendatahub.io \ + odhdashboardconfigs.opendatahub.io \ + odhdocuments.dashboard.opendatahub.io \ + odhquickstarts.console.openshift.io \ + --ignore-not-found 2>/dev/null || true + +echo " Done" +echo "" + +echo "================================================" +echo "RHOAI cleanup completed" +echo "================================================" diff --git a/components/operators/openshift-ai/instance-3.x/components/cleanup-on-delete/delete-rhoai-resources.yaml b/components/operators/openshift-ai/instance-3.x/components/cleanup-on-delete/delete-rhoai-resources.yaml new file mode 100644 index 000000000..a533f0ce5 --- /dev/null +++ b/components/operators/openshift-ai/instance-3.x/components/cleanup-on-delete/delete-rhoai-resources.yaml @@ -0,0 +1,141 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + argocd.argoproj.io/hook: PreDelete + argocd.argoproj.io/sync-wave: "-1" + name: rhoai-cleanup + namespace: openshift-gitops +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + argocd.argoproj.io/hook: PreDelete + argocd.argoproj.io/sync-wave: "-1" + name: rhoai-cleanup +rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - delete + - get + - list + - apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - delete + - get + - list + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - delete + - get + - list + - apiGroups: + - operators.coreos.com + resources: + - clusterserviceversions + - installplans + - operatorgroups + - subscriptions + verbs: + - delete + - get + - list + - apiGroups: + - components.platform.opendatahub.io + - dashboard.opendatahub.io + - datasciencecluster.opendatahub.io + - datasciencepipelinesapplications.opendatahub.io + - dscinitialization.opendatahub.io + - features.opendatahub.io + - infrastructure.opendatahub.io + - modelregistry.opendatahub.io + - nim.opendatahub.io + - opendatahub.io + - services.platform.opendatahub.io + - trustyai.opendatahub.io + resources: + - "*" + verbs: + - delete + - get + - list + - apiGroups: + - app.k8s.io + - argoproj.io + - console.openshift.io + - feast.dev + - inference.networking.x-k8s.io + - kubeflow.org + - kueue.x-k8s.io + - llamastack.io + - ray.io + - serving.kserve.io + - workload.codeflare.dev + resources: + - "*" + verbs: + - delete + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: + argocd.argoproj.io/hook: PreDelete + argocd.argoproj.io/sync-wave: "-1" + name: rhoai-cleanup +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rhoai-cleanup +subjects: + - kind: ServiceAccount + name: rhoai-cleanup + namespace: openshift-gitops +--- +apiVersion: batch/v1 +kind: Job +metadata: + annotations: + argocd.argoproj.io/hook: PreDelete + argocd.argoproj.io/hook-delete-policy: HookSucceeded,HookFailed + argocd.argoproj.io/sync-options: Force=true + argocd.argoproj.io/sync-wave: "0" + name: delete-rhoai-resources + namespace: openshift-gitops +spec: + backoffLimit: 0 + template: + metadata: + name: delete-rhoai-resources + spec: + containers: + - command: + - /bin/bash + - /scripts/delete-rhoai-resources.sh + image: registry.redhat.io/openshift4/ose-cli:latest + name: delete-rhoai-resources + volumeMounts: + - mountPath: /scripts + name: scripts + restartPolicy: Never + serviceAccountName: rhoai-cleanup + terminationGracePeriodSeconds: 30 + volumes: + - configMap: + defaultMode: 0755 + name: rhoai-cleanup-script + name: scripts diff --git a/components/operators/openshift-ai/instance-3.x/components/cleanup-on-delete/kustomization.yaml b/components/operators/openshift-ai/instance-3.x/components/cleanup-on-delete/kustomization.yaml new file mode 100644 index 000000000..cb0b406c7 --- /dev/null +++ b/components/operators/openshift-ai/instance-3.x/components/cleanup-on-delete/kustomization.yaml @@ -0,0 +1,26 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: + - files: + - delete-rhoai-resources.sh + name: rhoai-cleanup-script + namespace: openshift-gitops + +patches: + - patch: |- + - op: add + path: /metadata/annotations + value: + argocd.argoproj.io/hook: PreDelete + argocd.argoproj.io/hook-delete-policy: HookSucceeded,HookFailed + argocd.argoproj.io/sync-wave: "-1" + target: + kind: ConfigMap + name: rhoai-cleanup-script + +resources: + - delete-rhoai-resources.yaml diff --git a/components/operators/openshift-ai/instance-3.x/overlays/stable-3.3-nvidia-gpu/kustomization.yaml b/components/operators/openshift-ai/instance-3.x/overlays/stable-3.3-nvidia-gpu/kustomization.yaml index 58910727f..630eb7fd5 100644 --- a/components/operators/openshift-ai/instance-3.x/overlays/stable-3.3-nvidia-gpu/kustomization.yaml +++ b/components/operators/openshift-ai/instance-3.x/overlays/stable-3.3-nvidia-gpu/kustomization.yaml @@ -5,6 +5,7 @@ resources: - ../../base components: + - ../../components/cleanup-on-delete - ../../components/components-aipipelines - ../../components/components-feast - ../../components/components-kserve