Skip to content

[Cluster] Refactor: Add FinalizeDeleteCluster for async cluster lifecycle management #1673

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
powerkimhub wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[Cluster] Refactor: Add FinalizeDeleteCluster for async cluster lifecycle management #1673

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 104 additions & 8 deletions api-runtime/common-runtime/ClusterManager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -960,8 +960,11 @@ func ListCluster(connectionName string, rsType string, kubeconfigType string) ([
if err != nil {
clusterSPLock.RUnlock(connectionName, iidInfo.NameId)
if checkNotFoundError(err) {
cblog.Error(err)
info = cres.ClusterInfo{IId: cres.IID{NameId: iidInfo.NameId, SystemId: iidInfo.SystemId}}
cblog.Infof("Cluster '%s' not found on CSP, marking as NotFound", iidInfo.NameId)
info = cres.ClusterInfo{
IId: cres.IID{NameId: iidInfo.NameId, SystemId: iidInfo.SystemId},
Status: cres.ClusterNotFound,
}
infoList2 = append(infoList2, &info)
continue
}
Expand Down Expand Up @@ -1078,6 +1081,14 @@ func GetCluster(connectionName string, rsType string, clusterName string, kubeco
// (2) get resource(SystemId)
info, err := handler.GetCluster(getDriverIID(cres.IID{NameId: iidInfo.NameId, SystemId: iidInfo.SystemId}))
if err != nil {
if checkNotFoundError(err) {
cblog.Info("Cluster not found on CSP, returning NotFound status")
notFoundInfo := cres.ClusterInfo{
IId: getUserIID(cres.IID{NameId: iidInfo.NameId, SystemId: iidInfo.SystemId}),
Status: cres.ClusterNotFound,
}
return &notFoundInfo, nil
}
cblog.Error(err)
return nil, err
}
Expand Down Expand Up @@ -1985,25 +1996,110 @@ func DeleteCluster(connectionName string, rsType string, nameID string, force st
}
}

// (3) delete IID
_, err = infostore.DeleteByConditions(&ClusterIIDInfo{}, CONNECTION_NAME_COLUMN, iidInfo.ConnectionName, NAME_ID_COLUMN, nameID)
return result, nil
}

// FinalizeDeleteCluster deletes the Spider meta information for a Cluster
// only when the Cluster no longer exists on the CSP.
// (1) Check the Cluster existence in MetaDB
// (2) Check the Cluster existence on CSP via GetCluster()
// (3) If the Cluster does not exist on CSP, delete the meta information
func FinalizeDeleteCluster(connectionName string, rsType string, nameID string) (bool, error) {
cblog.Info("call FinalizeDeleteCluster()")

// check empty and trim user inputs
connectionName, err := EmptyCheckAndTrim("connectionName", connectionName)
if err != nil {
cblog.Error(err)
if force != "true" {
return false, err
}

if err := checkCapability(connectionName, CLUSTER_HANDLER); err != nil {
return false, err
}

nameID, err = EmptyCheckAndTrim("nameID", nameID)
if err != nil {
cblog.Error(err)
return false, err
}

cldConn, err := ccm.GetCloudConnection(connectionName)
if err != nil {
cblog.Error(err)
return false, err
}

handler, err := cldConn.CreateClusterHandler()
if err != nil {
cblog.Error(err)
return false, err
}

clusterSPLock.Lock(connectionName, nameID)
defer clusterSPLock.Unlock(connectionName, nameID)

// (1) get spiderIID for creating driverIID
var iidInfo *ClusterIIDInfo
var iidInfoList []*ClusterIIDInfo
if os.Getenv("PERMISSION_BASED_CONTROL_MODE") != "" {
err = getAuthIIDInfoList(connectionName, &iidInfoList)
if err != nil {
cblog.Error(err)
return false, err
}
} else {
err = infostore.ListByCondition(&iidInfoList, CONNECTION_NAME_COLUMN, connectionName)
if err != nil {
cblog.Error(err)
return false, err
}
}
var found = false
for _, OneIIdInfo := range iidInfoList {
if OneIIdInfo.NameId == nameID {
iidInfo = OneIIdInfo
found = true
break
}
}
if !found {
err := fmt.Errorf("%s '%s' does not exist in Spider's MetaDB for connection '%s'", RSTypeString(rsType), nameID, connectionName)
cblog.Error(err)
return false, err
}

// for NodeGroup list
// delete all nodegroups of target Cluster
// (2) Check the Cluster existence on CSP via GetCluster()
driverIId := getDriverIID(cres.IID{NameId: iidInfo.NameId, SystemId: iidInfo.SystemId})
_, err = handler.(cres.ClusterHandler).GetCluster(driverIId)
if err != nil {
if !checkNotFoundError(err) {
// unexpected error from CSP
cblog.Error(err)
return false, fmt.Errorf("failed to check Cluster existence on CSP: %w", err)
}
// Cluster does not exist on CSP => proceed to finalize
} else {
// Cluster still exists on CSP => cannot finalize
return false, fmt.Errorf("cannot finalize: Cluster '%s' still exists on CSP (status is not %s)", nameID, cres.ClusterNotFound)
}

// (3) delete IID from MetaDB
_, err = infostore.DeleteByConditions(&ClusterIIDInfo{}, CONNECTION_NAME_COLUMN, iidInfo.ConnectionName, NAME_ID_COLUMN, nameID)
if err != nil {
cblog.Error(err)
return false, err
}

// delete all nodegroups of target Cluster from MetaDB
_, err = infostore.DeleteByConditions(&NodeGroupIIDInfo{}, CONNECTION_NAME_COLUMN, iidInfo.ConnectionName,
OWNER_CLUSTER_NAME_COLUMN, iidInfo.NameId)
if err != nil {
cblog.Error(err)
return false, err
}

return result, nil
return true, nil
}

func CountAllClusters() (int64, error) {
Expand Down
1 change: 1 addition & 0 deletions api-runtime/rest-runtime/CBSpiderRuntime.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,6 +473,7 @@ func getRoutes() []route {
{"GET", "/cluster", ListCluster},
{"GET", "/cluster/:Name", GetCluster},
{"DELETE", "/cluster/:Name", DeleteCluster},
{"DELETE", "/cluster/:Name/finalize", FinalizeDeleteCluster},
{"GET", "/cluster/:Name/token", GetClusterToken},
//-- for NodeGroup
{"POST", "/cluster/:Name/nodegroup", AddNodeGroup},
Expand Down
42 changes: 41 additions & 1 deletion api-runtime/rest-runtime/ClusterRest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -549,7 +549,7 @@ func ChangeNodeGroupScaling(c echo.Context) error {
// deleteCluster godoc
// @ID delete-cluster
// @Summary Delete Cluster
// @Description Delete a specified Cluster.
// @Description Delete a specified Cluster from the CSP. <br> This API only deletes the CSP resource and does not remove Spider meta information. <br> After deletion, call **DELETE /cluster/{Name}/finalize** to clean up Spider's internal metadata once the CSP resource no longer exists.
// @Tags [Cluster Management]
// @Accept json
// @Produce json
Expand Down Expand Up @@ -585,6 +585,46 @@ func DeleteCluster(c echo.Context) error {
return c.JSON(http.StatusOK, &resultInfo)
}

// finalizeDeleteCluster godoc
// @ID finalize-delete-cluster
// @Summary Finalize Delete Cluster
// @Description Finalize the deletion of a Cluster by removing its Spider meta information.
// @Description This API only succeeds when the Cluster no longer exists on the CSP.
// @Description Use this after DeleteCluster to clean up Spider's internal metadata.
// @Tags [Cluster Management]
// @Accept json
// @Produce json
// @Param ConnectionRequest body restruntime.ConnectionRequest true "Request body for finalizing Cluster deletion"
// @Param Name path string true "The name of the Cluster to finalize deletion"
// @Success 200 {object} BooleanInfo "Result of the finalize delete operation"
// @Failure 400 {object} SimpleMsg "Bad Request, possibly due to invalid JSON structure or missing fields"
// @Failure 404 {object} SimpleMsg "Resource Not Found"
// @Failure 500 {object} SimpleMsg "Internal Server Error"
// @Router /cluster/{Name}/finalize [delete]
func FinalizeDeleteCluster(c echo.Context) error {
cblog.Info("call FinalizeDeleteCluster()")

var req ConnectionRequest

if err := c.Bind(&req); err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
}

clusterName := c.Param("Name")

// Call common-runtime API
result, err := cmrt.FinalizeDeleteCluster(req.ConnectionName, CLUSTER, clusterName)
if err != nil {
return echo.NewHTTPError(http.StatusInternalServerError, err.Error())
}

resultInfo := BooleanInfo{
Result: strconv.FormatBool(result),
}

return c.JSON(http.StatusOK, &resultInfo)
}

// deleteCSPCluster godoc
// @ID delete-csp-cluster
// @Summary Delete CSP Cluster
Expand Down
Loading