fix race condition between forget and failover

internal/controller/valkeycluster_controller.go

@@ -805,14 +805,25 @@ func (r *ValkeyClusterReconciler) forgetStaleNodes(ctx context.Context, cluster
 				idx := slices.IndexFunc(nodes.Items, func(n valkeyiov1alpha1.ValkeyNode) bool {
 					return n.Status.PodIP == failing.Address
 				})
-				if idx == -1 {
-					log.V(1).Info("forget a failing node", "address", failing.Address, "Id", failing.Id)
-					if err := node.Client.Do(ctx, node.Client.B().ClusterForget().NodeId(failing.Id).Build()).Error(); err != nil {
-						log.Error(err, "command failed: CLUSTER FORGET")
-						r.Recorder.Eventf(cluster, nil, corev1.EventTypeWarning, "NodeForgetFailed", "ForgetNode", "Failed to forget node: %v", err)
-					} else {
-						r.Recorder.Eventf(cluster, nil, corev1.EventTypeNormal, "StaleNodeForgotten", "ForgetNode", "Forgot stale node %v", failing.Address)
-					}
+				if idx != -1 {
+					continue
+				}
+				// A live replica still considers this failing node its
+				// primary. Forgetting it from the other primaries now would
+				// remove it from their node tables and prevent them from
+				// voting in the auto-failover election, permanently
+				// blocking the replica's promotion.
+				if state.HasReplicaOf(failing.Id) {
+					log.V(1).Info("skipping forget; failover pending for node",
+						"address", failing.Address, "Id", failing.Id)
+					continue
+				}
+				log.V(1).Info("forget a failing node", "address", failing.Address, "Id", failing.Id)
+				if err := node.Client.Do(ctx, node.Client.B().ClusterForget().NodeId(failing.Id).Build()).Error(); err != nil {
+					log.Error(err, "command failed: CLUSTER FORGET")
+					r.Recorder.Eventf(cluster, nil, corev1.EventTypeWarning, "NodeForgetFailed", "ForgetNode", "Failed to forget node: %v", err)
+				} else {
+					r.Recorder.Eventf(cluster, nil, corev1.EventTypeNormal, "StaleNodeForgotten", "ForgetNode", "Forgot stale node %v", failing.Address)
 				}
 			}
 		}

internal/valkey/clusterstate.go

@@ -193,6 +193,38 @@ func (n *NodeState) IsReplicationInSync() bool {
 	return n.Info["master_link_status"] == "up"
 }
 
+// HasReplicaOf returns true if any live node in the cluster state reports
+// itself as a replica of the given node ID. This is used to prevent
+// CLUSTER FORGET from racing with auto-failover: forgetting a failed
+// primary from other primaries removes it from their node tables, which
+// prevents them from voting in the replica's failover election.
+func (s *ClusterState) HasReplicaOf(nodeId string) bool {
+	for _, shard := range s.Shards {
+		for _, node := range shard.Nodes {
+			if node.PrimaryIdFromSelf() == nodeId {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// PrimaryIdFromSelf returns the primary node ID that this node reports as its
+// own primary in CLUSTER NODES (fields[3] of the "myself" line). Returns "-"
+// for primaries and the primary's node ID for replicas.
+func (n *NodeState) PrimaryIdFromSelf() string {
+	for line := range strings.SplitSeq(n.ClusterNodes, "\n") {
+		fields := strings.Fields(line)
+		if len(fields) < 8 {
+			continue
+		}
+		if strings.Contains(fields[2], "myself") {
+			return fields[3]
+		}
+	}
+	return ""
+}
+
 // GetFailingNodes returns all known nodes that are failing.
 func (n *NodeState) GetFailingNodes() []NodeState {
 	nodes := []NodeState{}