Skip to content

Commit cc41886

Browse files
jianrongzhang89jianrongzhang89
authored
DBAAS-906 [RHODA][R4] Operator Metrics - Instance with Pending status on Thanos loaded on Grafana with status True (#62)
* DBAAS-906 Operator Metrics - Instance with Pending status on Thanos loaded on Grafana with status True * DBAAS-906 Operator Metrics - Instance with Pending status on Thanos loaded on Grafana with status True: address review comments * DBAAS-906 Operator Metrics - Instance with Pending status on Thanos loaded on Grafana with status True: address review comments
1 parent 0aca5b6 commit cc41886

File tree

5 files changed

+88
-26
lines changed

5 files changed

+88
-26
lines changed

‎cmd/manager/main.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,11 @@ func main() {
135135
watch.SelectNamespacesPredicate(config.WatchedNamespaces), // select only desired namespaces
136136
}
137137

138+
globalPredicatesWithAnnotations := []predicate.Predicate{
139+
watch.CommonPredicatesWithAnnotations(), // ignore spurious changes. status changes etc but allow annotation changes
140+
watch.SelectNamespacesPredicate(config.WatchedNamespaces), // select only desired namespaces
141+
}
142+
138143
cfg := mgr.GetConfig()
139144
clientset, err := kubernetes.NewForConfig(cfg)
140145
if err != nil {
@@ -200,7 +205,7 @@ func main() {
200205
AtlasDomain: config.AtlasDomain,
201206
GlobalAPISecret: config.GlobalAPISecret,
202207
ResourceWatcher: watch.NewResourceWatcher(),
203-
GlobalPredicates: globalPredicates,
208+
GlobalPredicates: globalPredicatesWithAnnotations,
204209
EventRecorder: mgr.GetEventRecorderFor("AtlasDeployment"),
205210
}).SetupWithManager(mgr); err != nil {
206211
setupLog.Error(err, "unable to create controller", "controller", "AtlasDeployment")

‎config/manager/manager.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,5 +60,5 @@ spec:
6060
fieldRef:
6161
fieldPath: metadata.namespace
6262
- name: SYNC_PERIOD_MIN
63-
value: 180
63+
value: "180"
6464
terminationGracePeriodSeconds: 10

‎pkg/controller/atlasinstance/atlasinstance_controller.go‎

Lines changed: 32 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"errors"
2222
"fmt"
2323
"strings"
24+
"time"
2425

2526
"go.uber.org/zap"
2627

@@ -58,6 +59,12 @@ import (
5859
"github.com/mongodb/mongodb-atlas-kubernetes/pkg/util/kube"
5960
)
6061

62+
const (
63+
instancePhaseChangedInAtlas = "InstancePhaseChangedInAtlas"
64+
instancePhaseChangedInAtlasMsg = "db instance phase has changed in Atlas"
65+
updateAnnotationKey = "atlas.mongodb.com/updated-at"
66+
)
67+
6168
// MongoDBAtlasInstanceReconciler reconciles a MongoDBAtlasInstance object
6269
type MongoDBAtlasInstanceReconciler struct {
6370
Client client.Client
@@ -227,11 +234,17 @@ func (r *MongoDBAtlasInstanceReconciler) reconcileAtlasDeployment(cx context.Con
227234
return ctrl.Result{}, err
228235
}
229236

230-
result := setInstanceStatusWithDeploymentInfo(atlasClient, inst, atlasDeployment, instData.ProjectName)
237+
stateChangedInAtlas, result := setInstanceStatusWithDeploymentInfo(atlasClient, inst, atlasDeployment, instData.ProjectName)
231238
if !result.IsOk() {
239+
if stateChangedInAtlas {
240+
// Update an annotation in the atlas deployment resource to trigger its reconciliation
241+
log.Infof("Trigger AtlasDeployment reconciliation. Reason: %v", result.Message())
242+
_ = r.annotateAtlasDeployment(cx, atlasDeployment)
243+
}
232244
log.Infof("Error setting instance status: %v", result.Message())
233245
return ctrl.Result{}, errors.New(result.Message())
234246
}
247+
235248
return ctrl.Result{}, nil
236249
}
237250

@@ -358,6 +371,16 @@ func (r *MongoDBAtlasInstanceReconciler) getAtlasProjectForCreation(instance *db
358371
}, nil
359372
}
360373

374+
func (r *MongoDBAtlasInstanceReconciler) annotateAtlasDeployment(cx context.Context, atlasDeployment *v1.AtlasDeployment) error {
375+
annotations := atlasDeployment.GetAnnotations()
376+
if annotations == nil {
377+
annotations = make(map[string]string)
378+
}
379+
annotations[updateAnnotationKey] = time.Now().Format(time.RFC3339)
380+
atlasDeployment.SetAnnotations(annotations)
381+
return r.Client.Update(cx, atlasDeployment, &client.UpdateOptions{})
382+
}
383+
361384
// getAtlasDeploymentSpec returns the spec for the desired cluster
362385
func getAtlasDeploymentSpec(atlasProject *v1.AtlasProject, data *InstanceData) *v1.AtlasDeploymentSpec {
363386
var providerSettingsSpec *v1.ProviderSettingsSpec
@@ -461,7 +484,7 @@ func instanceMutateFn(atlasProject *v1.AtlasProject, atlasDeployment *v1.AtlasDe
461484
}
462485
}
463486

464-
func setInstanceStatusWithDeploymentInfo(atlasClient *mongodbatlas.Client, inst *dbaas.MongoDBAtlasInstance, atlasDeployment *v1.AtlasDeployment, project string) workflow.Result {
487+
func setInstanceStatusWithDeploymentInfo(atlasClient *mongodbatlas.Client, inst *dbaas.MongoDBAtlasInstance, atlasDeployment *v1.AtlasDeployment, project string) (bool, workflow.Result) {
465488
instInfo, result := atlasinventory.GetClusterInfo(atlasClient, project, inst.Spec.Name)
466489
if result.IsOk() {
467490
// Stores the phase info in inst.Status.Phase and remove from instInfo.InstanceInf map
@@ -479,7 +502,12 @@ func setInstanceStatusWithDeploymentInfo(atlasClient *mongodbatlas.Client, inst
479502
if cond.Type == status.DeploymentReadyType {
480503
statusFound = true
481504
if cond.Status == corev1.ConditionTrue {
482-
dbaas.SetInstanceCondition(inst, dbaasv1alpha1.DBaaSInstanceProviderSyncType, metav1.ConditionStatus(cond.Status), "Ready", cond.Message)
505+
if inst.Status.Phase == dbaasv1alpha1.InstancePhaseReady {
506+
dbaas.SetInstanceCondition(inst, dbaasv1alpha1.DBaaSInstanceProviderSyncType, metav1.ConditionStatus(cond.Status), "Ready", cond.Message)
507+
return false, result
508+
}
509+
dbaas.SetInstanceCondition(inst, dbaasv1alpha1.DBaaSInstanceProviderSyncType, metav1.ConditionFalse, instancePhaseChangedInAtlas, instancePhaseChangedInAtlasMsg)
510+
return true, result
483511
} else {
484512
if strings.Contains(cond.Message, FreeClusterFailed) {
485513
inst.Status.Phase = dbaasv1alpha1.InstancePhaseFailed
@@ -491,6 +519,5 @@ func setInstanceStatusWithDeploymentInfo(atlasClient *mongodbatlas.Client, inst
491519
if !statusFound {
492520
dbaas.SetInstanceCondition(inst, dbaasv1alpha1.DBaaSInstanceProviderSyncType, metav1.ConditionFalse, string(dbaasv1alpha1.InstancePhasePending), "Waiting for cluster creation to start")
493521
}
494-
495-
return result
522+
return false, result
496523
}

‎pkg/controller/atlasinstance/atlasinstance_test.go‎

Lines changed: 35 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -250,25 +250,40 @@ func TestSetInstanceStatusWithDeploymentInfo(t *testing.T) {
250250

251251
namespace := "default"
252252
testCase := map[string]struct {
253-
deploymentName string
254-
projectName string
255-
expErrMsg string
256-
expPhase dbaasv1alpha1.DBaasInstancePhase
257-
expStatus string
253+
deploymentName string
254+
projectName string
255+
deploymentCRStatus corev1.ConditionStatus
256+
expErrMsg string
257+
expPhase dbaasv1alpha1.DBaasInstancePhase
258+
expStatus string
259+
expStateChangedInAtlas bool
258260
}{
259-
"DeploymentCreating": {
260-
deploymentName: "mydeploymentcreating",
261-
projectName: "myproject",
262-
expErrMsg: "",
263-
expPhase: dbaasv1alpha1.InstancePhaseCreating,
264-
expStatus: "True",
261+
"DeploymentCreatingNorminal": {
262+
deploymentName: "mydeploymentcreating",
263+
projectName: "myproject",
264+
deploymentCRStatus: corev1.ConditionFalse,
265+
expErrMsg: "",
266+
expPhase: dbaasv1alpha1.InstancePhaseCreating,
267+
expStatus: "False",
268+
expStateChangedInAtlas: false,
269+
},
270+
"DeploymentCreatingChangedInAtlas": {
271+
deploymentName: "mydeploymentcreating",
272+
projectName: "myproject",
273+
deploymentCRStatus: corev1.ConditionTrue,
274+
expErrMsg: "",
275+
expPhase: dbaasv1alpha1.InstancePhaseCreating,
276+
expStatus: "False",
277+
expStateChangedInAtlas: true,
265278
},
266279
"DeploymentReady": {
267-
deploymentName: "mydeploymentready",
268-
projectName: "myproject",
269-
expErrMsg: "",
270-
expPhase: dbaasv1alpha1.InstancePhaseReady,
271-
expStatus: "True",
280+
deploymentName: "mydeploymentready",
281+
projectName: "myproject",
282+
deploymentCRStatus: corev1.ConditionTrue,
283+
expErrMsg: "",
284+
expPhase: dbaasv1alpha1.InstancePhaseReady,
285+
expStatus: "True",
286+
expStateChangedInAtlas: false,
272287
},
273288
"InvalidProject": {
274289
deploymentName: "mydeploymentready",
@@ -303,12 +318,12 @@ func TestSetInstanceStatusWithDeploymentInfo(t *testing.T) {
303318
Conditions: []status.Condition{
304319
{
305320
Type: status.ConditionType("Ready"),
306-
Status: corev1.ConditionTrue,
321+
Status: tc.deploymentCRStatus,
307322
LastTransitionTime: metav1.Now(),
308323
},
309324
{
310325
Type: status.ConditionType("DeploymentReady"),
311-
Status: corev1.ConditionTrue,
326+
Status: tc.deploymentCRStatus,
312327
LastTransitionTime: metav1.Now(),
313328
},
314329
},
@@ -331,13 +346,14 @@ func TestSetInstanceStatusWithDeploymentInfo(t *testing.T) {
331346
},
332347
},
333348
}
334-
result := setInstanceStatusWithDeploymentInfo(atlasClient, inst, atlasDeployment, tc.projectName)
349+
stateChangedInAtlas, result := setInstanceStatusWithDeploymentInfo(atlasClient, inst, atlasDeployment, tc.projectName)
335350
if len(tc.expErrMsg) == 0 {
336351
cond := dbaas.GetInstanceCondition(inst, dbaasv1alpha1.DBaaSInstanceProviderSyncType)
337352
assert.NotNil(t, cond)
338353
assert.True(t, result.IsOk())
339354
assert.Equal(t, inst.Status.Phase, tc.expPhase)
340355
assert.Equal(t, string(cond.Status), tc.expStatus)
356+
assert.Equal(t, stateChangedInAtlas, tc.expStateChangedInAtlas)
341357
} else {
342358
assert.Contains(t, result.Message(), tc.expErrMsg)
343359
}

‎pkg/controller/watch/predicates.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,20 @@ func CommonPredicates() predicate.Funcs {
2121
}
2222
}
2323

24+
// CommonPredicatesWithAnnotations returns the predicate which filter out the changes done to any field except for spec (e.g. status)
25+
// Also we should reconcile if finalizers have changed (see https://blog.openshift.com/kubernetes-operators-best-practices/)
26+
// and we should reconcile if the annotations have changed to allow other controllers to trigger the reconciliation using annotations.
27+
func CommonPredicatesWithAnnotations() predicate.Funcs {
28+
return predicate.Funcs{
29+
UpdateFunc: func(e event.UpdateEvent) bool {
30+
if e.ObjectOld.GetGeneration() == e.ObjectNew.GetGeneration() && reflect.DeepEqual(e.ObjectNew.GetFinalizers(), e.ObjectOld.GetFinalizers()) && reflect.DeepEqual(e.ObjectNew.GetAnnotations(), e.ObjectOld.GetAnnotations()) {
31+
return false
32+
}
33+
return true
34+
},
35+
}
36+
}
37+
2438
// DeleteOnly returns a predicate that will filter out everything except the Delete event
2539
func DeleteOnly() predicate.Funcs {
2640
return predicate.Funcs{

0 commit comments

Comments
 (0)