From 91ba9abb23a070cd190d2f2aaadadbabc2df227d Mon Sep 17 00:00:00 2001 From: dayuwang Date: Tue, 25 Oct 2022 23:29:52 -0400 Subject: [PATCH 1/4] add digest of image --- deploy/deploy.go | 4 ++-- events/patch.go | 4 ++-- registry/dockerhub/provider.go | 6 +++--- registry/ecr/provider.go | 15 ++++++++------- registry/export.go | 5 ++++- resource/sync.go | 4 ++-- verify/image.go | 2 +- 7 files changed, 22 insertions(+), 18 deletions(-) diff --git a/deploy/deploy.go b/deploy/deploy.go index 7035fdb2..14bfa48b 100644 --- a/deploy/deploy.go +++ b/deploy/deploy.go @@ -2,12 +2,12 @@ package deploy import ( "github.com/golang/glog" + "github.com/pkg/errors" kcd1 "github.com/wish/kcd/gok8s/apis/custom/v1" "github.com/wish/kcd/gok8s/workload" k8s "github.com/wish/kcd/gok8s/workload" "github.com/wish/kcd/registry" "github.com/wish/kcd/state" - "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -99,7 +99,7 @@ func CheckPods(cs kubernetes.Interface, namespace string, target RolloutTarget, // glog.V(2).Infof("Still waiting for rollout: pod %s phase is %v", pod.Name, pod.Status.Phase) // return false, nil //} - glog.V(4).Infof("Check pod spec version %v, $v", pod.Name, pod.Namespace) + glog.V(4).Infof("Check pod spec version %v, %v", pod.Name, pod.Namespace) ok, err := workload.CheckPodSpecVersion(pod.Spec, kcd, version) if err != nil { diff --git a/events/patch.go b/events/patch.go index 82f70656..dd12e576 100644 --- a/events/patch.go +++ b/events/patch.go @@ -270,13 +270,13 @@ func patchForContainer(cName string, current, replacement Record, stats stats.St glog.Errorf("Failed to build registry for image repo: %v", imageRepo) return nil, false } else { - versions, err := registry.Versions(context.Background(), fluxTag) + versions, digest, err := registry.Versions(context.Background(), fluxTag) if err != nil { glog.Errorf("Syncer failed to get version from registry using tag=%s", fluxTag) return nil , false } version := versions[0] - glog.Infof("Got registry versions for container=%s, tag=%s, rolloutVersion=%s", cName, fluxTag, version) + glog.Infof("Got registry versions for container=%s, tag=%s, rolloutVersion=%s, imageDigest=%s", cName, fluxTag, version, *digest) patchOp.Value = imageDataFlux[0] + ":" + version glog.Infof("Replacing path=%v old tag=%v to patched version=%v", pathToPatch, fluxTag, version) diff --git a/registry/dockerhub/provider.go b/registry/dockerhub/provider.go index a8d42640..a8ac9649 100644 --- a/registry/dockerhub/provider.go +++ b/registry/dockerhub/provider.go @@ -84,15 +84,15 @@ func (vp *V2Provider) RegistryFor(imageRepo string) (kcdregistry.Registry, error } // Versions implements the Registry interface. -func (vp *V2Provider) Versions(ctx context.Context, tag string) ([]string, error) { +func (vp *V2Provider) Versions(ctx context.Context, tag string) (kcdregistry.Versions, kcdregistry.Digest, error) { tags := make([]string, 0, 5) newVersion, err := vp.getDigest(tag) if err != nil { vp.opts.Stats.IncCount("registry.failure", vp.repository) - return tags, errors.Errorf("No version found for tag %s", tag) + return tags, &newVersion, errors.Errorf("No version found for tag %s", tag) } tags = append(tags, newVersion) - return tags, nil + return tags, &newVersion, nil } // Add adds list of tags to the image identified with version diff --git a/registry/ecr/provider.go b/registry/ecr/provider.go index 8ff0f162..74f23fd2 100644 --- a/registry/ecr/provider.go +++ b/registry/ecr/provider.go @@ -7,14 +7,14 @@ import ( "strings" "time" - "github.com/wish/kcd/registry" - "github.com/wish/kcd/stats" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ecr" "github.com/golang/glog" "github.com/pkg/errors" + "github.com/wish/kcd/registry" + "github.com/wish/kcd/stats" ) var ecrRule, _ = regexp.Compile("([0-9]*).dkr.ecr.([a-z0-9-]*).amazonaws.com/([a-zA-Z0-9/\\_-]*)") @@ -95,7 +95,7 @@ func (ep *Provider) RegistryFor(imageRepo string) (registry.Registry, error) { } // Version implements the Registry interface. -func (ep *Provider) Versions(ctx context.Context, tag string) ([]string, error) { +func (ep *Provider) Versions(ctx context.Context, tag string) (registry.Versions, registry.Digest, error) { // TODO: parameterize timeout var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, time.Second*15) @@ -118,26 +118,27 @@ func (ep *Provider) Versions(ctx context.Context, tag string) ([]string, error) result, err := ep.ecr.DescribeImagesWithContext(ctx, req) if err != nil { glog.Errorf("Failed to get ECR: %v", err) - return nil, errors.Wrap(err, "failed to get ecr") + return nil, nil, errors.Wrap(err, "failed to get ecr") } if len(result.ImageDetails) != 1 { ep.stats.Event(fmt.Sprintf("registry.%s.sync.failure", ep.repoName), fmt.Sprintf("Failed to sync with ECR for tag %s", tag), "", "error", time.Now().UTC(), tag) - return nil, errors.Errorf("Bad state: More than one image was tagged with %s", tag) + return nil, nil, errors.Errorf("Bad state: More than one image was tagged with %s", tag) } img := result.ImageDetails[0] versions := ep.currentVersions(img) + digest := img.ImageDigest if len(versions) == 0 { ep.stats.IncCount("registry.failure", ep.repoName) - return nil, errors.Errorf("No version found for tag %s", tag) + return nil, nil, errors.Errorf("No version found for tag %s", tag) } glog.V(2).Infof("Got currentVersions=%s from ECR", strings.Join(versions, ", ")) - return versions, nil + return versions, digest, nil } // Add a list of tags to the image identified with version diff --git a/registry/export.go b/registry/export.go index 19dcd755..72daade9 100644 --- a/registry/export.go +++ b/registry/export.go @@ -5,6 +5,9 @@ import ( "strings" ) +type Versions []string +type Digest *string + // ProviderByRepo generates Type based on image ARN func ProviderByRepo(repoARN string) string { if strings.Contains(repoARN, "amazonaws.com") { @@ -24,7 +27,7 @@ type Registry interface { // on each commit and a commit that does not change the resulting image is made // and is tagged. In this case, the syncers check all the tags for the existing // version before determining if a rollout should occur. - Versions(ctx context.Context, tag string) ([]string, error) + Versions(ctx context.Context, tag string) (Versions, Digest, error) } // Tagger provides capability of adding/removing environment tags on ECR diff --git a/resource/sync.go b/resource/sync.go index 922ccc86..82e47346 100644 --- a/resource/sync.go +++ b/resource/sync.go @@ -99,7 +99,7 @@ func (s *Syncer) initialState() state.StateFunc { // refresh kcd resource state s.kcd = kcd - versions, err := s.registry.Versions(ctx, s.kcd.Spec.Tag) + versions, digest, err := s.registry.Versions(ctx, s.kcd.Spec.Tag) if err != nil { glog.Errorf("Syncer failed to get version from registry, kcd=%s, tag=%s: %v", s.kcd.Name, kcd.Spec.Tag, err) s.options.Recorder.Event(events.Warning, "KCDSyncFailed", "Failed to get versions from registry") @@ -108,7 +108,7 @@ func (s *Syncer) initialState() state.StateFunc { version := versions[0] if glog.V(4) { - glog.V(4).Infof("Got registry versions for kcd=%s, tag=%s, versions=%v, rolloutVersion=%s", s.kcd.Name, kcd.Spec.Tag, strings.Join(versions, ", "), version) + glog.V(4).Infof("Got registry versions for kcd=%s, tag=%s, versions=%v, rolloutVersion=%s, imageDigest=%s", s.kcd.Name, kcd.Spec.Tag, strings.Join(versions, ", "), version, *digest) } deployer, err := deploy.New(s.workloadProvider, s.registryProvider, s.kcd, versions[0]) diff --git a/verify/image.go b/verify/image.go index 8a91a90b..1bbe9c62 100644 --- a/verify/image.go +++ b/verify/image.go @@ -121,7 +121,7 @@ func (iv *ImageVerifier) getImage(ctx context.Context, spec kcd1.VerifySpec) (st return "", errors.Wrapf(err, "failed to get registry for %s", spec.Image) } - versions, err := registry.Versions(ctx, spec.Tag) + versions, _, err := registry.Versions(ctx, spec.Tag) if err != nil { return "", errors.Wrapf(err, "failed to get version from registry for %+v", spec) } From a6f6895b87342244d422d64ac8864ab1c0a6c5c2 Mon Sep 17 00:00:00 2001 From: dayuwang Date: Thu, 27 Oct 2022 02:06:14 -0400 Subject: [PATCH 2/4] add client poster --- resource/robbieClient.go | 38 ++++++++++++++++++++++++++++ resource/sync.go | 54 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 resource/robbieClient.go diff --git a/resource/robbieClient.go b/resource/robbieClient.go new file mode 100644 index 00000000..de3ff125 --- /dev/null +++ b/resource/robbieClient.go @@ -0,0 +1,38 @@ +package resource + +import ( + "bytes" + "encoding/json" + "net/http" +) + +type signOffReview struct { + result bool `json:"result"` + uuid string `json:"uuid"` +} + +func signOffPost(signoffReq *robbieSignOffRequest) (bool, error) { + requestBody, err := json.Marshal(&signoffReq) + if err != nil { + return false, err + } + authAuthenticatorUrl := "" + r, err := http.NewRequest("POST", authAuthenticatorUrl, bytes.NewBuffer(requestBody)) + r.Header.Add("Content-Type", "application/json") + + client := &http.Client{} + res, err := client.Do(r) + if err != nil { + return false, err + } + + defer res.Body.Close() + + review := &signOffReview{} + derr := json.NewDecoder(res.Body).Decode(review) + if derr != nil { + return false, derr + } + return review.result, nil + +} diff --git a/resource/sync.go b/resource/sync.go index 82e47346..4db9f120 100644 --- a/resource/sync.go +++ b/resource/sync.go @@ -6,6 +6,8 @@ import ( "strings" "time" + "github.com/golang/glog" + "github.com/pkg/errors" "github.com/wish/kcd/config" "github.com/wish/kcd/deploy" "github.com/wish/kcd/events" @@ -15,8 +17,6 @@ import ( "github.com/wish/kcd/registry" "github.com/wish/kcd/state" "github.com/wish/kcd/verify" - "github.com/golang/glog" - "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" k8serr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -98,7 +98,8 @@ func (s *Syncer) initialState() state.StateFunc { // refresh kcd resource state s.kcd = kcd - + + // get digest from ECR Describe Image in order to have more strict image version validation with Robbie versions, digest, err := s.registry.Versions(ctx, s.kcd.Spec.Tag) if err != nil { glog.Errorf("Syncer failed to get version from registry, kcd=%s, tag=%s: %v", s.kcd.Name, kcd.Spec.Tag, err) @@ -127,8 +128,22 @@ func (s *Syncer) initialState() state.StateFunc { glog.V(4).Infof("Not attempting %s rollout of version %s: %+v", s.kcd.Name, version, s.kcd.Status) return state.None() } - - glog.V(4).Infof("Creating rollout state for kcd=%s", s.kcd.Name) + + process, err = s.acquireSignOffFromRobbieRetryable(&robbieSignOffRequest{ + kcdName: s.kcd.GetName(), + kcdNameSpace: s.kcd.GetNamespace(), + kcdLables: s.kcd.GetLabels(), + kcdTag: s.kcd.Spec.Tag, + kcdImageRepo: s.kcd.Spec.ImageRepo, + versions: versions, + digest: digest, + }) + + if err != nil || !process { + glog.V(4).Infof("Fail to acquire sign-off by Robbie, so not attempting %s rollout of version %s: %+v", s.kcd.Name, version, s.kcd.Status) + return state.None() + } + glog.V(4).Infof("Successfully acquire sign-off by Robbie and creating rollout state for kcd=%s", s.kcd.Name) syncState := s.verify(version, s.updateRolloutStatus(version, StatusProgressing, @@ -371,3 +386,32 @@ func (s *Syncer) addHistory(deployer deploy.Deployer, version string, next state return state.Single(next) } } + +type robbieSignOffRequest struct { + kcdName string + kcdNameSpace string + kcdLables map[string]string + kcdTag string + kcdImageRepo string + versions registry.Versions + digest registry.Digest +} + +func (s *Syncer) acquireSignOffFromRobbieRetryable(signoffReq *robbieSignOffRequest) (bool, error) { + // max attempts to Robbie Sign off is set as 3, and init sleep duration is 3 seconds + attempts := 3 + sleep := 3 + result := false + var err error + for i := 0; i < attempts; i++ { + result, err = signOffPost(signoffReq) + if err == nil { + return result, nil + } else { + time.Sleep(time.Duration(sleep) * time.Second) + sleep *= 2 + } + } + return result, err + +} From e3850df6a448f0aaba3cbd68c143a3bf105285b8 Mon Sep 17 00:00:00 2001 From: dayuwang Date: Thu, 27 Oct 2022 02:15:16 -0400 Subject: [PATCH 3/4] add uuid --- resource/robbieClient.go | 10 +++++----- resource/sync.go | 9 ++++++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/resource/robbieClient.go b/resource/robbieClient.go index de3ff125..cc6acda9 100644 --- a/resource/robbieClient.go +++ b/resource/robbieClient.go @@ -11,10 +11,10 @@ type signOffReview struct { uuid string `json:"uuid"` } -func signOffPost(signoffReq *robbieSignOffRequest) (bool, error) { +func signOffPost(signoffReq *robbieSignOffRequest) (*signOffReview, error) { requestBody, err := json.Marshal(&signoffReq) if err != nil { - return false, err + return nil, err } authAuthenticatorUrl := "" r, err := http.NewRequest("POST", authAuthenticatorUrl, bytes.NewBuffer(requestBody)) @@ -23,7 +23,7 @@ func signOffPost(signoffReq *robbieSignOffRequest) (bool, error) { client := &http.Client{} res, err := client.Do(r) if err != nil { - return false, err + return nil, err } defer res.Body.Close() @@ -31,8 +31,8 @@ func signOffPost(signoffReq *robbieSignOffRequest) (bool, error) { review := &signOffReview{} derr := json.NewDecoder(res.Body).Decode(review) if derr != nil { - return false, derr + return nil, derr } - return review.result, nil + return review, nil } diff --git a/resource/sync.go b/resource/sync.go index 4db9f120..92879d64 100644 --- a/resource/sync.go +++ b/resource/sync.go @@ -401,17 +401,20 @@ func (s *Syncer) acquireSignOffFromRobbieRetryable(signoffReq *robbieSignOffRequ // max attempts to Robbie Sign off is set as 3, and init sleep duration is 3 seconds attempts := 3 sleep := 3 - result := false + var result *signOffReview var err error for i := 0; i < attempts; i++ { + glog.V(2).Infof("Querying with Robbie to get sign-off review for the %v attempt", i) result, err = signOffPost(signoffReq) if err == nil { - return result, nil + glog.V(2).Infof("Successfully get with Robbie sign-off review as %v, with UUId as %v", result.result, result.uuid) + return result.result, nil } else { + glog.V(2).Infof("Querying with Robbie to get sign-off review fails, sleep for %v second to retry", sleep) time.Sleep(time.Duration(sleep) * time.Second) sleep *= 2 } } - return result, err + return false, err } From 12c58d8e965a415993cd433469dc05531b0f4a4a Mon Sep 17 00:00:00 2001 From: dayuwang Date: Mon, 31 Oct 2022 21:24:55 -0400 Subject: [PATCH 4/4] add robbie client for sign off --- deploy/deploy.go | 2 +- registry/ecr/provider.go | 4 +- resource/robbieClient.go | 47 ++++++++++++++-- resource/robbieClient_test.go | 103 ++++++++++++++++++++++++++++++++++ resource/sync.go | 53 ++++------------- 5 files changed, 158 insertions(+), 51 deletions(-) create mode 100644 resource/robbieClient_test.go diff --git a/deploy/deploy.go b/deploy/deploy.go index 14bfa48b..3ecbeaa2 100644 --- a/deploy/deploy.go +++ b/deploy/deploy.go @@ -2,12 +2,12 @@ package deploy import ( "github.com/golang/glog" - "github.com/pkg/errors" kcd1 "github.com/wish/kcd/gok8s/apis/custom/v1" "github.com/wish/kcd/gok8s/workload" k8s "github.com/wish/kcd/gok8s/workload" "github.com/wish/kcd/registry" "github.com/wish/kcd/state" + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" diff --git a/registry/ecr/provider.go b/registry/ecr/provider.go index 74f23fd2..114850a8 100644 --- a/registry/ecr/provider.go +++ b/registry/ecr/provider.go @@ -7,14 +7,14 @@ import ( "strings" "time" + "github.com/wish/kcd/registry" + "github.com/wish/kcd/stats" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ecr" "github.com/golang/glog" "github.com/pkg/errors" - "github.com/wish/kcd/registry" - "github.com/wish/kcd/stats" ) var ecrRule, _ = regexp.Compile("([0-9]*).dkr.ecr.([a-z0-9-]*).amazonaws.com/([a-zA-Z0-9/\\_-]*)") diff --git a/resource/robbieClient.go b/resource/robbieClient.go index cc6acda9..cc907584 100644 --- a/resource/robbieClient.go +++ b/resource/robbieClient.go @@ -3,21 +3,35 @@ package resource import ( "bytes" "encoding/json" + "github.com/golang/glog" + "github.com/wish/kcd/registry" "net/http" + "time" ) +const RobbieEndpoint = "https://robbie-dev.i.wish.com/signoff" + +type robbieSignOffRequest struct { + KcdName string + KcdNameSpace string + KcdLables map[string]string + KcdTag string + KcdImageRepo string + Versions registry.Versions + Digest registry.Digest +} + type signOffReview struct { - result bool `json:"result"` - uuid string `json:"uuid"` + Result bool `json:"result"` + Uuid string `json:"uuid"` } -func signOffPost(signoffReq *robbieSignOffRequest) (*signOffReview, error) { - requestBody, err := json.Marshal(&signoffReq) +func signOffPost(signoffReq *robbieSignOffRequest, endpoint string) (*signOffReview, error) { + requestBody, err := json.Marshal(signoffReq) if err != nil { return nil, err } - authAuthenticatorUrl := "" - r, err := http.NewRequest("POST", authAuthenticatorUrl, bytes.NewBuffer(requestBody)) + r, err := http.NewRequest("POST", endpoint, bytes.NewBuffer(requestBody)) r.Header.Add("Content-Type", "application/json") client := &http.Client{} @@ -36,3 +50,24 @@ func signOffPost(signoffReq *robbieSignOffRequest) (*signOffReview, error) { return review, nil } + +func signOffRetryalbe(signoffReq *robbieSignOffRequest, endpoint string) (bool, error) { + // max attempts to Robbie Sign off is set as 3, and init sleep duration is 3 seconds + attempts := 3 + sleep := 3 + var result *signOffReview + var err error + for i := 0; i < attempts; i++ { + glog.V(2).Infof("Querying with Robbie to get sign-off review for the %v attempt", i) + result, err = signOffPost(signoffReq, endpoint) + if err == nil { + glog.V(2).Infof("Successfully get with Robbie sign-off review as %v, with UUId as %v", result.Result, result.Uuid) + return result.Result, nil + } else { + glog.V(2).Infof("Querying with Robbie to get sign-off review fails, sleep for %v second to retry", sleep) + time.Sleep(time.Duration(sleep) * time.Second) + sleep *= 2 + } + } + return false, err +} diff --git a/resource/robbieClient_test.go b/resource/robbieClient_test.go new file mode 100644 index 00000000..18fe437c --- /dev/null +++ b/resource/robbieClient_test.go @@ -0,0 +1,103 @@ +package resource + +import ( + "encoding/json" + "github.com/coreos/etcd/pkg/testutil" + "io" + "net/http" + "net/http/httptest" + "testing" +) + +func Test_signOffPost(t *testing.T) { + var digest = "digest123" + var signOffReq = robbieSignOffRequest{ + KcdName: "my_kcd", + KcdNameSpace: "my_namespace", + KcdLables: map[string]string{ + "a": "a1", + "b": "b1", + }, + KcdTag: "my_tag", + KcdImageRepo: "my_repo", + Versions: []string{"123", "456", "789"}, + Digest: &digest, + } + + var server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, err := io.ReadAll(r.Body) + if err != nil { + panic(err) + } + + signOffReq := robbieSignOffRequest{} + _ = json.Unmarshal(b, &signOffReq) + + testutil.AssertEqual(t, "my_kcd", signOffReq.KcdName) + testutil.AssertEqual(t, "my_namespace", signOffReq.KcdNameSpace) + testutil.AssertEqual(t, "my_tag", signOffReq.KcdTag) + testutil.AssertEqual(t, "my_repo", signOffReq.KcdImageRepo) + testutil.AssertEqual(t, "digest123", *(signOffReq.Digest)) + testutil.AssertEqual(t, 2, len(signOffReq.KcdLables)) + testutil.AssertEqual(t, 3, len(signOffReq.Versions)) + + var review = signOffReview{ + Result: true, + Uuid: signOffReq.KcdName + "-" + signOffReq.KcdImageRepo, + } + var responseBody, _ = json.Marshal(&review) + io.WriteString(w, string(responseBody)) + // mock here + })) + + var review, err = signOffPost(&signOffReq, server.URL) + testutil.AssertNil(t, err) + testutil.AssertEqual(t, "my_kcd-my_repo", review.Uuid) + testutil.AssertTrue(t, review.Result) +} + +func Test_signOffPostRetryable_Good(t *testing.T) { + var digest = "digest123" + var signOffReq = robbieSignOffRequest{ + KcdName: "my_kcd", + KcdNameSpace: "my_namespace", + KcdLables: map[string]string{ + "a": "a1", + "b": "b1", + }, + KcdTag: "my_tag", + KcdImageRepo: "my_repo", + Versions: []string{"123", "456", "789"}, + Digest: &digest, + } + + var server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, err := io.ReadAll(r.Body) + if err != nil { + panic(err) + } + + signOffReq := robbieSignOffRequest{} + _ = json.Unmarshal(b, &signOffReq) + + testutil.AssertEqual(t, "my_kcd", signOffReq.KcdName) + testutil.AssertEqual(t, "my_namespace", signOffReq.KcdNameSpace) + testutil.AssertEqual(t, "my_tag", signOffReq.KcdTag) + testutil.AssertEqual(t, "my_repo", signOffReq.KcdImageRepo) + testutil.AssertEqual(t, "digest123", *(signOffReq.Digest)) + testutil.AssertEqual(t, 2, len(signOffReq.KcdLables)) + testutil.AssertEqual(t, 3, len(signOffReq.Versions)) + + var review = signOffReview{ + Result: true, + Uuid: signOffReq.KcdName + "-" + signOffReq.KcdImageRepo, + } + var responseBody, _ = json.Marshal(&review) + io.WriteString(w, string(responseBody)) + // mock here + })) + + var res, err = signOffRetryalbe(&signOffReq, server.URL) + testutil.AssertNil(t, err) + testutil.AssertTrue(t, res) +} diff --git a/resource/sync.go b/resource/sync.go index 92879d64..f32d675c 100644 --- a/resource/sync.go +++ b/resource/sync.go @@ -128,17 +128,17 @@ func (s *Syncer) initialState() state.StateFunc { glog.V(4).Infof("Not attempting %s rollout of version %s: %+v", s.kcd.Name, version, s.kcd.Status) return state.None() } - - process, err = s.acquireSignOffFromRobbieRetryable(&robbieSignOffRequest{ - kcdName: s.kcd.GetName(), - kcdNameSpace: s.kcd.GetNamespace(), - kcdLables: s.kcd.GetLabels(), - kcdTag: s.kcd.Spec.Tag, - kcdImageRepo: s.kcd.Spec.ImageRepo, - versions: versions, - digest: digest, - }) - + + process, err = signOffRetryalbe(&robbieSignOffRequest{ + KcdName: s.kcd.GetName(), + KcdNameSpace: s.kcd.GetNamespace(), + KcdLables: s.kcd.GetLabels(), + KcdTag: s.kcd.Spec.Tag, + KcdImageRepo: s.kcd.Spec.ImageRepo, + Versions: versions, + Digest: digest, + }, RobbieEndpoint) + if err != nil || !process { glog.V(4).Infof("Fail to acquire sign-off by Robbie, so not attempting %s rollout of version %s: %+v", s.kcd.Name, version, s.kcd.Status) return state.None() @@ -387,34 +387,3 @@ func (s *Syncer) addHistory(deployer deploy.Deployer, version string, next state } } -type robbieSignOffRequest struct { - kcdName string - kcdNameSpace string - kcdLables map[string]string - kcdTag string - kcdImageRepo string - versions registry.Versions - digest registry.Digest -} - -func (s *Syncer) acquireSignOffFromRobbieRetryable(signoffReq *robbieSignOffRequest) (bool, error) { - // max attempts to Robbie Sign off is set as 3, and init sleep duration is 3 seconds - attempts := 3 - sleep := 3 - var result *signOffReview - var err error - for i := 0; i < attempts; i++ { - glog.V(2).Infof("Querying with Robbie to get sign-off review for the %v attempt", i) - result, err = signOffPost(signoffReq) - if err == nil { - glog.V(2).Infof("Successfully get with Robbie sign-off review as %v, with UUId as %v", result.result, result.uuid) - return result.result, nil - } else { - glog.V(2).Infof("Querying with Robbie to get sign-off review fails, sleep for %v second to retry", sleep) - time.Sleep(time.Duration(sleep) * time.Second) - sleep *= 2 - } - } - return false, err - -}