Skip to content

Commit

Permalink
Support custom variables on metric templates
Browse files Browse the repository at this point in the history 
Signed-off-by: Nelson Johnstone <[email protected]>
  • Loading branch information
@njohnstone2
Nelson Johnstone authored and njohnstone2 committed Feb 8, 2023
1 parent e7d8ade commit 27eb21e
Show file tree
Hide file tree
Showing 5 changed files with 70 additions and 12 deletions.
4 changes: 4 additions & 0 deletions pkg/apis/flagger/v1beta1/canary.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,10 @@ type CanaryMetric struct {
// TemplateRef references a metric template object
// +optional
TemplateRef *CrossNamespaceObjectReference `json:"templateRef,omitempty"`

// TemplateVariables provides a map of key/value pairs to be used on a metric template object
// +optional
TemplateVariables map[string]string `json:"variables,omitempty"`
}

// CanaryThresholdRange defines the range used for metrics validation
Expand Down
16 changes: 9 additions & 7 deletions pkg/apis/flagger/v1beta1/metric.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,13 +82,14 @@ type MetricTemplateProvider struct {

// MetricTemplateModel is the query template model
type MetricTemplateModel struct {
Name string `json:"name"`
Namespace string `json:"namespace"`
Target string `json:"target"`
Service string `json:"service"`
Ingress string `json:"ingress"`
Route string `json:"route"`
Interval string `json:"interval"`
Name string `json:"name"`
Namespace string `json:"namespace"`
Target string `json:"target"`
Service string `json:"service"`
Ingress string `json:"ingress"`
Route string `json:"route"`
Interval string `json:"interval"`
Variables map[string]string `json:"variables"`
}

// TemplateFunctions returns a map of functions, one for each model field
Expand All @@ -101,6 +102,7 @@ func (mtm *MetricTemplateModel) TemplateFunctions() template.FuncMap {
"ingress": func() string { return mtm.Ingress },
"route": func() string { return mtm.Route },
"interval": func() string { return mtm.Interval },
"variables": func() map[string]string { return mtm.Variables },
}
}

Expand Down
25 changes: 25 additions & 0 deletions pkg/controller/scheduler_deployment_fixture_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ func newDeploymentFixture(c *flaggerv1.Canary) fixture {
flaggerClient := fakeFlagger.NewSimpleClientset(
c,
newDeploymentTestMetricTemplate(),
newDeploymentTestMetricTemplateCustomVars(),
newDeploymentTestAlertProvider(),
)

Expand Down Expand Up @@ -152,6 +153,7 @@ func newDeploymentFixture(c *flaggerv1.Canary) fixture {
ctrl.flaggerSynced = alwaysReady
ctrl.flaggerInformers.CanaryInformer.Informer().GetIndexer().Add(c)
ctrl.flaggerInformers.MetricInformer.Informer().GetIndexer().Add(newDeploymentTestMetricTemplate())
ctrl.flaggerInformers.MetricInformer.Informer().GetIndexer().Add(newDeploymentTestMetricTemplateCustomVars())
ctrl.flaggerInformers.AlertInformer.Informer().GetIndexer().Add(newDeploymentTestAlertProvider())

meshRouter := rf.MeshRouter("istio", "")
Expand Down Expand Up @@ -746,6 +748,29 @@ func newDeploymentTestMetricTemplate() *flaggerv1.MetricTemplate {
return template
}

func newDeploymentTestMetricTemplateCustomVars() *flaggerv1.MetricTemplate {
provider := flaggerv1.MetricTemplateProvider{
Type: "prometheus",
Address: testMetricsServerURL,
SecretRef: &corev1.LocalObjectReference{
Name: "podinfo-secret-env",
},
}

template := &flaggerv1.MetricTemplate{
TypeMeta: metav1.TypeMeta{APIVersion: flaggerv1.SchemeGroupVersion.String()},
ObjectMeta: metav1.ObjectMeta{
Namespace: "default",
Name: "custom-vars",
},
Spec: flaggerv1.MetricTemplateSpec{
Provider: provider,
Query: `sum(envoy_cluster_upstream_rq{envoy_cluster_name=~"{{ namespace }}_{{ target }},custom_label!={{ variables.second }}"})`,
},
}
return template
}

func newDeploymentTestAlertProviderSecret() *corev1.Secret {
return &corev1.Secret{
TypeMeta: metav1.TypeMeta{APIVersion: corev1.SchemeGroupVersion.String()},
Expand Down
11 changes: 6 additions & 5 deletions pkg/controller/scheduler_metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ func (c *Controller) runBuiltinMetricChecks(canary *flaggerv1.Canary) bool {
}

if metric.Name == "request-success-rate" {
val, err := observer.GetRequestSuccessRate(toMetricModel(canary, metric.Interval))
val, err := observer.GetRequestSuccessRate(toMetricModel(canary, metric.Interval, metric.TemplateVariables))
if err != nil {
if errors.Is(err, providers.ErrNoValuesFound) {
c.recordEventWarningf(canary,
Expand Down Expand Up @@ -167,7 +167,7 @@ func (c *Controller) runBuiltinMetricChecks(canary *flaggerv1.Canary) bool {
}

if metric.Name == "request-duration" {
val, err := observer.GetRequestDuration(toMetricModel(canary, metric.Interval))
val, err := observer.GetRequestDuration(toMetricModel(canary, metric.Interval, metric.TemplateVariables))
if err != nil {
if errors.Is(err, providers.ErrNoValuesFound) {
c.recordEventWarningf(canary, "Halt advancement no values found for %s metric %s probably %s.%s is not receiving traffic",
Expand Down Expand Up @@ -199,7 +199,7 @@ func (c *Controller) runBuiltinMetricChecks(canary *flaggerv1.Canary) bool {

// in-line PromQL
if metric.Query != "" {
query, err := observers.RenderQuery(metric.Query, toMetricModel(canary, metric.Interval))
query, err := observers.RenderQuery(metric.Query, toMetricModel(canary, metric.Interval, metric.TemplateVariables))
val, err := observerFactory.Client.RunQuery(query)
if err != nil {
if errors.Is(err, providers.ErrNoValuesFound) {
Expand Down Expand Up @@ -267,7 +267,7 @@ func (c *Controller) runMetricChecks(canary *flaggerv1.Canary) bool {
return false
}

query, err := observers.RenderQuery(template.Spec.Query, toMetricModel(canary, metric.Interval))
query, err := observers.RenderQuery(template.Spec.Query, toMetricModel(canary, metric.Interval, metric.TemplateVariables))
if err != nil {
c.recordEventErrorf(canary, "Metric template %s.%s query render error: %v",
metric.TemplateRef.Name, namespace, err)
Expand Down Expand Up @@ -310,7 +310,7 @@ func (c *Controller) runMetricChecks(canary *flaggerv1.Canary) bool {
return true
}

func toMetricModel(r *flaggerv1.Canary, interval string) flaggerv1.MetricTemplateModel {
func toMetricModel(r *flaggerv1.Canary, interval string, variables map[string]string) flaggerv1.MetricTemplateModel {
service := r.Spec.TargetRef.Name
if r.Spec.Service.Name != "" {
service = r.Spec.Service.Name
Expand All @@ -331,5 +331,6 @@ func toMetricModel(r *flaggerv1.Canary, interval string) flaggerv1.MetricTemplat
Ingress: ingress,
Route: route,
Interval: interval,
Variables: variables,
}
}
26 changes: 26 additions & 0 deletions pkg/controller/scheduler_metrics_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package controller
import (
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
"k8s.io/client-go/tools/record"
Expand Down Expand Up @@ -82,3 +83,28 @@ func TestController_checkMetricProviderAvailability(t *testing.T) {
require.NoError(t, ctrl.checkMetricProviderAvailability(canary))
})
}

func TestController_runMetricChecks(t *testing.T) {
t.Run("customVariables", func(t *testing.T) {
ctrl := newDeploymentFixture(nil).ctrl
analysis := &flaggerv1.CanaryAnalysis{Metrics: []flaggerv1.CanaryMetric{{
Name: "", TemplateVariables: map[string]string{
"first": "abc",
"second": "def",
},
TemplateRef: &flaggerv1.CrossNamespaceObjectReference{
Name: "custom-vars",
Namespace: "default",
},
ThresholdRange: &flaggerv1.CanaryThresholdRange{
Min: toFloatPtr(0),
Max: toFloatPtr(100),
},
}}}
canary := &flaggerv1.Canary{
ObjectMeta: metav1.ObjectMeta{Namespace: "default"},
Spec: flaggerv1.CanarySpec{Analysis: analysis},
}
assert.Equal(t, true, ctrl.runMetricChecks(canary))
})
}

0 comments on commit 27eb21e

Please sign in to comment.