Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SLO test & tech debt #1697

Draft
wants to merge 7 commits into
base: main
Choose a base branch
from
Draft

SLO test & tech debt #1697

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8fd43a8
moving some stuff
alexandreLamarre Aug 31, 2023
ec2d9d9
rework alerting integration test setup
alexandreLamarre Sep 14, 2023
2da63c3
changes to SLO metric generation
alexandreLamarre Sep 14, 2023
cc55985
add mwmb alerts to prometheus gen
alexandreLamarre Sep 18, 2023
74e122e
purge2
alexandreLamarre Sep 18, 2023
c53598e
mwmb improvements
alexandreLamarre Sep 18, 2023
a485bde
fix lint
alexandreLamarre Sep 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions dc3d8690-0f9b-446a-b936-7da6b853de11-alerts.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
name: dc3d8690-0f9b-446a-b936-7da6b853de11-alerts
interval: 1m
rules:
- record: slo:alert:ticket_window
expr: (max(slo:sli_error:ratio_rate5m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000347222 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate30m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000115741 * 5.000000000)) without (slo_window)) or (max(slo:sli_error:ratio_rate2h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000347222 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate6h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000115741 * 5.000000000)) without (slo_window))
labels:
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
slo_severity: ticket
- record: slo:alert:page_window
expr: (max(slo:sli_error:ratio_rate5m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.001666667 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate30m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000694444 * 5.000000000)) without (slo_window)) or (max(slo:sli_error:ratio_rate2h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.001666667 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate6h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000694444 * 5.000000000)) without (slo_window))
labels:
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
slo_severity: page
- alert: dc3d8690-0f9b-446a-b936-7da6b853de11-page-alerts
expr: (max(slo:sli_error:ratio_rate5m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.001666667 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate30m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000694444 * 5.000000000)) without (slo_window)) or (max(slo:sli_error:ratio_rate2h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.001666667 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate6h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000694444 * 5.000000000)) without (slo_window))
labels:
opni_uuid: dc3d8690-0f9b-446a-b936-7da6b853de11-page-alerts
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
slo_severity: page
annotations:
opni_uuid: dc3d8690-0f9b-446a-b936-7da6b853de11-page-alerts
- alert: dc3d8690-0f9b-446a-b936-7da6b853de11-ticket-alerts
expr: (max(slo:sli_error:ratio_rate5m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000347222 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate30m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000115741 * 5.000000000)) without (slo_window)) or (max(slo:sli_error:ratio_rate2h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000347222 * 5.000000000)) without (slo_window) and max(slo:sli_error:ratio_rate6h{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} > bool (0.000115741 * 5.000000000)) without (slo_window))
labels:
opni_uuid: dc3d8690-0f9b-446a-b936-7da6b853de11-ticket-alerts
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
slo_severity: ticket
annotations:
opni_uuid: dc3d8690-0f9b-446a-b936-7da6b853de11-ticket-alerts
49 changes: 49 additions & 0 deletions dc3d8690-0f9b-446a-b936-7da6b853de11-metadata.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
name: dc3d8690-0f9b-446a-b936-7da6b853de11-metadata
interval: 1m
rules:
- record: slo:objective:ratio
expr: vector(0.950000000)
labels:
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
- record: slo:error_budget:ratio
expr: vector(1-0.950000000)
labels:
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
- record: slo:time_period:days
expr: vector(7)
labels:
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
- record: slo:current_burn_rate:ratio
expr: |-
slo:sli_error:ratio_rate5m{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} / on(slo_opni_id, slo_opni_service, slo_opni_name) group_left
slo:error_budget:ratio{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"}
labels:
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
- record: slo:period_burn_rate:ratio
expr: |-
slo:sli_error:ratio_rate7d{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"} / on(slo_opni_id, slo_opni_service, slo_opni_name) group_left
slo:error_budget:ratio{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"}
labels:
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
- record: slo:period_error_budget_remaining:ratio
expr: 1 - slo:period_burn_rate:ratio{slo_opni_id="dc3d8690-0f9b-446a-b936-7da6b853de11", slo_opni_service="opni-gateway", slo_opni_name="slo"}
labels:
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
- record: opni_slo_info
expr: vector(1)
labels:
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
52 changes: 52 additions & 0 deletions dc3d8690-0f9b-446a-b936-7da6b853de11-recording.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
name: dc3d8690-0f9b-446a-b936-7da6b853de11-recording
interval: 1m
rules:
- record: slo:sli_error:ratio_rate5m
expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[5m])))/(sum(rate(process_max_fds{job="opni-gateway"}[5m]))))
labels:
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
slo_window: 5m
- record: slo:sli_error:ratio_rate30m
expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[30m])))/(sum(rate(process_max_fds{job="opni-gateway"}[30m]))))
labels:
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
slo_window: 30m
- record: slo:sli_error:ratio_rate1h
expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[1h])))/(sum(rate(process_max_fds{job="opni-gateway"}[1h]))))
labels:
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
slo_window: 1h
- record: slo:sli_error:ratio_rate2h
expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[2h])))/(sum(rate(process_max_fds{job="opni-gateway"}[2h]))))
labels:
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
slo_window: 2h
- record: slo:sli_error:ratio_rate6h
expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[6h])))/(sum(rate(process_max_fds{job="opni-gateway"}[6h]))))
labels:
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
slo_window: 6h
- record: slo:sli_error:ratio_rate1d
expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[1d])))/(sum(rate(process_max_fds{job="opni-gateway"}[1d]))))
labels:
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
slo_window: 1d
- record: slo:sli_error:ratio_rate7d
expr: 1 - ((sum(rate(process_max_fds{job="opni-gateway",__tenant_id__=~"0194fdc2-fa2f-4cc0-81d3-ff12045b73c8"}[7d])))/(sum(rate(process_max_fds{job="opni-gateway"}[7d]))))
labels:
slo_opni_id: dc3d8690-0f9b-446a-b936-7da6b853de11
slo_opni_name: slo
slo_opni_service: opni-gateway
slo_window: 7d
6 changes: 0 additions & 6 deletions internal/cmd/testenv/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ import (
_ "github.com/rancher/opni/plugins/alerting/test"
_ "github.com/rancher/opni/plugins/logging/test"
_ "github.com/rancher/opni/plugins/metrics/test"
_ "github.com/rancher/opni/plugins/slo/test"
)

func main() {
Expand Down Expand Up @@ -190,11 +189,6 @@ func main() {
c := make(chan os.Signal, 2)
var closeOnce sync.Once
signal.Notify(c, os.Interrupt)
iPort, _ = environment.StartInstrumentationServer()
kPort = environment.StartMockKubernetesMetricServer()
for i := 0; i < 100; i++ {
environment.SimulateKubeObject(kPort)
}
testlog.Log.Infof(chalk.Green.Color("Instrumentation server listening on %d"), iPort)
var client managementv1.ManagementClient
if enableGateway {
Expand Down
9 changes: 9 additions & 0 deletions pkg/apis/core/v1/extensions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,15 @@ import (
"google.golang.org/protobuf/types/known/timestamppb"
)

func (c *ClusterList) Has(clusterId string) bool {
for _, cluster := range c.Items {
if cluster.Id == clusterId {
return true
}
}
return false
}

func (s *HealthStatus) Summary() string {
if s.Status == nil || s.Health == nil {
return "Unknown"
Expand Down
12 changes: 10 additions & 2 deletions plugins/slo/apis/slo/extension.go → pkg/apis/slo/v1/extensions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package slo
package v1

import (
"slices"
Expand All @@ -19,5 +19,13 @@ func (s *ServiceList) ContainsId(id string) bool {
for _, svc := range s.Items {
svcIds = append(svcIds, svc.ServiceId)
}
return !slices.Contains(svcIds, id)
return slices.Contains(svcIds, id)
}

func (e *EventList) ContainsId(id string) bool {
var eventIds []string
for _, event := range e.Items {
eventIds = append(eventIds, event.Key)
}
return slices.Contains(eventIds, id)
}
Loading