From 47a07f7b075caea25775cd921c4a6297f149bb10 Mon Sep 17 00:00:00 2001
From: Joshua Hoblitt <josh@hoblitt.com>
Date: Thu, 25 Apr 2024 10:44:50 -0700
Subject: [PATCH 1/2] (fleet/kyverno) enable grafana dashboard

---
 fleet/lib/kyverno/fleet.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fleet/lib/kyverno/fleet.yaml b/fleet/lib/kyverno/fleet.yaml
index a759a3c75..cebfd0cf1 100644
--- a/fleet/lib/kyverno/fleet.yaml
+++ b/fleet/lib/kyverno/fleet.yaml
@@ -58,6 +58,8 @@ helm:
           lsst.io/monitor: "true"
       metricsService:
         create: true
+    grafana:
+      enabled: true
 dependsOn:
   - selector:
       matchLabels:

From 5bb52ace028844594d21d701f89ffd87cc23d5d7 Mon Sep 17 00:00:00 2001
From: Joshua Hoblitt <josh@hoblitt.com>
Date: Thu, 25 Apr 2024 10:54:34 -0700
Subject: [PATCH 2/2] (fleet/kyverno) add alerts

---
 .../kyverno-conf/prometheusrule-kyverno.yaml  | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 fleet/lib/kyverno-conf/prometheusrule-kyverno.yaml

diff --git a/fleet/lib/kyverno-conf/prometheusrule-kyverno.yaml b/fleet/lib/kyverno-conf/prometheusrule-kyverno.yaml
new file mode 100644
index 000000000..bbca1fe57
--- /dev/null
+++ b/fleet/lib/kyverno-conf/prometheusrule-kyverno.yaml
@@ -0,0 +1,27 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    lsst.io/rule: "true"
+  name: kyverno
+spec:
+  groups:
+    - name: kyverno.rules
+      rules:
+        - alert: KyvernoPolicyExecutionDurationHigh
+          annotations:
+            summary: High mean Kyverno policy execution time of {{ $value }} seconds
+          expr: sum(kyverno_policy_execution_duration_seconds_sum{cluster=~".*"}) / sum(kyverno_policy_execution_duration_seconds_count{cluster=~".*"}) > 0.1
+          for: 15s
+          labels:
+            severity: warning
+
+        - alert: KyvernoDeploymentIsOnFire
+          annotations:
+            summary: Kyverno deployment {{ $labels.namespace }}/{{ $labels.deployment }} is on fire
+          # XXX is this the correct way to determine if a deployment is unhappy?
+          expr: kube_deployment_status_condition{namespace="kyverno",condition="Available",status="true"} != 1
+          for: 5m
+          labels:
+            severity: warning