fix: add kube-ovn alerts and labels cleanip (#38)

Author: nahsi

flux/apps/kube-system/kube-ovn/observability/alerts.yml

@@ -8,6 +8,9 @@ spec:
     replicas: 1
   alloy:
     enableReporting: false
+    extraEnv:
+      - name: CLUSTER
+        value: spectrum
     configMap:
       content: |-
         logging {
@@ -78,6 +81,7 @@ spec:
 
         ksm.scrape "metrics" {
           targets = ksm.kubernetes.targets.output
+          job_label = "kube-state-metrics"
           forward_to = [prometheus.relabel.relable.receiver]
         }
 
@@ -136,7 +140,7 @@ spec:
 
         prometheus.scrape "nodeexporter" {
           targets    = discovery.relabel.nodeexporter_targets.output
-          job_name   = "integrations/node_exporter"
+          job_name   = "node"
           forward_to = [prometheus.relabel.relable.receiver]
         }
 
@@ -179,6 +183,11 @@ spec:
         
         // RELABELING
         prometheus.relabel "relable" {
+          rule {
+            action = "replace"
+            target_label = "cluster"
+            replacement = env("CLUSTER")
+          }
           forward_to = [prometheus.remote_write.victoriametrics.receiver]
         }
 

flux/apps/observability/alloy/app/metrics/alloy.yml

@@ -2,8 +2,6 @@ apiVersion: operator.victoriametrics.com/v1beta1
 kind: VMRule
 metadata:
   name: kube-state-alerts
-  labels:
-    app: kube-state-alerts
 spec:
   groups:
   - name: kube-state-metrics

flux/apps/observability/kube-state-metrics/app/monitoring/kube-state-alerts.yml

@@ -2,8 +2,6 @@ apiVersion: operator.victoriametrics.com/v1beta1
 kind: VMRule
 metadata:
   name: kubernetes-alerts
-  labels:
-    app: kubernetes-alerts
 spec:
   groups:
   - name: kubernetes-apps

flux/apps/observability/kube-state-metrics/app/monitoring/kubernetes-alerts.yml

@@ -2,8 +2,6 @@ apiVersion: operator.victoriametrics.com/v1beta1
 kind: VMRule
 metadata:
   name: kubernetes-rules
-  labels:
-    app: kubernetes-rules
 spec:
   groups:
   - interval: 3m

flux/apps/observability/kube-state-metrics/app/monitoring/kubernetes-rules.yml

@@ -1,6 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-namespace: observability
 resources:
   - dashboards.yml
   - kube-state-alerts.yml

flux/apps/observability/kube-state-metrics/app/monitoring/kustomization.yml

@@ -2,11 +2,9 @@ apiVersion: operator.victoriametrics.com/v1beta1
 kind: VMRule
 metadata:
   name: node-exporter-alerts
-  labels:
-    app: node-exporter-alerts
 spec:
   groups:
-  - name: node-exporter.rules
+  - name: node-exporter.alerts
     rules:
     - alert: NodeFilesystemSpaceFillingUp
       annotations:

flux/apps/observability/node-exporter/app/monitoring/alerts.yml

@@ -2,8 +2,6 @@ apiVersion: operator.victoriametrics.com/v1beta1
 kind: VMRule
 metadata:
   name: node-exporter-rules
-  labels:
-    app: node-exporter-rules
 spec:
   groups:
   - name: node-exporter.rules