fixup! Prefer kube-scheduler's resource metrics to kube-state-metrics'

rexagod · rexagod · commit 86d83ae4e619 · 2024-09-24T04:25:18.000+05:30
Refactor kube_pod_status_phase, since statuses other than "Pending" or
"Running" are excluded or deprecated.

Signed-off-by: Pranshu Srivastava &lt;rexagod@gmail.com&gt;
diff --git a/DESIGN.md b/DESIGN.md
@@ -54,7 +54,7 @@ Jsonnet offers the ability to parameterise configuration, allowing for basic cus
           alert: "KubePodNotReady", 
           expr: ||| 
             sum by (namespace, pod) ( 
-              kube_pod_status_phase{%(kubeStateMetricsSelector)s, phase!~"Running|Succeeded"} 
+              kube_pod_status_phase{%(kubeStateMetricsSelector)s, phase!~"Running"} 
             ) > $(allowedNotReadyPods)s 
           ||| % $._config, 
           "for": "1h", 
diff --git a/alerts/apps_alerts.libsonnet b/alerts/apps_alerts.libsonnet
@@ -33,7 +33,7 @@
             expr: |||
               sum by (namespace, pod, %(clusterLabel)s) (
                 max by(namespace, pod, %(clusterLabel)s) (
-                  kube_pod_status_phase{%(prefixedNamespaceSelector)s%(kubeStateMetricsSelector)s, phase=~"Pending|Unknown|Failed"}
+                  kube_pod_status_phase{%(prefixedNamespaceSelector)s%(kubeStateMetricsSelector)s, phase="Pending"}
                 ) * on(namespace, pod, %(clusterLabel)s) group_left(owner_kind) topk by(namespace, pod, %(clusterLabel)s) (
                   1, max by(namespace, pod, owner_kind, %(clusterLabel)s) (kube_pod_owner{owner_kind!="Job"})
                 )
diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
@@ -86,10 +86,7 @@
           {
             record: 'cluster:namespace:pod_memory:active:kube_pod_resource_request_or_kube_pod_container_resource_requests',
             expr: |||
-              (kube_pod_resource_request{resource="memory",%(kubeSchedulerSelector)s} or (kube_pod_container_resource_requests{resource="memory",%(kubeStateMetricsSelector)s}) * on (namespace, pod, %(clusterLabel)s)
-              group_left() max by (namespace, pod, %(clusterLabel)s) (
-                (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
-              )
+              (kube_pod_resource_request{resource="memory",%(kubeSchedulerSelector)s} or kube_pod_container_resource_requests{resource="memory",%(kubeStateMetricsSelector)s})
             ||| % $._config,
           },
           {
@@ -99,8 +96,6 @@
                   sum by (namespace, pod, %(clusterLabel)s) (
                       max by (namespace, pod, container, %(clusterLabel)s) (
                         kube_pod_resource_request{resource="memory",%(kubeSchedulerSelector)s} or kube_pod_container_resource_requests{resource="memory",%(kubeStateMetricsSelector)s}
-                      ) * on(namespace, pod, %(clusterLabel)s) group_left() max by (namespace, pod, %(clusterLabel)s) (
-                        kube_pod_status_phase{phase=~"Pending|Running"} == 1
                       )
                   )
               )
@@ -114,10 +109,7 @@
           {
             record: 'cluster:namespace:pod_cpu:active:kube_pod_resource_request_or_kube_pod_container_resource_requests',
             expr: |||
-              (kube_pod_resource_request{resource="memory",%(kubeSchedulerSelector)s} or (kube_pod_container_resource_requests{resource="cpu",%(kubeStateMetricsSelector)s}) * on (namespace, pod, %(clusterLabel)s)
-              group_left() max by (namespace, pod, %(clusterLabel)s) (
-                (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
-              )
+              (kube_pod_resource_request{resource="memory",%(kubeSchedulerSelector)s} or kube_pod_container_resource_requests{resource="cpu",%(kubeStateMetricsSelector)s})
             ||| % $._config,
           },
           {
@@ -127,8 +119,6 @@
                   sum by (namespace, pod, %(clusterLabel)s) (
                       max by (namespace, pod, container, %(clusterLabel)s) (
                         kube_pod_resource_request{resource="cpu",%(kubeSchedulerSelector)s} or kube_pod_container_resource_requests{resource="cpu",%(kubeStateMetricsSelector)s}
-                      ) * on(namespace, pod, %(clusterLabel)s) group_left() max by (namespace, pod, %(clusterLabel)s) (
-                        kube_pod_status_phase{phase=~"Pending|Running"} == 1
                       )
                   )
               )
@@ -142,10 +132,7 @@
           {
             record: 'cluster:namespace:pod_memory:active:kube_pod_resource_limit_or_kube_pod_container_resource_limits',
             expr: |||
-              (kube_pod_resource_limit{resource="memory",%(kubeSchedulerSelector)s} or kube_pod_container_resource_limits{resource="memory",%(kubeStateMetricsSelector)s}) * on (namespace, pod, %(clusterLabel)s)
-              group_left() max by (namespace, pod, %(clusterLabel)s) (
-                (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
-              )
+              (kube_pod_resource_limit{resource="memory",%(kubeSchedulerSelector)s} or kube_pod_container_resource_limits{resource="memory",%(kubeStateMetricsSelector)s})
             ||| % $._config,
           },
           {
@@ -155,8 +142,6 @@
                   sum by (namespace, pod, %(clusterLabel)s) (
                       max by (namespace, pod, container, %(clusterLabel)s) (
                         kube_pod_container_resource_limits{resource="memory",%(kubeStateMetricsSelector)s}
-                      ) * on(namespace, pod, %(clusterLabel)s) group_left() max by (namespace, pod, %(clusterLabel)s) (
-                        kube_pod_status_phase{phase=~"Pending|Running"} == 1
                       )
                   )
               )
@@ -170,10 +155,7 @@
           {
             record: 'cluster:namespace:pod_cpu:active:kube_pod_resource_limit_or_kube_pod_container_resource_limits',
             expr: |||
-              (kube_pod_resource_limit{resource="memory",%(kubeSchedulerSelector)s} or kube_pod_container_resource_limits{resource="cpu",%(kubeStateMetricsSelector)s}) * on (namespace, pod, %(clusterLabel)s)
-              group_left() max by (namespace, pod, %(clusterLabel)s) (
-               (kube_pod_status_phase{phase=~"Pending|Running"} == 1)
-               )
+              (kube_pod_resource_limit{resource="memory",%(kubeSchedulerSelector)s} or kube_pod_container_resource_limits{resource="cpu",%(kubeStateMetricsSelector)s})
             ||| % $._config,
           },
           {
@@ -183,8 +165,6 @@
                   sum by (namespace, pod, %(clusterLabel)s) (
                       max by (namespace, pod, container, %(clusterLabel)s) (
                         kube_pod_resource_limit{resource="memory",%(kubeSchedulerSelector)s} or kube_pod_container_resource_limits{resource="cpu",%(kubeStateMetricsSelector)s}
-                      ) * on(namespace, pod, %(clusterLabel)s) group_left() max by (namespace, pod, %(clusterLabel)s) (
-                        kube_pod_status_phase{phase=~"Pending|Running"} == 1
                       )
                   )
               )
diff --git a/tests.yaml b/tests.yaml
@@ -461,23 +461,13 @@ tests:
   - eval_time: 0m
     expr: namespace_cpu:kube_pod_resource_request_or_kube_pod_container_resource_requests:sum
     exp_samples:
-    - value: 0.15
+    - value: 0.3
       labels: 'namespace_cpu:kube_pod_resource_request_or_kube_pod_container_resource_requests:sum{cluster="kubernetes",namespace="kube-apiserver"}'
-  - eval_time: 0m
-    expr: namespace_memory:kube_pod_container_resource_requests:sum
-    exp_samples:
-    - value: 1.0e+9
-      labels: 'namespace_memory:kube_pod_container_resource_requests:sum{cluster="kubernetes",namespace="kube-apiserver"}'
   - eval_time: 1m
     expr: namespace_cpu:kube_pod_resource_request_or_kube_pod_container_resource_requests:sum
     exp_samples:
-    - value: 0.15
+    - value: 0.3
       labels: 'namespace_cpu:kube_pod_resource_request_or_kube_pod_container_resource_requests:sum{cluster="kubernetes",namespace="kube-apiserver"}'
-  - eval_time: 1m
-    expr: namespace_memory:kube_pod_container_resource_requests:sum
-    exp_samples:
-    - value: 1.0e+9
-      labels: 'namespace_memory:kube_pod_container_resource_requests:sum{cluster="kubernetes",namespace="kube-apiserver"}'
 
 - interval: 1m
   input_series:
@@ -506,23 +496,13 @@ tests:
   - eval_time: 0m
     expr: namespace_cpu:kube_pod_resource_request_or_kube_pod_container_resource_requests:sum
     exp_samples:
-    - value: 0.15
+    - value: 0.3
       labels: 'namespace_cpu:kube_pod_resource_request_or_kube_pod_container_resource_requests:sum{cluster="kubernetes",namespace="kube-apiserver"}'
-  - eval_time: 0m
-    expr: namespace_memory:kube_pod_container_resource_requests:sum
-    exp_samples:
-    - value: 1.0e+9
-      labels: 'namespace_memory:kube_pod_container_resource_requests:sum{cluster="kubernetes",namespace="kube-apiserver"}'
   - eval_time: 1m
     expr: namespace_cpu:kube_pod_resource_request_or_kube_pod_container_resource_requests:sum
     exp_samples:
-    - value: 0.15
+    - value: 0.3
       labels: 'namespace_cpu:kube_pod_resource_request_or_kube_pod_container_resource_requests:sum{cluster="kubernetes",namespace="kube-apiserver"}'
-  - eval_time: 1m
-    expr: namespace_memory:kube_pod_container_resource_requests:sum
-    exp_samples:
-    - value: 1.0e+9
-      labels: 'namespace_memory:kube_pod_container_resource_requests:sum{cluster="kubernetes",namespace="kube-apiserver"}'
 
 - interval: 1m
   input_series:

Original file line number	Diff line number	Diff line change
`@@ -33,7 +33,7 @@`
`33`	`33`	`expr: \|\|\|`
`34`	`34`	`sum by (namespace, pod, %(clusterLabel)s) (`
`35`	`35`	`max by(namespace, pod, %(clusterLabel)s) (`
`36`		`- kube_pod_status_phase{%(prefixedNamespaceSelector)s%(kubeStateMetricsSelector)s, phase=~"Pending\|Unknown\|Failed"}`
	`36`	`+ kube_pod_status_phase{%(prefixedNamespaceSelector)s%(kubeStateMetricsSelector)s, phase="Pending"}`
`37`	`37`	`) * on(namespace, pod, %(clusterLabel)s) group_left(owner_kind) topk by(namespace, pod, %(clusterLabel)s) (`
`38`	`38`	`1, max by(namespace, pod, owner_kind, %(clusterLabel)s) (kube_pod_owner{owner_kind!="Job"})`
`39`	`39`	`)`
Original file line number	Diff line number	Diff line change
`@@ -86,10 +86,7 @@`
`86`	`86`	`{`
`87`	`87`	`record: 'cluster:namespace:pod_memory:active:kube_pod_resource_request_or_kube_pod_container_resource_requests',`
`88`	`88`	`expr: \|\|\|`
`89`		`- (kube_pod_resource_request{resource="memory",%(kubeSchedulerSelector)s} or (kube_pod_container_resource_requests{resource="memory",%(kubeStateMetricsSelector)s}) * on (namespace, pod, %(clusterLabel)s)`
`90`		`- group_left() max by (namespace, pod, %(clusterLabel)s) (`
`91`		`- (kube_pod_status_phase{phase=~"Pending\|Running"} == 1)`
`92`		`- )`
	`89`	`+ (kube_pod_resource_request{resource="memory",%(kubeSchedulerSelector)s} or kube_pod_container_resource_requests{resource="memory",%(kubeStateMetricsSelector)s})`
`93`	`90`	`\|\|\| % $._config,`
`94`	`91`	`},`
`95`	`92`	`{`
`@@ -99,8 +96,6 @@`
`99`	`96`	`sum by (namespace, pod, %(clusterLabel)s) (`
`100`	`97`	`max by (namespace, pod, container, %(clusterLabel)s) (`
`101`	`98`	`kube_pod_resource_request{resource="memory",%(kubeSchedulerSelector)s} or kube_pod_container_resource_requests{resource="memory",%(kubeStateMetricsSelector)s}`
`102`		`- ) * on(namespace, pod, %(clusterLabel)s) group_left() max by (namespace, pod, %(clusterLabel)s) (`
`103`		`- kube_pod_status_phase{phase=~"Pending\|Running"} == 1`
`104`	`99`	`)`
`105`	`100`	`)`
`106`	`101`	`)`
`@@ -114,10 +109,7 @@`
`114`	`109`	`{`
`115`	`110`	`record: 'cluster:namespace:pod_cpu:active:kube_pod_resource_request_or_kube_pod_container_resource_requests',`
`116`	`111`	`expr: \|\|\|`
`117`		`- (kube_pod_resource_request{resource="memory",%(kubeSchedulerSelector)s} or (kube_pod_container_resource_requests{resource="cpu",%(kubeStateMetricsSelector)s}) * on (namespace, pod, %(clusterLabel)s)`
`118`		`- group_left() max by (namespace, pod, %(clusterLabel)s) (`
`119`		`- (kube_pod_status_phase{phase=~"Pending\|Running"} == 1)`
`120`		`- )`
	`112`	`+ (kube_pod_resource_request{resource="memory",%(kubeSchedulerSelector)s} or kube_pod_container_resource_requests{resource="cpu",%(kubeStateMetricsSelector)s})`
`121`	`113`	`\|\|\| % $._config,`
`122`	`114`	`},`
`123`	`115`	`{`
`@@ -127,8 +119,6 @@`
`127`	`119`	`sum by (namespace, pod, %(clusterLabel)s) (`
`128`	`120`	`max by (namespace, pod, container, %(clusterLabel)s) (`
`129`	`121`	`kube_pod_resource_request{resource="cpu",%(kubeSchedulerSelector)s} or kube_pod_container_resource_requests{resource="cpu",%(kubeStateMetricsSelector)s}`
`130`		`- ) * on(namespace, pod, %(clusterLabel)s) group_left() max by (namespace, pod, %(clusterLabel)s) (`
`131`		`- kube_pod_status_phase{phase=~"Pending\|Running"} == 1`
`132`	`122`	`)`
`133`	`123`	`)`
`134`	`124`	`)`
`@@ -142,10 +132,7 @@`
`142`	`132`	`{`
`143`	`133`	`record: 'cluster:namespace:pod_memory:active:kube_pod_resource_limit_or_kube_pod_container_resource_limits',`
`144`	`134`	`expr: \|\|\|`
`145`		`- (kube_pod_resource_limit{resource="memory",%(kubeSchedulerSelector)s} or kube_pod_container_resource_limits{resource="memory",%(kubeStateMetricsSelector)s}) * on (namespace, pod, %(clusterLabel)s)`
`146`		`- group_left() max by (namespace, pod, %(clusterLabel)s) (`
`147`		`- (kube_pod_status_phase{phase=~"Pending\|Running"} == 1)`
`148`		`- )`
	`135`	`+ (kube_pod_resource_limit{resource="memory",%(kubeSchedulerSelector)s} or kube_pod_container_resource_limits{resource="memory",%(kubeStateMetricsSelector)s})`
`149`	`136`	`\|\|\| % $._config,`
`150`	`137`	`},`
`151`	`138`	`{`
`@@ -155,8 +142,6 @@`
`155`	`142`	`sum by (namespace, pod, %(clusterLabel)s) (`
`156`	`143`	`max by (namespace, pod, container, %(clusterLabel)s) (`
`157`	`144`	`kube_pod_container_resource_limits{resource="memory",%(kubeStateMetricsSelector)s}`
`158`		`- ) * on(namespace, pod, %(clusterLabel)s) group_left() max by (namespace, pod, %(clusterLabel)s) (`
`159`		`- kube_pod_status_phase{phase=~"Pending\|Running"} == 1`
`160`	`145`	`)`
`161`	`146`	`)`
`162`	`147`	`)`
`@@ -170,10 +155,7 @@`
`170`	`155`	`{`
`171`	`156`	`record: 'cluster:namespace:pod_cpu:active:kube_pod_resource_limit_or_kube_pod_container_resource_limits',`
`172`	`157`	`expr: \|\|\|`
`173`		`- (kube_pod_resource_limit{resource="memory",%(kubeSchedulerSelector)s} or kube_pod_container_resource_limits{resource="cpu",%(kubeStateMetricsSelector)s}) * on (namespace, pod, %(clusterLabel)s)`
`174`		`- group_left() max by (namespace, pod, %(clusterLabel)s) (`
`175`		`- (kube_pod_status_phase{phase=~"Pending\|Running"} == 1)`
`176`		`- )`
	`158`	`+ (kube_pod_resource_limit{resource="memory",%(kubeSchedulerSelector)s} or kube_pod_container_resource_limits{resource="cpu",%(kubeStateMetricsSelector)s})`
`177`	`159`	`\|\|\| % $._config,`
`178`	`160`	`},`
`179`	`161`	`{`
`@@ -183,8 +165,6 @@`
`183`	`165`	`sum by (namespace, pod, %(clusterLabel)s) (`
`184`	`166`	`max by (namespace, pod, container, %(clusterLabel)s) (`
`185`	`167`	`kube_pod_resource_limit{resource="memory",%(kubeSchedulerSelector)s} or kube_pod_container_resource_limits{resource="cpu",%(kubeStateMetricsSelector)s}`
`186`		`- ) * on(namespace, pod, %(clusterLabel)s) group_left() max by (namespace, pod, %(clusterLabel)s) (`
`187`		`- kube_pod_status_phase{phase=~"Pending\|Running"} == 1`
`188`	`168`	`)`
`189`	`169`	`)`
`190`	`170`	`)`