Skip to content

Provides backwards compatibility for Kubernetes 1.34 control plane metrics #1836

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
+269 −0
Draft

Provides backwards compatibility for Kubernetes 1.34 control plane metrics #1836

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
218 changes: 218 additions & 0 deletions plugins/inputs/prometheus/k8s134_compatibility_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,218 @@
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: MIT

package prometheus

import (
"testing"
)

func TestApplyK8s134Compatibility(t *testing.T) {
tests := []struct {
name string
input PrometheusMetricBatch
expected PrometheusMetricBatch
}{
{
name: "apiserver_resource_objects renamed to apiserver_storage_objects with group",
input: PrometheusMetricBatch{
&PrometheusMetric{
metricName: "apiserver_resource_objects",
tags: map[string]string{
"resource": "pods",
"group": "v1",
},
},
},
expected: PrometheusMetricBatch{
&PrometheusMetric{
metricName: "apiserver_storage_objects",
tags: map[string]string{
"resource": "pods.v1",
},
},
},
},
{
name: "apiserver_resource_objects renamed to apiserver_storage_objects without group",
input: PrometheusMetricBatch{
&PrometheusMetric{
metricName: "apiserver_resource_objects",
tags: map[string]string{
"resource": "nodes",
},
},
},
expected: PrometheusMetricBatch{
&PrometheusMetric{
metricName: "apiserver_storage_objects",
tags: map[string]string{
"resource": "nodes",
},
},
},
},
{
name: "etcd_request metric gets type label",
input: PrometheusMetricBatch{
&PrometheusMetric{
metricName: "etcd_request_duration_seconds",
tags: map[string]string{
"resource": "pods",
"group": "v1",
},
},
},
expected: PrometheusMetricBatch{
&PrometheusMetric{
metricName: "etcd_request_duration_seconds",
tags: map[string]string{
"resource": "pods",
"group": "v1",
"resource_prefix": "pods.v1",
"type": "pods.v1",
},
},
},
},
{
name: "apiserver_watch metric gets kind label",
input: PrometheusMetricBatch{
&PrometheusMetric{
metricName: "apiserver_watch_events_total",
tags: map[string]string{
"resource": "pods",
"group": "v1",
},
},
},
expected: PrometheusMetricBatch{
&PrometheusMetric{
metricName: "apiserver_watch_events_total",
tags: map[string]string{
"resource": "pods",
"group": "v1",
"resource_prefix": "pods.v1",
"kind": "pods",
},
},
},
},
{
name: "resource without group",
input: PrometheusMetricBatch{
&PrometheusMetric{
metricName: "etcd_request_total",
tags: map[string]string{
"resource": "nodes",
},
},
},
expected: PrometheusMetricBatch{
&PrometheusMetric{
metricName: "etcd_request_total",
tags: map[string]string{
"resource": "nodes",
"resource_prefix": "nodes",
"type": "nodes",
},
},
},
},
{
name: "non-control-plane metric unchanged",
input: PrometheusMetricBatch{
&PrometheusMetric{
metricName: "some_other_metric",
tags: map[string]string{
"label": "value",
},
},
},
expected: PrometheusMetricBatch{
&PrometheusMetric{
metricName: "some_other_metric",
tags: map[string]string{
"label": "value",
},
},
},
},
{
name: "empty group treated as missing",
input: PrometheusMetricBatch{
&PrometheusMetric{
metricName: "etcd_request_errors",
tags: map[string]string{
"resource": "pods",
"group": "",
},
},
},
expected: PrometheusMetricBatch{
&PrometheusMetric{
metricName: "etcd_request_errors",
tags: map[string]string{
"resource": "pods",
"group": "",
"resource_prefix": "pods",
"type": "pods",
},
},
},
},
{
name: "metric without resource label unchanged",
input: PrometheusMetricBatch{
&PrometheusMetric{
metricName: "etcd_request_duration_seconds",
tags: map[string]string{
"operation": "get",
},
},
},
expected: PrometheusMetricBatch{
&PrometheusMetric{
metricName: "etcd_request_duration_seconds",
tags: map[string]string{
"operation": "get",
},
},
},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := applyK8s134Compatibility(tt.input)

if len(result) != len(tt.expected) {
t.Errorf("Expected %d metrics, got %d", len(tt.expected), len(result))
return
}

for i, metric := range result {
expected := tt.expected[i]

if metric.metricName != expected.metricName {
t.Errorf("Expected metric name %s, got %s", expected.metricName, metric.metricName)
}

for key, expectedValue := range expected.tags {
if actualValue, exists := metric.tags[key]; !exists {
t.Errorf("Expected tag %s to exist", key)
} else if actualValue != expectedValue {
t.Errorf("Expected tag %s=%s, got %s=%s", key, expectedValue, key, actualValue)
}
}

// Check that no unexpected tags exist (except for the ones we know should be removed)
for key := range metric.tags {
if key == "group" && metric.metricName == "apiserver_storage_objects" {
t.Errorf("Group tag should be removed for apiserver_storage_objects")
}
}
}
})
}
}
51 changes: 51 additions & 0 deletions plugins/inputs/prometheus/metrics_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package prometheus

import (
"log"
"strings"
"sync"
"time"

Expand Down Expand Up @@ -52,6 +53,9 @@ func (mh *metricsHandler) handle(pmb PrometheusMetricBatch) {
// do calculation: calculate delta for counter
pmb = mh.calculator.Calculate(pmb)

// Apply K8s 1.34 compatibility transformations
pmb = applyK8s134Compatibility(pmb)

// do merge: merge metrics which are sharing same tags
metricMaterials := mergeMetrics(pmb)

Expand Down Expand Up @@ -93,3 +97,50 @@ func (mh *metricsHandler) setEmfMetadata(mms []*metricMaterial) {
}
}
}

// applyK8s134Compatibility applies transformations for K8s 1.34 compatibility
// K8s 1.34 splits labels into group+resource and renames apiserver_storage_objects → apiserver_resource_objects
// We maintain pre-1.34 behavior with no user config changes
func applyK8s134Compatibility(pmb PrometheusMetricBatch) PrometheusMetricBatch {
for _, pm := range pmb {
// Rename: If metric name == apiserver_resource_objects, report it as apiserver_storage_objects
if pm.metricName == "apiserver_resource_objects" {
pm.metricName = "apiserver_storage_objects"

// Set legacy resource = resource + (group != "" ? "." + group : ""); drop/ignore group
if resource, hasResource := pm.tags["resource"]; hasResource {
if group, hasGroup := pm.tags["group"]; hasGroup && group != "" {
pm.tags["resource"] = resource + "." + group
}
delete(pm.tags, "group")
}
}

// Label shims for control-plane metrics that carry a resource label
if resource, hasResource := pm.tags["resource"]; hasResource {
group := pm.tags["group"] // treat missing group as ""

// Add resource_prefix = resource + (group != "" ? "." + group : "")
if group != "" {
pm.tags["resource_prefix"] = resource + "." + group
} else {
pm.tags["resource_prefix"] = resource
}

// For etcd_request metrics, set type = resource + (group != "" ? "." + group : "")
if strings.HasPrefix(pm.metricName, "etcd_request") {
if group != "" {
pm.tags["type"] = resource + "." + group
} else {
pm.tags["type"] = resource
}
}

// For apiserver_watch_ metrics, set kind = resource
if strings.HasPrefix(pm.metricName, "apiserver_watch_") {
pm.tags["kind"] = resource
}
}
}
return pmb
}
Loading