From e8ef333081c2d8a8ae2c390df2754f2e0bd22c2a Mon Sep 17 00:00:00 2001
From: Evan Nemerson <evan.nemerson@cloudzero.com>
Date: Wed, 12 Nov 2025 15:55:03 -0500
Subject: [PATCH] CP-35087: Upgrade Prometheus to 3.x with backward
 compatibility
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Prometheus 3.0 was released and includes performance improvements and new
features. We want to upgrade to the latest stable version (3.7.3) while
maintaining backward compatibility for customers who may be using custom
Prometheus 2.x images.

Impact:

Before: Chart used Prometheus 2.55.1 by default. Prometheus 3.x images would
produce warnings about unknown flag "--enable-feature=agent" since the agent
mode flag changed between versions.

After: Chart uses Prometheus 3.7.3 by default. Both 2.x and 3.x versions work
correctly with automatic flag detection based on image version.

Scope: All customers, but particularly those using custom Prometheus images
via server.image.tag or components.prometheus.image.tag overrides.

Implementation Approach:

Prometheus 3.x changed the agent mode flag from `--enable-feature=agent` to
just `--agent`. To support both versions seamlessly, we needed version-aware
flag selection that respects the same image tag fallback chain used for
actual image generation.

Solution:

1. Updated Chart.AppVersion from v2.55.1 to v3.7.3 in helm/Chart.yaml

2. Created cloudzero-agent.prometheusAgentFlag helper in helm/templates/_helpers.tpl
   that:
   - Checks if mode is "agent" or "federated" (using existing mode derivation logic)
   - Uses same tag fallback as image generation: server.image.tag →
     components.prometheus.image.tag → Chart.AppVersion
   - Returns "--enable-feature=agent" for v2.x tags
   - Returns "--agent" for v3.x and newer tags
   - Returns empty string for server/clustered modes

3. Updated helm/templates/agent-deploy.yaml and helm/templates/agent-daemonset.yaml
   to use the new helper instead of hardcoded flags

4. Updated helm/tests/agent_mode_derivation_test.yaml to expect "--agent" flag
   for default Chart.AppVersion (now 3.7.3)

5. Added comprehensive test suite in helm/tests/prometheus_version_flag_test.yaml
   covering:
   - Prometheus 2.x uses --enable-feature=agent
   - Prometheus 3.x uses --agent
   - server.image.tag precedence over components.prometheus.image.tag
   - Both agent and federated modes
   - Server mode (no agent flag)
   - Fallback to Chart.AppVersion

Validation:

- All 255 Helm unit tests pass (added 8 new tests)
- Schema validation tests pass
- Go unit tests pass
- Deployed to brahms cluster successfully
- Verified Prometheus 3.7.3 starts correctly in agent mode with no warnings
- Confirmed all expected metrics (container, node, pod, GPU) flowing correctly
- Zero dropped metrics in production deployment
- Manually tested both v2.x and v3.x tags produce correct flags
- Verified server.image.tag override precedence works correctly
---
 helm/Chart.yaml                            |  2 +-
 helm/templates/_helpers.tpl                | 31 +++++++++
 helm/templates/agent-daemonset.yaml        |  6 +-
 helm/templates/agent-deploy.yaml           |  5 +-
 helm/tests/agent_mode_derivation_test.yaml | 10 +--
 tests/helm/template/alloy.yaml             | 64 +++++++++---------
 tests/helm/template/cert-manager.yaml      | 70 +++++++++----------
 tests/helm/template/federated.yaml         | 78 +++++++++++-----------
 tests/helm/template/manifest.yaml          | 68 +++++++++----------
 9 files changed, 183 insertions(+), 151 deletions(-)

diff --git a/helm/Chart.yaml b/helm/Chart.yaml
index dc55dae7..740ac190 100644
--- a/helm/Chart.yaml
+++ b/helm/Chart.yaml
@@ -8,7 +8,7 @@ kubeVersion: ">= 1.21.0-0"
 maintainers:
   - name: CloudZero
     email: support@cloudzero.com
-appVersion: "v2.55.1"
+appVersion: "v3.7.3"
 dependencies:
   - name: kube-state-metrics
     version: "5.36.*"
diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl
index 4230bbce..a5f16b48 100644
--- a/helm/templates/_helpers.tpl
+++ b/helm/templates/_helpers.tpl
@@ -1294,3 +1294,34 @@ alloy-config.river
 prometheus.yml
 {{- end -}}
 {{- end -}}
+
+{{/*
+Get the appropriate Prometheus agent mode flag based on version and mode
+
+Determines whether Prometheus should run in agent mode and which flag to use:
+- Prometheus 2.x uses --enable-feature=agent
+- Prometheus 3.x uses --agent
+- Returns empty string if not in agent/federated mode
+
+The cloudzero-agent.Values.components.agent.mode helper already handles all the
+complex mode derivation logic, so we just check if it returns "agent" or "federated"
+and then determine the appropriate version-specific flag.
+
+Uses the same tag fallback chain as image generation:
+server.image.tag -> components.prometheus.image.tag -> Chart.AppVersion
+
+Usage: {{ include "cloudzero-agent.prometheusAgentFlag" . }}
+Returns: string (either "--agent", "--enable-feature=agent", or empty string)
+*/}}
+{{- define "cloudzero-agent.prometheusAgentFlag" -}}
+  {{- $mode := include "cloudzero-agent.Values.components.agent.mode" . -}}
+  {{- if or (eq $mode "agent") (eq $mode "federated") -}}
+    {{- /* Use same fallback chain as image generation: server.image.tag -> components.prometheus.image.tag -> Chart.AppVersion */ -}}
+    {{- $tag := .Values.server.image.tag | default .Values.components.prometheus.image.tag | default .Chart.AppVersion -}}
+    {{- if hasPrefix "v2." $tag -}}
+      --enable-feature=agent
+    {{- else -}}
+      --agent
+    {{- end -}}
+  {{- end -}}
+{{- end -}}
diff --git a/helm/templates/agent-daemonset.yaml b/helm/templates/agent-daemonset.yaml
index 670aaed6..4a0906b6 100644
--- a/helm/templates/agent-daemonset.yaml
+++ b/helm/templates/agent-daemonset.yaml
@@ -111,9 +111,9 @@ spec:
             - --web.enable-lifecycle
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
-            {{- /* In federated mode, default to Prometheus agent unless server.agentMode is explicitly false */ -}}
-            {{- if or (not .Values.server) (eq .Values.server.agentMode nil) (eq .Values.server.agentMode true) }}
-            - --enable-feature=agent
+            {{- $agentFlag := include "cloudzero-agent.prometheusAgentFlag" . }}
+            {{- if $agentFlag }}
+            - {{ $agentFlag }}
             {{- end }}
           ports:
             - containerPort: 9090
diff --git a/helm/templates/agent-deploy.yaml b/helm/templates/agent-deploy.yaml
index 67be1652..68080eea 100644
--- a/helm/templates/agent-deploy.yaml
+++ b/helm/templates/agent-deploy.yaml
@@ -148,8 +148,9 @@ spec:
                   - /checks/app/config/validator.yml
           args:
             {{ toYaml .Values.server.args | nindent 12}}
-            {{- if or (eq (include "cloudzero-agent.Values.components.agent.mode" .) "agent") (eq (include "cloudzero-agent.Values.components.agent.mode" .) "federated") }}
-            - --enable-feature=agent
+            {{- $agentFlag := include "cloudzero-agent.prometheusAgentFlag" . }}
+            {{- if $agentFlag }}
+            - {{ $agentFlag }}
             {{- end }}
             - --log.level={{ .Values.server.logging.level | default "info" }}
           ports:
diff --git a/helm/tests/agent_mode_derivation_test.yaml b/helm/tests/agent_mode_derivation_test.yaml
index 8baf8278..d98c0e42 100644
--- a/helm/tests/agent_mode_derivation_test.yaml
+++ b/helm/tests/agent_mode_derivation_test.yaml
@@ -68,7 +68,7 @@ tests:
       # Should have agent mode flag (default)
       - contains:
           path: spec.template.spec.containers[1].args
-          content: --enable-feature=agent
+          content: --agent
         template: templates/agent-deploy.yaml
 
   # Test automatic mode: server.agentMode=false derives mode as "server"
@@ -85,7 +85,7 @@ tests:
       # Should NOT have agent mode flag (server mode)
       - notContains:
           path: spec.template.spec.containers[1].args
-          content: --enable-feature=agent
+          content: --agent
         template: templates/agent-deploy.yaml
       # Should have Prometheus config (not Alloy)
       - isNotEmpty:
@@ -106,7 +106,7 @@ tests:
       # Should have agent mode flag (default)
       - contains:
           path: spec.template.spec.containers[1].args
-          content: --enable-feature=agent
+          content: --agent
         template: templates/agent-deploy.yaml
       # Should have Prometheus config (not Alloy)
       - isNotEmpty:
@@ -155,7 +155,7 @@ tests:
       # Should have agent mode flag
       - contains:
           path: spec.template.spec.containers[1].args
-          content: --enable-feature=agent
+          content: --agent
         template: templates/agent-deploy.yaml
 
   # Test that components.agent.mode="server" works
@@ -170,7 +170,7 @@ tests:
       # Should NOT have agent mode flag
       - notContains:
           path: spec.template.spec.containers[1].args
-          content: --enable-feature=agent
+          content: --agent
         template: templates/agent-deploy.yaml
 
   # Test that components.agent.mode="clustered" works
diff --git a/tests/helm/template/alloy.yaml b/tests/helm/template/alloy.yaml
index c2ae9896..d22f401e 100644
--- a/tests/helm/template/alloy.yaml
+++ b/tests/helm/template/alloy.yaml
@@ -31,7 +31,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -54,7 +54,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -77,7 +77,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -114,7 +114,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -130,7 +130,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -146,7 +146,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-api-key
@@ -164,7 +164,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-tls
@@ -180,7 +180,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-configuration
   namespace: cz-agent
@@ -408,7 +408,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -760,7 +760,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -1540,7 +1540,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-validator-configuration
   namespace: cz-agent
@@ -1623,7 +1623,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-webhook-configuration
   namespace: cz-agent
@@ -1853,7 +1853,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-server
 rules:
@@ -1945,7 +1945,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     checkov.io/skip_1: CKV_K8S_155
@@ -2013,7 +2013,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2036,7 +2036,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -2088,7 +2088,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cz-agent-cz-aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -2113,7 +2113,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     nginx.ingress.kubernetes.io/ssl-redirect: "false"
@@ -2232,7 +2232,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2253,7 +2253,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
     spec:
       
@@ -2504,7 +2504,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cz-agent-cz-aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   selector:
@@ -2523,7 +2523,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cz-agent-cz-aggregator
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
     spec:
       serviceAccountName: cz-agent-cz-server
@@ -2663,7 +2663,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Deployment annotations: Merge defaults with webhook-specific annotations
   # Supports monitoring, backup policies, and operational tooling integration
@@ -2683,7 +2683,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       annotations:
         checksum/config: DEADBEEF-FEED-FACE-CAFE-FEE10D15EA5E
@@ -2786,7 +2786,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-type: backfill
     job-category: onetime
@@ -2866,7 +2866,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -2879,7 +2879,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -2990,7 +2990,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3003,7 +3003,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3060,7 +3060,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3123,7 +3123,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-type: backfill
     job-category: cronjob
@@ -3207,7 +3207,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Certificate management annotations for automatic TLS certificate injection
   # When cert-manager is enabled, automatically injects CA bundle for webhook TLS validation
diff --git a/tests/helm/template/cert-manager.yaml b/tests/helm/template/cert-manager.yaml
index 40b3cf16..9de14cd8 100644
--- a/tests/helm/template/cert-manager.yaml
+++ b/tests/helm/template/cert-manager.yaml
@@ -31,7 +31,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -54,7 +54,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -77,7 +77,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -114,7 +114,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -130,7 +130,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -146,7 +146,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-api-key
@@ -164,7 +164,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-configuration
   namespace: cz-agent
@@ -369,7 +369,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -721,7 +721,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -1500,7 +1500,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-validator-configuration
   namespace: cz-agent
@@ -1583,7 +1583,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-webhook-configuration
   namespace: cz-agent
@@ -1813,7 +1813,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-server
 rules:
@@ -1905,7 +1905,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     checkov.io/skip_1: CKV_K8S_155
@@ -1973,7 +1973,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -1996,7 +1996,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -2048,7 +2048,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cz-agent-cz-aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -2073,7 +2073,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     nginx.ingress.kubernetes.io/ssl-redirect: "false"
@@ -2192,7 +2192,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2213,7 +2213,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
     spec:
       
@@ -2339,7 +2339,7 @@ spec:
         # Prometheus server container
         - name: cloudzero-agent-server
           
-          image: "quay.io/prometheus/prometheus:v2.55.1"
+          image: "quay.io/prometheus/prometheus:v3.7.3"
           imagePullPolicy: "IfNotPresent"
           lifecycle:
             postStart:
@@ -2364,7 +2364,7 @@ spec:
             - --web.enable-lifecycle
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
-            - --enable-feature=agent
+            - --agent
             - --log.level=info
           ports:
             - containerPort: 9090
@@ -2457,7 +2457,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cz-agent-cz-aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   selector:
@@ -2476,7 +2476,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cz-agent-cz-aggregator
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
     spec:
       serviceAccountName: cz-agent-cz-server
@@ -2616,7 +2616,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Deployment annotations: Merge defaults with webhook-specific annotations
   # Supports monitoring, backup policies, and operational tooling integration
@@ -2636,7 +2636,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       annotations:
         checksum/config: DEADBEEF-FEED-FACE-CAFE-FEE10D15EA5E
@@ -2739,7 +2739,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-type: backfill
     job-category: onetime
@@ -2819,7 +2819,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -2832,7 +2832,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -2943,7 +2943,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -2956,7 +2956,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3013,7 +3013,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-type: backfill
     job-category: cronjob
@@ -3096,7 +3096,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -3108,7 +3108,7 @@ spec:
       app.kubernetes.io/managed-by: Helm
       app.kubernetes.io/name: cloudzero-agent
       app.kubernetes.io/part-of: cloudzero-agent
-      app.kubernetes.io/version: v2.55.1
+      app.kubernetes.io/version: v3.7.3
       helm.sh/chart: cloudzero-agent-1.1.0-dev
   privateKey:
     algorithm: RSA
@@ -3135,7 +3135,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -3153,7 +3153,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Certificate management annotations for automatic TLS certificate injection
   # When cert-manager is enabled, automatically injects CA bundle for webhook TLS validation
diff --git a/tests/helm/template/federated.yaml b/tests/helm/template/federated.yaml
index b55b701e..5d824927 100644
--- a/tests/helm/template/federated.yaml
+++ b/tests/helm/template/federated.yaml
@@ -31,7 +31,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -54,7 +54,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -77,7 +77,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -114,7 +114,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -130,7 +130,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -146,7 +146,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-api-key
@@ -164,7 +164,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-tls
@@ -180,7 +180,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-configuration
   namespace: cz-agent
@@ -330,7 +330,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-daemonset-cm
   namespace: cz-agent
@@ -438,7 +438,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -790,7 +790,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -1569,7 +1569,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-validator-configuration
   namespace: cz-agent
@@ -1652,7 +1652,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-webhook-configuration
   namespace: cz-agent
@@ -1882,7 +1882,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-server
 rules:
@@ -1974,7 +1974,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     checkov.io/skip_1: CKV_K8S_155
@@ -2042,7 +2042,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2065,7 +2065,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -2117,7 +2117,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cz-agent-cz-aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -2142,7 +2142,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     nginx.ingress.kubernetes.io/ssl-redirect: "false"
@@ -2170,7 +2170,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-daemonset
@@ -2190,7 +2190,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
     spec:
       
@@ -2275,7 +2275,7 @@ spec:
               readOnly: true
         - name: cloudzero-agent-server
           
-          image: "quay.io/prometheus/prometheus:v2.55.1"
+          image: "quay.io/prometheus/prometheus:v3.7.3"
           imagePullPolicy: "IfNotPresent"
           env:
             - name: NODE_NAME
@@ -2287,7 +2287,7 @@ spec:
             - --web.enable-lifecycle
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
-            - --enable-feature=agent
+            - --agent
           ports:
             - containerPort: 9090
           readinessProbe:
@@ -2460,7 +2460,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2481,7 +2481,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
     spec:
       
@@ -2607,7 +2607,7 @@ spec:
         # Prometheus server container
         - name: cloudzero-agent-server
           
-          image: "quay.io/prometheus/prometheus:v2.55.1"
+          image: "quay.io/prometheus/prometheus:v3.7.3"
           imagePullPolicy: "IfNotPresent"
           lifecycle:
             postStart:
@@ -2632,7 +2632,7 @@ spec:
             - --web.enable-lifecycle
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
-            - --enable-feature=agent
+            - --agent
             - --log.level=info
           ports:
             - containerPort: 9090
@@ -2725,7 +2725,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cz-agent-cz-aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   selector:
@@ -2744,7 +2744,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cz-agent-cz-aggregator
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
     spec:
       serviceAccountName: cz-agent-cz-server
@@ -2884,7 +2884,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Deployment annotations: Merge defaults with webhook-specific annotations
   # Supports monitoring, backup policies, and operational tooling integration
@@ -2904,7 +2904,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       annotations:
         checksum/config: DEADBEEF-FEED-FACE-CAFE-FEE10D15EA5E
@@ -3007,7 +3007,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-type: backfill
     job-category: onetime
@@ -3087,7 +3087,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3100,7 +3100,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3211,7 +3211,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3224,7 +3224,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3281,7 +3281,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3344,7 +3344,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-type: backfill
     job-category: cronjob
@@ -3428,7 +3428,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Certificate management annotations for automatic TLS certificate injection
   # When cert-manager is enabled, automatically injects CA bundle for webhook TLS validation
diff --git a/tests/helm/template/manifest.yaml b/tests/helm/template/manifest.yaml
index e8030219..e3fb2747 100644
--- a/tests/helm/template/manifest.yaml
+++ b/tests/helm/template/manifest.yaml
@@ -31,7 +31,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -54,7 +54,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -77,7 +77,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -114,7 +114,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -130,7 +130,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -146,7 +146,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-api-key
@@ -164,7 +164,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-tls
@@ -180,7 +180,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-configuration
   namespace: cz-agent
@@ -385,7 +385,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -737,7 +737,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 data:
@@ -1516,7 +1516,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-validator-configuration
   namespace: cz-agent
@@ -1599,7 +1599,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-webhook-configuration
   namespace: cz-agent
@@ -1829,7 +1829,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   name: cz-agent-cz-server
 rules:
@@ -1921,7 +1921,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     checkov.io/skip_1: CKV_K8S_155
@@ -1989,7 +1989,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2012,7 +2012,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-webhook-init-cert
@@ -2064,7 +2064,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cz-agent-cz-aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
 spec:
@@ -2089,7 +2089,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   annotations:
     nginx.ingress.kubernetes.io/ssl-redirect: "false"
@@ -2208,7 +2208,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   
   name: cz-agent-cz-server
@@ -2229,7 +2229,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
     spec:
       
@@ -2355,7 +2355,7 @@ spec:
         # Prometheus server container
         - name: cloudzero-agent-server
           
-          image: "quay.io/prometheus/prometheus:v2.55.1"
+          image: "quay.io/prometheus/prometheus:v3.7.3"
           imagePullPolicy: "IfNotPresent"
           lifecycle:
             postStart:
@@ -2380,7 +2380,7 @@ spec:
             - --web.enable-lifecycle
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
-            - --enable-feature=agent
+            - --agent
             - --log.level=info
           ports:
             - containerPort: 9090
@@ -2473,7 +2473,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cz-agent-cz-aggregator
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   selector:
@@ -2492,7 +2492,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cz-agent-cz-aggregator
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
     spec:
       serviceAccountName: cz-agent-cz-server
@@ -2632,7 +2632,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Deployment annotations: Merge defaults with webhook-specific annotations
   # Supports monitoring, backup policies, and operational tooling integration
@@ -2652,7 +2652,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       annotations:
         checksum/config: DEADBEEF-FEED-FACE-CAFE-FEE10D15EA5E
@@ -2755,7 +2755,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-type: backfill
     job-category: onetime
@@ -2835,7 +2835,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -2848,7 +2848,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -2959,7 +2959,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -2972,7 +2972,7 @@ spec:
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: cloudzero-agent
         app.kubernetes.io/part-of: cloudzero-agent
-        app.kubernetes.io/version: v2.55.1
+        app.kubernetes.io/version: v3.7.3
         helm.sh/chart: cloudzero-agent-1.1.0-dev
       
     spec:
@@ -3029,7 +3029,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
 spec:
   template:
@@ -3092,7 +3092,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
     job-type: backfill
     job-category: cronjob
@@ -3176,7 +3176,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: cloudzero-agent
     app.kubernetes.io/part-of: cloudzero-agent
-    app.kubernetes.io/version: v2.55.1
+    app.kubernetes.io/version: v3.7.3
     helm.sh/chart: cloudzero-agent-1.1.0-dev
   # Certificate management annotations for automatic TLS certificate injection
   # When cert-manager is enabled, automatically injects CA bundle for webhook TLS validation