CP-35087: Upgrade Prometheus to 3.x with backward compatibility

helm/Chart.yaml

@@ -8,7 +8,7 @@ kubeVersion: ">= 1.21.0-0"
 maintainers:
   - name: CloudZero
     email: [email protected]
-appVersion: "v2.55.1"
+appVersion: "v3.7.3"
 dependencies:
   - name: kube-state-metrics
     version: "5.36.*"

helm/templates/_helpers.tpl

@@ -1294,3 +1294,34 @@ alloy-config.river
 prometheus.yml
 {{- end -}}
 {{- end -}}
+
+{{/*
+Get the appropriate Prometheus agent mode flag based on version and mode
+
+Determines whether Prometheus should run in agent mode and which flag to use:
+- Prometheus 2.x uses --enable-feature=agent
+- Prometheus 3.x uses --agent
+- Returns empty string if not in agent/federated mode
+
+The cloudzero-agent.Values.components.agent.mode helper already handles all the
+complex mode derivation logic, so we just check if it returns "agent" or "federated"
+and then determine the appropriate version-specific flag.
+
+Uses the same tag fallback chain as image generation:
+server.image.tag -> components.prometheus.image.tag -> Chart.AppVersion
+
+Usage: {{ include "cloudzero-agent.prometheusAgentFlag" . }}
+Returns: string (either "--agent", "--enable-feature=agent", or empty string)
+*/}}
+{{- define "cloudzero-agent.prometheusAgentFlag" -}}
+  {{- $mode := include "cloudzero-agent.Values.components.agent.mode" . -}}
+  {{- if or (eq $mode "agent") (eq $mode "federated") -}}
+    {{- /* Use same fallback chain as image generation: server.image.tag -> components.prometheus.image.tag -> Chart.AppVersion */ -}}
+    {{- $tag := .Values.server.image.tag | default .Values.components.prometheus.image.tag | default .Chart.AppVersion -}}
+    {{- if hasPrefix "v2." $tag -}}
+      --enable-feature=agent
+    {{- else -}}
+      --agent
+    {{- end -}}
+  {{- end -}}
+{{- end -}}

helm/templates/agent-daemonset.yaml

@@ -111,9 +111,9 @@ spec:
             - --web.enable-lifecycle
             - --web.console.libraries=/etc/prometheus/console_libraries
             - --web.console.templates=/etc/prometheus/consoles
-            {{- /* In federated mode, default to Prometheus agent unless server.agentMode is explicitly false */ -}}
-            {{- if or (not .Values.server) (eq .Values.server.agentMode nil) (eq .Values.server.agentMode true) }}
-            - --enable-feature=agent
+            {{- $agentFlag := include "cloudzero-agent.prometheusAgentFlag" . }}
+            {{- if $agentFlag }}
+            - {{ $agentFlag }}
             {{- end }}
           ports:
             - containerPort: 9090

helm/templates/agent-deploy.yaml

@@ -148,8 +148,9 @@ spec:
                   - /checks/app/config/validator.yml
           args:
             {{ toYaml .Values.server.args | nindent 12}}
-            {{- if or (eq (include "cloudzero-agent.Values.components.agent.mode" .) "agent") (eq (include "cloudzero-agent.Values.components.agent.mode" .) "federated") }}
-            - --enable-feature=agent
+            {{- $agentFlag := include "cloudzero-agent.prometheusAgentFlag" . }}
+            {{- if $agentFlag }}
+            - {{ $agentFlag }}
             {{- end }}
             - --log.level={{ .Values.server.logging.level | default "info" }}
           ports:

helm/tests/agent_mode_derivation_test.yaml

@@ -68,7 +68,7 @@ tests:
       # Should have agent mode flag (default)
       - contains:
           path: spec.template.spec.containers[1].args
-          content: --enable-feature=agent
+          content: --agent
         template: templates/agent-deploy.yaml
 
   # Test automatic mode: server.agentMode=false derives mode as "server"
@@ -85,7 +85,7 @@ tests:
       # Should NOT have agent mode flag (server mode)
       - notContains:
           path: spec.template.spec.containers[1].args
-          content: --enable-feature=agent
+          content: --agent
         template: templates/agent-deploy.yaml
       # Should have Prometheus config (not Alloy)
       - isNotEmpty:
@@ -106,7 +106,7 @@ tests:
       # Should have agent mode flag (default)
       - contains:
           path: spec.template.spec.containers[1].args
-          content: --enable-feature=agent
+          content: --agent
         template: templates/agent-deploy.yaml
       # Should have Prometheus config (not Alloy)
       - isNotEmpty:
@@ -155,7 +155,7 @@ tests:
       # Should have agent mode flag
       - contains:
           path: spec.template.spec.containers[1].args
-          content: --enable-feature=agent
+          content: --agent
         template: templates/agent-deploy.yaml
 
   # Test that components.agent.mode="server" works
@@ -170,7 +170,7 @@ tests:
       # Should NOT have agent mode flag
       - notContains:
           path: spec.template.spec.containers[1].args
-          content: --enable-feature=agent
+          content: --agent
         template: templates/agent-deploy.yaml
 
   # Test that components.agent.mode="clustered" works