chore: cherry-pick sync upstream to downstream rhoai-3.5

deployment/base/maas-controller/overlays/xks/cert-manager/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - maas-webhook-certificate.yaml

deployment/base/maas-controller/overlays/xks/cert-manager/maas-webhook-certificate.yaml

@@ -0,0 +1,16 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: maas-controller-webhook-server
+spec:
+  commonName: maas-controller-webhook-service.$(NAMESPACE).svc
+  dnsNames:
+    - maas-controller-webhook-service.$(NAMESPACE).svc
+    - maas-controller-webhook-service.$(NAMESPACE).svc.cluster.local
+  duration: 8760h # 1 year
+  renewBefore: 720h # 30 days
+  issuerRef:
+    name: $(ISSUER_REF_NAME)
+    kind: $(ISSUER_REF_KIND)
+    group: $(ISSUER_REF_GROUP)
+  secretName: maas-controller-webhook-cert

deployment/base/maas-controller/overlays/xks/kustomization.yaml

@@ -0,0 +1,122 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../crd
+  - ../../rbac
+  - ../../manager
+  - ../../webhook
+  - cert-manager/
+  # monitoring is excluded on xKS (OpenShift-specific ServiceMonitor/PodMonitor)
+
+labels:
+  - pairs:
+      app.kubernetes.io/name: maas-controller
+      app.kubernetes.io/component: models-as-a-service
+      app.kubernetes.io/part-of: models-as-a-service
+
+configMapGenerator:
+  - name: maas-parameters
+    envs:
+      - params.env
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+replacements:
+  - source:
+      kind: ConfigMap
+      name: maas-parameters
+      fieldPath: data.maas-controller-image
+    targets:
+      - select:
+          kind: Deployment
+          name: maas-controller
+        fieldPaths:
+          - spec.template.spec.containers.[name=manager].image
+  - source:
+      kind: ConfigMap
+      name: maas-parameters
+      fieldPath: data.infrastructure-namespace
+    targets:
+      - select:
+          kind: Deployment
+          name: maas-controller
+        fieldPaths:
+          - spec.template.spec.containers.[name=manager].env.[name=INFRA_NAMESPACE].value
+  - source:
+      kind: ConfigMap
+      name: maas-parameters
+      fieldPath: data.gateway-namespace
+    targets:
+      - select:
+          kind: Deployment
+          name: maas-controller
+        fieldPaths:
+          - spec.template.spec.containers.[name=manager].env.[name=GATEWAY_NAMESPACE].value
+
+patches:
+  # Remove OCP service-serving cert annotation (cert-manager handles TLS on xKS)
+  - target:
+      kind: Service
+      name: maas-controller-webhook-service
+    patch: |
+      - op: remove
+        path: /metadata/annotations/service.beta.openshift.io~1serving-cert-secret-name
+      - op: remove
+        path: /metadata/annotations/service.beta.openshift.io~1inject-cabundle
+
+  # Replace OCP CA injection with cert-manager CA injection on webhook
+  - target:
+      kind: ValidatingWebhookConfiguration
+      name: maas-validating-webhook-configuration
+    patch: |
+      - op: remove
+        path: /metadata/annotations/service.beta.openshift.io~1inject-cabundle
+      - op: add
+        path: /metadata/annotations/cert-manager.io~1inject-ca-from
+        value: "$(NAMESPACE)/maas-controller-webhook-server"
+
+  # Set platform manifests path so maas-controller uses xKS tenant overlay
+  - target:
+      kind: Deployment
+      name: maas-controller
+    patch: |
+      - op: add
+        path: /spec/template/spec/containers/0/env/-
+        value:
+          name: MAAS_PLATFORM_MANIFESTS
+          value: /maas-api/deploy/overlays/xks
+
+vars:
+  - name: NAMESPACE
+    objref:
+      apiVersion: v1
+      kind: ConfigMap
+      name: maas-parameters
+    fieldref:
+      fieldpath: data.namespace
+  - name: ISSUER_REF_NAME
+    objref:
+      apiVersion: v1
+      kind: ConfigMap
+      name: maas-parameters
+    fieldref:
+      fieldpath: data.ISSUER_REF_NAME
+  - name: ISSUER_REF_KIND
+    objref:
+      apiVersion: v1
+      kind: ConfigMap
+      name: maas-parameters
+    fieldref:
+      fieldpath: data.ISSUER_REF_KIND
+  - name: ISSUER_REF_GROUP
+    objref:
+      apiVersion: v1
+      kind: ConfigMap
+      name: maas-parameters
+    fieldref:
+      fieldpath: data.ISSUER_REF_GROUP
+
+configurations:
+  - params.yaml

deployment/base/maas-controller/overlays/xks/params.env

@@ -0,0 +1,11 @@
+maas-api-image=quay.io/opendatahub/maas-api:odh-stable
+maas-controller-image=quay.io/opendatahub/maas-controller:odh-stable
+payload-processing-image=quay.io/opendatahub/odh-ai-gateway-payload-processing:odh-stable
+maas-api-key-cleanup-image=registry.redhat.io/ubi9/ubi-minimal:9.7
+monitoring-namespace=
+infrastructure-namespace=AUTO
+gateway-namespace=redhat-ods-applications
+namespace=redhat-ods-applications
+ISSUER_REF_NAME=rhai-ca-issuer
+ISSUER_REF_KIND=ClusterIssuer
+ISSUER_REF_GROUP=cert-manager.io

deployment/base/maas-controller/overlays/xks/params.yaml

@@ -0,0 +1,13 @@
+varReference:
+  - path: spec/commonName
+    kind: Certificate
+  - path: spec/dnsNames
+    kind: Certificate
+  - path: spec/issuerRef/name
+    kind: Certificate
+  - path: spec/issuerRef/kind
+    kind: Certificate
+  - path: spec/issuerRef/group
+    kind: Certificate
+  - path: metadata/annotations
+    kind: ValidatingWebhookConfiguration

deployment/base/maas-controller/rbac/clusterrole.yaml

@@ -118,6 +118,17 @@ rules:
   - create
   - delete
   - get
+- apiGroups:
+  - cert-manager.io
+  resources:
+  - certificates
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - watch
 - apiGroups:
   - config.openshift.io
   resources:

deployment/base/networking/maas/maas-gateway-api.yaml

@@ -21,7 +21,7 @@ spec:
      protocol: HTTPS
      allowedRoutes:
        namespaces:
-         from: All
+         from: Same
      tls:
        certificateRefs:
        - group: ''

deployment/base/payload-processing/manager/envoy-filter.yaml

@@ -4,10 +4,19 @@ metadata:
   name: payload-processing
   namespace: openshift-ingress
 spec:
-  targetRefs:
-    - group: gateway.networking.k8s.io
-      kind: Gateway
-      name: maas-default-gateway
+  # Must run AFTER Kuadrant's EnvoyFilter (default priority 0), which INSERT_BEFOREs
+  # envoy.filters.http.wasm relative to the router. Without a positive priority, istiod
+  # applies this resource first (same priority 0, older creationTimestamp) and the RHCL
+  # wasm anchors miss — ext_proc never enters the gateway filter chain (404 NR on
+  # body-routed /v1/*). Multi-tenant gateways are especially affected: their
+  # payload-processing-* EFs are often created before Kuadrant's per-gateway EF.
+  # ODH WasmPlugin anchors are unaffected: WasmPlugin is in the base chain before
+  # any EnvoyFilter patches.
+  # See: https://istio.io/latest/docs/reference/config/networking/envoy-filter/#EnvoyFilter
+  priority: 10
+  workloadSelector:
+    labels:
+      gateway.networking.k8s.io/gateway-name: maas-default-gateway
   configPatches:
     # Stage 1: must run BEFORE the WasmPlugin so auth sees X-Gateway-Model-Name.
     - applyTo: HTTP_FILTER
@@ -70,6 +79,7 @@ spec:
             message_timeout: 300s
     # RHCL 1.4 injects auth via envoy.filters.http.wasm (no WasmPlugin CR). Only one anchor
     # pair matches per cluster: WasmPlugin on ODH/community Kuadrant, wasm filter on RHCL 1.4.
+    # These patches require priority > Kuadrant's EF (see spec.priority above).
     - applyTo: HTTP_FILTER
       match:
         context: GATEWAY

deployment/base/payload-processing/manager/plugins-configmap.yaml

@@ -3,6 +3,11 @@ kind: ConfigMap
 metadata:
   name: payload-processing-plugins
   namespace: openshift-ingress
+  # Do NOT set opendatahub.io/managed=false here: PostRender drops any resource that
+  # already carries that annotation, which would prevent first-time creation.
+  # The tenant reconciler stamps managed=false after bootstrap/migrate so later
+  # SSA leaves this ConfigMap alone (see ApplyRendered). Set managed=true on the
+  # live ConfigMap to opt back into continuous reconciler management.
 data:
   custom-pre-processing-ipp-config.yaml: |
     apiVersion: llm-d.ai/v1alpha1
@@ -39,5 +44,10 @@ data:
         - pluginRef: model-provider-resolver
         - pluginRef: api-translation
         - pluginRef: apikey-injection
-        response:
-        - pluginRef: api-translation
+        # Response api-translation is off by default so SSE streaming for
+        # internal/OpenAI-compatible models is not held by a response processor.
+        # Re-enable for providers that need response rewrite (e.g. Anthropic):
+        #   response:
+        #   - pluginRef: api-translation
+        # See docs/content/install/external-model-setup.md (IPP response translation).
+        response: []

deployment/components/observability/observability/dashboards/usage-dashboard.yaml

@@ -174,6 +174,7 @@ spec:
       spec:
         display:
           name: Token consumption chart
+          # [2h] matches the Loki dashboards' step for consistent cross-dashboard granularity.
           description: >-
             Tokens over time by model or subscription.
             View by drives chart sum by (${view_by:raw}).
@@ -209,7 +210,7 @@ spec:
                   query: >-
                     round(sum by (${view_by:raw}) (increase(authorized_hits_total{user!="",
                     user=~"$user", subscription=~"$subscription",
-                    model=~"$model"}[30m])))
+                    model=~"$model"}[2h])))
     tokenConsumptionByUser:
       kind: Panel
       spec:

deployment/components/observability/usage-logs/envoy-otel-access-log.yaml

@@ -37,10 +37,9 @@ metadata:
     app.kubernetes.io/part-of: maas-observability
     app.kubernetes.io/managed-by: maas-controller
 spec:
-  targetRefs:
-  - group: gateway.networking.k8s.io
-    kind: Gateway
-    name: maas-default-gateway
+  workloadSelector:
+    labels:
+      gateway.networking.k8s.io/gateway-name: maas-default-gateway
   configPatches:
 
   # ── Cluster: gRPC endpoint to OTel Collector ──────────────────────────

deployment/components/observability/usage-logs/usage-logs-admin-dashboard.yaml

@@ -1,8 +1,9 @@
 # Admin usage dashboard — queries Loki structured logs via LogQL.
 # Uses $__range to bind LogQL windows to the native time picker.
-# LokiLabelValuesVariable for model/subscription dropdowns (COO 1.5+).
-# User filter uses TextVariable (regex input) to avoid Loki cardinality issues
-# from indexing high-cardinality user_id as a label. Admins enter regex patterns.
+# All three dropdowns (user, subscription, model) follow the dashboard time
+# picker. User filter uses LokiLogQLVariable with [$__range] — auto-populates
+# from structured metadata via count by (user_id), bypassing /label/values
+# (which requires stream labels). Subscription/model use LokiLabelValuesVariable.
 # customAllValue: ".*" on ListVariables ensures "All" matches entries with
 # absent labels (e.g. subscription when capturing is disabled). Do not remove.
 apiVersion: perses.dev/v1alpha1
@@ -21,13 +22,24 @@ spec:
       rate limiting, and per-user breakdown across all users and subscriptions.
   duration: 1h
   variables:
-    - kind: TextVariable
+    - kind: ListVariable
       spec:
         name: user
         display:
-          name: "User (regex)"
-          description: "Filter by user ID. Use regex: 'user-1' (single), 'user-1|user-2' (multiple), '.*' (all)"
-        defaultValue: ".*"
+          name: "User"
+          description: "Filter by user. Populated from the selected time range."
+        allowMultiple: true
+        allowAllValue: true
+        customAllValue: ".*"
+        defaultValue: "$__all"
+        plugin:
+          kind: LokiLogQLVariable
+          spec:
+            datasource:
+              kind: LokiDatasource
+              name: usage-logs-all
+            expr: 'count by (user_id) (count_over_time({service_name="models-as-a-service"} | user_id!="" | user_id!="-" | keep user_id [$__range]))'
+            labelName: user_id
     - kind: ListVariable
       spec:
         name: subscription
@@ -240,11 +252,11 @@ spec:
       spec:
         display:
           name: "Token consumption chart"
+          # [2h] is the minimal stable step. Loki split_queries_by_interval is 30m; 1h still produces
+          # gaps at chunk boundaries. 2h is the lowest value that returns consistent results.
           description: >-
             Tokens from successful requests (2xx) over time by model or subscription.
-            **View by** drives chart ``sum by (${view_by:raw})``. Table uses hardcoded ``sum by (model, subscription)``.
-            LogQL ``[30m]`` is the range window for ``sum_over_time`` (Perses time-series step).
-            Visual: OpenShift Perses pattern — ``line`` + ``areaOpacity: 1`` + ``stack: all``.
+            **View by** drives the chart grouping.
         plugin:
           kind: TimeSeriesChart
           spec:
@@ -277,7 +289,7 @@ spec:
                     sum by (${view_by:raw}) (sum_over_time({service_name="models-as-a-service",
                     subscription=~"$subscription", model=~"$model", response_type="hit"}
                     | user_id=~"$user"
-                    | unwrap tokens_total [30m]))
+                    | unwrap tokens_total [2h]))
 
     tokenConsumptionTable:
       kind: Panel

deployment/components/observability/usage-logs/usage-logs-dashboard.yaml

@@ -204,6 +204,8 @@ spec:
       spec:
         display:
           name: "Token consumption chart"
+          # [2h] is the minimal stable step. Loki split_queries_by_interval is 30m; 1h still produces
+          # gaps at chunk boundaries. 2h is the lowest value that returns consistent results.
           description: >-
             Tokens from successful requests (2xx) over time by model or subscription.
         plugin:
@@ -237,7 +239,7 @@ spec:
                   query: >-
                     sum by (${view_by:raw}) (sum_over_time({service_name="models-as-a-service",
                     subscription=~"$subscription", model=~"$model", response_type="hit"}
-                    | unwrap tokens_total [30m]))
+                    | unwrap tokens_total [2h]))
 
     tokenConsumptionTable:
       kind: Panel