feat(uat): add argocd deployer variant on aws-h100 training

.github/workflows/uat-aws.yaml

@@ -68,6 +68,13 @@ on:
         description: 'Skip UAT test execution (manual runs only; the nightly batch always tests).'
         type: boolean
         default: false
+      deployer:
+        # Empty = <accelerator>-<intent>-config.yaml (helmfile). Non-empty
+        # inserts -<deployer> before -config.yaml (e.g. deployer=argocd →
+        # h100-training-argocd-config.yaml). See issue #2194.
+        description: 'Deployer variant (empty = helmfile).'
+        type: string
+        default: ''
 
 permissions:
   contents: read
@@ -139,10 +146,18 @@ jobs:
         && format('aicr-uat-day-{0}-{1}-{2}', inputs.slug, inputs.slot, github.run_id)
         || format('aicr-uat-{0}', github.run_id) }}
       CLUSTER_CONFIG: ${{ inputs.cluster_config_path }}
-      # The AICRConfig is selected by accelerator + intent
-      # (<accelerator>-<intent>-config.yaml); both intents drive the same
-      # cluster-config (GPU pool from the reservation, system/CPU pools dynamic).
-      TEST_CONFIG: ${{ inputs.test_config_dir }}/${{ inputs.accelerator }}-${{ inputs.intent }}-config.yaml
+      # The AICRConfig is selected by accelerator + intent + (optional) deployer:
+      #   deployer='' → <accelerator>-<intent>-config.yaml (helmfile, the default;
+      #                 backwards-compatible with every existing cell)
+      #   deployer=X  → <accelerator>-<intent>-X-config.yaml (e.g. argocd, issue #2194)
+      # Both variants drive the same cluster-config (GPU pool from the
+      # reservation, system/CPU pools dynamic). The step-level check below asserts
+      # the resolved file exists so an unwired deployer variant fails early
+      # (before any provisioning) rather than deep in phase_prep.
+      TEST_CONFIG: >-
+        ${{ inputs.test_config_dir }}/${{ inputs.accelerator }}-${{ inputs.intent }}${{
+          inputs.deployer != '' && format('-{0}', inputs.deployer) || ''
+        }}-config.yaml
       # v0.4.27 — required for node-group taints in the cluster-config schema
       # (GPU pool carries skyhook.nvidia.com=runtime-required:NoSchedule until
       # NodeWrite finishes tuning + reboot). Earlier v0.4.x pins predate taint
@@ -190,6 +205,7 @@ jobs:
           LIFECYCLE: ${{ inputs.lifecycle }}
           SLUG: ${{ inputs.slug }}
           SLOT: ${{ inputs.slot }}
+          DEPLOYER: ${{ inputs.deployer }}
         run: |
           set -euo pipefail
           case "${INTENT}" in
@@ -233,6 +249,15 @@ jobs:
             echo "::error::${CLUSTER_CONFIG} has no .deployment.tenancy"; exit 1
           fi
           echo "AWS_ACCOUNT_ID=${TENANCY}" >> "${GITHUB_ENV}"
+          # Validate deployer after the account export (which teardown depends on).
+          # workflow_dispatch constrains this to a choice, but workflow_call passes
+          # free strings, so a typo from a caller must surface as a clear error here
+          # rather than as a missing test-config file below or a mysterious dispatch
+          # deep in phases.sh. Allowlist mirrors uat-run.yaml's choice input.
+          case "${DEPLOYER}" in
+            ''|argocd) ;;
+            *) echo "::error::unsupported deployer '${DEPLOYER}' (want empty|argocd)"; exit 1 ;;
+          esac
           # The test config only matters for the run phases (nightly / daytime-up
           # provision+deploy). daytime-down just tears the held cluster down and
           # never reads it, so skip the existence check there — a missing sibling
@@ -685,21 +710,34 @@ jobs:
           aws-region: ${{ env.AWS_REGION }}
           role-session-name: GitHubActions-UAT-AICR-Install
 
-      - name: UAT - install (helmfile apply)
+      - name: UAT - install (helmfile apply or argocd sync)
         id: install
         if: steps.prep.outcome == 'success'
-        # helmfile apply (up to HELMFILE_TIMEOUT_SECONDS, 20m) + the post-install
-        # readiness gate, which runs `aicr validate --phase deployment` until it
-        # passes READINESS_CONSECUTIVE_PASSES times (up to READINESS_TIMEOUT_SECONDS,
-        # 60m, spanning nodewright tuning + reboots, which can run past 30m on a
-        # cold GPU node). Sized to exceed one helmfile attempt + the full gate
-        # window (20 + 60 = 80, + margin) so the gate's own fail-closed path runs
-        # before GitHub Actions kills the step.
-        timeout-minutes: 90
+        # Step-level timeout must fit the LARGER of the two deployer branches,
+        # because both dispatch from `./run install` and share this cap:
+        #   helmfile: HELMFILE_TIMEOUT_SECONDS 20m + gate 60m = 80m
+        #   argocd:   helm-install 5m + apply+sync 30m + root-grace 2m + gate 60m = 97m
+        # ARGOCD_HELM_TIMEOUT_SECONDS + ARGOCD_SYNC_TIMEOUT_SECONDS +
+        # ARGOCD_ROOT_APP_GRACE_SECONDS + READINESS_TIMEOUT_SECONDS in
+        # tests/uat/lib/phases.sh drive the argocd side. Sized at 110m so the
+        # argocd fail-closed paths (each shared-budget retry loop's `::error::`)
+        # run before GitHub Actions kills the step. The gate's own fail-closed
+        # path still fires within budget on both branches.
+        timeout-minutes: 110
         shell: bash
         env:
           AICR_BIN: ${{ github.workspace }}/aicr
           RUN_ID: ${{ github.run_id }}
+          # Consumed only by the argocd deployer branch (install_argocd in
+          # tests/uat/lib/phases.sh), which provisions an in-cluster ghcr.io
+          # repo-creds Secret so Argo CD's repo-server can pull the bundle
+          # pushed by phase_prep. GITHUB_ACTOR is auto-injected by the
+          # Actions runtime; GITHUB_TOKEN is not, so pass it explicitly.
+          # The token is scoped to `packages: write` at the job level (see
+          # permissions block above), which is exactly what the Secret
+          # needs. The helmfile branch is unaffected — install_helmfile
+          # never reads GITHUB_TOKEN.
+          GITHUB_TOKEN: ${{ github.token }}
         run: ./tests/uat/aws/run install "${TEST_CONFIG}"
 
       # Runs ALL validate phases (deployment + conformance + performance), not

.github/workflows/uat-run.yaml

@@ -84,6 +84,16 @@ on:
         description: 'Skip UAT test execution (manual debugging only).'
         type: boolean
         default: false
+      deployer:
+        # Empty = default helmfile via <accelerator>-<intent>-config.yaml.
+        # Non-empty selects <accelerator>-<intent>-<deployer>-config.yaml
+        # (e.g. argocd → h100-training-argocd-config.yaml). Only aws-h100
+        # training ships an argocd variant today; other cells fail with a
+        # missing test-config. See issue #2194.
+        description: 'Deployer variant (empty = helmfile). See workflow file for the variant matrix.'
+        type: choice
+        options: ['', argocd]
+        default: ''
   workflow_call:
     inputs:
       reservation:
@@ -110,6 +120,9 @@ on:
       skip_tests:
         type: boolean
         default: false
+      deployer:
+        type: string
+        default: ''
 
 permissions:
   contents: read
@@ -223,6 +236,7 @@ jobs:
       accelerator: ${{ needs.resolve.outputs.accelerator }}
       skip_delete: ${{ inputs.skip_delete }}
       skip_tests: ${{ inputs.skip_tests }}
+      deployer: ${{ inputs.deployer }}
 
   run-gcp:
     needs: resolve

docs/contributor/uat.md

@@ -84,6 +84,22 @@ The single nightly cron (`uat-nightly-batch.yaml`, `0 4 * * *`) runs **both inte
 
 Semantics: **`main` is never gated** (it is built from source and carries the newest fixes, so it always runs every listed intent); a **release** cell drops any intent whose minimum version is newer than the tag (semver; a tag `>=` the minimum runs). The gate lives in the schedule (`uat-broker schedule` attaches each cell's eligible `intents`), so the controller simply never dispatches a gated `(version × intent)` — no per-version workflow logic. Pointing the floor at a **not-yet-tagged** release is intentional and self-resolving: until that release ships, the intent runs on **`main` only** (green, continuous coverage of the fix), and the release enrolls automatically once it exists. `Validate` rejects a floor for an intent the row does not run, or a non-semver value. Bump the floor if the real first-fixed tag differs — an over-low floor surfaces as a visible red (safe), an over-high floor silently skips a good release (bump down).
 
+## Selecting the deployer
+
+The `deployer` input picks which deployer variant of the intent's test config the pipeline consumes. Left empty (the default), the pipeline resolves `tests/uat/<cloud>/tests/<accelerator>-<intent>-config.yaml` — the helmfile config every existing cell has always run against. Non-empty, it resolves `<accelerator>-<intent>-<deployer>-config.yaml` — for example `deployer=argocd` on `aws-h100` training loads `tests/uat/aws/tests/h100-training-argocd-config.yaml`.
+
+```bash
+# ArgoCD variant of the aws-h100 training cell (issue #2194)
+gh workflow run uat-run.yaml --repo NVIDIA/aicr --ref main \
+  -f reservation=aws-h100 -f intent=training -f deployer=argocd
+```
+
+The AICRConfig field `spec.bundle.deployment.deployer` is the source of truth `phase_prep`/`phase_install` read; the workflow input is only how the correct config file is *selected*. `phases.sh:phase_install` dispatches to `install_helmfile` (helmfile lane, unchanged) or `install_argocd` (Argo CD install + repo-creds Secret from `GITHUB_TOKEN` + `kubectl apply` of the `nvidia-stack` app-of-apps + terminal-pass wait on every `Application`). The post-install readiness gate is deployer-agnostic — it validates deployed cluster state (`aicr validate --phase deployment`), not the deployment mechanism — so a green ArgoCD cell means the GitOps deploy path converges on the same operator-managed stack the helmfile lane validates.
+
+**How the bundle reaches Argo CD.** `phase_prep` calls `aicr bundle --output oci://ghcr.io/nvidia/aicr-bundle-scratch/<recipe>:run-<id> --repo oci://ghcr.io/nvidia/aicr-bundle-scratch` — the `--output` flag pushes the rendered bundle to GHCR (the job already has `packages: write`), and `--repo` sets the `source.repoURL` baked into every generated `Application`. `install_argocd` provisions a prefix-matched `argocd.argoproj.io/secret-type: repo-creds` Secret from `GITHUB_TOKEN` so Argo CD's repo-server can pull the pushed artifact. Concurrent runs on the same recipe are isolated by the `:run-<id>` tag.
+
+**Coverage today (issue #2194).** Only `aws-h100` training carries a `-argocd` config file; other cells fail closed in the `Validate inputs` step with `test config not found` if dispatched with `deployer=argocd`. Nightly enrollment is deliberately deferred until a manual dispatch is green on hardware — mirroring the `azure-h100` (#1722) and `kind-h100` (#1843) onboarding pattern. The `argocd-helm` variant, and extension to gcp/azure/kind, are separate follow-ups.
+
 ## Cluster lifecycles
 
 The `lifecycle` input selects one of three cluster lifecycles, all sharing the reservation lease:

tests/uat/aws/tests/h100-training-argocd-config.yaml

@@ -0,0 +1,106 @@
+# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# AICRConfig consumed by `aicr {snapshot,recipe,bundle,validate} --config`.
+#
+# ArgoCD deployer variant of h100-training-config.yaml. The only functional
+# difference is `spec.bundle.deployment.deployer: argocd` — every other field
+# (recipe criteria, scheduling, snapshot/validate agent) is identical to the
+# helmfile sibling so a green run here is direct evidence the `--deployer
+# argocd` GitOps path works on the same recipe the helmfile lane validates.
+#
+# See ../../lib/phases.sh (install_argocd) for the install-phase flow:
+# `aicr bundle` pushes the same content to ghcr.io/nvidia/aicr-bundle-scratch
+# and Argo CD's repo-server pulls from that OCI ref via a prefix-matched
+# repo-creds Secret provisioned in-cluster from the workflow's GITHUB_TOKEN.
+#
+# Manual dispatch only for now (issue #2194): enroll in the nightly batch as
+# a separate follow-up after a green manual run mirrors the azure-h100 /
+# kind onboarding pattern (PRs #1722, #1843).
+#
+# Evidence push target is deliberately shared with the helmfile config: the
+# recipe (and therefore the evidence coordinate) is identical, and RUN_ID
+# tag isolation prevents same-recipe cross-run collisions. If the dashboard
+# grows a deployer facet later, tag the run there — not by forking the OCI
+# repo per deployer.
+kind: AICRConfig
+apiVersion: aicr.run/v1alpha2
+metadata:
+  name: eks-h100-training-uat-argocd
+
+spec:
+  snapshot:
+    output:
+      path: snapshot.yaml
+    agent:
+      namespace: aicr-validation
+      nodeSelector:
+        nodeGroup: gpu-worker
+      tolerations:
+        - dedicated=worker-workload:NoSchedule
+        - dedicated=worker-workload:NoExecute
+        - nvidia.com/gpu=present:NoSchedule
+        - skyhook.nvidia.com=runtime-required:NoSchedule
+
+  recipe:
+    criteria:
+      service: eks
+      accelerator: h100
+      os: ubuntu
+      intent: training
+      platform: kubeflow
+    output:
+      path: recipe.yaml
+
+  bundle:
+    input:
+      recipe: recipe.yaml
+    output:
+      # phase_prep overrides --output at CLI time with an oci:// target
+      # (ARGOCD_OCI_PREFIX in lib/phases.sh + a run-scoped tag). The local
+      # ./bundle directory still receives app-of-apps.yaml so `kubectl apply`
+      # has a manifest to consume.
+      target: ./bundle
+    deployment:
+      deployer: argocd
+    scheduling:
+      acceleratedNodeSelector:
+        nodeGroup: gpu-worker
+      acceleratedNodeTolerations:
+        - dedicated=worker-workload:NoSchedule
+        - dedicated=worker-workload:NoExecute
+        - nvidia.com/gpu=present:NoSchedule
+      systemNodeSelector:
+        nodeGroup: system-worker
+      systemNodeTolerations:
+        - dedicated=system-workload:NoSchedule
+        - dedicated=system-workload:NoExecute
+      storageClass: ""
+
+  validate:
+    execution:
+      failFast: true
+    input:
+      recipe: recipe.yaml
+      snapshot: snapshot.yaml
+    agent:
+      namespace: aicr-validation
+      tolerations:
+        - dedicated=worker-workload:NoSchedule
+        - dedicated=worker-workload:NoExecute
+        - nvidia.com/gpu=present:NoSchedule
+    evidence:
+      attestation:
+        out: ./evidence
+        push: ghcr.io/nvidia/aicr-evidence/h100-eks-ubuntu-training-kubeflow