diff --git a/.github/workflows/uat-aws.yaml b/.github/workflows/uat-aws.yaml index 63133a02c..1fca6dfec 100644 --- a/.github/workflows/uat-aws.yaml +++ b/.github/workflows/uat-aws.yaml @@ -68,6 +68,14 @@ on: description: 'Skip UAT test execution (manual runs only; the nightly batch always tests).' type: boolean default: false + deployer: + # helmfile = --config.yaml (the existing + # filename shape every cell has always used). argocd = insert + # -argocd before -config.yaml (e.g. h100-training-argocd-config.yaml). + # See issue #2194. + description: 'Deployer variant.' + type: string + default: helmfile permissions: contents: read @@ -88,17 +96,28 @@ jobs: if: always() && github.repository == 'nvidia/aicr' runs-on: ubuntu-latest # Budget = uncapped non-UAT steps (build, validator image push, EKS - # provisioning, evidence upload ~30m) + UAT phase steps (prep 15 + install 90 - # + validate 40 + the intent-selected CUJ (train 25 OR serve 40) + verify 5 - # = up to 190) + ~8 failure-path debug collection (Collect cluster debug - # bundle) + the always()-run Destroy Cluster teardown. Only one of - # train/serve runs per intent, so the budget counts the larger (serve). - # Teardown = up to 3 retries × ~18m real destroy = up to ~54m. The teardown - # MUST fit in this budget: a job-level timeout cancels pending always() - # steps, so an undersized cap would skip teardown and leak the GPU node / - # capacity reservation even after a LATE failure. ~30 setup + 190 UAT + ~8 - # debug = 228, + ~54 teardown = 282, so 300 leaves the teardown clear - # headroom on a worst-case run. + # provisioning, evidence upload ~30m) + UAT phase steps + ~8 failure-path + # debug collection + the always()-run Destroy Cluster teardown (up to 3 + # retries × ~18m real destroy = up to ~54m). The teardown MUST fit in this + # cap: a job-level timeout cancels pending always() steps, so an undersized + # cap would skip teardown and leak the GPU node / capacity reservation + # even after a LATE failure. Only one of train/serve runs per intent, so + # the UAT phase-step count uses the larger. + # + # UAT-phase-step budgets by (deployer, intent), using the current + # timeout-minutes on each step below: + # helmfile+training: prep 15 + install 90 + validate 40 + train 25 + verify 5 = 175 + # helmfile+inference: prep 15 + install 90 + validate 40 + serve 40 + verify 5 = 190 + # argocd+training: prep 15 + install 110 + validate 40 + train 25 + verify 5 = 195 (issue #2194, aws-h100 only) + # argocd+inference: not shipped today; would be 15 + 110 + 40 + 40 + 5 = 210 (needs cap bump below) + # + # Total = ~30 setup + + ~8 debug + ~54 teardown. On today's + # worst-case (argocd+training or helmfile+inference), ~30 + max(195,190) + # + ~8 + ~54 = ~287, so 300 leaves ~13m of teardown headroom. + # NOTE: the "not shipped today" argocd+inference row would push total to + # ~302, over the cap. If a future PR wires the argocd variant for the + # inference cell (or re-enables phase_serve alongside argocd), raise this + # to ~330 in the same PR. timeout-minutes: 300 permissions: contents: read @@ -139,10 +158,19 @@ jobs: && format('aicr-uat-day-{0}-{1}-{2}', inputs.slug, inputs.slot, github.run_id) || format('aicr-uat-{0}', github.run_id) }} CLUSTER_CONFIG: ${{ inputs.cluster_config_path }} - # The AICRConfig is selected by accelerator + intent - # (--config.yaml); both intents drive the same - # cluster-config (GPU pool from the reservation, system/CPU pools dynamic). - TEST_CONFIG: ${{ inputs.test_config_dir }}/${{ inputs.accelerator }}-${{ inputs.intent }}-config.yaml + # The AICRConfig is selected by accelerator + intent + deployer: + # deployer=helmfile → --config.yaml (the default; + # backwards-compatible with every existing cell) + # deployer=X (X!=helmfile) → --X-config.yaml + # (e.g. deployer=argocd → h100-training-argocd-config.yaml, issue #2194) + # Both variants drive the same cluster-config (GPU pool from the + # reservation, system/CPU pools dynamic). The step-level check below asserts + # the resolved file exists so an unwired deployer variant fails early + # (before any provisioning) rather than deep in phase_prep. + TEST_CONFIG: >- + ${{ inputs.test_config_dir }}/${{ inputs.accelerator }}-${{ inputs.intent }}${{ + inputs.deployer != 'helmfile' && format('-{0}', inputs.deployer) || '' + }}-config.yaml # v0.4.27 — required for node-group taints in the cluster-config schema # (GPU pool carries skyhook.nvidia.com=runtime-required:NoSchedule until # NodeWrite finishes tuning + reboot). Earlier v0.4.x pins predate taint @@ -190,6 +218,7 @@ jobs: LIFECYCLE: ${{ inputs.lifecycle }} SLUG: ${{ inputs.slug }} SLOT: ${{ inputs.slot }} + DEPLOYER: ${{ inputs.deployer }} run: | set -euo pipefail case "${INTENT}" in @@ -233,6 +262,15 @@ jobs: echo "::error::${CLUSTER_CONFIG} has no .deployment.tenancy"; exit 1 fi echo "AWS_ACCOUNT_ID=${TENANCY}" >> "${GITHUB_ENV}" + # Validate deployer after the account export (which teardown depends on). + # workflow_dispatch constrains this to a choice, but workflow_call passes + # free strings, so a typo from a caller must surface as a clear error here + # rather than as a missing test-config file below or a mysterious dispatch + # deep in phases.sh. Allowlist mirrors uat-run.yaml's choice input. + case "${DEPLOYER}" in + helmfile|argocd) ;; + *) echo "::error::unsupported deployer '${DEPLOYER}' (want helmfile|argocd)"; exit 1 ;; + esac # The test config only matters for the run phases (nightly / daytime-up # provision+deploy). daytime-down just tears the held cluster down and # never reads it, so skip the existence check there — a missing sibling @@ -685,21 +723,36 @@ jobs: aws-region: ${{ env.AWS_REGION }} role-session-name: GitHubActions-UAT-AICR-Install - - name: UAT - install (helmfile apply) + - name: UAT - install (helmfile apply or argocd sync) id: install if: steps.prep.outcome == 'success' - # helmfile apply (up to HELMFILE_TIMEOUT_SECONDS, 20m) + the post-install - # readiness gate, which runs `aicr validate --phase deployment` until it - # passes READINESS_CONSECUTIVE_PASSES times (up to READINESS_TIMEOUT_SECONDS, - # 60m, spanning nodewright tuning + reboots, which can run past 30m on a - # cold GPU node). Sized to exceed one helmfile attempt + the full gate - # window (20 + 60 = 80, + margin) so the gate's own fail-closed path runs - # before GitHub Actions kills the step. - timeout-minutes: 90 + # Step-level timeout must fit the LARGER of the two deployer branches, + # because both dispatch from `./run install` and share this cap: + # helmfile: HELMFILE_TIMEOUT_SECONDS 20m + gate 60m = 80m + # argocd: helm-install 5m + apply+sync 30m + root-grace 2m + gate 60m = 97m + # ARGOCD_HELM_TIMEOUT_SECONDS + ARGOCD_SYNC_TIMEOUT_SECONDS + + # ARGOCD_ROOT_APP_GRACE_SECONDS + READINESS_TIMEOUT_SECONDS in + # tests/uat/lib/phases.sh drive the argocd side. Sized at 110m so the + # argocd fail-closed paths (each shared-budget retry loop's `::error::`) + # run before GitHub Actions kills the step. The gate's own fail-closed + # path still fires within budget on both branches. + timeout-minutes: 110 shell: bash env: AICR_BIN: ${{ github.workspace }}/aicr RUN_ID: ${{ github.run_id }} + # Consumed only by the argocd deployer branch (install_argocd in + # tests/uat/lib/phases.sh), which provisions an in-cluster ghcr.io + # repo-creds Secret so Argo CD's repo-server can pull the bundle + # pushed by phase_prep. GITHUB_ACTOR is auto-injected by the + # Actions runtime; GITHUB_TOKEN is not, so pass it explicitly. + # The token is scoped to `packages: write` at the job level (see + # permissions block above), which is exactly what the Secret + # needs. Set the token ONLY for the argocd branch so helmfile's + # kubectl/aicr/chart-hook children don't inherit it in-env; + # install_argocd's `:?` guard fires on the empty default, which + # is what we want if a caller misconfigures the deployer input. + GITHUB_TOKEN: ${{ inputs.deployer == 'argocd' && github.token || '' }} run: ./tests/uat/aws/run install "${TEST_CONFIG}" # Runs ALL validate phases (deployment + conformance + performance), not @@ -883,12 +936,20 @@ jobs: SUMMARY_AICR_VERSION: ${{ inputs.aicr_version }} SUMMARY_INTENT: ${{ inputs.intent }} SUMMARY_LIFECYCLE: ${{ inputs.lifecycle }} + SUMMARY_DEPLOYER: ${{ inputs.deployer }} run: | { echo "## UAT Results (AWS)" echo "" printf '**Reservation:** `%s` · **Intent:** `%s` · **Lifecycle:** `%s`\n' \ "$SUMMARY_RESERVATION" "$SUMMARY_INTENT" "$SUMMARY_LIFECYCLE" + # Render the deployer only when it deviates from the default + # (helmfile), so existing cells' summaries are unchanged and + # only argocd cells surface the extra line — makes it easy to + # spot which lane a run was in from the Test Summary alone. + if [[ "$SUMMARY_DEPLOYER" != "" && "$SUMMARY_DEPLOYER" != "helmfile" ]]; then + printf '**Deployer:** `%s`\n' "$SUMMARY_DEPLOYER" + fi printf '**Cluster:** `%s`\n' "$DEPLOYMENT_ID" printf '**AICR version:** `%s`\n' "${SUMMARY_AICR_VERSION:-main (build from source)}" echo "**Build:** \`${{ github.sha }}\` (branch: \`${{ github.ref_name }}\`)" diff --git a/.github/workflows/uat-run.yaml b/.github/workflows/uat-run.yaml index 0d1d07806..a982f0de9 100644 --- a/.github/workflows/uat-run.yaml +++ b/.github/workflows/uat-run.yaml @@ -24,11 +24,26 @@ name: UAT Run # # The intent (training|inference) is rendered so nightly/daytime runs are # self-describing in the Actions list. It defaults to `training` on both the -# dispatch and call paths, so it is always present. IMPORTANT: the nightly -# (uat-nightly-batch.yaml) and daytime (uat-daytime.yaml) controllers rebuild -# this exact title to resolve the run they dispatched — keep their `title=` -# construction in lockstep with this format string. -run-name: "UAT ${{ inputs.reservation }} ${{ inputs.intent }} @ ${{ inputs.aicr_version || 'main' }}${{ inputs.dispatch_key != '' && format(' #{0}', inputs.dispatch_key) || '' }}" +# dispatch and call paths, so it is always present. The deployer is rendered +# as `[]` ONLY when it is non-empty and non-default (i.e. argocd +# today) so nightly/daytime titles, which never dispatch a non-default +# deployer, are unchanged. IMPORTANT: the nightly (uat-nightly-batch.yaml) +# and daytime (uat-daytime.yaml) controllers rebuild this exact title to +# resolve the run they dispatched — keep their `title=` construction in +# lockstep with this format string; if either controller ever starts passing +# `-f deployer=` (x != helmfile), also add the `[]` segment there. +# Folded scalar (>-): each newline below joins as a single space, which +# corresponds exactly to the literal single-space separators between the +# segments (`UAT · reservation · intent · @ · version`). Line-break points +# are chosen so no newline lands INSIDE an adjacent expression pair where +# the absence of a literal separator is load-bearing (e.g. the intent+ +# deployer pair — deployer renders ' []' or '', its leading space is the +# separator on its own — and version+dispatch_key, same shape). Keeps the +# rendered title identical to the pre-fold single-line form. +run-name: >- + UAT ${{ inputs.reservation }} + ${{ inputs.intent }}${{ inputs.deployer != 'helmfile' && format(' [{0}]', inputs.deployer) || '' }} + @ ${{ inputs.aicr_version || 'main' }}${{ inputs.dispatch_key != '' && format(' #{0}', inputs.dispatch_key) || '' }} # The shared UAT dispatch surface and reservation-lease owner (#1274, DC1). # A human (workflow_dispatch) or the nightly batch (workflow_call) requests a @@ -84,6 +99,20 @@ on: description: 'Skip UAT test execution (manual debugging only).' type: boolean default: false + deployer: + # helmfile = default via --config.yaml (the + # existing filename shape every cell has always used, kept for + # backwards-compat). argocd = --argocd-config.yaml + # (e.g. h100-training-argocd-config.yaml). Only aws-h100 training + # ships an argocd variant today; other cells fail with a missing + # test-config. See issue #2194. (actionlint rejects an empty + # string in a choice's options, so `helmfile` is the explicit + # default value here; the TEST_CONFIG resolver in uat-aws.yaml + # maps it to the unsuffixed filename.) + description: 'Deployer variant. See workflow file for the variant matrix.' + type: choice + options: [helmfile, argocd] + default: helmfile workflow_call: inputs: reservation: @@ -110,6 +139,9 @@ on: skip_tests: type: boolean default: false + deployer: + type: string + default: helmfile permissions: contents: read @@ -223,10 +255,16 @@ jobs: accelerator: ${{ needs.resolve.outputs.accelerator }} skip_delete: ${{ inputs.skip_delete }} skip_tests: ${{ inputs.skip_tests }} + deployer: ${{ inputs.deployer }} run-gcp: needs: resolve - if: always() && needs.resolve.outputs.cloud == 'gcp' + # deployer=argocd is aws-h100-scoped today (issue #2194): uat-gcp.yaml declares + # no deployer input, so passing one here would be silently ignored (green + # helmfile run against a request the user did not make). Skip this job when + # a non-helmfile deployer is dispatched; the unsupported-deployer-for-cloud + # guard job below emits the red-workflow signal for that combination. + if: always() && needs.resolve.outputs.cloud == 'gcp' && inputs.deployer == 'helmfile' # Superset ceiling for the nested pipeline (a reusable workflow's jobs # cannot exceed the caller's permissions). permissions: @@ -252,7 +290,8 @@ jobs: run-azure: needs: resolve - if: always() && needs.resolve.outputs.cloud == 'azure' + # See run-gcp for the deployer-scope rationale. + if: always() && needs.resolve.outputs.cloud == 'azure' && inputs.deployer == 'helmfile' # Superset ceiling for the nested pipeline (a reusable workflow's jobs # cannot exceed the caller's permissions). permissions: @@ -285,7 +324,8 @@ jobs: # cloud lanes. run-kind: needs: resolve - if: needs.resolve.outputs.cloud == 'kind' + # See run-gcp for the deployer-scope rationale. + if: needs.resolve.outputs.cloud == 'kind' && inputs.deployer == 'helmfile' permissions: contents: read # checkout inside the called pipeline actions: write # nested evidence-ingest dashboard-publish dispatch (a reusable workflow cannot elevate the caller token, so grant the ceiling here) @@ -302,6 +342,36 @@ jobs: skip_delete: ${{ inputs.skip_delete }} skip_tests: ${{ inputs.skip_tests }} + # Fail closed on a dispatched deployer variant this cell doesn't yet wire. + # Today only run-aws forwards the `deployer` input to its reusable pipeline; + # gcp/azure/kind reusable workflows declare no such input and would run the + # helmfile lane if we let them fire. The run-*.if:s above skip those jobs + # when a non-helmfile deployer is dispatched to a non-aws cell; this job + # provides the paired red-workflow signal so the requester never sees a + # green run for a request that was silently downgraded. + # Mirrors the unmapped-cloud pattern for the same "no silent no-op" reason. + unsupported-deployer-for-cloud: + needs: resolve + if: >- + inputs.deployer != 'helmfile' && + needs.resolve.outputs.cloud != 'aws' + runs-on: ubuntu-latest + steps: + - name: Fail on unsupported (deployer, cloud) combination + env: + CLOUD: ${{ needs.resolve.outputs.cloud }} + DEPLOYER: ${{ inputs.deployer }} + run: | + set -euo pipefail + echo "::error::deployer='${DEPLOYER}' is only wired for cloud='aws' today" + echo "::error::(this reservation resolved to cloud='${CLOUD}'). Only run-aws" + echo "::error::forwards the deployer input to its reusable pipeline; other" + echo "::error::run- jobs would run the helmfile lane silently. See" + echo "::error::issue #2194 for the coverage roadmap. The run-.if:" + echo "::error::guards above skipped the dispatched cell to prevent that" + echo "::error::silent downgrade." + exit 1 + # Fail closed on an unmapped cloud. The broker validates cloud membership # against its own set, but a cloud can be added there (and to the registry) # before its run- job exists here — without this job the dispatch diff --git a/docs/contributor/uat.md b/docs/contributor/uat.md index 3ef6da18b..b73c6ad64 100644 --- a/docs/contributor/uat.md +++ b/docs/contributor/uat.md @@ -84,6 +84,22 @@ The single nightly cron (`uat-nightly-batch.yaml`, `0 4 * * *`) runs **both inte Semantics: **`main` is never gated** (it is built from source and carries the newest fixes, so it always runs every listed intent); a **release** cell drops any intent whose minimum version is newer than the tag (semver; a tag `>=` the minimum runs). The gate lives in the schedule (`uat-broker schedule` attaches each cell's eligible `intents`), so the controller simply never dispatches a gated `(version × intent)` — no per-version workflow logic. Pointing the floor at a **not-yet-tagged** release is intentional and self-resolving: until that release ships, the intent runs on **`main` only** (green, continuous coverage of the fix), and the release enrolls automatically once it exists. `Validate` rejects a floor for an intent the row does not run, or a non-semver value. Bump the floor if the real first-fixed tag differs — an over-low floor surfaces as a visible red (safe), an over-high floor silently skips a good release (bump down). +## Selecting the deployer + +The `deployer` input picks which deployer variant of the intent's test config the pipeline consumes. Set to `helmfile` (the default), the pipeline resolves `tests/uat//tests/--config.yaml` — the config every existing cell has always run against. Any other value (currently only `argocd`) resolves `---config.yaml` — for example `deployer=argocd` on `aws-h100` training loads `tests/uat/aws/tests/h100-training-argocd-config.yaml`. + +```bash +# ArgoCD variant of the aws-h100 training cell (issue #2194) +gh workflow run uat-run.yaml --repo NVIDIA/aicr --ref main \ + -f reservation=aws-h100 -f intent=training -f deployer=argocd +``` + +The AICRConfig field `spec.bundle.deployment.deployer` is the source of truth `phase_prep`/`phase_install` read; the workflow input is only how the correct config file is *selected*. `phases.sh:phase_install` dispatches to `install_helmfile` (helmfile lane, unchanged) or `install_argocd` (Argo CD install + repo-creds Secret from `GITHUB_TOKEN` + `kubectl apply` of the `nvidia-stack` app-of-apps + terminal-pass wait on every `Application`). The post-install readiness gate is deployer-agnostic — it validates deployed cluster state (`aicr validate --phase deployment`), not the deployment mechanism — so a green ArgoCD cell means the GitOps deploy path converges on the same operator-managed stack the helmfile lane validates. + +**How the bundle reaches Argo CD.** `phase_prep` calls `aicr bundle --output oci://ghcr.io/nvidia/aicr-bundle-scratch/:run- --repo oci://ghcr.io/nvidia/aicr-bundle-scratch/` — the path segment is the AICRConfig's `metadata.name` (yq-read from the test-config in `phase_prep`), not the recipe coordinate. The `--output` flag pushes the rendered bundle to GHCR (the job already has `packages: write`), and `--repo` sets the `source.repoURL` baked into every generated `Application`. `install_argocd` provisions a prefix-matched `argocd.argoproj.io/secret-type: repo-creds` Secret from `GITHUB_TOKEN` so Argo CD's repo-server can pull the pushed artifact. Concurrent runs on the same recipe are isolated by the `:run-` tag. + +**Coverage today (issue #2194).** Only `aws-h100` training carries a `-argocd` config file. Dispatching `deployer=argocd` against a non-AWS reservation (`gcp-h100`, `azure-h100`, `kind-h100`) fails closed at the top level: `uat-run.yaml`'s `unsupported-deployer-for-cloud` guard job emits a red workflow, and the per-cloud `run-.if:` skips the reusable pipeline so no cluster is provisioned for a request that couldn't have been served. Only `run-aws` forwards the `deployer` input to its reusable pipeline; the other reusable workflows declare no such input. Nightly enrollment is deliberately deferred until a manual dispatch is green on hardware — mirroring the `azure-h100` (#1722) and `kind-h100` (#1843) onboarding pattern. The `argocd-helm` variant, and extension to gcp/azure/kind, are separate follow-ups. + ## Cluster lifecycles The `lifecycle` input selects one of three cluster lifecycles, all sharing the reservation lease: diff --git a/tests/uat/aws/tests/h100-training-argocd-config.yaml b/tests/uat/aws/tests/h100-training-argocd-config.yaml new file mode 100644 index 000000000..a1b88b769 --- /dev/null +++ b/tests/uat/aws/tests/h100-training-argocd-config.yaml @@ -0,0 +1,128 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# AICRConfig consumed by `aicr {snapshot,recipe,bundle,validate} --config`. +# +# ArgoCD deployer variant of h100-training-config.yaml. The only functional +# difference is `spec.bundle.deployment.deployer: argocd` — every other field +# (recipe criteria, scheduling, snapshot/validate agent) is identical to the +# helmfile sibling so a green run here is direct evidence the `--deployer +# argocd` GitOps path works on the same recipe the helmfile lane validates. +# +# See ../../lib/phases.sh (install_argocd) for the install-phase flow: +# `aicr bundle` pushes the same content to ghcr.io/nvidia/aicr-bundle-scratch +# and Argo CD's repo-server pulls from that OCI ref via a prefix-matched +# repo-creds Secret provisioned in-cluster from the workflow's GITHUB_TOKEN. +# +# Manual dispatch only for now (issue #2194): enroll in the nightly batch as +# a separate follow-up after a green manual run mirrors the azure-h100 / +# kind onboarding pattern (PRs #1722, #1843). +# +# Evidence push target is deliberately shared with the helmfile config: the +# recipe (and therefore the evidence coordinate) is identical, and RUN_ID +# tag isolation prevents same-recipe cross-run collisions. If the dashboard +# grows a deployer facet later, tag the run there — not by forking the OCI +# repo per deployer. +kind: AICRConfig +apiVersion: aicr.run/v1alpha2 +metadata: + name: eks-h100-training-uat-argocd + +spec: + snapshot: + output: + path: snapshot.yaml + agent: + namespace: aicr-validation + nodeSelector: + nodeGroup: gpu-worker + tolerations: + - dedicated=worker-workload:NoSchedule + - dedicated=worker-workload:NoExecute + - nvidia.com/gpu=present:NoSchedule + # The GPU pool carries skyhook.nvidia.com=runtime-required:NoSchedule + # (cluster-config.yaml) until NodeWright finishes tuning + reboot. The + # snapshot runs pre-deployment, so this taint is still present and the + # agent — pinned to the GPU pool above — must tolerate it to schedule. + - skyhook.nvidia.com=runtime-required:NoSchedule + + recipe: + criteria: + service: eks + accelerator: h100 + os: ubuntu + intent: training + platform: kubeflow + output: + path: recipe.yaml + + bundle: + input: + recipe: recipe.yaml + output: + # phase_prep overrides --output at CLI time with an oci:// target + # (ARGOCD_OCI_PREFIX in lib/phases.sh + a run-scoped tag). The local + # ./bundle directory still receives app-of-apps.yaml so `kubectl apply` + # has a manifest to consume. + target: ./bundle + deployment: + deployer: argocd + scheduling: + acceleratedNodeSelector: + nodeGroup: gpu-worker + acceleratedNodeTolerations: + - dedicated=worker-workload:NoSchedule + - dedicated=worker-workload:NoExecute + - nvidia.com/gpu=present:NoSchedule + systemNodeSelector: + nodeGroup: system-worker + systemNodeTolerations: + - dedicated=system-workload:NoSchedule + - dedicated=system-workload:NoExecute + # Consume the cluster's default StorageClass rather than hardcoding a + # name. The aws-ebs-csi-driver component provisions a gp3-backed default + # named "ebs-csi-default-sc" (annotated is-default-class=true), so an + # empty value binds PVCs to it automatically. Pinning a literal "gp3" + # only works on clusters using the AWS managed EBS CSI addon (which + # auto-creates a class named "gp3") — not on an aicr-bundle-deployed + # cluster, where the StatefulSet would otherwise never be created (#1455). + storageClass: "" + + validate: + execution: + # The deployment phase is the readiness barrier (gpu-operator ClusterPolicy + # reaches state=ready, the DRA kubelet-plugin DaemonSet rolls out, nodewright + # node tuning completes). failFast stops the run if that phase fails, so the + # conformance/performance phases are not evaluated against a not-yet-converged + # cluster — which previously surfaced as misleading accelerator-metrics / + # ai-service-metrics / pod-autoscaling failures. The `../run` readiness gate + # gives the operator-driven stack time to converge before validate, so on a + # healthy cluster the deployment phase passes and all phases still run. + failFast: true + input: + recipe: recipe.yaml + snapshot: snapshot.yaml + agent: + namespace: aicr-validation + tolerations: + - dedicated=worker-workload:NoSchedule + - dedicated=worker-workload:NoExecute + - nvidia.com/gpu=present:NoSchedule + evidence: + attestation: + # Setting `out` enables emit. The push target below is the stable + # per-recipe repo; ../run appends a `:run-${RUN_ID}` tag for per-run + # isolation. Consumers pin immutably via the artifact digest. + out: ./evidence + push: ghcr.io/nvidia/aicr-evidence/h100-eks-ubuntu-training-kubeflow diff --git a/tests/uat/lib/phases.sh b/tests/uat/lib/phases.sh index 677a77dc8..319f922ad 100644 --- a/tests/uat/lib/phases.sh +++ b/tests/uat/lib/phases.sh @@ -70,6 +70,24 @@ TRAINJOB_TIMEOUT_SECONDS="${TRAINJOB_TIMEOUT_SECONDS:-1200}" # 20 min # (tests/uat/kind/run) overrides to 1. TRAINJOB_NUM_NODES="${TRAINJOB_NUM_NODES:-2}" HELMFILE_TIMEOUT_SECONDS="${HELMFILE_TIMEOUT_SECONDS:-1200}" # 20 min +# ArgoCD deployer knobs (see install_argocd). ARGOCD_HELM_TIMEOUT_SECONDS +# bounds the `helm upgrade --install` of the argo-cd chart itself; +# ARGOCD_SYNC_TIMEOUT_SECONDS bounds the wait for every Application to +# reach a terminal-pass state. Both are shared wall-clock budgets, matching +# the discipline HELMFILE_TIMEOUT_SECONDS uses for the helmfile lane. +# ARGOCD_ROOT_APP_GRACE_SECONDS gives Argo CD a short window to reify the +# root `nvidia-stack` Application after `kubectl apply` -- a missing root +# after the grace means the apply silently produced no Application (RBAC +# collision, CRD not yet Established, etc.) and we should fail closed. +ARGOCD_HELM_TIMEOUT_SECONDS="${ARGOCD_HELM_TIMEOUT_SECONDS:-300}" # 5 min +ARGOCD_SYNC_TIMEOUT_SECONDS="${ARGOCD_SYNC_TIMEOUT_SECONDS:-1800}" # 30 min +ARGOCD_ROOT_APP_GRACE_SECONDS="${ARGOCD_ROOT_APP_GRACE_SECONDS:-120}" +# Prefix for the OCI target `aicr bundle --output` pushes to and the +# `--repo` baseURL baked into every Application. The path is under +# ghcr.io/nvidia (workflow already grants `packages: write` for evidence +# push) in a distinct namespace so bundle artifacts don't collide with +# signed evidence. See phase_prep's argocd branch. +ARGOCD_OCI_PREFIX="${ARGOCD_OCI_PREFIX:-oci://ghcr.io/nvidia/aicr-bundle-scratch}" # Budget for the post-install readiness gate (see phase_install), which runs # `aicr validate --phase deployment` until it passes READINESS_CONSECUTIVE_PASSES # times in a row. This is the gate window ONLY -- it is entered AFTER helmfile @@ -390,12 +408,75 @@ phase_prep() { echo "::endgroup::" echo "::group::Generate bundle" - "${AICR_BIN}" bundle --config "${config}" - test -f bundle/helmfile.yaml || { - echo "expected bundle/helmfile.yaml (deployer: helmfile) — got:" >&2 - ls -la bundle >&2 || true - exit 1 - } + # The bundle shape (and how it is delivered to the cluster) is deployer- + # specific: helmfile emits bundle/helmfile.yaml consumed off the local + # filesystem; argocd emits bundle/app-of-apps.yaml AND pushes the same + # content to an OCI registry so Argo CD's repo-server can pull it. Read + # the deployer straight from the AICRConfig -- keeping selection in the + # test-config (not a workflow env var) means the artifact under review + # is self-describing. + local deployer + deployer="$(yq -r '.spec.bundle.deployment.deployer // "helmfile"' "${config}")" + case "${deployer}" in + helmfile) + "${AICR_BIN}" bundle --config "${config}" + test -f bundle/helmfile.yaml || { + echo "expected bundle/helmfile.yaml (deployer: helmfile) — got:" >&2 + ls -la bundle >&2 || true + exit 1 + } + ;; + argocd) + # --output pushes bundle content to OCI; --repo sets the source.repoURL + # baked into the rendered nvidia-stack Application. Both flags override + # the config's spec.bundle.output.target (pkg/cli/bundle.go:583-604) + # so the AICRConfig stays deployer-neutral. + # + # URL shape (matches KWOK argocd-oci precedent at + # kwok/scripts/validate-scheduling.sh:951+): + # oci_repo = / (per-recipe, no tag) + # oci_target = /:run- (same repo, with tag) + # `aicr bundle` renders root Application spec.source.repoURL= + # and targetRevision=, so Argo CD pulls from oci_target — the same + # URL we pushed to. Passing a bare-prefix --repo (without the slug) + # would leave Argo CD chasing a non-existent artifact at + # oci://:main — verified locally with `aicr bundle --deployer + # argocd` inspection during Tier 1 validation of issue #2194. + # + # RUN_ID isolates concurrent runs on the same recipe. Argo CD prefix- + # matches repo-creds (util/db/repository_secrets.go: + # getRepositoryCredentialIndex → HasPrefix), so a single Secret at + # ARGOCD_OCI_PREFIX covers every pushed under that + # prefix — provisioned once by install_argocd. + # + # Retention follow-up (#2194): this pushes a new run-tagged artifact + # every dispatch, and there is no cleanup on the successful path. Left + # deferred while this cell is manual-dispatch-only (accumulation rate + # low, cost bounded); to be addressed at nightly enrollment by EITHER + # a workflow teardown step that `gh api DELETE`s the tag it stashed as + # a job output, OR an org-level retention policy on + # ghcr.io/nvidia/aicr-bundle-scratch. The `-scratch` namespace name + # already signals ephemeral intent to any operator inspecting GHCR. + local bundle_slug oci_repo oci_target + bundle_slug="$(yq -r '.metadata.name' "${config}")" + oci_repo="${ARGOCD_OCI_PREFIX}/${bundle_slug}" + oci_target="${oci_repo}:run-${RUN_ID}" + echo "argocd bundle push target: ${oci_target}" + echo "argocd Application source.repoURL: ${oci_repo}" + "${AICR_BIN}" bundle --config "${config}" \ + --output "${oci_target}" \ + --repo "${oci_repo}" + test -f bundle/app-of-apps.yaml || { + echo "expected bundle/app-of-apps.yaml (deployer: argocd) — got:" >&2 + ls -la bundle >&2 || true + exit 1 + } + ;; + *) + echo "::error::unsupported deployer: ${deployer} (supported: helmfile, argocd)" >&2 + exit 1 + ;; + esac echo "::endgroup::" } @@ -434,6 +515,30 @@ uat_helm_diff_platform() { } phase_install() { + # Dispatch to the deployer-specific install body. The readiness gate below + # is deployer-agnostic (it validates deployed cluster state, not the + # deployment mechanism) so it stays in phase_install; only the "get the + # stack onto the cluster" step differs. + local deployer + deployer="$(yq -r '.spec.bundle.deployment.deployer // "helmfile"' "${config}")" + case "${deployer}" in + helmfile) install_helmfile ;; + argocd) install_argocd ;; + *) + echo "::error::unsupported deployer: ${deployer} (supported: helmfile, argocd)" >&2 + exit 1 + ;; + esac + + echo "::group::Cluster state post-install" + kubectl get nodes -o wide + kubectl get pods -A | grep -Ev '\s+Running\s+|\s+Completed\s+' || true + echo "::endgroup::" + + install_readiness_gate +} + +install_helmfile() { command -v helmfile >/dev/null || { echo "helmfile not on PATH" >&2; exit 1; } command -v helm >/dev/null || { echo "helm not on PATH" >&2; exit 1; } command -v curl >/dev/null || { echo "curl not on PATH" >&2; exit 1; } @@ -631,12 +736,269 @@ phase_install() { echo "::error::helmfile apply failed after 3 attempts" >&2 exit 1 fi +} - echo "::group::Cluster state post-install" - kubectl get nodes -o wide - kubectl get pods -A | grep -Ev '\s+Running\s+|\s+Completed\s+' || true +install_argocd() { + command -v helm >/dev/null || { echo "helm not on PATH" >&2; exit 1; } + command -v kubectl >/dev/null || { echo "kubectl not on PATH" >&2; exit 1; } + + local SCRIPT_DIR + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + local REPO_ROOT="${SCRIPT_DIR}/../../.." + local ARGOCD_CHART_VERSION + ARGOCD_CHART_VERSION="$(yq -r '.testing_tools.argocd_chart' "${REPO_ROOT}/.settings.yaml")" + + echo "::group::Install Argo CD (chart ${ARGOCD_CHART_VERSION})" + # KWOK precedent (kwok/scripts/install-infra.sh:install_argocd) — same + # chart, same namespace/release. `helm upgrade --install` is idempotent + # so a re-run (or a cluster that already has Argo CD baked in) converges + # cleanly. `--wait --timeout` blocks until argocd-server + application- + # controller are Ready so downstream steps (repo-creds, apply) don't + # race a partially-rolled-out install. + helm repo add argo https://argoproj.github.io/argo-helm --force-update >/dev/null 2>&1 || true + # Every fail-closed path in this function emits `::error::` and closes the + # ::group:: before exiting so a triager gets a breadcrumb + no dangling + # log group. `set -euo pipefail` would otherwise abort here silently. + if ! helm repo update argo >/dev/null; then + echo "::error::helm repo update argo failed" >&2 + echo "::endgroup::" + exit 1 + fi + if ! helm upgrade --install argocd argo/argo-cd \ + --namespace argocd --create-namespace \ + --version "${ARGOCD_CHART_VERSION}" \ + --wait --timeout "${ARGOCD_HELM_TIMEOUT_SECONDS}s"; then + echo "::error::Argo CD helm install failed" >&2 + kubectl -n argocd get pods || true + echo "::endgroup::" + exit 1 + fi + # CRD race guard — applications.argoproj.io must be Established before we + # apply the app-of-apps, else kubectl apply races the CRD controller. + if ! kubectl wait --for=condition=Established \ + crd/applications.argoproj.io --timeout=120s; then + echo "::error::applications.argoproj.io CRD did not reach Established within 120s" >&2 + kubectl describe crd applications.argoproj.io 2>&1 | head -60 >&2 || true + echo "::endgroup::" + exit 1 + fi echo "::endgroup::" + # Start the shared ARGOCD_SYNC_TIMEOUT_SECONDS wall clock HERE, before the + # repo-creds Secret apply, so a hung apiserver on any single step (Secret + # apply, app-of-apps apply, root-app grace, terminal-pass poll) draws from + # the same 30m budget install_helmfile enforces with its own shared clock. + # Without this, a stalled Secret apply would burn indefinite time before + # the downstream retry loop even started measuring. + local argocd_deadline=$(( SECONDS + ARGOCD_SYNC_TIMEOUT_SECONDS )) + local secret_remaining + + echo "::group::Provision ghcr.io repo-creds Secret (prefix match)" + # Argo CD prefix-matches repo-creds against Application source URLs + # (util/db/repository_secrets.go::getRepositoryCredentialIndex → HasPrefix), + # so ONE Secret annotated argocd.argoproj.io/secret-type=repo-creds at + # url=ARGOCD_OCI_PREFIX covers every recipe pushed under that prefix. + # GITHUB_TOKEN + GITHUB_ACTOR are provided by the UAT workflow env + # (packages: write is already declared on every uat-*.yaml job). + : "${GITHUB_TOKEN:?GITHUB_TOKEN required for ghcr.io pull creds}" + : "${GITHUB_ACTOR:?GITHUB_ACTOR required for ghcr.io pull creds}" + # Field shape mirrors kwok/scripts/install-infra.sh:apply_repo_secret + # for chart 9.5.x / Argo CD v3.x — type: oci is the direct OCI credential + # kind (chart 7.x needed type: helm + enableOCI=true; superfluous here + # and dropped to avoid confusion). username/password are the only fields + # KWOK omits, because its in-cluster registry is unauthenticated HTTP; + # ghcr.io is HTTPS + auth, so we supply the GITHUB_ACTOR/GITHUB_TOKEN + # already scoped to `packages: write` on every UAT job. The whole apply is + # bounded by the remaining shared budget so a hung apiserver cannot outlast + # the sync-wait window; clamp >= 1 for the `timeout 0 == no timeout` + # boundary condition install_helmfile guards the same way. + # + # `kubectl create ... --from-literal | kubectl label --local | kubectl apply` + # carries the token/actor as argv, never through a YAML parser -- defense-in- + # depth against any future GH-token/actor value with YAML-breaking bytes + # (safe by construction today, but the belt-and-suspenders is cheap). + secret_remaining=$(( argocd_deadline - SECONDS )) + (( secret_remaining < 1 )) && secret_remaining=1 + if ! kubectl create secret generic aicr-oci-repo-creds \ + --namespace argocd \ + --from-literal=type=oci \ + --from-literal="url=${ARGOCD_OCI_PREFIX}" \ + --from-literal="username=${GITHUB_ACTOR}" \ + --from-literal="password=${GITHUB_TOKEN}" \ + --dry-run=client -o yaml \ + | kubectl label --local -f - argocd.argoproj.io/secret-type=repo-creds -o yaml \ + | timeout "${secret_remaining}" kubectl apply -f - + then + echo "::error::repo-creds Secret apply failed (timeout ${secret_remaining}s of ${ARGOCD_SYNC_TIMEOUT_SECONDS}s shared budget)" >&2 + echo "::endgroup::" + exit 1 + fi + echo "::endgroup::" + + # Retry `kubectl apply -f bundle/app-of-apps.yaml` on transient apiserver + # errors, mirroring install_helmfile's shared-budget discipline. In practice + # `kubectl apply` on a single manifest almost never needs the retry, but the + # shared-budget shape keeps the two branches behaviorally symmetric and + # bounds the install step's worst-case wall clock the same way. + # + # Each attempt is capped at the remaining shared ARGOCD_SYNC_TIMEOUT_SECONDS + # budget with `timeout`, and the deadline is re-checked BEFORE each attempt + # so a stalled apiserver cannot spend the whole 30m budget on kubectl apply + # and starve the downstream sync-wait. Clamp `remaining` to >= 1 for the + # same reason install_helmfile does: `timeout 0` means "no timeout". + local applied=false apply_remaining apply_nap + for attempt in 1 2 3; do + if (( SECONDS >= argocd_deadline )); then + echo "argocd shared ${ARGOCD_SYNC_TIMEOUT_SECONDS}s budget exhausted after attempt $(( attempt - 1 )); not starting attempt ${attempt}" + break + fi + apply_remaining=$(( argocd_deadline - SECONDS )) + (( apply_remaining < 1 )) && apply_remaining=1 + echo "::group::kubectl apply app-of-apps (attempt ${attempt}/3, timeout ${apply_remaining}s of ${ARGOCD_SYNC_TIMEOUT_SECONDS}s shared budget)" + if timeout "${apply_remaining}" kubectl apply -f bundle/app-of-apps.yaml; then + applied=true + echo "::endgroup::" + break + fi + echo "::endgroup::" + # Cap the retry delay to remaining shared budget so `sleep 15` near the + # deadline can't overrun it — same discipline as the root-app and sync- + # wait loops. Positive-only guard so the loop exits cleanly at budget=0. + if (( attempt < 3 )); then + apply_nap=15 + apply_remaining=$(( argocd_deadline - SECONDS )) + (( apply_nap > apply_remaining )) && apply_nap=${apply_remaining} + if (( apply_nap > 0 )); then + echo "waiting ${apply_nap}s before retry" + sleep "${apply_nap}" + fi + fi + done + if [[ "${applied}" != "true" ]]; then + echo "::error::kubectl apply app-of-apps.yaml failed after 3 attempts (or budget exhausted)" >&2 + exit 1 + fi + + echo "::group::Wait for root Application '${ARGOCD_ROOT_APP:-nvidia-stack}' to be reified" + # A missing root after the grace window means the apply silently produced + # no Application (RBAC race, CRD version skew, malformed manifest). Fail + # closed rather than time out downstream on an empty controller queue. + # Cap the root-app grace at the shared argocd_deadline so this loop can + # never run past the ARGOCD_SYNC_TIMEOUT_SECONDS budget that spans the + # whole install path (Secret apply + apply retries + this grace + sync + # poll). Without the cap, an upstream step that spent most of the shared + # budget could still let the root-app grace add its full 2m on top. + # `root_grace_effective` is the actual window this loop ran under (nominal + # OR shorter if the shared cap fired) — used by the failure diagnostic + # below so a budget-starved run doesn't look like a 120s hang. + local root_deadline=$(( SECONDS + ARGOCD_ROOT_APP_GRACE_SECONDS )) + (( root_deadline > argocd_deadline )) && root_deadline=${argocd_deadline} + local root_grace_effective=$(( root_deadline - SECONDS )) + local root_app="${ARGOCD_ROOT_APP:-nvidia-stack}" + local root_ready=false root_remaining root_nap + while (( SECONDS < root_deadline )); do + # Bound each kubectl by remaining budget so a hung apiserver cannot burn + # the whole grace window on one call. Clamp to >= 1 (same rationale as + # install_helmfile: `timeout 0` means no timeout). + root_remaining=$(( root_deadline - SECONDS )) + (( root_remaining < 1 )) && root_remaining=1 + if timeout "${root_remaining}" kubectl -n argocd get application "${root_app}" >/dev/null 2>&1; then + root_ready=true + break + fi + # Cap the sleep to remaining budget so we do not overrun the deadline + # waiting between polls. + root_nap=5 + root_remaining=$(( root_deadline - SECONDS )) + (( root_nap > root_remaining )) && root_nap=${root_remaining} + (( root_nap > 0 )) && sleep "${root_nap}" + done + if [[ "${root_ready}" != "true" ]]; then + if (( root_grace_effective < ARGOCD_ROOT_APP_GRACE_SECONDS )); then + echo "::error::root Application '${root_app}' not reified within effective ${root_grace_effective}s (nominal ${ARGOCD_ROOT_APP_GRACE_SECONDS}s; shared sync budget capped this loop early)" >&2 + else + echo "::error::root Application '${root_app}' not reified within ${ARGOCD_ROOT_APP_GRACE_SECONDS}s" >&2 + fi + kubectl -n argocd get applications || true + echo "::endgroup::" + exit 1 + fi + echo "::endgroup::" + + echo "::group::Wait for all Argo CD Applications to reach terminal-pass (budget $(( argocd_deadline - SECONDS ))s)" + # Two premature-convergence guards run before the 4-arm terminal-pass + # predicate (which mirrors tests/chainsaw/kwok/argocd-sync/chainsaw-test.yaml, + # source-agnostic per that test's SYNC NOTE header): + # 1. items == [] -> "no Applications yet" (CRD present but + # neither root nor children reified yet). + # 2. items == [root] only, or root not Synced -> in the race window where + # the app-of-apps root is OutOfSync+Healthy while its child Applications + # have not yet been generated, arm 2 (OutOfSync+Healthy) would satisfy + # `bad=""` and return 0 before gpu-operator/DRA even exist. Require the + # root Application to be Synced (not merely present) as the crossover. + # Otherwise apply the 4-arm predicate: Synced+Healthy (canonical); OutOfSync+ + # Healthy (operator mutation — gpu-operator ClusterPolicy, ResourceSlice + # injection); Synced+Progressing / Synced+Degraded (Argo health-controller + # divergence post-op — tolerated because the ultimate verdict lives in the + # deployment readiness gate that follows). + local root_app_name="${ARGOCD_ROOT_APP:-nvidia-stack}" + local jq_bad + jq_bad=' + if (.items | length) == 0 then + "no Applications yet" + elif ([.items[] | select(.metadata.name == "'"${root_app_name}"'" and .status.sync.status == "Synced")] | length) == 0 then + "root '"${root_app_name}"' not Synced yet (children may not be reified)" + else + ([ .items[] | + select( + ((.status.sync.status == "Synced") and (.status.health.status | IN("Healthy","Progressing","Degraded"))) or + ((.status.sync.status == "OutOfSync") and (.status.health.status == "Healthy")) + | not) + | (.metadata.name + "[" + (.status.sync.status // "?") + "/" + (.status.health.status // "?") + "]") + ] | join(", ")) + end' + # Initialize `bad` with a sentinel so the failure diagnostic below reads + # sensibly even in the pathological case where the while guard is false on + # the first check (SECONDS already >= argocd_deadline because kubectl-apply + # + root-grace consumed the whole shared budget). Loop iterations overwrite + # this with real values. + local bad="not sampled (sync-wait loop never ran — budget spent by upstream steps)" + local sync_remaining sync_nap + while (( SECONDS < argocd_deadline )); do + # Bound each poll by remaining shared budget so a hung apiserver on `get` + # cannot outlast the sync window (same discipline as the apply-retry loop + # above). Clamp to >= 1 for the `timeout 0 == no timeout` boundary. + sync_remaining=$(( argocd_deadline - SECONDS )) + (( sync_remaining < 1 )) && sync_remaining=1 + bad="$(timeout "${sync_remaining}" kubectl -n argocd get applications -o json 2>/dev/null | jq -r "${jq_bad}" || echo "ERR")" + if [[ -z "${bad}" ]]; then + echo "all Applications in terminal-pass state" + echo "::endgroup::" + return 0 + fi + echo "waiting for: ${bad}" + # Cap the poll interval to remaining budget so `sleep 15` near the + # deadline can't overrun it. Positive-only guard so we exit the loop + # cleanly when budget hits 0. + sync_nap=15 + sync_remaining=$(( argocd_deadline - SECONDS )) + (( sync_nap > sync_remaining )) && sync_nap=${sync_remaining} + (( sync_nap > 0 )) && sleep "${sync_nap}" + done + echo "::error::Argo CD sync did not converge within ${ARGOCD_SYNC_TIMEOUT_SECONDS}s; last bad: ${bad}" >&2 + # Best-effort failure diagnostic — every call `|| true` so a transient + # apiserver error on `get` doesn't skip the `describe` and repo-server + # logs that follow (which are what a reviewer actually needs to diagnose + # an OCI-pull auth error or a sync-wave block). Matches the KWOK failure + # dump semantics. + kubectl -n argocd get applications || true + kubectl -n argocd describe applications 2>&1 | head -200 >&2 || true + kubectl -n argocd logs -l app.kubernetes.io/name=argocd-repo-server --tail=100 2>&1 >&2 || true + echo "::endgroup::" + exit 1 +} + +install_readiness_gate() { # Readiness gate: run the deployment validation phase -- the authoritative # expected-resources / ClusterPolicy / DRA / nodewright checks the later # `--phase all` run gates on -- in a retry loop until it passes diff --git a/tests/uat/workflow_security_test.go b/tests/uat/workflow_security_test.go index 849603789..d3fd96105 100644 --- a/tests/uat/workflow_security_test.go +++ b/tests/uat/workflow_security_test.go @@ -71,6 +71,18 @@ var credentialApplyExpectations = []credentialApplyExpectation{ var actuatorStepNames = []string{"Bringup Infra", "Destroy Cluster"} +// awsTokenBearingStepNames enumerates AWS-lane steps that carry credentials +// via env (not credentials_file_path). Adding the argocd deployer variant +// wired GITHUB_TOKEN into the install step so install_argocd can provision +// the ghcr.io repo-creds Secret (see .github/workflows/uat-aws.yaml + issue +// #2194). The install path never enables `set -x` today and the token is +// scoped to `packages: write`, so this is a low-value invariant pin rather +// than a gap in protection — the pin exists so a future set -x addition in +// the install step (an addition invisible in an env-only diff) is caught by +// TestCredentialBearingUATStepsDisableXtrace instead of leaking the token +// into log lines. +var awsTokenBearingStepNames = []string{"UAT - install (helmfile apply or argocd sync)"} + type workflowDocument struct { Env map[string]string `yaml:"env"` Defaults workflowDefaults `yaml:"defaults"` @@ -391,10 +403,16 @@ func TestCredentialBearingUATStepsDisableXtrace(t *testing.T) { { "AWS", "uat-aws.yaml", "uat-aws", func(t *testing.T, steps []workflowStep) []workflowStep { - selected := make([]workflowStep, 0, len(actuatorStepNames)) + selected := make([]workflowStep, 0, len(actuatorStepNames)+len(awsTokenBearingStepNames)) for _, stepName := range actuatorStepNames { selected = append(selected, uniqueStepNamed(t, steps, stepName)) } + // Token-bearing steps (env-only credentials) — see the + // awsTokenBearingStepNames comment for the argocd deployer + // rationale. + for _, stepName := range awsTokenBearingStepNames { + selected = append(selected, uniqueStepNamed(t, steps, stepName)) + } return selected }, },