From e46cc5871e7383da660ea594a8bc59f78c10d033 Mon Sep 17 00:00:00 2001 From: framsouza Date: Thu, 20 Aug 2026 17:01:01 +0200 Subject: [PATCH] feat(uat): add argocd deployer variant on aws-h100 training Adds end-to-end UAT coverage for the `--deployer argocd` GitOps path, which today has unit + KWOK-sync coverage but has never been exercised on real GPU hardware. Refactors `phase_install` to dispatch on the config's `spec.bundle.deployment.deployer` (existing helmfile body moved into `install_helmfile`, byte-equivalent), and adds `install_argocd`: helm-installs the pinned argo-cd chart from `.settings.yaml`, provisions a prefix-matched ghcr.io repo-creds Secret from `GITHUB_TOKEN`, `kubectl apply`s the app-of-apps under a shared-budget retry loop, waits for the root `nvidia-stack` Application to be reified (grace capped at the shared budget so the loop can never outlast it), and waits for every Application to reach a terminal-pass state using the same 4-arm predicate the KWOK chainsaw sync gate encodes. Shared-budget discipline (mirroring install_helmfile): a single ARGOCD_SYNC_TIMEOUT_SECONDS wall clock spans the whole install path from repo-creds Secret apply through terminal-pass. Every kubectl invocation (Secret apply, app-of-apps apply retries, root-app grace poll, terminal-pass poll) is bounded by `timeout ${remaining}`; every sleep between polls (app-of-apps retry 15s, root-app 5s, sync-wait 15s) is capped to the remaining budget so a nap near the deadline cannot overrun it. Each retry loop re-checks `SECONDS >= deadline` before starting the next attempt. Root-app grace deadline is additionally capped at the shared argocd_deadline so an upstream step that spent most of the shared budget can never let the root grace add its full 2m on top. Fails closed at every step; the whole install step also has a step-level 110m cap in the workflow to fit the argocd branch's total budget (helm 5m + apply/sync 30m + root grace 2m + gate 60m = 97m) and keep phases.sh's own `::error::` paths reachable. Failure-diagnostic path is best-effort throughout (kubectl calls all `|| true`) so a transient apiserver hiccup can't skip the describe / repo-server logs a reviewer needs. Sync-wait `bad` variable is initialized with a sentinel so the timeout diagnostic reads sensibly even if the loop never executed. The bundle reaches Argo CD via `aicr bundle --output oci://... --repo oci://...` -- pushed to ghcr.io/nvidia/aicr-bundle-scratch under the existing `packages: write` scope; a per-run tag isolates concurrent runs. `GITHUB_TOKEN` is explicitly passed into the install step's env (Actions auto-injects GITHUB_ACTOR but not the token) so the in-cluster repo-creds Secret can be provisioned from it; the helmfile branch never reads it. The post-install readiness gate stays deployer-agnostic (it validates deployed cluster state, not deployment mechanism), so a green argocd cell is direct evidence the GitOps path converges on the same operator-managed stack the helmfile lane validates. Workflow surface: adds a `deployer` dispatch input to uat-run.yaml + uat-aws.yaml (values `helmfile|argocd`; `helmfile` is the default and selects the existing `--config.yaml` filename, `argocd` selects `--argocd-config.yaml`). The choice uses explicit `helmfile` rather than an empty string because actionlint rejects empty strings in a choice's `options:`. The `Validate inputs` step allowlists the deployer value after AWS_ACCOUNT_ID export so daytime-down teardown still authenticates. Retention follow-up: run-scoped bundle artifacts under ghcr.io/nvidia/aicr-bundle-scratch are not cleaned up on the success path. Deferred while this cell is manual-dispatch-only (low accumulation); noted inline for nightly-enrollment follow-up via either a workflow teardown step or an org-level retention policy. Manual dispatch only on `aws-h100` training for now -- nightly enrollment and extension to other cells are follow-ups after a green manual run, mirroring the azure-h100 (#1722) and kind-h100 (#1843) onboarding pattern. `argocd-helm` variant deferred. Fixes: #2194 Signed-off-by: framsouza --- .github/workflows/uat-aws.yaml | 66 +++- .github/workflows/uat-run.yaml | 18 + docs/contributor/uat.md | 16 + .../tests/h100-training-argocd-config.yaml | 106 ++++++ tests/uat/lib/phases.sh | 344 +++++++++++++++++- 5 files changed, 528 insertions(+), 22 deletions(-) create mode 100644 tests/uat/aws/tests/h100-training-argocd-config.yaml diff --git a/.github/workflows/uat-aws.yaml b/.github/workflows/uat-aws.yaml index 63133a02c..c6aee1e90 100644 --- a/.github/workflows/uat-aws.yaml +++ b/.github/workflows/uat-aws.yaml @@ -68,6 +68,14 @@ on: description: 'Skip UAT test execution (manual runs only; the nightly batch always tests).' type: boolean default: false + deployer: + # helmfile = --config.yaml (the existing + # filename shape every cell has always used). argocd = insert + # -argocd before -config.yaml (e.g. h100-training-argocd-config.yaml). + # See issue #2194. + description: 'Deployer variant.' + type: string + default: helmfile permissions: contents: read @@ -139,10 +147,19 @@ jobs: && format('aicr-uat-day-{0}-{1}-{2}', inputs.slug, inputs.slot, github.run_id) || format('aicr-uat-{0}', github.run_id) }} CLUSTER_CONFIG: ${{ inputs.cluster_config_path }} - # The AICRConfig is selected by accelerator + intent - # (--config.yaml); both intents drive the same - # cluster-config (GPU pool from the reservation, system/CPU pools dynamic). - TEST_CONFIG: ${{ inputs.test_config_dir }}/${{ inputs.accelerator }}-${{ inputs.intent }}-config.yaml + # The AICRConfig is selected by accelerator + intent + deployer: + # deployer=helmfile → --config.yaml (the default; + # backwards-compatible with every existing cell) + # deployer=X (X!=helmfile) → --X-config.yaml + # (e.g. deployer=argocd → h100-training-argocd-config.yaml, issue #2194) + # Both variants drive the same cluster-config (GPU pool from the + # reservation, system/CPU pools dynamic). The step-level check below asserts + # the resolved file exists so an unwired deployer variant fails early + # (before any provisioning) rather than deep in phase_prep. + TEST_CONFIG: >- + ${{ inputs.test_config_dir }}/${{ inputs.accelerator }}-${{ inputs.intent }}${{ + inputs.deployer != 'helmfile' && format('-{0}', inputs.deployer) || '' + }}-config.yaml # v0.4.27 — required for node-group taints in the cluster-config schema # (GPU pool carries skyhook.nvidia.com=runtime-required:NoSchedule until # NodeWrite finishes tuning + reboot). Earlier v0.4.x pins predate taint @@ -190,6 +207,7 @@ jobs: LIFECYCLE: ${{ inputs.lifecycle }} SLUG: ${{ inputs.slug }} SLOT: ${{ inputs.slot }} + DEPLOYER: ${{ inputs.deployer }} run: | set -euo pipefail case "${INTENT}" in @@ -233,6 +251,15 @@ jobs: echo "::error::${CLUSTER_CONFIG} has no .deployment.tenancy"; exit 1 fi echo "AWS_ACCOUNT_ID=${TENANCY}" >> "${GITHUB_ENV}" + # Validate deployer after the account export (which teardown depends on). + # workflow_dispatch constrains this to a choice, but workflow_call passes + # free strings, so a typo from a caller must surface as a clear error here + # rather than as a missing test-config file below or a mysterious dispatch + # deep in phases.sh. Allowlist mirrors uat-run.yaml's choice input. + case "${DEPLOYER}" in + helmfile|argocd) ;; + *) echo "::error::unsupported deployer '${DEPLOYER}' (want helmfile|argocd)"; exit 1 ;; + esac # The test config only matters for the run phases (nightly / daytime-up # provision+deploy). daytime-down just tears the held cluster down and # never reads it, so skip the existence check there — a missing sibling @@ -685,21 +712,34 @@ jobs: aws-region: ${{ env.AWS_REGION }} role-session-name: GitHubActions-UAT-AICR-Install - - name: UAT - install (helmfile apply) + - name: UAT - install (helmfile apply or argocd sync) id: install if: steps.prep.outcome == 'success' - # helmfile apply (up to HELMFILE_TIMEOUT_SECONDS, 20m) + the post-install - # readiness gate, which runs `aicr validate --phase deployment` until it - # passes READINESS_CONSECUTIVE_PASSES times (up to READINESS_TIMEOUT_SECONDS, - # 60m, spanning nodewright tuning + reboots, which can run past 30m on a - # cold GPU node). Sized to exceed one helmfile attempt + the full gate - # window (20 + 60 = 80, + margin) so the gate's own fail-closed path runs - # before GitHub Actions kills the step. - timeout-minutes: 90 + # Step-level timeout must fit the LARGER of the two deployer branches, + # because both dispatch from `./run install` and share this cap: + # helmfile: HELMFILE_TIMEOUT_SECONDS 20m + gate 60m = 80m + # argocd: helm-install 5m + apply+sync 30m + root-grace 2m + gate 60m = 97m + # ARGOCD_HELM_TIMEOUT_SECONDS + ARGOCD_SYNC_TIMEOUT_SECONDS + + # ARGOCD_ROOT_APP_GRACE_SECONDS + READINESS_TIMEOUT_SECONDS in + # tests/uat/lib/phases.sh drive the argocd side. Sized at 110m so the + # argocd fail-closed paths (each shared-budget retry loop's `::error::`) + # run before GitHub Actions kills the step. The gate's own fail-closed + # path still fires within budget on both branches. + timeout-minutes: 110 shell: bash env: AICR_BIN: ${{ github.workspace }}/aicr RUN_ID: ${{ github.run_id }} + # Consumed only by the argocd deployer branch (install_argocd in + # tests/uat/lib/phases.sh), which provisions an in-cluster ghcr.io + # repo-creds Secret so Argo CD's repo-server can pull the bundle + # pushed by phase_prep. GITHUB_ACTOR is auto-injected by the + # Actions runtime; GITHUB_TOKEN is not, so pass it explicitly. + # The token is scoped to `packages: write` at the job level (see + # permissions block above), which is exactly what the Secret + # needs. The helmfile branch is unaffected — install_helmfile + # never reads GITHUB_TOKEN. + GITHUB_TOKEN: ${{ github.token }} run: ./tests/uat/aws/run install "${TEST_CONFIG}" # Runs ALL validate phases (deployment + conformance + performance), not diff --git a/.github/workflows/uat-run.yaml b/.github/workflows/uat-run.yaml index 0d1d07806..a76864756 100644 --- a/.github/workflows/uat-run.yaml +++ b/.github/workflows/uat-run.yaml @@ -84,6 +84,20 @@ on: description: 'Skip UAT test execution (manual debugging only).' type: boolean default: false + deployer: + # helmfile = default via --config.yaml (the + # existing filename shape every cell has always used, kept for + # backwards-compat). argocd = --argocd-config.yaml + # (e.g. h100-training-argocd-config.yaml). Only aws-h100 training + # ships an argocd variant today; other cells fail with a missing + # test-config. See issue #2194. (actionlint rejects an empty + # string in a choice's options, so `helmfile` is the explicit + # default value here; the TEST_CONFIG resolver in uat-aws.yaml + # maps it to the unsuffixed filename.) + description: 'Deployer variant. See workflow file for the variant matrix.' + type: choice + options: [helmfile, argocd] + default: helmfile workflow_call: inputs: reservation: @@ -110,6 +124,9 @@ on: skip_tests: type: boolean default: false + deployer: + type: string + default: helmfile permissions: contents: read @@ -223,6 +240,7 @@ jobs: accelerator: ${{ needs.resolve.outputs.accelerator }} skip_delete: ${{ inputs.skip_delete }} skip_tests: ${{ inputs.skip_tests }} + deployer: ${{ inputs.deployer }} run-gcp: needs: resolve diff --git a/docs/contributor/uat.md b/docs/contributor/uat.md index 3ef6da18b..84943a2ea 100644 --- a/docs/contributor/uat.md +++ b/docs/contributor/uat.md @@ -84,6 +84,22 @@ The single nightly cron (`uat-nightly-batch.yaml`, `0 4 * * *`) runs **both inte Semantics: **`main` is never gated** (it is built from source and carries the newest fixes, so it always runs every listed intent); a **release** cell drops any intent whose minimum version is newer than the tag (semver; a tag `>=` the minimum runs). The gate lives in the schedule (`uat-broker schedule` attaches each cell's eligible `intents`), so the controller simply never dispatches a gated `(version × intent)` — no per-version workflow logic. Pointing the floor at a **not-yet-tagged** release is intentional and self-resolving: until that release ships, the intent runs on **`main` only** (green, continuous coverage of the fix), and the release enrolls automatically once it exists. `Validate` rejects a floor for an intent the row does not run, or a non-semver value. Bump the floor if the real first-fixed tag differs — an over-low floor surfaces as a visible red (safe), an over-high floor silently skips a good release (bump down). +## Selecting the deployer + +The `deployer` input picks which deployer variant of the intent's test config the pipeline consumes. Set to `helmfile` (the default), the pipeline resolves `tests/uat//tests/--config.yaml` — the config every existing cell has always run against. Any other value (currently only `argocd`) resolves `---config.yaml` — for example `deployer=argocd` on `aws-h100` training loads `tests/uat/aws/tests/h100-training-argocd-config.yaml`. + +```bash +# ArgoCD variant of the aws-h100 training cell (issue #2194) +gh workflow run uat-run.yaml --repo NVIDIA/aicr --ref main \ + -f reservation=aws-h100 -f intent=training -f deployer=argocd +``` + +The AICRConfig field `spec.bundle.deployment.deployer` is the source of truth `phase_prep`/`phase_install` read; the workflow input is only how the correct config file is *selected*. `phases.sh:phase_install` dispatches to `install_helmfile` (helmfile lane, unchanged) or `install_argocd` (Argo CD install + repo-creds Secret from `GITHUB_TOKEN` + `kubectl apply` of the `nvidia-stack` app-of-apps + terminal-pass wait on every `Application`). The post-install readiness gate is deployer-agnostic — it validates deployed cluster state (`aicr validate --phase deployment`), not the deployment mechanism — so a green ArgoCD cell means the GitOps deploy path converges on the same operator-managed stack the helmfile lane validates. + +**How the bundle reaches Argo CD.** `phase_prep` calls `aicr bundle --output oci://ghcr.io/nvidia/aicr-bundle-scratch/:run- --repo oci://ghcr.io/nvidia/aicr-bundle-scratch` — the `--output` flag pushes the rendered bundle to GHCR (the job already has `packages: write`), and `--repo` sets the `source.repoURL` baked into every generated `Application`. `install_argocd` provisions a prefix-matched `argocd.argoproj.io/secret-type: repo-creds` Secret from `GITHUB_TOKEN` so Argo CD's repo-server can pull the pushed artifact. Concurrent runs on the same recipe are isolated by the `:run-` tag. + +**Coverage today (issue #2194).** Only `aws-h100` training carries a `-argocd` config file; other cells fail closed in the `Validate inputs` step with `test config not found` if dispatched with `deployer=argocd`. Nightly enrollment is deliberately deferred until a manual dispatch is green on hardware — mirroring the `azure-h100` (#1722) and `kind-h100` (#1843) onboarding pattern. The `argocd-helm` variant, and extension to gcp/azure/kind, are separate follow-ups. + ## Cluster lifecycles The `lifecycle` input selects one of three cluster lifecycles, all sharing the reservation lease: diff --git a/tests/uat/aws/tests/h100-training-argocd-config.yaml b/tests/uat/aws/tests/h100-training-argocd-config.yaml new file mode 100644 index 000000000..c7702cf67 --- /dev/null +++ b/tests/uat/aws/tests/h100-training-argocd-config.yaml @@ -0,0 +1,106 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# AICRConfig consumed by `aicr {snapshot,recipe,bundle,validate} --config`. +# +# ArgoCD deployer variant of h100-training-config.yaml. The only functional +# difference is `spec.bundle.deployment.deployer: argocd` — every other field +# (recipe criteria, scheduling, snapshot/validate agent) is identical to the +# helmfile sibling so a green run here is direct evidence the `--deployer +# argocd` GitOps path works on the same recipe the helmfile lane validates. +# +# See ../../lib/phases.sh (install_argocd) for the install-phase flow: +# `aicr bundle` pushes the same content to ghcr.io/nvidia/aicr-bundle-scratch +# and Argo CD's repo-server pulls from that OCI ref via a prefix-matched +# repo-creds Secret provisioned in-cluster from the workflow's GITHUB_TOKEN. +# +# Manual dispatch only for now (issue #2194): enroll in the nightly batch as +# a separate follow-up after a green manual run mirrors the azure-h100 / +# kind onboarding pattern (PRs #1722, #1843). +# +# Evidence push target is deliberately shared with the helmfile config: the +# recipe (and therefore the evidence coordinate) is identical, and RUN_ID +# tag isolation prevents same-recipe cross-run collisions. If the dashboard +# grows a deployer facet later, tag the run there — not by forking the OCI +# repo per deployer. +kind: AICRConfig +apiVersion: aicr.run/v1alpha2 +metadata: + name: eks-h100-training-uat-argocd + +spec: + snapshot: + output: + path: snapshot.yaml + agent: + namespace: aicr-validation + nodeSelector: + nodeGroup: gpu-worker + tolerations: + - dedicated=worker-workload:NoSchedule + - dedicated=worker-workload:NoExecute + - nvidia.com/gpu=present:NoSchedule + - skyhook.nvidia.com=runtime-required:NoSchedule + + recipe: + criteria: + service: eks + accelerator: h100 + os: ubuntu + intent: training + platform: kubeflow + output: + path: recipe.yaml + + bundle: + input: + recipe: recipe.yaml + output: + # phase_prep overrides --output at CLI time with an oci:// target + # (ARGOCD_OCI_PREFIX in lib/phases.sh + a run-scoped tag). The local + # ./bundle directory still receives app-of-apps.yaml so `kubectl apply` + # has a manifest to consume. + target: ./bundle + deployment: + deployer: argocd + scheduling: + acceleratedNodeSelector: + nodeGroup: gpu-worker + acceleratedNodeTolerations: + - dedicated=worker-workload:NoSchedule + - dedicated=worker-workload:NoExecute + - nvidia.com/gpu=present:NoSchedule + systemNodeSelector: + nodeGroup: system-worker + systemNodeTolerations: + - dedicated=system-workload:NoSchedule + - dedicated=system-workload:NoExecute + storageClass: "" + + validate: + execution: + failFast: true + input: + recipe: recipe.yaml + snapshot: snapshot.yaml + agent: + namespace: aicr-validation + tolerations: + - dedicated=worker-workload:NoSchedule + - dedicated=worker-workload:NoExecute + - nvidia.com/gpu=present:NoSchedule + evidence: + attestation: + out: ./evidence + push: ghcr.io/nvidia/aicr-evidence/h100-eks-ubuntu-training-kubeflow diff --git a/tests/uat/lib/phases.sh b/tests/uat/lib/phases.sh index 30662056c..a6cf9e565 100644 --- a/tests/uat/lib/phases.sh +++ b/tests/uat/lib/phases.sh @@ -70,6 +70,24 @@ TRAINJOB_TIMEOUT_SECONDS="${TRAINJOB_TIMEOUT_SECONDS:-1200}" # 20 min # (tests/uat/kind/run) overrides to 1. TRAINJOB_NUM_NODES="${TRAINJOB_NUM_NODES:-2}" HELMFILE_TIMEOUT_SECONDS="${HELMFILE_TIMEOUT_SECONDS:-1200}" # 20 min +# ArgoCD deployer knobs (see install_argocd). ARGOCD_HELM_TIMEOUT_SECONDS +# bounds the `helm upgrade --install` of the argo-cd chart itself; +# ARGOCD_SYNC_TIMEOUT_SECONDS bounds the wait for every Application to +# reach a terminal-pass state. Both are shared wall-clock budgets, matching +# the discipline HELMFILE_TIMEOUT_SECONDS uses for the helmfile lane. +# ARGOCD_ROOT_APP_GRACE_SECONDS gives Argo CD a short window to reify the +# root `nvidia-stack` Application after `kubectl apply` -- a missing root +# after the grace means the apply silently produced no Application (RBAC +# collision, CRD not yet Established, etc.) and we should fail closed. +ARGOCD_HELM_TIMEOUT_SECONDS="${ARGOCD_HELM_TIMEOUT_SECONDS:-300}" # 5 min +ARGOCD_SYNC_TIMEOUT_SECONDS="${ARGOCD_SYNC_TIMEOUT_SECONDS:-1800}" # 30 min +ARGOCD_ROOT_APP_GRACE_SECONDS="${ARGOCD_ROOT_APP_GRACE_SECONDS:-120}" +# Prefix for the OCI target `aicr bundle --output` pushes to and the +# `--repo` baseURL baked into every Application. The path is under +# ghcr.io/nvidia (workflow already grants `packages: write` for evidence +# push) in a distinct namespace so bundle artifacts don't collide with +# signed evidence. See phase_prep's argocd branch. +ARGOCD_OCI_PREFIX="${ARGOCD_OCI_PREFIX:-oci://ghcr.io/nvidia/aicr-bundle-scratch}" # Budget for the post-install readiness gate (see phase_install), which runs # `aicr validate --phase deployment` until it passes READINESS_CONSECUTIVE_PASSES # times in a row. This is the gate window ONLY -- it is entered AFTER helmfile @@ -390,16 +408,103 @@ phase_prep() { echo "::endgroup::" echo "::group::Generate bundle" - "${AICR_BIN}" bundle --config "${config}" - test -f bundle/helmfile.yaml || { - echo "expected bundle/helmfile.yaml (deployer: helmfile) — got:" >&2 - ls -la bundle >&2 || true - exit 1 - } + # The bundle shape (and how it is delivered to the cluster) is deployer- + # specific: helmfile emits bundle/helmfile.yaml consumed off the local + # filesystem; argocd emits bundle/app-of-apps.yaml AND pushes the same + # content to an OCI registry so Argo CD's repo-server can pull it. Read + # the deployer straight from the AICRConfig -- keeping selection in the + # test-config (not a workflow env var) means the artifact under review + # is self-describing. + local deployer + deployer="$(yq -r '.spec.bundle.deployment.deployer // "helmfile"' "${config}")" + case "${deployer}" in + helmfile) + "${AICR_BIN}" bundle --config "${config}" + test -f bundle/helmfile.yaml || { + echo "expected bundle/helmfile.yaml (deployer: helmfile) — got:" >&2 + ls -la bundle >&2 || true + exit 1 + } + ;; + argocd) + # --output pushes bundle content to OCI; --repo sets the source.repoURL + # baked into the rendered nvidia-stack Application. Both flags override + # the config's spec.bundle.output.target (pkg/cli/bundle.go:583-604) + # so the AICRConfig stays deployer-neutral. + # + # URL shape (matches KWOK argocd-oci precedent at + # kwok/scripts/validate-scheduling.sh:951+): + # oci_repo = / (per-recipe, no tag) + # oci_target = /:run- (same repo, with tag) + # `aicr bundle` renders root Application spec.source.repoURL= + # and targetRevision=, so Argo CD pulls from oci_target — the same + # URL we pushed to. Passing a bare-prefix --repo (without the slug) + # would leave Argo CD chasing a non-existent artifact at + # oci://:main — verified locally with `aicr bundle --deployer + # argocd` inspection during Tier 1 validation of issue #2194. + # + # RUN_ID isolates concurrent runs on the same recipe. Argo CD prefix- + # matches repo-creds (util/db/repository_secrets.go: + # getRepositoryCredentialIndex → HasPrefix), so a single Secret at + # ARGOCD_OCI_PREFIX covers every pushed under that + # prefix — provisioned once by install_argocd. + # + # Retention follow-up (#2194): this pushes a new run-tagged artifact + # every dispatch, and there is no cleanup on the successful path. Left + # deferred while this cell is manual-dispatch-only (accumulation rate + # low, cost bounded); to be addressed at nightly enrollment by EITHER + # a workflow teardown step that `gh api DELETE`s the tag it stashed as + # a job output, OR an org-level retention policy on + # ghcr.io/nvidia/aicr-bundle-scratch. The `-scratch` namespace name + # already signals ephemeral intent to any operator inspecting GHCR. + local bundle_slug oci_repo oci_target + bundle_slug="$(yq -r '.metadata.name' "${config}")" + oci_repo="${ARGOCD_OCI_PREFIX}/${bundle_slug}" + oci_target="${oci_repo}:run-${RUN_ID}" + echo "argocd bundle push target: ${oci_target}" + echo "argocd Application source.repoURL: ${oci_repo}" + "${AICR_BIN}" bundle --config "${config}" \ + --output "${oci_target}" \ + --repo "${oci_repo}" + test -f bundle/app-of-apps.yaml || { + echo "expected bundle/app-of-apps.yaml (deployer: argocd) — got:" >&2 + ls -la bundle >&2 || true + exit 1 + } + ;; + *) + echo "::error::unsupported deployer: ${deployer} (supported: helmfile, argocd)" >&2 + exit 1 + ;; + esac echo "::endgroup::" } phase_install() { + # Dispatch to the deployer-specific install body. The readiness gate below + # is deployer-agnostic (it validates deployed cluster state, not the + # deployment mechanism) so it stays in phase_install; only the "get the + # stack onto the cluster" step differs. + local deployer + deployer="$(yq -r '.spec.bundle.deployment.deployer // "helmfile"' "${config}")" + case "${deployer}" in + helmfile) install_helmfile ;; + argocd) install_argocd ;; + *) + echo "::error::unsupported deployer: ${deployer} (supported: helmfile, argocd)" >&2 + exit 1 + ;; + esac + + echo "::group::Cluster state post-install" + kubectl get nodes -o wide + kubectl get pods -A | grep -Ev '\s+Running\s+|\s+Completed\s+' || true + echo "::endgroup::" + + install_readiness_gate +} + +install_helmfile() { command -v helmfile >/dev/null || { echo "helmfile not on PATH" >&2; exit 1; } command -v helm >/dev/null || { echo "helm not on PATH" >&2; exit 1; } @@ -487,12 +592,233 @@ phase_install() { echo "::error::helmfile apply failed after 3 attempts" >&2 exit 1 fi +} - echo "::group::Cluster state post-install" - kubectl get nodes -o wide - kubectl get pods -A | grep -Ev '\s+Running\s+|\s+Completed\s+' || true +install_argocd() { + command -v helm >/dev/null || { echo "helm not on PATH" >&2; exit 1; } + command -v kubectl >/dev/null || { echo "kubectl not on PATH" >&2; exit 1; } + + local SCRIPT_DIR + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + local REPO_ROOT="${SCRIPT_DIR}/../../.." + local ARGOCD_CHART_VERSION + ARGOCD_CHART_VERSION="$(yq -r '.testing_tools.argocd_chart' "${REPO_ROOT}/.settings.yaml")" + + echo "::group::Install Argo CD (chart ${ARGOCD_CHART_VERSION})" + # KWOK precedent (kwok/scripts/install-infra.sh:install_argocd) — same + # chart, same namespace/release. `helm upgrade --install` is idempotent + # so a re-run (or a cluster that already has Argo CD baked in) converges + # cleanly. `--wait --timeout` blocks until argocd-server + application- + # controller are Ready so downstream steps (repo-creds, apply) don't + # race a partially-rolled-out install. + helm repo add argo https://argoproj.github.io/argo-helm --force-update >/dev/null 2>&1 || true + helm repo update argo >/dev/null + if ! helm upgrade --install argocd argo/argo-cd \ + --namespace argocd --create-namespace \ + --version "${ARGOCD_CHART_VERSION}" \ + --wait --timeout "${ARGOCD_HELM_TIMEOUT_SECONDS}s"; then + echo "::error::Argo CD helm install failed" >&2 + kubectl -n argocd get pods + exit 1 + fi + # CRD race guard — applications.argoproj.io must be Established before we + # apply the app-of-apps, else kubectl apply races the CRD controller. + kubectl wait --for=condition=Established \ + crd/applications.argoproj.io --timeout=120s + echo "::endgroup::" + + # Start the shared ARGOCD_SYNC_TIMEOUT_SECONDS wall clock HERE, before the + # repo-creds Secret apply, so a hung apiserver on any single step (Secret + # apply, app-of-apps apply, root-app grace, terminal-pass poll) draws from + # the same 30m budget install_helmfile enforces with its own shared clock. + # Without this, a stalled Secret apply would burn indefinite time before + # the downstream retry loop even started measuring. + local argocd_deadline=$(( SECONDS + ARGOCD_SYNC_TIMEOUT_SECONDS )) + local secret_remaining + + echo "::group::Provision ghcr.io repo-creds Secret (prefix match)" + # Argo CD prefix-matches repo-creds against Application source URLs + # (util/db/repository_secrets.go::getRepositoryCredentialIndex → HasPrefix), + # so ONE Secret annotated argocd.argoproj.io/secret-type=repo-creds at + # url=ARGOCD_OCI_PREFIX covers every recipe pushed under that prefix. + # GITHUB_TOKEN + GITHUB_ACTOR are provided by the UAT workflow env + # (packages: write is already declared on every uat-*.yaml job). + : "${GITHUB_TOKEN:?GITHUB_TOKEN required for ghcr.io pull creds}" + : "${GITHUB_ACTOR:?GITHUB_ACTOR required for ghcr.io pull creds}" + # Field shape mirrors kwok/scripts/install-infra.sh:apply_repo_secret + # for chart 9.5.x / Argo CD v3.x — type: oci is the direct OCI credential + # kind (chart 7.x needed type: helm + enableOCI=true; superfluous here + # and dropped to avoid confusion). username/password are the only fields + # KWOK omits, because its in-cluster registry is unauthenticated HTTP; + # ghcr.io is HTTPS + auth, so we supply the GITHUB_ACTOR/GITHUB_TOKEN + # already scoped to `packages: write` on every UAT job. The whole apply is + # bounded by the remaining shared budget so a hung apiserver cannot outlast + # the sync-wait window; clamp >= 1 for the `timeout 0 == no timeout` + # boundary condition install_helmfile guards the same way. + secret_remaining=$(( argocd_deadline - SECONDS )) + (( secret_remaining < 1 )) && secret_remaining=1 + if ! timeout "${secret_remaining}" kubectl apply -f - <&2 + exit 1 + fi + echo "::endgroup::" + + # Retry `kubectl apply -f bundle/app-of-apps.yaml` on transient apiserver + # errors, mirroring install_helmfile's shared-budget discipline. In practice + # `kubectl apply` on a single manifest almost never needs the retry, but the + # shared-budget shape keeps the two branches behaviorally symmetric and + # bounds the install step's worst-case wall clock the same way. + # + # Each attempt is capped at the remaining shared ARGOCD_SYNC_TIMEOUT_SECONDS + # budget with `timeout`, and the deadline is re-checked BEFORE each attempt + # so a stalled apiserver cannot spend the whole 30m budget on kubectl apply + # and starve the downstream sync-wait. Clamp `remaining` to >= 1 for the + # same reason install_helmfile does: `timeout 0` means "no timeout". + local applied=false apply_remaining apply_nap + for attempt in 1 2 3; do + if (( SECONDS >= argocd_deadline )); then + echo "argocd shared ${ARGOCD_SYNC_TIMEOUT_SECONDS}s budget exhausted after attempt $(( attempt - 1 )); not starting attempt ${attempt}" + break + fi + apply_remaining=$(( argocd_deadline - SECONDS )) + (( apply_remaining < 1 )) && apply_remaining=1 + echo "::group::kubectl apply app-of-apps (attempt ${attempt}/3, timeout ${apply_remaining}s of ${ARGOCD_SYNC_TIMEOUT_SECONDS}s shared budget)" + if timeout "${apply_remaining}" kubectl apply -f bundle/app-of-apps.yaml; then + applied=true + echo "::endgroup::" + break + fi + echo "::endgroup::" + # Cap the retry delay to remaining shared budget so `sleep 15` near the + # deadline can't overrun it — same discipline as the root-app and sync- + # wait loops. Positive-only guard so the loop exits cleanly at budget=0. + if (( attempt < 3 )); then + apply_nap=15 + apply_remaining=$(( argocd_deadline - SECONDS )) + (( apply_nap > apply_remaining )) && apply_nap=${apply_remaining} + if (( apply_nap > 0 )); then + echo "waiting ${apply_nap}s before retry" + sleep "${apply_nap}" + fi + fi + done + if [[ "${applied}" != "true" ]]; then + echo "::error::kubectl apply app-of-apps.yaml failed after 3 attempts (or budget exhausted)" >&2 + exit 1 + fi + + echo "::group::Wait for root Application '${ARGOCD_ROOT_APP:-nvidia-stack}' to be reified" + # A missing root after the grace window means the apply silently produced + # no Application (RBAC race, CRD version skew, malformed manifest). Fail + # closed rather than time out downstream on an empty controller queue. + # Cap the root-app grace at the shared argocd_deadline so this loop can + # never run past the ARGOCD_SYNC_TIMEOUT_SECONDS budget that spans the + # whole install path (Secret apply + apply retries + this grace + sync + # poll). Without the cap, an upstream step that spent most of the shared + # budget could still let the root-app grace add its full 2m on top. + local root_deadline=$(( SECONDS + ARGOCD_ROOT_APP_GRACE_SECONDS )) + (( root_deadline > argocd_deadline )) && root_deadline=${argocd_deadline} + local root_app="${ARGOCD_ROOT_APP:-nvidia-stack}" + local root_ready=false root_remaining root_nap + while (( SECONDS < root_deadline )); do + # Bound each kubectl by remaining budget so a hung apiserver cannot burn + # the whole grace window on one call. Clamp to >= 1 (same rationale as + # install_helmfile: `timeout 0` means no timeout). + root_remaining=$(( root_deadline - SECONDS )) + (( root_remaining < 1 )) && root_remaining=1 + if timeout "${root_remaining}" kubectl -n argocd get application "${root_app}" >/dev/null 2>&1; then + root_ready=true + break + fi + # Cap the sleep to remaining budget so we do not overrun the deadline + # waiting between polls. + root_nap=5 + root_remaining=$(( root_deadline - SECONDS )) + (( root_nap > root_remaining )) && root_nap=${root_remaining} + (( root_nap > 0 )) && sleep "${root_nap}" + done + if [[ "${root_ready}" != "true" ]]; then + echo "::error::root Application '${root_app}' not reified within ${ARGOCD_ROOT_APP_GRACE_SECONDS}s" >&2 + kubectl -n argocd get applications || true + exit 1 + fi + echo "::endgroup::" + + echo "::group::Wait for all Argo CD Applications to reach terminal-pass (budget $(( argocd_deadline - SECONDS ))s)" + # 4-arm terminal-pass predicate, copied from + # tests/chainsaw/kwok/argocd-sync/chainsaw-test.yaml (same 4 states the + # KWOK chainsaw sync gate encodes; source-agnostic per that test's SYNC + # NOTE header). Real hardware should mostly land on state 1 + # (Synced+Healthy); operator mutation (gpu-operator ClusterPolicy, + # ResourceSlice injection) legitimately leaves individual apps at + # OutOfSync+Healthy; Progressing/Degraded are tolerated because Argo + # health controllers occasionally diverge post-op — the ultimate + # verdict lives in the deployment readiness gate that follows. + local jq_bad + jq_bad=' + [ .items[] | + select( + ((.status.sync.status == "Synced") and (.status.health.status | IN("Healthy","Progressing","Degraded"))) or + ((.status.sync.status == "OutOfSync") and (.status.health.status == "Healthy")) + | not) + | (.metadata.name + "[" + (.status.sync.status // "?") + "/" + (.status.health.status // "?") + "]") + ] | join(", ")' + # Initialize `bad` with a sentinel so the failure diagnostic below reads + # sensibly even in the pathological case where the while guard is false on + # the first check (SECONDS already >= argocd_deadline because kubectl-apply + # + root-grace consumed the whole shared budget). Loop iterations overwrite + # this with real values. + local bad="not sampled (sync-wait loop never ran — budget spent by upstream steps)" + local sync_remaining sync_nap + while (( SECONDS < argocd_deadline )); do + # Bound each poll by remaining shared budget so a hung apiserver on `get` + # cannot outlast the sync window (same discipline as the apply-retry loop + # above). Clamp to >= 1 for the `timeout 0 == no timeout` boundary. + sync_remaining=$(( argocd_deadline - SECONDS )) + (( sync_remaining < 1 )) && sync_remaining=1 + bad="$(timeout "${sync_remaining}" kubectl -n argocd get applications -o json 2>/dev/null | jq -r "${jq_bad}" || echo "ERR")" + if [[ -z "${bad}" ]]; then + echo "all Applications in terminal-pass state" + echo "::endgroup::" + return 0 + fi + echo "waiting for: ${bad}" + # Cap the poll interval to remaining budget so `sleep 15` near the + # deadline can't overrun it. Positive-only guard so we exit the loop + # cleanly when budget hits 0. + sync_nap=15 + sync_remaining=$(( argocd_deadline - SECONDS )) + (( sync_nap > sync_remaining )) && sync_nap=${sync_remaining} + (( sync_nap > 0 )) && sleep "${sync_nap}" + done + echo "::error::Argo CD sync did not converge within ${ARGOCD_SYNC_TIMEOUT_SECONDS}s; last bad: ${bad}" >&2 + # Best-effort failure diagnostic — every call `|| true` so a transient + # apiserver error on `get` doesn't skip the `describe` and repo-server + # logs that follow (which are what a reviewer actually needs to diagnose + # an OCI-pull auth error or a sync-wave block). Matches the KWOK failure + # dump semantics. + kubectl -n argocd get applications || true + kubectl -n argocd describe applications 2>&1 | head -200 >&2 || true + kubectl -n argocd logs -l app.kubernetes.io/name=argocd-repo-server --tail=100 2>&1 >&2 || true echo "::endgroup::" + exit 1 +} +install_readiness_gate() { # Readiness gate: run the deployment validation phase -- the authoritative # expected-resources / ClusterPolicy / DRA / nodewright checks the later # `--phase all` run gates on -- in a retry loop until it passes