diff --git a/deploy/stacks/nvcf-compute-plane/README.md b/deploy/stacks/nvcf-compute-plane/README.md index a38172da7..39f6aafee 100644 --- a/deploy/stacks/nvcf-compute-plane/README.md +++ b/deploy/stacks/nvcf-compute-plane/README.md @@ -43,8 +43,10 @@ without the `.yaml` suffix. ## Observability The stack defaults `observability.profile` to `compute`. The `compute` and -`all` profiles enable the NVCA collector and `BYOObservability` feature gate. -The `control` and `disabled` profiles leave both off. +`all` profiles enable the `BYOObservability` feature gate. The optional NVCA +collector stays disabled for every profile until the operator opts in. This +prevents a self-hosted install from depending on an image that was not mirrored +into its registry. One value selects the normal behavior: @@ -53,16 +55,15 @@ observability: profile: compute ``` -Explicit NVCA values override the profile defaults: +Enable the collector after publishing it under `global.image`. Its repository +defaults to `${global.image.registry}/${global.image.repository}/nvcf-otel-collector`: ```yaml global: nvcaOperator: selfManaged: otelCollector: - enabled: false - featureGateValues: - - "-BYOObservability" + enabled: true ``` ## Chart and Image Sources diff --git a/deploy/stacks/nvcf-compute-plane/environments/base.yaml b/deploy/stacks/nvcf-compute-plane/environments/base.yaml index c7a5f46a2..b04d29ff8 100644 --- a/deploy/stacks/nvcf-compute-plane/environments/base.yaml +++ b/deploy/stacks/nvcf-compute-plane/environments/base.yaml @@ -35,14 +35,17 @@ global: selfManaged: nvcaVersion: "3.2.7" otelCollector: + # Self-hosted registries do not always mirror this optional image. + # Enable it only after publishing the collector in global.image. + enabled: false imageTag: "0.157.9" - # ICMS (SIS) service URL — required; set per environment. + # ICMS (SIS) service URL: required; set per environment. icmsServiceURL: "" icmsServiceHostHeaderOverride: "" - # Reval service URL — required; set per environment. + # Reval service URL: required; set per environment. revalServiceURL: "" revalServiceHostHeaderOverride: "" - # NATS URL — required; set per environment. + # NATS URL: required; set per environment. natsURL: "" natsHostOverride: "" # Feature gates enabled on this cluster. Extra gates set here are added on diff --git a/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl b/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl index e99565306..85b207eae 100644 --- a/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl +++ b/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl @@ -90,7 +90,9 @@ helmDefaults: {{- $nvcaOp := dig "nvcaOperator" dict .Values.global }} {{- $selfManaged := dig "selfManaged" dict $nvcaOp }} {{- $otelCollector := dig "otelCollector" dict $selfManaged }} -{{- $otelCollectorEnabled := dig "enabled" $computeObservabilityEnabled $otelCollector }} +{{- $otelCollectorEnabled := dig "enabled" false $otelCollector }} +{{- $defaultOTelCollectorImageRepository := printf "%s/%s/nvcf-otel-collector" .Values.global.image.registry .Values.global.image.repository }} +{{- $otelCollectorImageRepository := dig "imageRepository" $defaultOTelCollectorImageRepository $otelCollector }} {{- $featureGateValues := dig "featureGateValues" list $selfManaged }} {{- $kaiSchedulerEnabled := dig "addons" "kaiScheduler" "enabled" false .Values }} @@ -224,9 +226,7 @@ releases: "dynamoOperatorEnabled" $dynamoOperatorEnabled) | nindent 10 }} otelCollector: enabled: {{ $otelCollectorEnabled }} - {{- with dig "imageRepository" "" $otelCollector }} - imageRepository: {{ . | quote }} - {{- end }} + imageRepository: {{ $otelCollectorImageRepository | quote }} {{- with dig "imageTag" "" $otelCollector }} imageTag: {{ . | quote }} {{- end }} diff --git a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml index be6ab8a0b..b584e6ae7 100644 --- a/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml +++ b/deploy/stacks/nvcf-compute-plane/testdata/golden/local/02-nvca.yaml-nvca-operator/helm-nvca-operator/templates/self-managed-nvcfbackend-cm.yaml @@ -61,9 +61,9 @@ data: imageRepository: "nvcr.io/0651155215864979/ncp-dev/samba" imageTag: "1.0.5" otelCollector: - enabled: true + enabled: false imageConfig: - repository: "nvcr.io/nvidia/nvcf-byoc/nvcf-otel-collector" + repository: "nvcr.io/0651155215864979/ncp-dev/nvcf-otel-collector" tag: "0.157.9" agent: natsURL: "nats://nats.nats-system.svc.cluster.local:4222" diff --git a/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh b/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh index 6c08e9353..6768b6cab 100755 --- a/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh +++ b/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh @@ -57,6 +57,14 @@ collector_image_tag() { ' "$1" } +collector_image_repository() { + awk ' + /^selfManaged:$/ { self_managed = 1; next } + self_managed && /^ otelCollector:$/ { collector = 1; next } + collector && /^ imageRepository:/ { gsub(/"/, "", $2); print $2; exit } + ' "$1" +} + operator_image_tag() { awk ' /^image:$/ { image = 1; next } @@ -76,13 +84,9 @@ for profile in default disabled control compute all; do render_values "$profile" "$values" case "$profile" in - default|compute|all) - test "$(collector_enabled "$values")" = "true" || - fail "$profile profile did not enable the NVCA collector" - ;; - disabled|control) + default|disabled|control|compute|all) test "$(collector_enabled "$values")" = "false" || - fail "$profile profile enabled the NVCA collector" + fail "$profile profile enabled the self-managed NVCA collector by default" ;; esac done @@ -93,6 +97,14 @@ test "$(operator_image_tag "$work_dir/default.yaml")" = "$expected_nvca_version" test "$(nvca_version "$work_dir/default.yaml")" = "$expected_nvca_version" || fail "default NVCA version is not $expected_nvca_version" +render_values compute "$work_dir/ncp-dev-default.yaml" \ + --state-values-set-string global.image.registry=nvcr.io \ + --state-values-set-string global.image.repository=0651155215864979/ncp-dev + +test "$(collector_image_repository "$work_dir/ncp-dev-default.yaml")" = \ + "nvcr.io/0651155215864979/ncp-dev/nvcf-otel-collector" || + fail "collector image repository did not inherit the global image repository" + render_values compute "$work_dir/compute-overrides.yaml" \ --state-values-set-string global.nvcaOperator.imageTag=operator-test-tag \ --state-values-set-string global.nvcaOperator.selfManaged.nvcaVersion=nvca-test-tag \ diff --git a/tests/bdd/features/observability-all.feature b/tests/bdd/features/observability-all.feature index fb230fb1e..432b1cf94 100644 --- a/tests/bdd/features/observability-all.feature +++ b/tests/bdd/features/observability-all.feature @@ -37,11 +37,11 @@ Feature: Install local Helmfile observability for both planes # Configure NVCA to join the same cluster and enable its collector. And I copy the file "tests/bdd/fixtures/nvcf-compute-plane-local-bdd.yaml" to "deploy/stacks/nvcf-compute-plane/environments/local-bdd-observability-all.yaml" And I update yaml file "deploy/stacks/nvcf-compute-plane/environments/local-bdd-observability-all.yaml" with keys: - | global.imagePullSecrets[0].name | nvcr-pull-secret | - | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | - | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | - | global.nvcaOperator.selfManaged.otelCollector.imageRepository | nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-otel-collector | - | observability.profile | all | + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.nvcaOperator.selfManaged.otelCollector.enabled | true | + | observability.profile | all | And I copy the file "deploy/stacks/self-managed/secrets/secrets.yaml.template" to "deploy/stacks/self-managed/secrets/local-bdd-observability-all-secrets.yaml" And I substitute "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" in file "deploy/stacks/self-managed/secrets/local-bdd-observability-all-secrets.yaml" with base64 of "$oauthtoken:${NGC_API_KEY}" # Conflict precheck: the split topology claims host ports used by the @@ -143,3 +143,10 @@ Feature: Install local Helmfile observability for both planes """ Then the command exit code should be 0 And the command output should contain "true" + + When I run command: + """ + bash -c 'set -eo pipefail; helm get values nvca-operator --namespace nvca-operator --kube-context k3d-ncp-local -o json | jq -r ".selfManaged.otelCollector.imageRepository"' + """ + Then the command exit code should be 0 + And the command output should contain "nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-otel-collector" diff --git a/tests/bdd/features/observability-compute.feature b/tests/bdd/features/observability-compute.feature index b0e14e81e..f7995362e 100644 --- a/tests/bdd/features/observability-compute.feature +++ b/tests/bdd/features/observability-compute.feature @@ -39,11 +39,11 @@ Feature: Install local Helmfile observability with the compute profile # Configure NVCA to use the same compute observability profile. And I copy the file "tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml" to "deploy/stacks/nvcf-compute-plane/environments/local-bdd-observability-compute.yaml" And I update yaml file "deploy/stacks/nvcf-compute-plane/environments/local-bdd-observability-compute.yaml" with keys: - | global.imagePullSecrets[0].name | nvcr-pull-secret | - | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | - | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | - | global.nvcaOperator.selfManaged.otelCollector.imageRepository | nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-otel-collector | - | observability.profile | compute | + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.nvcaOperator.selfManaged.otelCollector.enabled | true | + | observability.profile | compute | And I copy the file "deploy/stacks/self-managed/secrets/secrets.yaml.template" to "deploy/stacks/self-managed/secrets/local-bdd-observability-compute-secrets.yaml" And I substitute "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" in file "deploy/stacks/self-managed/secrets/local-bdd-observability-compute-secrets.yaml" with base64 of "$oauthtoken:${NGC_API_KEY}" # Conflict precheck: single-cluster ncp-local claims host ports used by the @@ -161,6 +161,13 @@ Feature: Install local Helmfile observability with the compute profile Then the command exit code should be 0 And the command output should contain "true" + When I run command: + """ + bash -c 'set -eo pipefail; helm get values nvca-operator --namespace nvca-operator --kube-context k3d-ncp-local-compute-1 -o json | jq -r ".selfManaged.otelCollector.imageRepository"' + """ + Then the command exit code should be 0 + And the command output should contain "nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-otel-collector" + When I run command "helm status function-autoscaler --namespace nvcf --kube-context k3d-ncp-local-compute-1" Then the command exit code should be 1 And the command output should contain "release: not found" diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 7b38fe879..4c3fa85ac 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -597,6 +597,7 @@ func TestObservabilityComputeFeatureFileWiresToSteps(t *testing.T) { podMonitorCommand = "kubectl get podmonitor/nvcf-default-monitors-worker --namespace monitoring --context k3d-ncp-local-compute-1 -o name" absentServiceMonitorCommand = "kubectl get servicemonitor/nvcf-default-monitors-state-metrics --namespace monitoring --context k3d-ncp-local-compute-1 --ignore-not-found -o name" collectorEnabledCommand = `bash -c 'set -eo pipefail; helm get values nvca-operator --namespace nvca-operator --kube-context k3d-ncp-local-compute-1 -o json | jq -r ".selfManaged.otelCollector.enabled"'` + collectorRepositoryCommand = `bash -c 'set -eo pipefail; helm get values nvca-operator --namespace nvca-operator --kube-context k3d-ncp-local-compute-1 -o json | jq -r ".selfManaged.otelCollector.imageRepository"'` collectorYAMLCommand = "kubectl get opentelemetrycollector/nvcf-observability --namespace monitoring --context k3d-ncp-local-compute-1 -o yaml" serviceKeyCommand = `bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" |` + ` kubectl --context k3d-ncp-local-compute-1 create secret generic ngc-service-api-key` + @@ -616,6 +617,7 @@ func TestObservabilityComputeFeatureFileWiresToSteps(t *testing.T) { podMonitorCommand: {ExitCode: 0}, absentServiceMonitorCommand: {ExitCode: 0}, collectorEnabledCommand: {ExitCode: 0, Stdout: "true\n"}, + collectorRepositoryCommand: {ExitCode: 0, Stdout: "nvcr.io/test-org/test-team/nvcf-otel-collector\n"}, "helm list --all-namespaces --kube-context k3d-ncp-local-compute-1 -o json": { ExitCode: 0, Stdout: observabilityComputeHelmListJSON(), @@ -687,7 +689,7 @@ func TestObservabilityComputeFeatureFileWiresToSteps(t *testing.T) { } computeEnvironmentPath := filepath.Join(suite.Config.RepoRoot, "deploy", "stacks", "nvcf-compute-plane", "environments", "local-bdd-observability-compute.yaml") for key, want := range map[string]string{ - "global.nvcaOperator.selfManaged.otelCollector.imageRepository": "nvcr.io/test-org/test-team/nvcf-otel-collector", + "global.nvcaOperator.selfManaged.otelCollector.enabled": "true", } { got, found, err := dsl.ReadYAMLKey(computeEnvironmentPath, key) if err != nil { @@ -715,12 +717,13 @@ func observabilityComputeHelmListJSON() string { // local context and verifies that one shared stack serves both monitor sets. func TestObservabilityAllFeatureFileWiresToSteps(t *testing.T) { const ( - registryLoginCommand = `bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" | helm registry login nvcr.io --username "\$oauthtoken" --password-stdin'` - serviceMonitorCommand = "kubectl get servicemonitor/nvcf-default-monitors-state-metrics --namespace monitoring --context k3d-ncp-local -o name" - podMonitorCommand = "kubectl get podmonitor/nvcf-default-monitors-worker --namespace monitoring --context k3d-ncp-local -o name" - collectorEnabledCommand = `bash -c 'set -eo pipefail; helm get values nvca-operator --namespace nvca-operator --kube-context k3d-ncp-local -o json | jq -r ".selfManaged.otelCollector.enabled"'` - collectorYAMLCommand = "kubectl get opentelemetrycollector/nvcf-observability --namespace monitoring --context k3d-ncp-local -o yaml" - serviceKeyCommand = `bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" |` + + registryLoginCommand = `bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" | helm registry login nvcr.io --username "\$oauthtoken" --password-stdin'` + serviceMonitorCommand = "kubectl get servicemonitor/nvcf-default-monitors-state-metrics --namespace monitoring --context k3d-ncp-local -o name" + podMonitorCommand = "kubectl get podmonitor/nvcf-default-monitors-worker --namespace monitoring --context k3d-ncp-local -o name" + collectorEnabledCommand = `bash -c 'set -eo pipefail; helm get values nvca-operator --namespace nvca-operator --kube-context k3d-ncp-local -o json | jq -r ".selfManaged.otelCollector.enabled"'` + collectorRepositoryCommand = `bash -c 'set -eo pipefail; helm get values nvca-operator --namespace nvca-operator --kube-context k3d-ncp-local -o json | jq -r ".selfManaged.otelCollector.imageRepository"'` + collectorYAMLCommand = "kubectl get opentelemetrycollector/nvcf-observability --namespace monitoring --context k3d-ncp-local -o yaml" + serviceKeyCommand = `bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" |` + ` kubectl --context k3d-ncp-local create secret generic ngc-service-api-key` + ` --namespace nvca-system --from-file=ngc-service-api-key=/dev/stdin --dry-run=client -o yaml |` + ` kubectl --context k3d-ncp-local apply -f -'` @@ -737,6 +740,7 @@ func TestObservabilityAllFeatureFileWiresToSteps(t *testing.T) { serviceMonitorCommand: {ExitCode: 0}, podMonitorCommand: {ExitCode: 0}, collectorEnabledCommand: {ExitCode: 0, Stdout: "true\n"}, + collectorRepositoryCommand: {ExitCode: 0, Stdout: "nvcr.io/test-org/test-team/nvcf-otel-collector\n"}, serviceKeyCommand: {ExitCode: 0}, restartNVCACommand: {ExitCode: 0}, "helm list --all-namespaces --kube-context k3d-ncp-local -o json": { @@ -820,7 +824,7 @@ func TestObservabilityAllFeatureFileWiresToSteps(t *testing.T) { want string }{ {stack: "self-managed", key: "functionAutoscaler.image.tag", want: "1.18.10"}, - {stack: "nvcf-compute-plane", key: "global.nvcaOperator.selfManaged.otelCollector.imageRepository", want: "nvcr.io/test-org/test-team/nvcf-otel-collector"}, + {stack: "nvcf-compute-plane", key: "global.nvcaOperator.selfManaged.otelCollector.enabled", want: "true"}, } for _, assertion := range assertions { environmentPath := filepath.Join(suite.Config.RepoRoot, "deploy", "stacks", assertion.stack, "environments", "local-bdd-observability-all.yaml")