diff --git a/docs/dev/best-practices/tracing.md b/docs/dev/best-practices/tracing.md index 125033007..18f0c4d5c 100644 --- a/docs/dev/best-practices/tracing.md +++ b/docs/dev/best-practices/tracing.md @@ -74,7 +74,10 @@ These are head sampling ratios that bound what leaves the process. Keep decision Set `OTEL_TRACES_SAMPLER=always_off` on the components under test (for ateom workers, via the controller's `--otel-traces-sampler` flag). `parentbased_always_off` is not enough under a load generator: boomer and locust send ratio-sampled trace context, and parent based samplers honor it. Alternatively set the generator's `trace_probability` to 0 and leave the servers alone. On kind, also override ateapi's `parentbased_always_on` pin. -The YAML manifest for your server should include the `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable to point the exporter to GKE's managed traces collector, e.g.: +The YAML manifest for your server needs `OTEL_EXPORTER_OTLP_ENDPOINT` set so the +exporter knows where to push spans. Do not hardcode it — consume the shared +`ate-otel-config` ConfigMap via `envFrom`, so your server follows the collector +address for whichever environment it is deployed to: ```yaml containers: @@ -82,12 +85,20 @@ The YAML manifest for your server should include the `OTEL_EXPORTER_OTLP_ENDPOIN image: ko://github.com/agent-substrate/substrate/cmd/ateapi ports: - containerPort: 443 - env: - # Tracing related environment variables - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317" + # Supplies OTEL_EXPORTER_OTLP_ENDPOINT (and, on kind, the metric + # export tunables) for every control plane component. + envFrom: + - configMapRef: + name: ate-otel-config ``` +The ConfigMap is defined in +[`manifests/ate-install/ate-otel-config.yaml`](../../../manifests/ate-install/ate-otel-config.yaml) +for GKE, with a kind replacement of the same name in +[`manifests/ate-install/kind/ate-otel-config.yaml`](../../../manifests/ate-install/kind/ate-otel-config.yaml) +that points at the in-cluster collector. Editing either one does not restart the +pods that consume it; follow a change with `kubectl rollout restart`. + For how to deploy that collector — the GKE managed option, a self-managed DaemonSet, and the constraints on what endpoints Substrate can talk to — see [OpenTelemetry Collector Best Practices](otel-collector.md). #### gRPC Servers diff --git a/docs/observability.md b/docs/observability.md index 2f07892c6..e3aabd00e 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -187,6 +187,10 @@ Telemetry is emitted the same way everywhere; only the backend differs between a | Dashboards | Not supported | Google Cloud Monitoring (see [Dashboards](#5-dashboards)) | > In Kind, `ateapi`, `atelet`, `ate-controller`, and `atenet-router` are pointed at the in-cluster collector, and the controller propagates the endpoint to the ateom worker pods it creates, so all component telemetry lands locally. +> +> Every component reads that endpoint from the shared `ate-otel-config` ConfigMap ([`manifests/ate-install/ate-otel-config.yaml`](../manifests/ate-install/ate-otel-config.yaml), with a Kind replacement of the same name under [`manifests/ate-install/kind/`](../manifests/ate-install/kind/ate-otel-config.yaml)). Editing it does not restart the pods that consume it — follow a change with `kubectl rollout restart`. +> +> ateom workers don't read the ConfigMap at all — `ate-controller` copies the value into each worker pod at creation. A new endpoint reaches them only once the controller itself restarts, and that restart then rolls every WorkerPool Deployment, replacing the running workers along with the actors on them. --- diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 49c61d827..5f43449de 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -206,6 +206,20 @@ render_atenet_router_manifest() { fi } +# Apply the ate-otel-config ConfigMap that every control plane component reads +# via envFrom. The full install gets it through render_ate_system_manifests, but +# the targeted single-component redeploys below apply raw manifests with no +# Kustomize, so they have to select the environment's copy themselves. Applying +# the base file unconditionally would overwrite a kind cluster's ConfigMap with +# the GKE endpoint and silently break telemetry for every component at once. +apply_otel_config() { + if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then + run_kubectl apply -f manifests/ate-install/kind/ate-otel-config.yaml + else + run_kubectl apply -f manifests/ate-install/ate-otel-config.yaml + fi +} + # Extract a CA pool secret's RootCertificateDER and emit it as a PEM certificate. ca_pool_root_pem() { local secret="$1" @@ -345,6 +359,14 @@ deploy_ate_system() { run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s + # Ahead of the bundle below, for the same reason as the namespace: every + # workload pulls this ConfigMap in via envFrom, and a container whose envFrom + # target is missing will not start. The bundle contains it, but a raw + # directory apply orders by filename, so ate-api-server.yaml and + # ate-controller.yaml would otherwise be created before it and sit in + # CreateContainerConfigError until it caught up. + apply_otel_config + ensure_apiserver_prerequisites # Deploy podcertificate-controller first so it starts signing and creating trust bundles immediately @@ -397,6 +419,7 @@ deploy_ate_apiserver() { && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s ensure_apiserver_prerequisites + apply_otel_config run_ko apply -f manifests/ate-install/ate-api-server.yaml run_kubectl rollout status deployment/ate-api-server -n ate-system --timeout=120s @@ -410,6 +433,8 @@ deploy_atelet() { run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s + apply_otel_config + local manifest="" if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then # Use Kustomize to build and resolve the atelet DaemonSet patch @@ -430,6 +455,8 @@ deploy_atenet() { run_kubectl apply -f manifests/ate-install/ate-system-namespace.yaml \ && run_kubectl wait --for=jsonpath='{.status.phase}'=Active namespace/ate-system --timeout=60s + apply_otel_config + local router_manifest="" router_manifest="$(render_atenet_router_manifest)" echo "${router_manifest}" | run_kubectl apply -f - diff --git a/manifests/ate-install/ate-api-server.yaml b/manifests/ate-install/ate-api-server.yaml index 0bb913c86..5ea4f41a6 100644 --- a/manifests/ate-install/ate-api-server.yaml +++ b/manifests/ate-install/ate-api-server.yaml @@ -120,13 +120,16 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317 + # OTLP exporter settings are shared by every control plane component; + # see manifests/ate-install/ate-otel-config.yaml. Listed first so the + # per-developer ConfigMap below can still override a key. + envFrom: + - configMapRef: + name: ate-otel-config # Inject env vars from a ConfigMap created by each developer. This lets # each developer customize their own redis address, etc, without having # to edit this manifest, which can remain constant across all # developers. - envFrom: - configMapRef: name: ate-api-server-envvars optional: true diff --git a/manifests/ate-install/ate-controller.yaml b/manifests/ate-install/ate-controller.yaml index 76ceacf64..5d6d45d52 100644 --- a/manifests/ate-install/ate-controller.yaml +++ b/manifests/ate-install/ate-controller.yaml @@ -91,9 +91,12 @@ spec: args: - --ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem - --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem - env: - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317 + # OTLP exporter settings are shared by every control plane component; + # see manifests/ate-install/ate-otel-config.yaml. The controller also + # propagates these to the ateom worker pods it creates. + envFrom: + - configMapRef: + name: ate-otel-config ports: - name: metrics containerPort: 8080 diff --git a/manifests/ate-install/ate-otel-config.yaml b/manifests/ate-install/ate-otel-config.yaml new file mode 100644 index 000000000..eb04e4f53 --- /dev/null +++ b/manifests/ate-install/ate-otel-config.yaml @@ -0,0 +1,35 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Single source of truth for the OTLP exporter settings shared by every control +# plane component (ate-api-server, ate-controller, atelet, atenet-router), which +# consume it via `envFrom`. ate-controller additionally propagates these values +# to the ateom worker pods it creates, which have no other way to receive them, +# and atenet-router's --otlp-collector-address defaults to the endpoint below, +# so Envoy's own spans follow it too. +# +# This file holds the GKE values. The kind overlay supplies a ConfigMap of the +# same name with the in-cluster collector address; see +# manifests/ate-install/kind/ate-otel-config.yaml. +# +# NOTE: editing this ConfigMap does not restart the pods that consume it — the +# pod template is unchanged, so no new rollout is triggered. Follow a change +# with `kubectl rollout restart` on the affected workloads. +apiVersion: v1 +kind: ConfigMap +metadata: + name: ate-otel-config + namespace: ate-system +data: + OTEL_EXPORTER_OTLP_ENDPOINT: http://opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317 diff --git a/manifests/ate-install/atelet.yaml b/manifests/ate-install/atelet.yaml index b8e057a86..6b2cc4b55 100644 --- a/manifests/ate-install/atelet.yaml +++ b/manifests/ate-install/atelet.yaml @@ -105,10 +105,13 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317 - name: ATE_STORAGE_BACKEND value: "gcs" + # OTLP exporter settings are shared by every control plane component; + # see manifests/ate-install/ate-otel-config.yaml. + envFrom: + - configMapRef: + name: ate-otel-config ports: - name: grpc containerPort: 8085 diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index 9c8c9d7e0..769d1c2e5 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -154,8 +154,13 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317 + # OTLP exporter settings are shared by every control plane component; + # see manifests/ate-install/ate-otel-config.yaml. This also covers + # Envoy's own spans: --otlp-collector-address defaults to + # $OTEL_EXPORTER_OTLP_ENDPOINT, so the flag stays out of the args above. + envFrom: + - configMapRef: + name: ate-otel-config ports: - name: xds containerPort: 18000 diff --git a/manifests/ate-install/base/kustomization.yaml b/manifests/ate-install/base/kustomization.yaml index ac1623f83..6a6040ac2 100644 --- a/manifests/ate-install/base/kustomization.yaml +++ b/manifests/ate-install/base/kustomization.yaml @@ -15,6 +15,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +# ate-otel-config.yaml carries the OTLP settings every component above consumes +# via envFrom. The kind overlay lists its own copy of the same ConfigMap name +# instead of building on this directory, so the two never collide. resources: - ../ate-api-server.yaml - ../ate-controller.yaml @@ -23,3 +26,4 @@ resources: - ../atenet-router.yaml - ../valkey.yaml - ../pod-certificate-controller.yaml + - ../ate-otel-config.yaml diff --git a/manifests/ate-install/kind/ate-otel-config.yaml b/manifests/ate-install/kind/ate-otel-config.yaml new file mode 100644 index 000000000..037485691 --- /dev/null +++ b/manifests/ate-install/kind/ate-otel-config.yaml @@ -0,0 +1,40 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Kind replacement for manifests/ate-install/ate-otel-config.yaml: same name, so +# the overlay lists this file instead of the base one. Points the control plane +# at the in-cluster collector deployed by kind/otel-collector.yaml. +# +# OTEL_METRIC_EXPORT_INTERVAL shortens the OTel SDK's 60s metric export tick to +# 10s. A component is invisible to the collector until its first tick, and the +# metrics e2e suite asserts against the collector's scrape on a bounded deadline, +# so the default leaves too little headroom on a loaded runner. +# +# OTEL_METRIC_EXPORT_TIMEOUT bounds a tick that fails rather than one that +# succeeds, and its 30s default swallows three shortened intervals: a collector +# blip or slow DNS drops a component back to roughly one attempt per 30s, undoing +# the shorter interval. Matching it to the interval keeps every tick an attempt. +# +# On ate-controller both do double duty: the controller also propagates them to +# the ateom worker pods it creates, which have no other way to receive them. +# Kind only; production keeps the SDK defaults. +apiVersion: v1 +kind: ConfigMap +metadata: + name: ate-otel-config + namespace: ate-system +data: + OTEL_EXPORTER_OTLP_ENDPOINT: http://opentelemetry-collector.otel-system.svc:4317 + OTEL_METRIC_EXPORT_INTERVAL: "10000" + OTEL_METRIC_EXPORT_TIMEOUT: "10000" diff --git a/manifests/ate-install/kind/atelet/kustomization.yaml b/manifests/ate-install/kind/atelet/kustomization.yaml index c70724fac..2a0cfa411 100644 --- a/manifests/ate-install/kind/atelet/kustomization.yaml +++ b/manifests/ate-install/kind/atelet/kustomization.yaml @@ -38,9 +38,11 @@ patches: # Kind clusters are dev/CI: run atelet at debug so e2e suites can # assert on per-item log lines. Production installs default to info. - --log-level=debug + # The OTLP endpoint comes from the ate-otel-config ConfigMap the + # atelet DaemonSet consumes via envFrom; the kind copy lives at + # ../ate-otel-config.yaml and is applied by the parent overlay or + # by hack/install-ate.sh for a targeted redeploy. env: - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://opentelemetry-collector.otel-system.svc:4317 - name: ATE_STORAGE_BACKEND value: s3 - name: AWS_REGION diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index ba668e079..54a6b0b33 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -15,6 +15,12 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +# ate-otel-config.yaml replaces the base ../ate-otel-config.yaml (same ConfigMap +# name) so every component's `envFrom` resolves to the in-cluster collector +# instead of the GKE one. It is listed here rather than in ./atelet because that +# overlay is also built standalone; listing it in both would be a duplicate +# resource. hack/install-ate.sh applies the ConfigMap directly for the targeted +# single-component redeploys. resources: - ../ate-api-server.yaml - ../ate-controller.yaml @@ -23,27 +29,17 @@ resources: - ../atenet-router.yaml - ../valkey.yaml - ../pod-certificate-controller.yaml + - ate-otel-config.yaml - rustfs.yaml - ./otel-collector.yaml - ./prometheus.yaml -# OTEL_METRIC_EXPORT_INTERVAL shortens the OTel SDK's 60s metric export tick to -# 10s. A component is invisible to the collector until its first tick, and the -# metrics e2e suite asserts against the collector's scrape on a bounded deadline, -# so the default leaves too little headroom on a loaded runner. -# -# OTEL_METRIC_EXPORT_TIMEOUT bounds a tick that fails rather than one that -# succeeds, and its 30s default swallows three shortened intervals: a collector -# blip or slow DNS drops a component back to roughly one attempt per 30s, undoing -# the shorter interval. Matching it to the interval keeps every tick an attempt. -# -# On ate-controller both do double duty: the controller also propagates them to -# the ateom worker pods it creates, which have no other way to receive them. -# Kind only; production keeps the SDK defaults. -# # OTEL_TRACES_SAMPLER pins ate-api-server to parentbased_always_on so every API # call it roots is traced; the other components keep their binary defaults and -# follow its sampled flag via their ParentBased samplers. +# follow its sampled flag via their ParentBased samplers. It stays an inline +# patch rather than a key in ate-otel-config.yaml because that ConfigMap is +# shared by every component via envFrom -- putting always_on there would pin the +# router and the control plane to 100% too, undoing their per-component ratios. patches: - patch: |- apiVersion: apps/v1 @@ -57,61 +53,5 @@ patches: containers: - name: ate-api-server env: - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://opentelemetry-collector.otel-system.svc:4317 - - name: OTEL_METRIC_EXPORT_INTERVAL - value: "10000" - - name: OTEL_METRIC_EXPORT_TIMEOUT - value: "10000" - name: OTEL_TRACES_SAMPLER value: parentbased_always_on - - patch: |- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: atelet - namespace: ate-system - spec: - template: - spec: - containers: - - name: atelet - env: - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://opentelemetry-collector.otel-system.svc:4317 - - name: OTEL_METRIC_EXPORT_INTERVAL - value: "10000" - - name: OTEL_METRIC_EXPORT_TIMEOUT - value: "10000" - - patch: |- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: ate-controller - namespace: ate-system - spec: - template: - spec: - containers: - - name: ate-controller - env: - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://opentelemetry-collector.otel-system.svc:4317 - - name: OTEL_METRIC_EXPORT_INTERVAL - value: "10000" - - name: OTEL_METRIC_EXPORT_TIMEOUT - value: "10000" - - patch: |- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: atenet-router - namespace: ate-system - spec: - template: - spec: - containers: - - name: atenet-router - env: - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: http://opentelemetry-collector.otel-system.svc:4317