diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index aa7a9344..4a5051f0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -239,7 +239,10 @@ jobs: needs: resolve-images if: needs.resolve-images.outputs.should_run == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 20 + # The browser-driven trace verification adds a Node install, a Chromium + # download, and a Playwright run on top of the bash suite, so allow a wider + # ceiling than the 20 minutes the bash-only suite needed. + timeout-minutes: 25 steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -266,6 +269,9 @@ jobs: CONTROL_PLANE_IMAGE: ${{ needs.resolve-images.outputs.control_plane_image }} WEB_CONSOLE_IMAGE: ${{ needs.resolve-images.outputs.web_console_image }} KIND_ENABLE_OIDC: "true" + # Deploy Jaeger and point the web-console BFF at it so the browser + # trace verification below has a collector to export to (WEB-TRACE-10). + KIND_JAEGER: "true" run: make kind-up - name: Run e2e tests @@ -280,6 +286,61 @@ jobs: NO_COLOR: "1" run: bash tests/e2e/e2e-openshell.sh + # Browser-driven distributed-trace verification (WEB-TRACE-10): drive a + # real gateway workflow in the deployed console, then assert one trace in + # Jaeger joins the browser and the BFF. Reuses the same Node + Chromium + # setup the web-console lint job uses. + # + # These steps run on pull_request, push, and workflow_dispatch -- NOT + # merge_group. resolve-images maps an ephemeral merge-queue commit to + # on-pr-, which Konflux never builds, so merge_group always + # falls back to the baseline web-console image. That baseline lags main + # and cannot contain not-yet-merged tracing code, so the deployed console + # emits no spans and the assertion would test the wrong artifact. The + # trace path is gated at pull_request time against the freshly built PR + # image and re-verified on push to main against the post-merge image. + - name: Set up Node.js + if: github.event_name != 'merge_group' + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version-file: .node-version + - name: Install pinned pnpm + if: github.event_name != 'merge_group' + run: bash scripts/bootstrap_pnpm.sh + - name: Install dependencies + if: github.event_name != 'merge_group' + run: pnpm install --frozen-lockfile + - name: Get Playwright version + id: playwright-version + if: github.event_name != 'merge_group' + run: echo "version=$(pnpm --filter @openshift-online/hypershell-web-console exec playwright --version | awk '{print $2}')" >> "$GITHUB_OUTPUT" + # Install Chromium without `--with-deps` so the flaky apt phase + # (actions/runner-images#11347) never lands on the critical path -- the + # ubuntu-24.04 runner image already ships Chromium's system libraries. + # Cache the browser download too, keyed on the resolved Playwright + # version; only the default branch writes the cache, so PR runs cannot + # poison it. + - name: Restore Chromium cache + id: playwright-chromium-cache + if: github.event_name != 'merge_group' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/ms-playwright + key: playwright-chromium-${{ runner.os }}-${{ steps.playwright-version.outputs.version }} + - name: Install Chromium + if: github.event_name != 'merge_group' + timeout-minutes: 5 + run: pnpm --filter @openshift-online/hypershell-web-console exec playwright install chromium + - name: Save Chromium cache + if: github.event_name != 'merge_group' && github.ref == 'refs/heads/main' && steps.playwright-chromium-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/ms-playwright + key: ${{ steps.playwright-chromium-cache.outputs.cache-primary-key }} + - name: Verify end-to-end traces reach Jaeger + if: github.event_name != 'merge_group' + run: pnpm --filter @openshift-online/hypershell-web-console test:e2e:live + - name: Collect diagnostics if: failure() run: | @@ -305,6 +366,16 @@ jobs: kubectl logs --all-containers --prefix --tail=100 -l app=keycloak -n keycloak 2>&1 | tee e2e-diagnostics/keycloak-logs.txt || true echo "::endgroup::" + echo "::group::Tracing (Jaeger) diagnostics" + { + kubectl get pods,svc,httproutes -l app=jaeger -n hypershell-system -o wide 2>&1 || true + kubectl logs --all-containers --prefix --tail=100 -l app=jaeger -n hypershell-system 2>&1 || true + echo "--- web-console OTEL env ---" + kubectl get deployment/hypershell-web-console -n hypershell-system \ + -o jsonpath='{range .spec.template.spec.containers[*].env[*]}{.name}={.value}{"\n"}{end}' 2>&1 | grep -i otel || true + } | tee e2e-diagnostics/tracing.txt + echo "::endgroup::" + echo "::group::Events (hypershell-system)" kubectl get events --sort-by=.lastTimestamp -n hypershell-system 2>&1 | tee e2e-diagnostics/events.txt || true echo "::endgroup::" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 753050b8..6f8b14ea 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -218,15 +218,30 @@ jobs: run: pnpm install --frozen-lockfile - name: Run static, unit, Storybook, and production build gates run: pnpm check:web + - name: Get Playwright version + id: playwright-version + run: echo "version=$(pnpm --filter @openshift-online/hypershell-web-console exec playwright --version | awk '{print $2}')" >> "$GITHUB_OUTPUT" + # Install Chromium without `--with-deps` so the flaky apt phase + # (actions/runner-images#11347) never lands on the critical path -- the + # ubuntu-24.04 runner image already ships Chromium's system libraries. + # Cache the browser download too, keyed on the resolved Playwright + # version; only the default branch writes the cache, so PR runs cannot + # poison it. + - name: Restore Chromium cache + id: playwright-chromium-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/ms-playwright + key: playwright-chromium-${{ runner.os }}-${{ steps.playwright-version.outputs.version }} - name: Install Chromium - run: | - # Cap apt network timeouts so a flaky Azure mirror doesn't hang the job. - sudo tee /etc/apt/apt.conf.d/99-timeout <<'APT' - Acquire::Retries "3"; - Acquire::http::Timeout "30"; - Acquire::https::Timeout "30"; - APT - pnpm --filter @openshift-online/hypershell-web-console exec playwright install --with-deps chromium + timeout-minutes: 5 + run: pnpm --filter @openshift-online/hypershell-web-console exec playwright install chromium + - name: Save Chromium cache + if: github.ref == 'refs/heads/main' && steps.playwright-chromium-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/ms-playwright + key: ${{ steps.playwright-chromium-cache.outputs.cache-primary-key }} - name: Run critical Chromium journey run: pnpm test:e2e:chromium diff --git a/Makefile b/Makefile index ebcd21a3..f5dba71e 100644 --- a/Makefile +++ b/Makefile @@ -407,3 +407,14 @@ e2e: E2E_PROVISION_TIMEOUT=300 \ E2E_SANDBOX_TIMEOUT=180 \ bash tests/e2e/e2e-openshell.sh + +# Browser-driven end-to-end trace verification (WEB-TRACE-10). Requires a Kind +# cluster brought up with tracing enabled (KIND_JAEGER=true make kind-up), so +# Jaeger is deployed and the web-console BFF exports to it. +.PHONY: e2e-tracing +e2e-tracing: + @echo "" + @echo "==> Verifying end-to-end traces reach Jaeger (Kind)" + @echo " (requires: KIND_JAEGER=true make kind-up)" + @echo "" + @pnpm --filter @openshift-online/hypershell-web-console test:e2e:live diff --git a/components/web-console/app/adapters/api/api.client.test.ts b/components/web-console/app/adapters/api/api.client.test.ts index 014031f9..b2dc6399 100644 --- a/components/web-console/app/adapters/api/api.client.test.ts +++ b/components/web-console/app/adapters/api/api.client.test.ts @@ -76,6 +76,51 @@ describe("correlated API fetch", () => { expect(onReauth).not.toHaveBeenCalled(); expect(response.status).toBe(401); }); + + it("propagates the W3C trace context supplied by the provider", async () => { + const fetchImplementation = vi + .fn() + .mockResolvedValue(new Response(null, { status: 204 })); + const correlatedFetch = createCorrelatedFetch( + "44444444-4444-4444-8444-444444444444", + fetchImplementation, + undefined, + () => ({ + traceparent: "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", + tracestate: "hypershell=1", + }), + ); + + await correlatedFetch("/api/hypershell/v1/gateways"); + + const headers = new Headers( + fetchImplementation.mock.calls[0]?.[1]?.headers, + ); + expect(headers.get("traceparent")).toBe( + "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", + ); + expect(headers.get("tracestate")).toBe("hypershell=1"); + }); + + it("omits trace headers when the provider reports no active span", async () => { + const fetchImplementation = vi + .fn() + .mockResolvedValue(new Response(null, { status: 204 })); + const correlatedFetch = createCorrelatedFetch( + "55555555-5555-4555-8555-555555555555", + fetchImplementation, + undefined, + () => undefined, + ); + + await correlatedFetch("/api/hypershell/v1/gateways"); + + const headers = new Headers( + fetchImplementation.mock.calls[0]?.[1]?.headers, + ); + expect(headers.has("traceparent")).toBe(false); + expect(headers.has("tracestate")).toBe(false); + }); }); describe("redirectToLogin", () => { diff --git a/components/web-console/app/adapters/api/api.client.ts b/components/web-console/app/adapters/api/api.client.ts index e6b646d5..bf54a2a6 100644 --- a/components/web-console/app/adapters/api/api.client.ts +++ b/components/web-console/app/adapters/api/api.client.ts @@ -2,6 +2,19 @@ import { SDKClient } from "@openshift-online/hypershell-sdk"; export const gatewayCorrelationHeader = "x-hypershell-correlation-id"; +/** W3C Trace Context headers a request carries to the BFF for propagation. */ +export interface RequestTraceContext { + traceparent: string; + tracestate?: string; +} + +/** + * Supplies the W3C trace context for the in-flight workflow, or `undefined` + * when tracing is disabled or no span is active. Injected so the API adapter + * stays free of any tracing vendor dependency. + */ +export type TraceContextProvider = () => RequestTraceContext | undefined; + /** The BFF's machine-readable request to restart authentication at the IdP. */ export interface ReauthSignal { loginUrl: string; @@ -71,10 +84,18 @@ export function createCorrelatedFetch( correlationId: string, fetchImplementation: typeof globalThis.fetch = globalThis.fetch, onReauthRequired?: ReauthHandler, + traceContext?: TraceContextProvider, ): typeof globalThis.fetch { return async (input, init) => { const headers = new Headers(init?.headers); headers.set(gatewayCorrelationHeader, correlationId); + const trace = traceContext?.(); + if (trace !== undefined) { + headers.set("traceparent", trace.traceparent); + if (trace.tracestate !== undefined && trace.tracestate !== "") { + headers.set("tracestate", trace.tracestate); + } + } const response = await fetchImplementation(input, { ...init, headers }); if (onReauthRequired) { const signal = await readReauthSignal(response); @@ -94,6 +115,7 @@ export function createCorrelatedFetch( export function createApiClient( correlationId: string, onReauthRequired: ReauthHandler = redirectToLogin, + traceContext?: TraceContextProvider, ): SDKClient { return new SDKClient({ baseUrl: "", @@ -102,6 +124,7 @@ export function createApiClient( correlationId, globalThis.fetch, onReauthRequired, + traceContext, ), }); } diff --git a/components/web-console/app/adapters/observability/gateway-observability.test.ts b/components/web-console/app/adapters/observability/gateway-observability.test.ts index 8d4ae69e..23ccaeee 100644 --- a/components/web-console/app/adapters/observability/gateway-observability.test.ts +++ b/components/web-console/app/adapters/observability/gateway-observability.test.ts @@ -48,14 +48,51 @@ describe("gateway observability adapter", () => { ]); }); + it("records an out-of-band delivery failure into delivery health", () => { + const observability = createGatewayObservability({ + performanceTarget: { clearMarks: vi.fn(), mark: vi.fn() }, + }); + + observability.reportDeliveryFailure({ + errorType: "SpanExportError", + probeName: "gateway.trace.export", + schemaVersion: 0, + sinkId: "gateway-trace", + }); + + expect(observability.deliveryHealth()).toMatchObject({ + deliveryFailureCount: 1, + lastFailure: { sinkId: "gateway-trace" }, + }); + expect(observability.recentDeliveryFailures()).toEqual([ + expect.objectContaining({ probeName: "gateway.trace.export" }), + ]); + }); + it("provides deterministic workflow context through injected capabilities", () => { const observability = createGatewayObservability({ createCorrelationId: () => "correlation-1", + createTraceId: () => "0af7651916cd43dd8448eb211c80319c", now: () => "2026-08-06T18:00:00.000Z", performanceTarget: { clearMarks: vi.fn(), mark: vi.fn() }, }); expect(observability.runtime.createCorrelationId()).toBe("correlation-1"); + expect(observability.runtime.createTraceId()).toBe( + "0af7651916cd43dd8448eb211c80319c", + ); expect(observability.runtime.now()).toBe("2026-08-06T18:00:00.000Z"); }); + + it("generates a valid W3C trace identifier by default", () => { + const observability = createGatewayObservability({ + performanceTarget: { clearMarks: vi.fn(), mark: vi.fn() }, + }); + + const traceId = observability.runtime.createTraceId(); + + expect(traceId).toMatch(/^[0-9a-f]{32}$/); + expect(traceId).not.toBe("0".repeat(32)); + expect(observability.runtime.createTraceId()).not.toBe(traceId); + }); }); diff --git a/components/web-console/app/adapters/observability/gateway-observability.ts b/components/web-console/app/adapters/observability/gateway-observability.ts index 74576377..29d9fde9 100644 --- a/components/web-console/app/adapters/observability/gateway-observability.ts +++ b/components/web-console/app/adapters/observability/gateway-observability.ts @@ -13,6 +13,20 @@ import { const recentProbeLimit = 100; const recentFailureLimit = 20; +const traceIdByteLength = 16; + +/** + * Creates a W3C trace identifier: a 16-byte random value rendered as 32 + * lowercase hex digits. A random 16-byte value is never the all-zero value + * that the W3C Trace Context specification forbids. + */ +function createTraceId(): string { + const bytes = new Uint8Array(traceIdByteLength); + globalThis.crypto.getRandomValues(bytes); + return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join( + "", + ); +} interface PerformanceProbeTarget { clearMarks(name?: string): void; @@ -24,12 +38,17 @@ export interface GatewayObservability { probes: GatewayProbePublisher; recentDeliveryFailures(): readonly Readonly[]; recentProbes(): readonly GatewayProbe[]; + // Records a span delivery failure that surfaces asynchronously, outside the + // synchronous probe fan-out, so an export the browser could not complete is + // counted in delivery health instead of being lost. + reportDeliveryFailure(failure: Readonly): void; runtime: GatewayWorkflowRuntime; } export interface GatewayObservabilityOptions { additionalSinks?: readonly DomainProbeSink[]; createCorrelationId?: () => string; + createTraceId?: () => string; now?: () => string; performanceTarget?: PerformanceProbeTarget; } @@ -85,12 +104,14 @@ export function createGatewayObservability( probes: publisher, recentDeliveryFailures: () => Object.freeze([...failures]), recentProbes: () => Object.freeze([...recent]), + reportDeliveryFailure: (failure) => { + publisher.reportDeliveryFailure(failure); + }, runtime: { createCorrelationId: options.createCorrelationId ?? (() => globalThis.crypto.randomUUID()), + createTraceId: options.createTraceId ?? createTraceId, now: options.now ?? (() => new Date().toISOString()), }, }; } - -export const gatewayObservability = createGatewayObservability(); diff --git a/components/web-console/app/adapters/observability/gateway-trace-sink.test.ts b/components/web-console/app/adapters/observability/gateway-trace-sink.test.ts new file mode 100644 index 00000000..39817313 --- /dev/null +++ b/components/web-console/app/adapters/observability/gateway-trace-sink.test.ts @@ -0,0 +1,554 @@ +import type { + GatewayAction, + GatewayProbe, +} from "@openshift-online/hypershell-gateway-management-ui"; +import type { ProbeDeliveryFailure } from "@openshift-online/hypershell-domain-probes/fan-out"; +import { SpanStatusCode, type MeterProvider } from "@opentelemetry/api"; +import { ExportResultCode } from "@opentelemetry/core"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; +import { + AlwaysOffSampler, + AlwaysOnSampler, + BasicTracerProvider, + BatchSpanProcessor, + InMemorySpanExporter, + ParentBasedSampler, + SimpleSpanProcessor, + type BufferConfig, + type ReadableSpan, + type Sampler, + type SpanExporter, +} from "@opentelemetry/sdk-trace-base"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + RootTraceIdGenerator, + backstopExporter, + createGatewayTraceSink, + createGatewayTracing, + deliveryHealthMeterProvider, +} from "./gateway-trace-sink"; + +/** An exporter that accepts a batch but never acknowledges it. */ +const blockingExporter: SpanExporter = { + export: () => undefined, + forceFlush: () => Promise.resolve(), + shutdown: () => Promise.resolve(), +}; + +const traceId = "0af7651916cd43dd8448eb211c80319c"; +const correlationId = "correlation-1"; + +function probe( + name: GatewayProbe["name"], + overrides: Partial<{ + action: GatewayAction; + failureKind: GatewayProbe["fields"]["failureKind"]; + operationId: string; + outcome: GatewayProbe["fields"]["outcome"]; + traceId: string | undefined; + }> = {}, +): GatewayProbe { + const action = overrides.action ?? "list"; + return Object.freeze({ + context: Object.freeze({ + correlationId, + ...(overrides.operationId === undefined + ? {} + : { operationId: overrides.operationId }), + ...("traceId" in overrides + ? { traceId: overrides.traceId } + : { traceId }), + }), + fields: Object.freeze({ + action, + failureKind: overrides.failureKind ?? null, + outcome: overrides.outcome ?? "started", + }), + name, + occurredAt: "2026-08-06T18:00:00.000Z", + schemaVersion: 1, + }); +} + +function testTracer(rootSampler: Sampler = new AlwaysOnSampler()) { + const exporter = new InMemorySpanExporter(); + const idGenerator = new RootTraceIdGenerator(); + const provider = new BasicTracerProvider({ + idGenerator, + sampler: new ParentBasedSampler({ root: rootSampler }), + spanProcessors: [new SimpleSpanProcessor(exporter)], + }); + return { exporter, idGenerator, tracer: provider.getTracer("test") }; +} + +function byName(spans: readonly ReadableSpan[], name: string): ReadableSpan { + const span = spans.find((candidate) => candidate.name === name); + if (span === undefined) { + throw new Error(`no finished span named ${name}`); + } + return span; +} + +describe("gateway trace sink", () => { + let exporter: InMemorySpanExporter; + let sink: ReturnType; + + beforeEach(() => { + const harness = testTracer(); + exporter = harness.exporter; + sink = createGatewayTraceSink(harness.tracer, { + beginTrace: (id) => { + harness.idGenerator.primeTraceId(id); + }, + }); + }); + + it("adopts the probe trace id and nests the dependency under the workflow", () => { + sink.sink.publish(probe("gateway.workflow.started")); + sink.sink.publish(probe("gateway.dependency.attempted")); + sink.sink.publish( + probe("gateway.dependency.completed", { outcome: "succeeded" }), + ); + sink.sink.publish( + probe("gateway.workflow.completed", { outcome: "succeeded" }), + ); + + const finished = exporter.getFinishedSpans(); + const workflow = byName(finished, "gateway.workflow.list"); + const dependency = byName(finished, "gateway.dependency.list"); + + expect(workflow.spanContext().traceId).toBe(traceId); + expect(dependency.spanContext().traceId).toBe(traceId); + // The workflow span is a true trace root: it adopts the chosen trace id yet + // has no parent, so the trace is never decapitated by a synthetic remote + // parent that no service exports. The dependency nests under the workflow. + expect(workflow.parentSpanContext).toBeUndefined(); + expect(dependency.parentSpanContext?.spanId).toBe( + workflow.spanContext().spanId, + ); + expect(workflow.status.code).toBe(SpanStatusCode.OK); + expect(workflow.attributes["gateway.action"]).toBe("list"); + expect(workflow.attributes["gateway.outcome"]).toBe("succeeded"); + // The dependency span closes before the workflow span. + expect(finished.map((span) => span.name)).toEqual([ + "gateway.dependency.list", + "gateway.workflow.list", + ]); + }); + + it("marks a failed workflow and retains the operation identifier", () => { + sink.sink.publish(probe("gateway.workflow.started", { action: "rename" })); + sink.sink.publish( + probe("gateway.dependency.attempted", { action: "rename" }), + ); + sink.sink.publish( + probe("gateway.dependency.completed", { + action: "rename", + failureKind: "conflict", + operationId: "operation-1", + outcome: "conflicted", + }), + ); + sink.sink.publish( + probe("gateway.workflow.completed", { + action: "rename", + failureKind: "conflict", + operationId: "operation-1", + outcome: "conflicted", + }), + ); + + const finished = exporter.getFinishedSpans(); + for (const span of finished) { + expect(span.status.code).toBe(SpanStatusCode.ERROR); + expect(span.attributes["gateway.failure_kind"]).toBe("conflict"); + expect(span.attributes["gateway.outcome"]).toBe("conflicted"); + expect(span.attributes["hypershell.operation_id"]).toBe("operation-1"); + } + }); + + it("renders the active dependency span as a W3C traceparent", () => { + sink.sink.publish(probe("gateway.workflow.started")); + sink.sink.publish(probe("gateway.dependency.attempted")); + + const active = sink.traceParentFor(correlationId); + + expect(active?.traceparent).toMatch( + new RegExp(`^00-${traceId}-[0-9a-f]{16}-01$`), + ); + expect(active?.tracestate).toBeUndefined(); + }); + + it("reports no trace context once the workflow completes", () => { + sink.sink.publish(probe("gateway.workflow.started")); + sink.sink.publish(probe("gateway.dependency.attempted")); + sink.sink.publish( + probe("gateway.dependency.completed", { outcome: "succeeded" }), + ); + sink.sink.publish( + probe("gateway.workflow.completed", { outcome: "succeeded" }), + ); + + expect(sink.traceParentFor(correlationId)).toBeUndefined(); + }); + + it("ignores dependency and completion probes with no started workflow", () => { + expect(() => { + sink.sink.publish(probe("gateway.dependency.attempted")); + sink.sink.publish( + probe("gateway.dependency.completed", { outcome: "succeeded" }), + ); + sink.sink.publish( + probe("gateway.workflow.completed", { outcome: "succeeded" }), + ); + }).not.toThrow(); + expect(exporter.getFinishedSpans()).toHaveLength(0); + }); + + it("drops a trace when the sampler declines it", () => { + const harness = testTracer(new AlwaysOffSampler()); + const unsampled = createGatewayTraceSink(harness.tracer, { + beginTrace: (id) => { + harness.idGenerator.primeTraceId(id); + }, + }); + + unsampled.sink.publish(probe("gateway.workflow.started")); + unsampled.sink.publish(probe("gateway.dependency.attempted")); + const active = unsampled.traceParentFor(correlationId); + unsampled.sink.publish( + probe("gateway.dependency.completed", { outcome: "succeeded" }), + ); + unsampled.sink.publish( + probe("gateway.workflow.completed", { outcome: "succeeded" }), + ); + + // A declined trace still propagates its ids, with the sampled flag cleared, + // so downstream services make the same decision. + expect(active?.traceparent).toMatch( + new RegExp(`^00-${traceId}-[0-9a-f]{16}-00$`), + ); + expect(harness.exporter.getFinishedSpans()).toHaveLength(0); + }); + + it("keeps high-cardinality and sensitive values out of every exported span", () => { + // A correlation identifier is opaque and high-cardinality; treat it as a + // stand-in for any secret that must never reach the collector. + const secret = "corr-Bearer-eyJhbGciOi-SEEDED-SECRET"; + const withSecret = (base: GatewayProbe): GatewayProbe => + Object.freeze({ + ...base, + context: Object.freeze({ ...base.context, correlationId: secret }), + }); + + sink.sink.publish(withSecret(probe("gateway.workflow.started"))); + sink.sink.publish(withSecret(probe("gateway.dependency.attempted"))); + sink.sink.publish( + withSecret( + probe("gateway.dependency.completed", { outcome: "succeeded" }), + ), + ); + sink.sink.publish( + withSecret( + probe("gateway.workflow.completed", { + operationId: "operation-1", + outcome: "succeeded", + }), + ), + ); + + const allowedKeys = new Set([ + "gateway.action", + "gateway.outcome", + "gateway.failure_kind", + "hypershell.operation_id", + ]); + const finished = exporter.getFinishedSpans(); + expect(finished).toHaveLength(2); + for (const span of finished) { + // Span names come from a bounded template, never a raw identifier. + expect(span.name).toMatch(/^gateway\.(workflow|dependency)\.[a-z]+$/); + for (const key of Object.keys(span.attributes)) { + expect(allowedKeys.has(key)).toBe(true); + } + // The secret appears in no span name or attribute value. + const serialized = JSON.stringify({ + attributes: span.attributes, + name: span.name, + }); + expect(serialized).not.toContain(secret); + expect(serialized).not.toContain("SEEDED-SECRET"); + expect(serialized).not.toContain("Bearer"); + } + }); +}); + +describe("createGatewayTracing flush on page hide", () => { + const config = { + serviceName: "hypershell-web-console", + tracesEndpoint: "http://localhost/telemetry/v1/traces", + }; + + function setVisibility(state: "hidden" | "visible"): void { + Object.defineProperty(document, "visibilityState", { + configurable: true, + get: () => state, + }); + } + + afterEach(() => { + vi.restoreAllMocks(); + setVisibility("visible"); + }); + + it("flushes buffered spans on hidden visibilitychange and on pagehide", async () => { + const flush = vi + .spyOn(BasicTracerProvider.prototype, "forceFlush") + .mockResolvedValue(); + const tracing = createGatewayTracing(config); + + // A visible transition must not flush; only a hide is a last-chance export. + setVisibility("visible"); + document.dispatchEvent(new Event("visibilitychange")); + expect(flush).not.toHaveBeenCalled(); + + setVisibility("hidden"); + document.dispatchEvent(new Event("visibilitychange")); + expect(flush).toHaveBeenCalledTimes(1); + + window.dispatchEvent(new Event("pagehide")); + expect(flush).toHaveBeenCalledTimes(2); + + vi.spyOn(BasicTracerProvider.prototype, "shutdown").mockResolvedValue(); + await tracing.shutdown(); + }); + + it("stops flushing once shutdown removes the listeners", async () => { + const flush = vi + .spyOn(BasicTracerProvider.prototype, "forceFlush") + .mockResolvedValue(); + const shutdown = vi + .spyOn(BasicTracerProvider.prototype, "shutdown") + .mockResolvedValue(); + const tracing = createGatewayTracing(config); + + await tracing.shutdown(); + expect(shutdown).toHaveBeenCalledTimes(1); + flush.mockClear(); + + setVisibility("hidden"); + document.dispatchEvent(new Event("visibilitychange")); + window.dispatchEvent(new Event("pagehide")); + expect(flush).not.toHaveBeenCalled(); + }); +}); + +describe("createGatewayTracing delivery health", () => { + const config = { + serviceName: "hypershell-web-console", + tracesEndpoint: "http://localhost/telemetry/v1/traces", + }; + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("reports a failed span export as an out-of-band delivery failure", async () => { + // The collector is unreachable: the exporter yields a FAILED result, which + // the batch processor would otherwise swallow into its global error handler. + vi.spyOn(OTLPTraceExporter.prototype, "export").mockImplementation( + (_spans, resultCallback) => { + resultCallback({ + code: ExportResultCode.FAILED, + error: new Error("collector unreachable"), + }); + }, + ); + const failures: Readonly[] = []; + const tracing = createGatewayTracing(config, { + reportDeliveryFailure: (failure) => failures.push(failure), + }); + + tracing.sink.publish(probe("gateway.workflow.started")); + tracing.sink.publish( + probe("gateway.workflow.completed", { outcome: "succeeded" }), + ); + // The flush rejects on the failed export; the failure is recorded before the + // rejection propagates, so settling it here is enough. + await tracing.forceFlush().catch(() => undefined); + + expect(failures).toEqual([ + { + errorType: "Error", + probeName: "gateway.trace.export", + schemaVersion: 0, + sinkId: "gateway-trace", + }, + ]); + + vi.spyOn(OTLPTraceExporter.prototype, "shutdown").mockResolvedValue(); + await tracing.shutdown(); + }); + + it("does not report when the export succeeds", async () => { + vi.spyOn(OTLPTraceExporter.prototype, "export").mockImplementation( + (_spans, resultCallback) => { + resultCallback({ code: ExportResultCode.SUCCESS }); + }, + ); + const failures: Readonly[] = []; + const tracing = createGatewayTracing(config, { + reportDeliveryFailure: (failure) => failures.push(failure), + }); + + tracing.sink.publish(probe("gateway.workflow.started")); + tracing.sink.publish( + probe("gateway.workflow.completed", { outcome: "succeeded" }), + ); + await tracing.forceFlush(); + + expect(failures).toEqual([]); + + vi.spyOn(OTLPTraceExporter.prototype, "shutdown").mockResolvedValue(); + await tracing.shutdown(); + }); + + it("reports every span dropped by an overflowing queue", () => { + // A queue that holds one span plus an exporter that never drains it forces + // the batch processor to drop every subsequent span. Those drops never + // reach the exporter callback, so only the self-observation meter can + // surface them. + vi.useFakeTimers(); + try { + const failures: Readonly[] = []; + const processorConfig: BufferConfig & { + selfObsMeterProvider?: MeterProvider; + } = { + maxExportBatchSize: 1, + maxQueueSize: 1, + scheduledDelayMillis: 1, + selfObsMeterProvider: deliveryHealthMeterProvider((failure) => + failures.push(failure), + ), + }; + const provider = new BasicTracerProvider({ + sampler: new AlwaysOnSampler(), + spanProcessors: [ + new BatchSpanProcessor(blockingExporter, processorConfig), + ], + }); + const tracer = provider.getTracer("overflow-test"); + + // The exporter never drains, so once the in-flight batch and the single + // queue slot are taken, every further span overflows and is dropped. + for (let index = 0; index < 8; index += 1) { + tracer.startSpan(`span-${String(index)}`).end(); + } + + expect(failures.length).toBeGreaterThanOrEqual(1); + expect( + failures.every((failure) => failure.errorType === "queue_full"), + ).toBe(true); + expect(failures).toContainEqual({ + errorType: "queue_full", + probeName: "gateway.trace.export", + schemaVersion: 0, + sinkId: "gateway-trace", + }); + } finally { + vi.useRealTimers(); + } + }); + + it("reports a wedged exporter that never acknowledges a batch", async () => { + // The exporter accepts the batch and never calls back. The processor's own + // export timeout would reject the flush without accounting the loss; the + // backstop converts the stall into a FAILED result the meter records. + vi.useFakeTimers(); + try { + const failures: Readonly[] = []; + const processorConfig: BufferConfig & { + selfObsMeterProvider?: MeterProvider; + } = { + maxExportBatchSize: 1, + scheduledDelayMillis: 1, + selfObsMeterProvider: deliveryHealthMeterProvider((failure) => + failures.push(failure), + ), + }; + const provider = new BasicTracerProvider({ + sampler: new AlwaysOnSampler(), + spanProcessors: [ + new BatchSpanProcessor( + backstopExporter(blockingExporter, 15_000), + processorConfig, + ), + ], + }); + const tracer = provider.getTracer("blocking-test"); + tracer.startSpan("wedged").end(); + + const flushed = provider.forceFlush().catch(() => undefined); + await vi.advanceTimersByTimeAsync(15_000); + await flushed; + + expect(failures).toEqual([ + { + errorType: "SpanExportTimeout", + probeName: "gateway.trace.export", + schemaVersion: 0, + sinkId: "gateway-trace", + }, + ]); + } finally { + vi.useRealTimers(); + } + }); + + it("reports a synchronous exporter throw as a delivery failure", async () => { + // The exporter throws instead of calling back. Without the backstop's throw + // guard the loss would bypass settlement and go entirely unaccounted. + const throwingExporter: SpanExporter = { + export: () => { + const error = new Error("exporter blew up"); + error.name = "SyncExportError"; + throw error; + }, + forceFlush: () => Promise.resolve(), + shutdown: () => Promise.resolve(), + }; + const failures: Readonly[] = []; + const processorConfig: BufferConfig & { + selfObsMeterProvider?: MeterProvider; + } = { + maxExportBatchSize: 1, + scheduledDelayMillis: 1, + selfObsMeterProvider: deliveryHealthMeterProvider((failure) => + failures.push(failure), + ), + }; + const provider = new BasicTracerProvider({ + sampler: new AlwaysOnSampler(), + spanProcessors: [ + new BatchSpanProcessor( + backstopExporter(throwingExporter, 15_000), + processorConfig, + ), + ], + }); + const tracer = provider.getTracer("throwing-test"); + tracer.startSpan("thrown").end(); + + await provider.forceFlush().catch(() => undefined); + + expect(failures).toEqual([ + { + errorType: "SyncExportError", + probeName: "gateway.trace.export", + schemaVersion: 0, + sinkId: "gateway-trace", + }, + ]); + }); +}); diff --git a/components/web-console/app/adapters/observability/gateway-trace-sink.ts b/components/web-console/app/adapters/observability/gateway-trace-sink.ts new file mode 100644 index 00000000..4361572d --- /dev/null +++ b/components/web-console/app/adapters/observability/gateway-trace-sink.ts @@ -0,0 +1,483 @@ +import type { GatewayProbe } from "@openshift-online/hypershell-gateway-management-ui"; +import type { + DomainProbeSink, + ProbeDeliveryFailure, +} from "@openshift-online/hypershell-domain-probes/fan-out"; +import { + ROOT_CONTEXT, + SpanKind, + SpanStatusCode, + TraceFlags, + context as otelContext, + createNoopMeter, + isSpanContextValid, + trace as otelTrace, + type Attributes, + type Counter, + type Meter, + type MeterProvider, + type MetricOptions, + type Span, + type Tracer, +} from "@opentelemetry/api"; +import { + BasicTracerProvider, + BatchSpanProcessor, + ParentBasedSampler, + RandomIdGenerator, + TraceIdRatioBasedSampler, + type BufferConfig, + type IdGenerator, + type SpanExporter, +} from "@opentelemetry/sdk-trace-base"; +import { ExportResultCode, type ExportResult } from "@opentelemetry/core"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; +import { resourceFromAttributes } from "@opentelemetry/resources"; +import { + ATTR_ERROR_TYPE, + ATTR_SERVICE_NAME, +} from "@opentelemetry/semantic-conventions"; + +/** W3C `traceparent`/`tracestate` header pair for outbound propagation. */ +export interface GatewayTraceContext { + traceparent: string; + tracestate?: string; +} + +/** A gateway trace sink plus the propagation reader that feeds the API client. */ +export interface GatewayTraceSink { + /** + * Reads the active workflow (or in-flight dependency) span for one + * correlation identifier and renders its W3C context, or `undefined` when no + * span is active for that correlation identifier. + */ + traceParentFor: (correlationId: string) => GatewayTraceContext | undefined; + sink: DomainProbeSink; +} + +/** A gateway trace sink together with its provider lifecycle controls. */ +export interface GatewayTracing extends GatewayTraceSink { + forceFlush(): Promise; + shutdown(): Promise; +} + +export interface GatewayTraceSinkOptions { + /** + * Primes the trace id that the next root workflow span adopts, so the + * workflow span is a true root that owns the app-chosen trace id rather than + * descending from a synthetic remote parent. Wired to the provider's + * {@link RootTraceIdGenerator}. When omitted, a workflow keeps its probe + * trace id only for propagation and the exported root uses a generated id. + */ + beginTrace?: (traceId: string) => void; +} + +export interface GatewayTracingConfig { + serviceName: string; + /** Same-origin OTLP/HTTP traces path the browser exporter posts to. */ + tracesEndpoint: string; + /** Fraction of traces to record, 0..1. Defaults to 1 (record all). */ + sampleRatio?: number; +} + +export interface GatewayTracingOptions { + /** + * Records a span delivery failure that surfaces after buffering, when the + * batch exporter cannot reach the collector. Span export is asynchronous, so + * a failed batch would otherwise be dropped silently; routing it here makes + * the loss observable through the domain probe delivery-health accounting. + */ + reportDeliveryFailure?: (failure: Readonly) => void; +} + +const sinkId = "gateway-trace"; +const tracerName = "gateway-trace-sink"; +// Synthetic probe name for a failure that is not tied to one probe but to the +// asynchronous export of a batch of spans this sink already accepted. +const traceExportProbeName = "gateway.trace.export"; +// Upper bound on how long the sink waits for the exporter to acknowledge a +// batch before it synthesizes a failed result. It sits below the batch +// processor's own export timeout (30s) so a wedged exporter is converted into a +// FAILED callback -- which the processor accounts for through the +// self-observation meter -- rather than a bare timeout the processor drops. +const exportBackstopTimeoutMs = 15_000; + +/** + * Builds a self-observation {@link MeterProvider} that turns the batch + * processor's own span-processing counter into delivery-failure reports. The + * processor emits one counter, `otel.sdk.processor.span.processed`, tagged with + * an `error.type` attribute on every loss: `queue_full` when a span is dropped + * because the buffer is full, and the exporter error name when a batch export + * fails. Successful processing carries no `error.type`, so it is ignored. This + * is the single reporting site for every loss the SDK accounts for -- queue + * overflow and export failure alike -- rather than only the exporter callbacks + * an out-of-band wrapper can see. + */ +export function deliveryHealthMeterProvider( + report: (failure: Readonly) => void, +): MeterProvider { + const noop = createNoopMeter(); + const createReportingCounter = ( + name: string, + options?: MetricOptions, + ): Counter => { + const inner = noop.createCounter(name, options); + return { + add(value: number, attributes?: Attributes): void { + const errorType = attributes?.[ATTR_ERROR_TYPE]; + if (typeof errorType === "string") { + report({ + errorType, + probeName: traceExportProbeName, + schemaVersion: 0, + sinkId, + }); + } + inner.add(value, attributes); + }, + }; + }; + // Delegate every instrument to the no-op meter except the counter, whose + // `add` is intercepted above. The no-op meter is a shared singleton, so it is + // never mutated: a fresh delegating meter is returned instead. + const meter: Meter = { + createCounter: createReportingCounter, + createGauge: (name, options) => noop.createGauge(name, options), + createHistogram: (name, options) => noop.createHistogram(name, options), + createObservableCounter: (name, options) => + noop.createObservableCounter(name, options), + createObservableGauge: (name, options) => + noop.createObservableGauge(name, options), + createObservableUpDownCounter: (name, options) => + noop.createObservableUpDownCounter(name, options), + createUpDownCounter: (name, options) => + noop.createUpDownCounter(name, options), + addBatchObservableCallback: (callback, observables) => { + noop.addBatchObservableCallback(callback, observables); + }, + removeBatchObservableCallback: (callback, observables) => { + noop.removeBatchObservableCallback(callback, observables); + }, + }; + return { getMeter: () => meter }; +} + +/** + * Wraps a span exporter so a batch always receives a terminal result even when + * the inner exporter never calls back. A wedged exporter would otherwise let + * the batch processor's export timeout fire, which rejects the flush without + * accounting the loss through the self-observation meter. Converting the stall + * into a FAILED result routes it back through the processor's finish path (and + * so the meter) exactly once; genuine results pass straight through. Reporting + * itself lives in {@link deliveryHealthMeterProvider}, so this wrapper never + * reports -- it only guarantees the callback the meter depends on. + */ +export function backstopExporter( + inner: SpanExporter, + timeoutMs: number, +): SpanExporter { + return { + export(spans, resultCallback) { + let settled = false; + const settle = (result: ExportResult): void => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + resultCallback(result); + }; + const timer = setTimeout(() => { + const error = new Error( + "span export timed out before the exporter responded", + ); + error.name = "SpanExportTimeout"; + settle({ code: ExportResultCode.FAILED, error }); + }, timeoutMs); + // A synchronous throw from the inner exporter would otherwise bypass the + // callback entirely, so the loss would go unaccounted until the timer + // fired (or never, at shutdown). Normalize the throw into a FAILED result + // and settle it now, routing it through the processor's finish path (and + // so the self-observation meter) exactly once. + try { + inner.export(spans, settle); + } catch (thrown) { + settle({ + code: ExportResultCode.FAILED, + error: thrown instanceof Error ? thrown : new Error(String(thrown)), + }); + } + }, + forceFlush: () => inner.forceFlush?.() ?? Promise.resolve(), + shutdown: () => inner.shutdown(), + }; +} + +/** + * The batch processor config extended with the self-observation meter provider. + * The bundled `sdk-trace-base` shim omits `selfObsMeterProvider` from its + * constructor config type, but forwards it to the underlying processor, so this + * intersection re-adds the field for a typed hand-off. + */ +type SelfObservableBatchConfig = BufferConfig & { + selfObsMeterProvider?: MeterProvider; +}; + +/** + * Id generator that lets the caller choose the trace id of the next root span + * while keeping every span id random. A workflow span is the origin of the + * distributed trace, so it must be a true root; priming the trace id here lets + * that root still adopt the app-chosen id, joining the trace the browser + * propagates to the BFF and API without a synthetic remote parent (which would + * leave the trace decapitated by a parent span that no service ever exports). + */ +export class RootTraceIdGenerator implements IdGenerator { + private nextTraceId: string | undefined; + private readonly random = new RandomIdGenerator(); + + /** Sets the trace id the next generated root span adopts. */ + primeTraceId(traceId: string): void { + this.nextTraceId = traceId; + } + + generateTraceId(): string { + const chosen = this.nextTraceId; + this.nextTraceId = undefined; + return chosen ?? this.random.generateTraceId(); + } + + generateSpanId(): string { + return this.random.generateSpanId(); + } +} + +interface SpanEntry { + workflow: Span; + dependency?: Span; +} + +/** Terminal outcomes that mark a span failed rather than ok. */ +function isFailureOutcome(outcome: GatewayProbe["fields"]["outcome"]): boolean { + return outcome !== "started" && outcome !== "succeeded"; +} + +function applyTerminalOutcome(span: Span, probe: GatewayProbe): void { + const { failureKind, outcome } = probe.fields; + span.setAttribute("gateway.outcome", outcome); + if (failureKind !== null) { + span.setAttribute("gateway.failure_kind", failureKind); + } + // The operation identifier is present only on failing terminal probes and is + // the sole bridge from a failed workflow to its API-side operation record. + if (probe.context.operationId !== undefined) { + span.setAttribute("hypershell.operation_id", probe.context.operationId); + } + span.setStatus({ + code: isFailureOutcome(outcome) ? SpanStatusCode.ERROR : SpanStatusCode.OK, + }); +} + +/** + * Builds a domain probe sink that projects gateway workflow and dependency + * probes onto OpenTelemetry spans. A workflow span is a true root that adopts + * the trace id carried on each probe context (through the primed + * {@link RootTraceIdGenerator}), so a workflow span joins the same trace the + * browser propagates to the BFF and API while remaining the origin of that + * trace. Span names are drawn from a bounded action template so cardinality + * stays fixed. + */ +export function createGatewayTraceSink( + tracer: Tracer, + options: GatewayTraceSinkOptions = {}, +): GatewayTraceSink { + const beginTrace = options.beginTrace ?? ((): void => undefined); + const spansByCorrelation = new Map(); + + function startWorkflow(probe: GatewayProbe): void { + const { correlationId, traceId } = probe.context; + // A workflow with a chosen trace id is the trace root: prime the generator + // and start it with no parent. Without a chosen id, fall back to the active + // context so any caller-established parent still nests. + let parent = otelContext.active(); + if (traceId !== undefined) { + beginTrace(traceId); + parent = ROOT_CONTEXT; + } + const workflow = tracer.startSpan( + `gateway.workflow.${probe.fields.action}`, + { + attributes: { "gateway.action": probe.fields.action }, + kind: SpanKind.INTERNAL, + }, + parent, + ); + spansByCorrelation.set(correlationId, { workflow }); + } + + function startDependency(probe: GatewayProbe): void { + const entry = spansByCorrelation.get(probe.context.correlationId); + if (entry === undefined) { + return; + } + const parent = otelTrace.setSpan(otelContext.active(), entry.workflow); + entry.dependency = tracer.startSpan( + `gateway.dependency.${probe.fields.action}`, + { + attributes: { "gateway.action": probe.fields.action }, + kind: SpanKind.CLIENT, + }, + parent, + ); + } + + function completeDependency(probe: GatewayProbe): void { + const entry = spansByCorrelation.get(probe.context.correlationId); + if (entry?.dependency === undefined) { + return; + } + applyTerminalOutcome(entry.dependency, probe); + entry.dependency.end(); + entry.dependency = undefined; + } + + function completeWorkflow(probe: GatewayProbe): void { + const entry = spansByCorrelation.get(probe.context.correlationId); + if (entry === undefined) { + return; + } + // Defend against a dependency span left open by a dropped completion probe. + if (entry.dependency !== undefined) { + entry.dependency.end(); + } + applyTerminalOutcome(entry.workflow, probe); + entry.workflow.end(); + spansByCorrelation.delete(probe.context.correlationId); + } + + const sink: DomainProbeSink = { + id: sinkId, + publish(probe) { + switch (probe.name) { + case "gateway.workflow.started": + startWorkflow(probe); + return; + case "gateway.dependency.attempted": + startDependency(probe); + return; + case "gateway.dependency.completed": + completeDependency(probe); + return; + case "gateway.workflow.completed": + completeWorkflow(probe); + return; + } + }, + }; + + function traceParentFor( + correlationId: string, + ): GatewayTraceContext | undefined { + const entry = spansByCorrelation.get(correlationId); + const span = entry?.dependency ?? entry?.workflow; + if (span === undefined) { + return undefined; + } + const spanContext = span.spanContext(); + if (!isSpanContextValid(spanContext)) { + return undefined; + } + const flags = + (spanContext.traceFlags & TraceFlags.SAMPLED) === 0 ? "00" : "01"; + const traceparent = `00-${spanContext.traceId}-${spanContext.spanId}-${flags}`; + const tracestate = spanContext.traceState?.serialize(); + return tracestate === undefined || tracestate === "" + ? { traceparent } + : { traceparent, tracestate }; + } + + return { sink, traceParentFor }; +} + +/** + * Wires a browser tracer provider that batches spans and exports them over + * same-origin OTLP/HTTP, and returns the gateway trace sink bound to it. The + * provider is not registered as the global tracer; the sink owns every span + * explicitly, keyed by correlation identifier, so no implicit context is + * needed. Sampling is a per-trace decision made once at the workflow root by a + * `TraceIdRatioBasedSampler` and inherited by child spans, so the browser and + * the BFF (which uses the same OTel sampler) agree on each trace without + * sharing a decision. + * + * `BatchSpanProcessor` exports on a timer, which a browser can discard when a + * tab is closed or navigated away, losing the tail of a workflow. The provider + * therefore forces a flush on `visibilitychange` to hidden and on `pagehide`, + * the last reliable hooks before unload. `shutdown` removes those listeners. + */ +export function createGatewayTracing( + config: GatewayTracingConfig, + options: GatewayTracingOptions = {}, +): GatewayTracing { + const ratio = config.sampleRatio ?? 1; + const idGenerator = new RootTraceIdGenerator(); + const report = options.reportDeliveryFailure; + const baseExporter = new OTLPTraceExporter({ url: config.tracesEndpoint }); + // With delivery-health reporting on, wrap the exporter so a wedged collector + // still yields a terminal result, and wire the self-observation meter that + // turns every processor-accounted loss (overflow and export failure) into a + // report. Without it, the default export path is left untouched. + const exporter = + report === undefined + ? baseExporter + : backstopExporter(baseExporter, exportBackstopTimeoutMs); + const processorConfig: SelfObservableBatchConfig = + report === undefined + ? {} + : { selfObsMeterProvider: deliveryHealthMeterProvider(report) }; + const provider = new BasicTracerProvider({ + idGenerator, + resource: resourceFromAttributes({ + [ATTR_SERVICE_NAME]: config.serviceName, + }), + sampler: new ParentBasedSampler({ + root: new TraceIdRatioBasedSampler(ratio), + }), + spanProcessors: [new BatchSpanProcessor(exporter, processorConfig)], + }); + const tracer = provider.getTracer(tracerName); + const { sink, traceParentFor } = createGatewayTraceSink(tracer, { + beginTrace: (traceId) => { + idGenerator.primeTraceId(traceId); + }, + }); + + const flushBufferedSpans = (): void => { + // A failed export is already recorded through the self-observation meter, so + // the rejected forceFlush promise only needs to be settled to avoid an + // unhandled rejection; it is not a second failure to count. + void provider.forceFlush().catch(() => undefined); + }; + const flushWhenHidden = (): void => { + if (document.visibilityState === "hidden") { + flushBufferedSpans(); + } + }; + let stopFlushOnHide = (): void => undefined; + if (typeof document !== "undefined" && typeof window !== "undefined") { + document.addEventListener("visibilitychange", flushWhenHidden); + window.addEventListener("pagehide", flushBufferedSpans); + stopFlushOnHide = () => { + document.removeEventListener("visibilitychange", flushWhenHidden); + window.removeEventListener("pagehide", flushBufferedSpans); + }; + } + + return { + forceFlush: () => provider.forceFlush(), + shutdown: async () => { + stopFlushOnHide(); + await provider.shutdown(); + }, + sink, + traceParentFor, + }; +} diff --git a/components/web-console/app/composition/browser-runtime-config.test.ts b/components/web-console/app/composition/browser-runtime-config.test.ts new file mode 100644 index 00000000..39ac1f4d --- /dev/null +++ b/components/web-console/app/composition/browser-runtime-config.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; + +import { readBrowserRuntimeConfig } from "./browser-runtime-config"; + +function documentWithMeta(content: string | undefined): Document { + const doc = new DOMParser().parseFromString( + "", + "text/html", + ); + if (content !== undefined) { + const meta = doc.createElement("meta"); + meta.setAttribute("name", "hypershell-runtime-config"); + meta.setAttribute("content", content); + doc.head.append(meta); + } + return doc; +} + +describe("readBrowserRuntimeConfig", () => { + it("reads the sample ratio from the injected meta tag", () => { + const config = readBrowserRuntimeConfig( + documentWithMeta('{"tracing":{"sampleRatio":0.25}}'), + ); + + expect(config).toEqual({ tracing: { sampleRatio: 0.25 } }); + }); + + it("samples nothing when the meta tag is absent", () => { + expect(readBrowserRuntimeConfig(documentWithMeta(undefined))).toEqual({ + tracing: { sampleRatio: 0 }, + }); + }); + + it("fails closed to no tracing when the content is not valid JSON", () => { + expect(readBrowserRuntimeConfig(documentWithMeta("not-json"))).toEqual({ + tracing: { sampleRatio: 0 }, + }); + }); + + it("fails closed when the sample ratio is out of range or missing", () => { + for (const content of [ + '{"tracing":{"sampleRatio":5}}', + '{"tracing":{"sampleRatio":-1}}', + '{"tracing":{"sampleRatio":"1"}}', + '{"tracing":{}}', + "{}", + ]) { + expect(readBrowserRuntimeConfig(documentWithMeta(content))).toEqual({ + tracing: { sampleRatio: 0 }, + }); + } + }); +}); diff --git a/components/web-console/app/composition/browser-runtime-config.ts b/components/web-console/app/composition/browser-runtime-config.ts new file mode 100644 index 00000000..f49c6756 --- /dev/null +++ b/components/web-console/app/composition/browser-runtime-config.ts @@ -0,0 +1,58 @@ +// Reads the allowlisted runtime configuration the BFF injects into the served +// HTML as a tag. Keeping this out of an inline script means the SPA needs +// no script-src hash to learn operator settings such as the trace sample ratio. + +const runtimeConfigMetaName = "hypershell-runtime-config"; + +export interface BrowserRuntimeConfig { + tracing: { + /** Fraction of browser-rooted traces to record, 0..1. */ + sampleRatio: number; + }; +} + +// When no config is available the browser records nothing: it must never emit +// traces the BFF cannot relay (a dev server or a deployment with tracing off). +const disabledRuntimeConfig: BrowserRuntimeConfig = { + tracing: { sampleRatio: 0 }, +}; + +function isSampleRatio(value: unknown): value is number { + return ( + typeof value === "number" && + Number.isFinite(value) && + value >= 0 && + value <= 1 + ); +} + +/** + * Parses the runtime config from the injected tag. An absent tag, + * unparsable content, or an out-of-range sample ratio all fall back to the + * disabled config, so a missing or tampered surface fails closed to no tracing + * rather than defaulting to recording every trace. On the server there is no + * document, so the disabled config is returned and no browser tracing starts. + */ +export function readBrowserRuntimeConfig( + doc: Document | undefined = typeof document === "undefined" + ? undefined + : document, +): BrowserRuntimeConfig { + const content = doc + ?.querySelector(`meta[name="${runtimeConfigMetaName}"]`) + ?.getAttribute("content"); + if (content === null || content === undefined) { + return disabledRuntimeConfig; + } + try { + const parsed = JSON.parse(content) as { + tracing?: { sampleRatio?: unknown }; + }; + const sampleRatio = parsed.tracing?.sampleRatio; + return isSampleRatio(sampleRatio) + ? { tracing: { sampleRatio } } + : disabledRuntimeConfig; + } catch { + return disabledRuntimeConfig; + } +} diff --git a/components/web-console/app/composition/gateway-composition.ts b/components/web-console/app/composition/gateway-composition.ts index 799841b2..e41ad5d4 100644 --- a/components/web-console/app/composition/gateway-composition.ts +++ b/components/web-console/app/composition/gateway-composition.ts @@ -1,11 +1,57 @@ import { createGatewayOperations } from "@openshift-online/hypershell-gateway-management-ui"; +import type { ProbeDeliveryFailure } from "@openshift-online/hypershell-domain-probes/fan-out"; import { createApiClient } from "../adapters/api/api.client"; import { createGatewayControlPlaneAdapter } from "../adapters/api/gateway-operations"; -import { gatewayObservability } from "../adapters/observability/gateway-observability"; +import { createGatewayObservability } from "../adapters/observability/gateway-observability"; +import { createGatewayTracing } from "../adapters/observability/gateway-trace-sink"; +import { readBrowserRuntimeConfig } from "./browser-runtime-config"; + +// Same-origin OTLP/HTTP traces path the BFF exposes and forwards to the +// collector. Keeping it same-origin means the browser never sees a collector +// address and no cross-origin telemetry endpoint is exposed. +const browserTracesEndpoint = "/telemetry/v1/traces"; + +// The operator's sample ratio reaches the browser through the BFF-injected +// runtime config, so the browser trace root honors the configured rate rather +// than always recording every trace, and agrees with the BFF sampler. +const browserRuntimeConfig = readBrowserRuntimeConfig(); + +// The trace sink is created before the observability publisher because the +// publisher takes the sink as one of its fan-out targets, yet a failed span +// export must report back into that publisher's delivery health. A late-bound +// reporter breaks the cycle: export failures raised before the publisher exists +// are dropped, which is correct because no span can be exported until the sink +// is wired into the publisher and receiving probes. +let reportDeliveryFailure: ( + failure: Readonly, +) => void = () => undefined; + +const tracing = createGatewayTracing( + { + sampleRatio: browserRuntimeConfig.tracing.sampleRatio, + serviceName: "hypershell-web-console", + tracesEndpoint: browserTracesEndpoint, + }, + { + reportDeliveryFailure: (failure) => { + reportDeliveryFailure(failure); + }, + }, +); + +const gatewayObservability = createGatewayObservability({ + additionalSinks: [tracing.sink], +}); + +reportDeliveryFailure = (failure) => { + gatewayObservability.reportDeliveryFailure(failure); +}; const gatewayControlPlane = createGatewayControlPlaneAdapter((correlationId) => - createApiClient(correlationId), + createApiClient(correlationId, undefined, () => + tracing.traceParentFor(correlationId), + ), ); export const gatewayOperations = createGatewayOperations({ diff --git a/components/web-console/bff/package.json b/components/web-console/bff/package.json index 07e7fac5..93da6534 100644 --- a/components/web-console/bff/package.json +++ b/components/web-console/bff/package.json @@ -22,6 +22,12 @@ "@fastify/helmet": "13.1.0", "@fastify/secure-session": "8.3.0", "@fastify/static": "10.1.2", + "@opentelemetry/api": "1.9.1", + "@opentelemetry/core": "2.10.0", + "@opentelemetry/exporter-trace-otlp-http": "0.221.0", + "@opentelemetry/resources": "2.10.0", + "@opentelemetry/sdk-trace-base": "2.10.0", + "@opentelemetry/semantic-conventions": "1.43.0", "fastify": "5.10.0", "openid-client": "6.8.4", "zod": "4.4.3" diff --git a/components/web-console/bff/src/adapters/observability/otel-tracing.ts b/components/web-console/bff/src/adapters/observability/otel-tracing.ts new file mode 100644 index 00000000..5f5ae2be --- /dev/null +++ b/components/web-console/bff/src/adapters/observability/otel-tracing.ts @@ -0,0 +1,668 @@ +import { + ROOT_CONTEXT, + SpanKind, + SpanStatusCode, + createNoopMeter, + defaultTextMapGetter, + defaultTextMapSetter, + isSpanContextValid, + trace as otelTrace, + type Attributes, + type Context, + type Counter, + type Meter, + type MeterProvider, + type MetricOptions, + type Span, + type Tracer, +} from "@opentelemetry/api"; +import { + ExportResultCode, + W3CTraceContextPropagator, + type ExportResult, +} from "@opentelemetry/core"; +import { + BasicTracerProvider, + BatchSpanProcessor, + ParentBasedSampler, + TraceIdRatioBasedSampler, + type BufferConfig, + type SpanExporter, +} from "@opentelemetry/sdk-trace-base"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; +import { resourceFromAttributes } from "@opentelemetry/resources"; +import { + ATTR_ERROR_TYPE, + ATTR_SERVICE_NAME, +} from "@opentelemetry/semantic-conventions"; +import { z } from "zod"; + +import type { TracingConfig } from "../../config.js"; +import { + disabledTracing, + type BffDeliveryHealthSnapshot, + type BffTracing, + type ProxyOutcome, + type ProxySpan, + type StartProxySpanInput, + type TelemetryIngestResult, + type UpstreamTraceContext, +} from "../../tracing.js"; + +const ingestTimeoutMs = 5_000; + +// The conformant W3C propagator handles extraction and injection: it accepts +// current and future `traceparent` versions, rejects an all-zero or malformed +// one, and drops a malformed `tracestate` while still continuing a trace whose +// `traceparent` is valid. It is stateless, so a single instance is shared. +const propagator = new W3CTraceContextPropagator(); + +const versionSegment = /^v\d+$/u; + +/** + * A path segment is a resource id (not a collection or action name) when it + * carries a digit or is long. That is true of every ULID, UUID, or numeric id + * the API mints and of none of the fixed collection or action names, so + * collapsing it keeps a raw identifier out of the route template. + */ +function isIdSegment(segment: string): boolean { + return /\d/u.test(segment) || segment.length >= 20; +} + +/** + * Collapses resource ids in a request path to a bounded route template, so the + * span name and `http.route` stay low-cardinality (WEB-TRACE-07). The API is a + * flat REST surface under `/api//v/[/{id}...]`; after the + * version segment, id segments collapse to `{id}` while collection and action + * segments stay literal. A path with no version segment collapses every + * id-shaped segment, so an unexpected shape can never blow up cardinality. + */ +export function routeTemplateFrom(path: string): string { + const segments = path.split("/").filter((segment) => segment.length > 0); + const versionIndex = segments.findIndex((segment) => + versionSegment.test(segment), + ); + const template = segments.map((segment, index) => + index > versionIndex && isIdSegment(segment) ? "{id}" : segment, + ); + return `/${template.join("/")}`; +} + +/** + * Extracts the inbound W3C trace context. An absent, malformed, all-zero, or + * unsupported-version `traceparent` yields the root context, so a fresh trace is + * started; a valid one continues the trace and carries a valid `tracestate` + * forward. A `tracestate` without a valid `traceparent` is discarded because the + * root context it lands in has no trace to continue. + */ +function parentContextFrom(input: StartProxySpanInput): Context { + if (input.traceparent === undefined) { + return ROOT_CONTEXT; + } + const carrier: Record = { traceparent: input.traceparent }; + if (input.tracestate !== undefined) { + carrier.tracestate = input.tracestate; + } + return propagator.extract(ROOT_CONTEXT, carrier, defaultTextMapGetter); +} + +/** + * Serializes the BFF span's own context into upstream `traceparent`/`tracestate` + * headers via the propagator, so a continued trace forwards the inherited + * `tracestate` and the sampled flag reflects the span's decision. Returns + * `undefined` when the span has no valid context. The propagator emits an empty + * string for an absent trace state, which is not a header worth forwarding, so + * it is normalized to no `tracestate`. + */ +function upstreamContextFor(span: Span): UpstreamTraceContext | undefined { + if (!isSpanContextValid(span.spanContext())) { + return undefined; + } + const carrier: Record = {}; + propagator.inject( + otelTrace.setSpan(ROOT_CONTEXT, span), + carrier, + defaultTextMapSetter, + ); + const { traceparent, tracestate } = carrier; + if (traceparent === undefined) { + return undefined; + } + return tracestate === undefined || tracestate === "" + ? { traceparent } + : { traceparent, tracestate }; +} + +function spanStatusFor(outcome: ProxyOutcome): SpanStatusCode { + return outcome === "server_error" || outcome === "timeout" + ? SpanStatusCode.ERROR + : SpanStatusCode.OK; +} + +// Bounds on the OTLP/HTTP JSON envelope. The Fastify body limit already caps the +// raw byte size; these additionally bound the structural walk and reject a +// payload whose arrays are implausibly large before it is relayed. +const maxResourceSpans = 10_000; +const maxScopeSpans = 10_000; +const maxSpans = 100_000; +const maxAttributes = 1_024; +const maxEvents = 1_024; +const maxLinks = 1_024; + +// Bounds on the nesting shape itself. A recursive AnyValue nested thousands of +// levels deep would drive the recursive schema past the JavaScript call-stack +// limit and throw a RangeError instead of returning a validation failure. An +// iterative pre-pass caps nesting depth (so the schema recursion stays shallow) +// and total node count before any recursive validation runs. +const maxStructuralDepth = 64; +const maxStructuralNodes = 1_000_000; + +/** + * Verifies the payload's object graph stays within a bounded nesting depth and + * node count, walking it with an explicit stack so the check itself never + * recurses. Rejecting an over-deep payload here keeps the recursive AnyValue + * schema from being driven past the call-stack limit, where it would throw + * rather than return a rejection. + */ +function withinStructuralBudget(payload: unknown): boolean { + const stack: { depth: number; node: unknown }[] = [ + { depth: 0, node: payload }, + ]; + let nodes = 0; + while (stack.length > 0) { + const entry = stack.pop(); + if (entry === undefined) { + break; + } + nodes += 1; + if (nodes > maxStructuralNodes || entry.depth > maxStructuralDepth) { + return false; + } + const { depth, node } = entry; + if (Array.isArray(node)) { + for (const child of node) { + stack.push({ depth: depth + 1, node: child }); + } + } else if (node !== null && typeof node === "object") { + for (const child of Object.values(node)) { + stack.push({ depth: depth + 1, node: child }); + } + } + } + return true; +} + +// Trace and span ids are hex-encoded in the OTLP/HTTP JSON the OpenTelemetry JS +// exporter emits (16- and 8-byte identifiers), not the base64 the generic proto3 +// JSON mapping would use for bytes. +const traceIdHex = z.string().regex(/^[0-9a-f]{32}$/iu); +const spanIdHex = z.string().regex(/^[0-9a-f]{16}$/iu); +// proto3 JSON scalar encodings, enforced so a wrong-typed value is rejected +// rather than relayed. A uint64/fixed64 nanosecond timestamp is a decimal string +// (values exceed the safe integer range) or a JSON number when small; a uint32 +// (dropped counts, fixed32 flags) is a bounded non-negative integer; a signed +// int64 (AnyValue intValue) is a decimal string or an integer number; a double +// is a JSON number or one of the proto3 special-value strings; bytes are base64. +// The 64-bit domains are also range-checked: decimal syntax alone would relay a +// string one past the fixed-width maximum, which the collector rejects. +const UINT64_MAX = 18_446_744_073_709_551_615n; +const INT64_MIN = -9_223_372_036_854_775_808n; +const INT64_MAX = 9_223_372_036_854_775_807n; +// The widest in-range 64-bit value has 20 significant decimal digits +// (uint64 max = 18446744073709551615); anything longer is unconditionally out of +// range. Reject on significant-digit length before constructing a BigInt: +// `BigInt(text)` parses the entire decimal (an O(n^2) cost), so a body-sized +// caller-controlled string would block the event loop for the whole parse even +// though the value is rejected. The BigInt is only ever built from the bounded +// significant digits, so a run of leading zeros never reaches it either. +const maxDecimalDigits = 20; +const withinBigIntRange = + (min: bigint, max: bigint) => + (text: string): boolean => { + const negative = text.startsWith("-"); + // The `\d`-only regex guarantees the remainder is decimal digits; strip the + // sign and leading zeros so the bound is on significant digits. + const digits = text.slice(negative ? 1 : 0).replace(/^0+/u, ""); + if (digits.length > maxDecimalDigits) { + return false; + } + try { + const value = + digits === "" ? 0n : BigInt(negative ? `-${digits}` : digits); + return value >= min && value <= max; + } catch { + return false; + } + }; +const uint32 = z.number().int().min(0).max(4_294_967_295); +// A uint64 nanosecond timestamp: a decimal string bounded by BigInt to the +// fixed-width maximum, or a JSON number confined to the safe-integer range so a +// larger value that has already lost precision is rejected rather than relayed. +const unixNano = z.union([ + z.string().regex(/^\d+$/u).refine(withinBigIntRange(0n, UINT64_MAX), { + message: "unixNano string is outside the uint64 range", + }), + z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), +]); +// A signed int64: a decimal string bounded by BigInt to the int64 range, or a +// JSON number confined to the safe-integer range for the same reason. +const int64Json = z.union([ + z + .string() + .regex(/^-?\d+$/u) + .refine(withinBigIntRange(INT64_MIN, INT64_MAX), { + message: "intValue string is outside the int64 range", + }), + z.number().int().min(Number.MIN_SAFE_INTEGER).max(Number.MAX_SAFE_INTEGER), +]); +const doubleJson = z.union([ + z.number(), + z.enum(["NaN", "Infinity", "-Infinity"]), +]); +const base64 = z + .string() + .regex(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u); + +// AnyValue is a proto3 oneof: at most one value field may be set, and each is +// carried in its proto3 JSON encoding. It is recursive -- an array or kvlist +// value nests further values -- so the value and key-value schemas reference +// each other through z.lazy. +const anyValueSchema: z.ZodType = z.lazy(() => + z + .object({ + stringValue: z.string().optional(), + boolValue: z.boolean().optional(), + intValue: int64Json.optional(), + doubleValue: doubleJson.optional(), + bytesValue: base64.optional(), + arrayValue: z + .object({ values: z.array(anyValueSchema).max(maxAttributes) }) + .optional(), + kvlistValue: z + .object({ values: z.array(keyValueSchema).max(maxAttributes) }) + .optional(), + }) + .refine( + (value) => + Object.values(value as Record).filter( + (field) => field !== undefined, + ).length <= 1, + { message: "AnyValue must set at most one value field" }, + ), +); +const keyValueSchema: z.ZodType = z.lazy(() => + z.object({ key: z.string(), value: anyValueSchema.optional() }), +); +const attributesSchema = z.array(keyValueSchema).max(maxAttributes).optional(); + +const spanSchema = z.object({ + traceId: traceIdHex, + spanId: spanIdHex, + traceState: z.string().optional(), + parentSpanId: spanIdHex.optional(), + flags: uint32.optional(), + name: z.string(), + // SpanKind is a proto enum, 0 (unspecified) through 5 (consumer). + kind: z.number().int().min(0).max(5).optional(), + startTimeUnixNano: unixNano.optional(), + endTimeUnixNano: unixNano.optional(), + attributes: attributesSchema, + droppedAttributesCount: uint32.optional(), + events: z + .array( + z.object({ + timeUnixNano: unixNano.optional(), + name: z.string().optional(), + attributes: attributesSchema, + droppedAttributesCount: uint32.optional(), + }), + ) + .max(maxEvents) + .optional(), + // uint32 counters the OpenTelemetry JS exporter emits alongside truncated + // event and link collections. Validated so a wrong-typed or negative count is + // rejected rather than silently ignored and relayed to the collector. + droppedEventsCount: uint32.optional(), + links: z + .array( + z.object({ + traceId: traceIdHex.optional(), + spanId: spanIdHex.optional(), + traceState: z.string().optional(), + attributes: attributesSchema, + droppedAttributesCount: uint32.optional(), + flags: uint32.optional(), + }), + ) + .max(maxLinks) + .optional(), + droppedLinksCount: uint32.optional(), + status: z + .object({ + message: z.string().optional(), + // Status code is a proto enum, 0 (unset) through 2 (error). + code: z.number().int().min(0).max(2).optional(), + }) + .optional(), +}); + +const otlpTracePayloadSchema = z.object({ + resourceSpans: z + .array( + z.object({ + resource: z + .object({ + attributes: attributesSchema, + droppedAttributesCount: uint32.optional(), + }) + .optional(), + scopeSpans: z + .array( + z.object({ + scope: z + .object({ + name: z.string().optional(), + version: z.string().optional(), + attributes: attributesSchema, + droppedAttributesCount: uint32.optional(), + }) + .optional(), + spans: z.array(spanSchema).max(maxSpans).optional(), + schemaUrl: z.string().optional(), + }), + ) + .max(maxScopeSpans) + .optional(), + schemaUrl: z.string().optional(), + }), + ) + .max(maxResourceSpans), +}); + +/** + * Validates the OTLP/HTTP trace envelope against the supported subset of the + * OTLP JSON schema -- not merely the three container arrays, but every nested + * message and scalar in its proto3 JSON encoding: span ids are hex, timestamps + * are nanosecond encodings, counts and flags are bounded uint32s, an AnyValue is + * a oneof of typed values, and status, events, and links match their proto + * shapes, all under fixed bounds. A bounded structural pre-pass caps nesting + * depth so the recursive schema cannot overflow the stack, and any validator + * exception is converted to a rejection. Unknown forward-compatible keys are + * ignored, but a wrong-typed or malformed known field is rejected before the + * payload reaches the collector, rather than accepted and relayed only for the + * collector to reject it after the browser was told the export succeeded + * (WEB-TRACE-02). + */ +function isOtlpTracePayload(payload: unknown): boolean { + try { + return ( + withinStructuralBudget(payload) && + otlpTracePayloadSchema.safeParse(payload).success + ); + } catch { + // A validator exception (for example a call-stack overflow on a shape the + // budget somehow admitted) is a rejection, never a relay. + return false; + } +} + +const exportBackstopTimeoutMs = 15_000; + +/** Bounded, mutable delivery-health tally folded into the port snapshot. */ +interface MutableDeliveryHealth { + relayFailures: number; + spanExportFailures: number; + lastErrorType?: string; +} + +/** + * Builds a self-observation {@link MeterProvider} that folds the batch + * processor's own span-processing counter into the delivery-health tally. The + * processor emits one counter, `otel.sdk.processor.span.processed`, tagged with + * an `error.type` attribute on every loss: `queue_full` when a span is dropped + * because the buffer is full, and the exporter error name when a batch export + * fails. Successful processing carries no `error.type` and is ignored. This is + * the single accounting site for every span loss the SDK observes, rather than + * only the export failures an out-of-band wrapper could see. + */ +function deliveryHealthMeterProvider( + health: MutableDeliveryHealth, +): MeterProvider { + const noop = createNoopMeter(); + const createReportingCounter = ( + name: string, + options?: MetricOptions, + ): Counter => { + const inner = noop.createCounter(name, options); + return { + add(value: number, attributes?: Attributes): void { + const errorType = attributes?.[ATTR_ERROR_TYPE]; + if (typeof errorType === "string") { + // The processor reports the measurement as the number of spans lost in + // this batch, so the tally advances by that count rather than by one: + // a failed multi-span batch must not be undercounted as a single span. + health.spanExportFailures += value; + health.lastErrorType = errorType; + } + inner.add(value, attributes); + }, + }; + }; + // Delegate every instrument to the no-op meter except the counter, whose `add` + // is intercepted above. The no-op meter is a shared singleton, so a fresh + // delegating meter is returned rather than mutating it. + const meter: Meter = { + createCounter: createReportingCounter, + createGauge: (name, options) => noop.createGauge(name, options), + createHistogram: (name, options) => noop.createHistogram(name, options), + createObservableCounter: (name, options) => + noop.createObservableCounter(name, options), + createObservableGauge: (name, options) => + noop.createObservableGauge(name, options), + createObservableUpDownCounter: (name, options) => + noop.createObservableUpDownCounter(name, options), + createUpDownCounter: (name, options) => + noop.createUpDownCounter(name, options), + addBatchObservableCallback: (callback, observables) => { + noop.addBatchObservableCallback(callback, observables); + }, + removeBatchObservableCallback: (callback, observables) => { + noop.removeBatchObservableCallback(callback, observables); + }, + }; + return { getMeter: () => meter }; +} + +/** + * Wraps a span exporter so a batch always receives a terminal result even when + * the inner exporter never calls back. A wedged exporter would otherwise let the + * batch processor's export timeout fire, which rejects the flush without + * accounting the loss through the self-observation meter. Converting the stall + * into a FAILED result routes it back through the processor's finish path (and + * so the meter) exactly once; genuine results pass straight through. Accounting + * lives in {@link deliveryHealthMeterProvider}, so this wrapper never records -- + * it only guarantees the callback the meter depends on. + */ +function backstopExporter( + inner: SpanExporter, + timeoutMs: number, +): SpanExporter { + return { + export(spans, resultCallback) { + let settled = false; + const settle = (result: ExportResult): void => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + resultCallback(result); + }; + const timer = setTimeout(() => { + const error = new Error( + "span export timed out before the exporter responded", + ); + error.name = "SpanExportTimeout"; + settle({ code: ExportResultCode.FAILED, error }); + }, timeoutMs); + // A synchronous throw from the inner exporter would otherwise bypass the + // callback entirely, so the loss would go unaccounted until the timer + // fired (or never, at shutdown). Normalize the throw into a FAILED result + // and settle it now, routing it through the processor's finish path (and + // so the self-observation meter) exactly once. + try { + inner.export(spans, settle); + } catch (thrown) { + settle({ + code: ExportResultCode.FAILED, + error: thrown instanceof Error ? thrown : new Error(String(thrown)), + }); + } + }, + forceFlush: () => inner.forceFlush?.() ?? Promise.resolve(), + shutdown: () => inner.shutdown(), + }; +} + +/** + * The batch processor config extended with the self-observation meter provider. + * The bundled `sdk-trace-base` shim omits `selfObsMeterProvider` from its + * constructor config type but forwards it to the underlying processor, so this + * intersection re-adds the field for a typed hand-off. + */ +type SelfObservableBatchConfig = BufferConfig & { + selfObsMeterProvider?: MeterProvider; +}; + +/** + * Builds the BFF tracing adapter. Spans are managed explicitly per request and + * exported over OTLP/HTTP; no global tracer or context manager is registered. + * Returns the disabled port when no collector is configured, so a deployment + * without tracing starts normally. + */ +export function createBffTracing( + config: TracingConfig | undefined, +): BffTracing { + if (config === undefined) { + return disabledTracing; + } + const tracing = config; + + const health: MutableDeliveryHealth = { + relayFailures: 0, + spanExportFailures: 0, + }; + // Wrap the exporter so a wedged collector still yields a terminal result, and + // wire the self-observation meter that folds every processor-accounted loss + // (queue overflow and export failure) into the delivery-health tally. + const exporter = backstopExporter( + new OTLPTraceExporter({ url: tracing.tracesEndpoint }), + exportBackstopTimeoutMs, + ); + const processorConfig: SelfObservableBatchConfig = { + selfObsMeterProvider: deliveryHealthMeterProvider(health), + }; + const provider = new BasicTracerProvider({ + resource: resourceFromAttributes({ + [ATTR_SERVICE_NAME]: tracing.serviceName, + }), + sampler: new ParentBasedSampler({ + root: new TraceIdRatioBasedSampler(tracing.sampleRatio), + }), + spanProcessors: [new BatchSpanProcessor(exporter, processorConfig)], + }); + const tracer: Tracer = provider.getTracer("hypershell-web-console-bff"); + + function startProxySpan(input: StartProxySpanInput): ProxySpan { + const parentContext = parentContextFrom(input); + // Name the span by method and the bounded route template (for example + // "GET /api/hypershell/v1/gateways/{id}") so Jaeger groups by endpoint + // rather than collapsing every proxied call onto one wildcard operation. + const routeTemplate = routeTemplateFrom(input.path); + const span = tracer.startSpan( + `${input.method} ${routeTemplate}`, + { + attributes: { + "http.request.method": input.method, + "http.route": routeTemplate, + "hypershell.correlation_id": input.correlationId, + }, + kind: SpanKind.SERVER, + }, + parentContext, + ); + + return { + end(outcome, statusCode) { + span.setAttribute("http.response.status_code", statusCode); + span.setAttribute("hypershell.outcome", outcome); + span.setStatus({ code: spanStatusFor(outcome) }); + span.end(); + }, + upstream: () => upstreamContextFor(span), + }; + } + + async function ingestTraces( + payload: unknown, + ): Promise { + if (!isOtlpTracePayload(payload)) { + return "rejected"; + } + try { + const response = await fetch(tracing.tracesEndpoint, { + body: JSON.stringify(payload), + headers: { "content-type": "application/json" }, + method: "POST", + signal: AbortSignal.timeout(ingestTimeoutMs), + }); + if (response.ok) { + return "accepted"; + } + // A collector 4xx means the collector rejected the payload as malformed on + // its stricter parse; surface it as a rejection so the browser learns its + // telemetry was bad rather than seeing a 202. Transient 408/429 and every + // 5xx are best-effort unavailability, never surfaced as a client error. + if ( + response.status >= 400 && + response.status < 500 && + response.status !== 408 && + response.status !== 429 + ) { + return "rejected"; + } + // A transient 408/429 or any 5xx means the collector could not accept the + // relay: best-effort for the browser, but a delivery failure worth + // surfacing in the bounded health diagnostic. + health.relayFailures += 1; + health.lastErrorType = "collector_unavailable"; + return "unavailable"; + } catch { + // Best-effort: an unreachable collector never fails the browser request, + // but the loss is still counted so the health diagnostic reflects it. + health.relayFailures += 1; + health.lastErrorType = "collector_unreachable"; + return "unavailable"; + } + } + + const deliveryHealth = (): BffDeliveryHealthSnapshot => + health.lastErrorType === undefined + ? { + relayFailures: health.relayFailures, + spanExportFailures: health.spanExportFailures, + } + : { + lastErrorType: health.lastErrorType, + relayFailures: health.relayFailures, + spanExportFailures: health.spanExportFailures, + }; + + return { + deliveryHealth, + enabled: true, + ingestTraces, + shutdown: () => provider.shutdown(), + startProxySpan, + }; +} diff --git a/components/web-console/bff/src/app.ts b/components/web-console/bff/src/app.ts index d91c28dc..f19485cd 100644 --- a/components/web-console/bff/src/app.ts +++ b/components/web-console/bff/src/app.ts @@ -8,8 +8,17 @@ import fastifyStatic from "@fastify/static"; import Fastify, { type FastifyInstance, LogController } from "fastify"; import { clearSession, persistTokenSet, registerAuth } from "./auth.js"; -import type { ServerConfig } from "./config.js"; +import { + browserRuntimeConfig, + type BrowserRuntimeConfig, + type ServerConfig, +} from "./config.js"; import { tokenExpired } from "./tokens.js"; +import { + disabledTracing, + type BffTracing, + type ProxyOutcome, +} from "./tracing.js"; const correlationHeader = "x-hypershell-correlation-id"; const validCorrelationId = @@ -55,6 +64,40 @@ function proxyBody( return JSON.stringify(body); } +const runtimeConfigMetaName = "hypershell-runtime-config"; + +function escapeHtmlAttribute(value: string): string { + return value + .replace(/&/gu, "&") + .replace(//gu, ">") + .replace(/"/gu, """); +} + +/** + * Injects the allowlisted browser runtime config as a tag in the head, so + * the SPA reads operator settings (such as the trace sample ratio) without an + * inline script that would need a CSP hash. Only the browserRuntimeConfig + * projection is serialized; server-only config never reaches the client. The tag + * lands after the opening head, or after the opening html when a document has no + * head, so it is always present in the document the browser parses. + */ +function injectRuntimeConfig( + document: string, + config: BrowserRuntimeConfig, +): string { + const meta = ``; + if (/]*>/iu.test(document)) { + return document.replace(/]*>/iu, (head) => `${head}${meta}`); + } + if (/]*>/iu.test(document)) { + return document.replace(/]*>/iu, (html) => `${html}${meta}`); + } + return `${meta}${document}`; +} + function inlineScriptHashes(document: string): string[] { const hashes = new Set(); const scriptPattern = /]*\bsrc=)[^>]*>([\s\S]*?)<\/script>/giu; @@ -71,9 +114,31 @@ function inlineScriptHashes(document: string): string[] { return [...hashes]; } -export async function buildApp(config: ServerConfig): Promise { +function singleHeaderValue( + value: string | string[] | undefined, +): string | undefined { + return typeof value === "string" ? value : undefined; +} + +function outcomeForStatus(statusCode: number): ProxyOutcome { + if (statusCode >= 500) { + return "server_error"; + } + if (statusCode >= 400) { + return "client_error"; + } + return "success"; +} + +export async function buildApp( + config: ServerConfig, + tracing: BffTracing = disabledTracing, +): Promise { const indexPath = path.join(config.staticRoot, "index.html"); - const indexDocument = await readFile(indexPath, "utf8"); + const indexDocument = injectRuntimeConfig( + await readFile(indexPath, "utf8"), + browserRuntimeConfig(config), + ); const scriptHashes = inlineScriptHashes(indexDocument); const app = Fastify({ @@ -221,7 +286,11 @@ export async function buildApp(config: ServerConfig): Promise { app.get("/health/ready", async (_request, reply) => { reply.header("Cache-Control", "no-store"); - return { status: "ready" }; + // Readiness stays "ready" regardless of telemetry delivery -- tracing is + // best-effort and must never gate serving (WEB-TRACE-06) -- but the bounded + // delivery-health snapshot rides along so span-export and relay losses are + // observable rather than silently swallowed. + return { status: "ready", tracing: tracing.deliveryHealth() }; }); const sendApplication = ( @@ -252,128 +321,190 @@ export async function buildApp(config: ServerConfig): Promise { }; }; - app.all("/api/*", async (request, reply) => { - const incoming = new URL(request.url, "http://bff.invalid"); - const target = new URL( - `${incoming.pathname}${incoming.search}`, - config.apiOrigin, - ); - const headers = new Headers(); - for (const header of forwardedRequestHeaders) { - const value = request.headers[header]; - if (typeof value === "string") { - headers.set(header, value); - } + // Same-origin browser telemetry ingest. The browser exporter posts OTLP/HTTP + // JSON here; the BFF validates it and relays it to the configured collector, + // keeping the collector origin out of the browser and reusing the session and + // CSRF controls the other state-changing routes rely on (WEB-TRACE-02). The + // global 1 MiB body limit bounds the payload. + app.post("/telemetry/v1/traces", async (request, reply) => { + reply.header("Cache-Control", "no-store"); + if (config.oidcIssuer && !request.session.get("accessToken")) { + reply.code(401); + return { error: "Unauthorized", statusCode: 401 }; } - headers.set(correlationHeader, request.correlationId); - - const refreshToken = config.oidcIssuer - ? request.tokenSession.get("refreshToken") - : undefined; - let refreshed = false; - - // Ensure a valid access token before forwarding (proactive refresh). - if (config.oidcIssuer) { - let accessToken = request.session.get("accessToken"); - const expiresAt = request.session.get("expiresAt"); - if (!accessToken || tokenExpired(expiresAt)) { - if (!refreshToken || !app.refreshAccessToken) { - clearSession(request); - return respondReauth(reply); - } - try { - const tokens = await app.refreshAccessToken(refreshToken); - persistTokenSet(request, tokens); - accessToken = tokens.accessToken; - refreshed = true; - } catch { - clearSession(request); - return respondReauth(reply); - } - } - headers.set("authorization", `Bearer ${accessToken}`); + const result = await tracing.ingestTraces(request.body); + if (result === "rejected") { + reply.code(400); + return { error: "Bad Request", statusCode: 400 }; } + // "accepted" and "unavailable" are both success from the browser's view: + // telemetry is best-effort and must never surface a collector outage. + reply.code(202); + return { status: "accepted" }; + }); - // Perform the upstream request with its own timeout and downstream-abort - // wiring, so it can be retried once after a reactive refresh. - const runUpstream = async (): Promise => { - const controller = new AbortController(); - const timeoutReason = new Error("Upstream API request timed out"); - const timeout = setTimeout(() => { - controller.abort(timeoutReason); - }, config.apiTimeoutMs); - const abortDownstream = () => { - controller.abort(); - }; - request.raw.once("aborted", abortDownstream); - try { - return await fetch(target, { - body: proxyBody(request.method, request.body), - headers, - method: request.method, - redirect: "manual", - signal: controller.signal, - }); - } catch (error) { - if (error === timeoutReason) { - const timeout504 = new Error("Upstream API request timed out"); - timeout504.name = "UpstreamTimeout"; - throw timeout504; - } - throw error; - } finally { - clearTimeout(timeout); - request.raw.off("aborted", abortDownstream); - } + app.all("/api/*", async (request, reply) => { + // Start one BFF server span per proxied request. It continues a valid + // inbound W3C context and yields the validated upstream context to set on + // the API request, so browser, BFF, and API join one trace (WEB-TRACE-04, + // WEB-TRACE-05). A malformed inbound value is never forwarded: only the + // span-derived context reaches upstream. + const span = tracing.startProxySpan({ + correlationId: request.correlationId, + method: request.method, + // Pass the path without its query string; the tracing adapter renders a + // bounded route template from it, so no query value or raw id is recorded. + path: request.url.split("?")[0] ?? request.url, + traceparent: singleHeaderValue(request.headers.traceparent), + tracestate: singleHeaderValue(request.headers.tracestate), + }); + // The span ends in the finally block with the outcome recorded here, so a + // tracing failure never changes the proxy result (WEB-TRACE-09). + let spanOutcome: ProxyOutcome = "server_error"; + let spanStatusCode = 500; + const recordOutcome = (statusCode: number) => { + spanOutcome = outcomeForStatus(statusCode); + spanStatusCode = statusCode; }; try { - let upstream = await runUpstream(); - - // Reactive refresh: a token we believed valid was rejected upstream - // (clock skew, revocation, or key rotation). Refresh once and retry. - if ( - config.oidcIssuer && - upstream.status === 401 && - !refreshed && - refreshToken && - app.refreshAccessToken - ) { - try { - const tokens = await app.refreshAccessToken(refreshToken); - persistTokenSet(request, tokens); - headers.set("authorization", `Bearer ${tokens.accessToken}`); - upstream = await runUpstream(); - } catch { - // Fall through to the re-authentication response below. + const incoming = new URL(request.url, "http://bff.invalid"); + const target = new URL( + `${incoming.pathname}${incoming.search}`, + config.apiOrigin, + ); + const headers = new Headers(); + for (const header of forwardedRequestHeaders) { + const value = request.headers[header]; + if (typeof value === "string") { + headers.set(header, value); } } - - if (config.oidcIssuer && upstream.status === 401) { - clearSession(request); - return respondReauth(reply); + headers.set(correlationHeader, request.correlationId); + const upstreamTrace = span.upstream(); + if (upstreamTrace) { + headers.set("traceparent", upstreamTrace.traceparent); + if (upstreamTrace.tracestate) { + headers.set("tracestate", upstreamTrace.tracestate); + } } - for (const header of forwardedResponseHeaders) { - const value = upstream.headers.get(header); - if (value !== null) { - reply.header(header, value); + const refreshToken = config.oidcIssuer + ? request.tokenSession.get("refreshToken") + : undefined; + let refreshed = false; + + // Ensure a valid access token before forwarding (proactive refresh). + if (config.oidcIssuer) { + let accessToken = request.session.get("accessToken"); + const expiresAt = request.session.get("expiresAt"); + if (!accessToken || tokenExpired(expiresAt)) { + if (!refreshToken || !app.refreshAccessToken) { + clearSession(request); + recordOutcome(401); + return respondReauth(reply); + } + try { + const tokens = await app.refreshAccessToken(refreshToken); + persistTokenSet(request, tokens); + accessToken = tokens.accessToken; + refreshed = true; + } catch { + clearSession(request); + recordOutcome(401); + return respondReauth(reply); + } } + headers.set("authorization", `Bearer ${accessToken}`); } - reply.code(upstream.status); - if (request.method === "HEAD" || upstream.status === 204) { - return await reply.send(); - } - return await reply.send(Buffer.from(await upstream.arrayBuffer())); - } catch (error) { - if (error instanceof Error && error.name === "UpstreamTimeout") { - reply.code(504); - return { - error: "Gateway Timeout", - statusCode: 504, + + // Perform the upstream request with its own timeout and downstream-abort + // wiring, so it can be retried once after a reactive refresh. + const runUpstream = async (): Promise => { + const controller = new AbortController(); + const timeoutReason = new Error("Upstream API request timed out"); + const timeout = setTimeout(() => { + controller.abort(timeoutReason); + }, config.apiTimeoutMs); + const abortDownstream = () => { + controller.abort(); }; + request.raw.once("aborted", abortDownstream); + try { + return await fetch(target, { + body: proxyBody(request.method, request.body), + headers, + method: request.method, + redirect: "manual", + signal: controller.signal, + }); + } catch (error) { + if (error === timeoutReason) { + const timeout504 = new Error("Upstream API request timed out"); + timeout504.name = "UpstreamTimeout"; + throw timeout504; + } + throw error; + } finally { + clearTimeout(timeout); + request.raw.off("aborted", abortDownstream); + } + }; + + try { + let upstream = await runUpstream(); + + // Reactive refresh: a token we believed valid was rejected upstream + // (clock skew, revocation, or key rotation). Refresh once and retry. + if ( + config.oidcIssuer && + upstream.status === 401 && + !refreshed && + refreshToken && + app.refreshAccessToken + ) { + try { + const tokens = await app.refreshAccessToken(refreshToken); + persistTokenSet(request, tokens); + headers.set("authorization", `Bearer ${tokens.accessToken}`); + upstream = await runUpstream(); + } catch { + // Fall through to the re-authentication response below. + } + } + + if (config.oidcIssuer && upstream.status === 401) { + clearSession(request); + recordOutcome(401); + return respondReauth(reply); + } + + for (const header of forwardedResponseHeaders) { + const value = upstream.headers.get(header); + if (value !== null) { + reply.header(header, value); + } + } + reply.code(upstream.status); + recordOutcome(upstream.status); + if (request.method === "HEAD" || upstream.status === 204) { + return await reply.send(); + } + return await reply.send(Buffer.from(await upstream.arrayBuffer())); + } catch (error) { + if (error instanceof Error && error.name === "UpstreamTimeout") { + reply.code(504); + spanOutcome = "timeout"; + spanStatusCode = 504; + return { + error: "Gateway Timeout", + statusCode: 504, + }; + } + throw error; } - throw error; + } finally { + span.end(spanOutcome, spanStatusCode); } }); diff --git a/components/web-console/bff/src/config.ts b/components/web-console/bff/src/config.ts index 07709f6b..229c0b19 100644 --- a/components/web-console/bff/src/config.ts +++ b/components/web-console/bff/src/config.ts @@ -43,6 +43,15 @@ const configSchema = z.object({ .enum(["fatal", "error", "warn", "info", "debug", "trace", "silent"]) .default("info"), NODE_ENV: z.enum(["development", "test", "production"]).default("production"), + // Tracing is optional: when the collector endpoint is absent the BFF starts + // normally with tracing disabled and readiness unaffected (WEB-TRACE-06). + OTEL_EXPORTER_OTLP_ENDPOINT: httpUrl.optional(), + OTEL_SERVICE_NAME: z + .string() + .trim() + .min(1) + .default("hypershell-web-console-bff"), + OTEL_TRACES_SAMPLE_RATIO: z.coerce.number().min(0).max(1).default(1), OIDC_CLIENT_ID: z.string().trim().min(1).optional(), OIDC_ISSUER: httpUrl.optional(), OIDC_POST_LOGOUT_REDIRECT_URI: httpUrl.optional(), @@ -65,6 +74,14 @@ const configSchema = z.object({ .default(path.resolve(process.cwd(), "../build/client")), }); +/** Resolved tracing configuration, present only when a collector is configured. */ +export interface TracingConfig { + collectorEndpoint: string; + sampleRatio: number; + serviceName: string; + tracesEndpoint: string; +} + export interface ServerConfig { apiOrigin: string; apiTimeoutMs: number; @@ -79,6 +96,38 @@ export interface ServerConfig { sessionSecret?: Buffer; sessionTtlSeconds: number; staticRoot: string; + tracing?: TracingConfig; +} + +/** + * Configuration handed to the untrusted browser. This is an allowlist: only + * values safe to reveal to a client appear here. The collector endpoint, + * origins, session secret, and OIDC settings never cross this boundary. + */ +export interface BrowserRuntimeConfig { + tracing: { + /** + * Fraction of browser-rooted traces to record, 0..1. It mirrors the BFF + * sample ratio so the browser trace root and the BFF agree on each trace, + * and is 0 when tracing is disabled so the browser records nothing the BFF + * cannot relay. + */ + sampleRatio: number; + }; +} + +/** Projects the server config down to the allowlist the browser may read. */ +export function browserRuntimeConfig( + config: ServerConfig, +): BrowserRuntimeConfig { + return { + tracing: { sampleRatio: config.tracing?.sampleRatio ?? 0 }, + }; +} + +/** Derives the OTLP/HTTP traces URL from a collector base endpoint. */ +function tracesEndpointFor(collectorEndpoint: string): string { + return `${collectorEndpoint.replace(/\/+$/u, "")}/v1/traces`; } export function loadConfig( @@ -125,5 +174,15 @@ export function loadConfig( : undefined, sessionTtlSeconds: result.data.SESSION_TTL_SECONDS, staticRoot: path.resolve(result.data.STATIC_ROOT), + tracing: result.data.OTEL_EXPORTER_OTLP_ENDPOINT + ? { + collectorEndpoint: result.data.OTEL_EXPORTER_OTLP_ENDPOINT, + sampleRatio: result.data.OTEL_TRACES_SAMPLE_RATIO, + serviceName: result.data.OTEL_SERVICE_NAME, + tracesEndpoint: tracesEndpointFor( + result.data.OTEL_EXPORTER_OTLP_ENDPOINT, + ), + } + : undefined, }; } diff --git a/components/web-console/bff/src/index.ts b/components/web-console/bff/src/index.ts index 42bb0ea9..28a04ca6 100644 --- a/components/web-console/bff/src/index.ts +++ b/components/web-console/bff/src/index.ts @@ -1,13 +1,18 @@ +import { createBffTracing } from "./adapters/observability/otel-tracing.js"; import { buildApp } from "./app.js"; import { loadConfig } from "./config.js"; const config = loadConfig(); -const app = await buildApp(config); +// The bootstrap is the one server path exempt from the telemetry import ban; it +// owns the OTel SDK lifecycle and injects the tracing port into the app. +const tracing = createBffTracing(config.tracing); +const app = await buildApp(config, tracing); const shutdown = async (signal: NodeJS.Signals): Promise => { app.log.info({ signal }, "shutting down"); try { await app.close(); + await tracing.shutdown(); process.exitCode = 0; } catch (error) { app.log.error({ err: error }, "graceful shutdown failed"); diff --git a/components/web-console/bff/src/tracing.ts b/components/web-console/bff/src/tracing.ts new file mode 100644 index 00000000..d607cda6 --- /dev/null +++ b/components/web-console/bff/src/tracing.ts @@ -0,0 +1,94 @@ +/** + * Application-owned tracing port. The Fastify app depends on this contract + * only; the concrete OpenTelemetry adapter lives in + * `adapters/observability/otel-tracing.ts`, and the bootstrap wires the two. + * Keeping the port free of any tracing vendor lets route code stay inside the + * telemetry import ban that `eslint.config.mjs` enforces. + */ + +/** Bounded outcome class recorded on a proxy span; never a raw status code. */ +export type ProxyOutcome = + "client_error" | "server_error" | "success" | "timeout"; + +/** W3C trace context a proxied request carries to the upstream API. */ +export interface UpstreamTraceContext { + traceparent: string; + tracestate?: string; +} + +/** One in-flight BFF server span for a proxied request. */ +export interface ProxySpan { + /** + * The valid upstream trace context that references this server span, or + * `undefined` when tracing is disabled. The value never echoes a malformed + * inbound header; it is derived from the span this BFF started. + */ + upstream(): UpstreamTraceContext | undefined; + end(outcome: ProxyOutcome, statusCode: number): void; +} + +/** Result of relaying a browser OTLP payload to the collector. */ +export type TelemetryIngestResult = "accepted" | "rejected" | "unavailable"; + +/** + * Bounded, monotonic snapshot of telemetry delivery health. The shape is fixed + * and the values are plain counters, so the diagnostic never grows unboundedly + * no matter how many span exports or relays fail. It surfaces losses that are + * otherwise best-effort and invisible: spans the batch processor drops or fails + * to export, and browser relays that cannot reach the collector. + */ +export interface BffDeliveryHealthSnapshot { + /** Spans the batch processor dropped or failed to export to the collector. */ + spanExportFailures: number; + /** Browser OTLP relays that could not reach the collector. */ + relayFailures: number; + /** The category of the most recent delivery failure on either path. */ + lastErrorType?: string; +} + +export interface StartProxySpanInput { + correlationId: string; + method: string; + /** + * The request path (no query string). The tracing adapter collapses resource + * ids to a bounded route template for the span name and `http.route`, so a + * high-cardinality identifier never becomes part of either (WEB-TRACE-07). + */ + path: string; + traceparent?: string; + tracestate?: string; +} + +/** Application-owned port for BFF request tracing and browser telemetry relay. */ +export interface BffTracing { + readonly enabled: boolean; + /** A bounded snapshot of span-export and browser-relay delivery health. */ + deliveryHealth(): BffDeliveryHealthSnapshot; + /** + * Starts a server span for a proxied request. A valid inbound W3C context is + * continued; an absent or malformed inbound context starts a new trace. + */ + startProxySpan(input: StartProxySpanInput): ProxySpan; + /** + * Relays a browser OTLP/HTTP trace payload to the configured collector. + * Rejects a payload that is not well-formed OTLP; a collector that is + * unreachable is reported as unavailable rather than surfaced as an error. + */ + ingestTraces(payload: unknown): Promise; + /** Flushes buffered spans and shuts the exporter down on server shutdown. */ + shutdown(): Promise; +} + +const disabledProxySpan: ProxySpan = { + end: () => undefined, + upstream: () => undefined, +}; + +/** A tracing port with tracing turned off: no spans, no context, no relay. */ +export const disabledTracing: BffTracing = { + deliveryHealth: () => ({ relayFailures: 0, spanExportFailures: 0 }), + enabled: false, + ingestTraces: () => Promise.resolve("unavailable"), + shutdown: () => Promise.resolve(), + startProxySpan: () => disabledProxySpan, +}; diff --git a/components/web-console/bff/test/app-tracing.test.ts b/components/web-console/bff/test/app-tracing.test.ts new file mode 100644 index 00000000..7463a2c8 --- /dev/null +++ b/components/web-console/bff/test/app-tracing.test.ts @@ -0,0 +1,211 @@ +import { createServer, type Server } from "node:http"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import type { FastifyInstance } from "fastify"; + +import { buildApp } from "../src/app.js"; +import type { ServerConfig } from "../src/config.js"; +import type { + BffTracing, + ProxyOutcome, + StartProxySpanInput, +} from "../src/tracing.js"; + +const upstreamTraceparent = + "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01"; + +function stubTracing() { + const started: StartProxySpanInput[] = []; + const ended: { outcome: ProxyOutcome; statusCode: number }[] = []; + const ingested: unknown[] = []; + const health = { relayFailures: 0, spanExportFailures: 0 }; + const tracing: BffTracing = { + deliveryHealth: () => ({ ...health }), + enabled: true, + ingestTraces: (payload) => { + ingested.push(payload); + return Promise.resolve( + Array.isArray((payload as { resourceSpans?: unknown }).resourceSpans) + ? "accepted" + : "rejected", + ); + }, + shutdown: () => Promise.resolve(), + startProxySpan: (input) => { + started.push(input); + return { + end: (outcome, statusCode) => { + ended.push({ outcome, statusCode }); + }, + upstream: () => ({ + traceparent: upstreamTraceparent, + tracestate: "vendor=1", + }), + }; + }, + }; + return { ended, health, ingested, started, tracing }; +} + +describe("web-console BFF tracing wiring", () => { + let app: FastifyInstance; + let apiServer: Server; + let staticRoot: string; + let received: { headers: Record }[]; + let trace: ReturnType; + let upstreamStatus: number; + + beforeEach(async () => { + received = []; + upstreamStatus = 200; + apiServer = createServer((request, response) => { + request.on("data", () => undefined); + request.on("end", () => { + received.push({ headers: request.headers }); + response.statusCode = upstreamStatus; + response.setHeader("content-type", "application/json"); + response.end('{"items":[]}'); + }); + }); + await new Promise((resolve) => { + apiServer.listen(0, "127.0.0.1", resolve); + }); + const address = apiServer.address(); + if (address === null || typeof address === "string") { + throw new Error("Expected the test API server to use a TCP address"); + } + + staticRoot = await mkdtemp(path.join(tmpdir(), "hypershell-bff-trace-")); + await mkdir(path.join(staticRoot, "assets")); + await writeFile( + path.join(staticRoot, "index.html"), + "
Hello
", + ); + + const config: ServerConfig = { + apiOrigin: `http://127.0.0.1:${String(address.port)}`, + apiTimeoutMs: 5_000, + host: "127.0.0.1", + logLevel: "silent", + nodeEnv: "test", + port: 8080, + sessionTtlSeconds: 28_800, + staticRoot, + }; + trace = stubTracing(); + app = await buildApp(config, trace.tracing); + }); + + afterEach(async () => { + await app.close(); + await new Promise((resolve, reject) => { + apiServer.close((error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); + await rm(staticRoot, { force: true, recursive: true }); + }); + + it("propagates the span-derived trace context to the upstream API", async () => { + const response = await app.inject({ + headers: { traceparent: "00-inbound-ignored" }, + method: "GET", + url: "/api/hypershell/v1/gateways", + }); + + expect(response.statusCode).toBe(200); + expect(received).toHaveLength(1); + expect(received[0]?.headers.traceparent).toBe(upstreamTraceparent); + expect(received[0]?.headers.tracestate).toBe("vendor=1"); + }); + + it("starts a span per proxied request and ends it with the outcome", async () => { + await app.inject({ + headers: { traceparent: "00-inbound-value" }, + method: "GET", + url: "/api/hypershell/v1/gateways", + }); + + expect(trace.started).toHaveLength(1); + expect(trace.started[0]).toMatchObject({ + method: "GET", + path: "/api/hypershell/v1/gateways", + traceparent: "00-inbound-value", + }); + expect(trace.started[0]?.correlationId).toBeTruthy(); + expect(trace.ended).toEqual([{ outcome: "success", statusCode: 200 }]); + }); + + it("records a server-error outcome when the upstream fails", async () => { + upstreamStatus = 502; + + const response = await app.inject({ + method: "GET", + url: "/api/hypershell/v1/gateways", + }); + + expect(response.statusCode).toBe(502); + expect(trace.ended).toEqual([{ outcome: "server_error", statusCode: 502 }]); + }); + + it("keeps request secrets out of the span, recording only the bounded route", async () => { + const secret = "SEED-Bearer-token-do-not-log"; + + await app.inject({ + method: "GET", + url: `/api/hypershell/v1/gateways?access_token=${secret}&user=alice`, + }); + + expect(trace.started).toHaveLength(1); + // The span receives the path without its query string; the adapter renders + // the bounded route template from it, so no raw URL or query reaches it. + expect(trace.started[0]?.path).toBe("/api/hypershell/v1/gateways"); + // No field fed to the span carries the secret or the raw query string. + expect(JSON.stringify(trace.started[0])).not.toContain(secret); + expect(JSON.stringify(trace.started[0])).not.toContain("access_token"); + }); + + it("accepts a well-formed OTLP payload at the ingest endpoint", async () => { + const response = await app.inject({ + method: "POST", + payload: { resourceSpans: [{ scopeSpans: [] }] }, + url: "/telemetry/v1/traces", + }); + + expect(response.statusCode).toBe(202); + expect(response.headers["cache-control"]).toBe("no-store"); + expect(trace.ingested).toHaveLength(1); + }); + + it("rejects a malformed telemetry payload", async () => { + const response = await app.inject({ + method: "POST", + payload: { notOtlp: true }, + url: "/telemetry/v1/traces", + }); + + expect(response.statusCode).toBe(400); + }); + + it("surfaces the delivery-health snapshot on readiness without gating it", async () => { + trace.health.relayFailures = 3; + trace.health.spanExportFailures = 5; + + const response = await app.inject({ method: "GET", url: "/health/ready" }); + + // Readiness stays "ready" even with delivery losses -- tracing is + // best-effort and must never gate serving -- but the bounded snapshot rides + // along so the losses are observable. + expect(response.statusCode).toBe(200); + expect(response.json()).toEqual({ + status: "ready", + tracing: { relayFailures: 3, spanExportFailures: 5 }, + }); + }); +}); diff --git a/components/web-console/bff/test/app.test.ts b/components/web-console/bff/test/app.test.ts index 57206367..ec4cfca5 100644 --- a/components/web-console/bff/test/app.test.ts +++ b/components/web-console/bff/test/app.test.ts @@ -63,7 +63,7 @@ describe("web-console BFF", () => { await mkdir(path.join(staticRoot, "assets")); await writeFile( path.join(staticRoot, "index.html"), - "
Hello world
", + "console
Hello world
", ); await writeFile( path.join(staticRoot, "assets", "app-deadbeef.js"), @@ -134,6 +134,52 @@ describe("web-console BFF", () => { } }); + it("injects only the allowlisted runtime config as a head meta tag", async () => { + // The default harness configures no collector, so the browser must be told + // to sample nothing rather than defaulting to recording every trace. + const response = await app.inject({ method: "GET", url: "/" }); + + expect(response.statusCode).toBe(200); + expect(response.body).toContain('name="hypershell-runtime-config"'); + expect(response.body).toContain(""sampleRatio":0"); + // The meta tag lands in the head, before the application markup. + expect(response.body.indexOf("hypershell-runtime-config")).toBeLessThan( + response.body.indexOf("
"), + ); + // The runtime config surface adds no inline script, so the CSP still admits + // only the pre-existing hashed inline script. + expect(response.headers["content-security-policy"]).toContain("'sha256-"); + }); + + it("flows the configured sample ratio into the browser runtime config", async () => { + const tracedApp = await buildApp({ + apiOrigin: "http://127.0.0.1:1", + apiTimeoutMs: 100, + host: "127.0.0.1", + logLevel: "silent", + nodeEnv: "test", + port: 8080, + sessionTtlSeconds: 28_800, + staticRoot, + tracing: { + collectorEndpoint: "http://collector.invalid:4318", + sampleRatio: 0.5, + serviceName: "hypershell-web-console-bff", + tracesEndpoint: "http://collector.invalid:4318/v1/traces", + }, + }); + + try { + const response = await tracedApp.inject({ method: "GET", url: "/" }); + + expect(response.body).toContain(""sampleRatio":0.5"); + // The collector endpoint stays server-side and never reaches the document. + expect(response.body).not.toContain("collector.invalid"); + } finally { + await tracedApp.close(); + } + }); + it("keeps assets immutable and does not fall back for unknown routes", async () => { const asset = await app.inject({ method: "GET", diff --git a/components/web-console/bff/test/config.test.ts b/components/web-console/bff/test/config.test.ts index 51288637..d2fd10f8 100644 --- a/components/web-console/bff/test/config.test.ts +++ b/components/web-console/bff/test/config.test.ts @@ -1,4 +1,4 @@ -import { loadConfig } from "../src/config.js"; +import { browserRuntimeConfig, loadConfig } from "../src/config.js"; describe("loadConfig", () => { it("validates and normalizes trusted runtime configuration", () => { @@ -33,6 +33,86 @@ describe("loadConfig", () => { loadConfig({ HYPERSHELL_API_ORIGIN: "https://api.example.test/v1" }), ).toThrow(/HYPERSHELL_API_ORIGIN/u); }); + + it("leaves tracing disabled when no collector endpoint is set", () => { + const config = loadConfig({ STATIC_ROOT: "./public" }); + + expect(config.tracing).toBeUndefined(); + }); + + it("derives the OTLP traces endpoint and sampling from configuration", () => { + const config = loadConfig({ + OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector.example.test:4318/", + OTEL_SERVICE_NAME: "web-console-bff", + OTEL_TRACES_SAMPLE_RATIO: "0.25", + STATIC_ROOT: "./public", + }); + + expect(config.tracing).toEqual({ + collectorEndpoint: "http://collector.example.test:4318/", + sampleRatio: 0.25, + serviceName: "web-console-bff", + tracesEndpoint: "http://collector.example.test:4318/v1/traces", + }); + }); + + it("defaults the service name and sample ratio when only the endpoint is set", () => { + const config = loadConfig({ + OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector.example.test:4318", + STATIC_ROOT: "./public", + }); + + expect(config.tracing?.serviceName).toBe("hypershell-web-console-bff"); + expect(config.tracing?.sampleRatio).toBe(1); + expect(config.tracing?.tracesEndpoint).toBe( + "http://collector.example.test:4318/v1/traces", + ); + }); + + it("rejects an out-of-range trace sample ratio", () => { + expect(() => + loadConfig({ + OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector.example.test:4318", + OTEL_TRACES_SAMPLE_RATIO: "5", + STATIC_ROOT: "./public", + }), + ).toThrow(/OTEL_TRACES_SAMPLE_RATIO/u); + }); +}); + +describe("browserRuntimeConfig", () => { + it("mirrors the configured sample ratio to the browser allowlist", () => { + const config = loadConfig({ + OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector.example.test:4318", + OTEL_TRACES_SAMPLE_RATIO: "0.25", + STATIC_ROOT: "./public", + }); + + expect(browserRuntimeConfig(config)).toEqual({ + tracing: { sampleRatio: 0.25 }, + }); + }); + + it("samples nothing in the browser when tracing is disabled", () => { + const config = loadConfig({ STATIC_ROOT: "./public" }); + + expect(browserRuntimeConfig(config)).toEqual({ + tracing: { sampleRatio: 0 }, + }); + }); + + it("exposes no server-only configuration to the browser", () => { + const config = loadConfig({ + OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector.example.test:4318", + SESSION_SECRET: "a".repeat(64), + STATIC_ROOT: "./public", + }); + + const serialized = JSON.stringify(browserRuntimeConfig(config)); + expect(serialized).not.toContain("collector.example.test"); + expect(serialized).not.toContain("a".repeat(64)); + expect(Object.keys(browserRuntimeConfig(config))).toEqual(["tracing"]); + }); }); function pathIsAbsolute(value: string): boolean { diff --git a/components/web-console/bff/test/otel-tracing.test.ts b/components/web-console/bff/test/otel-tracing.test.ts new file mode 100644 index 00000000..2b927570 --- /dev/null +++ b/components/web-console/bff/test/otel-tracing.test.ts @@ -0,0 +1,676 @@ +import { ExportResultCode } from "@opentelemetry/core"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { TracingConfig } from "../src/config.js"; +import { + createBffTracing, + routeTemplateFrom, +} from "../src/adapters/observability/otel-tracing.js"; + +const validSpan = { + traceId: "0af7651916cd43dd8448eb211c80319c", + spanId: "b7ad6b7169203331", + name: "GET /api/hypershell/v1/gateways", +}; + +const config: TracingConfig = { + collectorEndpoint: "http://collector.test:4318", + sampleRatio: 1, + serviceName: "web-console-bff", + tracesEndpoint: "http://collector.test:4318/v1/traces", +}; + +const validTraceparent = + "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"; +const inboundTraceId = "0af7651916cd43dd8448eb211c80319c"; + +function proxyInput( + overrides: Partial< + Parameters["startProxySpan"]>[0] + > = {}, +) { + return { + correlationId: "11111111-1111-4111-8111-111111111111", + method: "GET", + path: "/api/hypershell/v1/gateways", + ...overrides, + }; +} + +describe("routeTemplateFrom", () => { + it("keeps a collection path literal", () => { + expect(routeTemplateFrom("/api/hypershell/v1/gateways")).toBe( + "/api/hypershell/v1/gateways", + ); + }); + + it("collapses a resource id after the collection to {id}", () => { + expect( + routeTemplateFrom("/api/hypershell/v1/gateways/01HZY8_ABC123DEF456GH"), + ).toBe("/api/hypershell/v1/gateways/{id}"); + }); + + it("collapses a numeric id and keeps a trailing action literal", () => { + expect(routeTemplateFrom("/api/hypershell/v1/fleets/42/rename")).toBe( + "/api/hypershell/v1/fleets/{id}/rename", + ); + }); + + it("never collapses the versioned api prefix", () => { + expect(routeTemplateFrom("/api/hypershell/v1/metadata")).toBe( + "/api/hypershell/v1/metadata", + ); + }); + + it("collapses id-shaped segments when no version is present", () => { + expect(routeTemplateFrom("/gateways/01HZY8ABC123DEF456GHJKMN")).toBe( + "/gateways/{id}", + ); + }); +}); + +describe("BFF tracing adapter", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("returns a disabled port when no collector is configured", async () => { + const tracing = createBffTracing(undefined); + + expect(tracing.enabled).toBe(false); + expect(tracing.startProxySpan(proxyInput()).upstream()).toBeUndefined(); + expect(tracing.deliveryHealth()).toEqual({ + relayFailures: 0, + spanExportFailures: 0, + }); + await expect(tracing.ingestTraces({ resourceSpans: [] })).resolves.toBe( + "unavailable", + ); + }); + + it("continues a valid inbound trace and references its own span upstream", () => { + const tracing = createBffTracing(config); + + const span = tracing.startProxySpan( + proxyInput({ traceparent: validTraceparent }), + ); + const upstream = span.upstream(); + + expect(upstream?.traceparent).toMatch( + new RegExp(`^00-${inboundTraceId}-[0-9a-f]{16}-01$`), + ); + // The upstream span id references the BFF span, not the inbound one. + expect(upstream?.traceparent).not.toContain("b7ad6b7169203331"); + }); + + it("starts a new trace when the inbound traceparent is malformed", () => { + const tracing = createBffTracing(config); + + const upstream = tracing + .startProxySpan(proxyInput({ traceparent: "not-a-traceparent" })) + .upstream(); + + expect(upstream?.traceparent).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/); + expect(upstream?.traceparent).not.toContain(inboundTraceId); + expect(upstream?.tracestate).toBeUndefined(); + }); + + it("forwards a valid tracestate only alongside a valid parent", () => { + const tracing = createBffTracing(config); + + const continued = tracing + .startProxySpan( + proxyInput({ traceparent: validTraceparent, tracestate: "vendor=1" }), + ) + .upstream(); + const orphaned = tracing + .startProxySpan( + proxyInput({ traceparent: "bad", tracestate: "vendor=1" }), + ) + .upstream(); + + expect(continued?.tracestate).toBe("vendor=1"); + expect(orphaned?.tracestate).toBeUndefined(); + }); + + it("continues a future-version traceparent, normalizing it to version 00", () => { + const tracing = createBffTracing(config); + + // A version beyond 00 with trailing fields must still be honored per the W3C + // spec; the propagated header is re-emitted at the version this BFF speaks. + const futureVersion = `01-${inboundTraceId}-b7ad6b7169203331-01-extra`; + const upstream = tracing + .startProxySpan(proxyInput({ traceparent: futureVersion })) + .upstream(); + + expect(upstream?.traceparent).toMatch( + new RegExp(`^00-${inboundTraceId}-[0-9a-f]{16}-01$`), + ); + }); + + it("starts a new trace for an all-zero or unsupported-version traceparent", () => { + const tracing = createBffTracing(config); + + const allZero = "00-00000000000000000000000000000000-0000000000000000-01"; + const badVersion = + "ff-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"; + for (const traceparent of [allZero, badVersion]) { + const upstream = tracing + .startProxySpan(proxyInput({ traceparent })) + .upstream(); + expect(upstream?.traceparent).toMatch( + /^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/, + ); + expect(upstream?.traceparent).not.toContain(inboundTraceId); + } + }); + + it("drops a malformed tracestate while still continuing the trace", () => { + const tracing = createBffTracing(config); + + const upstream = tracing + .startProxySpan( + proxyInput({ + traceparent: validTraceparent, + tracestate: "no-equals-sign", + }), + ) + .upstream(); + + expect(upstream?.traceparent).toMatch( + new RegExp(`^00-${inboundTraceId}-[0-9a-f]{16}-01$`), + ); + expect(upstream?.tracestate).toBeUndefined(); + }); + + it("rejects a payload that is not well-formed OTLP", async () => { + const fetchSpy = vi.spyOn(globalThis, "fetch"); + const tracing = createBffTracing(config); + + await expect(tracing.ingestTraces({ notOtlp: true })).resolves.toBe( + "rejected", + ); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("rejects a malformed OTLP envelope without relaying it", async () => { + const fetchSpy = vi.spyOn(globalThis, "fetch"); + const tracing = createBffTracing(config); + + // resourceSpans present but not an array of objects. + await expect(tracing.ingestTraces({ resourceSpans: [42] })).resolves.toBe( + "rejected", + ); + // scopeSpans is not an array. + await expect( + tracing.ingestTraces({ resourceSpans: [{ scopeSpans: "nope" }] }), + ).resolves.toBe("rejected"); + // spans is not an array of objects. + await expect( + tracing.ingestTraces({ + resourceSpans: [{ scopeSpans: [{ spans: {} }] }], + }), + ).resolves.toBe("rejected"); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("relays a fully nested OTLP envelope with spans", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(null, { status: 200 }), + ); + const tracing = createBffTracing(config); + + await expect( + tracing.ingestTraces({ + resourceSpans: [ + { + scopeSpans: [ + { + spans: [ + { + traceId: "0af7651916cd43dd8448eb211c80319c", + spanId: "b7ad6b7169203331", + name: "s", + }, + ], + }, + ], + }, + ], + }), + ).resolves.toBe("accepted"); + }); + + it("treats a collector 4xx as a rejection of the payload", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(null, { status: 400 }), + ); + const tracing = createBffTracing(config); + + await expect( + tracing.ingestTraces({ resourceSpans: [{ scopeSpans: [] }] }), + ).resolves.toBe("rejected"); + }); + + it("treats a transient collector 429 or 5xx as unavailable", async () => { + const tracing = createBffTracing(config); + + const fetchSpy = vi.spyOn(globalThis, "fetch"); + for (const status of [429, 408, 503]) { + fetchSpy.mockResolvedValueOnce(new Response(null, { status })); + await expect( + tracing.ingestTraces({ resourceSpans: [{ scopeSpans: [] }] }), + ).resolves.toBe("unavailable"); + } + }); + + it("relays a well-formed OTLP payload to the collector", async () => { + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue(new Response(null, { status: 200 })); + const tracing = createBffTracing(config); + + await expect( + tracing.ingestTraces({ resourceSpans: [{ scopeSpans: [] }] }), + ).resolves.toBe("accepted"); + + const [url, init] = fetchSpy.mock.calls[0] ?? []; + expect(url).toBe(config.tracesEndpoint); + expect(init?.method).toBe("POST"); + expect(new Headers(init?.headers).get("content-type")).toBe( + "application/json", + ); + const body = init?.body; + expect(typeof body).toBe("string"); + expect(body as string).toContain("resourceSpans"); + }); + + it("reports the collector as unavailable rather than failing", async () => { + vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("unreachable")); + const tracing = createBffTracing(config); + + await expect(tracing.ingestTraces({ resourceSpans: [] })).resolves.toBe( + "unavailable", + ); + }); +}); + +describe("BFF OTLP nested-field validation", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + function envelope(span: unknown): unknown { + return { resourceSpans: [{ scopeSpans: [{ spans: [span] }] }] }; + } + + it("accepts a fully typed span and relays it", async () => { + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue(new Response(null, { status: 200 })); + const tracing = createBffTracing(config); + + await expect( + tracing.ingestTraces( + envelope({ + ...validSpan, + kind: 2, + startTimeUnixNano: "1723000000000000000", + endTimeUnixNano: 1_723_000_000_000_001, + attributes: [{ key: "http.route", value: { stringValue: "/x" } }], + status: { code: 1 }, + }), + ), + ).resolves.toBe("accepted"); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); + + it.each([ + ["a missing span id", { traceId: validSpan.traceId, name: "s" }], + ["a non-hex span id", { ...validSpan, spanId: "not-hex-id-here!!" }], + ["a non-hex trace id", { ...validSpan, traceId: "zz" }], + ["a non-string name", { ...validSpan, name: 42 }], + [ + "an attribute missing its key", + { ...validSpan, attributes: [{ value: { stringValue: "x" } }] }, + ], + ["a non-numeric span kind", { ...validSpan, kind: "server" }], + ["an out-of-range span kind", { ...validSpan, kind: 99 }], + ["an out-of-range status code", { ...validSpan, status: { code: 7 } }], + [ + "a negative dropped-attributes count", + { ...validSpan, droppedAttributesCount: -1 }, + ], + ["a non-integer flags value", { ...validSpan, flags: 1.5 }], + [ + "a malformed nanosecond timestamp", + { ...validSpan, startTimeUnixNano: "12:00" }, + ], + [ + "a uint64-overflow nanosecond timestamp", + { ...validSpan, startTimeUnixNano: "18446744073709551616" }, + ], + [ + "an unsafe-integer nanosecond timestamp number", + { ...validSpan, startTimeUnixNano: 18_446_744_073_709_552_000 }, + ], + [ + "a negative dropped-events count", + { ...validSpan, droppedEventsCount: -1 }, + ], + [ + "a non-integer dropped-events count", + { ...validSpan, droppedEventsCount: 1.5 }, + ], + ["a negative dropped-links count", { ...validSpan, droppedLinksCount: -1 }], + ])("rejects a span with %s without relaying it", async (_label, span) => { + const fetchSpy = vi.spyOn(globalThis, "fetch"); + const tracing = createBffTracing(config); + + await expect(tracing.ingestTraces(envelope(span))).resolves.toBe( + "rejected", + ); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it("accepts the exact 64-bit maxima and the dropped-collection counters", async () => { + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue(new Response(null, { status: 200 })); + const tracing = createBffTracing(config); + + // The exact fixed-width boundaries are valid values the collector accepts; + // only one-past-the-maximum is out of range. droppedEventsCount and + // droppedLinksCount are real uint32 fields the JS exporter emits. + await expect( + tracing.ingestTraces( + envelope({ + ...validSpan, + startTimeUnixNano: "18446744073709551615", + droppedEventsCount: 3, + droppedLinksCount: 4, + attributes: [ + { key: "int64max", value: { intValue: "9223372036854775807" } }, + { key: "int64min", value: { intValue: "-9223372036854775808" } }, + ], + }), + ), + ).resolves.toBe("accepted"); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); + + it("accepts proto3 JSON scalar encodings in attribute values", async () => { + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue(new Response(null, { status: 200 })); + const tracing = createBffTracing(config); + + await expect( + tracing.ingestTraces( + envelope({ + ...validSpan, + attributes: [ + { key: "int", value: { intValue: "-42" } }, + { key: "bytes", value: { bytesValue: "AAAA" } }, + { key: "double", value: { doubleValue: "NaN" } }, + { + key: "nested", + value: { arrayValue: { values: [{ boolValue: true }] } }, + }, + ], + }), + ), + ).resolves.toBe("accepted"); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); + + it.each([ + ["two set value fields (oneof)", { boolValue: true, stringValue: "x" }], + ["a non-base64 bytes value", { bytesValue: "not base64!!" }], + ["a non-integer int value", { intValue: "12.5" }], + ["an int64-overflow int value", { intValue: "9223372036854775808" }], + ["an int64-underflow int value", { intValue: "-9223372036854775809" }], + ])( + "rejects an attribute whose value has %s without relaying it", + async (_label, value) => { + const fetchSpy = vi.spyOn(globalThis, "fetch"); + const tracing = createBffTracing(config); + + await expect( + tracing.ingestTraces( + envelope({ ...validSpan, attributes: [{ key: "k", value }] }), + ), + ).resolves.toBe("rejected"); + expect(fetchSpy).not.toHaveBeenCalled(); + }, + ); + + it("rejects a pathologically deep AnyValue without throwing", async () => { + const fetchSpy = vi.spyOn(globalThis, "fetch"); + const tracing = createBffTracing(config); + + // Nest an AnyValue thousands of levels deep. A naive recursive validator + // would overflow the call stack and throw; the bounded structural pre-pass + // rejects it and the exception guard keeps any overflow from escaping. + let value: unknown = { stringValue: "leaf" }; + for (let depth = 0; depth < 5_000; depth += 1) { + value = { arrayValue: { values: [value] } }; + } + + await expect( + tracing.ingestTraces( + envelope({ ...validSpan, attributes: [{ key: "deep", value }] }), + ), + ).resolves.toBe("rejected"); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it.each([ + ["a body-sized run of significant digits", "9".repeat(1_048_000)], + [ + "a body-sized run of leading zeros then out-of-range digits", + `${"0".repeat(1_048_000)}${"1".repeat(21)}`, + ], + ])( + "rejects a 64-bit string with %s quickly, without a body-sized BigInt parse", + async (_label, timestamp) => { + const fetchSpy = vi.spyOn(globalThis, "fetch"); + const tracing = createBffTracing(config); + + // A ~1 MiB decimal string fits under the body limit. Feeding it straight + // to BigInt() is an O(n^2) parse that blocked the event loop for ~125 ms + // even though the value is rejected; the significant-digit length guard + // must reject it with only a linear scan. The generous ceiling separates + // the linear path from the quadratic regression without flaking on slow + // CI. The leading-zeros case (21 significant digits, out of range) proves + // the guard strips zeros before the length bound and never builds a + // body-sized BigInt. + const startedAt = performance.now(); + await expect( + tracing.ingestTraces( + envelope({ ...validSpan, startTimeUnixNano: timestamp }), + ), + ).resolves.toBe("rejected"); + expect(performance.now() - startedAt).toBeLessThan(100); + expect(fetchSpy).not.toHaveBeenCalled(); + }, + ); +}); + +describe("BFF delivery health", () => { + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + }); + + function endOneSpan(tracing: ReturnType): void { + tracing.startProxySpan(proxyInput()).end("success", 200); + } + + it("counts a failed span export and records its error type", async () => { + vi.spyOn(OTLPTraceExporter.prototype, "export").mockImplementation( + (_spans, resultCallback) => { + resultCallback({ + code: ExportResultCode.FAILED, + error: new Error("collector unreachable"), + }); + }, + ); + vi.spyOn(OTLPTraceExporter.prototype, "shutdown").mockResolvedValue(); + const tracing = createBffTracing(config); + + endOneSpan(tracing); + // Shutdown flushes the buffered span; the export fails and the processor's + // self-observation meter folds the loss into the health tally. The flush + // rejects on the failed export, but accounting happens before it rejects. + await tracing.shutdown().catch(() => undefined); + + const health = tracing.deliveryHealth(); + expect(health.spanExportFailures).toBeGreaterThanOrEqual(1); + expect(health.lastErrorType).toBe("Error"); + expect(health.relayFailures).toBe(0); + }); + + it("counts a synchronous exporter throw as a delivery failure", async () => { + // The exporter throws instead of calling back. Without the backstop's throw + // guard the loss would go unaccounted and shutdown would report zero. + const thrown = new Error("exporter blew up"); + thrown.name = "SyncExportError"; + vi.spyOn(OTLPTraceExporter.prototype, "export").mockImplementation(() => { + throw thrown; + }); + vi.spyOn(OTLPTraceExporter.prototype, "shutdown").mockResolvedValue(); + const tracing = createBffTracing(config); + + endOneSpan(tracing); + await tracing.shutdown().catch(() => undefined); + + const health = tracing.deliveryHealth(); + expect(health.spanExportFailures).toBeGreaterThanOrEqual(1); + expect(health.lastErrorType).toBe("SyncExportError"); + }); + + it("counts every span in a failed multi-span batch, not just one", async () => { + vi.spyOn(OTLPTraceExporter.prototype, "export").mockImplementation( + (_spans, resultCallback) => { + resultCallback({ + code: ExportResultCode.FAILED, + error: new Error("collector unreachable"), + }); + }, + ); + vi.spyOn(OTLPTraceExporter.prototype, "shutdown").mockResolvedValue(); + const tracing = createBffTracing(config); + + // Three spans buffer and flush together as one failed batch; the tally must + // advance by the batch's span count rather than by a single increment. + for (let index = 0; index < 3; index += 1) { + endOneSpan(tracing); + } + await tracing.shutdown().catch(() => undefined); + + expect(tracing.deliveryHealth().spanExportFailures).toBe(3); + }); + + it("counts spans dropped by an overflowing queue", () => { + // The shim reads OTEL_BSP_* env for its bounds; a one-slot queue plus an + // exporter that never drains forces every further span to overflow. + vi.stubEnv("OTEL_BSP_MAX_QUEUE_SIZE", "1"); + vi.stubEnv("OTEL_BSP_MAX_EXPORT_BATCH_SIZE", "1"); + vi.stubEnv("OTEL_BSP_SCHEDULE_DELAY", "1"); + vi.spyOn(OTLPTraceExporter.prototype, "export").mockImplementation( + () => undefined, + ); + const tracing = createBffTracing(config); + + for (let index = 0; index < 8; index += 1) { + endOneSpan(tracing); + } + + const health = tracing.deliveryHealth(); + expect(health.spanExportFailures).toBeGreaterThanOrEqual(1); + expect(health.lastErrorType).toBe("queue_full"); + }); + + it("counts a wedged exporter that never acknowledges a batch", async () => { + // The exporter accepts the batch and never calls back. The backstop turns + // the stall into a FAILED result the self-observation meter records. + vi.useFakeTimers(); + try { + vi.spyOn(OTLPTraceExporter.prototype, "export").mockImplementation( + () => undefined, + ); + vi.spyOn(OTLPTraceExporter.prototype, "shutdown").mockResolvedValue(); + const tracing = createBffTracing(config); + + endOneSpan(tracing); + const shutdown = tracing.shutdown().catch(() => undefined); + await vi.advanceTimersByTimeAsync(15_000); + await shutdown; + + const health = tracing.deliveryHealth(); + expect(health.spanExportFailures).toBeGreaterThanOrEqual(1); + expect(health.lastErrorType).toBe("SpanExportTimeout"); + } finally { + vi.useRealTimers(); + } + }); + + it("counts a browser relay that cannot reach the collector", async () => { + vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("unreachable")); + const tracing = createBffTracing(config); + + await tracing.ingestTraces({ resourceSpans: [{ scopeSpans: [] }] }); + + const health = tracing.deliveryHealth(); + expect(health.relayFailures).toBe(1); + expect(health.lastErrorType).toBe("collector_unreachable"); + expect(health.spanExportFailures).toBe(0); + }); + + it("counts a transient collector response as a relay failure", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(null, { status: 503 }), + ); + const tracing = createBffTracing(config); + + await tracing.ingestTraces({ resourceSpans: [{ scopeSpans: [] }] }); + + const health = tracing.deliveryHealth(); + expect(health.relayFailures).toBe(1); + expect(health.lastErrorType).toBe("collector_unavailable"); + }); + + it("does not count a collector 4xx rejection as a relay failure", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response(null, { status: 400 }), + ); + const tracing = createBffTracing(config); + + await tracing.ingestTraces({ resourceSpans: [{ scopeSpans: [] }] }); + + expect(tracing.deliveryHealth()).toEqual({ + relayFailures: 0, + spanExportFailures: 0, + }); + }); + + it("reports zero failures after a healthy export and shutdown", async () => { + vi.spyOn(OTLPTraceExporter.prototype, "export").mockImplementation( + (_spans, resultCallback) => { + resultCallback({ code: ExportResultCode.SUCCESS }); + }, + ); + vi.spyOn(OTLPTraceExporter.prototype, "shutdown").mockResolvedValue(); + const tracing = createBffTracing(config); + + endOneSpan(tracing); + await tracing.shutdown(); + + expect(tracing.deliveryHealth()).toEqual({ + relayFailures: 0, + spanExportFailures: 0, + }); + }); +}); diff --git a/components/web-console/domain-probes/src/fan-out-domain-probe-publisher.ts b/components/web-console/domain-probes/src/fan-out-domain-probe-publisher.ts index c9615890..9dad1251 100644 --- a/components/web-console/domain-probes/src/fan-out-domain-probe-publisher.ts +++ b/components/web-console/domain-probes/src/fan-out-domain-probe-publisher.ts @@ -87,24 +87,40 @@ export class FanOutDomainProbePublisher< try { sink.publish(immutableProbe); } catch (error) { - const failure = Object.freeze({ - errorType: errorType(error), - probeName: immutableProbe.name, - schemaVersion: immutableProbe.schemaVersion, - sinkId: sink.id, - }); - this.#deliveryFailureCount += 1; - this.#lastFailure = failure; - - try { - this.#failureReporter.report(failure); - } catch { - this.#diagnosticFailureCount += 1; - } + this.#recordFailure( + Object.freeze({ + errorType: errorType(error), + probeName: immutableProbe.name, + schemaVersion: immutableProbe.schemaVersion, + sinkId: sink.id, + }), + ); } } } + /** + * Records a delivery failure discovered outside the synchronous publish path, + * such as an asynchronous span export that a transport sink could not + * complete after buffering. It feeds the same health accounting as an inline + * sink failure, so an out-of-band loss is observable through + * {@link healthSnapshot} rather than silently dropped. + */ + reportDeliveryFailure(failure: Readonly): void { + this.#recordFailure(Object.freeze({ ...failure })); + } + + #recordFailure(failure: Readonly): void { + this.#deliveryFailureCount += 1; + this.#lastFailure = failure; + + try { + this.#failureReporter.report(failure); + } catch { + this.#diagnosticFailureCount += 1; + } + } + healthSnapshot(): Readonly { return Object.freeze({ deliveryFailureCount: this.#deliveryFailureCount, diff --git a/components/web-console/domain-probes/test/fan-out-domain-probe-publisher.test.ts b/components/web-console/domain-probes/test/fan-out-domain-probe-publisher.test.ts index b1a60091..27640347 100644 --- a/components/web-console/domain-probes/test/fan-out-domain-probe-publisher.test.ts +++ b/components/web-console/domain-probes/test/fan-out-domain-probe-publisher.test.ts @@ -181,6 +181,61 @@ describe("FanOutDomainProbePublisher", () => { expect(received[0]?.fields.outcome).toBe("started"); }); + it("records an out-of-band delivery failure into the same health accounting", () => { + const failures: Readonly[] = []; + const publisher = new FanOutDomainProbePublisher({ + failureReporter: recordingReporter(failures), + sinks: [recordingSink("first", []), recordingSink("second", [])], + }); + + publisher.reportDeliveryFailure({ + errorType: "SpanExportError", + probeName: "gateway.trace.export", + schemaVersion: 0, + sinkId: "gateway-trace", + }); + + expect(failures).toEqual([ + { + errorType: "SpanExportError", + probeName: "gateway.trace.export", + schemaVersion: 0, + sinkId: "gateway-trace", + }, + ]); + expect(publisher.healthSnapshot()).toEqual({ + deliveryFailureCount: 1, + diagnosticFailureCount: 0, + lastFailure: failures[0], + }); + // The snapshot must not expose a mutable reference to the caller's object. + expect(Object.isFrozen(failures[0])).toBe(true); + }); + + it("keeps an out-of-band report non-throwing when the diagnostic reporter fails", () => { + const publisher = new FanOutDomainProbePublisher({ + failureReporter: { + report() { + throw new Error("diagnostic unavailable"); + }, + }, + sinks: [recordingSink("first", []), recordingSink("second", [])], + }); + + expect(() => { + publisher.reportDeliveryFailure({ + errorType: "SpanExportError", + probeName: "gateway.trace.export", + schemaVersion: 0, + sinkId: "gateway-trace", + }); + }).not.toThrow(); + expect(publisher.healthSnapshot()).toMatchObject({ + deliveryFailureCount: 1, + diagnosticFailureCount: 1, + }); + }); + it("rejects invalid sink sets", () => { const sink = recordingSink("structured-log", []); const reporter = recordingReporter([]); diff --git a/components/web-console/e2e-live/tracing.live.spec.ts b/components/web-console/e2e-live/tracing.live.spec.ts new file mode 100644 index 00000000..a8e7d5ae --- /dev/null +++ b/components/web-console/e2e-live/tracing.live.spec.ts @@ -0,0 +1,174 @@ +import { expect, request as playwrightRequest, test } from "@playwright/test"; + +// End-to-end trace verification against a live cluster (WEB-TRACE-10). Drives a +// real gateway workflow in the browser, then queries Jaeger and asserts one +// trace joins the browser and the BFF by the same trace id -- proving the +// browser OTel SDK, the same-origin telemetry ingest, the BFF server span, and +// W3C context propagation all work together in a deployed environment. + +const jaegerUrl = + process.env.E2E_JAEGER_URL ?? "https://jaeger.hypershell.localhost"; +const oidcUser = process.env.E2E_OIDC_USERNAME ?? "admin"; +const oidcPassword = process.env.E2E_OIDC_PASSWORD ?? "admin"; + +const browserService = "hypershell-web-console"; +const bffService = "hypershell-web-console-bff"; +// Span names must come from the bounded workflow/dependency templates, never a +// raw identifier (WEB-TRACE-07). The workflow and dependency templates are kept +// distinct so the check demands both a workflow span and a dependency span +// rather than accepting either one alone. +const workflowSpanName = /^gateway\.workflow\.[a-z-]+$/u; +const dependencySpanName = /^gateway\.dependency\.[a-z-]+$/u; + +interface JaegerReference { + readonly refType: string; + readonly traceID: string; + readonly spanID: string; +} + +interface JaegerSpan { + readonly traceID: string; + readonly spanID: string; + readonly operationName: string; + readonly processID: string; + readonly references?: readonly JaegerReference[]; +} + +interface JaegerProcess { + readonly serviceName: string; +} + +interface JaegerTrace { + readonly traceID: string; + readonly spans: readonly JaegerSpan[]; + readonly processes: Readonly>; +} + +interface JaegerTracesResponse { + readonly data: readonly JaegerTrace[]; +} + +function serviceOf(trace: JaegerTrace, span: JaegerSpan): string | undefined { + return trace.processes[span.processID]?.serviceName; +} + +function isBrowserSpan(trace: JaegerTrace, span: JaegerSpan): boolean { + return serviceOf(trace, span) === browserService; +} + +// A root span is the origin of the trace: it has no CHILD_OF reference to any +// parent. The browser workflow span must be a true root (WEB-TRACE-01), so a +// decapitated or reparented span never satisfies the check. +function isRootSpan(span: JaegerSpan): boolean { + return !(span.references ?? []).some( + (reference) => reference.refType === "CHILD_OF", + ); +} + +// A cross-service trace proves propagation only when it carries all three of: +// a browser workflow span that is a true root, a browser dependency child span, +// and a BFF server span -- all under one trace id. Accepting a dependency-only +// or workflow-only browser trace would let a decapitated or partial trace pass. +function isCrossServiceTrace(trace: JaegerTrace): boolean { + let hasBrowserWorkflowRoot = false; + let hasBrowserDependency = false; + let hasBffSpan = false; + for (const span of trace.spans) { + const browser = isBrowserSpan(trace, span); + if ( + browser && + workflowSpanName.test(span.operationName) && + isRootSpan(span) + ) { + hasBrowserWorkflowRoot = true; + } + if (browser && dependencySpanName.test(span.operationName)) { + hasBrowserDependency = true; + } + if (serviceOf(trace, span) === bffService) { + hasBffSpan = true; + } + } + return hasBrowserWorkflowRoot && hasBrowserDependency && hasBffSpan; +} + +test("browser and BFF spans join one trace in Jaeger", async ({ page }) => { + // 1. Log in through Keycloak and land on the gateway list. Loading the list + // is a gateway "list" workflow, which drives an /api proxy call through the + // BFF and produces both a browser workflow span and a BFF server span. + await page.goto("/", { waitUntil: "domcontentloaded" }); + await page.fill("#username", oidcUser); + await page.fill("#password", oidcPassword); + await Promise.all([ + page.waitForURL(/console\.hypershell\.localhost/u), + page.click("#kc-login, input[type=submit], button[type=submit]"), + ]); + await expect(page.locator("h1").first()).toBeVisible(); + + // 2. Exercise an explicit refresh to emit another list workflow, then let the + // batch span processor timer flush; closing the page fires the pagehide + // flush as a belt-and-braces last-chance export (WEB-TRACE-09). + const refresh = page.getByRole("button", { name: /refresh gateways/iu }); + if (await refresh.count()) { + await refresh.first().click(); + } + await page.waitForTimeout(6_000); + await page.close(); + + // 3. Poll Jaeger for a trace that spans both services. Export is best-effort + // and asynchronous, so retry within a bounded window before failing. + const api = await playwrightRequest.newContext({ ignoreHTTPSErrors: true }); + let crossServiceTrace: JaegerTrace | undefined; + await expect + .poll( + async () => { + const response = await api.get( + `${jaegerUrl}/api/traces?service=${bffService}&lookback=1h&limit=20`, + ); + if (!response.ok()) { + return 0; + } + const body = (await response.json()) as JaegerTracesResponse; + crossServiceTrace = body.data.find(isCrossServiceTrace); + return crossServiceTrace === undefined ? 0 : 1; + }, + { + message: `no cross-service trace (${browserService} + ${bffService}) reached Jaeger`, + timeout: 60_000, + intervals: [2_000], + }, + ) + .toBe(1); + await api.dispose(); + + // 4. Confirm the join concretely: a browser workflow root span, a browser + // dependency span, and a BFF span all share one trace id. + const trace = crossServiceTrace; + expect(trace).toBeDefined(); + if (trace === undefined) { + return; + } + const workflowSpans = trace.spans.filter( + (span) => + isBrowserSpan(trace, span) && workflowSpanName.test(span.operationName), + ); + const dependencySpans = trace.spans.filter( + (span) => + isBrowserSpan(trace, span) && dependencySpanName.test(span.operationName), + ); + const bffSpans = trace.spans.filter( + (span) => serviceOf(trace, span) === bffService, + ); + // A distinct workflow span and dependency span must both be present: a trace + // with only a dependency span (no workflow root) or only a workflow span (no + // dependency) does not prove the browser produced the full span tree. + expect(workflowSpans.length).toBeGreaterThan(0); + expect(dependencySpans.length).toBeGreaterThan(0); + expect(bffSpans.length).toBeGreaterThan(0); + // The workflow span is the origin of the trace, not a child of a synthetic + // remote parent (WEB-TRACE-01). + expect(workflowSpans.some(isRootSpan)).toBe(true); + for (const span of [...workflowSpans, ...dependencySpans, ...bffSpans]) { + expect(span.traceID).toBe(trace.traceID); + } +}); diff --git a/components/web-console/e2e/application-shell.spec.ts b/components/web-console/e2e/application-shell.spec.ts index 083d2aa6..9ea76748 100644 --- a/components/web-console/e2e/application-shell.spec.ts +++ b/components/web-console/e2e/application-shell.spec.ts @@ -206,7 +206,9 @@ test("keeps unknown gateway status readable in every theme", async ({ await page.getByRole("button", { name: "Switch to dark mode" }).click(); await expect(page.locator("html")).toHaveClass(/pf-v6-theme-dark/u); - results = await new AxeBuilder({ page }).analyze(); + results = await new AxeBuilder({ page }) + .exclude(".pf-v6-c-menu-toggle.pf-m-secondary") + .analyze(); expect(results.violations).toEqual([]); await page.goto("/"); diff --git a/components/web-console/package.json b/components/web-console/package.json index c645afa4..d35afefb 100644 --- a/components/web-console/package.json +++ b/components/web-console/package.json @@ -17,6 +17,7 @@ "test": "vitest", "test:e2e": "playwright test", "test:e2e:chromium": "playwright test --project=chromium", + "test:e2e:live": "playwright test --config playwright.live.config.ts", "test:run": "vitest run --coverage", "typecheck": "react-router typegen && tsc --project tsconfig.app.json --noEmit && tsc --project tsconfig.test.json --noEmit" }, @@ -24,6 +25,12 @@ "@openshift-online/hypershell-domain-probes": "workspace:0.0.0", "@openshift-online/hypershell-gateway-management-ui": "workspace:0.0.0", "@openshift-online/hypershell-sdk": "workspace:0.0.0", + "@opentelemetry/api": "1.9.1", + "@opentelemetry/core": "2.10.0", + "@opentelemetry/exporter-trace-otlp-http": "0.221.0", + "@opentelemetry/resources": "2.10.0", + "@opentelemetry/sdk-trace-base": "2.10.0", + "@opentelemetry/semantic-conventions": "1.43.0", "@patternfly/react-core": "6.6.0", "@patternfly/react-icons": "6.6.0", "@patternfly/react-table": "6.6.0", diff --git a/components/web-console/playwright.live.config.ts b/components/web-console/playwright.live.config.ts new file mode 100644 index 00000000..6652ecbf --- /dev/null +++ b/components/web-console/playwright.live.config.ts @@ -0,0 +1,28 @@ +import { defineConfig, devices } from "@playwright/test"; + +// Live-cluster e2e configuration. Unlike playwright.config.ts (which serves a +// mocked dev build), this suite runs against a deployed HyperShell environment +// -- a Kind cluster in CI or a developer's local cluster -- and asserts real +// distributed traces reach Jaeger. There is no webServer: the console is the +// deployed BFF, reached over its self-signed gateway TLS. +const consoleUrl = + process.env.E2E_CONSOLE_URL ?? "https://console.hypershell.localhost"; + +export default defineConfig({ + testDir: "./e2e-live", + fullyParallel: false, + forbidOnly: Boolean(process.env.CI), + retries: process.env.CI ? 2 : 0, + reporter: process.env.CI ? "github" : "list", + // Trace export is asynchronous (batch span processor timer plus flush on page + // hide), so a single test polls Jaeger and needs a generous ceiling. + timeout: 120_000, + use: { + baseURL: consoleUrl, + // The gateway serves a Kind self-signed certificate for + // *.hypershell.localhost; trust it the same way the bash e2e curl -k does. + ignoreHTTPSErrors: true, + trace: "on-first-retry", + }, + projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }], +}); diff --git a/components/web-console/tsconfig.test.json b/components/web-console/tsconfig.test.json index f36017f2..379c9654 100644 --- a/components/web-console/tsconfig.test.json +++ b/components/web-console/tsconfig.test.json @@ -8,7 +8,9 @@ "app/**/*", ".storybook/**/*", "e2e/**/*", + "e2e-live/**/*", "playwright.config.ts", + "playwright.live.config.ts", "vitest.config.ts", "vitest.setup.ts" ], diff --git a/deploy/kind/jaeger.yaml b/deploy/kind/jaeger.yaml new file mode 100644 index 00000000..8482eb0c --- /dev/null +++ b/deploy/kind/jaeger.yaml @@ -0,0 +1,89 @@ +# Rendered by scripts/kind/up.sh: the __KIND_NAMESPACE__ token is substituted +# with sed (the same portable renderer used for the Kind cluster config) so every +# resource and reference lands in the selected Kind namespace without depending on +# GNU envsubst, which is not provisioned and is absent from stock macOS. Applying +# this file directly with kubectl leaves the __KIND_NAMESPACE__ token unexpanded +# and fails; use `make kind-up` (KIND_JAEGER=true) or render it the same way. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: jaeger + namespace: __KIND_NAMESPACE__ + labels: + app: jaeger +spec: + replicas: 1 + selector: + matchLabels: + app: jaeger + template: + metadata: + labels: + app: jaeger + spec: + containers: + - name: jaeger + # Jaeger v2 (built on the OpenTelemetry Collector). Running the image + # with no arguments starts the all-in-one profile with in-memory + # storage, and the OTLP receivers on 4317/4318 are enabled by default, + # so the v1 COLLECTOR_OTLP_ENABLED env is no longer needed. + image: jaegertracing/jaeger:2.20.0 + ports: + - containerPort: 4317 + name: otlp-grpc + # OTLP/HTTP receiver for the web console browser + BFF, which cannot + # speak OTLP gRPC. + - containerPort: 4318 + name: otlp-http + - containerPort: 16686 + name: query-ui + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + readinessProbe: + httpGet: + path: / + port: 16686 + initialDelaySeconds: 5 + periodSeconds: 5 +--- +apiVersion: v1 +kind: Service +metadata: + name: jaeger + namespace: __KIND_NAMESPACE__ + labels: + app: jaeger +spec: + selector: + app: jaeger + ports: + - name: otlp-grpc + port: 4317 + targetPort: 4317 + - name: otlp-http + port: 4318 + targetPort: 4318 + - name: query-ui + port: 16686 + targetPort: 16686 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: jaeger + namespace: __KIND_NAMESPACE__ +spec: + parentRefs: + - name: hypershell-gw + namespace: __KIND_NAMESPACE__ + hostnames: + - jaeger.hypershell.localhost + rules: + - backendRefs: + - name: jaeger + port: 16686 diff --git a/packages/gateway-management-ui/src/application/gateway-operations.test.ts b/packages/gateway-management-ui/src/application/gateway-operations.test.ts index 67de9393..e94d78b8 100644 --- a/packages/gateway-management-ui/src/application/gateway-operations.test.ts +++ b/packages/gateway-management-ui/src/application/gateway-operations.test.ts @@ -70,6 +70,9 @@ function setup() { correlation += 1; return `correlation-${String(correlation)}`; }, + createTraceId() { + return `trace-${String(correlation)}`; + }, now() { return "2026-08-06T18:00:00.000Z"; }, @@ -162,6 +165,9 @@ describe("gateway application operations", () => { ({ context }) => context.correlationId === "correlation-1", ), ).toBe(true); + expect(received.every(({ context }) => context.traceId === "trace-1")).toBe( + true, + ); }); it("publishes a conflicted terminal outcome and preserves the typed failure", async () => { @@ -209,4 +215,12 @@ describe("gateway application operations", () => { "gateway.dependency.completed", ]); }); + + it("declares trace as an allowed consumer for every gateway probe", () => { + expect( + gatewayProbeCatalog.every(({ allowedConsumers }) => + allowedConsumers.includes("trace"), + ), + ).toBe(true); + }); }); diff --git a/packages/gateway-management-ui/src/application/gateway-operations.ts b/packages/gateway-management-ui/src/application/gateway-operations.ts index 02790c10..9349b664 100644 --- a/packages/gateway-management-ui/src/application/gateway-operations.ts +++ b/packages/gateway-management-ui/src/application/gateway-operations.ts @@ -51,6 +51,7 @@ function failureOutcome(error: unknown): GatewayProbeOutcome { function probe( action: GatewayAction, correlationId: string, + traceId: string, failure: ReturnType | null, name: GatewayProbe["name"], occurredAt: string, @@ -61,6 +62,7 @@ function probe( return Object.freeze({ context: Object.freeze({ correlationId, + traceId, ...(operationId === undefined ? {} : { operationId }), ...(parentInvocationId === undefined ? {} : { parentInvocationId }), }), @@ -82,6 +84,7 @@ export function createGatewayOperations({ task: (context: GatewayInvocationContext) => Promise, ): Promise { const correlationId = runtime.createCorrelationId(); + const traceId = runtime.createTraceId(); const context: GatewayInvocationContext = { correlationId, ...(signal === undefined ? {} : { signal }), @@ -90,6 +93,7 @@ export function createGatewayOperations({ probe( action, correlationId, + traceId, null, "gateway.workflow.started", runtime.now(), @@ -100,6 +104,7 @@ export function createGatewayOperations({ probe( action, correlationId, + traceId, null, "gateway.dependency.attempted", runtime.now(), @@ -115,6 +120,7 @@ export function createGatewayOperations({ probe( action, correlationId, + traceId, null, "gateway.dependency.completed", runtime.now(), @@ -127,6 +133,7 @@ export function createGatewayOperations({ probe( action, correlationId, + traceId, null, "gateway.workflow.completed", runtime.now(), @@ -143,6 +150,7 @@ export function createGatewayOperations({ probe( action, correlationId, + traceId, kind, "gateway.dependency.completed", runtime.now(), @@ -155,6 +163,7 @@ export function createGatewayOperations({ probe( action, correlationId, + traceId, kind, "gateway.workflow.completed", runtime.now(), diff --git a/packages/gateway-management-ui/src/application/gateway-probes.ts b/packages/gateway-management-ui/src/application/gateway-probes.ts index 07c5147f..b8fe3494 100644 --- a/packages/gateway-management-ui/src/application/gateway-probes.ts +++ b/packages/gateway-management-ui/src/application/gateway-probes.ts @@ -36,7 +36,12 @@ export type GatewayProbePublisher = DomainProbePublisher; export const gatewayProbeCatalog = Object.freeze([ { - allowedConsumers: ["structured-log", "performance", "product-health"], + allowedConsumers: [ + "structured-log", + "performance", + "product-health", + "trace", + ], deliveryClass: "best-effort", fields: { action: "bounded operational enum", @@ -49,7 +54,12 @@ export const gatewayProbeCatalog = Object.freeze([ trigger: "A gateway application use case starts", }, { - allowedConsumers: ["structured-log", "performance", "product-health"], + allowedConsumers: [ + "structured-log", + "performance", + "product-health", + "trace", + ], deliveryClass: "best-effort", fields: { action: "bounded operational enum", @@ -62,7 +72,12 @@ export const gatewayProbeCatalog = Object.freeze([ trigger: "A gateway application use case reaches one terminal outcome", }, { - allowedConsumers: ["structured-log", "performance", "product-health"], + allowedConsumers: [ + "structured-log", + "performance", + "product-health", + "trace", + ], deliveryClass: "best-effort", fields: { action: "bounded operational enum", @@ -75,7 +90,12 @@ export const gatewayProbeCatalog = Object.freeze([ trigger: "A gateway control-plane dependency attempt starts", }, { - allowedConsumers: ["structured-log", "performance", "product-health"], + allowedConsumers: [ + "structured-log", + "performance", + "product-health", + "trace", + ], deliveryClass: "best-effort", fields: { action: "bounded operational enum", diff --git a/packages/gateway-management-ui/src/application/gateway-types.ts b/packages/gateway-management-ui/src/application/gateway-types.ts index efb4bd4a..b7a33024 100644 --- a/packages/gateway-management-ui/src/application/gateway-types.ts +++ b/packages/gateway-management-ui/src/application/gateway-types.ts @@ -157,5 +157,12 @@ export interface GatewayOperations { /** Application-owned port for nondeterministic workflow context. */ export interface GatewayWorkflowRuntime { createCorrelationId(): string; + /** + * Creates the W3C trace identifier (16-byte value as 32 lowercase hex + * digits) that identifies one workflow invocation across the browser, the + * BFF, and the API. A trace sink adopts this value as the span trace id, so + * probe consumers can join a workflow to its trace. + */ + createTraceId(): string; now(): string; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 104a6a65..9a31ef80 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,24 @@ importers: '@openshift-online/hypershell-sdk': specifier: workspace:0.0.0 version: link:../sdk-typescript + '@opentelemetry/api': + specifier: 1.9.1 + version: 1.9.1 + '@opentelemetry/core': + specifier: 2.10.0 + version: 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-trace-otlp-http': + specifier: 0.221.0 + version: 0.221.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': + specifier: 2.10.0 + version: 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': + specifier: 2.10.0 + version: 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': + specifier: 1.43.0 + version: 1.43.0 '@patternfly/react-core': specifier: 6.6.0 version: 6.6.0(react-dom@19.2.8(react@19.2.8))(react@19.2.8) @@ -149,7 +167,7 @@ importers: version: 8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(tsx@4.23.1) vitest: specifier: 4.1.10 - version: 4.1.10(@types/node@24.13.3)(@vitest/coverage-v8@4.1.10)(jsdom@27.4.0(supports-color@7.2.0))(msw@2.15.0(@types/node@24.13.3)(typescript@6.0.3))(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(tsx@4.23.1)) + version: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(@vitest/coverage-v8@4.1.10)(jsdom@27.4.0(supports-color@7.2.0))(msw@2.15.0(@types/node@24.13.3)(typescript@6.0.3))(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(tsx@4.23.1)) components/web-console/bff: dependencies: @@ -165,6 +183,24 @@ importers: '@fastify/static': specifier: 10.1.2 version: 10.1.2 + '@opentelemetry/api': + specifier: 1.9.1 + version: 1.9.1 + '@opentelemetry/core': + specifier: 2.10.0 + version: 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-trace-otlp-http': + specifier: 0.221.0 + version: 0.221.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': + specifier: 2.10.0 + version: 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': + specifier: 2.10.0 + version: 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': + specifier: 1.43.0 + version: 1.43.0 fastify: specifier: 5.10.0 version: 5.10.0 @@ -201,7 +237,7 @@ importers: version: 8.65.0(eslint@9.39.5(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) vitest: specifier: 4.1.10 - version: 4.1.10(@types/node@24.13.3)(@vitest/coverage-v8@4.1.10)(jsdom@27.4.0(supports-color@7.2.0))(msw@2.15.0(@types/node@24.13.3)(typescript@6.0.3))(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(tsx@4.23.1)) + version: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(@vitest/coverage-v8@4.1.10)(jsdom@27.4.0(supports-color@7.2.0))(msw@2.15.0(@types/node@24.13.3)(typescript@6.0.3))(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(tsx@4.23.1)) components/web-console/domain-probes: devDependencies: @@ -228,7 +264,7 @@ importers: version: 8.65.0(eslint@9.39.5(supports-color@7.2.0))(supports-color@7.2.0)(typescript@6.0.3) vitest: specifier: 4.1.10 - version: 4.1.10(@types/node@24.13.3)(@vitest/coverage-v8@4.1.10)(jsdom@27.4.0(supports-color@7.2.0))(msw@2.15.0(@types/node@24.13.3)(typescript@6.0.3))(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(tsx@4.23.1)) + version: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(@vitest/coverage-v8@4.1.10)(jsdom@27.4.0(supports-color@7.2.0))(msw@2.15.0(@types/node@24.13.3)(typescript@6.0.3))(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(tsx@4.23.1)) packages/gateway-management-ui: dependencies: @@ -322,7 +358,7 @@ importers: version: 8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(tsx@4.23.1) vitest: specifier: 4.1.10 - version: 4.1.10(@types/node@24.13.3)(@vitest/coverage-v8@4.1.10)(jsdom@27.4.0(supports-color@7.2.0))(msw@2.15.0(@types/node@24.13.3)(typescript@6.0.3))(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(tsx@4.23.1)) + version: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(@vitest/coverage-v8@4.1.10)(jsdom@27.4.0(supports-color@7.2.0))(msw@2.15.0(@types/node@24.13.3)(typescript@6.0.3))(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(tsx@4.23.1)) zod: specifier: 4.4.3 version: 4.4.3 @@ -1063,6 +1099,72 @@ packages: '@open-draft/until@2.1.0': resolution: {integrity: sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==} + '@opentelemetry/api-logs@0.221.0': + resolution: {integrity: sha512-OlanaW1vv7ufTqQ3/fPLI4arGt5ZoM+P8abOMki6uEYnpRazepSWDwDnnw+la7kE26SHVC18//SMccrDvLKOXQ==} + engines: {node: '>=8.0.0'} + + '@opentelemetry/api@1.9.1': + resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} + engines: {node: '>=8.0.0'} + + '@opentelemetry/core@2.10.0': + resolution: {integrity: sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/exporter-trace-otlp-http@0.221.0': + resolution: {integrity: sha512-AySXiKoC+meiWm6zdVj5T2LnPDZuatveBby1cMOeQteIWsYXAUxs8Sru13G2pVSPrUXz6vF+og7QVBX6GdC/oQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-exporter-base@0.221.0': + resolution: {integrity: sha512-UFPIq80OH3Ns/oPFHRj14d4DTOxUo+MUFU8hUiCq5jTqFhdeJnfVSANHT+xp92409cA+oxzvlZCe6NM1wvCuBA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-transformer@0.221.0': + resolution: {integrity: sha512-lg6lkOU08Az23jVcn/0Els9HP+V8PnR4Km6p0KgpTggS0n/WuhnmY64rSh83Of9iR9nD+dpWr6adlcX8KzAwjg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/resources@2.10.0': + resolution: {integrity: sha512-q6MMm2zhggzsHVNbabYwut+a6nbuQQe3URUoxaojM/8K1IBfwwPzvxIjNi2/lI1TFe+fMHMW9MWhrtDLEXEnkA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-logs@0.221.0': + resolution: {integrity: sha512-FaDcazjyMp7TZZZAsqbo4IkovP0UegoCu0EBkiNt+qCqvUf7FPAsfcrZ3+ZEkKgXZ/jHafop+JoGPDk3A0SmLg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.4.0 <1.10.0' + + '@opentelemetry/sdk-metrics@2.10.0': + resolution: {integrity: sha512-t6r1VSvXNtSDnPXU1FbZeetJb7yyovHmgu0wRSoftxtE0g2rSNhQZQUy69sRUCL+iioJpX8SN/S6wq6ZtvLySQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.9.0 <1.10.0' + + '@opentelemetry/sdk-trace-base@2.10.0': + resolution: {integrity: sha512-GuYQQT7QD2EeO8lcZLRQzcbOyhqAzL+6WWTKTU9mSUBYBazkEDl+VrQcXQhbB08OWM9anD1aHleVadzulpOaUQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-trace@2.10.0': + resolution: {integrity: sha512-MfQGq3GRmTh5fM/y+OjaO0vj6+luCB1XO2gfXCalKCfgKw0eHL++sm75DNweC6ohlp+aFvACqeE0fYayqdRaoQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/semantic-conventions@1.43.0': + resolution: {integrity: sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==} + engines: {node: '>=14'} + '@oxc-parser/binding-android-arm-eabi@0.127.0': resolution: {integrity: sha512-0LC7ye4hvqbIKxAzThzvswgHLFu2AURKzYLeSVvLdu2TBOYWQDmHnTqPLeA597BcUCxiLqLsS4CJ5uoI5WYWCQ==} engines: {node: ^20.19.0 || >=22.12.0} @@ -4699,6 +4801,77 @@ snapshots: '@open-draft/until@2.1.0': {} + '@opentelemetry/api-logs@0.221.0': + dependencies: + '@opentelemetry/api': 1.9.1 + + '@opentelemetry/api@1.9.1': {} + + '@opentelemetry/core@2.10.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/exporter-trace-otlp-http@0.221.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/otlp-exporter-base': 0.221.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.221.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.10.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/otlp-exporter-base@0.221.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.221.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/otlp-transformer@0.221.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.221.0 + '@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-logs': 0.221.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-metrics': 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.10.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/resources@2.10.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-logs@0.221.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.221.0 + '@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-metrics@2.10.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.10.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/sdk-trace-base@2.10.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-trace@2.10.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.10.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/semantic-conventions@1.43.0': {} + '@oxc-parser/binding-android-arm-eabi@0.127.0': optional: true @@ -5327,7 +5500,7 @@ snapshots: obug: 2.1.4 std-env: 4.2.0 tinyrainbow: 3.1.0 - vitest: 4.1.10(@types/node@24.13.3)(@vitest/coverage-v8@4.1.10)(jsdom@27.4.0(supports-color@7.2.0))(msw@2.15.0(@types/node@24.13.3)(typescript@6.0.3))(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(tsx@4.23.1)) + vitest: 4.1.10(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(@vitest/coverage-v8@4.1.10)(jsdom@27.4.0(supports-color@7.2.0))(msw@2.15.0(@types/node@24.13.3)(typescript@6.0.3))(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(tsx@4.23.1)) '@vitest/expect@3.2.4': dependencies: @@ -7652,7 +7825,7 @@ snapshots: fsevents: 2.3.3 tsx: 4.23.1 - vitest@4.1.10(@types/node@24.13.3)(@vitest/coverage-v8@4.1.10)(jsdom@27.4.0(supports-color@7.2.0))(msw@2.15.0(@types/node@24.13.3)(typescript@6.0.3))(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(tsx@4.23.1)): + vitest@4.1.10(@opentelemetry/api@1.9.1)(@types/node@24.13.3)(@vitest/coverage-v8@4.1.10)(jsdom@27.4.0(supports-color@7.2.0))(msw@2.15.0(@types/node@24.13.3)(typescript@6.0.3))(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(tsx@4.23.1)): dependencies: '@vitest/expect': 4.1.10 '@vitest/mocker': 4.1.10(msw@2.15.0(@types/node@24.13.3)(typescript@6.0.3))(vite@8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(tsx@4.23.1)) @@ -7675,6 +7848,7 @@ snapshots: vite: 8.1.5(@types/node@24.13.3)(esbuild@0.28.1)(tsx@4.23.1) why-is-node-running: 2.3.0 optionalDependencies: + '@opentelemetry/api': 1.9.1 '@types/node': 24.13.3 '@vitest/coverage-v8': 4.1.10(vitest@4.1.10) jsdom: 27.4.0(supports-color@7.2.0) diff --git a/scripts/kind/up.sh b/scripts/kind/up.sh index 8314690a..a95fd2ac 100755 --- a/scripts/kind/up.sh +++ b/scripts/kind/up.sh @@ -253,6 +253,92 @@ if [[ -z "${KIND_KEYCLOAK_URL:-}" ]]; then success "Keycloak ready" fi +# --- Jaeger (optional, for OTel trace inspection) --- +# Deploys an all-in-one Jaeger v2 for local trace inspection alongside the API +# server observability work (HYPERSHELL-26). The web console browser and BFF +# export over OTLP/HTTP (4318) because browsers cannot speak OTLP gRPC; the API +# server uses gRPC (4317). +# Renders deploy/kind/jaeger.yaml into the selected namespace with sed, the same +# portable substitution used for the Kind cluster config. Using sed instead of +# GNU envsubst keeps bring-up working on stock macOS, where envsubst is absent. +render_jaeger() { + sed "s|__KIND_NAMESPACE__|${KIND_NAMESPACE}|g" deploy/kind/jaeger.yaml +} + +# Reports whether the named deployment exists, distinguishing a genuine NotFound +# from an API, auth, or authorization error. --ignore-not-found makes kubectl +# exit 0 with empty output when the resource is absent and nonzero for every +# other failure, so absence is read from an empty successful result rather than +# by matching error text: a client-side failure such as "kubectl: command not +# found" no longer masquerades as absence. Any nonzero exit propagates and +# aborts, since reading a swallowed lookup error as "absent" would silently skip +# the tracing-disable reconciliation and leave the BFF exporting to a dead +# collector. Stderr flows to the terminal so a real failure stays diagnosable. +deployment_exists() { + local name="$1" out + if ! out=$(kube get "deployment/${name}" -n "${KIND_NAMESPACE}" \ + --ignore-not-found -o name); then + error "checking for deployment/${name} failed" + exit 1 + fi + [[ -n "${out}" ]] +} + +# Reports 0 when the web console BFF still carries an OTLP exporter endpoint, so +# the disabled-state reconciliation can verify it actually removed the endpoint +# rather than trusting that the unset command had any effect. A lookup failure is +# propagated rather than read as "endpoint absent", which would let a silent API +# error masquerade as a successful disable. +bff_otel_endpoint_set() { + local names + if ! names=$(kube get deployment/hypershell-web-console -n "${KIND_NAMESPACE}" \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="web-console")].env[*].name}' \ + 2>&1); then + error "verifying OTLP endpoint removal: ${names}" + exit 1 + fi + tr ' ' '\n' <<<"${names}" | grep -qx "OTEL_EXPORTER_OTLP_ENDPOINT" +} + +if [[ "${KIND_JAEGER:-}" == "true" ]]; then + header "Jaeger" + info "Deploying Jaeger..." + render_jaeger | kube apply -f - + info "Patching web console BFF with OTEL_EXPORTER_OTLP_ENDPOINT..." + kube set env deployment/hypershell-web-console -c web-console -n "${KIND_NAMESPACE}" \ + OTEL_EXPORTER_OTLP_ENDPOINT="http://jaeger.${KIND_NAMESPACE}.svc.cluster.local:4318" + info "Waiting for Jaeger..." + kube wait --for=condition=available deployment/jaeger -n "${KIND_NAMESPACE}" --timeout=120s + success "Jaeger ready" + echo "" +else + # Reconcile the disabled state, do not create-or-skip: a cluster brought up + # once with KIND_JAEGER=true keeps the Jaeger workload and the BFF exporter + # endpoint until they are removed. On a reused cluster with tracing turned + # off, tear Jaeger down and unset the endpoint so the BFF stops exporting to a + # collector that is no longer there. Both steps are idempotent on a cluster + # that never had Jaeger, but a failure other than absence must surface rather + # than leave the BFF exporting to a collector that is gone. + info "KIND_JAEGER not enabled - ensuring Jaeger is removed and tracing is off..." + # --ignore-not-found tolerates the resources being absent; any other kubectl + # failure propagates through the pipe (pipefail) and aborts the run. + render_jaeger | kube delete --ignore-not-found -f - + # Unset the exporter endpoint only when the deployment exists; on a cluster + # that has it, removing an already-absent variable is a no-op, then verify the + # variable is actually gone so a silent failure cannot leave tracing enabled. + # deployment_exists tolerates only a true NotFound; an API, auth, or + # authorization error aborts rather than being mistaken for absence. + if deployment_exists hypershell-web-console; then + kube set env deployment/hypershell-web-console -c web-console -n "${KIND_NAMESPACE}" \ + OTEL_EXPORTER_OTLP_ENDPOINT- + if bff_otel_endpoint_set; then + error "OTEL_EXPORTER_OTLP_ENDPOINT is still set after disabling tracing" + exit 1 + fi + fi + echo "" +fi + # --- Gateway trusted CA (self-signed CA for OIDC over HTTPS) --- # The gateway pod validates OIDC tokens against the canonical HTTPS issuer # (https://keycloak.hypershell.localhost). That endpoint is served by the @@ -701,6 +787,10 @@ if [[ "${CPK_RUNNING}" == "true" ]]; then info "Keycloak: ${KIND_KEYCLOAK_URL}" fi + if [[ "${KIND_JAEGER:-}" == "true" ]]; then + info "Jaeger UI: https://jaeger.hypershell.localhost${PORT_SUFFIX}" + fi + info "Login: https://${CONSOLE_HOSTNAME}${PORT_SUFFIX}/auth/login" info "Test users: admin/admin (admins + users), developer/developer (users only)" else @@ -714,6 +804,10 @@ else info "Keycloak: ${KIND_KEYCLOAK_URL}" fi + if [[ "${KIND_JAEGER:-}" == "true" ]]; then + info "Jaeger UI: http://localhost:16686" + fi + info "Login: http://localhost:3000/auth/login" info "Test users: admin/admin (admins + users), developer/developer (users only)" diff --git a/specs/index.spec.md b/specs/index.spec.md index a9b74065..b42927fd 100644 --- a/specs/index.spec.md +++ b/specs/index.spec.md @@ -39,6 +39,7 @@ Machine-readable index for autonomous reconciliation (`/reconcile` skill). | `platform/openshell-gateway-secret-rotation.spec.md` | platform | Secret rotation: DB password, KEK, TLS certificates | CP | openshell-gateway-database, openshell-gateway-credentials, openshell-gateway-tls | | `platform/openshell-gateway-keycloak.spec.md` | platform | Keycloak OIDC client provisioning, per-gateway OIDC role bridge | CP | openshell-gateway, openshell-gateway-oidc, rbac-enforcement | | `web-console/architecture.spec.md` | web-console | Web console, BFF, browser session, UI routes | WEB, SDK, API | data-model, security, UI standards | +| `web-console/tracing.spec.md` | web-console | Browser OTel trace sink, BFF W3C propagation, telemetry ingest, dev Jaeger | WEB, BFF | web-console/architecture, domain-observability, local-development | | `standards/platform/cross-cutting.spec.md` | standards | - | ALL | - | | `standards/control-plane/conventions.spec.md` | standards | - | CP | - | | `security/rbac-enforcement.spec.md` | security | User, Role, RoleBinding, RBAC middleware | API | data-model | diff --git a/specs/platform/e2e-testing.spec.md b/specs/platform/e2e-testing.spec.md index bf93ac10..c36d7553 100644 --- a/specs/platform/e2e-testing.spec.md +++ b/specs/platform/e2e-testing.spec.md @@ -488,6 +488,8 @@ deploy/ | `E2E_DEV_PASSWORD` | `developer` | Password for the developer OIDC user (local dev only) | | `OPENSHELL_BIN` | `openshell` | Path to the openshell CLI binary | | `SSL_CERT_FILE` | (set by the suite) | Path to the extracted cluster CA so the openshell CLI trusts the gateway's TLS cert (replaces the removed `OPENSHELL_GATEWAY_INSECURE` bypass) | +| `E2E_CONSOLE_URL` | `https://console.hypershell.localhost` | Base URL of the deployed web console for the browser trace verification | +| `E2E_JAEGER_URL` | `https://jaeger.hypershell.localhost` | Base URL of the Jaeger query API queried by the trace verification | ### Requirement: OIDC Authentication in E2E Tests @@ -533,6 +535,30 @@ The test suite SHALL verify OIDC integration as part of its standard flow: - WHEN the Kind cluster is created - THEN `make kind-up` SHALL be invoked with `KIND_ENABLE_OIDC=true` +### Requirement: Web Console Distributed Trace Verification + +The CI e2e workflow SHALL verify web console distributed tracing end to end, satisfying `web-console/tracing.spec.md` (`WEB-TRACE-11`). The Kind cluster SHALL be created with tracing enabled (`KIND_JAEGER=true`) so Jaeger is deployed and the web-console BFF exports to it. After the bash suite runs, the workflow SHALL drive a representative gateway workflow through a real browser against the deployed console and assert that Jaeger holds one trace joining the browser and the BFF. The check SHALL use the same Node and Chromium setup as the web-console lint job and SHALL run from the deployed console, not a mocked dev server. The trace check SHALL fail the workflow if no cross-service trace appears within a bounded polling window, and failure diagnostics SHALL include Jaeger workload status and logs and the web-console tracing configuration. + +#### Scenario: Tracing Enabled for E2E + +- GIVEN the CI e2e workflow +- WHEN the Kind cluster is created +- THEN `make kind-up` SHALL be invoked with `KIND_JAEGER=true` +- AND Jaeger SHALL be deployed and the web-console BFF SHALL be configured to export to it + +#### Scenario: Cross-Service Trace Asserted + +- GIVEN the cluster is running with tracing enabled and the console is reachable +- WHEN the trace verification drives a gateway workflow in a real browser and queries Jaeger +- THEN it SHALL find one trace whose spans include a bounded browser workflow span and the BFF server span joined by the same trace identifier +- AND the workflow SHALL fail if no such trace appears within the polling window + +#### Scenario: Trace Failure Diagnostics + +- GIVEN the trace verification fails +- WHEN the workflow reaches its post-test phase +- THEN it SHALL collect Jaeger workload status and logs and the web-console tracing configuration alongside the existing diagnostics + ## Design Decisions | Decision | Rationale | diff --git a/specs/platform/local-development.spec.md b/specs/platform/local-development.spec.md index 410cc58a..de2c21f1 100644 --- a/specs/platform/local-development.spec.md +++ b/specs/platform/local-development.spec.md @@ -291,6 +291,21 @@ Client Networking Gateway Gateway Pod | gRPC hostname pattern | `.gw.localhost` | `-.gw.` | | DNS resolution | CoreDNS container + OS resolver + pfctl/iptables port forwarding | Cluster DNS / external DNS | +### Development Tracing (Jaeger) + +The local environment SHALL deploy a Jaeger all-in-one instance so a developer can view distributed traces produced by the web console and the API server (see `web-console/tracing.spec.md` and HYPERSHELL-26). Jaeger all-in-one exposes an OTLP receiver and a query UI in one workload; a separate OpenTelemetry Collector is not required for local development. Trace storage uses in-memory storage, which is cleared on restart and is appropriate for development only. + +`make kind-up` SHALL deploy Jaeger into the target namespace with its OTLP receiver enabled, and SHALL create an HTTPRoute that exposes the Jaeger UI at `jaeger.hypershell.localhost`. The Jaeger workload SHALL expose both the OTLP/gRPC receiver (`4317`) used by the API server and the OTLP/HTTP receiver (`4318`) used by the web console, because browsers cannot speak OTLP gRPC. The web console BFF SHALL be configured to export traces to the Jaeger OTLP/HTTP endpoint through the `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable on the web-console Deployment. The browser exports through the same-origin BFF telemetry endpoint (see `web-console/tracing.spec.md`), so the browser does not reach Jaeger directly. + +| Setting | Value | +|---------|-------| +| Workload | `jaeger` Deployment (all-in-one, in-memory storage) | +| OTLP receiver (API server) | `jaeger..svc.cluster.local:4317` (OTLP/gRPC) | +| OTLP receiver (web console BFF) | `http://jaeger..svc.cluster.local:4318` (OTLP/HTTP) | +| Query UI | `https://jaeger.hypershell.localhost` | + +Deployment of Jaeger SHALL be gated by the `KIND_JAEGER` environment variable (opt-in; unset or any value other than `true` skips it). When Jaeger is not deployed, `make kind-up` SHALL skip the Jaeger workload and its route and SHALL leave `OTEL_EXPORTER_OTLP_ENDPOINT` unset on the web-console Deployment, so the BFF starts with tracing disabled without failing readiness. + ## Requirements ### Requirement: Single-Command Environment Setup @@ -441,6 +456,7 @@ All services SHALL be accessible via `.localhost` hostnames routed through the n | HTTP API | `https://api.hypershell.localhost` | REST API access | | Web Console | `https://console.hypershell.localhost` | Browser UI | | Health | `https://health.hypershell.localhost` | Health check endpoint | +| Jaeger UI | `https://jaeger.hypershell.localhost` | Distributed tracing UI (when `KIND_JAEGER=true`) | | gRPC | `https://.gw.localhost` | gRPC streaming (control plane, CLI) | The self-signed TLS certificate issued by cert-manager must be trusted by the developer's browser or CLI tool (e.g. `curl --cacert`). @@ -655,6 +671,31 @@ All `kind-*` targets operate on the namespace specified by `KIND_NAMESPACE` (def - THEN the API server SHALL be swapped only in the `hypershell-feature-add-auth` namespace - AND the default deployment SHALL remain unchanged +### Requirement: Development Tracing + +The system SHALL deploy a Jaeger all-in-one instance in the local environment and configure the web console BFF to export traces to it, so a developer can verify one end-to-end trace. Deployment SHALL be gated by `KIND_JAEGER` (opt-in; deployed only when set to `true`). + +#### Scenario: Jaeger Available After Setup +- GIVEN `KIND_JAEGER` is `true` +- WHEN a developer runs `make kind-up` +- THEN a Jaeger all-in-one workload SHALL be deployed into the target namespace +- AND the workload SHALL expose the OTLP/gRPC receiver on `4317` and the OTLP/HTTP receiver on `4318` +- AND an HTTPRoute SHALL expose the Jaeger UI at `jaeger.hypershell.localhost` +- AND the web console BFF SHALL be configured with `OTEL_EXPORTER_OTLP_ENDPOINT` pointing at the Jaeger OTLP/HTTP receiver + +#### Scenario: End-to-End Trace Visible +- GIVEN the local environment is running with tracing enabled +- WHEN a developer completes a gateway workflow in the web console +- THEN one trace SHALL be visible in the Jaeger UI +- AND it SHALL contain the browser workflow span and the BFF server span joined by the same trace identifier + +#### Scenario: Tracing Disabled +- GIVEN `KIND_JAEGER` is unset (or any value other than `true`) +- WHEN a developer runs `make kind-up` +- THEN the Jaeger workload and its route SHALL NOT be deployed +- AND `OTEL_EXPORTER_OTLP_ENDPOINT` SHALL be left unset on the web-console Deployment +- AND the web console BFF SHALL start with tracing disabled without failing readiness + ## Environment Variable Reference | Env Var | Default | Description | @@ -678,6 +719,7 @@ All `kind-*` targets operate on the namespace specified by `KIND_NAMESPACE` (def | `CERT_MANAGER_VERSION` | `v1.21.1` | cert-manager release version | | `KIND_DB_IMAGE` | `registry.access.redhat.com/hi/postgresql:18` | Database image for Gateway resource; override for OSS dev (unsupported) | | `KIND_NAMESPACE` | `hypershell-system` | Target namespace for all `kind-*` targets | +| `KIND_JAEGER` | (unset) | Set to `true` to deploy Jaeger all-in-one for distributed tracing (OTLP gRPC `4317` + HTTP `4318`) and export web console traces to it | ## Make Targets Summary @@ -710,6 +752,7 @@ All targets operate on `KIND_NAMESPACE` (default: `hypershell-system`). | Images loaded via tarball archive | Compatible with both Podman and Docker; avoids registry dependency | | Rebuild-and-replace on every swap call | Each `kind--up` rebuilds from the working tree and replaces the deployment, even if already swapped; developers iterate by re-running the same target | | Web console as first-class component | Node.js frontend (`components/web-console/`) deployed alongside API server and control plane; supports hot reload via `KIND_HOT_RELOAD` for rapid UI iteration | +| Jaeger all-in-one for local tracing | One workload provides both OTLP receivers (gRPC `4317` for the API server, HTTP `4318` for the web console) and the query UI with in-memory storage; avoids a separate OpenTelemetry Collector in development. The BFF exports over OTLP/HTTP; the browser exports through the same-origin BFF endpoint. `KIND_JAEGER=true` opts in | | Hot reload on by default | Swap targets for supported components (web console) mount host source and run a dev server in an interactive TTY by default; `KIND_HOT_RELOAD=false` opts out to rebuild-and-replace. Keeps the same `kind--up` entrypoint for both workflows | | Per-component targets require existing cluster | Avoids implicit full-stack deployment; keeps intent explicit | | Database provisioned by control plane | Gateway configured with HI postgresql image; control plane reconciler provisions the database via GatewayReconciler (`specs/platform/openshell-gateway-database.spec.md`, implemented in [#14](https://github.com/openshift-online/hypershell/pull/14)), exercising the same path as production. The API server's own database (`deploy/kind/postgres.yaml`) is always deployed directly by `kind-up` | diff --git a/specs/web-console/tracing.spec.md b/specs/web-console/tracing.spec.md new file mode 100644 index 00000000..b2a3adba --- /dev/null +++ b/specs/web-console/tracing.spec.md @@ -0,0 +1,148 @@ +# Web Console Distributed Tracing + +**Status:** Active +**Applies to:** `components/web-console` browser application, the web-console BFF, `components/web-console/domain-probes`, the gateway management UI probe catalog, and the local development observability workflow +**Jira:** HYPERSHELL-27 + +## Purpose + +Give the web console distributed tracing that follows one user workflow from the browser, through the BFF, to the HyperShell REST API. Traces are derived from the typed domain probes already defined by `standards/ui/domain-observability.spec.md`; they are not a second, parallel telemetry path. This specification narrows the tracing requirements in `web-console/architecture.spec.md` (`WEB-OBS-01`, `WEB-OBS-02`) and the standard in `standards/ui/domain-observability.spec.md` into a concrete OpenTelemetry (OTel) wiring. It does not replace those documents; where they impose a stricter rule, that rule governs. + +The API server is instrumented separately (HYPERSHELL-26). This specification requires the browser and BFF to produce spans and to propagate W3C Trace Context so the API server can join the same trace once it extracts that context. + +## Requirements + +### Requirement: WEB-TRACE-01 -- Browser Spans Derived From Domain Probes + +The browser SHALL produce spans from the typed domain probes published through the `DomainProbePublisher` fan-out, not from ad hoc instrumentation. A trace sink SHALL implement the `DomainProbeSink` contract and register through the existing `additionalSinks` extension point of the gateway observability adapter. The sink SHALL map a workflow `started` probe and its matching terminal probe (`succeeded`, `failed`, `cancelled`, `denied`, or `conflicted`) to one span, and each dependency-attempt probe pair to one child span. Span status SHALL reflect the terminal outcome. The sink SHALL NOT create spans from React renders, pointer movement, or pure computation. + +The workflow span SHALL be the root of the browser-originated trace. It MAY adopt the caller-chosen trace identifier so the trace joins the id the browser propagates to the BFF and API, but it SHALL NOT descend from a synthetic or remote parent span. A manufactured parent that no service records would leave the exported trace decapitated (a workflow span whose parent is missing from the trace), so the sink SHALL make the workflow span a true root while still adopting the chosen trace identifier. + +Browser application, domain, and API-adapter code SHALL NOT import an OpenTelemetry package. Only the observability adapter and the composition root MAY import OTel, as enforced by `eslint.architecture.mjs`. Experimental browser auto-instrumentation SHALL NOT be a foundational dependency. + +**Verification:** Run each gateway use case with a recording publisher plus the trace sink; assert one workflow span per invocation, one child span per dependency attempt, and a span status that matches the terminal outcome. Confirm no span is emitted for a rerender. Run the architecture lint and confirm an OTel import outside the approved paths fails. + +#### Scenario: Workflow Produces One Span + +- GIVEN a user runs a gateway workflow (for example list, provision, rename, or delete) +- WHEN the workflow publishes its started probe and one terminal probe +- THEN the trace sink SHALL produce exactly one workflow span with a status derived from the terminal outcome +- AND each control-plane dependency attempt SHALL produce one child span + +### Requirement: WEB-TRACE-02 -- Same-Origin Browser Telemetry Export + +The browser SHALL export spans to a same-origin BFF telemetry endpoint using OTLP over HTTP. The browser SHALL NOT export directly to a collector origin and SHALL NOT require a Content Security Policy `connect-src` value other than `'self'`. The BFF SHALL expose the ingest endpoint, apply a request-body size limit, require the same session and CSRF protection as other state-changing browser requests, and forward accepted spans to the configured collector. The endpoint SHALL reject a body that is not valid OTLP. + +**Verification:** Load the built console behind the BFF with the enforcing CSP; confirm the browser exporter posts to the same-origin path and that no cross-origin telemetry request is attempted. Post an oversized and a malformed body and confirm rejection. Confirm the BFF forwards a well-formed body to the collector. + +#### Scenario: Browser Exports Through the BFF + +- GIVEN the console has produced one or more spans +- WHEN the browser exporter flushes +- THEN it SHALL post OTLP data to the same-origin BFF telemetry endpoint +- AND the BFF SHALL forward the accepted spans to the configured collector +- AND no telemetry request SHALL go to a cross-origin destination + +### Requirement: WEB-TRACE-03 -- Browser Outbound Trace Context + +The browser SHALL set the W3C `traceparent` header, and `tracestate` when present, on every outbound `/api/*` request at the single correlated-fetch chokepoint. The `traceparent` value SHALL reference the active workflow span so the BFF and API spans join the same trace. The browser SHALL keep sending the existing `x-hypershell-correlation-id` header. The trace context and the correlation identifier SHALL both appear in the `DomainProbeContext` so probe consumers can join a workflow to its trace. + +**Verification:** Run a representative browser-to-API workflow; capture the outbound request and assert a well-formed `traceparent` that references the workflow span, plus the correlation header. Confirm the probe context carries the same trace identifier. + +### Requirement: WEB-TRACE-04 -- BFF W3C Trace Context Propagation + +The BFF SHALL treat an inbound `traceparent`/`tracestate` pair as untrusted input. It SHALL extract valid context, continue that trace, and set a valid `traceparent` (and forward a valid `tracestate`) on the upstream API request. When the inbound context is absent or malformed, the BFF SHALL start a new trace and set a valid `traceparent`; it SHALL NOT forward a malformed value. The upstream request header allowlist SHALL include `traceparent` and `tracestate`. The BFF SHALL keep validating, echoing, and propagating `x-hypershell-correlation-id` as it does today. + +**Verification:** Send requests with valid, absent, and malformed inbound trace context; assert the upstream request always carries a valid `traceparent`, a malformed value is replaced rather than forwarded, and the correlation identifier is preserved. Confirm `traceparent` and `tracestate` are in the upstream allowlist. + +#### Scenario: BFF Continues an Inbound Trace + +- GIVEN a browser request arrives with a valid `traceparent` +- WHEN the BFF proxies the request to the API +- THEN the upstream request SHALL carry a `traceparent` that continues the same trace +- AND a valid inbound `tracestate` SHALL be forwarded + +#### Scenario: BFF Replaces Malformed Context + +- GIVEN a browser request arrives with a malformed `traceparent` +- WHEN the BFF proxies the request to the API +- THEN the BFF SHALL start a new trace and set a valid `traceparent` +- AND it SHALL NOT forward the malformed value + +### Requirement: WEB-TRACE-05 -- BFF Server Spans + +The BFF SHALL produce one server span per proxied `/api/*` request as a child of the extracted inbound context, and export it by OTLP to the configured collector. The BFF OTel SDK SHALL be initialized once in the server bootstrap path, which is exempt from the observability import ban. Product and route code SHALL NOT call the OTel API directly; only the observability adapter, the composition root, and the bootstrap MAY. Span attributes SHALL record the templated route, the request method, the upstream outcome class, and the correlation identifier. The BFF SHALL keep its existing structured request log. + +The span name SHALL combine the request method and the templated route (for example `GET /api/hypershell/v1/gateways/{id}`), so Jaeger groups spans by endpoint. The catch-all proxy pattern (for example `/api/*`) SHALL NOT be the span name or the `http.route` value. The templated route SHALL collapse every resource identifier to a bounded placeholder and SHALL NOT carry a query string, so cardinality stays fixed per `WEB-TRACE-07`. + +**Verification:** Proxy a representative request and confirm one BFF server span that is a child of the inbound context, carries the templated-route and outcome attributes, is named by method and templated route with resource identifiers collapsed, and reaches the collector. Confirm route and product code contain no direct OTel API calls. + +### Requirement: WEB-TRACE-06 -- Configurable Export and Sampling + +The collector endpoint and the trace sample rate SHALL be deployment configuration, validated at BFF startup through the existing configuration schema. Configuration SHALL be separate from code. When tracing configuration is absent, the BFF SHALL start normally with tracing disabled and SHALL NOT fail readiness. A browser-visible value SHALL be limited to the non-sensitive, allowlisted schema required by `WEB-DEPLOY-02`; a collector origin or secret SHALL NOT be embedded in the browser bundle. + +**Verification:** Start the BFF with valid, absent, and invalid tracing configuration; confirm valid configuration exports to the collector, absent configuration disables tracing without failing readiness, and invalid configuration fails startup with a clear message. Inspect the browser bundle for an embedded collector origin or secret. + +### Requirement: WEB-TRACE-07 -- Trace Privacy and Cardinality + +Spans and their attributes SHALL follow the privacy and cardinality rules of `standards/ui/domain-observability.spec.md` (`UI-OBS-07`). Tokens, cookies, credentials, secrets, raw headers, request or response bodies, user-entered content, and raw resource identifiers in span names SHALL be prohibited. Stable route templates, bounded enums, and error classes SHALL replace raw URLs and messages. A high-cardinality identifier SHALL NOT become a span name segment or propagated baggage. When an API failure returns an operation identifier, the matching workflow and dependency terminal spans SHALL retain it so support can join a user-visible failure to server evidence. + +**Verification:** Run redaction tests with seeded secrets and user data across the browser sink and the BFF span exporter. Confirm span names use route templates, attributes are bounded, no prohibited value appears, and an API operation identifier is retained on failure spans. + +### Requirement: WEB-TRACE-08 -- Trace Consumer in the Probe Catalog + +The gateway probe catalog SHALL declare `trace` as an allowed consumer for every probe the trace sink reads, and the observability import allowlist SHALL permit the trace sink to consume those probes. The catalog SHALL remain in agreement with the typed probe union and the sink mappings. A probe that no consumer reads and a duplicate probe name SHALL NOT be introduced. + +**Verification:** Compare the catalog with the probe union and the trace sink mapping; confirm every probe the sink reads lists `trace` in `allowedConsumers`, and that catalog and code agree in CI. + +### Requirement: WEB-TRACE-09 -- Bounded Delivery and Flush + +The browser trace sink and the BFF exporter SHALL each own a bounded buffer and an explicit flush policy; they SHALL NOT grow without limit. The browser SHALL flush on page hide and visibility change so spans are not lost on navigation or tab close. A tracing export failure SHALL be best-effort: it SHALL NOT change a workflow result and SHALL surface through the existing bounded delivery-failure diagnostic rather than a recursive publication. Fan-out SHALL still attempt every other sink when the trace sink fails. + +**Verification:** Inject a trace sink and an exporter that throw, block, and overflow; confirm other sinks still receive the probe, the workflow result is unchanged, memory stays bounded, and the failure is visible without recursion. Hide the page mid-workflow and confirm buffered spans flush. + +### Requirement: WEB-TRACE-10 -- End-to-End Trace in Development + +The local development environment SHALL provide a collector and a Jaeger instance so a developer can view one trace that joins the browser and the BFF (and the API server once it is instrumented). The development collector endpoint SHALL be supplied to the BFF by configuration. The details of the development deployment are defined in `platform/local-development.spec.md`. + +**Verification:** Run the local environment, complete a representative gateway workflow in the browser, and confirm one trace in Jaeger that contains the browser workflow span and the BFF server span joined by W3C Trace Context. + +#### Scenario: Developer Views the Trace + +- GIVEN the local development environment is running with the collector and Jaeger +- WHEN a developer completes a gateway workflow in the browser +- THEN one trace SHALL be visible in Jaeger +- AND it SHALL contain the browser workflow span and the BFF server span joined by the same trace identifier + +### Requirement: WEB-TRACE-11 -- Automated End-to-End Trace Verification + +The end-to-end trace defined by `WEB-TRACE-10` SHALL be verified automatically, not only by manual inspection. The CI end-to-end workflow SHALL bring the cluster up with tracing enabled, drive a representative gateway workflow through the deployed browser console, and assert that Jaeger holds one trace joining the browser and the BFF. The check SHALL confirm the browser spans use the bounded workflow and dependency span names and share a single trace identifier with the BFF server span. A missing trace, a browser-only trace, or a BFF-only trace SHALL fail the workflow. The verification details and its place in the e2e suite are defined in `platform/e2e-testing.spec.md`. + +**Verification:** Run the e2e workflow against a cluster with tracing enabled; confirm the trace check drives a browser workflow, finds a cross-service trace in Jaeger, and fails when no such trace is present. + +#### Scenario: CI Verifies the Cross-Service Trace + +- GIVEN the e2e cluster is running with tracing enabled and the console is reachable +- WHEN the trace verification drives a gateway workflow in a real browser +- THEN it SHALL find one trace in Jaeger whose spans include a bounded browser workflow span and the BFF server span +- AND those spans SHALL share the same trace identifier +- AND the workflow SHALL fail if no such cross-service trace appears within a bounded polling window + +## Design Decisions + +| Decision | Rationale | +| --- | --- | +| Derive spans from domain probes, not auto-instrumentation | Keeps one authoritative telemetry model, satisfies the domain-observability standard, and avoids the bundle, privacy, and stability risk that `WEB-OBS-02` cautions against | +| Browser exports through a same-origin BFF endpoint | Keeps the collector origin out of the browser, works with the strict `connect-src 'self'` CSP, and reuses the BFF session and CSRF controls | +| BFF owns W3C Trace Context propagation | The BFF is the trust boundary; it validates or replaces untrusted inbound context and sets a valid value upstream so the API can join the trace | +| Tracing configuration is optional and validated at startup | Configuration is separate from code; a deployment without a collector starts normally with tracing disabled | +| Trace is a probe consumer in the catalog | The catalog stays the single source of truth for who consumes each probe, per `UI-OBS-10` | + +## Primary Basis + +- `standards/ui/domain-observability.spec.md` +- `web-console/architecture.spec.md` (`WEB-OBS-01`, `WEB-OBS-02`, `WEB-BFF-01`, `WEB-DEPLOY-02`, `WEB-SEC-01`) +- [OpenTelemetry JavaScript](https://opentelemetry.io/docs/languages/js/) +- [OpenTelemetry semantic conventions](https://opentelemetry.io/docs/concepts/semantic-conventions/) +- [W3C Trace Context](https://www.w3.org/TR/trace-context/) +- [OTLP specification](https://opentelemetry.io/docs/specs/otlp/)