[HYPERSHELL-107] feat(web-console): distributed tracing for browser + BFF

.github/workflows/e2e.yml

@@ -239,7 +239,10 @@ jobs:
     needs: resolve-images
     if: needs.resolve-images.outputs.should_run == 'true'
     runs-on: ubuntu-24.04
-    timeout-minutes: 20
+    # The browser-driven trace verification adds a Node install, a Chromium
+    # download, and a Playwright run on top of the bash suite, so allow a wider
+    # ceiling than the 20 minutes the bash-only suite needed.
+    timeout-minutes: 25
     steps:
       - name: Check out repository
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -266,6 +269,9 @@ jobs:
           CONTROL_PLANE_IMAGE: ${{ needs.resolve-images.outputs.control_plane_image }}
           WEB_CONSOLE_IMAGE: ${{ needs.resolve-images.outputs.web_console_image }}
           KIND_ENABLE_OIDC: "true"
+          # Deploy Jaeger and point the web-console BFF at it so the browser
+          # trace verification below has a collector to export to (WEB-TRACE-10).
+          KIND_JAEGER: "true"
         run: make kind-up
 
       - name: Run e2e tests
@@ -280,6 +286,61 @@ jobs:
           NO_COLOR: "1"
         run: bash tests/e2e/e2e-openshell.sh
 
+      # Browser-driven distributed-trace verification (WEB-TRACE-10): drive a
+      # real gateway workflow in the deployed console, then assert one trace in
+      # Jaeger joins the browser and the BFF. Reuses the same Node + Chromium
+      # setup the web-console lint job uses.
+      #
+      # These steps run on pull_request, push, and workflow_dispatch -- NOT
+      # merge_group. resolve-images maps an ephemeral merge-queue commit to
+      # on-pr-<merge_sha>, which Konflux never builds, so merge_group always
+      # falls back to the baseline web-console image. That baseline lags main
+      # and cannot contain not-yet-merged tracing code, so the deployed console
+      # emits no spans and the assertion would test the wrong artifact. The
+      # trace path is gated at pull_request time against the freshly built PR
+      # image and re-verified on push to main against the post-merge image.
+      - name: Set up Node.js
+        if: github.event_name != 'merge_group'
+        uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0
+        with:
+          node-version-file: .node-version
+      - name: Install pinned pnpm
+        if: github.event_name != 'merge_group'
+        run: bash scripts/bootstrap_pnpm.sh
+      - name: Install dependencies
+        if: github.event_name != 'merge_group'
+        run: pnpm install --frozen-lockfile
+      - name: Get Playwright version
+        id: playwright-version
+        if: github.event_name != 'merge_group'
+        run: echo "version=$(pnpm --filter @openshift-online/hypershell-web-console exec playwright --version | awk '{print $2}')" >> "$GITHUB_OUTPUT"
+      # Install Chromium without `--with-deps` so the flaky apt phase
+      # (actions/runner-images#11347) never lands on the critical path -- the
+      # ubuntu-24.04 runner image already ships Chromium's system libraries.
+      # Cache the browser download too, keyed on the resolved Playwright
+      # version; only the default branch writes the cache, so PR runs cannot
+      # poison it.
+      - name: Restore Chromium cache
+        id: playwright-chromium-cache
+        if: github.event_name != 'merge_group'
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-chromium-${{ runner.os }}-${{ steps.playwright-version.outputs.version }}
+      - name: Install Chromium
+        if: github.event_name != 'merge_group'
+        timeout-minutes: 5
+        run: pnpm --filter @openshift-online/hypershell-web-console exec playwright install chromium
+      - name: Save Chromium cache
+        if: github.event_name != 'merge_group' && github.ref == 'refs/heads/main' && steps.playwright-chromium-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ steps.playwright-chromium-cache.outputs.cache-primary-key }}
+      - name: Verify end-to-end traces reach Jaeger
+        if: github.event_name != 'merge_group'
+        run: pnpm --filter @openshift-online/hypershell-web-console test:e2e:live
+
       - name: Collect diagnostics
         if: failure()
         run: |
@@ -305,6 +366,16 @@ jobs:
           kubectl logs --all-containers --prefix --tail=100 -l app=keycloak -n keycloak 2>&1 | tee e2e-diagnostics/keycloak-logs.txt || true
           echo "::endgroup::"
 
+          echo "::group::Tracing (Jaeger) diagnostics"
+          {
+            kubectl get pods,svc,httproutes -l app=jaeger -n hypershell-system -o wide 2>&1 || true
+            kubectl logs --all-containers --prefix --tail=100 -l app=jaeger -n hypershell-system 2>&1 || true
+            echo "--- web-console OTEL env ---"
+            kubectl get deployment/hypershell-web-console -n hypershell-system \
+              -o jsonpath='{range .spec.template.spec.containers[*].env[*]}{.name}={.value}{"\n"}{end}' 2>&1 | grep -i otel || true
+          } | tee e2e-diagnostics/tracing.txt
+          echo "::endgroup::"
+
           echo "::group::Events (hypershell-system)"
           kubectl get events --sort-by=.lastTimestamp -n hypershell-system 2>&1 | tee e2e-diagnostics/events.txt || true
           echo "::endgroup::"

.github/workflows/lint.yml

@@ -218,15 +218,30 @@ jobs:
         run: pnpm install --frozen-lockfile
       - name: Run static, unit, Storybook, and production build gates
         run: pnpm check:web
+      - name: Get Playwright version
+        id: playwright-version
+        run: echo "version=$(pnpm --filter @openshift-online/hypershell-web-console exec playwright --version | awk '{print $2}')" >> "$GITHUB_OUTPUT"
+      # Install Chromium without `--with-deps` so the flaky apt phase
+      # (actions/runner-images#11347) never lands on the critical path -- the
+      # ubuntu-24.04 runner image already ships Chromium's system libraries.
+      # Cache the browser download too, keyed on the resolved Playwright
+      # version; only the default branch writes the cache, so PR runs cannot
+      # poison it.
+      - name: Restore Chromium cache
+        id: playwright-chromium-cache
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-chromium-${{ runner.os }}-${{ steps.playwright-version.outputs.version }}
       - name: Install Chromium
-        run: |
-          # Cap apt network timeouts so a flaky Azure mirror doesn't hang the job.
-          sudo tee /etc/apt/apt.conf.d/99-timeout <<'APT'
-          Acquire::Retries "3";
-          Acquire::http::Timeout "30";
-          Acquire::https::Timeout "30";
-          APT
-          pnpm --filter @openshift-online/hypershell-web-console exec playwright install --with-deps chromium
+        timeout-minutes: 5
+        run: pnpm --filter @openshift-online/hypershell-web-console exec playwright install chromium
+      - name: Save Chromium cache
+        if: github.ref == 'refs/heads/main' && steps.playwright-chromium-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ steps.playwright-chromium-cache.outputs.cache-primary-key }}
       - name: Run critical Chromium journey
         run: pnpm test:e2e:chromium
 

Makefile

@@ -407,3 +407,14 @@ e2e:
 		E2E_PROVISION_TIMEOUT=300 \
 		E2E_SANDBOX_TIMEOUT=180 \
 		bash tests/e2e/e2e-openshell.sh
+
+# Browser-driven end-to-end trace verification (WEB-TRACE-10). Requires a Kind
+# cluster brought up with tracing enabled (KIND_JAEGER=true make kind-up), so
+# Jaeger is deployed and the web-console BFF exports to it.
+.PHONY: e2e-tracing
+e2e-tracing:
+	@echo ""
+	@echo "==> Verifying end-to-end traces reach Jaeger (Kind)"
+	@echo "    (requires: KIND_JAEGER=true make kind-up)"
+	@echo ""
+	@pnpm --filter @openshift-online/hypershell-web-console test:e2e:live

components/web-console/app/adapters/api/api.client.test.ts

@@ -76,6 +76,51 @@ describe("correlated API fetch", () => {
     expect(onReauth).not.toHaveBeenCalled();
     expect(response.status).toBe(401);
   });
+
+  it("propagates the W3C trace context supplied by the provider", async () => {
+    const fetchImplementation = vi
+      .fn<typeof globalThis.fetch>()
+      .mockResolvedValue(new Response(null, { status: 204 }));
+    const correlatedFetch = createCorrelatedFetch(
+      "44444444-4444-4444-8444-444444444444",
+      fetchImplementation,
+      undefined,
+      () => ({
+        traceparent: "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
+        tracestate: "hypershell=1",
+      }),
+    );
+
+    await correlatedFetch("/api/hypershell/v1/gateways");
+
+    const headers = new Headers(
+      fetchImplementation.mock.calls[0]?.[1]?.headers,
+    );
+    expect(headers.get("traceparent")).toBe(
+      "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
+    );
+    expect(headers.get("tracestate")).toBe("hypershell=1");
+  });
+
+  it("omits trace headers when the provider reports no active span", async () => {
+    const fetchImplementation = vi
+      .fn<typeof globalThis.fetch>()
+      .mockResolvedValue(new Response(null, { status: 204 }));
+    const correlatedFetch = createCorrelatedFetch(
+      "55555555-5555-4555-8555-555555555555",
+      fetchImplementation,
+      undefined,
+      () => undefined,
+    );
+
+    await correlatedFetch("/api/hypershell/v1/gateways");
+
+    const headers = new Headers(
+      fetchImplementation.mock.calls[0]?.[1]?.headers,
+    );
+    expect(headers.has("traceparent")).toBe(false);
+    expect(headers.has("tracestate")).toBe(false);
+  });
 });
 
 describe("redirectToLogin", () => {

components/web-console/app/adapters/api/api.client.ts

@@ -2,6 +2,19 @@ import { SDKClient } from "@openshift-online/hypershell-sdk";
 
 export const gatewayCorrelationHeader = "x-hypershell-correlation-id";
 
+/** W3C Trace Context headers a request carries to the BFF for propagation. */
+export interface RequestTraceContext {
+  traceparent: string;
+  tracestate?: string;
+}
+
+/**
+ * Supplies the W3C trace context for the in-flight workflow, or `undefined`
+ * when tracing is disabled or no span is active. Injected so the API adapter
+ * stays free of any tracing vendor dependency.
+ */
+export type TraceContextProvider = () => RequestTraceContext | undefined;
+
 /** The BFF's machine-readable request to restart authentication at the IdP. */
 export interface ReauthSignal {
   loginUrl: string;
@@ -71,10 +84,18 @@ export function createCorrelatedFetch(
   correlationId: string,
   fetchImplementation: typeof globalThis.fetch = globalThis.fetch,
   onReauthRequired?: ReauthHandler,
+  traceContext?: TraceContextProvider,
 ): typeof globalThis.fetch {
   return async (input, init) => {
     const headers = new Headers(init?.headers);
     headers.set(gatewayCorrelationHeader, correlationId);
+    const trace = traceContext?.();
+    if (trace !== undefined) {
+      headers.set("traceparent", trace.traceparent);
+      if (trace.tracestate !== undefined && trace.tracestate !== "") {
+        headers.set("tracestate", trace.tracestate);
+      }
+    }
     const response = await fetchImplementation(input, { ...init, headers });
     if (onReauthRequired) {
       const signal = await readReauthSignal(response);
@@ -94,6 +115,7 @@ export function createCorrelatedFetch(
 export function createApiClient(
   correlationId: string,
   onReauthRequired: ReauthHandler = redirectToLogin,
+  traceContext?: TraceContextProvider,
 ): SDKClient {
   return new SDKClient({
     baseUrl: "",
@@ -102,6 +124,7 @@ export function createApiClient(
       correlationId,
       globalThis.fetch,
       onReauthRequired,
+      traceContext,
     ),
   });
 }

components/web-console/app/adapters/observability/gateway-observability.test.ts

@@ -48,14 +48,51 @@ describe("gateway observability adapter", () => {
     ]);
   });
 
+  it("records an out-of-band delivery failure into delivery health", () => {
+    const observability = createGatewayObservability({
+      performanceTarget: { clearMarks: vi.fn(), mark: vi.fn() },
+    });
+
+    observability.reportDeliveryFailure({
+      errorType: "SpanExportError",
+      probeName: "gateway.trace.export",
+      schemaVersion: 0,
+      sinkId: "gateway-trace",
+    });
+
+    expect(observability.deliveryHealth()).toMatchObject({
+      deliveryFailureCount: 1,
+      lastFailure: { sinkId: "gateway-trace" },
+    });
+    expect(observability.recentDeliveryFailures()).toEqual([
+      expect.objectContaining({ probeName: "gateway.trace.export" }),
+    ]);
+  });
+
   it("provides deterministic workflow context through injected capabilities", () => {
     const observability = createGatewayObservability({
       createCorrelationId: () => "correlation-1",
+      createTraceId: () => "0af7651916cd43dd8448eb211c80319c",
       now: () => "2026-08-06T18:00:00.000Z",
       performanceTarget: { clearMarks: vi.fn(), mark: vi.fn() },
     });
 
     expect(observability.runtime.createCorrelationId()).toBe("correlation-1");
+    expect(observability.runtime.createTraceId()).toBe(
+      "0af7651916cd43dd8448eb211c80319c",
+    );
     expect(observability.runtime.now()).toBe("2026-08-06T18:00:00.000Z");
   });
+
+  it("generates a valid W3C trace identifier by default", () => {
+    const observability = createGatewayObservability({
+      performanceTarget: { clearMarks: vi.fn(), mark: vi.fn() },
+    });
+
+    const traceId = observability.runtime.createTraceId();
+
+    expect(traceId).toMatch(/^[0-9a-f]{32}$/);
+    expect(traceId).not.toBe("0".repeat(32));
+    expect(observability.runtime.createTraceId()).not.toBe(traceId);
+  });
 });

components/web-console/app/adapters/observability/gateway-observability.ts

@@ -13,6 +13,20 @@ import {
 
 const recentProbeLimit = 100;
 const recentFailureLimit = 20;
+const traceIdByteLength = 16;
+
+/**
+ * Creates a W3C trace identifier: a 16-byte random value rendered as 32
+ * lowercase hex digits. A random 16-byte value is never the all-zero value
+ * that the W3C Trace Context specification forbids.
+ */
+function createTraceId(): string {
+  const bytes = new Uint8Array(traceIdByteLength);
+  globalThis.crypto.getRandomValues(bytes);
+  return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(
+    "",
+  );
+}
 
 interface PerformanceProbeTarget {
   clearMarks(name?: string): void;
@@ -24,12 +38,17 @@ export interface GatewayObservability {
   probes: GatewayProbePublisher;
   recentDeliveryFailures(): readonly Readonly<ProbeDeliveryFailure>[];
   recentProbes(): readonly GatewayProbe[];
+  // Records a span delivery failure that surfaces asynchronously, outside the
+  // synchronous probe fan-out, so an export the browser could not complete is
+  // counted in delivery health instead of being lost.
+  reportDeliveryFailure(failure: Readonly<ProbeDeliveryFailure>): void;
   runtime: GatewayWorkflowRuntime;
 }
 
 export interface GatewayObservabilityOptions {
   additionalSinks?: readonly DomainProbeSink<GatewayProbe>[];
   createCorrelationId?: () => string;
+  createTraceId?: () => string;
   now?: () => string;
   performanceTarget?: PerformanceProbeTarget;
 }
@@ -85,12 +104,14 @@ export function createGatewayObservability(
     probes: publisher,
     recentDeliveryFailures: () => Object.freeze([...failures]),
     recentProbes: () => Object.freeze([...recent]),
+    reportDeliveryFailure: (failure) => {
+      publisher.reportDeliveryFailure(failure);
+    },
     runtime: {
       createCorrelationId:
         options.createCorrelationId ?? (() => globalThis.crypto.randomUUID()),
+      createTraceId: options.createTraceId ?? createTraceId,
       now: options.now ?? (() => new Date().toISOString()),
     },
   };
 }
-
-export const gatewayObservability = createGatewayObservability();