From e6dc2bbb2d56fea2c0c956a7115f735f6af440ea Mon Sep 17 00:00:00 2001 From: Brian McKenna Date: Sun, 3 May 2026 07:19:52 +0000 Subject: [PATCH] Fix case-sensitive HTTP headers in W3C traces When a header did not match "traceparent" case-sensitively, a new trace ID would be generated and used, severing the tie to other services. This commit introduces a value type which normalises header name case, for comparison. Before: curl -v -H 'Traceparent: 00-aaaa...-bbbb...-01' http://:5003/api/validator/readyz # traceparent: 00-cccc...-dddd...-03 curl -H 'traceparent: 00-aaaa...-bbbb...-01' http://:5003/api/validator/readyz # traceparent: 00-aaaa...-bbbb...-01 After: curl -v -H 'Traceparent: 00-aaaa...-bbbb...-01' http://:5003/api/validator/readyz # traceparent: 00-aaaa...-bbbb...-01 curl -H 'traceparent: 00-aaaa...-bbbb...-01' http://:5003/api/validator/readyz # traceparent: 00-aaaa...-bbbb...-01 --- .../admin/api/TraceContextDirectives.scala | 3 +- .../api/client/TraceContextPropagation.scala | 6 ++-- .../splice/http/HttpClient.scala | 4 +-- .../jsonapi/HttpServiceUserFixture.scala | 2 +- .../canton/http/json/v2/Endpoints.scala | 6 ++-- .../canton/http/json/v2/V2Routes.scala | 6 ++-- .../canton/tracing/HeaderName.scala | 21 ++++++++++++ .../canton/tracing/W3CTraceContext.scala | 32 ++++++++++--------- 8 files changed, 53 insertions(+), 27 deletions(-) create mode 100644 canton/community/util-observability/src/main/scala/com/digitalasset/canton/tracing/HeaderName.scala diff --git a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/admin/api/TraceContextDirectives.scala b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/admin/api/TraceContextDirectives.scala index c5ca27b24f..a25b1fd1a8 100644 --- a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/admin/api/TraceContextDirectives.scala +++ b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/admin/api/TraceContextDirectives.scala @@ -6,6 +6,7 @@ package org.lfdecentralizedtrust.splice.admin.api import org.apache.pekko.http.scaladsl.server import org.apache.pekko.http.scaladsl.server.{Directive, Directive1, RequestContext, RouteResult} import org.lfdecentralizedtrust.splice.admin.api.client.TraceContextPropagation.* +import com.digitalasset.canton.tracing.HeaderName import com.digitalasset.canton.tracing.TraceContext import com.digitalasset.canton.tracing.W3CTraceContext @@ -21,7 +22,7 @@ object TraceContextDirectives { def withTraceContext: Directive1[TraceContext] = { Directive { inner => ctx => val headersMap = - ctx.request.headers.map(h => h.name -> h.value).toMap + ctx.request.headers.map(h => HeaderName(h.name) -> h.value).toMap W3CTraceContext .fromHeaders(headersMap) diff --git a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/admin/api/client/TraceContextPropagation.scala b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/admin/api/client/TraceContextPropagation.scala index 2ad896ecf5..b681ee9e80 100644 --- a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/admin/api/client/TraceContextPropagation.scala +++ b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/admin/api/client/TraceContextPropagation.scala @@ -5,7 +5,7 @@ package org.lfdecentralizedtrust.splice.admin.api.client import org.apache.pekko.http.scaladsl.model.HttpHeader import org.apache.pekko.http.scaladsl.model.headers.RawHeader -import com.digitalasset.canton.tracing.TraceContext +import com.digitalasset.canton.tracing.{HeaderName, TraceContext} object TraceContextPropagation { implicit class TraceContextExtension(tc: TraceContext) { @@ -16,9 +16,9 @@ object TraceContextPropagation { .map { w3Ctx => val headersMap = w3Ctx.asHeaders val w3CtxHeaders = headersMap.map { case (name, value) => - RawHeader(name, value) + RawHeader(name.value, value) } - headers.filterNot(h => headersMap.contains(h.name)) ++ w3CtxHeaders.toSeq + headers.filterNot(h => headersMap.contains(HeaderName(h.name))) ++ w3CtxHeaders.toSeq } .getOrElse(headers) } diff --git a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/http/HttpClient.scala b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/http/HttpClient.scala index 157d0d7ba9..c8eff10a11 100644 --- a/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/http/HttpClient.scala +++ b/apps/common/src/main/scala/org/lfdecentralizedtrust/splice/http/HttpClient.scala @@ -8,7 +8,7 @@ import com.daml.metrics.api.MetricQualification.Latency import com.daml.metrics.api.{MetricHandle, MetricInfo, MetricName, MetricsContext} import com.digitalasset.canton.config.{ApiLoggingConfig, NonNegativeDuration} import com.digitalasset.canton.logging.{NamedLoggerFactory, TracedLogger} -import com.digitalasset.canton.tracing.{TraceContext, W3CTraceContext} +import com.digitalasset.canton.tracing.{HeaderName, TraceContext, W3CTraceContext} import org.apache.pekko.actor.ActorSystem import org.apache.pekko.http.scaladsl.{ClientTransport, ConnectionContext, Http} import org.apache.pekko.http.scaladsl.model.{ @@ -281,7 +281,7 @@ object HttpClient { private def traceContextFromHeaders(headers: immutable.Seq[HttpHeader]) = { W3CTraceContext - .fromHeaders(headers.map(h => h.name() -> h.value()).toMap) + .fromHeaders(headers.map(h => HeaderName(h.name()) -> h.value()).toMap) .map(_.toTraceContext) .getOrElse(TraceContext.empty) } diff --git a/canton/community/app/src/test/scala/com/digitalasset/canton/integration/tests/jsonapi/HttpServiceUserFixture.scala b/canton/community/app/src/test/scala/com/digitalasset/canton/integration/tests/jsonapi/HttpServiceUserFixture.scala index 01723dad99..f28df265e0 100644 --- a/canton/community/app/src/test/scala/com/digitalasset/canton/integration/tests/jsonapi/HttpServiceUserFixture.scala +++ b/canton/community/app/src/test/scala/com/digitalasset/canton/integration/tests/jsonapi/HttpServiceUserFixture.scala @@ -43,7 +43,7 @@ trait HttpServiceUserFixture extends PekkoBeforeAndAfterAll { this: Suite with C } protected def extractHeaders(w3cContext: W3CTraceContext): Seq[HttpHeader] = - w3cContext.asHeaders.toSeq.map(h => HttpHeader.parse(h._1, h._2)).collect { + w3cContext.asHeaders.toSeq.map(h => HttpHeader.parse(h._1.value, h._2)).collect { case ParsingResult.Ok(header, _) => header } diff --git a/canton/community/ledger/ledger-json-api/src/main/scala/com/digitalasset/canton/http/json/v2/Endpoints.scala b/canton/community/ledger/ledger-json-api/src/main/scala/com/digitalasset/canton/http/json/v2/Endpoints.scala index 757fe2b274..2ee846574e 100644 --- a/canton/community/ledger/ledger-json-api/src/main/scala/com/digitalasset/canton/http/json/v2/Endpoints.scala +++ b/canton/community/ledger/ledger-json-api/src/main/scala/com/digitalasset/canton/http/json/v2/Endpoints.scala @@ -15,7 +15,7 @@ import com.digitalasset.canton.ledger.error.{JsonApiErrors, LedgerApiErrors} import com.digitalasset.canton.logging.audit.ApiRequestLogger import com.digitalasset.canton.logging.{LoggingContextWithTrace, NamedLogging} import com.digitalasset.canton.networking.grpc.CallMetadata -import com.digitalasset.canton.tracing.{TraceContext, W3CTraceContext} +import com.digitalasset.canton.tracing.{HeaderName, TraceContext, W3CTraceContext} import com.digitalasset.daml.lf.data.Ref import com.digitalasset.daml.lf.engine.Error.Preprocessing import com.digitalasset.daml.lf.language.Ast.TVar @@ -465,7 +465,9 @@ object Endpoints { ) ) .map { case (headersList: Seq[Header], addr) => - val z = W3CTraceContext.fromHeaders(headersList.map(h => (h.name, h.value)).toMap) + val z = W3CTraceContext.fromHeaders( + headersList.map(h => (HeaderName(h.name), h.value)).toMap + ) (z.map(_.toTraceContext), addr) } { case (tc1, addr) => ( diff --git a/canton/community/ledger/ledger-json-api/src/main/scala/com/digitalasset/canton/http/json/v2/V2Routes.scala b/canton/community/ledger/ledger-json-api/src/main/scala/com/digitalasset/canton/http/json/v2/V2Routes.scala index 463bf91813..f68997df4c 100644 --- a/canton/community/ledger/ledger-json-api/src/main/scala/com/digitalasset/canton/http/json/v2/V2Routes.scala +++ b/canton/community/ledger/ledger-json-api/src/main/scala/com/digitalasset/canton/http/json/v2/V2Routes.scala @@ -16,7 +16,7 @@ import com.digitalasset.canton.logging.audit.{ApiRequestLogger, ResponseKind, Tr import com.digitalasset.canton.logging.{NamedLoggerFactory, NamedLogging} import com.digitalasset.canton.networking.grpc.CallMetadata import com.digitalasset.canton.platform.PackagePreferenceBackend -import com.digitalasset.canton.tracing.{TraceContext, W3CTraceContext} +import com.digitalasset.canton.tracing.{HeaderName, TraceContext, W3CTraceContext} import org.apache.pekko.http.scaladsl.model.{AttributeKeys, HttpRequest, MediaType, MediaTypes} import org.apache.pekko.http.scaladsl.server.RequestContext import org.apache.pekko.stream.Materializer @@ -193,7 +193,7 @@ class RequestInterceptors( def loggingInterceptor() = RequestInterceptor.transformServerRequest { request => - val incomingHeaders = request.headers.map(h => (h.name, h.value)).toMap + val incomingHeaders = request.headers.map(h => (HeaderName(h.name), h.value)).toMap val extractedW3cTrace = W3CTraceContext.fromHeaders(incomingHeaders) val requestParameters = s"[${request.queryParameters.toSeq.map { case (k, v) => s"$k=$v" }.mkString(", ")}]" @@ -289,7 +289,7 @@ class RequestInterceptors( def apply[B](request: ServerRequest, result: RequestResult[B]): Future[RequestResult[B]] = { val addr = RequestInterceptorsUtil.extractAddress(request) - val incomingHeaders = request.headers.map(h => (h.name, h.value)).toMap + val incomingHeaders = request.headers.map(h => (HeaderName(h.name), h.value)).toMap val extractedW3cTrace = W3CTraceContext.fromHeaders(incomingHeaders) val callMetadata = CallMetadata( apiEndpoint = request.showShort, diff --git a/canton/community/util-observability/src/main/scala/com/digitalasset/canton/tracing/HeaderName.scala b/canton/community/util-observability/src/main/scala/com/digitalasset/canton/tracing/HeaderName.scala new file mode 100644 index 0000000000..889e38e19e --- /dev/null +++ b/canton/community/util-observability/src/main/scala/com/digitalasset/canton/tracing/HeaderName.scala @@ -0,0 +1,21 @@ +// Copyright (c) 2026 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.digitalasset.canton.tracing + +import java.util.Locale + +/** HTTP header name with case-insensitive equality (RFC 7230 ยง3.2). The + * apply factory normalizes to lowercase, so equality and hashing on the + * underlying String are case-insensitive by construction. Use as the key + * type for any map of HTTP headers to avoid lookup misses caused by + * upstream case variation. + */ +final class HeaderName private (val value: String) extends AnyVal { + override def toString: String = value +} + +object HeaderName { + def apply(name: String): HeaderName = + new HeaderName(name.toLowerCase(Locale.ROOT)) +} diff --git a/canton/community/util-observability/src/main/scala/com/digitalasset/canton/tracing/W3CTraceContext.scala b/canton/community/util-observability/src/main/scala/com/digitalasset/canton/tracing/W3CTraceContext.scala index 711b2e708d..09df3c9f6f 100644 --- a/canton/community/util-observability/src/main/scala/com/digitalasset/canton/tracing/W3CTraceContext.scala +++ b/canton/community/util-observability/src/main/scala/com/digitalasset/canton/tracing/W3CTraceContext.scala @@ -24,8 +24,8 @@ final case class W3CTraceContext(parent: String, state: Option[String] = None) /** HTTP headers of this trace context. Marked transient as the headers do not need to be * serialized when using java serialization. */ - @transient lazy val asHeaders: Map[String, String] = - Map(TRACEPARENT_HEADER_NAME -> parent) ++ state.map(TRACESTATE_HEADER_NAME -> _).toList + @transient lazy val asHeaders: Map[HeaderName, String] = + Map(TraceparentHeader -> parent) ++ state.map(TracestateHeader -> _).toList override def toString: String = { val sb = new mutable.StringBuilder() @@ -41,27 +41,27 @@ final case class W3CTraceContext(parent: String, state: Option[String] = None) object W3CTraceContext { // https://www.w3.org/TR/trace-context/ private val propagator = W3CTraceContextPropagator.getInstance() - private val TRACEPARENT_HEADER_NAME = - "traceparent" // same as W3CTraceContextPropagator.TRACE_PARENT - private val TRACESTATE_HEADER_NAME = "tracestate" // same as W3CTraceContextPropagator.TRACE_STATE + // values match W3CTraceContextPropagator.TRACE_PARENT / TRACE_STATE + private val TraceparentHeader = HeaderName("traceparent") + private val TracestateHeader = HeaderName("tracestate") @SuppressWarnings(Array("org.wartremover.warts.Var")) def fromOpenTelemetryContext(context: OpenTelemetryContext): Option[W3CTraceContext] = { var builder = new W3CTraceContextBuilder val setter: TextMapSetter[W3CTraceContextBuilder] = (carrier, key, value) => - builder = key match { - case TRACEPARENT_HEADER_NAME => carrier.copy(parent = Some(value)) - case TRACESTATE_HEADER_NAME => carrier.copy(state = Some(value)) + builder = HeaderName(key) match { + case TraceparentHeader => carrier.copy(parent = Some(value)) + case TracestateHeader => carrier.copy(state = Some(value)) case _ => carrier } propagator.inject(context, builder, setter) builder.build } - def fromHeaders(headers: Map[String, String]): Option[W3CTraceContext] = + def fromHeaders(headers: Map[HeaderName, String]): Option[W3CTraceContext] = W3CTraceContextBuilder( - headers.get(TRACEPARENT_HEADER_NAME), - headers.get(TRACESTATE_HEADER_NAME), + headers.get(TraceparentHeader), + headers.get(TracestateHeader), ).build private final case class W3CTraceContextBuilder( @@ -96,10 +96,12 @@ object W3CTraceContext { * returned but the current span will be invalid and [[TraceContext.traceId]] will return None. */ @SuppressWarnings(Array("org.wartremover.warts.Null")) - def toTraceContext(parent: Option[String], state: Option[String]): TraceContext = extract { - case `TRACEPARENT_HEADER_NAME` => parent.orNull - case `TRACESTATE_HEADER_NAME` => state.orNull - case _ => null + def toTraceContext(parent: Option[String], state: Option[String]): TraceContext = extract { key => + HeaderName(key) match { + case TraceparentHeader => parent.orNull + case TracestateHeader => state.orNull + case _ => null + } } private def extract(getter: String => String): TraceContext =