Skip to content

Make terminal-host telemetry opt-in #20005

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Mitch Denny (mitchdenny) merged 3 commits into from
Sep 11, 2026
Merged

Make terminal-host telemetry opt-in #20005

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
78972bb
Make terminal-host telemetry opt-in
mitchdenny Sep 9, 2026
6ac0f05
Fail closed for malformed terminal telemetry opt-in
mitchdenny Sep 11, 2026
ad937d4
Use shared configuration parsing for terminal telemetry
mitchdenny Sep 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions src/Aspire.Hosting/ApplicationModel/TerminalAnnotation.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,17 +129,19 @@ public int Rows

/// <summary>
/// Gets or sets a value indicating whether the per-replica terminal host resources
/// (named <c>{parent}-terminalhost-{index}</c>) should appear in the resource list.
/// (named <c>{parent}-terminalhost-{index}</c>) should appear in the resource list
/// and export diagnostic logs, metrics, and traces.
/// </summary>
/// <remarks>
/// Defaults to <c>false</c>: terminal host resources are hidden from the dashboard
/// and CLI resource list because they are an implementation detail of the
/// and CLI resource list and do not export telemetry because they are an implementation detail of the
/// <see cref="TerminalResourceBuilderExtensions.WithTerminal{T}(IResourceBuilder{T}, Action{TerminalOptions}?)"/>
/// feature, not something the user explicitly added to their app model.
/// <para>
/// Set to <c>true</c> when diagnosing terminal-host startup / connectivity issues so
/// the host's state, exit code, logs, and (eventually) telemetry are visible alongside
/// the parent resource. This is useful when investigating cases like "DCP never dialed
/// the host's state, exit code, and diagnostic telemetry are visible alongside
/// the parent resource. Telemetry is exported when a dashboard OTLP endpoint is available.
/// This is useful when investigating cases like "DCP never dialed
/// the producer UDS" or "the host crashed during recycle".
/// </para>
/// </remarks>
Expand Down
42 changes: 16 additions & 26 deletions src/Aspire.Hosting/TerminalResourceBuilderExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,34 +222,20 @@ private static async Task MaterializeTerminalHostsAsync(
appHostPid,
appHostProcessIdentity);

// Wire OTLP env vars onto each terminal host so it can ship logs/traces/metrics to
// the dashboard. Without this, terminal host failures (DCP never dials, control
// socket bind fails, replica recycles in a loop) are only visible by attaching a
// debugger — there is no other log sink: the host explicitly does not write to
// stderr because DCP captures stderr into the consumer's resource log stream
// and any host-generated bytes would corrupt that view.
//
// AddOtlpEnvironment also injects OTEL_RESOURCE_ATTRIBUTES=service.instance.id=...,
// which combined with DCP's OTEL_SERVICE_NAME annotation gives each replica's
// host process a unique identity in the dashboard. We do not pin a protocol here
// (gRPC vs HTTP/protobuf); the env-callback picks the dashboard's preferred
// protocol and the host's composite `UseOtlpExporter()` honours
// OTEL_EXPORTER_OTLP_PROTOCOL — same path every ServiceDefaults-wired Aspire
// project takes.
OtlpConfigurationExtensions.AddOtlpEnvironment(terminalHost, configuration, @event.Services.GetRequiredService<IHostEnvironment>());

// Propagate ASPIRE_TERMINAL_HOST_LOG_LEVEL from the AppHost process so playground/dev
// can dial up host verbosity (e.g. "Debug") from launchSettings.json without code
// changes. The terminal host itself reads this env var and applies it to
// ILoggingBuilder.SetMinimumLevel; only relevant when OTLP is wired so the resulting
// log records reach the dashboard.
var hostLogLevel = Environment.GetEnvironmentVariable("ASPIRE_TERMINAL_HOST_LOG_LEVEL");
if (!string.IsNullOrWhiteSpace(hostLogLevel))
// Telemetry creates dashboard resource entries independently of IsHidden.
// Only export helper diagnostics when the user explicitly makes the host visible.
if (options.ShowTerminalHost)
{
terminalHost.Annotations.Add(new EnvironmentCallbackAnnotation(ctx =>
OtlpConfigurationExtensions.AddOtlpEnvironment(terminalHost, configuration, @event.Services.GetRequiredService<IHostEnvironment>());

var hostLogLevel = Environment.GetEnvironmentVariable("ASPIRE_TERMINAL_HOST_LOG_LEVEL");
if (!string.IsNullOrWhiteSpace(hostLogLevel))
{
ctx.EnvironmentVariables["ASPIRE_TERMINAL_HOST_LOG_LEVEL"] = hostLogLevel;
}));
terminalHost.Annotations.Add(new EnvironmentCallbackAnnotation(ctx =>
{
ctx.EnvironmentVariables["ASPIRE_TERMINAL_HOST_LOG_LEVEL"] = hostLogLevel;
}));
}
}

@event.Model.Resources.Add(terminalHost);
Expand Down Expand Up @@ -400,6 +386,10 @@ private static void ConfigureTerminalHostAnnotations(

host.Annotations.Add(new EnvironmentCallbackAnnotation(context =>
{
// Explicitly disable telemetry for hidden hosts even if OTLP settings or a
// diagnostic opt-in were inherited from the AppHost's environment.
context.EnvironmentVariables[KnownConfigNames.TerminalHostTelemetryEnabled] =
options.ShowTerminalHost ? "true" : "false";
context.EnvironmentVariables[KnownConfigNames.TerminalHostParentProcessId] =
appHostPid.ToString(CultureInfo.InvariantCulture);
context.EnvironmentVariables[KnownConfigNames.TerminalHostParentProcessStartedStable] =
Expand Down
2 changes: 2 additions & 0 deletions src/Aspire.TerminalHost/Aspire.TerminalHost.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
<RuntimeIdentifiers>win-x64;win-arm64;linux-x64;linux-arm64;linux-musl-x64;osx-x64;osx-arm64</RuntimeIdentifiers>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<DefineConstants>$(DefineConstants);ASPIRE_TERMINAL_HOST</DefineConstants>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<RollForward>Major</RollForward>
<!--
Expand Down Expand Up @@ -38,6 +39,7 @@
</ItemGroup>

<ItemGroup>
<Compile Include="..\Shared\IConfigurationExtensions.cs" Link="Shared\IConfigurationExtensions.cs" />
<Compile Include="..\Shared\KnownConfigNames.cs" Link="Shared\KnownConfigNames.cs" />
<Compile Include="..\Shared\ParentProcessLivenessMonitor.cs" Link="Shared\ParentProcessLivenessMonitor.cs" />
<Compile Include="..\Shared\ParentProcessWatchdog.cs" Link="Shared\ParentProcessWatchdog.cs" />
Expand Down
133 changes: 52 additions & 81 deletions src/Aspire.TerminalHost/TerminalHostApp.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
// The .NET Foundation licenses this file to you under the MIT license.

using System.Diagnostics.Tracing;
using Aspire.Hosting;
using Aspire.Shared.TerminalHost;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
Expand Down Expand Up @@ -263,99 +265,34 @@ await Console.Error.WriteLineAsync($"[Aspire.TerminalHost] {ex.Message}")
return 64; // EX_USAGE
}

// The Aspire AppHost wires OTEL_EXPORTER_OTLP_ENDPOINT (and protocol/headers) into the
// host environment via OtlpConfigurationExtensions.AddOtlpEnvironment on each
// TerminalHostResource. When that variable isn't set — e.g. a standalone
// `dotnet run --project src/Aspire.TerminalHost` invocation for local debugging — we
// intentionally fall back to NullLoggerFactory rather than scribbling on stderr, since
// DCP captures stderr into the resource log stream and any accidental log line would
// surface as noisy resource output. The dashboard is the only intended sink.
var otlpEndpoint = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT");
var otlpProtocol = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_PROTOCOL");
var otlpHeaders = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_HEADERS");
var serviceName = Environment.GetEnvironmentVariable("OTEL_SERVICE_NAME");
var resourceAttrs = Environment.GetEnvironmentVariable("OTEL_RESOURCE_ATTRIBUTES");
var otelEnabled = !string.IsNullOrEmpty(otlpEndpoint);

// One-shot stderr diagnostic at startup so the dashboard's resource log tab for
// *-terminalhost-N shows whether OTLP is wired. Single line; subsequent operational
// logs go through the OTel pipeline (or NullLoggerFactory) per the gating below.
// headers length is logged (not the value) because it contains the dashboard's x-otlp-api-key:
// a missing/empty header yields 401 from the dashboard OTLP listener and silently drops
// every signal, which presents as "telemetry is wired but nothing shows up".
await Console.Error.WriteLineAsync(
$"[Aspire.TerminalHost] startup pid={Environment.ProcessId} otel={(otelEnabled ? "on" : "off")} endpoint='{otlpEndpoint}' protocol='{otlpProtocol}' headers.len={otlpHeaders?.Length ?? 0} service='{serviceName}' resource='{resourceAttrs}'")
.ConfigureAwait(false);
var configuration = new ConfigurationBuilder().AddEnvironmentVariables().Build();
var hostBuilder = CreateTelemetryHostBuilder(configuration);

ILoggerFactory loggerFactory;
IHost? host = null;
OtelSelfDiagnosticsListener? selfDiag = null;
if (otelEnabled)
if (hostBuilder is not null)
{
var otlpEndpoint = configuration["OTEL_EXPORTER_OTLP_ENDPOINT"];
var otlpProtocol = configuration["OTEL_EXPORTER_OTLP_PROTOCOL"];
var otlpHeaders = configuration["OTEL_EXPORTER_OTLP_HEADERS"];
var serviceName = configuration["OTEL_SERVICE_NAME"];
var resourceAttrs = configuration["OTEL_RESOURCE_ATTRIBUTES"];

// Log only the header length: its value contains the dashboard OTLP API key.
await Console.Error.WriteLineAsync(
$"[Aspire.TerminalHost] startup pid={Environment.ProcessId} otel=on endpoint='{otlpEndpoint}' protocol='{otlpProtocol}' headers.len={otlpHeaders?.Length ?? 0} service='{serviceName}' resource='{resourceAttrs}'")
.ConfigureAwait(false);

// Surface OTLP exporter failures (cert trust, connection refused, 401, schema
// mismatches) to stderr. Without this, every Warning/Error from
// OpenTelemetry-Exporter-OpenTelemetryProtocol (and the SDK proper) is swallowed
// and the dashboard just stays empty — the symptom that brought us here originally.
// Listener is disposed in the finally below.
selfDiag = new OtelSelfDiagnosticsListener();

// Configure OTel via the same composite pattern the Aspire ServiceDefaults template
// emits: a single `services.AddOpenTelemetry()...UseOtlpExporter()` chain that wires
// logs, metrics, and traces to one shared OtlpExporterOptions. The legacy per-signal
// `.AddOtlpExporter()` shorthand (three separate calls on `Sdk.Create*Builder()`)
// resolves endpoints inconsistently — under gRPC it sends to the root path and the
// dashboard's gRPC OTLP listener returns `Status(Unimplemented, "Service is
// unimplemented.")` for executable consumers, silently dropping every signal.
// UseOtlpExporter goes through the same code path every ServiceDefaults-wired
// project uses, so by definition it talks to the dashboard the way the dashboard
// expects.
//
// OTEL_SERVICE_NAME and the service.instance.id resource attribute are set by DCP
// via CustomResource.OtelServiceNameAnnotation /
// CustomResource.OtelServiceInstanceIdAnnotation on each executable; the default
// resource detector picks them up from the environment, so we don't override them.
//
// We build an IHost (via Host.CreateEmptyApplicationBuilder so we don't inherit a
// console logger — DCP captures stderr into the consumer log stream and any
// accidental log line would corrupt that view) and start it, rather than using a
// bare ServiceCollection + BuildServiceProvider. The reason: OpenTelemetry's
// tracer and meter providers are registered as DI singletons by
// `services.AddOpenTelemetry().With{Tracing,Metrics}()`, but the thing that
// instantiates them — and thereby starts the OTLP export pipelines — is
// OpenTelemetry.Extensions.Hosting's `TelemetryHostedService.StartAsync`. Without
// an IHost to run that hosted service, metrics and spans never reach the
// exporter even though the code looks correctly wired. Logs happen to work
// without IHost because resolving ILoggerFactory transitively builds the logging
// pipeline, but tracer/meter providers have no such eager resolver.

// Minimum log level — honour ASPIRE_TERMINAL_HOST_LOG_LEVEL so playground/dev
// can crank verbosity from launchSettings.json without code changes. Recognised
// values match the Microsoft.Extensions.Logging.LogLevel enum (Trace, Debug,
// Information, Warning, Error, Critical, None). Default: Information.
var minLevel = ParseLogLevel(Environment.GetEnvironmentVariable("ASPIRE_TERMINAL_HOST_LOG_LEVEL"));

// Use Host.CreateApplicationBuilder so we get the standard set of services every
// other .NET host gets: configuration providers, logging registrations (console +
// debug + eventsource), and ILoggerFactory wiring. The console logger writes to
// the host process's own stdout/stderr, which DCP captures into this terminal
// host's "Console logs" tab in the dashboard — completely separate from the PTY
// consumer stream, which is over the consumer UDS. So we're not corrupting
// anything by emitting console output here.
var hostBuilder = Host.CreateApplicationBuilder();

hostBuilder.Logging.SetMinimumLevel(minLevel);
hostBuilder.Logging.AddOpenTelemetry(logging =>
{
logging.IncludeFormattedMessage = true;
logging.IncludeScopes = true;
});

hostBuilder.Services.AddOpenTelemetry()
.ConfigureResource(r => r.AddService(TerminalHostTelemetry.SourceName))
.WithTracing(t => t.AddSource(TerminalHostTelemetry.SourceName))
.WithMetrics(m => m.AddMeter(TerminalHostTelemetry.SourceName))
.UseOtlpExporter();

// Starting the host eagerly initializes the tracer and meter providers through
// OpenTelemetry's hosted service, rather than only activating the logging pipeline.
host = hostBuilder.Build();
await host.StartAsync(cancellationToken).ConfigureAwait(false);

Expand Down Expand Up @@ -415,6 +352,40 @@ await Console.Error.WriteLineAsync(
}
}

internal static HostApplicationBuilder? CreateTelemetryHostBuilder(IConfiguration configuration)
{
// An inherited OTLP endpoint must not turn a hidden implementation detail into
// a telemetry resource. The AppHost explicitly sets this flag from ShowTerminalHost.
// Malformed values (e.g. "not-a-bool") must leave telemetry disabled, not stop the terminal.
if (!configuration.GetBool(KnownConfigNames.TerminalHostTelemetryEnabled, defaultValue: false) ||
string.IsNullOrEmpty(configuration["OTEL_EXPORTER_OTLP_ENDPOINT"]))
{
return null;
}

var hostBuilder = Host.CreateApplicationBuilder();
hostBuilder.Configuration.AddConfiguration(configuration);
hostBuilder.Logging.SetMinimumLevel(ParseLogLevel(configuration["ASPIRE_TERMINAL_HOST_LOG_LEVEL"]));
hostBuilder.Logging.AddOpenTelemetry(logging =>
{
logging.IncludeFormattedMessage = true;
logging.IncludeScopes = true;
});

// Preserve DCP's service name and instance ID so telemetry matches the helper
// resource instead of creating an unrelated resource in the dashboard.
var serviceName = configuration["OTEL_SERVICE_NAME"];
hostBuilder.Services.AddOpenTelemetry()
.ConfigureResource(r => r.AddService(
string.IsNullOrEmpty(serviceName) ? TerminalHostTelemetry.SourceName : serviceName,
autoGenerateServiceInstanceId: false))
.WithTracing(t => t.AddSource(TerminalHostTelemetry.SourceName))
.WithMetrics(m => m.AddMeter(TerminalHostTelemetry.SourceName))
.UseOtlpExporter();

return hostBuilder;
}

// Parse a friendly LogLevel string from the env var. Case-insensitive; bad/empty values
// fall back to Information. We accept the standard Microsoft.Extensions.Logging.LogLevel
// names (Trace, Debug, Information, Warning, Error, Critical, None).
Expand Down
10 changes: 5 additions & 5 deletions src/Aspire.TerminalHost/TerminalHostTelemetry.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,17 @@ namespace Aspire.TerminalHost;

/// <summary>
/// Shared <see cref="System.Diagnostics.ActivitySource"/> and <see cref="System.Diagnostics.Metrics.Meter"/>
/// for the Aspire terminal host. Telemetry is exported via OTLP to the Aspire dashboard so failures
/// for the Aspire terminal host. Opted-in telemetry is exported via OTLP to the Aspire dashboard so failures
/// like "DCP never dialed in" or "control socket bound but no clients" are diagnosable without
/// resorting to attaching a debugger.
/// </summary>
/// <remarks>
/// <para>
/// The OTLP exporter wiring in <see cref="TerminalHostApp.RunAsync(string[], System.Threading.CancellationToken)"/>
/// only attaches when <c>OTEL_EXPORTER_OTLP_ENDPOINT</c> is set in the environment. The Aspire
/// AppHost injects that via <c>OtlpConfigurationExtensions.AddOtlpEnvironment</c> on each
/// <c>TerminalHostResource</c>, so production runs always have it. Standalone debug runs of the
/// host (<c>dotnet run --project src/Aspire.TerminalHost</c>) drop telemetry silently.
/// only attaches when <c>ASPIRE_TERMINAL_HOST_TELEMETRY_ENABLED</c> is <c>true</c> and
/// <c>OTEL_EXPORTER_OTLP_ENDPOINT</c> is set. The Aspire AppHost configures these for each
/// <c>TerminalHostResource</c> when <c>TerminalOptions.ShowTerminalHost</c> is enabled.
/// Standalone diagnostic runs must also explicitly enable telemetry and configure an endpoint.
/// </para>
/// <para>
/// Source / meter names follow the assembly name convention
Expand Down
4 changes: 2 additions & 2 deletions src/Shared/IConfigurationExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@
// The .NET Foundation licenses this file to you under the MIT license.

using System.Diagnostics.CodeAnalysis;
#if !CLI && !ASPIRE_DASHBOARD
#if !CLI && !ASPIRE_DASHBOARD && !ASPIRE_TERMINAL_HOST
using Aspire.Hosting;
#endif
using Microsoft.Extensions.Configuration;

namespace Aspire;

#if CLI || ASPIRE_DASHBOARD
#if CLI || ASPIRE_DASHBOARD || ASPIRE_TERMINAL_HOST
[AttributeUsage(AttributeTargets.All)]
internal sealed class AspireExportIgnoreAttribute : Attribute
{
Expand Down
1 change: 1 addition & 0 deletions src/Shared/KnownConfigNames.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ internal static class KnownConfigNames
// this identity so it can shut down and unlink its sockets if the AppHost disappears.
public const string TerminalHostParentProcessId = "ASPIRE_TERMINAL_HOST_PARENT_PID";
public const string TerminalHostParentProcessStartedStable = "ASPIRE_TERMINAL_HOST_PARENT_STARTED_STABLE";
public const string TerminalHostTelemetryEnabled = "ASPIRE_TERMINAL_HOST_TELEMETRY_ENABLED";

// Identity (PID + start time) of the foreground CLI that spawned a detached `aspire start` /
// `aspire run --detach` child. The detached child watches this during startup and tears the
Expand Down
Loading
Loading