diff --git a/docs/images/openai-tracing-with-opentelemetry.png b/docs/images/openai-tracing-with-opentelemetry.png new file mode 100644 index 00000000..2bc40499 Binary files /dev/null and b/docs/images/openai-tracing-with-opentelemetry.png differ diff --git a/docs/observability.md b/docs/observability.md index 8dd3290a..acd88399 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -10,48 +10,61 @@ OpenAI .NET instrumentation follows [OpenTelemetry Semantic Conventions for Gene ### How to enable -The instrumentation is **experimental** - volume and semantics of the telemetry items may change. +> [!NOTE] +> The instrumentation is **experimental** - semantics of telemetry items +> (such as metric or attribute names, types, value range or other properties) may change. -To enable the instrumentation: +The following code snippet shows how to enable OpenAI traces and metrics: -1. Set instrumentation feature-flag using one of the following options: +```csharp +builder.Services.AddOpenTelemetry() + .WithTracing(b => + { + b.AddSource("Experimental.OpenAI.*", "OpenAI.*") + ... + .AddOtlpExporter(); + }) + .WithMetrics(b => + { + b.AddMeter("Experimental.OpenAI.*", "OpenAI.*") + ... + .AddOtlpExporter(); + }); +``` - - set the `OPENAI_EXPERIMENTAL_ENABLE_OPEN_TELEMETRY` environment variable to `"true"` - - set the `OpenAI.Experimental.EnableOpenTelemetry` context switch to true in your application code when application - is starting and before initializing any OpenAI clients. For example: +Distributed tracing is enabled with `AddSource("Experimental.OpenAI.*", "OpenAI.*")` which tells OpenTelemetry to listen to all [ActivitySources](https://learn.microsoft.com/dotnet/api/system.diagnostics.activitysource) with names starting with `Experimental.OpenAI.` (experimental ones) or sources which names start with `"OpenAI.*"` (stable ones). - ```csharp - AppContext.SetSwitch("OpenAI.Experimental.EnableOpenTelemetry", true); - ``` +Similarly, metrics are configured with `AddMeter("Experimental.OpenAI.*", "OpenAI.*")` which enables all OpenAI-related [Meters](https://learn.microsoft.com/dotnet/api/system.diagnostics.metrics.meter). -2. Enable OpenAI telemetry: +Once experimental telemetry stabilizes, experimental sources and meters will be renamed (for example, `Experimental.OpenAI.ChatClient` will become `OpenAI.ChatClient`), so it's +recommended to enable both to avoid changing the source code later. - ```csharp - builder.Services.AddOpenTelemetry() - .WithTracing(b => - { - b.AddSource("OpenAI.*") - ... - .AddOtlpExporter(); - }) - .WithMetrics(b => - { - b.AddMeter("OpenAI.*") - ... - .AddOtlpExporter(); - }); - ``` +Consider enabling [HTTP client instrumentation](https://www.nuget.org/packages/OpenTelemetry.Instrumentation.Http) to see all HTTP client +calls made by your application including those done by the OpenAI SDK. - Distributed tracing is enabled with `AddSource("OpenAI.*")` which tells OpenTelemetry to listen to all [ActivitySources](https://learn.microsoft.com/dotnet/api/system.diagnostics.activitysource) with names starting with `OpenAI.*`. +Check out [full example](../examples/OpenTelemetryExamples.cs) and [OpenTelemetry documentation](https://opentelemetry.io/docs/languages/net/getting-started/) for more details. - Similarly, metrics are configured with `AddMeter("OpenAI.*")` which enables all OpenAI-related [Meters](https://learn.microsoft.com/dotnet/api/system.diagnostics.metrics.meter). +### How to view telemetry -Consider enabling [HTTP client instrumentation](https://www.nuget.org/packages/OpenTelemetry.Instrumentation.Http) to see all HTTP client -calls made by your application including those done by the OpenAI SDK. -Check out [OpenTelemetry documentation](https://opentelemetry.io/docs/languages/net/getting-started/) for more details. +You can view traces and metrics in any [telemetry system](https://opentelemetry.io/ecosystem/vendors/) compatible with OpenTelemetry. + +You may use [Aspire dashboard](https://learn.microsoft.com/dotnet/aspire/fundamentals/dashboard/standalone) to test things out locally. +You can run it as a docker container with the following command: + +```bash +docker run --rm -it \ + -p 18888:18888 \ + -p 4317:18889 -d \ + --name aspire-dashboard \ + mcr.microsoft.com/dotnet/aspire-dashboard:latest +``` + +Here's a trace produced by [OpenTelemetry sample](../examples/OpenTelemetryExamples.cs): + +![](./images/openai-tracing-with-opentelemetry.png) ### Available sources and meters The following sources and meters are available: -- `OpenAI.ChatClient` - records traces and metrics for `ChatClient` operations (except streaming and protocol methods which are not instrumented yet) +- `Experimental.OpenAI.ChatClient` - records traces and metrics for `ChatClient` operations (except streaming and protocol methods which are not instrumented yet) diff --git a/examples/OpenAI.Examples.csproj b/examples/OpenAI.Examples.csproj index 74bf8c48..9924607e 100644 --- a/examples/OpenAI.Examples.csproj +++ b/examples/OpenAI.Examples.csproj @@ -19,5 +19,8 @@ + + + \ No newline at end of file diff --git a/examples/OpenTelemetryExamples.cs b/examples/OpenTelemetryExamples.cs new file mode 100644 index 00000000..41be8d15 --- /dev/null +++ b/examples/OpenTelemetryExamples.cs @@ -0,0 +1,49 @@ +using NUnit.Framework; +using OpenAI.Chat; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; +using System; +using System.Threading.Tasks; + +namespace OpenAI.Examples; + +public partial class ChatExamples +{ + [Test] + public async Task OpenTelemetryExamples() + { + // Let's configure OpenTelemetry to collect OpenAI and HTTP client traces and metrics + // and export them to console and also to the local OTLP endpoint. + // + // If you have some local OTLP listener (e.g. Aspire dashboard) running, + // you can explore traces and metrics produced by the test there. + // + // Check out https://opentelemetry.io/docs/languages/net/getting-started/ for more details and + // examples on how to set up OpenTelemetry with ASP.NET Core. + + ResourceBuilder resourceBuilder = ResourceBuilder.CreateDefault().AddService("test"); + using TracerProvider tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .SetResourceBuilder(resourceBuilder) + .AddSource("Experimental.OpenAI.*", "OpenAI.*") + .AddHttpClientInstrumentation() + .AddConsoleExporter() + .AddOtlpExporter() + .Build(); + + using MeterProvider meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .SetResourceBuilder(resourceBuilder) + .AddView("gen_ai.client.operation.duration", new ExplicitBucketHistogramConfiguration { Boundaries = [0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92] }) + .AddMeter("Experimental.OpenAI.*", "OpenAI.*") + .AddHttpClientInstrumentation() + .AddConsoleExporter() + .AddOtlpExporter() + .Build(); + + ChatClient client = new("gpt-4o-mini", Environment.GetEnvironmentVariable("OPENAI_API_KEY")); + + ChatCompletion completion = await client.CompleteChatAsync("Say 'this is a test.'"); + + Console.WriteLine($"{completion}"); + } +} diff --git a/src/Utility/AppContextSwitchHelper.cs b/src/Utility/AppContextSwitchHelper.cs deleted file mode 100644 index 34d98529..00000000 --- a/src/Utility/AppContextSwitchHelper.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System; - -namespace OpenAI; - -internal static class AppContextSwitchHelper -{ - /// - /// Determines if either an AppContext switch or its corresponding Environment Variable is set - /// - /// Name of the AppContext switch. - /// Name of the Environment variable. - /// If the AppContext switch has been set, returns the value of the switch. - /// If the AppContext switch has not been set, returns the value of the environment variable. - /// False if neither is set. - /// - public static bool GetConfigValue(string appContexSwitchName, string environmentVariableName) - { - // First check for the AppContext switch, giving it priority over the environment variable. - if (AppContext.TryGetSwitch(appContexSwitchName, out bool value)) - { - return value; - } - // AppContext switch wasn't used. Check the environment variable. - string envVar = Environment.GetEnvironmentVariable(environmentVariableName); - if (envVar != null && (envVar.Equals("true", StringComparison.OrdinalIgnoreCase) || envVar.Equals("1"))) - { - return true; - } - - // Default to false. - return false; - } -} diff --git a/src/Utility/Telemetry/OpenTelemetryScope.cs b/src/Utility/Telemetry/OpenTelemetryScope.cs index 6a975c64..0b22ffa8 100644 --- a/src/Utility/Telemetry/OpenTelemetryScope.cs +++ b/src/Utility/Telemetry/OpenTelemetryScope.cs @@ -10,8 +10,8 @@ namespace OpenAI.Telemetry; internal class OpenTelemetryScope : IDisposable { - private static readonly ActivitySource s_chatSource = new ActivitySource("OpenAI.ChatClient"); - private static readonly Meter s_chatMeter = new Meter("OpenAI.ChatClient"); + private static readonly ActivitySource s_chatSource = new ActivitySource("Experimental.OpenAI.ChatClient"); + private static readonly Meter s_chatMeter = new Meter("Experimental.OpenAI.ChatClient"); // TODO: add explicit histogram buckets once System.Diagnostics.DiagnosticSource 9.0 is used private static readonly Histogram s_duration = s_chatMeter.CreateHistogram(GenAiClientOperationDurationMetricName, "s", "Measures GenAI operation duration."); diff --git a/src/Utility/Telemetry/OpenTelemetrySource.cs b/src/Utility/Telemetry/OpenTelemetrySource.cs index a0ac1fe4..e3719c82 100644 --- a/src/Utility/Telemetry/OpenTelemetrySource.cs +++ b/src/Utility/Telemetry/OpenTelemetrySource.cs @@ -6,9 +6,6 @@ namespace OpenAI.Telemetry; internal class OpenTelemetrySource { private const string ChatOperationName = "chat"; - private readonly bool IsOTelEnabled = AppContextSwitchHelper - .GetConfigValue("OpenAI.Experimental.EnableOpenTelemetry", "OPENAI_EXPERIMENTAL_ENABLE_OPEN_TELEMETRY"); - private readonly string _serverAddress; private readonly int _serverPort; private readonly string _model; @@ -22,9 +19,6 @@ public OpenTelemetrySource(string model, Uri endpoint) public OpenTelemetryScope StartChatScope(ChatCompletionOptions completionsOptions) { - return IsOTelEnabled - ? OpenTelemetryScope.StartChat(_model, ChatOperationName, _serverAddress, _serverPort, completionsOptions) - : null; + return OpenTelemetryScope.StartChat(_model, ChatOperationName, _serverAddress, _serverPort, completionsOptions); } - } diff --git a/tests/Chat/ChatTests.cs b/tests/Chat/ChatTests.cs index 614e028f..b4d9a31e 100644 --- a/tests/Chat/ChatTests.cs +++ b/tests/Chat/ChatTests.cs @@ -754,9 +754,8 @@ void HandleUpdate(StreamingChatCompletionUpdate update) [NonParallelizable] public async Task HelloWorldChatWithTracingAndMetrics() { - using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); - using TestActivityListener activityListener = new TestActivityListener("OpenAI.ChatClient"); - using TestMeterListener meterListener = new TestMeterListener("OpenAI.ChatClient"); + using TestActivityListener activityListener = new TestActivityListener("Experimental.OpenAI.ChatClient"); + using TestMeterListener meterListener = new TestMeterListener("Experimental.OpenAI.ChatClient"); ChatClient client = GetTestClient(TestScenario.Chat); IEnumerable messages = [new UserChatMessage("Hello, world!")]; diff --git a/tests/OpenAI.Tests.csproj b/tests/OpenAI.Tests.csproj index 711df03c..304965a1 100644 --- a/tests/OpenAI.Tests.csproj +++ b/tests/OpenAI.Tests.csproj @@ -24,6 +24,5 @@ - \ No newline at end of file diff --git a/tests/Telemetry/ChatTelemetryTests.cs b/tests/Telemetry/ChatTelemetryTests.cs index 36b9f415..7fbd5d75 100644 --- a/tests/Telemetry/ChatTelemetryTests.cs +++ b/tests/Telemetry/ChatTelemetryTests.cs @@ -41,24 +41,12 @@ public void AllTelemetryOff() Assert.IsNull(Activity.Current); } - [Test] - public void SwitchOffAllTelemetryOn() - { - using var activityListener = new TestActivityListener("OpenAI.ChatClient"); - using var meterListener = new TestMeterListener("OpenAI.ChatClient"); - var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); - Assert.IsNull(telemetry.StartChatScope(new ChatCompletionOptions())); - Assert.IsNull(Activity.Current); - } - [Test] public void MetricsOnTracingOff() { - using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); - var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); - using var meterListener = new TestMeterListener("OpenAI.ChatClient"); + using var meterListener = new TestMeterListener("Experimental.OpenAI.ChatClient"); var elapsedMax = Stopwatch.StartNew(); using var scope = telemetry.StartChatScope(new ChatCompletionOptions()); @@ -83,10 +71,8 @@ public void MetricsOnTracingOff() [Test] public void MetricsOnTracingOffException() { - using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); - var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); - using var meterListener = new TestMeterListener("OpenAI.ChatClient"); + using var meterListener = new TestMeterListener("Experimental.OpenAI.ChatClient"); using (var scope = telemetry.StartChatScope(new ChatCompletionOptions())) { @@ -100,10 +86,8 @@ public void MetricsOnTracingOffException() [Test] public void TracingOnMetricsOff() { - using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); - var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); - using var listener = new TestActivityListener("OpenAI.ChatClient"); + using var listener = new TestActivityListener("Experimental.OpenAI.ChatClient"); var chatCompletion = CreateChatCompletion(); @@ -129,9 +113,8 @@ public void TracingOnMetricsOff() [Test] public void ChatTracingAllAttributes() { - using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); - using var listener = new TestActivityListener("OpenAI.ChatClient"); + using var listener = new TestActivityListener("Experimental.OpenAI.ChatClient"); var options = new ChatCompletionOptions() { Temperature = 0.42f, @@ -157,10 +140,8 @@ public void ChatTracingAllAttributes() [Test] public void ChatTracingException() { - using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); - var telemetry = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); - using var listener = new TestActivityListener("OpenAI.ChatClient"); + using var listener = new TestActivityListener("Experimental.OpenAI.ChatClient"); var error = new SocketException(42, "test error"); using (var scope = telemetry.StartChatScope(new ChatCompletionOptions())) @@ -176,11 +157,10 @@ public void ChatTracingException() [Test] public async Task ChatTracingAndMetricsMultiple() { - using var _ = TestAppContextSwitchHelper.EnableOpenTelemetry(); var source = new OpenTelemetrySource(RequestModel, new Uri(Endpoint)); - using var activityListener = new TestActivityListener("OpenAI.ChatClient"); - using var meterListener = new TestMeterListener("OpenAI.ChatClient"); + using var activityListener = new TestActivityListener("Experimental.OpenAI.ChatClient"); + using var meterListener = new TestMeterListener("Experimental.OpenAI.ChatClient"); var options = new ChatCompletionOptions(); diff --git a/tests/Telemetry/TestAppContextSwitchHelper.cs b/tests/Telemetry/TestAppContextSwitchHelper.cs deleted file mode 100644 index 5faf5eca..00000000 --- a/tests/Telemetry/TestAppContextSwitchHelper.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System; - -namespace OpenAI.Tests.Telemetry; - -internal class TestAppContextSwitchHelper : IDisposable -{ - private const string OpenTelemetrySwitchName = "OpenAI.Experimental.EnableOpenTelemetry"; - - private string _switchName; - private TestAppContextSwitchHelper(string switchName) - { - _switchName = switchName; - AppContext.SetSwitch(_switchName, true); - } - - public static IDisposable EnableOpenTelemetry() - { - return new TestAppContextSwitchHelper(OpenTelemetrySwitchName); - } - - public void Dispose() - { - AppContext.SetSwitch(_switchName, false); - } -} diff --git a/tests/Telemetry/TestMeterListener.cs b/tests/Telemetry/TestMeterListener.cs index a8a5cdc0..05343596 100644 --- a/tests/Telemetry/TestMeterListener.cs +++ b/tests/Telemetry/TestMeterListener.cs @@ -4,6 +4,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.Metrics; +using System.Linq; namespace OpenAI.Tests.Telemetry; @@ -11,7 +12,7 @@ internal class TestMeterListener : IDisposable { public record TestMeasurement(object value, Dictionary tags); - private readonly ConcurrentDictionary> _measurements = new(); + private readonly ConcurrentDictionary> _measurements = new(); private readonly ConcurrentDictionary _instruments = new(); private readonly MeterListener _listener; public TestMeterListener(string meterName) @@ -31,8 +32,8 @@ public TestMeterListener(string meterName) public List GetMeasurements(string instrumentName) { - _measurements.TryGetValue(instrumentName, out var list); - return list; + _measurements.TryGetValue(instrumentName, out var queue); + return queue?.ToList(); } public Instrument GetInstrument(string instrumentName) @@ -46,11 +47,11 @@ private void OnMeasurementRecorded(Instrument instrument, T measurement, Read _instruments.TryAdd(instrument.Name, instrument); var testMeasurement = new TestMeasurement(measurement, new Dictionary(tags.ToArray())); - _measurements.AddOrUpdate(instrument.Name, - k => new() { testMeasurement }, + _measurements.AddOrUpdate(instrument.Name, + k => new ConcurrentQueue([ testMeasurement ]), (k, l) => { - l.Add(testMeasurement); + l.Enqueue(testMeasurement); return l; }); }