diff --git a/examples/EvalExample/Program.cs b/examples/EvalExample/Program.cs index 225d4b9..67241b7 100644 --- a/examples/EvalExample/Program.cs +++ b/examples/EvalExample/Program.cs @@ -1,6 +1,3 @@ -using System; -using System.Linq; -using Braintrust.Sdk; using Braintrust.Sdk.Eval; using Braintrust.Sdk.Instrumentation.OpenAI; using OpenAI; @@ -10,7 +7,7 @@ namespace Braintrust.Sdk.Examples.EvalExample; class Program { - static void Main(string[] args) + static async Task Main(string[] args) { var openAIApiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY"); if (string.IsNullOrEmpty(openAIApiKey)) @@ -44,23 +41,23 @@ string GetFoodType(string food) } // Create and run the evaluation - var eval = braintrust + var eval = await braintrust .EvalBuilder() .Name($"dotnet-eval-x-{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}") .Cases( - DatasetCase.Of("strawberry", "fruit"), - DatasetCase.Of("asparagus", "vegetable"), - DatasetCase.Of("apple", "fruit"), - DatasetCase.Of("banana", "fruit") + DatasetCase.Of("strawberry", "fruit"), + DatasetCase.Of("asparagus", "vegetable"), + DatasetCase.Of("apple", "fruit"), + DatasetCase.Of("banana", "fruit") ) .TaskFunction(GetFoodType) .Scorers( - Scorer.Of("exact_match", (expected, actual) => expected == actual ? 1.0 : 0.0), - Scorer.Of("close_enough_match", (expected, actual) => expected.Trim().ToLowerInvariant() == actual.Trim().ToLowerInvariant() ? 1.0 : 0.0) + new FunctionScorer("exact_match", (expected, actual) => expected == actual ? 1.0 : 0.0), + new FunctionScorer("close_enough_match", (expected, actual) => expected.Trim().ToLowerInvariant() == actual.Trim().ToLowerInvariant() ? 1.0 : 0.0) ) - .Build(); + .BuildAsync(); - var result = eval.Run(); + var result = await eval.RunAsync(); Console.WriteLine($"\n\n{result.CreateReportString()}"); } } diff --git a/examples/OpenAIInstrumentation/Program.cs b/examples/OpenAIInstrumentation/Program.cs index b8646b0..b8ffb7b 100644 --- a/examples/OpenAIInstrumentation/Program.cs +++ b/examples/OpenAIInstrumentation/Program.cs @@ -31,7 +31,7 @@ static async Task Main(string[] args) if (rootActivity != null) { await ChatCompletionsExample(instrumentedClient); - var url = braintrust.ProjectUri() + var url = await braintrust.GetProjectUriAsync() + $"/logs?r={rootActivity.TraceId}&s={rootActivity.SpanId}"; Console.WriteLine($"\n\n Example complete! View your data in Braintrust: {url}\n"); } diff --git a/examples/SimpleOpenTelemetry/Program.cs b/examples/SimpleOpenTelemetry/Program.cs index d65c159..d968db0 100644 --- a/examples/SimpleOpenTelemetry/Program.cs +++ b/examples/SimpleOpenTelemetry/Program.cs @@ -1,12 +1,8 @@ -using System; -using System.Threading; -using Braintrust.Sdk; - namespace Braintrust.Sdk.Examples.SimpleOpenTelemetry; class Program { - static void Main(string[] args) + static async Task Main(string[] args) { var braintrust = Braintrust.Get(); var activitySource = braintrust.GetActivitySource(); @@ -17,7 +13,7 @@ static void Main(string[] args) ArgumentNullException.ThrowIfNull(activity); Console.WriteLine("Performing simple operation..."); activity.SetTag("some boolean attribute", true); - url = braintrust.ProjectUri() + $"/logs?r={activity.TraceId}&s={activity.SpanId}"; + url = await braintrust.GetProjectUriAsync() + $"/logs?r={activity.TraceId}&s={activity.SpanId}"; } Console.WriteLine($"\n\n Example complete! View your data in Braintrust: {url}"); } diff --git a/src/Braintrust.Sdk/Api/ApiException.cs b/src/Braintrust.Sdk/Api/ApiException.cs new file mode 100644 index 0000000..6edd749 --- /dev/null +++ b/src/Braintrust.Sdk/Api/ApiException.cs @@ -0,0 +1,18 @@ +namespace Braintrust.Sdk.Api; + +/// +/// Exception thrown when an API request fails. +/// +public class ApiException : Exception +{ + public int? StatusCode { get; } + + public ApiException(string message) : base(message) { } + + public ApiException(string message, Exception innerException) : base(message, innerException) { } + + public ApiException(int statusCode, string message) : base(message) + { + StatusCode = statusCode; + } +} \ No newline at end of file diff --git a/src/Braintrust.Sdk/Api/BraintrustApiClient.cs b/src/Braintrust.Sdk/Api/BraintrustApiClient.cs index 5cbe3fe..8931e07 100644 --- a/src/Braintrust.Sdk/Api/BraintrustApiClient.cs +++ b/src/Braintrust.Sdk/Api/BraintrustApiClient.cs @@ -1,34 +1,11 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; using System.Net.Http.Headers; using System.Net.Http.Json; using System.Text.Json; using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; using Braintrust.Sdk.Config; namespace Braintrust.Sdk.Api; -/// -/// Exception thrown when an API request fails. -/// -public class ApiException : Exception -{ - public int? StatusCode { get; } - - public ApiException(string message) : base(message) { } - - public ApiException(string message, Exception innerException) : base(message, innerException) { } - - public ApiException(int statusCode, string message) : base(message) - { - StatusCode = statusCode; - } -} - /// /// Implementation of Braintrust API client. /// @@ -64,12 +41,11 @@ internal BraintrustApiClient(BraintrustConfig config, HttpClient? httpClient = n private static HttpClient CreateDefaultHttpClient(BraintrustConfig config) { - var client = new HttpClient + return new HttpClient { BaseAddress = new Uri(config.ApiUrl), Timeout = config.RequestTimeout }; - return client; } private static JsonSerializerOptions CreateJsonOptions() @@ -82,19 +58,17 @@ private static JsonSerializerOptions CreateJsonOptions() }; } - public Project GetOrCreateProject(string projectName) + public async Task GetOrCreateProject(string projectName) { var request = new CreateProjectRequest(projectName); - return PostAsync("/v1/project", request, default) - .ConfigureAwait(false).GetAwaiter().GetResult(); + return await PostAsync("/v1/project", request).ConfigureAwait(false); } - public Project? GetProject(string projectId) + public async Task GetProject(string projectId) { try { - return GetAsync($"/v1/project/{projectId}", default) - .ConfigureAwait(false).GetAwaiter().GetResult(); + return await GetAsync($"/v1/project/{projectId}").ConfigureAwait(false); } catch (ApiException ex) when (ex.StatusCode == 404) { @@ -102,37 +76,37 @@ public Project GetOrCreateProject(string projectName) } } - public Experiment GetOrCreateExperiment(CreateExperimentRequest request) + public async Task GetOrCreateExperiment(CreateExperimentRequest request) { - return PostAsync("/v1/experiment", request, default) - .ConfigureAwait(false).GetAwaiter().GetResult(); + return await PostAsync("/v1/experiment", request) + .ConfigureAwait(false); } - public OrganizationAndProjectInfo? GetProjectAndOrgInfo() + public async Task GetProjectAndOrgInfo() { if (_config.DefaultProjectId != null) { - return GetProjectAndOrgInfo(_config.DefaultProjectId); + return await GetProjectAndOrgInfo(_config.DefaultProjectId).ConfigureAwait(false); } if (_config.DefaultProjectName != null) { - var project = GetOrCreateProject(_config.DefaultProjectName); - return GetProjectAndOrgInfo(project.Id); + var project = await GetOrCreateProject(_config.DefaultProjectName).ConfigureAwait(false); + return await GetProjectAndOrgInfo(project.Id).ConfigureAwait(false); } return null; } - public OrganizationAndProjectInfo? GetProjectAndOrgInfo(string projectId) + public async Task GetProjectAndOrgInfo(string projectId) { - var project = GetProject(projectId); + var project = await GetProject(projectId).ConfigureAwait(false); if (project == null) { return null; } - var loginResponse = Login(); + var loginResponse = await Login().ConfigureAwait(false); var orgInfo = loginResponse.OrgInfo.FirstOrDefault(org => string.Equals(org.Id, project.OrgId, StringComparison.OrdinalIgnoreCase)); @@ -144,13 +118,13 @@ public Experiment GetOrCreateExperiment(CreateExperimentRequest request) return new OrganizationAndProjectInfo(orgInfo, project); } - public OrganizationAndProjectInfo GetOrCreateProjectAndOrgInfo() + public async Task GetOrCreateProjectAndOrgInfo() { Project project; if (_config.DefaultProjectId != null) { - var existingProject = GetProject(_config.DefaultProjectId); + var existingProject = await GetProject(_config.DefaultProjectId).ConfigureAwait(false); if (existingProject == null) { throw new ApiException($"Project with ID {_config.DefaultProjectId} not found"); @@ -159,14 +133,14 @@ public OrganizationAndProjectInfo GetOrCreateProjectAndOrgInfo() } else if (_config.DefaultProjectName != null) { - project = GetOrCreateProject(_config.DefaultProjectName); + project = await GetOrCreateProject(_config.DefaultProjectName).ConfigureAwait(false); } else { throw new InvalidOperationException("Either DefaultProjectId or DefaultProjectName must be set in config"); } - var loginResponse = Login(); + var loginResponse = await Login().ConfigureAwait(false); var orgInfo = loginResponse.OrgInfo.FirstOrDefault(org => string.Equals(org.Id, project.OrgId, StringComparison.OrdinalIgnoreCase)); @@ -178,11 +152,11 @@ public OrganizationAndProjectInfo GetOrCreateProjectAndOrgInfo() return new OrganizationAndProjectInfo(orgInfo, project); } - private LoginResponse Login() + private async Task Login() { var request = new LoginRequest(_config.ApiKey); - return PostAsync("/api/apikey/login", request, default) - .ConfigureAwait(false).GetAwaiter().GetResult(); + return await PostAsync("/api/apikey/login", request) + .ConfigureAwait(false); } private async Task GetAsync(string path, CancellationToken cancellationToken = default) @@ -191,8 +165,8 @@ private async Task GetAsync(string path, CancellationToken request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _config.ApiKey); request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - using var response = await _httpClient.SendAsync(request, cancellationToken); - return await HandleResponseAsync(response, cancellationToken); + using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + return await HandleResponseAsync(response, cancellationToken).ConfigureAwait(false); } private async Task PostAsync( @@ -205,15 +179,15 @@ private async Task PostAsync( request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); request.Content = JsonContent.Create(body, options: _jsonOptions); - using var response = await _httpClient.SendAsync(request, cancellationToken); - return await HandleResponseAsync(response, cancellationToken); + using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + return await HandleResponseAsync(response, cancellationToken).ConfigureAwait(false); } private async Task HandleResponseAsync(HttpResponseMessage response, CancellationToken cancellationToken) { if (response.IsSuccessStatusCode) { - var content = await response.Content.ReadAsStringAsync(cancellationToken); + var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); var result = JsonSerializer.Deserialize(content, _jsonOptions); if (result == null) @@ -225,7 +199,7 @@ private async Task HandleResponseAsync(HttpResponseMessage response, Cance } else { - var content = await response.Content.ReadAsStringAsync(cancellationToken); + var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); throw new ApiException( (int)response.StatusCode, $"API request failed with status {(int)response.StatusCode}: {content}"); @@ -236,7 +210,7 @@ public void Dispose() { if (_ownsHttpClient) { - _httpClient?.Dispose(); + _httpClient.Dispose(); } } } diff --git a/src/Braintrust.Sdk/Api/IBraintrustApiClient.cs b/src/Braintrust.Sdk/Api/IBraintrustApiClient.cs index d6f7eec..bb603f2 100644 --- a/src/Braintrust.Sdk/Api/IBraintrustApiClient.cs +++ b/src/Braintrust.Sdk/Api/IBraintrustApiClient.cs @@ -8,30 +8,30 @@ public interface IBraintrustApiClient /// /// Get or create a project by name. /// - Project GetOrCreateProject(string projectName); + Task GetOrCreateProject(string projectName); /// /// Get a project by ID. /// - Project? GetProject(string projectId); + Task GetProject(string projectId); /// /// Get or create an experiment. /// - Experiment GetOrCreateExperiment(CreateExperimentRequest request); + Task GetOrCreateExperiment(CreateExperimentRequest request); /// /// Get project and organization information using the default project from config. /// - OrganizationAndProjectInfo? GetProjectAndOrgInfo(); + Task GetProjectAndOrgInfo(); /// /// Get project and organization information for a specific project ID. /// - OrganizationAndProjectInfo? GetProjectAndOrgInfo(string projectId); + Task GetProjectAndOrgInfo(string projectId); /// /// Get or create project and organization information from config. /// - OrganizationAndProjectInfo GetOrCreateProjectAndOrgInfo(); + Task GetOrCreateProjectAndOrgInfo(); } diff --git a/src/Braintrust.Sdk/Api/Models.cs b/src/Braintrust.Sdk/Api/Models.cs index cdd981b..b446d33 100644 --- a/src/Braintrust.Sdk/Api/Models.cs +++ b/src/Braintrust.Sdk/Api/Models.cs @@ -1,5 +1,3 @@ -using System; -using System.Collections.Generic; using System.Text.Json.Serialization; namespace Braintrust.Sdk.Api; diff --git a/src/Braintrust.Sdk/Braintrust.cs b/src/Braintrust.Sdk/Braintrust.cs index 4920130..08f28c1 100644 --- a/src/Braintrust.Sdk/Braintrust.cs +++ b/src/Braintrust.Sdk/Braintrust.cs @@ -1,5 +1,3 @@ -using System; -using System.Threading; using Braintrust.Sdk.Api; using Braintrust.Sdk.Config; using Microsoft.Extensions.Logging; @@ -47,7 +45,7 @@ public static Braintrust Get() /// /// Braintrust configuration /// When true, automatically set up Braintrust connection and shutdown hooks - public static Braintrust Get(BraintrustConfig config, Boolean autoManageOpenTelemetry = true) + public static Braintrust Get(BraintrustConfig config, bool autoManageOpenTelemetry = true) { var current = _instance; if (current == null) @@ -84,7 +82,7 @@ internal static void ResetForTest() /// /// Create a new Braintrust instance from the given config. /// - public static Braintrust Of(BraintrustConfig config, Boolean autoManageOpenTelemetry = true) + public static Braintrust Of(BraintrustConfig config, bool autoManageOpenTelemetry = true) { var apiClient = BraintrustApiClient.Of(config); return new Braintrust(config, apiClient, autoManageOpenTelemetry); @@ -94,22 +92,22 @@ public static Braintrust Of(BraintrustConfig config, Boolean autoManageOpenTelem public IBraintrustApiClient ApiClient { get; } private volatile OpenTelemetry.Trace.TracerProvider? _tracer; - private Braintrust(BraintrustConfig config, IBraintrustApiClient apiClient, Boolean autoManageOpenTelemetry) + private Braintrust(BraintrustConfig config, IBraintrustApiClient apiClient, bool autoManageOpenTelemetry) { Config = config ?? throw new ArgumentNullException(nameof(config)); ApiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient)); if (autoManageOpenTelemetry) { - this._tracer = Trace.BraintrustTracing.CreateTracerProvider(this.Config); + _tracer = Trace.BraintrustTracing.CreateTracerProvider(this.Config); } } /// /// Get the URI to the configured Braintrust org and project. /// - public Uri ProjectUri() + public async Task GetProjectUriAsync() { - var orgAndProject = ApiClient.GetOrCreateProjectAndOrgInfo(); + var orgAndProject = await ApiClient.GetOrCreateProjectAndOrgInfo().ConfigureAwait(false); return new Uri($"{Config.AppUrl}/app/{orgAndProject.OrgInfo.Name}/p/{orgAndProject.Project.Name}"); } @@ -122,9 +120,9 @@ public Uri ProjectUri() /// public void OpenTelemetryEnable(OpenTelemetry.Trace.TracerProviderBuilder tracerProviderBuilder, ILoggingBuilder loggingBuilder, MeterProviderBuilder meterProviderBuilder) { - if (this._tracer != null) + if (_tracer != null) { - throw new System.InvalidOperationException("cannot call enable for Braintrusts which autoManage Open Telemetry"); + throw new InvalidOperationException("cannot call enable for Braintrusts which autoManage Open Telemetry"); } Trace.BraintrustTracing.Enable(Config, tracerProviderBuilder, loggingBuilder, meterProviderBuilder); } diff --git a/src/Braintrust.Sdk/Config/BaseConfig.cs b/src/Braintrust.Sdk/Config/BaseConfig.cs index ca7aa54..127d1e1 100644 --- a/src/Braintrust.Sdk/Config/BaseConfig.cs +++ b/src/Braintrust.Sdk/Config/BaseConfig.cs @@ -1,40 +1,28 @@ -using System; -using System.Collections.Generic; -using System.Linq; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; namespace Braintrust.Sdk.Config; -public class BaseConfig +public abstract class BaseConfig { /// /// Sentinel used to set null in the env. Only used for testing. /// - internal static readonly string NullOverride = $"BRAINTRUST_NULL_SENTINAL_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}"; + internal static readonly string NullOverride = $"BRAINTRUST_NULL_SENTINEL_{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}"; - protected readonly IReadOnlyDictionary EnvOverrides; + protected readonly IReadOnlyDictionary EnvOverrides; - protected BaseConfig(IDictionary envOverrides) + protected BaseConfig(IDictionary envOverrides) { - EnvOverrides = new Dictionary(envOverrides); + EnvOverrides = new Dictionary(envOverrides); } - protected T GetConfig(string settingName, T defaultValue) where T : notnull - { - ArgumentNullException.ThrowIfNull(defaultValue); - return GetConfig(settingName, defaultValue, typeof(T))!; - } - - protected T? GetConfig(string settingName, T? defaultValue, Type settingType) + [return: NotNullIfNotNull(nameof(defaultValue))] + protected T? GetConfig(string settingName, T? defaultValue) + where T : IParsable { var rawVal = GetEnvValue(settingName); - if (rawVal == null) - { - return defaultValue; - } - else - { - return (T?)Cast(rawVal, settingType); - } + return rawVal == null ? defaultValue : T.Parse(rawVal, CultureInfo.InvariantCulture); } protected string GetRequiredConfig(string settingName) @@ -43,46 +31,10 @@ protected string GetRequiredConfig(string settingName) } protected T GetRequiredConfig(string settingName) + where T : IParsable { - var value = GetConfig(settingName, default, typeof(T)); - if (value == null) - { - throw new InvalidOperationException($"{settingName} is required"); - } - return value; - } - - protected object Cast(string value, Type type) - { - if (type == typeof(string)) - { - return value; - } - else if (type == typeof(bool)) - { - return bool.Parse(value); - } - else if (type == typeof(int)) - { - return int.Parse(value); - } - else if (type == typeof(long)) - { - return long.Parse(value); - } - else if (type == typeof(float)) - { - return float.Parse(value); - } - else if (type == typeof(double)) - { - return double.Parse(value); - } - else - { - throw new InvalidOperationException( - $"Unsupported default class: {type} -- please implement or use a different default"); - } + var value = GetConfig(settingName, default); + return value ?? throw new InvalidOperationException($"{settingName} is required"); } protected string? GetEnvValue(string settingName) diff --git a/src/Braintrust.Sdk/Config/BraintrustConfig.cs b/src/Braintrust.Sdk/Config/BraintrustConfig.cs index 214ec8b..17da39c 100644 --- a/src/Braintrust.Sdk/Config/BraintrustConfig.cs +++ b/src/Braintrust.Sdk/Config/BraintrustConfig.cs @@ -1,6 +1,3 @@ -using System; -using System.Collections.Generic; - namespace Braintrust.Sdk.Config; /// @@ -21,44 +18,33 @@ public sealed class BraintrustConfig : BaseConfig public string? DefaultProjectName { get; } public bool EnableTraceConsoleLog { get; } public bool Debug { get; } - public bool ExperimentalOtelLogs { get; } public TimeSpan RequestTimeout { get; } - /// - /// Setting for unit testing. Do not use in production. - /// - public bool ExportSpansInMemoryForUnitTest { get; } - public static BraintrustConfig FromEnvironment() { return Of(); } - public static BraintrustConfig Of(params string[] envOverrides) + public static BraintrustConfig Of(params (string Key, string? Value)[] envOverrides) { - if (envOverrides.Length % 2 != 0) - { - throw new ArgumentException( - $"config overrides require key-value pairs. Found dangling key: {envOverrides[^1]}"); - } - - var overridesMap = new Dictionary(); - for (int i = 0; i < envOverrides.Length - 1; i += 2) + var overridesMap = new Dictionary(); + + foreach (var (key, value) in envOverrides) { - overridesMap[envOverrides[i]] = envOverrides[i + 1]; + overridesMap[key] = value; } return new BraintrustConfig(overridesMap); } - private BraintrustConfig(IDictionary envOverrides) : base(envOverrides) + private BraintrustConfig(IDictionary envOverrides) : base(envOverrides) { ApiKey = GetRequiredConfig("BRAINTRUST_API_KEY"); ApiUrl = GetConfig("BRAINTRUST_API_URL", "https://api.braintrust.dev"); AppUrl = GetConfig("BRAINTRUST_APP_URL", "https://www.braintrust.dev"); TracesPath = GetConfig("BRAINTRUST_TRACES_PATH", "/otel/v1/traces"); LogsPath = GetConfig("BRAINTRUST_LOGS_PATH", "/otel/v1/logs"); - DefaultProjectId = GetConfig("BRAINTRUST_DEFAULT_PROJECT_ID", null, typeof(string)); + DefaultProjectId = GetConfig("BRAINTRUST_DEFAULT_PROJECT_ID", null); DefaultProjectName = GetConfig("BRAINTRUST_DEFAULT_PROJECT_NAME", "default-dotnet-project"); EnableTraceConsoleLog = GetConfig("BRAINTRUST_ENABLE_TRACE_CONSOLE_LOG", false); Debug = GetConfig("BRAINTRUST_DEBUG", false); diff --git a/src/Braintrust.Sdk/Eval/DatasetCase.cs b/src/Braintrust.Sdk/Eval/DatasetCase.cs index 143b896..909bec3 100644 --- a/src/Braintrust.Sdk/Eval/DatasetCase.cs +++ b/src/Braintrust.Sdk/Eval/DatasetCase.cs @@ -1,9 +1,43 @@ -using System; -using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; namespace Braintrust.Sdk.Eval; +public static class DatasetCase +{ + /// + /// Creates a new dataset case. + /// + public static DatasetCase Of( + TInput input, + TOutput expected, + IReadOnlyList tags, + IReadOnlyDictionary metadata) + where TInput : notnull + where TOutput : notnull + => new(input, expected, tags, metadata); + + /// + /// Creates a new dataset case. + /// + public static DatasetCase Of( + TInput input, + TOutput expected, + IReadOnlyList tags) + where TInput : notnull + where TOutput : notnull + => new(input, expected, tags); + + /// + /// Creates a new dataset case. + /// + public static DatasetCase Of( + TInput input, + TOutput expected) + where TInput : notnull + where TOutput : notnull + => new(input, expected); +} + /// /// A single row in a dataset. /// @@ -39,8 +73,20 @@ public DatasetCase( this.Metadata = metadata; } - public static DatasetCase Of(TInput input, TOutput expected) + [SetsRequiredMembers] + public DatasetCase( + TInput input, + TOutput expected, + IReadOnlyList tags) + : this(input, expected, tags, new Dictionary()) + { + } + + [SetsRequiredMembers] + public DatasetCase( + TInput input, + TOutput expected) + : this(input, expected, []) { - return new DatasetCase(input, expected, Array.Empty(), new Dictionary()); } } diff --git a/src/Braintrust.Sdk/Eval/DatasetInMemoryImpl.cs b/src/Braintrust.Sdk/Eval/DatasetInMemoryImpl.cs index 705d324..7a69127 100644 --- a/src/Braintrust.Sdk/Eval/DatasetInMemoryImpl.cs +++ b/src/Braintrust.Sdk/Eval/DatasetInMemoryImpl.cs @@ -1,68 +1,29 @@ -using System; -using System.Collections.Generic; -using System.Linq; - namespace Braintrust.Sdk.Eval; /// /// A dataset held entirely in memory. /// -internal class DatasetInMemoryImpl : Dataset +internal class DatasetInMemoryImpl : IDataset where TInput : notnull where TOutput : notnull { private readonly IReadOnlyList> _cases; - private readonly string _id; public DatasetInMemoryImpl(IEnumerable> cases) { _cases = cases.ToList(); - _id = $"in-memory-dataset<{_cases.GetHashCode()}>"; + Id = $"in-memory-dataset<{_cases.GetHashCode()}>"; } - public string Id => _id; + public string Id { get; } public string Version => "0"; - public ICursor> OpenCursor() - { - return new InMemoryCursor(_cases); - } - - private class InMemoryCursor : ICursor> + public async IAsyncEnumerable> GetCasesAsync() { - private readonly IReadOnlyList> _cases; - private int _nextIndex = 0; - private bool _closed = false; - - public InMemoryCursor(IReadOnlyList> cases) - { - _cases = cases; - } - - public DatasetCase? Next() - { - if (_closed) - { - throw new InvalidOperationException("This method may not be invoked after Close"); - } - - if (_nextIndex < _cases.Count) - { - return _cases[_nextIndex++]; - } - - return default; - } - - public void Close() - { - _closed = true; - } - - public void Dispose() + foreach (var item in _cases) { - Close(); + yield return item; } } } diff --git a/src/Braintrust.Sdk/Eval/Eval.cs b/src/Braintrust.Sdk/Eval/Eval.cs index e65cb31..b893937 100644 --- a/src/Braintrust.Sdk/Eval/Eval.cs +++ b/src/Braintrust.Sdk/Eval/Eval.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; using System.Diagnostics; -using System.Linq; using System.Text.Json; using Braintrust.Sdk.Api; using Braintrust.Sdk.Config; @@ -29,26 +26,16 @@ public sealed class Eval private readonly IBraintrustApiClient _client; private readonly OrganizationAndProjectInfo _orgAndProject; private readonly ActivitySource _activitySource; - private readonly Dataset _dataset; - private readonly Task _task; - private readonly IReadOnlyList> _scorers; + private readonly IDataset _dataset; + private readonly ITask _task; + private readonly IReadOnlyList> _scorers; - private Eval(Builder builder) + private Eval(Builder builder, OrganizationAndProjectInfo orgAndProject) { _experimentName = builder._experimentName; _config = builder._config ?? throw new ArgumentNullException(nameof(builder._config)); _client = builder._apiClient ?? throw new ArgumentNullException(nameof(builder._apiClient)); - - if (builder._projectId == null) - { - _orgAndProject = _client.GetProjectAndOrgInfo() - ?? throw new InvalidOperationException("Unable to retrieve project and org info"); - } - else - { - _orgAndProject = _client.GetProjectAndOrgInfo(builder._projectId) - ?? throw new InvalidOperationException($"Invalid project id: {builder._projectId}"); - } + _orgAndProject = orgAndProject ?? throw new ArgumentNullException(nameof(orgAndProject)); _activitySource = builder._activitySource ?? throw new ArgumentNullException(nameof(builder._activitySource)); _dataset = builder._dataset ?? throw new ArgumentNullException(nameof(builder._dataset)); @@ -59,24 +46,19 @@ private Eval(Builder builder) /// /// Runs the evaluation and returns results. /// - public EvalResult Run() + public async Task RunAsync() { - var experiment = _client.GetOrCreateExperiment( + var experiment = await _client.GetOrCreateExperiment( new CreateExperimentRequest( _orgAndProject.Project.Id, - _experimentName, - null, - null)); + _experimentName)) + .ConfigureAwait(false); var experimentId = experiment.Id; - using (var cursor = _dataset.OpenCursor()) + await foreach (var datasetCase in _dataset.GetCasesAsync()) { - DatasetCase? datasetCase; - while ((datasetCase = cursor.Next()) != null) - { - EvalOne(experimentId, datasetCase); - } + EvalOne(experimentId, datasetCase); } var experimentUrl = CreateExperimentUrl(_config.AppUrl, _orgAndProject, _experimentName); @@ -144,7 +126,7 @@ private void EvalOne(string experimentId, DatasetCase datasetCa if (score.Value < 0.0 || score.Value > 1.0) { throw new InvalidOperationException( - $"Score must be between 0 and 1: {scorer.GetName()} : {score}"); + $"Score must be between 0 and 1: {scorer.Name} : {score}"); } nameToScore[score.Name] = score.Value; } @@ -205,14 +187,14 @@ public sealed class Builder internal IBraintrustApiClient? _apiClient; internal string? _projectId; internal ActivitySource? _activitySource; - internal Dataset? _dataset; - internal Task? _task; - internal List> _scorers = new(); + internal IDataset? _dataset; + internal ITask? _task; + internal List> _scorers = new(); /// /// Build the Eval instance. /// - public Eval Build() + public async Task> BuildAsync() { _config ??= BraintrustConfig.FromEnvironment(); _activitySource ??= BraintrustTracing.GetActivitySource(); @@ -234,7 +216,20 @@ public Eval Build() throw new InvalidOperationException("Must provide a task"); } - return new Eval(this); + OrganizationAndProjectInfo? orgAndProject; + + if (_projectId == null) + { + orgAndProject = await _apiClient.GetProjectAndOrgInfo().ConfigureAwait(false) + ?? throw new InvalidOperationException("Unable to retrieve project and org info"); + } + else + { + orgAndProject = await _apiClient.GetProjectAndOrgInfo(_projectId).ConfigureAwait(false) + ?? throw new InvalidOperationException($"Invalid project id: {_projectId}"); + } + + return new Eval(this, orgAndProject); } /// @@ -285,7 +280,7 @@ public Builder ActivitySource(ActivitySource activitySource) /// /// Set the dataset. /// - public Builder Dataset(Dataset dataset) + public Builder Dataset(IDataset dataset) { _dataset = dataset; return this; @@ -300,13 +295,13 @@ public Builder Cases(params DatasetCase[] cases) { throw new ArgumentException("Must provide at least one case", nameof(cases)); } - return Dataset(Eval.Dataset.Of(cases)); + return Dataset(Eval.Dataset.Of(cases)); } /// /// Set the task. /// - public Builder Task(Task task) + public Builder Task(ITask task) { _task = task; return this; @@ -324,13 +319,13 @@ public Builder TaskFunction(Func taskFn) /// /// Set the scorers. /// - public Builder Scorers(params Scorer[] scorers) + public Builder Scorers(params IScorer[] scorers) { _scorers = scorers.ToList(); return this; } - private class FunctionTask : Task + private class FunctionTask : ITask { private readonly Func _taskFn; diff --git a/src/Braintrust.Sdk/Eval/EvalResult.cs b/src/Braintrust.Sdk/Eval/EvalResult.cs index a44b6c0..dcd808b 100644 --- a/src/Braintrust.Sdk/Eval/EvalResult.cs +++ b/src/Braintrust.Sdk/Eval/EvalResult.cs @@ -3,7 +3,7 @@ namespace Braintrust.Sdk.Eval; /// /// Results of all eval cases of an experiment. /// -public class EvalResult +public readonly struct EvalResult { /// /// URL to view the experiment results in Braintrust. @@ -22,4 +22,6 @@ public string CreateReportString() { return $"Experiment complete. View results in braintrust: {ExperimentUrl}"; } + + public override string ToString() => CreateReportString(); } diff --git a/src/Braintrust.Sdk/Eval/FunctionScorer.cs b/src/Braintrust.Sdk/Eval/FunctionScorer.cs new file mode 100644 index 0000000..fe750a9 --- /dev/null +++ b/src/Braintrust.Sdk/Eval/FunctionScorer.cs @@ -0,0 +1,24 @@ +namespace Braintrust.Sdk.Eval; + +/// +/// Implementation of a scorer from a function. +/// +public class FunctionScorer : IScorer + where TInput : notnull + where TOutput : notnull +{ + private readonly Func _scorerFn; + + public FunctionScorer(string name, Func scorerFn) + { + Name = name; + _scorerFn = scorerFn; + } + + public string Name { get; } + + public IReadOnlyList Score(TaskResult taskResult) + { + return [new Score(Name, _scorerFn(taskResult.DatasetCase.Expected, taskResult.Result))]; + } +} \ No newline at end of file diff --git a/src/Braintrust.Sdk/Eval/Dataset.cs b/src/Braintrust.Sdk/Eval/IDataset.cs similarity index 54% rename from src/Braintrust.Sdk/Eval/Dataset.cs rename to src/Braintrust.Sdk/Eval/IDataset.cs index 45d0245..b0a8686 100644 --- a/src/Braintrust.Sdk/Eval/Dataset.cs +++ b/src/Braintrust.Sdk/Eval/IDataset.cs @@ -1,5 +1,3 @@ -using System; - namespace Braintrust.Sdk.Eval; /// @@ -10,14 +8,14 @@ namespace Braintrust.Sdk.Eval; /// /// Type of the input data /// Type of the output data -public interface Dataset +public interface IDataset where TInput : notnull where TOutput : notnull { /// /// Open a cursor to iterate through dataset cases. /// - ICursor> OpenCursor(); + IAsyncEnumerable> GetCasesAsync(); /// /// Gets the dataset ID. @@ -28,34 +26,22 @@ public interface Dataset /// Gets the dataset version. /// string Version { get; } - - /// - /// Create an in-memory Dataset containing the provided cases. - /// - public static Dataset Of(params DatasetCase[] cases) - { - return new DatasetInMemoryImpl(cases); - } } /// -/// A cursor for iterating through dataset cases. -/// Not thread-safe. +/// Datasets define the cases for evals. This class provides factories for in-memory datasets. /// -/// The type of case being iterated -public interface ICursor : IDisposable +public static class Dataset { /// - /// Fetch the next case. Returns null if there are no more cases to fetch. - /// - /// Implementations may make external requests to fetch data. - /// - /// If this method is invoked after Close() or Dispose(), an InvalidOperationException will be thrown. - /// - TCase? Next(); - - /// - /// Close the cursor and release all resources. + /// Create an in-memory Dataset containing the provided cases. /// - void Close(); -} + /// Type of the input data + /// Type of the output data + public static IDataset Of(params DatasetCase[] cases) + where TInput : notnull + where TOutput : notnull + { + return new DatasetInMemoryImpl(cases); + } +} \ No newline at end of file diff --git a/src/Braintrust.Sdk/Eval/IScorer.cs b/src/Braintrust.Sdk/Eval/IScorer.cs new file mode 100644 index 0000000..293cd16 --- /dev/null +++ b/src/Braintrust.Sdk/Eval/IScorer.cs @@ -0,0 +1,21 @@ +namespace Braintrust.Sdk.Eval; + +/// +/// A scorer evaluates the result of a test case with a score between 0 (inclusive) and 1 (inclusive). +/// +/// Type of the input data +/// Type of the output data +public interface IScorer + where TInput : notnull + where TOutput : notnull +{ + /// + /// Gets the name of this scorer. + /// + string Name { get; } + + /// + /// Score the task result and return one or more scores. + /// + IReadOnlyList Score(TaskResult taskResult); +} \ No newline at end of file diff --git a/src/Braintrust.Sdk/Eval/Task.cs b/src/Braintrust.Sdk/Eval/ITask.cs similarity index 92% rename from src/Braintrust.Sdk/Eval/Task.cs rename to src/Braintrust.Sdk/Eval/ITask.cs index 58c11d8..0cab335 100644 --- a/src/Braintrust.Sdk/Eval/Task.cs +++ b/src/Braintrust.Sdk/Eval/ITask.cs @@ -5,7 +5,7 @@ namespace Braintrust.Sdk.Eval; /// /// Type of the input data /// Type of the output data -public interface Task +public interface ITask where TInput : notnull where TOutput : notnull { diff --git a/src/Braintrust.Sdk/Eval/Score.cs b/src/Braintrust.Sdk/Eval/Score.cs index 3e66686..8c19f9a 100644 --- a/src/Braintrust.Sdk/Eval/Score.cs +++ b/src/Braintrust.Sdk/Eval/Score.cs @@ -5,4 +5,4 @@ namespace Braintrust.Sdk.Eval; /// /// Name of the metric being scored. This does not have to be the same as the scorer name, but it often will be. /// Numeric representation of how well the task performed. Must be between 0.0 (inclusive) and 1.0 (inclusive). 0 is completely incorrect. 1 is completely correct. -public record Score(string Name, double Value); +public readonly record struct Score(string Name, double Value); diff --git a/src/Braintrust.Sdk/Eval/Scorer.cs b/src/Braintrust.Sdk/Eval/Scorer.cs deleted file mode 100644 index e8a092f..0000000 --- a/src/Braintrust.Sdk/Eval/Scorer.cs +++ /dev/null @@ -1,59 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace Braintrust.Sdk.Eval; - -/// -/// A scorer evaluates the result of a test case with a score between 0 (inclusive) and 1 (inclusive). -/// -/// Type of the input data -/// Type of the output data -public interface Scorer - where TInput : notnull - where TOutput : notnull -{ - /// - /// Gets the name of this scorer. - /// - string GetName(); - - /// - /// Score the task result and return one or more scores. - /// - List Score(TaskResult taskResult); - - /// - /// Create a scorer from a function that takes the output and returns a score. - /// - /// Name of the scorer - /// Function that takes (expectedValue, actualValue) and returns a score between 0.0 and 1.0 - /// A scorer instance - public static Scorer Of(string scorerName, Func scorerFn) - { - return new FunctionScorer(scorerName, scorerFn); - } -} - -/// -/// Internal implementation of a scorer from a function. -/// -internal class FunctionScorer : Scorer - where TInput : notnull - where TOutput : notnull -{ - private readonly string _name; - private readonly Func _scorerFn; - - public FunctionScorer(string name, Func scorerFn) - { - _name = name; - _scorerFn = scorerFn; - } - - public string GetName() => _name; - - public List Score(TaskResult taskResult) - { - return new List { new Score(_name, _scorerFn(taskResult.DatasetCase.Expected, taskResult.Result)) }; - } -} diff --git a/src/Braintrust.Sdk/Eval/TaskResult.cs b/src/Braintrust.Sdk/Eval/TaskResult.cs index 14cd839..dbefc30 100644 --- a/src/Braintrust.Sdk/Eval/TaskResult.cs +++ b/src/Braintrust.Sdk/Eval/TaskResult.cs @@ -7,7 +7,7 @@ namespace Braintrust.Sdk.Eval; /// The type of output data /// Task output /// The dataset case the task ran against to produce the result -public record TaskResult( +public readonly record struct TaskResult( TOutput Result, DatasetCase DatasetCase) where TInput : notnull diff --git a/src/Braintrust.Sdk/Instrumentation/OpenAI/BraintrustOpenAI.cs b/src/Braintrust.Sdk/Instrumentation/OpenAI/BraintrustOpenAI.cs index 2dc7ec5..e5b156d 100644 --- a/src/Braintrust.Sdk/Instrumentation/OpenAI/BraintrustOpenAI.cs +++ b/src/Braintrust.Sdk/Instrumentation/OpenAI/BraintrustOpenAI.cs @@ -1,8 +1,6 @@ -using System; using System.ClientModel; using System.ClientModel.Primitives; using System.Diagnostics; -using System.Net.Http; using OpenAI; namespace Braintrust.Sdk.Instrumentation.OpenAI; diff --git a/src/Braintrust.Sdk/Instrumentation/OpenAI/BraintrustPipelineTransport.cs b/src/Braintrust.Sdk/Instrumentation/OpenAI/BraintrustPipelineTransport.cs index bbf41fc..3dc510d 100644 --- a/src/Braintrust.Sdk/Instrumentation/OpenAI/BraintrustPipelineTransport.cs +++ b/src/Braintrust.Sdk/Instrumentation/OpenAI/BraintrustPipelineTransport.cs @@ -1,10 +1,6 @@ -using System; using System.ClientModel; using System.ClientModel.Primitives; using System.Diagnostics; -using System.IO; -using System.Threading; -using System.Threading.Tasks; namespace Braintrust.Sdk.Instrumentation.OpenAI; diff --git a/src/Braintrust.Sdk/Instrumentation/OpenAI/InstrumentedChatClient.cs b/src/Braintrust.Sdk/Instrumentation/OpenAI/InstrumentedChatClient.cs new file mode 100644 index 0000000..d8194e8 --- /dev/null +++ b/src/Braintrust.Sdk/Instrumentation/OpenAI/InstrumentedChatClient.cs @@ -0,0 +1,175 @@ +using System.ClientModel; +using System.Diagnostics; +using System.Text.Json.Nodes; +using OpenAI.Chat; +using OpenTelemetry.Trace; + +namespace Braintrust.Sdk.Instrumentation.OpenAI; + +/// +/// Decorator wrapper for ChatClient that adds telemetry instrumentation. +/// +/// This class intercepts chat completion calls and wraps them with spans +/// that capture request and response data. +/// +internal sealed class InstrumentedChatClient : ChatClient +{ + private readonly ChatClient _client; + private readonly ActivitySource _activitySource; + private readonly bool _captureMessageContent; + + /// + /// Creates an instrumented wrapper for the given ChatClient. + /// + internal static ChatClient Create( + ChatClient client, + ActivitySource activitySource, + bool captureMessageContent) + { + return new InstrumentedChatClient(client, activitySource, captureMessageContent); + } + + private InstrumentedChatClient( + ChatClient client, + ActivitySource activitySource, + bool captureMessageContent) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + _activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); + _captureMessageContent = captureMessageContent; + } + + /// + /// Intercepts CompleteChat to add span instrumentation (synchronous). + /// + public override ClientResult CompleteChat(IEnumerable messages, ChatCompletionOptions? options = null, CancellationToken cancellationToken = default) + { + // Start a span for the chat completion + var activity = _activitySource.StartActivity("Chat Completion", ActivityKind.Client); + var startTime = DateTime.UtcNow; + + try + { + // Call the underlying client - this will trigger HTTP call and capture JSON + var result = _client.CompleteChat(messages, options, cancellationToken); + + // Calculate time to first token + var timeToFirstToken = (DateTime.UtcNow - startTime).TotalSeconds; + + // Get the captured HTTP JSON from Activity baggage + if (activity != null && _captureMessageContent) + { + TagActivity(activity, timeToFirstToken); + } + + return result; + } + catch (Exception ex) + { + // Record the exception in the span + if (activity != null) + { + activity.SetStatus(ActivityStatusCode.Error, ex.Message); + activity.RecordException(ex); + } + // intentionally re-throwing original exception + throw; + } + finally + { + // Always dispose the activity + activity?.Dispose(); + } + } + + /// + /// Intercepts CompleteChatAsync to add span instrumentation. + /// + public override async Task> CompleteChatAsync(IEnumerable messages, ChatCompletionOptions? options = null, CancellationToken cancellationToken = default) + { + // Start a span for the chat completion + var activity = _activitySource.StartActivity("Chat Completion", ActivityKind.Client); + var startTime = DateTime.UtcNow; + + try + { + // Call the underlying client - this will trigger HTTP call and capture JSON + var result = await _client.CompleteChatAsync(messages, options, cancellationToken).ConfigureAwait(false); + + // Calculate time to first token + var timeToFirstToken = (DateTime.UtcNow - startTime).TotalSeconds; + + // Get the captured HTTP JSON from Activity baggage + if (activity != null && _captureMessageContent) + { + TagActivity(activity, timeToFirstToken); + } + + return result; + } + catch (Exception ex) + { + // Record the exception in the span + if (activity != null) + { + activity.SetStatus(ActivityStatusCode.Error, ex.Message); + activity.RecordException(ex); + } + // intentionally re-throwing original exception + throw; + } + finally + { + // Always dispose the activity after the async operation completes + activity?.Dispose(); + } + } + + // TODO: Override other methods as needed (CompleteChatStreaming, etc.) + private void TagActivity(Activity activity, double? timeToFirstToken = null) + { + activity.SetTag("provider", "openai"); + { + var requestRaw = activity.GetBaggageItem("braintrust.http.request"); + if (requestRaw != null) + { + var requestJson = JsonNode.Parse(requestRaw); + activity.SetTag("gen_ai.request.model", requestJson?["model"]?.ToString()); + activity.SetTag("braintrust.input_json", requestJson?["messages"]?.ToString()); + } + } + { + var responseRaw = activity.GetBaggageItem("braintrust.http.response"); + if (responseRaw != null) + { + var responseJson = JsonNode.Parse(responseRaw); + activity.SetTag("gen_ai.response.model", responseJson?["model"]?.ToString()); + activity.SetTag("braintrust.output_json", responseJson?["choices"]?.ToString()); + + // Extract token usage metrics + var usage = responseJson?["usage"]; + if (usage != null) + { + var promptTokens = usage["prompt_tokens"]?.GetValue(); + var completionTokens = usage["completion_tokens"]?.GetValue(); + var totalTokens = usage["total_tokens"]?.GetValue(); + + if (promptTokens.HasValue) + activity.SetTag("braintrust.metrics.prompt_tokens", promptTokens.Value); + if (completionTokens.HasValue) + activity.SetTag("braintrust.metrics.completion_tokens", completionTokens.Value); + if (totalTokens.HasValue) + activity.SetTag("braintrust.metrics.tokens", totalTokens.Value); + } + + // Set time_to_first_token metric + // For non-streaming responses, this is the total response time + if (timeToFirstToken.HasValue && timeToFirstToken.Value > 0) + { + activity.SetTag("braintrust.metrics.time_to_first_token", timeToFirstToken.Value); + } + } + } + } + +} \ No newline at end of file diff --git a/src/Braintrust.Sdk/Instrumentation/OpenAI/InstrumentedOpenAIClient.cs b/src/Braintrust.Sdk/Instrumentation/OpenAI/InstrumentedOpenAIClient.cs index 23edc1b..4451118 100644 --- a/src/Braintrust.Sdk/Instrumentation/OpenAI/InstrumentedOpenAIClient.cs +++ b/src/Braintrust.Sdk/Instrumentation/OpenAI/InstrumentedOpenAIClient.cs @@ -1,10 +1,4 @@ -using System; -using System.ClientModel; -using System.Collections.Generic; using System.Diagnostics; -using System.Text.Json.Nodes; -using System.Threading; -using System.Threading.Tasks; using OpenAI; using OpenAI.Audio; using OpenAI.Chat; @@ -12,7 +6,6 @@ using OpenAI.Files; using OpenAI.Images; using OpenAI.Moderations; -using OpenTelemetry.Trace; namespace Braintrust.Sdk.Instrumentation.OpenAI; @@ -105,172 +98,4 @@ public override ModerationClient GetModerationClient(string model) // Note: GetAssistantClient, GetVectorStoreClient, and GetBatchClient are experimental APIs // that may not exist in all versions. They will fall through to the base class if called. // TODO: Add delegation for experimental APIs if they become stable -} - -/// -/// Decorator wrapper for ChatClient that adds telemetry instrumentation. -/// -/// This class intercepts chat completion calls and wraps them with spans -/// that capture request and response data. -/// -internal sealed class InstrumentedChatClient : ChatClient -{ - private readonly ChatClient _client; - private readonly ActivitySource _activitySource; - private readonly bool _captureMessageContent; - - /// - /// Creates an instrumented wrapper for the given ChatClient. - /// - internal static ChatClient Create( - ChatClient client, - ActivitySource activitySource, - bool captureMessageContent) - { - return new InstrumentedChatClient(client, activitySource, captureMessageContent); - } - - private InstrumentedChatClient( - ChatClient client, - ActivitySource activitySource, - bool captureMessageContent) - { - _client = client ?? throw new ArgumentNullException(nameof(client)); - _activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); - _captureMessageContent = captureMessageContent; - } - - /// - /// Intercepts CompleteChat to add span instrumentation (synchronous). - /// - public override ClientResult CompleteChat(IEnumerable messages, ChatCompletionOptions? options = null, CancellationToken cancellationToken = default) - { - // Start a span for the chat completion - var activity = _activitySource.StartActivity("Chat Completion", ActivityKind.Client); - var startTime = DateTime.UtcNow; - - try - { - // Call the underlying client - this will trigger HTTP call and capture JSON - var result = _client.CompleteChat(messages, options, cancellationToken); - - // Calculate time to first token - var timeToFirstToken = (DateTime.UtcNow - startTime).TotalSeconds; - - // Get the captured HTTP JSON from Activity baggage - if (activity != null && _captureMessageContent) - { - TagActivity(activity, timeToFirstToken); - } - - return result; - } - catch (Exception ex) - { - // Record the exception in the span - if (activity != null) - { - activity.SetStatus(ActivityStatusCode.Error, ex.Message); - activity.RecordException(ex); - } - // intentionally re-throwing original exception - throw; - } - finally - { - // Always dispose the activity - activity?.Dispose(); - } - } - - /// - /// Intercepts CompleteChatAsync to add span instrumentation. - /// - public override async Task> CompleteChatAsync(IEnumerable messages, ChatCompletionOptions? options = null, CancellationToken cancellationToken = default) - { - // Start a span for the chat completion - var activity = _activitySource.StartActivity("Chat Completion", ActivityKind.Client); - var startTime = DateTime.UtcNow; - - try - { - // Call the underlying client - this will trigger HTTP call and capture JSON - var result = await _client.CompleteChatAsync(messages, options, cancellationToken).ConfigureAwait(false); - - // Calculate time to first token - var timeToFirstToken = (DateTime.UtcNow - startTime).TotalSeconds; - - // Get the captured HTTP JSON from Activity baggage - if (activity != null && _captureMessageContent) - { - TagActivity(activity, timeToFirstToken); - } - - return result; - } - catch (Exception ex) - { - // Record the exception in the span - if (activity != null) - { - activity.SetStatus(ActivityStatusCode.Error, ex.Message); - activity.RecordException(ex); - } - // intentionally re-throwing original exception - throw; - } - finally - { - // Always dispose the activity after the async operation completes - activity?.Dispose(); - } - } - - // TODO: Override other methods as needed (CompleteChatStreaming, etc.) - private void TagActivity(Activity activity, double? timeToFirstToken = null) - { - activity.SetTag("provider", "openai"); - { - var requestRaw = activity.GetBaggageItem("braintrust.http.request"); - if (requestRaw != null) - { - var requestJson = JsonNode.Parse(requestRaw); - activity.SetTag("gen_ai.request.model", requestJson?["model"]?.ToString()); - activity.SetTag("braintrust.input_json", requestJson?["messages"]?.ToString()); - } - } - { - var responseRaw = activity.GetBaggageItem("braintrust.http.response"); - if (responseRaw != null) - { - var responseJson = JsonNode.Parse(responseRaw); - activity.SetTag("gen_ai.response.model", responseJson?["model"]?.ToString()); - activity.SetTag("braintrust.output_json", responseJson?["choices"]?.ToString()); - - // Extract token usage metrics - var usage = responseJson?["usage"]; - if (usage != null) - { - var promptTokens = usage["prompt_tokens"]?.GetValue(); - var completionTokens = usage["completion_tokens"]?.GetValue(); - var totalTokens = usage["total_tokens"]?.GetValue(); - - if (promptTokens.HasValue) - activity.SetTag("braintrust.metrics.prompt_tokens", promptTokens.Value); - if (completionTokens.HasValue) - activity.SetTag("braintrust.metrics.completion_tokens", completionTokens.Value); - if (totalTokens.HasValue) - activity.SetTag("braintrust.metrics.tokens", totalTokens.Value); - } - - // Set time_to_first_token metric - // For non-streaming responses, this is the total response time - if (timeToFirstToken.HasValue && timeToFirstToken.Value > 0) - { - activity.SetTag("braintrust.metrics.time_to_first_token", timeToFirstToken.Value); - } - } - } - } - -} +} \ No newline at end of file diff --git a/src/Braintrust.Sdk/Instrumentation/OpenAI/OpenAITelemetry.cs b/src/Braintrust.Sdk/Instrumentation/OpenAI/OpenAITelemetry.cs index d0b2f09..f5832ff 100644 --- a/src/Braintrust.Sdk/Instrumentation/OpenAI/OpenAITelemetry.cs +++ b/src/Braintrust.Sdk/Instrumentation/OpenAI/OpenAITelemetry.cs @@ -1,4 +1,3 @@ -using System; using System.Diagnostics; using OpenAI; diff --git a/src/Braintrust.Sdk/SdkVersion.cs b/src/Braintrust.Sdk/SdkVersion.cs index d700aea..576de31 100644 --- a/src/Braintrust.Sdk/SdkVersion.cs +++ b/src/Braintrust.Sdk/SdkVersion.cs @@ -1,5 +1,3 @@ -using System; -using System.IO; using System.Reflection; namespace Braintrust.Sdk; diff --git a/src/Braintrust.Sdk/Trace/BraintrustContext.cs b/src/Braintrust.Sdk/Trace/BraintrustContext.cs index 4ab4b1e..e6381fa 100644 --- a/src/Braintrust.Sdk/Trace/BraintrustContext.cs +++ b/src/Braintrust.Sdk/Trace/BraintrustContext.cs @@ -1,6 +1,4 @@ -using System; using System.Diagnostics; -using System.Threading; namespace Braintrust.Sdk.Trace; diff --git a/src/Braintrust.Sdk/Trace/BraintrustSpanProcessor.cs b/src/Braintrust.Sdk/Trace/BraintrustSpanProcessor.cs index d137a2f..cb74940 100644 --- a/src/Braintrust.Sdk/Trace/BraintrustSpanProcessor.cs +++ b/src/Braintrust.Sdk/Trace/BraintrustSpanProcessor.cs @@ -1,4 +1,3 @@ -using System; using System.Diagnostics; using Braintrust.Sdk.Config; using OpenTelemetry; diff --git a/src/Braintrust.Sdk/Trace/BraintrustTracing.cs b/src/Braintrust.Sdk/Trace/BraintrustTracing.cs index e6e3b06..7d3fa40 100644 --- a/src/Braintrust.Sdk/Trace/BraintrustTracing.cs +++ b/src/Braintrust.Sdk/Trace/BraintrustTracing.cs @@ -1,9 +1,6 @@ -using System; -using System.Collections.Generic; using System.Diagnostics; using Braintrust.Sdk.Config; using Microsoft.Extensions.Logging; -using OpenTelemetry; using OpenTelemetry.Exporter; using OpenTelemetry.Metrics; using OpenTelemetry.Resources; @@ -88,7 +85,6 @@ public static void Enable(BraintrustConfig config, TracerProviderBuilder tracerP .SetSampler(new AlwaysOnSampler()); } - /// /// Get the singleton ActivitySource for instrumentation. /// @@ -97,7 +93,6 @@ public static ActivitySource GetActivitySource() return _activitySource.Value; } - private static string BuildHeaders(BraintrustConfig config) { var headers = new List diff --git a/tests/Braintrust.Sdk.Tests/Api/BraintrustApiClientTest.cs b/tests/Braintrust.Sdk.Tests/Api/BraintrustApiClientTest.cs index 620cc0a..60e7c25 100644 --- a/tests/Braintrust.Sdk.Tests/Api/BraintrustApiClientTest.cs +++ b/tests/Braintrust.Sdk.Tests/Api/BraintrustApiClientTest.cs @@ -1,13 +1,8 @@ -using System; using System.Net; -using System.Net.Http; using System.Text; using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; using Braintrust.Sdk.Api; using Braintrust.Sdk.Config; -using Xunit; namespace Braintrust.Sdk.Tests.Api; @@ -15,7 +10,6 @@ public class BraintrustApiClientTest : IDisposable { private readonly TestHttpMessageHandler _handler; private readonly HttpClient _httpClient; - private readonly BraintrustConfig _config; private readonly BraintrustApiClient _apiClient; public BraintrustApiClientTest() @@ -26,29 +20,29 @@ public BraintrustApiClientTest() BaseAddress = new Uri("https://test-api.example.com") }; - _config = BraintrustConfig.Of( - "BRAINTRUST_API_KEY", "test-api-key", - "BRAINTRUST_API_URL", "https://test-api.example.com", - "BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project" + var config = BraintrustConfig.Of( + ("BRAINTRUST_API_KEY", "test-api-key"), + ("BRAINTRUST_API_URL", "https://test-api.example.com"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); - _apiClient = new BraintrustApiClient(_config, _httpClient); + _apiClient = new BraintrustApiClient(config, _httpClient); } public void Dispose() { - _apiClient?.Dispose(); - _httpClient?.Dispose(); - _handler?.Dispose(); + _apiClient.Dispose(); + _httpClient.Dispose(); + _handler.Dispose(); } [Fact] - public void GetOrCreateProject_CreatesProject() + public async Task GetOrCreateProject_CreatesProject() { var expectedProject = new Project("proj-123", "test-project", "org-456"); _handler.SetResponse(HttpStatusCode.OK, expectedProject); - var result = _apiClient.GetOrCreateProject("test-project"); + var result = await _apiClient.GetOrCreateProject("test-project"); Assert.NotNull(result); Assert.Equal("proj-123", result.Id); @@ -62,37 +56,37 @@ public void GetOrCreateProject_CreatesProject() } [Fact] - public void GetProject_ReturnsProject() + public async Task GetProject_ReturnsProject() { var expectedProject = new Project("proj-123", "test-project", "org-456"); _handler.SetResponse(HttpStatusCode.OK, expectedProject); - var result = _apiClient.GetProject("proj-123"); + var result = await _apiClient.GetProject("proj-123"); Assert.NotNull(result); - Assert.Equal("proj-123", result!.Id); + Assert.Equal("proj-123", result.Id); Assert.Equal(HttpMethod.Get, _handler.LastRequest?.Method); Assert.Equal("/v1/project/proj-123", _handler.LastRequest?.RequestUri?.AbsolutePath); } [Fact] - public void GetProject_ReturnsNull_When404() + public async Task GetProject_ReturnsNull_When404() { _handler.SetResponse(HttpStatusCode.NotFound, "Not found"); - var result = _apiClient.GetProject("missing-project"); + var result = await _apiClient.GetProject("missing-project"); Assert.Null(result); } [Fact] - public void GetOrCreateExperiment_CreatesExperiment() + public async Task GetOrCreateExperiment_CreatesExperiment() { var expectedExperiment = new Experiment("exp-123", "proj-456", "test-experiment"); _handler.SetResponse(HttpStatusCode.OK, expectedExperiment); var request = new CreateExperimentRequest("proj-456", "test-experiment", "Test description"); - var result = _apiClient.GetOrCreateExperiment(request); + var result = await _apiClient.GetOrCreateExperiment(request); Assert.NotNull(result); Assert.Equal("exp-123", result.Id); @@ -103,21 +97,18 @@ public void GetOrCreateExperiment_CreatesExperiment() } [Fact] - public void GetOrCreateProjectAndOrgInfo_ReturnsInfo() + public async Task GetOrCreateProjectAndOrgInfo_ReturnsInfo() { // Setup: First call creates/gets project, second call is login var project = new Project("proj-123", "test-project", "org-456"); - var loginResponse = new LoginResponse(new System.Collections.Generic.List - { - new OrganizationInfo("org-456", "Test Org") - }); + var loginResponse = new LoginResponse([new OrganizationInfo("org-456", "Test Org")]); _handler.SetResponses( (HttpStatusCode.OK, project), (HttpStatusCode.OK, loginResponse) ); - var result = _apiClient.GetOrCreateProjectAndOrgInfo(); + var result = await _apiClient.GetOrCreateProjectAndOrgInfo(); Assert.NotNull(result); Assert.Equal("proj-123", result.Project.Id); @@ -126,11 +117,11 @@ public void GetOrCreateProjectAndOrgInfo_ReturnsInfo() } [Fact] - public void ApiException_ThrownOn_HttpError() + public async Task ApiException_ThrownOn_HttpError() { _handler.SetResponse(HttpStatusCode.BadRequest, "Bad request"); - var exception = Assert.Throws(() => + var exception = await Assert.ThrowsAsync(() => _apiClient.GetProject("test")); Assert.Equal(400, exception.StatusCode); diff --git a/tests/Braintrust.Sdk.Tests/BraintrustTest.cs b/tests/Braintrust.Sdk.Tests/BraintrustTest.cs index 7026366..91de382 100644 --- a/tests/Braintrust.Sdk.Tests/BraintrustTest.cs +++ b/tests/Braintrust.Sdk.Tests/BraintrustTest.cs @@ -1,6 +1,4 @@ -using System; using Braintrust.Sdk.Config; -using Xunit; namespace Braintrust.Sdk.Tests; @@ -42,7 +40,7 @@ public void GetCreatesGlobalInstance() [Fact] public void GetWithConfigCreatesGlobalInstance() { - var config = BraintrustConfig.Of("BRAINTRUST_API_KEY", "custom-key"); + var config = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "custom-key")); var instance1 = Braintrust.Get(config); var instance2 = Braintrust.Get(); @@ -55,8 +53,8 @@ public void GetWithConfigCreatesGlobalInstance() [Fact] public void GetWithConfigOnlyCreatesOnce() { - var config1 = BraintrustConfig.Of("BRAINTRUST_API_KEY", "key-1"); - var config2 = BraintrustConfig.Of("BRAINTRUST_API_KEY", "key-2"); + var config1 = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "key-1")); + var config2 = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "key-2")); var instance1 = Braintrust.Get(config1); var instance2 = Braintrust.Get(config2); @@ -69,8 +67,8 @@ public void GetWithConfigOnlyCreatesOnce() [Fact] public void OfCreatesNewInstance() { - var config1 = BraintrustConfig.Of("BRAINTRUST_API_KEY", "key-1"); - var config2 = BraintrustConfig.Of("BRAINTRUST_API_KEY", "key-2"); + var config1 = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "key-1")); + var config2 = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "key-2")); var instance1 = Braintrust.Of(config1); var instance2 = Braintrust.Of(config2); @@ -85,8 +83,8 @@ public void OfCreatesNewInstance() [Fact] public void OfDoesNotAffectGlobalInstance() { - var globalConfig = BraintrustConfig.Of("BRAINTRUST_API_KEY", "global-key"); - var localConfig = BraintrustConfig.Of("BRAINTRUST_API_KEY", "local-key"); + var globalConfig = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "global-key")); + var localConfig = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "local-key")); var globalInstance = Braintrust.Get(globalConfig); var localInstance = Braintrust.Of(localConfig); @@ -103,8 +101,8 @@ public void OfDoesNotAffectGlobalInstance() public void ConfigIsAccessible() { var config = BraintrustConfig.Of( - "BRAINTRUST_API_KEY", "test-key", - "BRAINTRUST_DEBUG", "true" + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_DEBUG", "true") ); var instance = Braintrust.Of(config); @@ -117,8 +115,8 @@ public void ConfigIsAccessible() [Fact] public void SetIsThreadSafe() { - var config1 = BraintrustConfig.Of("BRAINTRUST_API_KEY", "key-1"); - var config2 = BraintrustConfig.Of("BRAINTRUST_API_KEY", "key-2"); + var config1 = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "key-1")); + var config2 = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "key-2")); Braintrust? result1 = null; Braintrust? result2 = null; @@ -145,12 +143,12 @@ public void SetIsThreadSafe() [Fact] public void ResetForTestClearsInstance() { - var config = BraintrustConfig.Of("BRAINTRUST_API_KEY", "test-key"); + var config = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "test-key")); var instance1 = Braintrust.Get(config); Braintrust.ResetForTest(); - var newConfig = BraintrustConfig.Of("BRAINTRUST_API_KEY", "new-key"); + var newConfig = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "new-key")); var instance2 = Braintrust.Get(newConfig); Assert.NotSame(instance1, instance2); diff --git a/tests/Braintrust.Sdk.Tests/Config/BaseConfigTest.cs b/tests/Braintrust.Sdk.Tests/Config/BaseConfigTest.cs index 48ad521..d808cd8 100644 --- a/tests/Braintrust.Sdk.Tests/Config/BaseConfigTest.cs +++ b/tests/Braintrust.Sdk.Tests/Config/BaseConfigTest.cs @@ -1,31 +1,21 @@ -using System; -using System.Collections.Generic; using Braintrust.Sdk.Config; -using Xunit; namespace Braintrust.Sdk.Tests.Config; public class BaseConfigTest { // Test implementation of BaseConfig for testing purposes - private class TestConfig : BaseConfig + private class TestConfig(IDictionary? envOverrides = null) + : BaseConfig(envOverrides ?? new Dictionary()) { - public TestConfig(IDictionary? envOverrides = null) - : base(envOverrides ?? new Dictionary()) - { - } - - public new T GetConfig(string settingName, T defaultValue) where T : notnull + public new T? GetConfig(string settingName, T? defaultValue) + where T : IParsable { return base.GetConfig(settingName, defaultValue); } - public new T? GetConfig(string settingName, T? defaultValue, Type settingType) - { - return base.GetConfig(settingName, defaultValue, settingType); - } - public new T GetRequiredConfig(string settingName) + where T : IParsable { return base.GetRequiredConfig(settingName); } @@ -35,82 +25,16 @@ public TestConfig(IDictionary? envOverrides = null) return base.GetRequiredConfig(settingName); } - public new object Cast(string value, Type type) - { - return base.Cast(value, type); - } - public new string? GetEnvValue(string settingName) { return base.GetEnvValue(settingName); } } - [Fact] - public void TestCastBoolean() - { - var config = new TestConfig(); - Assert.True((bool)config.Cast("true", typeof(bool))); - Assert.False((bool)config.Cast("false", typeof(bool))); - Assert.True((bool)config.Cast("True", typeof(bool))); - Assert.False((bool)config.Cast("False", typeof(bool))); - } - - [Fact] - public void TestCastInteger() - { - var config = new TestConfig(); - Assert.Equal(123, (int)config.Cast("123", typeof(int))); - Assert.Equal(-456, (int)config.Cast("-456", typeof(int))); - Assert.Equal(0, (int)config.Cast("0", typeof(int))); - } - - [Fact] - public void TestCastLong() - { - var config = new TestConfig(); - Assert.Equal(123L, (long)config.Cast("123", typeof(long))); - Assert.Equal(long.MaxValue, (long)config.Cast(long.MaxValue.ToString(), typeof(long))); - Assert.Equal(long.MinValue, (long)config.Cast(long.MinValue.ToString(), typeof(long))); - } - - [Fact] - public void TestCastFloat() - { - var config = new TestConfig(); - Assert.Equal(123.45f, (float)config.Cast("123.45", typeof(float)), precision: 2); - Assert.Equal(-678.90f, (float)config.Cast("-678.90", typeof(float)), precision: 2); - } - - [Fact] - public void TestCastDouble() - { - var config = new TestConfig(); - Assert.Equal(123.456789, (double)config.Cast("123.456789", typeof(double)), precision: 6); - Assert.Equal(-987.654321, (double)config.Cast("-987.654321", typeof(double)), precision: 6); - } - - [Fact] - public void TestCastString() - { - var config = new TestConfig(); - Assert.Equal("hello", (string)config.Cast("hello", typeof(string))); - Assert.Equal("world", (string)config.Cast("world", typeof(string))); - } - - [Fact] - public void TestCastUnsupportedType() - { - var config = new TestConfig(); - var exception = Assert.Throws(() => - config.Cast("test", typeof(DateTime))); - Assert.Contains("Unsupported default class", exception.Message); - } - [Fact] public void TestGetRequiredConfigSuccess() { - var overrides = new Dictionary + var overrides = new Dictionary { { "REQUIRED_VAR", "test_value" } }; @@ -131,7 +55,7 @@ public void TestGetRequiredConfigFailure() [Fact] public void TestGetRequiredConfigWithType() { - var overrides = new Dictionary + var overrides = new Dictionary { { "INT_VAR", "42" }, { "BOOL_VAR", "true" } @@ -146,7 +70,7 @@ public void TestGetRequiredConfigWithType() public void TestGetConfigWithNullDefault() { var config = new TestConfig(); - var result = config.GetConfig("MISSING_VAR", null, typeof(string)); + var result = config.GetConfig("MISSING_VAR", null); Assert.Null(result); } @@ -162,7 +86,7 @@ public void TestGetConfigWithDefaultValue() [Fact] public void TestGetEnvValueFromOverrides() { - var overrides = new Dictionary + var overrides = new Dictionary { { "TEST_VAR", "override_value" } }; @@ -183,7 +107,7 @@ public void TestGetEnvValueNonExistent() [Fact] public void TestNullSentinelHandling() { - var overrides = new Dictionary + var overrides = new Dictionary { { "NULL_VAR", BaseConfig.NullOverride } }; @@ -196,7 +120,7 @@ public void TestNullSentinelHandling() public void TestGetConfigHierarchy() { // Test that overrides take precedence over default values - var overrides = new Dictionary + var overrides = new Dictionary { { "OVERRIDE_VAR", "from_override" } }; @@ -209,7 +133,7 @@ public void TestGetConfigHierarchy() [Fact] public void TestIntegrationWithAllTypes() { - var overrides = new Dictionary + var overrides = new Dictionary { { "STR_VAR", "hello" }, { "BOOL_VAR", "true" }, diff --git a/tests/Braintrust.Sdk.Tests/Config/BraintrustConfigTest.cs b/tests/Braintrust.Sdk.Tests/Config/BraintrustConfigTest.cs index 9ccfcf6..ef35a48 100644 --- a/tests/Braintrust.Sdk.Tests/Config/BraintrustConfigTest.cs +++ b/tests/Braintrust.Sdk.Tests/Config/BraintrustConfigTest.cs @@ -1,6 +1,4 @@ -using System; using Braintrust.Sdk.Config; -using Xunit; namespace Braintrust.Sdk.Tests.Config; @@ -10,8 +8,8 @@ public class BraintrustConfigTest public void ParentDefaultsToProjectName() { var config = BraintrustConfig.Of( - "BRAINTRUST_API_KEY", "test-key", - "BRAINTRUST_DEFAULT_PROJECT_NAME", "my-project" + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "my-project") ); Assert.Equal("project_name:my-project", config.GetBraintrustParentValue()); @@ -21,9 +19,9 @@ public void ParentDefaultsToProjectName() public void ParentUsesProjectId() { var config = BraintrustConfig.Of( - "BRAINTRUST_API_KEY", "test-key", - "BRAINTRUST_DEFAULT_PROJECT_NAME", "my-project", - "BRAINTRUST_DEFAULT_PROJECT_ID", "proj-123" + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "my-project"), + ("BRAINTRUST_DEFAULT_PROJECT_ID", "proj-123") ); // Project ID takes precedence over project name @@ -35,7 +33,7 @@ public void RequiresApiKey() { var exception = Assert.Throws(() => BraintrustConfig.Of( - "BRAINTRUST_API_KEY", BaseConfig.NullOverride)); + ("BRAINTRUST_API_KEY", BaseConfig.NullOverride))); Assert.Contains("BRAINTRUST_API_KEY is required", exception.Message); } @@ -45,16 +43,16 @@ public void HasDefaultValues() { // Use NULL_OVERRIDE to force defaults even if env vars are set var config = BraintrustConfig.Of( - "BRAINTRUST_API_KEY", "test-key", - "BRAINTRUST_API_URL", BaseConfig.NullOverride, - "BRAINTRUST_APP_URL", BaseConfig.NullOverride, - "BRAINTRUST_TRACES_PATH", BaseConfig.NullOverride, - "BRAINTRUST_LOGS_PATH", BaseConfig.NullOverride, - "BRAINTRUST_DEFAULT_PROJECT_NAME", BaseConfig.NullOverride, - "BRAINTRUST_DEFAULT_PROJECT_ID", BaseConfig.NullOverride, - "BRAINTRUST_DEBUG", BaseConfig.NullOverride, - "BRAINTRUST_ENABLE_TRACE_CONSOLE_LOG", BaseConfig.NullOverride, - "BRAINTRUST_REQUEST_TIMEOUT", BaseConfig.NullOverride + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_API_URL", BaseConfig.NullOverride), + ("BRAINTRUST_APP_URL", BaseConfig.NullOverride), + ("BRAINTRUST_TRACES_PATH", BaseConfig.NullOverride), + ("BRAINTRUST_LOGS_PATH", BaseConfig.NullOverride), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", BaseConfig.NullOverride), + ("BRAINTRUST_DEFAULT_PROJECT_ID", BaseConfig.NullOverride), + ("BRAINTRUST_DEBUG", BaseConfig.NullOverride), + ("BRAINTRUST_ENABLE_TRACE_CONSOLE_LOG", BaseConfig.NullOverride), + ("BRAINTRUST_REQUEST_TIMEOUT", BaseConfig.NullOverride) ); Assert.Equal("https://api.braintrust.dev", config.ApiUrl); @@ -71,11 +69,11 @@ public void HasDefaultValues() public void CanOverrideDefaults() { var config = BraintrustConfig.Of( - "BRAINTRUST_API_KEY", "test-key", - "BRAINTRUST_API_URL", "https://custom.api.url", - "BRAINTRUST_APP_URL", "https://custom.app.url", - "BRAINTRUST_DEBUG", "true", - "BRAINTRUST_REQUEST_TIMEOUT", "60" + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_API_URL", "https://custom.api.url"), + ("BRAINTRUST_APP_URL", "https://custom.app.url"), + ("BRAINTRUST_DEBUG", "true"), + ("BRAINTRUST_REQUEST_TIMEOUT", "60") ); Assert.Equal("https://custom.api.url", config.ApiUrl); @@ -85,17 +83,32 @@ public void CanOverrideDefaults() } [Fact] - public void OddNumberOfOverridesThrows() + public void FromEnvironmentUsesEnvironmentVariables() { - var exception = Assert.Throws(() => - BraintrustConfig.Of("BRAINTRUST_API_KEY")); + // Set a temporary environment variable for testing + var originalApiKey = Environment.GetEnvironmentVariable("BRAINTRUST_API_KEY"); + var originalProjectName = Environment.GetEnvironmentVariable("BRAINTRUST_DEFAULT_PROJECT_NAME"); + + Environment.SetEnvironmentVariable("BRAINTRUST_API_KEY", "env-test-key"); + Environment.SetEnvironmentVariable("BRAINTRUST_DEFAULT_PROJECT_NAME", "env-project"); - Assert.Contains("config overrides require key-value pairs", exception.Message); - Assert.Contains("BRAINTRUST_API_KEY", exception.Message); + try + { + var config = BraintrustConfig.FromEnvironment(); + + Assert.Equal("env-test-key", config.ApiKey); + Assert.Equal("env-project", config.DefaultProjectName); + } + finally + { + // Restore original values + Environment.SetEnvironmentVariable("BRAINTRUST_API_KEY", originalApiKey); + Environment.SetEnvironmentVariable("BRAINTRUST_DEFAULT_PROJECT_NAME", originalProjectName); + } } [Fact] - public void FromEnvironmentUsesEnvironmentVariables() + public void NullOverrides() { // Set a temporary environment variable for testing var originalApiKey = Environment.GetEnvironmentVariable("BRAINTRUST_API_KEY"); @@ -106,10 +119,11 @@ public void FromEnvironmentUsesEnvironmentVariables() try { - var config = BraintrustConfig.FromEnvironment(); - + var config = BraintrustConfig.Of(("BRAINTRUST_DEFAULT_PROJECT_NAME", null)); Assert.Equal("env-test-key", config.ApiKey); - Assert.Equal("env-project", config.DefaultProjectName); + + // The project name should fall back to the default since we passed null override + Assert.Equal("default-dotnet-project", config.DefaultProjectName); } finally { @@ -123,8 +137,8 @@ public void FromEnvironmentUsesEnvironmentVariables() public void ProjectIdCanBeNull() { var config = BraintrustConfig.Of( - "BRAINTRUST_API_KEY", "test-key", - "BRAINTRUST_DEFAULT_PROJECT_NAME", "my-project" + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "my-project") ); Assert.Null(config.DefaultProjectId); @@ -135,10 +149,10 @@ public void ProjectIdCanBeNull() public void BooleanConfigsParsedCorrectly() { var config = BraintrustConfig.Of( - "BRAINTRUST_API_KEY", "test-key", - "BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project", - "BRAINTRUST_DEBUG", "true", - "BRAINTRUST_ENABLE_TRACE_CONSOLE_LOG", "true" + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project"), + ("BRAINTRUST_DEBUG", "true"), + ("BRAINTRUST_ENABLE_TRACE_CONSOLE_LOG", "true") ); Assert.True(config.Debug); diff --git a/tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs b/tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs index a2fec6e..80550ad 100644 --- a/tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs +++ b/tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs @@ -1,10 +1,6 @@ -using System; using System.Diagnostics; -using System.Linq; -using Braintrust.Sdk.Api; using Braintrust.Sdk.Config; using Braintrust.Sdk.Eval; -using Xunit; namespace Braintrust.Sdk.Tests.Eval; @@ -33,38 +29,38 @@ public void Dispose() } [Fact] - public void BasicEvalBuildsAndRuns() + public async Task BasicEvalBuildsAndRuns() { // Arrange var config = BraintrustConfig.Of( - "BRAINTRUST_API_KEY", "test-key", - "BRAINTRUST_APP_URL", "https://braintrust.dev", - "BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project" + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_APP_URL", "https://braintrust.dev"), + ("BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project") ); // Create a mock API client that doesn't make real API calls var mockClient = new MockBraintrustApiClient(); - var cases = new[] + var cases = new DatasetCase[] { - DatasetCase.Of("strawberry", "fruit"), - DatasetCase.Of("asparagus", "vegetable") + new("strawberry", "fruit"), + new("asparagus", "vegetable") }; // Act - var eval = Eval.NewBuilder() + var eval = await Eval.NewBuilder() .Name("test-eval") .Config(config) .ApiClient(mockClient) .Cases(cases) .TaskFunction(food => "fruit") .Scorers( - Scorer.Of("fruit_scorer", (expected, actual) => expected == "fruit" && actual == "fruit" ? 1.0 : 0.0), - Scorer.Of("vegetable_scorer", (expected, actual) => expected == "vegetable" && actual == "vegetable" ? 1.0 : 0.0) + new FunctionScorer("fruit_scorer", (expected, actual) => expected == "fruit" && actual == "fruit" ? 1.0 : 0.0), + new FunctionScorer("vegetable_scorer", (expected, actual) => expected == "vegetable" && actual == "vegetable" ? 1.0 : 0.0) ) - .Build(); + .BuildAsync(); - var result = eval.Run(); + var result = await eval.RunAsync(); // Assert Assert.NotNull(result); @@ -74,57 +70,57 @@ public void BasicEvalBuildsAndRuns() } [Fact] - public void EvalRequiresAtLeastOneScorer() + public async Task EvalRequiresAtLeastOneScorer() { - var config = BraintrustConfig.Of("BRAINTRUST_API_KEY", "test-key"); + var config = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "test-key")); var mockClient = new MockBraintrustApiClient(); - Assert.Throws(() => + await Assert.ThrowsAsync(() => Eval.NewBuilder() .Name("test-eval") .Config(config) .ApiClient(mockClient) - .Cases(DatasetCase.Of("input", "expected")) + .Cases(DatasetCase.Of("input", "expected")) .TaskFunction(x => x) - .Build()); + .BuildAsync()); } [Fact] - public void EvalRequiresDataset() + public async Task EvalRequiresDataset() { - var config = BraintrustConfig.Of("BRAINTRUST_API_KEY", "test-key"); + var config = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "test-key")); var mockClient = new MockBraintrustApiClient(); - Assert.Throws(() => + await Assert.ThrowsAsync(() => Eval.NewBuilder() .Name("test-eval") .Config(config) .ApiClient(mockClient) .TaskFunction(x => x) - .Scorers(Scorer.Of("test", (_, _) => 1.0)) - .Build()); + .Scorers(new FunctionScorer("test", (_, _) => 1.0)) + .BuildAsync()); } [Fact] - public void EvalRequiresTask() + public async Task EvalRequiresTask() { - var config = BraintrustConfig.Of("BRAINTRUST_API_KEY", "test-key"); + var config = BraintrustConfig.Of(("BRAINTRUST_API_KEY", "test-key")); var mockClient = new MockBraintrustApiClient(); - Assert.Throws(() => + await Assert.ThrowsAsync(() => Eval.NewBuilder() .Name("test-eval") .Config(config) .ApiClient(mockClient) - .Cases(DatasetCase.Of("input", "expected")) - .Scorers(Scorer.Of("test", (_, _) => 1.0)) - .Build()); + .Cases(DatasetCase.Of("input", "expected")) + .Scorers(new FunctionScorer("test", (_, _) => 1.0)) + .BuildAsync()); } [Fact] public void DatasetCaseOfCreatesWithEmptyTagsAndMetadata() { - var datasetCase = DatasetCase.Of("input", "expected"); + var datasetCase = DatasetCase.Of("input", "expected"); Assert.Equal("input", datasetCase.Input); Assert.Equal("expected", datasetCase.Expected); @@ -135,10 +131,10 @@ public void DatasetCaseOfCreatesWithEmptyTagsAndMetadata() [Fact] public void ScorerCreatesValidScore() { - var scorer = Scorer.Of("test_scorer", (expected, actual) => expected == actual ? 1.0 : 0.0); + var scorer = new FunctionScorer("test_scorer", (expected, actual) => expected == actual ? 1.0 : 0.0); var taskResult = new TaskResult( "expected", - DatasetCase.Of("input", "expected") + DatasetCase.Of("input", "expected") ); var scores = scorer.Score(taskResult); @@ -149,65 +145,28 @@ public void ScorerCreatesValidScore() } [Fact] - public void DatasetOfCreatesInMemoryDataset() + public async Task DatasetOfCreatesInMemoryDataset() { - var dataset = Dataset.Of( - DatasetCase.Of("input1", "output1"), - DatasetCase.Of("input2", "output2") + var dataset = Dataset.Of( + DatasetCase.Of("input1", "output1"), + DatasetCase.Of("input2", "output2") ); Assert.NotNull(dataset); Assert.NotNull(dataset.Id); Assert.NotNull(dataset.Version); - using var cursor = dataset.OpenCursor(); - var case1 = cursor.Next(); - var case2 = cursor.Next(); - var case3 = cursor.Next(); + await using var cursor = dataset.GetCasesAsync().GetAsyncEnumerator(); + + Assert.True(await cursor.MoveNextAsync()); + var case1 = cursor.Current; + Assert.True(await cursor.MoveNextAsync()); + var case2 = cursor.Current; + Assert.False(await cursor.MoveNextAsync()); Assert.NotNull(case1); Assert.Equal("input1", case1.Input); Assert.NotNull(case2); Assert.Equal("input2", case2.Input); - Assert.Null(case3); - } -} - -/// -/// Mock API client for testing that doesn't make real HTTP calls. -/// -internal class MockBraintrustApiClient : IBraintrustApiClient -{ - private readonly OrganizationInfo _orgInfo = new OrganizationInfo("test-org-id", "test-org"); - private readonly Project _project = new Project("test-project-id", "test-project", "test-org-id", null, null); - - public Project GetOrCreateProject(string projectName) - { - return _project; - } - - public Project? GetProject(string projectId) - { - return _project; - } - - public Experiment GetOrCreateExperiment(CreateExperimentRequest request) - { - return new Experiment("test-experiment-id", request.ProjectId, request.Name, request.Description, null, null); - } - - public OrganizationAndProjectInfo? GetProjectAndOrgInfo() - { - return new OrganizationAndProjectInfo(_orgInfo, _project); - } - - public OrganizationAndProjectInfo? GetProjectAndOrgInfo(string projectId) - { - return new OrganizationAndProjectInfo(_orgInfo, _project); - } - - public OrganizationAndProjectInfo GetOrCreateProjectAndOrgInfo() - { - return new OrganizationAndProjectInfo(_orgInfo, _project); } -} +} \ No newline at end of file diff --git a/tests/Braintrust.Sdk.Tests/Eval/MockBraintrustApiClient.cs b/tests/Braintrust.Sdk.Tests/Eval/MockBraintrustApiClient.cs new file mode 100644 index 0000000..75ff5cf --- /dev/null +++ b/tests/Braintrust.Sdk.Tests/Eval/MockBraintrustApiClient.cs @@ -0,0 +1,42 @@ +using Braintrust.Sdk.Api; + +namespace Braintrust.Sdk.Tests.Eval; + +/// +/// Mock API client for testing that doesn't make real HTTP calls. +/// +internal class MockBraintrustApiClient : IBraintrustApiClient +{ + private readonly OrganizationInfo _orgInfo = new OrganizationInfo("test-org-id", "test-org"); + private readonly Project _project = new Project("test-project-id", "test-project", "test-org-id"); + + public Task GetOrCreateProject(string projectName) + { + return Task.FromResult(_project); + } + + public Task GetProject(string projectId) + { + return Task.FromResult(_project); + } + + public Task GetOrCreateExperiment(CreateExperimentRequest request) + { + return Task.FromResult(new Experiment("test-experiment-id", request.ProjectId, request.Name, request.Description)); + } + + public Task GetProjectAndOrgInfo() + { + return Task.FromResult(new OrganizationAndProjectInfo(_orgInfo, _project)); + } + + public Task GetProjectAndOrgInfo(string projectId) + { + return Task.FromResult(new OrganizationAndProjectInfo(_orgInfo, _project)); + } + + public Task GetOrCreateProjectAndOrgInfo() + { + return Task.FromResult(new OrganizationAndProjectInfo(_orgInfo, _project)); + } +} \ No newline at end of file diff --git a/tests/Braintrust.Sdk.Tests/Instrumentation/OpenAI/BraintrustOpenAITest.cs b/tests/Braintrust.Sdk.Tests/Instrumentation/OpenAI/BraintrustOpenAITest.cs index febc5d2..d88e7ba 100644 --- a/tests/Braintrust.Sdk.Tests/Instrumentation/OpenAI/BraintrustOpenAITest.cs +++ b/tests/Braintrust.Sdk.Tests/Instrumentation/OpenAI/BraintrustOpenAITest.cs @@ -1,15 +1,7 @@ -using System; -using System.ClientModel; using System.ClientModel.Primitives; -using System.Collections.Generic; using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Net.Http; using System.Text; using System.Text.Json.Nodes; -using System.Threading; -using System.Threading.Tasks; using Braintrust.Sdk.Config; using Braintrust.Sdk.Instrumentation.OpenAI; using Braintrust.Sdk.Trace; @@ -19,7 +11,6 @@ using OpenTelemetry; using OpenTelemetry.Resources; using OpenTelemetry.Trace; -using Xunit; namespace Braintrust.Sdk.Tests.Instrumentation.OpenAI; @@ -71,8 +62,8 @@ public void Dispose() private TracerProvider SetupOpenTelemetry() { var config = BraintrustConfig.Of( - "BRAINTRUST_API_KEY", "test-key", - "BRAINTRUST_EXPORT_SPANS_IN_MEMORY_FOR_UNIT_TEST", "true" + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_EXPORT_SPANS_IN_MEMORY_FOR_UNIT_TEST", "true") ); var braintrust = Braintrust.Of(config); diff --git a/tests/Braintrust.Sdk.Tests/Trace/BraintrustSpanProcessorTest.cs b/tests/Braintrust.Sdk.Tests/Trace/BraintrustSpanProcessorTest.cs index a76d539..05f11ff 100644 --- a/tests/Braintrust.Sdk.Tests/Trace/BraintrustSpanProcessorTest.cs +++ b/tests/Braintrust.Sdk.Tests/Trace/BraintrustSpanProcessorTest.cs @@ -1,8 +1,6 @@ -using System; using System.Diagnostics; using Braintrust.Sdk.Config; using Braintrust.Sdk.Trace; -using Xunit; namespace Braintrust.Sdk.Tests.Trace; @@ -34,7 +32,7 @@ public void Dispose() public void OnStart_AddsParentFromContext() { var config = BraintrustConfig.Of( - "BRAINTRUST_API_KEY", "test-key" + ("BRAINTRUST_API_KEY", "test-key") ); var processor = new BraintrustSpanProcessor(config); @@ -54,7 +52,7 @@ public void OnStart_AddsParentFromContext() public void OnStart_AddsExperimentParentFromContext() { var config = BraintrustConfig.Of( - "BRAINTRUST_API_KEY", "test-key" + ("BRAINTRUST_API_KEY", "test-key") ); var processor = new BraintrustSpanProcessor(config); @@ -74,8 +72,8 @@ public void OnStart_AddsExperimentParentFromContext() public void OnStart_AddsParentFromConfig() { var config = BraintrustConfig.Of( - "BRAINTRUST_API_KEY", "test-key", - "BRAINTRUST_DEFAULT_PROJECT_ID", "proj-config" + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_DEFAULT_PROJECT_ID", "proj-config") ); var processor = new BraintrustSpanProcessor(config); @@ -93,8 +91,8 @@ public void OnStart_AddsParentFromConfig() public void OnStart_ContextOverridesConfig() { var config = BraintrustConfig.Of( - "BRAINTRUST_API_KEY", "test-key", - "BRAINTRUST_DEFAULT_PROJECT_ID", "proj-config" + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_DEFAULT_PROJECT_ID", "proj-config") ); var processor = new BraintrustSpanProcessor(config); @@ -114,8 +112,8 @@ public void OnStart_ContextOverridesConfig() public void OnStart_DoesNotOverrideExistingParent() { var config = BraintrustConfig.Of( - "BRAINTRUST_API_KEY", "test-key", - "BRAINTRUST_DEFAULT_PROJECT_ID", "proj-config" + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_DEFAULT_PROJECT_ID", "proj-config") ); var processor = new BraintrustSpanProcessor(config); diff --git a/tests/Braintrust.Sdk.Tests/Trace/BraintrustTracingTest.cs b/tests/Braintrust.Sdk.Tests/Trace/BraintrustTracingTest.cs index 46f374c..bc894a7 100644 --- a/tests/Braintrust.Sdk.Tests/Trace/BraintrustTracingTest.cs +++ b/tests/Braintrust.Sdk.Tests/Trace/BraintrustTracingTest.cs @@ -1,7 +1,5 @@ -using System; using Braintrust.Sdk.Config; using Braintrust.Sdk.Trace; -using Xunit; namespace Braintrust.Sdk.Tests.Trace; @@ -12,8 +10,8 @@ public class BraintrustTracingTest public void Of_CreatesTracerProvider() { var config = BraintrustConfig.Of( - "BRAINTRUST_API_KEY", "test-key", - "BRAINTRUST_API_URL", "https://test-api.example.com" + ("BRAINTRUST_API_KEY", "test-key"), + ("BRAINTRUST_API_URL", "https://test-api.example.com") ); using var tracerProvider = BraintrustTracing.CreateTracerProvider(config);