diff --git a/examples/EvalExample/Program.cs b/examples/EvalExample/Program.cs index 225d4b9..41e12c3 100644 --- a/examples/EvalExample/Program.cs +++ b/examples/EvalExample/Program.cs @@ -1,6 +1,3 @@ -using System; -using System.Linq; -using Braintrust.Sdk; using Braintrust.Sdk.Eval; using Braintrust.Sdk.Instrumentation.OpenAI; using OpenAI; @@ -10,7 +7,7 @@ namespace Braintrust.Sdk.Examples.EvalExample; class Program { - static void Main(string[] args) + static async Task Main(string[] args) { var openAIApiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY"); if (string.IsNullOrEmpty(openAIApiKey)) @@ -44,7 +41,7 @@ string GetFoodType(string food) } // Create and run the evaluation - var eval = braintrust + var eval = await braintrust .EvalBuilder() .Name($"dotnet-eval-x-{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}") .Cases( @@ -58,9 +55,9 @@ string GetFoodType(string food) Scorer.Of("exact_match", (expected, actual) => expected == actual ? 1.0 : 0.0), Scorer.Of("close_enough_match", (expected, actual) => expected.Trim().ToLowerInvariant() == actual.Trim().ToLowerInvariant() ? 1.0 : 0.0) ) - .Build(); + .BuildAsync(); - var result = eval.Run(); + var result = await eval.RunAsync(); Console.WriteLine($"\n\n{result.CreateReportString()}"); } } diff --git a/examples/OpenAIInstrumentation/Program.cs b/examples/OpenAIInstrumentation/Program.cs index b8646b0..b8ffb7b 100644 --- a/examples/OpenAIInstrumentation/Program.cs +++ b/examples/OpenAIInstrumentation/Program.cs @@ -31,7 +31,7 @@ static async Task Main(string[] args) if (rootActivity != null) { await ChatCompletionsExample(instrumentedClient); - var url = braintrust.ProjectUri() + var url = await braintrust.GetProjectUriAsync() + $"/logs?r={rootActivity.TraceId}&s={rootActivity.SpanId}"; Console.WriteLine($"\n\n Example complete! View your data in Braintrust: {url}\n"); } diff --git a/examples/SimpleOpenTelemetry/Program.cs b/examples/SimpleOpenTelemetry/Program.cs index d65c159..d968db0 100644 --- a/examples/SimpleOpenTelemetry/Program.cs +++ b/examples/SimpleOpenTelemetry/Program.cs @@ -1,12 +1,8 @@ -using System; -using System.Threading; -using Braintrust.Sdk; - namespace Braintrust.Sdk.Examples.SimpleOpenTelemetry; class Program { - static void Main(string[] args) + static async Task Main(string[] args) { var braintrust = Braintrust.Get(); var activitySource = braintrust.GetActivitySource(); @@ -17,7 +13,7 @@ static void Main(string[] args) ArgumentNullException.ThrowIfNull(activity); Console.WriteLine("Performing simple operation..."); activity.SetTag("some boolean attribute", true); - url = braintrust.ProjectUri() + $"/logs?r={activity.TraceId}&s={activity.SpanId}"; + url = await braintrust.GetProjectUriAsync() + $"/logs?r={activity.TraceId}&s={activity.SpanId}"; } Console.WriteLine($"\n\n Example complete! View your data in Braintrust: {url}"); } diff --git a/src/Braintrust.Sdk/Api/ApiException.cs b/src/Braintrust.Sdk/Api/ApiException.cs new file mode 100644 index 0000000..6edd749 --- /dev/null +++ b/src/Braintrust.Sdk/Api/ApiException.cs @@ -0,0 +1,18 @@ +namespace Braintrust.Sdk.Api; + +/// +/// Exception thrown when an API request fails. +/// +public class ApiException : Exception +{ + public int? StatusCode { get; } + + public ApiException(string message) : base(message) { } + + public ApiException(string message, Exception innerException) : base(message, innerException) { } + + public ApiException(int statusCode, string message) : base(message) + { + StatusCode = statusCode; + } +} \ No newline at end of file diff --git a/src/Braintrust.Sdk/Api/BraintrustApiClient.cs b/src/Braintrust.Sdk/Api/BraintrustApiClient.cs index 5cbe3fe..8931e07 100644 --- a/src/Braintrust.Sdk/Api/BraintrustApiClient.cs +++ b/src/Braintrust.Sdk/Api/BraintrustApiClient.cs @@ -1,34 +1,11 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; using System.Net.Http.Headers; using System.Net.Http.Json; using System.Text.Json; using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; using Braintrust.Sdk.Config; namespace Braintrust.Sdk.Api; -/// -/// Exception thrown when an API request fails. -/// -public class ApiException : Exception -{ - public int? StatusCode { get; } - - public ApiException(string message) : base(message) { } - - public ApiException(string message, Exception innerException) : base(message, innerException) { } - - public ApiException(int statusCode, string message) : base(message) - { - StatusCode = statusCode; - } -} - /// /// Implementation of Braintrust API client. /// @@ -64,12 +41,11 @@ internal BraintrustApiClient(BraintrustConfig config, HttpClient? httpClient = n private static HttpClient CreateDefaultHttpClient(BraintrustConfig config) { - var client = new HttpClient + return new HttpClient { BaseAddress = new Uri(config.ApiUrl), Timeout = config.RequestTimeout }; - return client; } private static JsonSerializerOptions CreateJsonOptions() @@ -82,19 +58,17 @@ private static JsonSerializerOptions CreateJsonOptions() }; } - public Project GetOrCreateProject(string projectName) + public async Task GetOrCreateProject(string projectName) { var request = new CreateProjectRequest(projectName); - return PostAsync("/v1/project", request, default) - .ConfigureAwait(false).GetAwaiter().GetResult(); + return await PostAsync("/v1/project", request).ConfigureAwait(false); } - public Project? GetProject(string projectId) + public async Task GetProject(string projectId) { try { - return GetAsync($"/v1/project/{projectId}", default) - .ConfigureAwait(false).GetAwaiter().GetResult(); + return await GetAsync($"/v1/project/{projectId}").ConfigureAwait(false); } catch (ApiException ex) when (ex.StatusCode == 404) { @@ -102,37 +76,37 @@ public Project GetOrCreateProject(string projectName) } } - public Experiment GetOrCreateExperiment(CreateExperimentRequest request) + public async Task GetOrCreateExperiment(CreateExperimentRequest request) { - return PostAsync("/v1/experiment", request, default) - .ConfigureAwait(false).GetAwaiter().GetResult(); + return await PostAsync("/v1/experiment", request) + .ConfigureAwait(false); } - public OrganizationAndProjectInfo? GetProjectAndOrgInfo() + public async Task GetProjectAndOrgInfo() { if (_config.DefaultProjectId != null) { - return GetProjectAndOrgInfo(_config.DefaultProjectId); + return await GetProjectAndOrgInfo(_config.DefaultProjectId).ConfigureAwait(false); } if (_config.DefaultProjectName != null) { - var project = GetOrCreateProject(_config.DefaultProjectName); - return GetProjectAndOrgInfo(project.Id); + var project = await GetOrCreateProject(_config.DefaultProjectName).ConfigureAwait(false); + return await GetProjectAndOrgInfo(project.Id).ConfigureAwait(false); } return null; } - public OrganizationAndProjectInfo? GetProjectAndOrgInfo(string projectId) + public async Task GetProjectAndOrgInfo(string projectId) { - var project = GetProject(projectId); + var project = await GetProject(projectId).ConfigureAwait(false); if (project == null) { return null; } - var loginResponse = Login(); + var loginResponse = await Login().ConfigureAwait(false); var orgInfo = loginResponse.OrgInfo.FirstOrDefault(org => string.Equals(org.Id, project.OrgId, StringComparison.OrdinalIgnoreCase)); @@ -144,13 +118,13 @@ public Experiment GetOrCreateExperiment(CreateExperimentRequest request) return new OrganizationAndProjectInfo(orgInfo, project); } - public OrganizationAndProjectInfo GetOrCreateProjectAndOrgInfo() + public async Task GetOrCreateProjectAndOrgInfo() { Project project; if (_config.DefaultProjectId != null) { - var existingProject = GetProject(_config.DefaultProjectId); + var existingProject = await GetProject(_config.DefaultProjectId).ConfigureAwait(false); if (existingProject == null) { throw new ApiException($"Project with ID {_config.DefaultProjectId} not found"); @@ -159,14 +133,14 @@ public OrganizationAndProjectInfo GetOrCreateProjectAndOrgInfo() } else if (_config.DefaultProjectName != null) { - project = GetOrCreateProject(_config.DefaultProjectName); + project = await GetOrCreateProject(_config.DefaultProjectName).ConfigureAwait(false); } else { throw new InvalidOperationException("Either DefaultProjectId or DefaultProjectName must be set in config"); } - var loginResponse = Login(); + var loginResponse = await Login().ConfigureAwait(false); var orgInfo = loginResponse.OrgInfo.FirstOrDefault(org => string.Equals(org.Id, project.OrgId, StringComparison.OrdinalIgnoreCase)); @@ -178,11 +152,11 @@ public OrganizationAndProjectInfo GetOrCreateProjectAndOrgInfo() return new OrganizationAndProjectInfo(orgInfo, project); } - private LoginResponse Login() + private async Task Login() { var request = new LoginRequest(_config.ApiKey); - return PostAsync("/api/apikey/login", request, default) - .ConfigureAwait(false).GetAwaiter().GetResult(); + return await PostAsync("/api/apikey/login", request) + .ConfigureAwait(false); } private async Task GetAsync(string path, CancellationToken cancellationToken = default) @@ -191,8 +165,8 @@ private async Task GetAsync(string path, CancellationToken request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _config.ApiKey); request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - using var response = await _httpClient.SendAsync(request, cancellationToken); - return await HandleResponseAsync(response, cancellationToken); + using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + return await HandleResponseAsync(response, cancellationToken).ConfigureAwait(false); } private async Task PostAsync( @@ -205,15 +179,15 @@ private async Task PostAsync( request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); request.Content = JsonContent.Create(body, options: _jsonOptions); - using var response = await _httpClient.SendAsync(request, cancellationToken); - return await HandleResponseAsync(response, cancellationToken); + using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + return await HandleResponseAsync(response, cancellationToken).ConfigureAwait(false); } private async Task HandleResponseAsync(HttpResponseMessage response, CancellationToken cancellationToken) { if (response.IsSuccessStatusCode) { - var content = await response.Content.ReadAsStringAsync(cancellationToken); + var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); var result = JsonSerializer.Deserialize(content, _jsonOptions); if (result == null) @@ -225,7 +199,7 @@ private async Task HandleResponseAsync(HttpResponseMessage response, Cance } else { - var content = await response.Content.ReadAsStringAsync(cancellationToken); + var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); throw new ApiException( (int)response.StatusCode, $"API request failed with status {(int)response.StatusCode}: {content}"); @@ -236,7 +210,7 @@ public void Dispose() { if (_ownsHttpClient) { - _httpClient?.Dispose(); + _httpClient.Dispose(); } } } diff --git a/src/Braintrust.Sdk/Api/IBraintrustApiClient.cs b/src/Braintrust.Sdk/Api/IBraintrustApiClient.cs index d6f7eec..bb603f2 100644 --- a/src/Braintrust.Sdk/Api/IBraintrustApiClient.cs +++ b/src/Braintrust.Sdk/Api/IBraintrustApiClient.cs @@ -8,30 +8,30 @@ public interface IBraintrustApiClient /// /// Get or create a project by name. /// - Project GetOrCreateProject(string projectName); + Task GetOrCreateProject(string projectName); /// /// Get a project by ID. /// - Project? GetProject(string projectId); + Task GetProject(string projectId); /// /// Get or create an experiment. /// - Experiment GetOrCreateExperiment(CreateExperimentRequest request); + Task GetOrCreateExperiment(CreateExperimentRequest request); /// /// Get project and organization information using the default project from config. /// - OrganizationAndProjectInfo? GetProjectAndOrgInfo(); + Task GetProjectAndOrgInfo(); /// /// Get project and organization information for a specific project ID. /// - OrganizationAndProjectInfo? GetProjectAndOrgInfo(string projectId); + Task GetProjectAndOrgInfo(string projectId); /// /// Get or create project and organization information from config. /// - OrganizationAndProjectInfo GetOrCreateProjectAndOrgInfo(); + Task GetOrCreateProjectAndOrgInfo(); } diff --git a/src/Braintrust.Sdk/Braintrust.cs b/src/Braintrust.Sdk/Braintrust.cs index 4920130..69dfde9 100644 --- a/src/Braintrust.Sdk/Braintrust.cs +++ b/src/Braintrust.Sdk/Braintrust.cs @@ -1,5 +1,3 @@ -using System; -using System.Threading; using Braintrust.Sdk.Api; using Braintrust.Sdk.Config; using Microsoft.Extensions.Logging; @@ -100,16 +98,16 @@ private Braintrust(BraintrustConfig config, IBraintrustApiClient apiClient, Bool ApiClient = apiClient ?? throw new ArgumentNullException(nameof(apiClient)); if (autoManageOpenTelemetry) { - this._tracer = Trace.BraintrustTracing.CreateTracerProvider(this.Config); + _tracer = Trace.BraintrustTracing.CreateTracerProvider(this.Config); } } /// /// Get the URI to the configured Braintrust org and project. /// - public Uri ProjectUri() + public async Task GetProjectUriAsync() { - var orgAndProject = ApiClient.GetOrCreateProjectAndOrgInfo(); + var orgAndProject = await ApiClient.GetOrCreateProjectAndOrgInfo().ConfigureAwait(false); return new Uri($"{Config.AppUrl}/app/{orgAndProject.OrgInfo.Name}/p/{orgAndProject.Project.Name}"); } @@ -122,9 +120,9 @@ public Uri ProjectUri() /// public void OpenTelemetryEnable(OpenTelemetry.Trace.TracerProviderBuilder tracerProviderBuilder, ILoggingBuilder loggingBuilder, MeterProviderBuilder meterProviderBuilder) { - if (this._tracer != null) + if (_tracer != null) { - throw new System.InvalidOperationException("cannot call enable for Braintrusts which autoManage Open Telemetry"); + throw new InvalidOperationException("cannot call enable for Braintrusts which autoManage Open Telemetry"); } Trace.BraintrustTracing.Enable(Config, tracerProviderBuilder, loggingBuilder, meterProviderBuilder); } diff --git a/src/Braintrust.Sdk/Eval/Eval.cs b/src/Braintrust.Sdk/Eval/Eval.cs index e65cb31..6b7335a 100644 --- a/src/Braintrust.Sdk/Eval/Eval.cs +++ b/src/Braintrust.Sdk/Eval/Eval.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; using System.Diagnostics; -using System.Linq; using System.Text.Json; using Braintrust.Sdk.Api; using Braintrust.Sdk.Config; @@ -33,22 +30,12 @@ public sealed class Eval private readonly Task _task; private readonly IReadOnlyList> _scorers; - private Eval(Builder builder) + private Eval(Builder builder, OrganizationAndProjectInfo orgAndProject) { _experimentName = builder._experimentName; _config = builder._config ?? throw new ArgumentNullException(nameof(builder._config)); _client = builder._apiClient ?? throw new ArgumentNullException(nameof(builder._apiClient)); - - if (builder._projectId == null) - { - _orgAndProject = _client.GetProjectAndOrgInfo() - ?? throw new InvalidOperationException("Unable to retrieve project and org info"); - } - else - { - _orgAndProject = _client.GetProjectAndOrgInfo(builder._projectId) - ?? throw new InvalidOperationException($"Invalid project id: {builder._projectId}"); - } + _orgAndProject = orgAndProject ?? throw new ArgumentNullException(nameof(orgAndProject)); _activitySource = builder._activitySource ?? throw new ArgumentNullException(nameof(builder._activitySource)); _dataset = builder._dataset ?? throw new ArgumentNullException(nameof(builder._dataset)); @@ -59,14 +46,13 @@ private Eval(Builder builder) /// /// Runs the evaluation and returns results. /// - public EvalResult Run() + public async Task RunAsync() { - var experiment = _client.GetOrCreateExperiment( + var experiment = await _client.GetOrCreateExperiment( new CreateExperimentRequest( _orgAndProject.Project.Id, - _experimentName, - null, - null)); + _experimentName)) + .ConfigureAwait(false); var experimentId = experiment.Id; @@ -212,7 +198,7 @@ public sealed class Builder /// /// Build the Eval instance. /// - public Eval Build() + public async Task> BuildAsync() { _config ??= BraintrustConfig.FromEnvironment(); _activitySource ??= BraintrustTracing.GetActivitySource(); @@ -234,7 +220,13 @@ public Eval Build() throw new InvalidOperationException("Must provide a task"); } - return new Eval(this); + OrganizationAndProjectInfo? orgAndProject = + _projectId == null + ? await _apiClient.GetProjectAndOrgInfo().ConfigureAwait(false) + ?? throw new InvalidOperationException("Unable to retrieve project and org info") + : await _apiClient.GetProjectAndOrgInfo(_projectId).ConfigureAwait(false) + ?? throw new InvalidOperationException($"Invalid project id: {_projectId}"); + return new Eval(this, orgAndProject); } /// diff --git a/tests/Braintrust.Sdk.Tests/Api/BraintrustApiClientTest.cs b/tests/Braintrust.Sdk.Tests/Api/BraintrustApiClientTest.cs index 620cc0a..13209d4 100644 --- a/tests/Braintrust.Sdk.Tests/Api/BraintrustApiClientTest.cs +++ b/tests/Braintrust.Sdk.Tests/Api/BraintrustApiClientTest.cs @@ -1,13 +1,8 @@ -using System; using System.Net; -using System.Net.Http; using System.Text; using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; using Braintrust.Sdk.Api; using Braintrust.Sdk.Config; -using Xunit; namespace Braintrust.Sdk.Tests.Api; @@ -15,7 +10,6 @@ public class BraintrustApiClientTest : IDisposable { private readonly TestHttpMessageHandler _handler; private readonly HttpClient _httpClient; - private readonly BraintrustConfig _config; private readonly BraintrustApiClient _apiClient; public BraintrustApiClientTest() @@ -26,29 +20,29 @@ public BraintrustApiClientTest() BaseAddress = new Uri("https://test-api.example.com") }; - _config = BraintrustConfig.Of( + var config = BraintrustConfig.Of( "BRAINTRUST_API_KEY", "test-api-key", "BRAINTRUST_API_URL", "https://test-api.example.com", "BRAINTRUST_DEFAULT_PROJECT_NAME", "test-project" ); - _apiClient = new BraintrustApiClient(_config, _httpClient); + _apiClient = new BraintrustApiClient(config, _httpClient); } public void Dispose() { - _apiClient?.Dispose(); - _httpClient?.Dispose(); - _handler?.Dispose(); + _apiClient.Dispose(); + _httpClient.Dispose(); + _handler.Dispose(); } [Fact] - public void GetOrCreateProject_CreatesProject() + public async Task GetOrCreateProject_CreatesProject() { var expectedProject = new Project("proj-123", "test-project", "org-456"); _handler.SetResponse(HttpStatusCode.OK, expectedProject); - var result = _apiClient.GetOrCreateProject("test-project"); + var result = await _apiClient.GetOrCreateProject("test-project"); Assert.NotNull(result); Assert.Equal("proj-123", result.Id); @@ -62,37 +56,37 @@ public void GetOrCreateProject_CreatesProject() } [Fact] - public void GetProject_ReturnsProject() + public async Task GetProject_ReturnsProject() { var expectedProject = new Project("proj-123", "test-project", "org-456"); _handler.SetResponse(HttpStatusCode.OK, expectedProject); - var result = _apiClient.GetProject("proj-123"); + var result = await _apiClient.GetProject("proj-123"); Assert.NotNull(result); - Assert.Equal("proj-123", result!.Id); + Assert.Equal("proj-123", result.Id); Assert.Equal(HttpMethod.Get, _handler.LastRequest?.Method); Assert.Equal("/v1/project/proj-123", _handler.LastRequest?.RequestUri?.AbsolutePath); } [Fact] - public void GetProject_ReturnsNull_When404() + public async Task GetProject_ReturnsNull_When404() { _handler.SetResponse(HttpStatusCode.NotFound, "Not found"); - var result = _apiClient.GetProject("missing-project"); + var result = await _apiClient.GetProject("missing-project"); Assert.Null(result); } [Fact] - public void GetOrCreateExperiment_CreatesExperiment() + public async Task GetOrCreateExperiment_CreatesExperiment() { var expectedExperiment = new Experiment("exp-123", "proj-456", "test-experiment"); _handler.SetResponse(HttpStatusCode.OK, expectedExperiment); var request = new CreateExperimentRequest("proj-456", "test-experiment", "Test description"); - var result = _apiClient.GetOrCreateExperiment(request); + var result = await _apiClient.GetOrCreateExperiment(request); Assert.NotNull(result); Assert.Equal("exp-123", result.Id); @@ -103,21 +97,18 @@ public void GetOrCreateExperiment_CreatesExperiment() } [Fact] - public void GetOrCreateProjectAndOrgInfo_ReturnsInfo() + public async Task GetOrCreateProjectAndOrgInfo_ReturnsInfo() { // Setup: First call creates/gets project, second call is login var project = new Project("proj-123", "test-project", "org-456"); - var loginResponse = new LoginResponse(new System.Collections.Generic.List - { - new OrganizationInfo("org-456", "Test Org") - }); + var loginResponse = new LoginResponse([new OrganizationInfo("org-456", "Test Org")]); _handler.SetResponses( (HttpStatusCode.OK, project), (HttpStatusCode.OK, loginResponse) ); - var result = _apiClient.GetOrCreateProjectAndOrgInfo(); + var result = await _apiClient.GetOrCreateProjectAndOrgInfo(); Assert.NotNull(result); Assert.Equal("proj-123", result.Project.Id); @@ -126,11 +117,11 @@ public void GetOrCreateProjectAndOrgInfo_ReturnsInfo() } [Fact] - public void ApiException_ThrownOn_HttpError() + public async Task ApiException_ThrownOn_HttpError() { _handler.SetResponse(HttpStatusCode.BadRequest, "Bad request"); - var exception = Assert.Throws(() => + var exception = await Assert.ThrowsAsync(() => _apiClient.GetProject("test")); Assert.Equal(400, exception.StatusCode); diff --git a/tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs b/tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs index a2fec6e..9ff95b2 100644 --- a/tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs +++ b/tests/Braintrust.Sdk.Tests/Eval/EvalTest.cs @@ -1,10 +1,6 @@ -using System; using System.Diagnostics; -using System.Linq; -using Braintrust.Sdk.Api; using Braintrust.Sdk.Config; using Braintrust.Sdk.Eval; -using Xunit; namespace Braintrust.Sdk.Tests.Eval; @@ -33,7 +29,7 @@ public void Dispose() } [Fact] - public void BasicEvalBuildsAndRuns() + public async Task BasicEvalBuildsAndRuns() { // Arrange var config = BraintrustConfig.Of( @@ -52,7 +48,7 @@ public void BasicEvalBuildsAndRuns() }; // Act - var eval = Eval.NewBuilder() + var eval = await Eval.NewBuilder() .Name("test-eval") .Config(config) .ApiClient(mockClient) @@ -62,9 +58,9 @@ public void BasicEvalBuildsAndRuns() Scorer.Of("fruit_scorer", (expected, actual) => expected == "fruit" && actual == "fruit" ? 1.0 : 0.0), Scorer.Of("vegetable_scorer", (expected, actual) => expected == "vegetable" && actual == "vegetable" ? 1.0 : 0.0) ) - .Build(); + .BuildAsync(); - var result = eval.Run(); + var result = await eval.RunAsync(); // Assert Assert.NotNull(result); @@ -74,51 +70,51 @@ public void BasicEvalBuildsAndRuns() } [Fact] - public void EvalRequiresAtLeastOneScorer() + public async Task EvalRequiresAtLeastOneScorer() { var config = BraintrustConfig.Of("BRAINTRUST_API_KEY", "test-key"); var mockClient = new MockBraintrustApiClient(); - Assert.Throws(() => + await Assert.ThrowsAsync(() => Eval.NewBuilder() .Name("test-eval") .Config(config) .ApiClient(mockClient) .Cases(DatasetCase.Of("input", "expected")) .TaskFunction(x => x) - .Build()); + .BuildAsync()); } [Fact] - public void EvalRequiresDataset() + public async Task EvalRequiresDataset() { var config = BraintrustConfig.Of("BRAINTRUST_API_KEY", "test-key"); var mockClient = new MockBraintrustApiClient(); - Assert.Throws(() => + await Assert.ThrowsAsync(() => Eval.NewBuilder() .Name("test-eval") .Config(config) .ApiClient(mockClient) .TaskFunction(x => x) .Scorers(Scorer.Of("test", (_, _) => 1.0)) - .Build()); + .BuildAsync()); } [Fact] - public void EvalRequiresTask() + public async Task EvalRequiresTask() { var config = BraintrustConfig.Of("BRAINTRUST_API_KEY", "test-key"); var mockClient = new MockBraintrustApiClient(); - Assert.Throws(() => + await Assert.ThrowsAsync(() => Eval.NewBuilder() .Name("test-eval") .Config(config) .ApiClient(mockClient) .Cases(DatasetCase.Of("input", "expected")) .Scorers(Scorer.Of("test", (_, _) => 1.0)) - .Build()); + .BuildAsync()); } [Fact] @@ -171,43 +167,4 @@ public void DatasetOfCreatesInMemoryDataset() Assert.Equal("input2", case2.Input); Assert.Null(case3); } -} - -/// -/// Mock API client for testing that doesn't make real HTTP calls. -/// -internal class MockBraintrustApiClient : IBraintrustApiClient -{ - private readonly OrganizationInfo _orgInfo = new OrganizationInfo("test-org-id", "test-org"); - private readonly Project _project = new Project("test-project-id", "test-project", "test-org-id", null, null); - - public Project GetOrCreateProject(string projectName) - { - return _project; - } - - public Project? GetProject(string projectId) - { - return _project; - } - - public Experiment GetOrCreateExperiment(CreateExperimentRequest request) - { - return new Experiment("test-experiment-id", request.ProjectId, request.Name, request.Description, null, null); - } - - public OrganizationAndProjectInfo? GetProjectAndOrgInfo() - { - return new OrganizationAndProjectInfo(_orgInfo, _project); - } - - public OrganizationAndProjectInfo? GetProjectAndOrgInfo(string projectId) - { - return new OrganizationAndProjectInfo(_orgInfo, _project); - } - - public OrganizationAndProjectInfo GetOrCreateProjectAndOrgInfo() - { - return new OrganizationAndProjectInfo(_orgInfo, _project); - } -} +} \ No newline at end of file diff --git a/tests/Braintrust.Sdk.Tests/Eval/MockBraintrustApiClient.cs b/tests/Braintrust.Sdk.Tests/Eval/MockBraintrustApiClient.cs new file mode 100644 index 0000000..75ff5cf --- /dev/null +++ b/tests/Braintrust.Sdk.Tests/Eval/MockBraintrustApiClient.cs @@ -0,0 +1,42 @@ +using Braintrust.Sdk.Api; + +namespace Braintrust.Sdk.Tests.Eval; + +/// +/// Mock API client for testing that doesn't make real HTTP calls. +/// +internal class MockBraintrustApiClient : IBraintrustApiClient +{ + private readonly OrganizationInfo _orgInfo = new OrganizationInfo("test-org-id", "test-org"); + private readonly Project _project = new Project("test-project-id", "test-project", "test-org-id"); + + public Task GetOrCreateProject(string projectName) + { + return Task.FromResult(_project); + } + + public Task GetProject(string projectId) + { + return Task.FromResult(_project); + } + + public Task GetOrCreateExperiment(CreateExperimentRequest request) + { + return Task.FromResult(new Experiment("test-experiment-id", request.ProjectId, request.Name, request.Description)); + } + + public Task GetProjectAndOrgInfo() + { + return Task.FromResult(new OrganizationAndProjectInfo(_orgInfo, _project)); + } + + public Task GetProjectAndOrgInfo(string projectId) + { + return Task.FromResult(new OrganizationAndProjectInfo(_orgInfo, _project)); + } + + public Task GetOrCreateProjectAndOrgInfo() + { + return Task.FromResult(new OrganizationAndProjectInfo(_orgInfo, _project)); + } +} \ No newline at end of file