From c1ff0debaa415609ee8b8da55be5e91691a0e1e6 Mon Sep 17 00:00:00 2001 From: Hurshal Patel Date: Thu, 20 Feb 2025 17:13:15 -0800 Subject: [PATCH 1/5] fix comments --- README.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 808e998..2e72f83 100644 --- a/README.md +++ b/README.md @@ -60,11 +60,11 @@ print(f"Factuality score: {result.score}") print(f"Factuality metadata: {result.metadata['rationale']}") ``` -#### Use with other AI providers through the Braintrust AI Proxy +#### Use with other AI providers through the AI Proxy -Autoevals will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI compatible API. If `OPENAI_BASE_URL` is not set, it will default to the [Braintrust AI Proxy](https://www.braintrust.dev/docs/guides/proxy). This provides numerous benefits like simplified access to many AI providers, reduced costs with automatic request caching, and increased observability when you enable logging to Braintrust. The proxy is free to use, even if you don't have a Braintrust account. +Autoevals will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI compatible API. If `OPENAI_BASE_URL` is not set, it will default to the [AI Proxy](https://www.braintrust.dev/docs/guides/proxy). This provides numerous benefits like simplified access to many AI providers, reduced costs with automatic request caching, and increased observability when you enable logging to Braintrust. The proxy is free to use, even if you don't have a Braintrust account. -If you have a Braintrust account, you can set the `BRAINTUST_API_KEY` environment variable instead of `OPENAI_API_KEY` to unlock additional features like logging and monitoring. Additionally, you can route requests to [supported AI providers and models](https://www.braintrust.dev/docs/guides/proxy#supported-models) or custom models you have configured in the Braintrust dashboard. +If you have a Braintrust account, you can set the `BRAINTUST_API_KEY` environment variable instead of `OPENAI_API_KEY` to unlock additional features like logging and monitoring. Additionally, you can route requests to [supported AI providers and models](https://www.braintrust.dev/docs/guides/proxy#supported-models) or custom models you have configured in Braintrust. ```python # NOTE: ensure BRAINTRUST_API_KEY is set in your environment and OPENAI_API_KEY is not set @@ -132,13 +132,14 @@ import { Factuality } from "autoevals"; })(); ``` -#### Use with other AI providers through the Braintrust AI Proxy +#### Use with other AI providers through the AI Proxy -Autoevals will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI compatible API. If `OPENAI_BASE_URL` is not set, it will default to the [Braintrust AI Proxy](https://www.braintrust.dev/docs/guides/proxy). This provides numerous benefits like simplified access to many AI providers, reduced costs with automatic request caching, and increased observability when you enable logging to Braintrust. The proxy is free to use, even if you don't have a Braintrust account. +Autoevals will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI compatible API. If `OPENAI_BASE_URL` is not set, it will default to the [AI Proxy](https://www.braintrust.dev/docs/guides/proxy). This provides numerous benefits like simplified access to many AI providers, reduced costs with automatic request caching, and increased observability when you enable logging to Braintrust. The proxy is free to use, even if you don't have a Braintrust account. -If you have a Braintrust account, you can set the `BRAINTUST_API_KEY` environment variable instead of `OPENAI_API_KEY` to unlock additional features like logging and monitoring. Additionally, you can route requests to [supported AI providers and models](https://www.braintrust.dev/docs/guides/proxy#supported-models) or custom models you have configured in the Braintrust dashboard. +If you have a Braintrust account, you can set the `BRAINTUST_API_KEY` environment variable instead of `OPENAI_API_KEY` to unlock additional features like logging and monitoring. Additionally, you can route requests to [supported AI providers and models](https://www.braintrust.dev/docs/guides/proxy#supported-models) or custom models you have configured in Braintrust. ```javascript +// NOTE: ensure BRAINTRUST_API_KEY is set in your environment and OPENAI_API_KEY is not set import { Factuality } from "autoevals"; (async () => { @@ -146,14 +147,17 @@ import { Factuality } from "autoevals"; const output = "People's Republic of China"; const expected = "China"; + // Run an LLM-based evaluator using the Claude 3.5 Sonnet model from Anthropic const result = await Factuality({ model: "claude-3-5-sonnet-latest", output, expected, input, }); + + // The evaluator returns a score from [0,1] and includes the raw outputs from the evaluator console.log(`Factuality score: ${result.score}`); - console.log(`Factuality metadata: ${result.metadata.rationale}`); + console.log(`Factuality metadata: ${result.metadata?.rationale}`); })(); ``` From 9cca8b5710fb66d25e62d87caa84475c3a796a77 Mon Sep 17 00:00:00 2001 From: Ornella Altunyan Date: Fri, 21 Feb 2025 10:27:11 -0800 Subject: [PATCH 2/5] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2e72f83..e165022 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ print(f"Factuality score: {result.score}") print(f"Factuality metadata: {result.metadata['rationale']}") ``` -#### Use with other AI providers through the AI Proxy +#### Use with other AI providers through the AI proxy Autoevals will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI compatible API. If `OPENAI_BASE_URL` is not set, it will default to the [AI Proxy](https://www.braintrust.dev/docs/guides/proxy). This provides numerous benefits like simplified access to many AI providers, reduced costs with automatic request caching, and increased observability when you enable logging to Braintrust. The proxy is free to use, even if you don't have a Braintrust account. From 557a80a838545233832b0a5cd403f056a0013ec7 Mon Sep 17 00:00:00 2001 From: Ornella Altunyan Date: Fri, 21 Feb 2025 10:27:19 -0800 Subject: [PATCH 3/5] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e165022..cfeb646 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ print(f"Factuality metadata: {result.metadata['rationale']}") #### Use with other AI providers through the AI proxy -Autoevals will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI compatible API. If `OPENAI_BASE_URL` is not set, it will default to the [AI Proxy](https://www.braintrust.dev/docs/guides/proxy). This provides numerous benefits like simplified access to many AI providers, reduced costs with automatic request caching, and increased observability when you enable logging to Braintrust. The proxy is free to use, even if you don't have a Braintrust account. +Autoevals will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI compatible API. If `OPENAI_BASE_URL` is not set, it will default to the [AI proxy](https://www.braintrust.dev/docs/guides/proxy). This provides numerous benefits like simplified access to many AI providers, reduced costs with automatic request caching, and increased observability when you enable logging to Braintrust. The proxy is free to use, even if you don't have a Braintrust account. If you have a Braintrust account, you can set the `BRAINTUST_API_KEY` environment variable instead of `OPENAI_API_KEY` to unlock additional features like logging and monitoring. Additionally, you can route requests to [supported AI providers and models](https://www.braintrust.dev/docs/guides/proxy#supported-models) or custom models you have configured in Braintrust. From 9ef1cdf6d180e59dc962514a4de86df0d3bf71f8 Mon Sep 17 00:00:00 2001 From: Ornella Altunyan Date: Fri, 21 Feb 2025 10:27:29 -0800 Subject: [PATCH 4/5] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cfeb646..2823460 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ import { Factuality } from "autoevals"; #### Use with other AI providers through the AI Proxy -Autoevals will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI compatible API. If `OPENAI_BASE_URL` is not set, it will default to the [AI Proxy](https://www.braintrust.dev/docs/guides/proxy). This provides numerous benefits like simplified access to many AI providers, reduced costs with automatic request caching, and increased observability when you enable logging to Braintrust. The proxy is free to use, even if you don't have a Braintrust account. +Autoevals will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI compatible API. If `OPENAI_BASE_URL` is not set, it will default to the [AI proxy](https://www.braintrust.dev/docs/guides/proxy). This provides numerous benefits like simplified access to many AI providers, reduced costs with automatic request caching, and increased observability when you enable logging to Braintrust. The proxy is free to use, even if you don't have a Braintrust account. If you have a Braintrust account, you can set the `BRAINTUST_API_KEY` environment variable instead of `OPENAI_API_KEY` to unlock additional features like logging and monitoring. Additionally, you can route requests to [supported AI providers and models](https://www.braintrust.dev/docs/guides/proxy#supported-models) or custom models you have configured in Braintrust. From dee95f3add643a8f6aa4baa364e535c68337a784 Mon Sep 17 00:00:00 2001 From: Ornella Altunyan Date: Fri, 21 Feb 2025 10:27:37 -0800 Subject: [PATCH 5/5] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2823460..71233e8 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ import { Factuality } from "autoevals"; })(); ``` -#### Use with other AI providers through the AI Proxy +#### Use with other AI providers through the AI proxy Autoevals will look for an `OPENAI_BASE_URL` environment variable to use as the base for requests to an OpenAI compatible API. If `OPENAI_BASE_URL` is not set, it will default to the [AI proxy](https://www.braintrust.dev/docs/guides/proxy). This provides numerous benefits like simplified access to many AI providers, reduced costs with automatic request caching, and increased observability when you enable logging to Braintrust. The proxy is free to use, even if you don't have a Braintrust account.