bump version

.github/workflows/js.yaml

@@ -1,8 +1,5 @@
 name: js
 
-env:
-  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-
 on:
   pull_request:
   push:
@@ -14,7 +11,8 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [18.x]
+        # duckdb has an incredibly slow install with 24.x
+        node-version: [20.x, 22.x]
 
     steps:
       - uses: actions/checkout@v3
@@ -34,4 +32,8 @@ jobs:
       - uses: pnpm/action-setup@v4
       - run: pnpm install
       - run: pnpm run test
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
+          BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
       - run: pnpm run build

.github/workflows/python.yaml

@@ -1,9 +1,6 @@
 # Modified from https://github.com/actions/starter-workflows/blob/main/ci/python-app.yml
 name: python
 
-env:
-  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-
 on:
   pull_request:
   push:
@@ -29,3 +26,7 @@ jobs:
       - name: Test with pytest
         run: |
           pytest
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
+          BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}

Makefile

@@ -41,4 +41,4 @@ test-py:
 	source env.sh && python3 -m pytest
 
 test-js:
-	npm install && npm run test
+	pnpm install && pnpm run test

js/embeddings.test.ts

@@ -21,6 +21,8 @@ const UNRELATED = [
   "I like to eat apples",
 ];
 
+import { test, expect } from "vitest";
+
 test("Embeddings Test", async () => {
   const prefix = "resource type: ";
   for (const { word, synonyms } of SYNONYMS) {

js/json.test.ts

@@ -2,6 +2,8 @@ import { JSONDiff, ValidJSON } from "./json";
 import { NumericDiff } from "./number";
 import { ExactMatch } from "./value";
 
+import { test, expect } from "vitest";
+
 test("JSON String Test", async () => {
   const cases = [
     { a: "", b: "", expected: 1 },

js/llm.test.ts

@@ -1,21 +1,30 @@
+import { bypass, http, HttpResponse } from "msw";
+import { setupServer } from "msw/node";
+import { OpenAI } from "openai";
 import { ChatCompletionMessageParam } from "openai/resources";
+import { afterAll, afterEach, beforeAll, describe, expect, test } from "vitest";
 import {
   Battle,
+  buildClassificationTools,
   LLMClassifierFromTemplate,
   OpenAIClassifier,
-  buildClassificationTools,
 } from "../js/llm";
-import { bypass, http, HttpResponse } from "msw";
-import { server } from "./test/setup";
-import { OpenAI } from "openai";
-import { init } from "./oai";
 import {
   openaiClassifierShouldEvaluateArithmeticExpressions,
   openaiClassifierShouldEvaluateTitles,
   openaiClassifierShouldEvaluateTitlesWithCoT,
 } from "./llm.fixtures";
+import { init } from "./oai";
+
+export const server = setupServer();
 
 beforeAll(() => {
+  server.listen({
+    onUnhandledRequest: (req) => {
+      throw new Error(`Unhandled request ${req.method}, ${req.url}`);
+    },
+  });
+
   init({
     client: new OpenAI({
       apiKey: "test-api-key",
@@ -24,7 +33,12 @@ beforeAll(() => {
   });
 });
 
+afterEach(() => {
+  server.resetHandlers();
+});
+
 afterAll(() => {
+  server.close();
   init();
 });
 

js/llm.ts

@@ -1,5 +1,3 @@
-import mustache from "mustache";
-
 import { Score, Scorer, ScorerArgs } from "@braintrust/core";
 import { ChatCache, OpenAIAuth, cachedChatCompletion } from "./oai";
 import { ModelGradedSpec, templates } from "./templates";

js/moderation.test.ts

@@ -1,5 +1,5 @@
-import exp from "constants";
 import { Moderation } from "./moderation";
+import { describe, expect, test } from "vitest";
 
 describe("Moderation", () => {
   const cases = [

js/oai.test.ts

@@ -1,7 +1,47 @@
-import { buildOpenAIClient, init } from "./oai";
 import { http, HttpResponse } from "msw";
-import { server } from "./test/setup";
 import OpenAI from "openai";
+import {
+  afterAll,
+  afterEach,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  test,
+  vi,
+} from "vitest";
+import { buildOpenAIClient, init } from "./oai";
+
+import { setupServer } from "msw/node";
+
+export const server = setupServer();
+
+beforeAll(() => {
+  server.listen({
+    onUnhandledRequest: (req) => {
+      throw new Error(`Unhandled request ${req.method}, ${req.url}`);
+    },
+  });
+});
+
+let OPENAI_API_KEY: string | undefined;
+let OPENAI_BASE_URL: string | undefined;
+
+beforeEach(() => {
+  OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+  OPENAI_BASE_URL = process.env.OPENAI_BASE_URL;
+});
+
+afterEach(() => {
+  server.resetHandlers();
+
+  process.env.OPENAI_API_KEY = OPENAI_API_KEY;
+  process.env.OPENAI_BASE_URL = OPENAI_BASE_URL;
+});
+
+afterAll(() => {
+  server.close();
+});
 
 const MOCK_OPENAI_COMPLETION_RESPONSE = {
   choices: [
@@ -78,8 +118,12 @@ describe("OAI", () => {
   });
 
   test("calls proxy if everything unset", async () => {
+    delete process.env.OPENAI_API_KEY;
+    delete process.env.OPENAI_BASE_URL;
+
     server.use(
       http.post("https://api.braintrust.dev/v1/proxy/chat/completions", () => {
+        debugger;
         return HttpResponse.json(MOCK_OPENAI_COMPLETION_RESPONSE);
       }),
     );
@@ -90,12 +134,17 @@ describe("OAI", () => {
       messages: [{ role: "user", content: "Hello" }],
     });
 
+    debugger;
+
     expect(response.choices[0].message.content).toBe(
       "Hello, I am a mock response!",
     );
   });
 
   test("default wraps", async () => {
+    delete process.env.OPENAI_API_KEY;
+    delete process.env.OPENAI_BASE_URL;
+
     server.use(
       http.post("https://api.braintrust.dev/v1/proxy/chat/completions", () => {
         return HttpResponse.json(MOCK_OPENAI_COMPLETION_RESPONSE);
@@ -119,6 +168,9 @@ describe("OAI", () => {
   });
 
   test("wraps once", async () => {
+    delete process.env.OPENAI_API_KEY;
+    delete process.env.OPENAI_BASE_URL;
+
     server.use(
       http.post("https://api.braintrust.dev/v1/proxy/chat/completions", () => {
         return HttpResponse.json(MOCK_OPENAI_COMPLETION_RESPONSE);
@@ -210,10 +262,10 @@ describe("OAI", () => {
 const withMockWrapper = async (
   fn: (args: {
     wrapperMock: (client: any) => any;
-    createSpy: jest.Mock;
+    createSpy: ReturnType<typeof vi.fn>;
   }) => Promise<void>,
 ) => {
-  const createSpy = jest.fn();
+  const createSpy = vi.fn();
   const wrapperMock = (client: any) => {
     return new Proxy(client, {
       get(target, prop) {

js/oai.ts

@@ -6,8 +6,6 @@ import {
 } from "openai/resources";
 import { AzureOpenAI, OpenAI } from "openai";
 
-import { Env } from "./env";
-
 export interface CachedLLMParams {
   /**
    Model to use for the completion.
@@ -102,21 +100,29 @@ const resolveOpenAIClient = (options: OpenAIAuth): OpenAI => {
     return globalThis.__client;
   }
 
-  return azureOpenAi
-    ? new AzureOpenAI({
-        apiKey: azureOpenAi.apiKey,
-        endpoint: azureOpenAi.endpoint,
-        apiVersion: azureOpenAi.apiVersion,
-        defaultHeaders: openAiDefaultHeaders,
-        dangerouslyAllowBrowser: openAiDangerouslyAllowBrowser,
-      })
-    : new OpenAI({
-        apiKey: openAiApiKey || Env.OPENAI_API_KEY || Env.BRAINTRUST_API_KEY,
-        organization: openAiOrganizationId,
-        baseURL: openAiBaseUrl || Env.OPENAI_BASE_URL || PROXY_URL,
-        defaultHeaders: openAiDefaultHeaders,
-        dangerouslyAllowBrowser: openAiDangerouslyAllowBrowser,
-      });
+  if (azureOpenAi) {
+    // if not unset will could raise an exception
+    delete process.env.OPENAI_BASE_URL;
+
+    return new AzureOpenAI({
+      apiKey: azureOpenAi.apiKey,
+      endpoint: azureOpenAi.endpoint,
+      apiVersion: azureOpenAi.apiVersion,
+      defaultHeaders: openAiDefaultHeaders,
+      dangerouslyAllowBrowser: openAiDangerouslyAllowBrowser,
+    });
+  }
+
+  return new OpenAI({
+    apiKey:
+      openAiApiKey ||
+      process.env.OPENAI_API_KEY ||
+      process.env.BRAINTRUST_API_KEY,
+    organization: openAiOrganizationId,
+    baseURL: openAiBaseUrl || process.env.OPENAI_BASE_URL || PROXY_URL,
+    defaultHeaders: openAiDefaultHeaders,
+    dangerouslyAllowBrowser: openAiDangerouslyAllowBrowser,
+  });
 };
 
 const isWrapped = (client: OpenAI): boolean => {

js/partial.test.ts

@@ -1,3 +1,4 @@
+import { expect, test } from "vitest";
 import { ClosedQA } from "./llm";
 import { Levenshtein } from "./string";
 

js/ragas.test.ts

@@ -1,3 +1,4 @@
+import { expect, test } from "vitest";
 import {
   AnswerCorrectness,
   AnswerRelevancy,

js/ragas.ts

@@ -66,7 +66,7 @@ const entitySchema = z.object({
 export const ContextEntityRecall: ScorerWithPartial<
   string,
   RagasArgs & {
-    pairwiseScorer?: Scorer<string, {}>;
+    pairwiseScorer?: Scorer<string, object>;
   }
 > = makePartial(async (args) => {
   const { chatArgs, client, ...inputs } = parseArgs(args);
@@ -775,7 +775,7 @@ export const AnswerCorrectness: ScorerWithPartial<
   RagasArgs & {
     factualityWeight?: number;
     answerSimilarityWeight?: number;
-    answerSimilarity?: Scorer<string, {}>;
+    answerSimilarity?: Scorer<string, object>;
   }
 > = makePartial(async (args) => {
   const { chatArgs, client, ...inputs } = parseArgs(args);
@@ -904,7 +904,8 @@ function checkRequired<T>(
     }
   }
 
-  return args as Record<string, T>;
+  // eslint-disable-next-line
+  return args as Record<string, any>;
 }
 
 function mustParseArgs(

js/render-messages.test.ts

@@ -1,3 +1,4 @@
+import { describe, expect, it } from "vitest";
 import { renderMessages } from "./render-messages";
 import { ChatCompletionMessageParam } from "openai/resources";
 

js/templates.ts

@@ -33,9 +33,12 @@ const templateStrings = {
   translation,
 } as const;
 
+// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
 export const templates = Object.fromEntries(
   Object.entries(templateStrings).map(([name, template]) => [
     name,
-    modelGradedSpecSchema.parse(yaml.load(template)),
+    modelGradedSpecSchema.parse(
+      typeof template === "string" ? yaml.load(template) : template,
+    ),
   ]),
 ) as Record<keyof typeof templateStrings, ModelGradedSpec>;