feat(ts): prompts ts sdk

.github/workflows/typescript-packages-publish-experimental.yml

@@ -0,0 +1,55 @@
+name: Publish Any Pull Request
+# TODO: Only publish on pull requests by arize team members
+on:
+  pull_request:
+
+jobs:
+  # JOB to run change detection
+  changes:
+    runs-on: ubuntu-latest
+    # Required permissions
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step
+    outputs:
+      workflow_file: ${{ steps.filter.outputs.workflow_file }}
+      phoenix-client: ${{ steps.filter.outputs.phoenix-client }}
+    steps:
+      # For pull requests it's not necessary to checkout the code
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            phoenix-client:
+              - 'js/packages/phoenix-client/**'
+            workflow_file:
+              - '.github/workflows/typescript-packages-publish-experimental.yml'
+
+  publish-experimental-packages:
+    needs: changes
+    if: ${{ needs.changes.outputs.phoenix-client == 'true' || needs.changes.outputs.workflow_file == 'true' }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - run: corepack enable
+        working-directory: ./js
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: "pnpm"
+          cache-dependency-path: ./js
+
+      - name: Install dependencies
+        working-directory: ./js
+        run: pnpm install
+
+      - name: Build
+        working-directory: ./js
+        run: pnpm -r build
+
+      - name: Publish
+        working-directory: ./js
+        run: pnpx pkg-pr-new publish ./packages/phoenix-client

js/examples/apps/phoenix-experiment-runner/index.ts

@@ -1,10 +1,10 @@
 /* eslint-disable no-console */
+import { createClient } from "@arizeai/phoenix-client";
 import {
   asEvaluator,
-  createClient,
   runExperiment,
   type RunExperimentParams,
-} from "@arizeai/phoenix-client";
+} from "@arizeai/phoenix-client/experimental";
 import { intro, outro, select, spinner, log, confirm } from "@clack/prompts";
 import { Factuality, Humor } from "autoevals";
 import dotenv from "dotenv";

js/package.json

@@ -28,6 +28,7 @@
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^6.21.0",
     "eslint": "^8.57.1",
+    "pkg-pr-new": "^0.0.39",
     "prettier": "^3.4.1",
     "rimraf": "^5.0.10",
     "tsc-alias": "^1.8.10",

js/packages/phoenix-client/README.md

@@ -18,13 +18,15 @@ The client will automatically read environment variables from your environment,
 The following environment variables are used:
 
 - `PHOENIX_HOST` - The base URL of the Phoenix API.
+- `PHOENIX_API_KEY` - The API key to use for authentication.
 - `PHOENIX_CLIENT_HEADERS` - Custom headers to add to all requests. A JSON stringified object.
 
 ```bash
-PHOENIX_HOST=http://localhost:6006 PHOENIX_CLIENT_HEADERS='{"Authorization": "bearer xxxxxx"}' pnpx tsx examples/list_datasets.ts
+PHOENIX_HOST='http://localhost:12345' PHOENIX_API_KEY='xxxxxx' PHOENIX_CLIENT_HEADERS='{"X-Custom-Header": "123"}' pnpx tsx examples/list_datasets.ts
 # emits the following request:
-# GET http://localhost:6006/v1/datasets
+# GET http://localhost:12345/v1/datasets
 # headers: {
+#   "X-Custom-Header": "123",
 #   "Authorization": "bearer xxxxxx",
 # }
 ```

js/packages/phoenix-client/examples/apply_prompt.ts

@@ -0,0 +1,57 @@
+/* eslint-disable no-console */
+import { createClient, getPrompt, toSDK } from "../src";
+import OpenAI from "openai";
+
+const PROMPT_NAME = process.env.PROMPT_NAME!;
+const OPENAI_API_KEY = process.env.OPENAI_API_KEY!;
+
+// get first argument from command line
+const question = process.argv[2];
+
+if (!question) {
+  throw new Error(
+    "Usage: pnpx tsx examples/apply_prompt.ts 'What is the capital of France?'\nAssumes that the prompt has a variable named 'question'\nAssumes that the prompt is openai with an openai model"
+  );
+}
+
+if (!OPENAI_API_KEY) {
+  throw new Error("OPENAI_API_KEY must be provided in the environment");
+}
+
+const client = createClient({
+  options: {
+    baseUrl: "http://localhost:6006",
+  },
+});
+
+const openai = new OpenAI({
+  apiKey: OPENAI_API_KEY,
+});
+
+const main = async () => {
+  const prompt = await getPrompt({
+    client,
+    prompt: { name: PROMPT_NAME },
+  });
+
+  const openAIParams = toSDK({
+    prompt,
+    sdk: "openai",
+    variables: {
+      question,
+    },
+  });
+
+  if (!openAIParams) {
+    throw new Error("Prompt could not be converted to OpenAI params");
+  }
+
+  const response = await openai.chat.completions.create({
+    ...openAIParams,
+    stream: false,
+  });
+
+  console.log(response.choices[0]?.message.content);
+};
+
+main();

js/packages/phoenix-client/examples/apply_prompt_anthropic.ts

@@ -0,0 +1,124 @@
+/* eslint-disable no-console */
+import { createClient, getPrompt, toSDK } from "../src";
+import { Anthropic } from "@anthropic-ai/sdk";
+import { PromptLike } from "../src/types/prompts";
+
+const PROMPT_NAME = process.env.PROMPT_NAME!;
+const PROMPT_TAG = process.env.PROMPT_TAG!;
+const PROMPT_VERSION_ID = process.env.PROMPT_VERSION_ID!;
+const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY!;
+
+// get first argument from command line
+const question = process.argv[2];
+
+if (!question) {
+  throw new Error(
+    "Usage: pnpx tsx examples/apply_prompt_anthropic.ts 'What is the capital of France?'\nAssumes that the prompt has a variable named 'question'"
+  );
+}
+
+if (!ANTHROPIC_API_KEY) {
+  throw new Error("ANTHROPIC_API_KEY must be provided in the environment");
+}
+
+const client = createClient({
+  options: {
+    baseUrl: "http://localhost:6006",
+  },
+});
+
+const anthropic = new Anthropic({
+  apiKey: ANTHROPIC_API_KEY,
+});
+
+const main = async () => {
+  const promptArgument: PromptLike | null = PROMPT_VERSION_ID
+    ? { versionId: PROMPT_VERSION_ID }
+    : PROMPT_TAG && PROMPT_NAME
+      ? { name: PROMPT_NAME, tag: PROMPT_TAG }
+      : PROMPT_NAME
+        ? { name: PROMPT_NAME }
+        : null;
+  if (!promptArgument) {
+    throw new Error(
+      `Either PROMPT_VERSION_ID, PROMPT_TAG and PROMPT_NAME, or PROMPT_NAME must be provided in the environment`
+    );
+  }
+  console.log(`Getting prompt ${PROMPT_VERSION_ID}`);
+
+  // TODO: Apply variable replacement to the prompt
+  const prompt = await getPrompt({
+    client,
+    prompt: promptArgument,
+  });
+
+  if (!prompt) {
+    throw new Error("Prompt not found");
+  }
+
+  console.log(
+    `Loaded prompt: ${prompt.id}\n${prompt.description ? `\n${prompt.description}` : ""}`
+  );
+
+  console.log(`Converting prompt to OpenAI params`);
+
+  const anthropicParams = toSDK({
+    prompt,
+    sdk: "anthropic",
+    variables: {
+      question,
+    },
+  });
+
+  if (!anthropicParams) {
+    throw new Error("Prompt could not be converted to Anthropic params");
+  }
+
+  // @ts-expect-error Anthropic doesn't support these parameters
+  delete anthropicParams.frequency_penalty;
+  // @ts-expect-error Anthropic doesn't support these parameters
+  delete anthropicParams.presence_penalty;
+
+  console.log(`Applying prompt to Anthropic`);
+  const response = await anthropic.messages.create({
+    ...anthropicParams,
+    // we may not have an anthropic model saved in the prompt
+    model: "claude-3-5-sonnet-20240620",
+    // TODO: should this be strongly typed inside of toSDK results if sdk: "anthropic"?
+    stream: true,
+  });
+
+  console.log(`Streaming response from OpenAI:\n\n`);
+
+  let responseText = "";
+  let responseJson = "";
+  for await (const chunk of response) {
+    if (chunk.type === "message_delta") {
+      console.clear();
+      console.log("Input:\n");
+      console.log(JSON.stringify(anthropicParams.messages, null, 2));
+      console.log("\nOutput:\n");
+      try {
+        console.log(JSON.stringify(JSON.parse(responseText), null, 2));
+        console.log(JSON.stringify(JSON.parse(responseJson), null, 2));
+      } catch {
+        console.log(responseText);
+        console.log(responseJson);
+      }
+    } else if (chunk.type === "content_block_delta") {
+      console.clear();
+      if (chunk.delta.type === "text_delta") {
+        responseText += String(chunk.delta.text);
+      }
+      if (chunk.delta.type === "input_json_delta") {
+        responseJson += chunk.delta.partial_json;
+      }
+      console.log(responseText);
+    }
+  }
+
+  console.log("\n\n");
+  console.log(`Done!`);
+};
+
+main();

js/packages/phoenix-client/examples/apply_prompt_openai.ts

@@ -0,0 +1,110 @@
+/* eslint-disable no-console */
+import { createClient, getPrompt, toSDK } from "../src";
+import OpenAI from "openai";
+import { PromptLike } from "../src/types/prompts";
+
+const PROMPT_NAME = process.env.PROMPT_NAME!;
+const PROMPT_TAG = process.env.PROMPT_TAG!;
+const PROMPT_VERSION_ID = process.env.PROMPT_VERSION_ID!;
+const OPENAI_API_KEY = process.env.OPENAI_API_KEY!;
+
+// get first argument from command line
+const question = process.argv[2];
+
+if (!question) {
+  throw new Error(
+    "Usage: pnpx tsx examples/apply_prompt_openai.ts 'What is the capital of France?'\nAssumes that the prompt has a variable named 'question'"
+  );
+}
+
+if (!OPENAI_API_KEY) {
+  throw new Error("OPENAI_API_KEY must be provided in the environment");
+}
+
+const client = createClient({
+  options: {
+    baseUrl: "http://localhost:6006",
+  },
+});
+
+const openai = new OpenAI({
+  apiKey: OPENAI_API_KEY,
+});
+
+const main = async () => {
+  const promptArgument: PromptLike | null = PROMPT_VERSION_ID
+    ? { versionId: PROMPT_VERSION_ID }
+    : PROMPT_TAG && PROMPT_NAME
+      ? { name: PROMPT_NAME, tag: PROMPT_TAG }
+      : PROMPT_NAME
+        ? { name: PROMPT_NAME }
+        : null;
+  if (!promptArgument) {
+    throw new Error(
+      `Either PROMPT_VERSION_ID, PROMPT_TAG and PROMPT_NAME, or PROMPT_NAME must be provided in the environment`
+    );
+  }
+  console.log(`Getting prompt ${PROMPT_VERSION_ID}`);
+
+  // TODO: Apply variable replacement to the prompt
+  const prompt = await getPrompt({
+    client,
+    prompt: promptArgument,
+  });
+
+  if (!prompt) {
+    throw new Error("Prompt not found");
+  }
+
+  console.log(
+    `Loaded prompt: ${prompt.id}\n${prompt.description ? `\n${prompt.description}` : ""}`
+  );
+
+  console.log(`Converting prompt to OpenAI params`);
+
+  const openAIParams = toSDK({
+    prompt,
+    sdk: "openai",
+    variables: {
+      question,
+    },
+  });
+
+  if (!openAIParams) {
+    throw new Error("Prompt could not be converted to OpenAI params");
+  }
+
+  console.log(`Applying prompt to OpenAI`);
+  const response = await openai.chat.completions.create({
+    ...openAIParams,
+    // we may not have an openai model saved in the prompt
+    model: "gpt-4o-mini",
+    // TODO: should this be strongly typed inside of toSDK results if sdk: "openai"?
+    stream: true,
+  });
+
+  console.log(`Streaming response from OpenAI:\n\n`);
+
+  let responseText = "";
+  for await (const chunk of response) {
+    if (chunk.choices[0]?.delta?.content) {
+      responseText += chunk.choices[0]?.delta?.content;
+      console.clear();
+      console.log("Input:\n");
+      console.log(JSON.stringify(openAIParams.messages, null, 2));
+      console.log("\nOutput:\n");
+      try {
+        console.log(JSON.stringify(JSON.parse(responseText), null, 2));
+      } catch {
+        console.log(responseText);
+      }
+    } else if (chunk.choices[0]?.delta?.tool_calls) {
+      console.log(chunk.choices[0]?.delta?.tool_calls);
+    }
+  }
+
+  console.log("\n\n");
+  console.log(`Done!`);
+};
+
+main();