diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 000000000..764a76855 --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,43 @@ +name: e2e + +on: + pull_request: + paths: + - "e2e/**" + - "js/**" + - ".github/workflows/e2e.yaml" + - "package.json" + - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" + - "turbo.json" + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 20 + + env: + BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + + - uses: pnpm/action-setup@v4 + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Run e2e tests + run: pnpm test:e2e diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 000000000..a96f097e9 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,80 @@ +# E2E Tests + +End-to-end tests that validate the Braintrust SDK by running real usage scenarios against a mock Braintrust server. + +## How It Works + +1. Each colocated `scenario.test.ts` file uses `withScenarioHarness(...)`, which starts an isolated mock Braintrust server. +2. The test resolves its own scenario folder and spawns a scenario entrypoint as a subprocess. +3. The scenario uses the SDK normally (init, create spans, log data, flush, or OTEL / OpenAI integrations). +4. The test inspects captured events, payloads, or raw HTTP requests, then normalizes and snapshots them where useful. + +Subprocess isolation keeps the SDK execution path close to production, including plain Node runs for auto-instrumentation hook coverage. + +## Structure + +```text +e2e/ +|- helpers/ # Shared harness, mock server, normalization, selectors, summaries +|- scenarios/ +| `- / +| |- scenario.ts # Default tsx entrypoint +| |- scenario.mjs # Default plain-Node entrypoint when needed +| |- scenario.test.ts # Colocated Vitest suite +| |- package.json # Optional slim scenario-local deps +| `- __snapshots__/ # Colocated snapshots +`- vitest.config.mts +``` + +Any extra files needed only by one scenario stay in that scenario folder. Anything reused by multiple scenarios belongs in `e2e/helpers/`. + +## Helpers (`helpers/`) + +- `scenario-harness.ts` - Starts the mock server, creates a unique test run id, resolves scenario directories, and runs scenario folders. +- `scenario-installer.ts` - Installs optional scenario-local dependencies from a colocated `package.json`. +- `mock-braintrust-server.ts` - Captures requests, merged log payloads, and parsed span-like events. +- `normalize.ts` - Makes snapshots deterministic by normalizing ids, timestamps, paths, and mock-server URLs. +- `trace-selectors.ts` / `trace-summary.ts` - Helpers for finding spans and snapshotting only the relevant shape. +- `scenario-runtime.ts` - Shared runtime utilities used by scenario entrypoints. +- `openai.ts` - Shared scenario lists and assertions for OpenAI wrapper and hook coverage across v4/v5/v6. + +### Writing a new test + +Most tests use this pattern: + +```ts +const scenarioDir = resolveScenarioDir(import.meta.url); + +beforeAll(async () => { + await installScenarioDependencies({ scenarioDir }); +}); +``` + +`installScenarioDependencies(...)` is optional and only needed when the scenario folder has its own `package.json`. + +`withScenarioHarness(async (harness) => { ... })` gives each test a fresh server plus helpers for running scenarios and reading what the server captured. + +The main utilities you'll use in test files: + +- `resolveScenarioDir(import.meta.url)` - Resolves the folder that contains the current test. +- `installScenarioDependencies({ scenarioDir })` - Installs optional scenario-local dependencies. +- `runScenarioDir({ scenarioDir, entry?, timeoutMs? })` - Runs a TypeScript scenario with `tsx`. +- `runNodeScenarioDir({ scenarioDir, entry?, nodeArgs?, timeoutMs? })` - Runs plain Node scenarios, used for `--import braintrust/hook.mjs`. +- `testRunEvents()` - Returns parsed events tagged with the current test run id. +- `events()`, `payloads()`, `requestCursor()`, `requestsAfter()` - Lower-level access for ingestion payloads and HTTP request flow assertions. +- `testRunId` - Useful when a scenario or assertion needs the exact run marker. + +Use `normalizeForSnapshot(...)` before snapshotting. It replaces timestamps and ids with stable tokens and strips machine-specific paths and localhost ports. + +### Scenario-local `package.json` + +Scenario-local manifests are optional and should stay slim. They are only for scenario-specific external dependencies, such as OpenAI version matrices. Shared test tooling and workspace-local packages stay in `e2e/package.json`. + +`workspace:` dependency specs are intentionally not supported in scenario-local manifests. If a scenario needs a workspace package, keep that dependency in `e2e/package.json`. + +## Running + +```bash +pnpm run test:e2e # Run tests +pnpm run test:e2e:update # Run tests and update snapshots +``` diff --git a/e2e/helpers/mock-braintrust-server.ts b/e2e/helpers/mock-braintrust-server.ts new file mode 100644 index 000000000..1de467465 --- /dev/null +++ b/e2e/helpers/mock-braintrust-server.ts @@ -0,0 +1,402 @@ +import { randomUUID } from "node:crypto"; +import { createServer } from "node:http"; +import type { + IncomingHttpHeaders, + IncomingMessage, + ServerResponse, +} from "node:http"; +import type { AddressInfo } from "node:net"; + +export type JsonValue = + | null + | boolean + | number + | string + | JsonValue[] + | { [key: string]: JsonValue }; + +export interface CapturedRequest { + method: string; + path: string; + query: Record; + headers: Record; + rawBody: string; + jsonBody: JsonValue | null; +} + +export type CapturedLogRow = Record; + +export type CapturedLogPayload = { + api_version: number; + rows: CapturedLogRow[]; +}; + +export type CapturedLogEvent = { + apiVersion: number; + context?: Record; + expected?: unknown; + experimentId?: string; + input?: unknown; + isMerge: boolean; + metadata?: Record; + metrics?: Record; + output?: unknown; + projectId?: string; + row: CapturedLogRow; + scores?: unknown; + span: { + ended: boolean; + id?: string; + name?: string; + parentIds: string[]; + rootId?: string; + started: boolean; + type?: string; + }; +}; + +interface MockBraintrustServer { + apiKey: string; + close: () => Promise; + events: CapturedLogEvent[]; + payloads: CapturedLogPayload[]; + requests: CapturedRequest[]; + url: string; +} + +const DEFAULT_API_KEY = "mock-braintrust-api-key"; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function clone(value: T): T { + return structuredClone(value); +} + +function normalizeHeaders( + headers: IncomingHttpHeaders, +): Record { + return Object.entries(headers).reduce>( + (normalized, [key, value]) => { + if (value === undefined) { + return normalized; + } + + normalized[key] = Array.isArray(value) ? value.join(", ") : value; + return normalized; + }, + {}, + ); +} + +function parseJson(rawBody: string): JsonValue | null { + if (!rawBody.trim()) { + return null; + } + + try { + return JSON.parse(rawBody) as JsonValue; + } catch { + return null; + } +} + +function parsePayloadBody(body: JsonValue | null): CapturedLogPayload | null { + if (!isRecord(body) || !Array.isArray(body.rows)) { + return null; + } + + return { + api_version: typeof body.api_version === "number" ? body.api_version : 0, + rows: body.rows.reduce((capturedRows, row) => { + if (isRecord(row)) { + capturedRows.push(clone(row)); + } + return capturedRows; + }, []), + }; +} + +function parsePayload(request: CapturedRequest): CapturedLogPayload | null { + return parsePayloadBody(request.jsonBody); +} + +function rowKey(row: CapturedLogRow): string { + return JSON.stringify( + [ + "org_id", + "project_id", + "experiment_id", + "dataset_id", + "prompt_session_id", + "log_id", + "id", + ].map((key) => row[key]), + ); +} + +function mergeValue(base: unknown, incoming: unknown): unknown { + if (isRecord(base) && isRecord(incoming)) { + const merged: Record = { ...base }; + for (const [key, value] of Object.entries(incoming)) { + merged[key] = key in merged ? mergeValue(merged[key], value) : value; + } + return merged; + } + + return incoming; +} + +function mergeRow( + existing: CapturedLogRow | undefined, + incoming: CapturedLogRow, +): CapturedLogRow { + if (!existing || !incoming._is_merge) { + return clone(incoming); + } + + const preserveNoMerge = !existing._is_merge; + const merged = mergeValue(existing, incoming) as CapturedLogRow; + if (preserveNoMerge) { + delete merged._is_merge; + } + return clone(merged); +} + +function recordField(value: unknown): Record | undefined { + return isRecord(value) ? clone(value) : undefined; +} + +function stringField(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined; +} + +function arrayOfStrings(value: unknown): string[] { + if (!Array.isArray(value)) { + return []; + } + + return value.filter((entry): entry is string => typeof entry === "string"); +} + +function toCapturedLogEvent( + apiVersion: number, + row: CapturedLogRow, + rawRow: CapturedLogRow, +): CapturedLogEvent { + const spanAttributes = recordField(row.span_attributes); + const metrics = recordField(row.metrics); + + return { + apiVersion, + context: recordField(row.context), + expected: clone(row.expected), + experimentId: stringField(row.experiment_id), + input: clone(row.input), + isMerge: rawRow._is_merge === true, + metadata: recordField(row.metadata), + metrics, + output: clone(row.output), + projectId: stringField(row.project_id), + row: clone(row), + scores: clone(row.scores), + span: { + ended: typeof metrics?.end === "number", + id: stringField(row.span_id), + name: stringField(spanAttributes?.name), + parentIds: arrayOfStrings(row.span_parents), + rootId: stringField(row.root_span_id), + started: typeof metrics?.start === "number", + type: stringField(spanAttributes?.type), + }, + }; +} + +function respondJson( + response: ServerResponse, + statusCode: number, + body: unknown, +): void { + response.writeHead(statusCode, { "Content-Type": "application/json" }); + response.end(JSON.stringify(body)); +} + +async function readRequestBody(req: IncomingMessage): Promise { + return await new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + req.on("error", reject); + }); +} + +function capturedRequestFrom( + method: string | undefined, + requestUrl: URL, + headers: IncomingHttpHeaders, + rawBody: string, +): CapturedRequest { + return { + method: method ?? "GET", + path: requestUrl.pathname, + query: Object.fromEntries(requestUrl.searchParams.entries()), + headers: normalizeHeaders(headers), + rawBody, + jsonBody: parseJson(rawBody), + }; +} + +export async function startMockBraintrustServer( + apiKey = DEFAULT_API_KEY, +): Promise { + const requests: CapturedRequest[] = []; + const payloads: CapturedLogPayload[] = []; + const events: CapturedLogEvent[] = []; + const mergedRows = new Map(); + const projectsByName = new Map(); + let serverUrl = ""; + let xactCursor = 0; + + function nextXactId(): string { + xactCursor += 1; + return String(xactCursor).padStart(12, "0"); + } + + function persistPayload(payload: CapturedLogPayload): void { + payloads.push(payload); + + for (const row of payload.rows) { + const persistedRow = clone(row); + if (typeof persistedRow._xact_id !== "string") { + persistedRow._xact_id = nextXactId(); + } + + const key = rowKey(persistedRow); + const mergedRow = mergeRow(mergedRows.get(key), persistedRow); + mergedRows.set(key, mergedRow); + events.push( + toCapturedLogEvent(payload.api_version, mergedRow, persistedRow), + ); + } + } + + function projectForName(name: string): { id: string; name: string } { + const existing = projectsByName.get(name); + if (existing) { + return existing; + } + + const created = { id: randomUUID(), name }; + projectsByName.set(name, created); + return created; + } + + const server = createServer((req, res) => { + void (async () => { + try { + const requestUrl = new URL( + req.url ?? "/", + serverUrl || "http://127.0.0.1", + ); + const rawBody = await readRequestBody(req); + const capturedRequest = capturedRequestFrom( + req.method, + requestUrl, + req.headers, + rawBody, + ); + + requests.push(capturedRequest); + + if ( + capturedRequest.method === "POST" && + capturedRequest.path === "/api/apikey/login" + ) { + respondJson(res, 200, { + org_info: [ + { + id: "mock-org-id", + name: "mock-org", + api_url: serverUrl, + proxy_url: serverUrl, + }, + ], + }); + return; + } + + if ( + capturedRequest.method === "POST" && + capturedRequest.path === "/api/project/register" + ) { + const projectName = + isRecord(capturedRequest.jsonBody) && + typeof capturedRequest.jsonBody.project_name === "string" + ? capturedRequest.jsonBody.project_name + : "project"; + + respondJson(res, 200, { + project: projectForName(projectName), + }); + return; + } + + if ( + capturedRequest.method === "GET" && + capturedRequest.path === "/version" + ) { + respondJson(res, 200, {}); + return; + } + + if ( + capturedRequest.method === "POST" && + capturedRequest.path === "/logs3" + ) { + const payload = parsePayload(capturedRequest); + if (payload) { + persistPayload(payload); + } + respondJson(res, 200, { ok: true }); + return; + } + + if ( + capturedRequest.method === "POST" && + capturedRequest.path === "/otel/v1/traces" + ) { + respondJson(res, 200, { ok: true }); + return; + } + + respondJson(res, 404, { + error: `Unhandled mock Braintrust route: ${capturedRequest.method} ${capturedRequest.path}`, + }); + } catch (error) { + respondJson(res, 500, { + error: error instanceof Error ? error.message : String(error), + }); + } + })(); + }); + + serverUrl = await new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const address = server.address() as AddressInfo; + resolve(`http://127.0.0.1:${address.port}`); + }); + }); + + return { + apiKey, + close: () => + new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }), + events, + payloads, + requests, + url: serverUrl, + }; +} diff --git a/e2e/helpers/normalize.ts b/e2e/helpers/normalize.ts new file mode 100644 index 000000000..4df975f57 --- /dev/null +++ b/e2e/helpers/normalize.ts @@ -0,0 +1,217 @@ +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; + +type Primitive = null | boolean | number | string; +export type Json = + | Primitive + | Json[] + | { + [key: string]: Json; + }; + +type TokenMaps = { + ids: Map; + runs: Map; + xacts: Map; +}; + +const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/; +const UUID_REGEX = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const UUID_SUBSTRING_REGEX = + /[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}/gi; +const TIME_KEYS = new Set(["created", "start", "end"]); +const SPAN_ID_KEYS = new Set(["id", "span_id", "root_span_id"]); +const ZERO_NUMBER_KEYS = new Set(["caller_lineno"]); +const XACT_VERSION_KEYS = new Set([ + "currentVersion", + "initialVersion", + "version", +]); +const HELPERS_DIR = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(HELPERS_DIR, "../..").replace(/\\/g, "/"); +const STACK_FRAME_REPO_PATH_REGEX = + /(?:[A-Za-z]:)?[^\s)\n]*braintrust-sdk-javascript(?:[\\/](?:braintrust-sdk-javascript|[^\\/\s)\n]+))?((?:[\\/](?:e2e|js)[^:\s)\n]+)):\d+:\d+/g; +const REPO_PATH_REGEX = + /(?:[A-Za-z]:)?[^\s)\n]*braintrust-sdk-javascript(?:[\\/](?:braintrust-sdk-javascript|[^\\/\s)\n]+))?((?:[\\/](?:e2e|js)[^:\s)\n]+))/g; +const NODE_INTERNAL_FRAME_REGEX = /node:[^)\n]+:\d+:\d+/g; + +function normalizeCallerFilename(value: string): string { + const e2eIndex = value.lastIndexOf("/e2e/"); + if (e2eIndex >= 0) { + return `${value.slice(e2eIndex)}`; + } + + return value; +} + +function normalizeMockServerUrl(value: string): string | undefined { + try { + const url = new URL(value); + if (url.protocol !== "http:" || url.hostname !== "127.0.0.1") { + return undefined; + } + + const suffix = `${url.pathname}${url.search}${url.hash}`; + return suffix === "/" ? "" : `${suffix}`; + } catch { + return undefined; + } +} + +function normalizeStackLikeString(value: string): string { + let normalized = value.replaceAll("file://", ""); + normalized = normalized.replaceAll(REPO_ROOT, ""); + + normalized = normalized.replace( + STACK_FRAME_REPO_PATH_REGEX, + (_, suffix: string) => `${suffix.replace(/\\/g, "/")}:0:0`, + ); + normalized = normalized.replace(REPO_PATH_REGEX, (_, suffix: string) => { + return `${suffix.replace(/\\/g, "/")}`; + }); + normalized = normalized.replace( + /((?:\/(?:e2e|js)\/[^:\s)\n]+)):\d+:\d+/g, + "$1:0:0", + ); + normalized = normalized.replace( + NODE_INTERNAL_FRAME_REGEX, + "node::0:0", + ); + + return normalized; +} + +function normalizeObject( + value: { [key: string]: Json }, + tokenMaps: TokenMaps, +): Json { + const callerFilename = + typeof value.caller_filename === "string" + ? value.caller_filename + : undefined; + const isNodeInternalCaller = callerFilename?.startsWith("node:"); + + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => { + if (isNodeInternalCaller) { + if (key === "caller_filename") { + return [key, ""]; + } + if (key === "caller_functionname") { + return [key, ""]; + } + if (key === "caller_lineno") { + return [key, 0]; + } + } + + return [key, normalizeValue(entry as Json, tokenMaps, key)]; + }), + ); +} + +function tokenFor( + map: Map, + rawValue: string, + prefix: string, +): string { + const existing = map.get(rawValue); + if (existing) { + return existing; + } + + const token = `<${prefix}:${map.size + 1}>`; + map.set(rawValue, token); + return token; +} + +function normalizeValue( + value: Json, + tokenMaps: TokenMaps, + currentKey?: string, +): Json { + if (Array.isArray(value)) { + if (currentKey === "span_parents") { + return value.map((entry) => + typeof entry === "string" + ? tokenFor(tokenMaps.ids, entry, "span") + : normalizeValue(entry, tokenMaps), + ); + } + + return value.map((entry) => normalizeValue(entry, tokenMaps)); + } + + if (value && typeof value === "object") { + return normalizeObject(value, tokenMaps); + } + + if (typeof value === "number") { + if (currentKey && ZERO_NUMBER_KEYS.has(currentKey)) { + return 0; + } + if (currentKey && TIME_KEYS.has(currentKey)) { + return 0; + } + return value; + } + + if (typeof value === "string") { + value = normalizeStackLikeString(value); + + const normalizedUrl = normalizeMockServerUrl(value); + if (normalizedUrl) { + return normalizedUrl; + } + + if (currentKey === "caller_filename") { + return normalizeCallerFilename(value); + } + + if (currentKey === "_xact_id") { + return tokenFor(tokenMaps.xacts, value, "xact"); + } + + if (currentKey && XACT_VERSION_KEYS.has(currentKey)) { + return tokenFor(tokenMaps.xacts, value, "xact"); + } + + if (currentKey === "testRunId") { + return tokenFor(tokenMaps.runs, value, "run"); + } + + if (currentKey && SPAN_ID_KEYS.has(currentKey)) { + return tokenFor(tokenMaps.ids, value, "span"); + } + + if (currentKey && TIME_KEYS.has(currentKey)) { + return ""; + } + + if (ISO_DATE_REGEX.test(value)) { + return ""; + } + + const withNormalizedUuids = value.replace(UUID_SUBSTRING_REGEX, (match) => + tokenFor(tokenMaps.ids, match, "uuid"), + ); + if (withNormalizedUuids !== value) { + return withNormalizedUuids; + } + + if (UUID_REGEX.test(value)) { + return tokenFor(tokenMaps.ids, value, "uuid"); + } + } + + return value; +} + +export function normalizeForSnapshot(value: Json): Json { + return normalizeValue(value, { + ids: new Map(), + runs: new Map(), + xacts: new Map(), + }); +} diff --git a/e2e/helpers/openai.ts b/e2e/helpers/openai.ts new file mode 100644 index 000000000..f41c21782 --- /dev/null +++ b/e2e/helpers/openai.ts @@ -0,0 +1,70 @@ +import type { CapturedLogEvent } from "./mock-braintrust-server"; +import type { Json } from "./normalize"; + +interface OpenAIScenario { + entry: string; + version: string; +} + +const OPENAI_VERSIONS = [ + { + suffix: "v4", + version: "4.104.0", + }, + { + suffix: "v5", + version: "5.11.0", + }, + { + suffix: "v6", + version: "6.25.0", + }, +] as const; + +export const OPENAI_SCENARIO_TIMEOUT_MS = 60_000; + +export const OPENAI_AUTO_HOOK_SCENARIOS: OpenAIScenario[] = OPENAI_VERSIONS.map( + ({ suffix, version }) => ({ + entry: `scenario.openai-${suffix}.mjs`, + version, + }), +); + +export const WRAP_OPENAI_SCENARIOS: OpenAIScenario[] = OPENAI_VERSIONS.map( + ({ suffix, version }) => ({ + entry: `scenario.openai-${suffix}.ts`, + version, + }), +); + +export function summarizeOpenAIContract(event: CapturedLogEvent): Json { + const metadata = event.row.metadata as + | { + metadata?: { operation?: string }; + model?: string; + openaiSdkVersion?: string; + provider?: string; + scenario?: string; + } + | undefined; + + return { + has_input: event.input !== undefined && event.input !== null, + has_output: event.output !== undefined && event.output !== null, + metadata: { + has_model: typeof metadata?.model === "string", + openaiSdkVersion: metadata?.openaiSdkVersion ?? null, + operation: metadata?.metadata?.operation ?? null, + provider: metadata?.provider ?? null, + scenario: metadata?.scenario ?? null, + }, + metric_keys: Object.keys(event.metrics ?? {}) + .filter((key) => key !== "start" && key !== "end") + .sort(), + name: event.span.name ?? null, + root_span_id: event.span.rootId ?? null, + span_id: event.span.id ?? null, + span_parents: event.span.parentIds, + type: event.span.type ?? null, + } satisfies Json; +} diff --git a/e2e/helpers/scenario-harness.ts b/e2e/helpers/scenario-harness.ts new file mode 100644 index 000000000..e4fadcb9c --- /dev/null +++ b/e2e/helpers/scenario-harness.ts @@ -0,0 +1,262 @@ +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { createRequire } from "node:module"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; +import { + startMockBraintrustServer, + type CapturedLogEvent, + type CapturedLogPayload, + type CapturedRequest, +} from "./mock-braintrust-server"; +import { + installScenarioDependencies, + type InstallScenarioDependenciesOptions, + type InstallScenarioDependenciesResult, +} from "./scenario-installer"; + +type EventPredicate = (event: CapturedLogEvent) => boolean; +type PayloadPredicate = (payload: CapturedLogPayload) => boolean; +type RequestPredicate = (request: CapturedRequest) => boolean; + +interface ScenarioResult { + exitCode: number; + stdout: string; + stderr: string; +} + +const tsxCliPath = createRequire(import.meta.url).resolve("tsx/cli"); +const DEFAULT_SCENARIO_TIMEOUT_MS = 15_000; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function hasTestRunId(value: unknown, testRunId: string): boolean { + if (Array.isArray(value)) { + return value.some((entry) => hasTestRunId(entry, testRunId)); + } + + if (!isRecord(value)) { + return false; + } + + if (value.testRunId === testRunId) { + return true; + } + + return Object.values(value).some((entry) => hasTestRunId(entry, testRunId)); +} + +function filterItems(items: T[], predicate?: (item: T) => boolean): T[] { + return predicate ? items.filter(predicate) : [...items]; +} + +function createTestRunId(): string { + return `e2e-${randomUUID()}`; +} + +function getTestServerEnv( + testRunId: string, + server: { apiKey: string; url: string }, +): Record { + return { + BRAINTRUST_API_KEY: server.apiKey, + BRAINTRUST_API_URL: server.url, + BRAINTRUST_APP_URL: server.url, + BRAINTRUST_APP_PUBLIC_URL: server.url, + BRAINTRUST_PROXY_URL: server.url, + BRAINTRUST_E2E_RUN_ID: testRunId, + }; +} + +async function runProcess( + args: string[], + cwd: string, + env: Record, + timeoutMs: number, +): Promise { + return await new Promise((resolve, reject) => { + const child = spawn(process.execPath, args, { + cwd, + env: { + ...process.env, + ...env, + }, + stdio: ["ignore", "pipe", "pipe"], + }); + const timeout = setTimeout(() => { + child.kill("SIGTERM"); + reject( + new Error(`Process ${args.join(" ")} timed out after ${timeoutMs}ms`), + ); + }, timeoutMs); + + let stdout = ""; + let stderr = ""; + + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + + child.on("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + child.on("close", (code) => { + clearTimeout(timeout); + resolve({ + exitCode: code ?? 0, + stdout, + stderr, + }); + }); + }); +} + +function resolveEntryPath(scenarioDir: string, entry: string): string { + return path.join(scenarioDir, entry); +} + +async function runScenarioDirOrThrow( + scenarioDir: string, + env: Record, + options: { + entry: string; + nodeArgs?: string[]; + timeoutMs?: number; + useTsx?: boolean; + } = { + entry: "scenario.ts", + }, +): Promise { + const scenarioPath = resolveEntryPath(scenarioDir, options.entry); + const args = + options.useTsx === false + ? [...(options.nodeArgs ?? []), scenarioPath] + : [tsxCliPath, scenarioPath]; + const result = await runProcess( + args, + scenarioDir, + env, + options.timeoutMs ?? DEFAULT_SCENARIO_TIMEOUT_MS, + ); + + if (result.exitCode !== 0) { + throw new Error( + `Scenario ${path.join(scenarioDir, options.entry)} failed with exit code ${result.exitCode}\nSTDOUT:\n${result.stdout}\nSTDERR:\n${result.stderr}`, + ); + } + + return result; +} + +export function resolveScenarioDir(importMetaUrl: string): string { + return path.dirname(fileURLToPath(importMetaUrl)); +} + +export async function runScenarioDir(options: { + env?: Record; + entry?: string; + scenarioDir: string; + timeoutMs?: number; +}): Promise { + return await runScenarioDirOrThrow(options.scenarioDir, options.env ?? {}, { + entry: options.entry ?? "scenario.ts", + timeoutMs: options.timeoutMs, + }); +} + +export async function runNodeScenarioDir(options: { + env?: Record; + entry?: string; + nodeArgs?: string[]; + scenarioDir: string; + timeoutMs?: number; +}): Promise { + return await runScenarioDirOrThrow(options.scenarioDir, options.env ?? {}, { + entry: options.entry ?? "scenario.mjs", + nodeArgs: options.nodeArgs, + timeoutMs: options.timeoutMs, + useTsx: false, + }); +} + +interface ScenarioHarness { + events: (predicate?: EventPredicate) => CapturedLogEvent[]; + payloads: (predicate?: PayloadPredicate) => CapturedLogPayload[]; + requestCursor: () => number; + requestsAfter: ( + after: number, + predicate?: RequestPredicate, + ) => CapturedRequest[]; + runNodeScenarioDir: (options: { + entry?: string; + env?: Record; + nodeArgs?: string[]; + scenarioDir: string; + timeoutMs?: number; + }) => Promise; + runScenarioDir: (options: { + entry?: string; + env?: Record; + scenarioDir: string; + timeoutMs?: number; + }) => Promise; + testRunEvents: (predicate?: EventPredicate) => CapturedLogEvent[]; + testRunId: string; +} + +export async function withScenarioHarness( + body: (harness: ScenarioHarness) => Promise, +): Promise { + const server = await startMockBraintrustServer(); + const testRunId = createTestRunId(); + const testEnv = getTestServerEnv(testRunId, server); + + try { + await body({ + events: (predicate) => filterItems(server.events, predicate), + payloads: (predicate) => filterItems(server.payloads, predicate), + requestCursor: () => server.requests.length, + requestsAfter: (after, predicate) => + filterItems(server.requests.slice(after), predicate), + runNodeScenarioDir: (options) => + runNodeScenarioDir({ + ...options, + env: { + ...testEnv, + ...(options.env ?? {}), + }, + }), + runScenarioDir: (options) => + runScenarioDir({ + ...options, + env: { + ...testEnv, + ...(options.env ?? {}), + }, + }), + testRunEvents: (predicate) => + filterItems( + server.events, + (event) => + hasTestRunId(event.row, testRunId) && + (predicate ? predicate(event) : true), + ), + testRunId, + }); + } finally { + await server.close(); + } +} + +export { + installScenarioDependencies, + type InstallScenarioDependenciesResult, + type InstallScenarioDependenciesOptions, +}; diff --git a/e2e/helpers/scenario-installer.ts b/e2e/helpers/scenario-installer.ts new file mode 100644 index 000000000..09b965ead --- /dev/null +++ b/e2e/helpers/scenario-installer.ts @@ -0,0 +1,123 @@ +import { promises as fs } from "node:fs"; +import { spawn } from "node:child_process"; +import * as path from "node:path"; + +export type InstallScenarioDependenciesResult = + | { status: "no-manifest" } + | { status: "installed" }; + +export interface InstallScenarioDependenciesOptions { + preferOffline?: boolean; + scenarioDir: string; +} + +const PNPM_COMMAND = process.platform === "win32" ? "pnpm.cmd" : "pnpm"; + +async function fileExists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +async function spawnOrThrow( + command: string, + args: string[], + cwd: string, +): Promise { + return await new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd, + env: process.env, + stdio: ["ignore", "pipe", "pipe"], + }); + + let stdout = ""; + let stderr = ""; + + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("error", reject); + child.on("close", (code) => { + if (code === 0) { + resolve(stdout.trim()); + return; + } + + reject( + new Error( + `${command} ${args.join(" ")} failed with exit code ${code ?? 0}\nSTDOUT:\n${stdout}\nSTDERR:\n${stderr}`, + ), + ); + }); + }); +} + +function findWorkspaceSpecs( + manifest: Record, +): Array<{ name: string; section: string; spec: string }> { + const dependencySections = [ + "dependencies", + "devDependencies", + "optionalDependencies", + "peerDependencies", + ] as const; + + return dependencySections.flatMap((section) => { + const value = manifest[section]; + if (!value || typeof value !== "object" || Array.isArray(value)) { + return []; + } + + return Object.entries(value).flatMap(([name, spec]) => { + if (typeof spec === "string" && spec.startsWith("workspace:")) { + return [{ name, section, spec }]; + } + return []; + }); + }); +} + +export async function installScenarioDependencies({ + preferOffline = true, + scenarioDir, +}: InstallScenarioDependenciesOptions): Promise { + const manifestPath = path.join(scenarioDir, "package.json"); + if (!(await fileExists(manifestPath))) { + return { status: "no-manifest" }; + } + + const manifestRaw = await fs.readFile(manifestPath, "utf8"); + const manifest = JSON.parse(manifestRaw) as Record; + const workspaceSpecs = findWorkspaceSpecs(manifest); + if (workspaceSpecs.length > 0) { + const details = workspaceSpecs + .map(({ name, section, spec }) => `${section}.${name} -> ${spec}`) + .join(", "); + throw new Error( + `Scenario package.json in ${scenarioDir} cannot use workspace: dependencies (${details}). Keep workspace packages in e2e/package.json or use a non-workspace spec.`, + ); + } + + const installArgs = [ + "install", + "--dir", + scenarioDir, + "--ignore-workspace", + "--no-lockfile", + "--no-frozen-lockfile", + "--strict-peer-dependencies=false", + ]; + if (preferOffline) { + installArgs.push("--prefer-offline"); + } + + await spawnOrThrow(PNPM_COMMAND, installArgs, scenarioDir); + return { status: "installed" }; +} diff --git a/e2e/helpers/scenario-runtime.ts b/e2e/helpers/scenario-runtime.ts new file mode 100644 index 000000000..c3cf246a6 --- /dev/null +++ b/e2e/helpers/scenario-runtime.ts @@ -0,0 +1,44 @@ +import { BasicTracerProvider } from "@opentelemetry/sdk-trace-base"; + +export async function collectAsync(records: AsyncIterable): Promise { + const items: T[] = []; + for await (const record of records) { + items.push(record); + } + return items; +} + +export function getTestRunId(): string { + return process.env.BRAINTRUST_E2E_RUN_ID!; +} + +export function scopedName(base: string, testRunId = getTestRunId()): string { + const suffix = testRunId.toLowerCase().replace(/[^a-z0-9-]/g, "-"); + return `${base}-${suffix}`; +} + +export function createTracerProvider(processors: unknown[]) { + const testProvider = new BasicTracerProvider(); + + if ( + typeof (testProvider as { addSpanProcessor?: unknown }).addSpanProcessor === + "function" + ) { + const provider = new BasicTracerProvider() as BasicTracerProvider & { + addSpanProcessor: (processor: unknown) => void; + }; + processors.forEach((processor) => provider.addSpanProcessor(processor)); + return provider; + } + + return new BasicTracerProvider({ + spanProcessors: processors as never, + }); +} + +export function runMain(main: () => Promise): void { + void main().catch((error) => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/e2e/helpers/trace-selectors.ts b/e2e/helpers/trace-selectors.ts new file mode 100644 index 000000000..c4b303ed3 --- /dev/null +++ b/e2e/helpers/trace-selectors.ts @@ -0,0 +1,40 @@ +import type { CapturedLogEvent } from "./mock-braintrust-server"; + +function findLatestEvent( + events: CapturedLogEvent[], + predicate: (event: CapturedLogEvent) => boolean, +): CapturedLogEvent | undefined { + return [...events].reverse().find(predicate); +} + +export function findLatestSpan( + events: CapturedLogEvent[], + name: string, +): CapturedLogEvent | undefined { + return findLatestEvent(events, (event) => event.span.name === name); +} + +export function findLatestChildSpan( + events: CapturedLogEvent[], + name: string, + parentId: string | undefined, +): CapturedLogEvent | undefined { + if (!parentId) { + return undefined; + } + + return ( + findLatestEvent( + events, + (event) => + event.span.name === name && + event.span.parentIds.includes(parentId) && + event.output !== undefined, + ) ?? + findLatestEvent( + events, + (event) => + event.span.name === name && event.span.parentIds.includes(parentId), + ) + ); +} diff --git a/e2e/helpers/trace-summary.ts b/e2e/helpers/trace-summary.ts new file mode 100644 index 000000000..09906921c --- /dev/null +++ b/e2e/helpers/trace-summary.ts @@ -0,0 +1,148 @@ +import type { + CapturedLogEvent, + CapturedRequest, + JsonValue, +} from "./mock-braintrust-server"; +import type { Json } from "./normalize"; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function summarizeEvent(event: CapturedLogEvent): Json { + const row = event.row as Record; + const error = + typeof row.error === "string" + ? row.error.split("\n\n")[0] + : row.error == null + ? null + : String(row.error); + + return { + error, + input: (row.input ?? null) as Json, + metadata: (row.metadata ?? null) as Json, + name: event.span.name ?? null, + output: (row.output ?? null) as Json, + span_attributes: (row.span_attributes ?? null) as Json, + span_id: (row.span_id ?? null) as Json, + span_parents: (row.span_parents ?? null) as Json, + root_span_id: (row.root_span_id ?? null) as Json, + }; +} + +export function summarizeRequest( + request: CapturedRequest, + options: { + includeHeaders?: string[]; + normalizeJsonRawBody?: boolean; + } = {}, +): Json { + const headers = + options.includeHeaders && options.includeHeaders.length > 0 + ? Object.fromEntries( + options.includeHeaders.flatMap((key) => { + const value = request.headers[key]; + return value === undefined ? [] : [[key, value]]; + }), + ) + : null; + + return { + headers: + headers && Object.keys(headers).length > 0 ? (headers as Json) : null, + jsonBody: (request.jsonBody ?? null) as Json, + method: request.method, + path: request.path, + query: + Object.keys(request.query).length === 0 ? null : (request.query as Json), + rawBody: + options.normalizeJsonRawBody && request.jsonBody + ? (request.jsonBody as Json) + : request.rawBody || null, + }; +} + +function otlpAttributeValue(value: unknown): Json { + if (!isRecord(value)) { + return null; + } + + if (typeof value.stringValue === "string") { + return value.stringValue; + } + if (typeof value.boolValue === "boolean") { + return value.boolValue; + } + if (typeof value.intValue === "string") { + return value.intValue; + } + if (typeof value.doubleValue === "number") { + return value.doubleValue; + } + const arrayValues = + isRecord(value.arrayValue) && Array.isArray(value.arrayValue.values) + ? value.arrayValue.values + : undefined; + if (arrayValues) { + return arrayValues.map((entry: unknown) => otlpAttributeValue(entry)); + } + + return null; +} + +type OtlpSpanSummary = { + attributes: Record; + name: string; + parentSpanId?: string; + spanId?: string; + traceId?: string; +}; + +export function extractOtelSpans(body: JsonValue | null): OtlpSpanSummary[] { + if (!isRecord(body) || !Array.isArray(body.resourceSpans)) { + return []; + } + + const spans: OtlpSpanSummary[] = []; + for (const resourceSpan of body.resourceSpans) { + if (!isRecord(resourceSpan) || !Array.isArray(resourceSpan.scopeSpans)) { + continue; + } + + for (const scopeSpan of resourceSpan.scopeSpans) { + if (!isRecord(scopeSpan) || !Array.isArray(scopeSpan.spans)) { + continue; + } + + for (const span of scopeSpan.spans) { + if (!isRecord(span) || typeof span.name !== "string") { + continue; + } + + const attributes: Record = {}; + if (Array.isArray(span.attributes)) { + for (const attribute of span.attributes) { + if (!isRecord(attribute) || typeof attribute.key !== "string") { + continue; + } + attributes[attribute.key] = otlpAttributeValue(attribute.value); + } + } + + spans.push({ + attributes, + name: span.name, + parentSpanId: + typeof span.parentSpanId === "string" + ? span.parentSpanId + : undefined, + spanId: typeof span.spanId === "string" ? span.spanId : undefined, + traceId: typeof span.traceId === "string" ? span.traceId : undefined, + }); + } + } + } + + return spans; +} diff --git a/e2e/package.json b/e2e/package.json new file mode 100644 index 000000000..5cd56b69d --- /dev/null +++ b/e2e/package.json @@ -0,0 +1,21 @@ +{ + "name": "@braintrust/js-e2e-tests", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "test:e2e": "vitest run", + "test:e2e:update": "vitest run --update" + }, + "devDependencies": { + "@braintrust/otel": "workspace:^", + "@opentelemetry/api": ">=1.9.0", + "@opentelemetry/context-async-hooks": ">=1.9.0", + "@opentelemetry/sdk-trace-base": ">=1.9.0", + "@types/node": "^20.10.5", + "braintrust": "workspace:^", + "tsx": "^3.14.0", + "typescript": "5.4.4", + "vitest": "^2.1.9" + } +} diff --git a/e2e/scenarios/openai-auto-instrumentation-node-hook/__snapshots__/scenario.test.ts.snap b/e2e/scenarios/openai-auto-instrumentation-node-hook/__snapshots__/scenario.test.ts.snap new file mode 100644 index 000000000..d7011e783 --- /dev/null +++ b/e2e/scenarios/openai-auto-instrumentation-node-hook/__snapshots__/scenario.test.ts.snap @@ -0,0 +1,157 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`openai auto-instrumentation via node hook collects traces without manual wrapping (openai 4.104.0) > span-events 1`] = ` +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "has_model": false, + "openaiSdkVersion": "4.104.0", + "operation": null, + "provider": null, + "scenario": "openai-auto-instrumentation-node-hook", + }, + "metric_keys": [], + "name": "openai-auto-hook-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task", + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "has_model": true, + "openaiSdkVersion": null, + "operation": null, + "provider": "openai", + "scenario": null, + }, + "metric_keys": [ + "completion_accepted_prediction_tokens", + "completion_audio_tokens", + "completion_reasoning_tokens", + "completion_rejected_prediction_tokens", + "completion_tokens", + "prompt_audio_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens", + ], + "name": "Chat Completion", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": "llm", + }, +] +`; + +exports[`openai auto-instrumentation via node hook collects traces without manual wrapping (openai 5.11.0) > span-events 1`] = ` +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "has_model": false, + "openaiSdkVersion": "5.11.0", + "operation": null, + "provider": null, + "scenario": "openai-auto-instrumentation-node-hook", + }, + "metric_keys": [], + "name": "openai-auto-hook-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task", + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "has_model": true, + "openaiSdkVersion": null, + "operation": null, + "provider": "openai", + "scenario": null, + }, + "metric_keys": [ + "completion_accepted_prediction_tokens", + "completion_audio_tokens", + "completion_reasoning_tokens", + "completion_rejected_prediction_tokens", + "completion_tokens", + "prompt_audio_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens", + ], + "name": "Chat Completion", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": "llm", + }, +] +`; + +exports[`openai auto-instrumentation via node hook collects traces without manual wrapping (openai 6.25.0) > span-events 1`] = ` +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "has_model": false, + "openaiSdkVersion": "6.25.0", + "operation": null, + "provider": null, + "scenario": "openai-auto-instrumentation-node-hook", + }, + "metric_keys": [], + "name": "openai-auto-hook-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task", + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "has_model": true, + "openaiSdkVersion": null, + "operation": null, + "provider": "openai", + "scenario": null, + }, + "metric_keys": [ + "completion_accepted_prediction_tokens", + "completion_audio_tokens", + "completion_reasoning_tokens", + "completion_rejected_prediction_tokens", + "completion_tokens", + "prompt_audio_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens", + ], + "name": "Chat Completion", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": "llm", + }, +] +`; diff --git a/e2e/scenarios/openai-auto-instrumentation-node-hook/package.json b/e2e/scenarios/openai-auto-instrumentation-node-hook/package.json new file mode 100644 index 000000000..08871ee43 --- /dev/null +++ b/e2e/scenarios/openai-auto-instrumentation-node-hook/package.json @@ -0,0 +1,9 @@ +{ + "name": "@braintrust/e2e-openai-auto-instrumentation-node-hook", + "private": true, + "dependencies": { + "openai": "6.25.0", + "openai-v4": "npm:openai@4.104.0", + "openai-v5": "npm:openai@5.11.0" + } +} diff --git a/e2e/scenarios/openai-auto-instrumentation-node-hook/scenario.impl.mjs b/e2e/scenarios/openai-auto-instrumentation-node-hook/scenario.impl.mjs new file mode 100644 index 000000000..8fba754eb --- /dev/null +++ b/e2e/scenarios/openai-auto-instrumentation-node-hook/scenario.impl.mjs @@ -0,0 +1,66 @@ +import { initLogger } from "braintrust"; + +const OPENAI_MODEL = "gpt-4o-mini"; + +function getTestRunId() { + return process.env.BRAINTRUST_E2E_RUN_ID; +} + +function scopedName(base, testRunId = getTestRunId()) { + const suffix = testRunId.toLowerCase().replace(/[^a-z0-9-]/g, "-"); + return `${base}-${suffix}`; +} + +export async function runOpenAIAutoInstrumentationNodeHook( + OpenAI, + openaiSdkVersion, +) { + const testRunId = getTestRunId(); + const logger = initLogger({ + projectName: scopedName("e2e-openai-auto-instrumentation-hook", testRunId), + }); + const client = new OpenAI({ + apiKey: process.env.OPENAI_API_KEY, + baseURL: process.env.OPENAI_BASE_URL, + }); + + await logger.traced( + async () => { + await client.chat.completions.create({ + model: OPENAI_MODEL, + messages: [ + { + role: "user", + content: "Auto-instrument this request.", + }, + ], + max_tokens: 8, + temperature: 0, + }); + }, + { + name: "openai-auto-hook-root", + event: { + metadata: { + scenario: "openai-auto-instrumentation-node-hook", + openaiSdkVersion, + testRunId, + }, + }, + }, + ); + + await logger.flush(); +} + +export function runOpenAIAutoInstrumentationNodeHookOrExit( + OpenAI, + openaiSdkVersion, +) { + void runOpenAIAutoInstrumentationNodeHook(OpenAI, openaiSdkVersion).catch( + (error) => { + console.error(error); + process.exitCode = 1; + }, + ); +} diff --git a/e2e/scenarios/openai-auto-instrumentation-node-hook/scenario.openai-v4.mjs b/e2e/scenarios/openai-auto-instrumentation-node-hook/scenario.openai-v4.mjs new file mode 100644 index 000000000..18be93ee3 --- /dev/null +++ b/e2e/scenarios/openai-auto-instrumentation-node-hook/scenario.openai-v4.mjs @@ -0,0 +1,4 @@ +import OpenAI from "openai-v4"; +import { runOpenAIAutoInstrumentationNodeHookOrExit } from "./scenario.impl.mjs"; + +runOpenAIAutoInstrumentationNodeHookOrExit(OpenAI, "4.104.0"); diff --git a/e2e/scenarios/openai-auto-instrumentation-node-hook/scenario.openai-v5.mjs b/e2e/scenarios/openai-auto-instrumentation-node-hook/scenario.openai-v5.mjs new file mode 100644 index 000000000..2df8abe18 --- /dev/null +++ b/e2e/scenarios/openai-auto-instrumentation-node-hook/scenario.openai-v5.mjs @@ -0,0 +1,4 @@ +import OpenAI from "openai-v5"; +import { runOpenAIAutoInstrumentationNodeHookOrExit } from "./scenario.impl.mjs"; + +runOpenAIAutoInstrumentationNodeHookOrExit(OpenAI, "5.11.0"); diff --git a/e2e/scenarios/openai-auto-instrumentation-node-hook/scenario.openai-v6.mjs b/e2e/scenarios/openai-auto-instrumentation-node-hook/scenario.openai-v6.mjs new file mode 100644 index 000000000..617498e34 --- /dev/null +++ b/e2e/scenarios/openai-auto-instrumentation-node-hook/scenario.openai-v6.mjs @@ -0,0 +1,4 @@ +import OpenAI from "openai"; +import { runOpenAIAutoInstrumentationNodeHookOrExit } from "./scenario.impl.mjs"; + +runOpenAIAutoInstrumentationNodeHookOrExit(OpenAI, "6.25.0"); diff --git a/e2e/scenarios/openai-auto-instrumentation-node-hook/scenario.test.ts b/e2e/scenarios/openai-auto-instrumentation-node-hook/scenario.test.ts new file mode 100644 index 000000000..4a763d803 --- /dev/null +++ b/e2e/scenarios/openai-auto-instrumentation-node-hook/scenario.test.ts @@ -0,0 +1,63 @@ +import { beforeAll, expect, test } from "vitest"; +import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import { + OPENAI_AUTO_HOOK_SCENARIOS, + OPENAI_SCENARIO_TIMEOUT_MS, + summarizeOpenAIContract, +} from "../../helpers/openai"; +import { + installScenarioDependencies, + resolveScenarioDir, + withScenarioHarness, +} from "../../helpers/scenario-harness"; +import { + findLatestChildSpan, + findLatestSpan, +} from "../../helpers/trace-selectors"; + +const scenarioDir = resolveScenarioDir(import.meta.url); + +beforeAll(async () => { + await installScenarioDependencies({ scenarioDir }); +}); + +for (const scenario of OPENAI_AUTO_HOOK_SCENARIOS) { + test(`openai auto-instrumentation via node hook collects traces without manual wrapping (openai ${scenario.version})`, async () => { + await withScenarioHarness(async ({ events, runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: scenario.entry, + nodeArgs: ["--import", "braintrust/hook.mjs"], + scenarioDir, + timeoutMs: OPENAI_SCENARIO_TIMEOUT_MS, + }); + + const capturedEvents = events(); + const root = findLatestSpan(capturedEvents, "openai-auto-hook-root"); + const chatCompletion = + findLatestChildSpan(capturedEvents, "Chat Completion", root?.span.id) ?? + findLatestSpan(capturedEvents, "Chat Completion"); + + expect(root).toBeDefined(); + expect(root?.row.metadata).toMatchObject({ + openaiSdkVersion: scenario.version, + }); + expect(chatCompletion).toBeDefined(); + expect(chatCompletion?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(chatCompletion?.row.metadata).toMatchObject({ + provider: "openai", + }); + expect( + typeof (chatCompletion?.row.metadata as { model?: unknown } | undefined) + ?.model, + ).toBe("string"); + + expect( + normalizeForSnapshot( + [root, chatCompletion].map((event) => + summarizeOpenAIContract(event!), + ) as Json, + ), + ).toMatchSnapshot("span-events"); + }); + }); +} diff --git a/e2e/scenarios/otel-compat-mixed-tracing/__snapshots__/scenario.test.ts.snap b/e2e/scenarios/otel-compat-mixed-tracing/__snapshots__/scenario.test.ts.snap new file mode 100644 index 000000000..8ee531240 --- /dev/null +++ b/e2e/scenarios/otel-compat-mixed-tracing/__snapshots__/scenario.test.ts.snap @@ -0,0 +1,46 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`otel-compat-mixed-tracing unifies Braintrust and OTEL spans into one trace > braintrust-span-events 1`] = ` +[ + { + "error": null, + "input": null, + "metadata": { + "scenario": "otel-compat-mixed-tracing", + "testRunId": "", + }, + "name": "bt-root", + "output": null, + "root_span_id": "", + "span_attributes": { + "exec_counter": 0, + "name": "bt-root", + "type": "task", + }, + "span_id": "", + "span_parents": null, + }, + { + "error": null, + "input": null, + "metadata": { + "kind": "bt-child-under-otel", + "testRunId": "", + }, + "name": "bt-child-under-otel", + "output": { + "source": "otel-child-context", + }, + "root_span_id": "", + "span_attributes": { + "exec_counter": 1, + "name": "bt-child-under-otel", + "type": "task", + }, + "span_id": "", + "span_parents": [ + "", + ], + }, +] +`; diff --git a/e2e/scenarios/otel-compat-mixed-tracing/scenario.test.ts b/e2e/scenarios/otel-compat-mixed-tracing/scenario.test.ts new file mode 100644 index 000000000..2084bb66f --- /dev/null +++ b/e2e/scenarios/otel-compat-mixed-tracing/scenario.test.ts @@ -0,0 +1,46 @@ +import { expect, test } from "vitest"; +import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import { + resolveScenarioDir, + withScenarioHarness, +} from "../../helpers/scenario-harness"; +import { findLatestSpan } from "../../helpers/trace-selectors"; +import { extractOtelSpans, summarizeEvent } from "../../helpers/trace-summary"; + +const scenarioDir = resolveScenarioDir(import.meta.url); + +test("otel-compat-mixed-tracing unifies Braintrust and OTEL spans into one trace", async () => { + await withScenarioHarness( + async ({ requestsAfter, runScenarioDir, testRunEvents }) => { + await runScenarioDir({ scenarioDir }); + + const btEvents = testRunEvents(); + const btRoot = findLatestSpan(btEvents, "bt-root"); + const btChild = findLatestSpan(btEvents, "bt-child-under-otel"); + + expect(btRoot).toBeDefined(); + expect(btChild).toBeDefined(); + + const otelRequests = requestsAfter( + 0, + (request) => request.path === "/otel/v1/traces", + ); + expect(otelRequests.length).toBeGreaterThanOrEqual(1); + + const otelSpans = extractOtelSpans(otelRequests[0].jsonBody); + const otelMiddle = otelSpans.find((span) => span.name === "otel-middle"); + + expect(otelMiddle).toBeDefined(); + expect(otelMiddle?.traceId).toBe(btRoot?.span.rootId); + expect(otelMiddle?.parentSpanId).toBe(btRoot?.span.id); + expect(btChild?.span.rootId).toBe(btRoot?.span.rootId); + expect(btChild?.span.parentIds).toContain(otelMiddle?.spanId ?? ""); + + expect( + normalizeForSnapshot( + [btRoot, btChild].map((event) => summarizeEvent(event!)) as Json, + ), + ).toMatchSnapshot("braintrust-span-events"); + }, + ); +}); diff --git a/e2e/scenarios/otel-compat-mixed-tracing/scenario.ts b/e2e/scenarios/otel-compat-mixed-tracing/scenario.ts new file mode 100644 index 000000000..f3a2d4bb1 --- /dev/null +++ b/e2e/scenarios/otel-compat-mixed-tracing/scenario.ts @@ -0,0 +1,75 @@ +import { context as otelContext, trace } from "@opentelemetry/api"; +import { AsyncHooksContextManager } from "@opentelemetry/context-async-hooks"; +import { BraintrustSpanProcessor, setupOtelCompat } from "@braintrust/otel"; +import { getContextManager, initLogger } from "braintrust"; +import { + createTracerProvider, + getTestRunId, + runMain, + scopedName, +} from "../../helpers/scenario-runtime"; + +async function main() { + const testRunId = getTestRunId(); + setupOtelCompat(); + + const contextManager = new AsyncHooksContextManager(); + contextManager.enable(); + otelContext.setGlobalContextManager(contextManager); + + try { + const processor = new BraintrustSpanProcessor({ + apiKey: process.env.BRAINTRUST_API_KEY!, + apiUrl: process.env.BRAINTRUST_API_URL!, + parent: `project_name:${scopedName("e2e-otel-compat-mixed-tracing", testRunId)}`, + }); + const provider = createTracerProvider([processor]); + trace.setGlobalTracerProvider(provider); + + const tracer = trace.getTracer("e2e-otel-compat"); + const logger = initLogger({ + projectName: scopedName("e2e-otel-compat-mixed-tracing", testRunId), + }); + const btRoot = logger.startSpan({ + name: "bt-root", + event: { + metadata: { + scenario: "otel-compat-mixed-tracing", + testRunId, + }, + }, + }); + const contextManagerFacade = getContextManager(); + + await contextManagerFacade.runInContext(btRoot, async () => { + await tracer.startActiveSpan("otel-middle", async (otelSpan) => { + const btChild = logger.startSpan({ + name: "bt-child-under-otel", + event: { + metadata: { + kind: "bt-child-under-otel", + testRunId, + }, + }, + }); + btChild.log({ + output: { + source: "otel-child-context", + }, + }); + btChild.end(); + otelSpan.end(); + }); + }); + btRoot.end(); + + await logger.flush(); + await processor.forceFlush(); + await (provider as { shutdown?: () => Promise }).shutdown?.(); + } finally { + otelContext.disable(); + contextManager.disable(); + } +} + +runMain(main); diff --git a/e2e/scenarios/otel-span-processor-export/scenario.test.ts b/e2e/scenarios/otel-span-processor-export/scenario.test.ts new file mode 100644 index 000000000..cc1d38d24 --- /dev/null +++ b/e2e/scenarios/otel-span-processor-export/scenario.test.ts @@ -0,0 +1,42 @@ +import { expect, test } from "vitest"; +import { + resolveScenarioDir, + withScenarioHarness, +} from "../../helpers/scenario-harness"; +import { + extractOtelSpans, + summarizeRequest, +} from "../../helpers/trace-summary"; + +const scenarioDir = resolveScenarioDir(import.meta.url); + +test("otel-span-processor-export sends filtered OTLP traces to Braintrust", async () => { + await withScenarioHarness( + async ({ requestsAfter, runScenarioDir, testRunId }) => { + await runScenarioDir({ scenarioDir }); + + const requests = requestsAfter( + 0, + (request) => request.path === "/otel/v1/traces", + ); + expect(requests).toHaveLength(1); + + const request = requests[0]; + const spans = extractOtelSpans(request.jsonBody); + + expect(request.headers["x-bt-parent"]).toContain(testRunId.toLowerCase()); + expect(spans.map((span) => span.name)).toContain("gen_ai.completion"); + expect(spans.map((span) => span.name)).not.toContain("root-operation"); + expect(spans[0]?.attributes["gen_ai.system"]).toBe("openai"); + + expect( + summarizeRequest(request, { + includeHeaders: ["content-type", "x-bt-parent"], + }), + ).toMatchObject({ + method: "POST", + path: "/otel/v1/traces", + }); + }, + ); +}); diff --git a/e2e/scenarios/otel-span-processor-export/scenario.ts b/e2e/scenarios/otel-span-processor-export/scenario.ts new file mode 100644 index 000000000..c7d972ee7 --- /dev/null +++ b/e2e/scenarios/otel-span-processor-export/scenario.ts @@ -0,0 +1,33 @@ +import { context, trace } from "@opentelemetry/api"; +import { BraintrustSpanProcessor } from "@braintrust/otel"; +import { + createTracerProvider, + getTestRunId, + runMain, + scopedName, +} from "../../helpers/scenario-runtime"; + +async function main() { + const testRunId = getTestRunId(); + const processor = new BraintrustSpanProcessor({ + apiKey: process.env.BRAINTRUST_API_KEY!, + apiUrl: process.env.BRAINTRUST_API_URL!, + filterAISpans: true, + parent: `project_name:${scopedName("e2e-otel-span-processor-export", testRunId)}`, + }); + const provider = createTracerProvider([processor]); + trace.setGlobalTracerProvider(provider); + + const tracer = trace.getTracer("e2e-otel-export"); + const rootSpan = tracer.startSpan("root-operation"); + const rootContext = trace.setSpan(context.active(), rootSpan); + const aiSpan = tracer.startSpan("gen_ai.completion", undefined, rootContext); + aiSpan.setAttribute("gen_ai.system", "openai"); + aiSpan.end(); + rootSpan.end(); + + await processor.forceFlush(); + await (provider as { shutdown?: () => Promise }).shutdown?.(); +} + +runMain(main); diff --git a/e2e/scenarios/trace-context-and-continuation/__snapshots__/scenario.test.ts.snap b/e2e/scenarios/trace-context-and-continuation/__snapshots__/scenario.test.ts.snap new file mode 100644 index 000000000..022911646 --- /dev/null +++ b/e2e/scenarios/trace-context-and-continuation/__snapshots__/scenario.test.ts.snap @@ -0,0 +1,134 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`trace-context-and-continuation supports reattachment and late span updates > late-update-payloads 1`] = ` +[ + { + "context": { + "caller_filename": "/e2e/scenarios/trace-context-and-continuation/scenario.ts", + "caller_functionname": "main", + "caller_lineno": 0, + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "kind": "late-update", + "testRunId": "", + }, + "metrics": { + "end": 0, + "start": 0, + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 3, + "name": "late-update", + "type": "task", + }, + "span_id": "", + }, + { + "_is_merge": true, + "id": "", + "log_id": "g", + "metadata": { + "kind": "late-update", + "patched": true, + "testRunId": "", + }, + "output": { + "state": "updated", + }, + "project_id": "", + "root_span_id": "", + "span_id": "", + }, +] +`; + +exports[`trace-context-and-continuation supports reattachment and late span updates > span-events 1`] = ` +[ + { + "error": null, + "input": null, + "metadata": { + "scenario": "trace-context-and-continuation", + "testRunId": "", + }, + "name": "context-root", + "output": null, + "root_span_id": "", + "span_attributes": { + "exec_counter": 0, + "name": "context-root", + "type": "task", + }, + "span_id": "", + "span_parents": null, + }, + { + "error": null, + "input": null, + "metadata": { + "kind": "current-child", + "testRunId": "", + }, + "name": "current-child", + "output": { + "source": "withCurrent", + }, + "root_span_id": "", + "span_attributes": { + "exec_counter": 1, + "name": "current-child", + }, + "span_id": "", + "span_parents": [ + "", + ], + }, + { + "error": null, + "input": null, + "metadata": { + "kind": "reattached-child", + "testRunId": "", + }, + "name": "reattached-child", + "output": { + "resumed": true, + }, + "root_span_id": "", + "span_attributes": { + "exec_counter": 2, + "name": "reattached-child", + }, + "span_id": "", + "span_parents": [ + "", + ], + }, + { + "error": null, + "input": null, + "metadata": { + "kind": "late-update", + "patched": true, + "testRunId": "", + }, + "name": "late-update", + "output": { + "state": "updated", + }, + "root_span_id": "", + "span_attributes": { + "exec_counter": 3, + "name": "late-update", + "type": "task", + }, + "span_id": "", + "span_parents": null, + }, +] +`; diff --git a/e2e/scenarios/trace-context-and-continuation/scenario.test.ts b/e2e/scenarios/trace-context-and-continuation/scenario.test.ts new file mode 100644 index 000000000..12a3b2bfb --- /dev/null +++ b/e2e/scenarios/trace-context-and-continuation/scenario.test.ts @@ -0,0 +1,75 @@ +import { expect, test } from "vitest"; +import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import { + resolveScenarioDir, + withScenarioHarness, +} from "../../helpers/scenario-harness"; +import { findLatestSpan } from "../../helpers/trace-selectors"; +import { summarizeEvent } from "../../helpers/trace-summary"; + +const scenarioDir = resolveScenarioDir(import.meta.url); + +test("trace-context-and-continuation supports reattachment and late span updates", async () => { + await withScenarioHarness( + async ({ payloads, runScenarioDir, testRunEvents, testRunId }) => { + await runScenarioDir({ scenarioDir }); + + const capturedEvents = testRunEvents(); + const root = findLatestSpan(capturedEvents, "context-root"); + const currentChild = findLatestSpan(capturedEvents, "current-child"); + const reattachedChild = findLatestSpan( + capturedEvents, + "reattached-child", + ); + const lateUpdate = findLatestSpan(capturedEvents, "late-update"); + + expect(root).toBeDefined(); + expect(currentChild).toBeDefined(); + expect(reattachedChild).toBeDefined(); + expect(lateUpdate).toBeDefined(); + + expect(currentChild?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(reattachedChild?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(reattachedChild?.span.rootId).toBe(root?.span.rootId); + expect(lateUpdate?.row.metadata).toMatchObject({ + patched: true, + testRunId, + }); + expect(lateUpdate?.row.output).toEqual({ + state: "updated", + }); + + expect( + normalizeForSnapshot( + [ + "context-root", + "current-child", + "reattached-child", + "late-update", + ].map((name) => + summarizeEvent(findLatestSpan(capturedEvents, name)!), + ) as Json, + ), + ).toMatchSnapshot("span-events"); + + const mutationRows = payloads() + .flatMap((payload) => payload.rows) + .filter((row) => { + const metadata = + row.metadata && typeof row.metadata === "object" + ? row.metadata + : null; + return ( + metadata !== null && + "testRunId" in metadata && + (metadata as Record).testRunId === testRunId && + row.id === lateUpdate?.row.id + ); + }); + + expect(normalizeForSnapshot(mutationRows as Json)).toMatchSnapshot( + "late-update-payloads", + ); + }, + ); +}); diff --git a/e2e/scenarios/trace-context-and-continuation/scenario.ts b/e2e/scenarios/trace-context-and-continuation/scenario.ts new file mode 100644 index 000000000..c0bb88ec3 --- /dev/null +++ b/e2e/scenarios/trace-context-and-continuation/scenario.ts @@ -0,0 +1,103 @@ +import { + flush, + initLogger, + startSpan, + traced, + updateSpan, + withCurrent, + withParent, +} from "braintrust"; +import { + getTestRunId, + runMain, + scopedName, +} from "../../helpers/scenario-runtime"; + +async function main() { + const testRunId = getTestRunId(); + const logger = initLogger({ + projectName: scopedName("e2e-trace-context-and-continuation", testRunId), + }); + + const rootSpan = logger.startSpan({ + name: "context-root", + event: { + metadata: { + scenario: "trace-context-and-continuation", + testRunId, + }, + }, + }); + const exportedRoot = await rootSpan.export(); + + await withCurrent(rootSpan, async () => { + const currentChild = startSpan({ + name: "current-child", + event: { + metadata: { + kind: "current-child", + testRunId, + }, + }, + }); + currentChild.log({ + output: { + source: "withCurrent", + }, + }); + currentChild.end(); + }); + + rootSpan.end(); + + await withParent(exportedRoot, async () => { + await traced( + (span) => { + span.log({ + output: { + resumed: true, + }, + }); + }, + { + name: "reattached-child", + event: { + metadata: { + kind: "reattached-child", + testRunId, + }, + }, + }, + ); + }); + + const updatableSpan = logger.startSpan({ + name: "late-update", + event: { + metadata: { + kind: "late-update", + testRunId, + }, + }, + }); + const exportedUpdatableSpan = await updatableSpan.export(); + updatableSpan.end(); + + await logger.flush(); + + updateSpan({ + exported: exportedUpdatableSpan, + metadata: { + kind: "late-update", + patched: true, + testRunId, + }, + output: { + state: "updated", + }, + }); + + await flush(); +} + +runMain(main); diff --git a/e2e/scenarios/trace-primitives-basic/__snapshots__/scenario.test.ts.snap b/e2e/scenarios/trace-primitives-basic/__snapshots__/scenario.test.ts.snap new file mode 100644 index 000000000..d62c9d559 --- /dev/null +++ b/e2e/scenarios/trace-primitives-basic/__snapshots__/scenario.test.ts.snap @@ -0,0 +1,379 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`trace-primitives-basic collects a minimal manual trace tree > request-flow 1`] = ` +[ + { + "headers": null, + "jsonBody": null, + "method": "POST", + "path": "/api/apikey/login", + "query": null, + "rawBody": null, + }, + { + "headers": null, + "jsonBody": { + "org_id": "mock-org-id", + "project_name": "e2e-trace-primitives-basic-e2e-", + }, + "method": "POST", + "path": "/api/project/register", + "query": null, + "rawBody": { + "org_id": "mock-org-id", + "project_name": "e2e-trace-primitives-basic-e2e-", + }, + }, + { + "headers": null, + "jsonBody": null, + "method": "GET", + "path": "/version", + "query": null, + "rawBody": null, + }, + { + "headers": null, + "jsonBody": { + "api_version": 2, + "rows": [ + { + "_is_merge": false, + "context": { + "caller_filename": "/e2e/scenarios/trace-primitives-basic/scenario.ts", + "caller_functionname": "main", + "caller_lineno": 0, + }, + "created": "", + "id": "", + "input": { + "scenario": "trace-primitives-basic", + "testRunId": "", + }, + "log_id": "g", + "metadata": { + "scenario": "trace-primitives-basic", + "testRunId": "", + }, + "metrics": { + "start": 0, + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 0, + "name": "trace-primitives-root", + "type": "task", + }, + "span_id": "", + }, + ], + }, + "method": "POST", + "path": "/logs3", + "query": null, + "rawBody": { + "api_version": 2, + "rows": [ + { + "_is_merge": false, + "context": { + "caller_filename": "/e2e/scenarios/trace-primitives-basic/scenario.ts", + "caller_functionname": "main", + "caller_lineno": 0, + }, + "created": "", + "id": "", + "input": { + "scenario": "trace-primitives-basic", + "testRunId": "", + }, + "log_id": "g", + "metadata": { + "scenario": "trace-primitives-basic", + "testRunId": "", + }, + "metrics": { + "start": 0, + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 0, + "name": "trace-primitives-root", + "type": "task", + }, + "span_id": "", + }, + ], + }, + }, + { + "headers": null, + "jsonBody": { + "api_version": 2, + "rows": [ + { + "context": { + "caller_filename": "/e2e/scenarios/trace-primitives-basic/scenario.ts", + "caller_functionname": "logger.traced.name", + "caller_lineno": 0, + }, + "created": "", + "id": "", + "input": { + "step": "child", + "testRunId": "", + }, + "log_id": "g", + "metadata": { + "kind": "basic-child", + "testRunId": "", + }, + "metrics": { + "end": 0, + "start": 0, + }, + "output": { + "ok": true, + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 1, + "name": "basic-child", + }, + "span_id": "", + "span_parents": [ + "", + ], + }, + { + "context": { + "caller_filename": "/e2e/scenarios/trace-primitives-basic/scenario.ts", + "caller_functionname": "logger.traced.name", + "caller_lineno": 0, + }, + "created": "", + "error": "basic boom + +Error: basic boom + at logger.traced.name (/e2e/scenarios/trace-primitives-basic/scenario.ts:0:0) + at /js/dist/index.mjs:0:0 + at AsyncLocalStorage.run (node::0:0) + at BraintrustContextManager.runInContext (/js/dist/index.mjs:0:0) + at withCurrent (/js/dist/index.mjs:0:0) + at /js/dist/index.mjs:0:0 + at runCatchFinally (/js/dist/index.mjs:0:0) + at Logger.traced (/js/dist/index.mjs:0:0) + at main (/e2e/scenarios/trace-primitives-basic/scenario.ts:0:0) + at runMain (/e2e/helpers/scenario-runtime.ts:0:0)", + "id": "", + "log_id": "g", + "metadata": { + "kind": "basic-error", + "testRunId": "", + }, + "metrics": { + "end": 0, + "start": 0, + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 2, + "name": "basic-error", + }, + "span_id": "", + "span_parents": [ + "", + ], + }, + { + "_is_merge": true, + "id": "", + "log_id": "g", + "metrics": { + "end": 0, + }, + "output": { + "status": "ok", + }, + "project_id": "", + "root_span_id": "", + "span_id": "", + }, + ], + }, + "method": "POST", + "path": "/logs3", + "query": null, + "rawBody": { + "api_version": 2, + "rows": [ + { + "context": { + "caller_filename": "/e2e/scenarios/trace-primitives-basic/scenario.ts", + "caller_functionname": "logger.traced.name", + "caller_lineno": 0, + }, + "created": "", + "id": "", + "input": { + "step": "child", + "testRunId": "", + }, + "log_id": "g", + "metadata": { + "kind": "basic-child", + "testRunId": "", + }, + "metrics": { + "end": 0, + "start": 0, + }, + "output": { + "ok": true, + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 1, + "name": "basic-child", + }, + "span_id": "", + "span_parents": [ + "", + ], + }, + { + "context": { + "caller_filename": "/e2e/scenarios/trace-primitives-basic/scenario.ts", + "caller_functionname": "logger.traced.name", + "caller_lineno": 0, + }, + "created": "", + "error": "basic boom + +Error: basic boom + at logger.traced.name (/e2e/scenarios/trace-primitives-basic/scenario.ts:0:0) + at /js/dist/index.mjs:0:0 + at AsyncLocalStorage.run (node::0:0) + at BraintrustContextManager.runInContext (/js/dist/index.mjs:0:0) + at withCurrent (/js/dist/index.mjs:0:0) + at /js/dist/index.mjs:0:0 + at runCatchFinally (/js/dist/index.mjs:0:0) + at Logger.traced (/js/dist/index.mjs:0:0) + at main (/e2e/scenarios/trace-primitives-basic/scenario.ts:0:0) + at runMain (/e2e/helpers/scenario-runtime.ts:0:0)", + "id": "", + "log_id": "g", + "metadata": { + "kind": "basic-error", + "testRunId": "", + }, + "metrics": { + "end": 0, + "start": 0, + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 2, + "name": "basic-error", + }, + "span_id": "", + "span_parents": [ + "", + ], + }, + { + "_is_merge": true, + "id": "", + "log_id": "g", + "metrics": { + "end": 0, + }, + "output": { + "status": "ok", + }, + "project_id": "", + "root_span_id": "", + "span_id": "", + }, + ], + }, + }, +] +`; + +exports[`trace-primitives-basic collects a minimal manual trace tree > span-events 1`] = ` +[ + { + "error": null, + "input": { + "scenario": "trace-primitives-basic", + "testRunId": "", + }, + "metadata": { + "scenario": "trace-primitives-basic", + "testRunId": "", + }, + "name": "trace-primitives-root", + "output": { + "status": "ok", + }, + "root_span_id": "", + "span_attributes": { + "exec_counter": 0, + "name": "trace-primitives-root", + "type": "task", + }, + "span_id": "", + "span_parents": null, + }, + { + "error": null, + "input": { + "step": "child", + "testRunId": "", + }, + "metadata": { + "kind": "basic-child", + "testRunId": "", + }, + "name": "basic-child", + "output": { + "ok": true, + }, + "root_span_id": "", + "span_attributes": { + "exec_counter": 1, + "name": "basic-child", + }, + "span_id": "", + "span_parents": [ + "", + ], + }, + { + "error": "basic boom", + "input": null, + "metadata": { + "kind": "basic-error", + "testRunId": "", + }, + "name": "basic-error", + "output": null, + "root_span_id": "", + "span_attributes": { + "exec_counter": 2, + "name": "basic-error", + }, + "span_id": "", + "span_parents": [ + "", + ], + }, +] +`; diff --git a/e2e/scenarios/trace-primitives-basic/scenario.test.ts b/e2e/scenarios/trace-primitives-basic/scenario.test.ts new file mode 100644 index 000000000..ac20a6c2c --- /dev/null +++ b/e2e/scenarios/trace-primitives-basic/scenario.test.ts @@ -0,0 +1,60 @@ +import { expect, test } from "vitest"; +import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import { + resolveScenarioDir, + withScenarioHarness, +} from "../../helpers/scenario-harness"; +import { findLatestSpan } from "../../helpers/trace-selectors"; +import { summarizeEvent, summarizeRequest } from "../../helpers/trace-summary"; + +const scenarioDir = resolveScenarioDir(import.meta.url); + +test("trace-primitives-basic collects a minimal manual trace tree", async () => { + await withScenarioHarness( + async ({ requestCursor, requestsAfter, runScenarioDir, testRunEvents }) => { + const cursor = requestCursor(); + + await runScenarioDir({ scenarioDir }); + + const capturedEvents = testRunEvents(); + const root = findLatestSpan(capturedEvents, "trace-primitives-root"); + const child = findLatestSpan(capturedEvents, "basic-child"); + const error = findLatestSpan(capturedEvents, "basic-error"); + + expect(root).toBeDefined(); + expect(child).toBeDefined(); + expect(error).toBeDefined(); + + expect(child?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(error?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(root?.span.rootId).toBe(root?.span.id); + + expect( + normalizeForSnapshot( + ["trace-primitives-root", "basic-child", "basic-error"].map((name) => + summarizeEvent(findLatestSpan(capturedEvents, name)!), + ) as Json, + ), + ).toMatchSnapshot("span-events"); + + const requests = requestsAfter( + cursor, + (request) => + request.path === "/api/apikey/login" || + request.path === "/api/project/register" || + request.path === "/version" || + request.path === "/logs3", + ); + + expect( + normalizeForSnapshot( + requests.map((request) => + summarizeRequest(request, { + normalizeJsonRawBody: true, + }), + ) as Json, + ), + ).toMatchSnapshot("request-flow"); + }, + ); +}); diff --git a/e2e/scenarios/trace-primitives-basic/scenario.ts b/e2e/scenarios/trace-primitives-basic/scenario.ts new file mode 100644 index 000000000..442ce6518 --- /dev/null +++ b/e2e/scenarios/trace-primitives-basic/scenario.ts @@ -0,0 +1,72 @@ +import { initLogger, logError, startSpan } from "braintrust"; +import { + getTestRunId, + runMain, + scopedName, +} from "../../helpers/scenario-runtime"; + +async function main() { + const testRunId = getTestRunId(); + const logger = initLogger({ + projectName: scopedName("e2e-trace-primitives-basic", testRunId), + }); + + await logger.traced( + async (rootSpan) => { + const childSpan = startSpan({ + name: "basic-child", + event: { + input: { + step: "child", + testRunId, + }, + metadata: { + kind: "basic-child", + testRunId, + }, + }, + }); + childSpan.log({ + output: { + ok: true, + }, + }); + childSpan.end(); + + const errorSpan = startSpan({ + name: "basic-error", + event: { + metadata: { + kind: "basic-error", + testRunId, + }, + }, + }); + logError(errorSpan, new Error("basic boom")); + errorSpan.end(); + + rootSpan.log({ + output: { + status: "ok", + }, + }); + }, + { + name: "trace-primitives-root", + event: { + input: { + scenario: "trace-primitives-basic", + testRunId, + }, + metadata: { + scenario: "trace-primitives-basic", + testRunId, + }, + }, + }, + ); + + await logger.flush(); +} + +runMain(main); diff --git a/e2e/scenarios/wrap-openai-conversation-traces/__snapshots__/scenario.test.ts.snap b/e2e/scenarios/wrap-openai-conversation-traces/__snapshots__/scenario.test.ts.snap new file mode 100644 index 000000000..77dec2548 --- /dev/null +++ b/e2e/scenarios/wrap-openai-conversation-traces/__snapshots__/scenario.test.ts.snap @@ -0,0 +1,496 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`wrap-openai-conversation-traces logs wrapped chat and responses traces (openai 4.104.0) > span-events 1`] = ` +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "has_model": false, + "openaiSdkVersion": "4.104.0", + "operation": null, + "provider": null, + "scenario": "wrap-openai-conversation-traces", + }, + "metric_keys": [], + "name": "openai-wrapper-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task", + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "has_model": false, + "openaiSdkVersion": null, + "operation": null, + "provider": null, + "scenario": null, + }, + "metric_keys": [], + "name": "openai-chat-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": null, + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "has_model": true, + "openaiSdkVersion": null, + "operation": null, + "provider": "openai", + "scenario": null, + }, + "metric_keys": [ + "completion_accepted_prediction_tokens", + "completion_audio_tokens", + "completion_reasoning_tokens", + "completion_rejected_prediction_tokens", + "completion_tokens", + "prompt_audio_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens", + ], + "name": "Chat Completion", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": "llm", + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "has_model": false, + "openaiSdkVersion": null, + "operation": null, + "provider": null, + "scenario": null, + }, + "metric_keys": [], + "name": "openai-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": null, + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "has_model": true, + "openaiSdkVersion": null, + "operation": null, + "provider": "openai", + "scenario": null, + }, + "metric_keys": [ + "completion_accepted_prediction_tokens", + "completion_audio_tokens", + "completion_reasoning_tokens", + "completion_rejected_prediction_tokens", + "completion_tokens", + "prompt_audio_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens", + ], + "name": "Chat Completion", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": "llm", + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "has_model": false, + "openaiSdkVersion": null, + "operation": null, + "provider": null, + "scenario": null, + }, + "metric_keys": [], + "name": "openai-responses-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": null, + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "has_model": true, + "openaiSdkVersion": null, + "operation": null, + "provider": "openai", + "scenario": null, + }, + "metric_keys": [ + "completion_reasoning_tokens", + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens", + ], + "name": "openai.responses.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": "llm", + }, +] +`; + +exports[`wrap-openai-conversation-traces logs wrapped chat and responses traces (openai 5.11.0) > span-events 1`] = ` +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "has_model": false, + "openaiSdkVersion": "5.11.0", + "operation": null, + "provider": null, + "scenario": "wrap-openai-conversation-traces", + }, + "metric_keys": [], + "name": "openai-wrapper-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task", + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "has_model": false, + "openaiSdkVersion": null, + "operation": null, + "provider": null, + "scenario": null, + }, + "metric_keys": [], + "name": "openai-chat-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": null, + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "has_model": true, + "openaiSdkVersion": null, + "operation": null, + "provider": "openai", + "scenario": null, + }, + "metric_keys": [ + "completion_accepted_prediction_tokens", + "completion_audio_tokens", + "completion_reasoning_tokens", + "completion_rejected_prediction_tokens", + "completion_tokens", + "prompt_audio_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens", + ], + "name": "Chat Completion", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": "llm", + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "has_model": false, + "openaiSdkVersion": null, + "operation": null, + "provider": null, + "scenario": null, + }, + "metric_keys": [], + "name": "openai-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": null, + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "has_model": true, + "openaiSdkVersion": null, + "operation": null, + "provider": "openai", + "scenario": null, + }, + "metric_keys": [ + "completion_accepted_prediction_tokens", + "completion_audio_tokens", + "completion_reasoning_tokens", + "completion_rejected_prediction_tokens", + "completion_tokens", + "prompt_audio_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens", + ], + "name": "Chat Completion", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": "llm", + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "has_model": false, + "openaiSdkVersion": null, + "operation": null, + "provider": null, + "scenario": null, + }, + "metric_keys": [], + "name": "openai-responses-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": null, + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "has_model": true, + "openaiSdkVersion": null, + "operation": null, + "provider": "openai", + "scenario": null, + }, + "metric_keys": [ + "completion_reasoning_tokens", + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens", + ], + "name": "openai.responses.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": "llm", + }, +] +`; + +exports[`wrap-openai-conversation-traces logs wrapped chat and responses traces (openai 6.25.0) > span-events 1`] = ` +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "has_model": false, + "openaiSdkVersion": "6.25.0", + "operation": null, + "provider": null, + "scenario": "wrap-openai-conversation-traces", + }, + "metric_keys": [], + "name": "openai-wrapper-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task", + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "has_model": false, + "openaiSdkVersion": null, + "operation": null, + "provider": null, + "scenario": null, + }, + "metric_keys": [], + "name": "openai-chat-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": null, + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "has_model": true, + "openaiSdkVersion": null, + "operation": null, + "provider": "openai", + "scenario": null, + }, + "metric_keys": [ + "completion_accepted_prediction_tokens", + "completion_audio_tokens", + "completion_reasoning_tokens", + "completion_rejected_prediction_tokens", + "completion_tokens", + "prompt_audio_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens", + ], + "name": "Chat Completion", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": "llm", + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "has_model": false, + "openaiSdkVersion": null, + "operation": null, + "provider": null, + "scenario": null, + }, + "metric_keys": [], + "name": "openai-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": null, + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "has_model": true, + "openaiSdkVersion": null, + "operation": null, + "provider": "openai", + "scenario": null, + }, + "metric_keys": [ + "completion_accepted_prediction_tokens", + "completion_audio_tokens", + "completion_reasoning_tokens", + "completion_rejected_prediction_tokens", + "completion_tokens", + "prompt_audio_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens", + ], + "name": "Chat Completion", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": "llm", + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "has_model": false, + "openaiSdkVersion": null, + "operation": null, + "provider": null, + "scenario": null, + }, + "metric_keys": [], + "name": "openai-responses-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": null, + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "has_model": true, + "openaiSdkVersion": null, + "operation": null, + "provider": "openai", + "scenario": null, + }, + "metric_keys": [ + "completion_reasoning_tokens", + "completion_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens", + ], + "name": "openai.responses.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "", + ], + "type": "llm", + }, +] +`; diff --git a/e2e/scenarios/wrap-openai-conversation-traces/package.json b/e2e/scenarios/wrap-openai-conversation-traces/package.json new file mode 100644 index 000000000..122bbbca6 --- /dev/null +++ b/e2e/scenarios/wrap-openai-conversation-traces/package.json @@ -0,0 +1,9 @@ +{ + "name": "@braintrust/e2e-wrap-openai-conversation-traces", + "private": true, + "dependencies": { + "openai": "6.25.0", + "openai-v4": "npm:openai@4.104.0", + "openai-v5": "npm:openai@5.11.0" + } +} diff --git a/e2e/scenarios/wrap-openai-conversation-traces/scenario.impl.ts b/e2e/scenarios/wrap-openai-conversation-traces/scenario.impl.ts new file mode 100644 index 000000000..4f380daf2 --- /dev/null +++ b/e2e/scenarios/wrap-openai-conversation-traces/scenario.impl.ts @@ -0,0 +1,111 @@ +import { initLogger, startSpan, withCurrent, wrapOpenAI } from "braintrust"; +import { + collectAsync, + getTestRunId, + scopedName, +} from "../../helpers/scenario-runtime"; + +const OPENAI_MODEL = "gpt-4o-mini"; + +export async function runWrapOpenAIConversationTraces( + OpenAI: any, + openaiSdkVersion: string, +) { + const testRunId = getTestRunId(); + const logger = initLogger({ + projectName: scopedName("e2e-wrap-openai-conversation", testRunId), + }); + const client = wrapOpenAI( + new OpenAI({ + apiKey: process.env.OPENAI_API_KEY!, + baseURL: process.env.OPENAI_BASE_URL, + }), + ); + + await logger.traced( + async () => { + const chatSpan = startSpan({ + name: "openai-chat-operation", + event: { + metadata: { + operation: "chat", + testRunId, + }, + }, + }); + await withCurrent(chatSpan, async () => { + await client.chat.completions.create({ + model: OPENAI_MODEL, + messages: [ + { + role: "user", + content: "Reply with exactly OK.", + }, + ], + max_tokens: 8, + temperature: 0, + }); + }); + chatSpan.end(); + + const streamSpan = startSpan({ + name: "openai-stream-operation", + event: { + metadata: { + operation: "stream", + testRunId, + }, + }, + }); + await withCurrent(streamSpan, async () => { + const chatStream = await client.chat.completions.create({ + model: OPENAI_MODEL, + messages: [ + { + role: "user", + content: "Reply with exactly STREAM.", + }, + ], + stream: true, + max_tokens: 8, + temperature: 0, + stream_options: { + include_usage: true, + }, + }); + await collectAsync(chatStream); + }); + streamSpan.end(); + + const responsesSpan = startSpan({ + name: "openai-responses-operation", + event: { + metadata: { + operation: "responses", + testRunId, + }, + }, + }); + await withCurrent(responsesSpan, async () => { + await client.responses.create({ + model: OPENAI_MODEL, + input: "Reply with exactly PARIS.", + max_output_tokens: 16, + }); + }); + responsesSpan.end(); + }, + { + name: "openai-wrapper-root", + event: { + metadata: { + scenario: "wrap-openai-conversation-traces", + openaiSdkVersion, + testRunId, + }, + }, + }, + ); + + await logger.flush(); +} diff --git a/e2e/scenarios/wrap-openai-conversation-traces/scenario.openai-v4.ts b/e2e/scenarios/wrap-openai-conversation-traces/scenario.openai-v4.ts new file mode 100644 index 000000000..6ac3405ca --- /dev/null +++ b/e2e/scenarios/wrap-openai-conversation-traces/scenario.openai-v4.ts @@ -0,0 +1,5 @@ +import OpenAI from "openai-v4"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrapOpenAIConversationTraces } from "./scenario.impl"; + +runMain(() => runWrapOpenAIConversationTraces(OpenAI, "4.104.0")); diff --git a/e2e/scenarios/wrap-openai-conversation-traces/scenario.openai-v5.ts b/e2e/scenarios/wrap-openai-conversation-traces/scenario.openai-v5.ts new file mode 100644 index 000000000..ac4ee2541 --- /dev/null +++ b/e2e/scenarios/wrap-openai-conversation-traces/scenario.openai-v5.ts @@ -0,0 +1,5 @@ +import OpenAI from "openai-v5"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrapOpenAIConversationTraces } from "./scenario.impl"; + +runMain(() => runWrapOpenAIConversationTraces(OpenAI, "5.11.0")); diff --git a/e2e/scenarios/wrap-openai-conversation-traces/scenario.openai-v6.ts b/e2e/scenarios/wrap-openai-conversation-traces/scenario.openai-v6.ts new file mode 100644 index 000000000..58701e96a --- /dev/null +++ b/e2e/scenarios/wrap-openai-conversation-traces/scenario.openai-v6.ts @@ -0,0 +1,5 @@ +import OpenAI from "openai"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrapOpenAIConversationTraces } from "./scenario.impl"; + +runMain(() => runWrapOpenAIConversationTraces(OpenAI, "6.25.0")); diff --git a/e2e/scenarios/wrap-openai-conversation-traces/scenario.test.ts b/e2e/scenarios/wrap-openai-conversation-traces/scenario.test.ts new file mode 100644 index 000000000..a63eb3572 --- /dev/null +++ b/e2e/scenarios/wrap-openai-conversation-traces/scenario.test.ts @@ -0,0 +1,143 @@ +import { beforeAll, expect, test } from "vitest"; +import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import { + OPENAI_SCENARIO_TIMEOUT_MS, + WRAP_OPENAI_SCENARIOS, + summarizeOpenAIContract, +} from "../../helpers/openai"; +import { + installScenarioDependencies, + resolveScenarioDir, + withScenarioHarness, +} from "../../helpers/scenario-harness"; +import { + findLatestChildSpan, + findLatestSpan, +} from "../../helpers/trace-selectors"; + +const scenarioDir = resolveScenarioDir(import.meta.url); + +beforeAll(async () => { + await installScenarioDependencies({ scenarioDir }); +}); + +test.each( + WRAP_OPENAI_SCENARIOS.map(({ entry, version }) => [version, entry] as const), +)( + "wrap-openai-conversation-traces logs wrapped chat and responses traces (openai %s)", + async (version, entry) => { + await withScenarioHarness(async ({ events, runScenarioDir }) => { + await runScenarioDir({ + entry, + scenarioDir, + timeoutMs: OPENAI_SCENARIO_TIMEOUT_MS, + }); + + const capturedEvents = events(); + + const root = findLatestSpan(capturedEvents, "openai-wrapper-root"); + const chatOperation = findLatestSpan( + capturedEvents, + "openai-chat-operation", + ); + const streamOperation = findLatestSpan( + capturedEvents, + "openai-stream-operation", + ); + const responsesOperation = findLatestSpan( + capturedEvents, + "openai-responses-operation", + ); + const chatCompletionSpan = findLatestChildSpan( + capturedEvents, + "Chat Completion", + chatOperation?.span.id, + ); + const streamCompletionSpan = findLatestChildSpan( + capturedEvents, + "Chat Completion", + streamOperation?.span.id, + ); + const responsesSpan = findLatestChildSpan( + capturedEvents, + "openai.responses.create", + responsesOperation?.span.id, + ); + + expect(root).toBeDefined(); + expect(chatOperation).toBeDefined(); + expect(streamOperation).toBeDefined(); + expect(responsesOperation).toBeDefined(); + expect(chatCompletionSpan).toBeDefined(); + expect(streamCompletionSpan).toBeDefined(); + expect(responsesSpan).toBeDefined(); + expect(root?.row.metadata).toMatchObject({ + openaiSdkVersion: version, + }); + expect(chatOperation?.row.metadata).toMatchObject({ + operation: "chat", + }); + expect(streamOperation?.row.metadata).toMatchObject({ + operation: "stream", + }); + expect(responsesOperation?.row.metadata).toMatchObject({ + operation: "responses", + }); + expect(chatCompletionSpan?.row.metadata).toMatchObject({ + provider: "openai", + }); + expect( + typeof ( + chatCompletionSpan?.row.metadata as { model?: unknown } | undefined + )?.model, + ).toBe("string"); + expect(streamCompletionSpan?.row.metadata).toMatchObject({ + provider: "openai", + }); + expect( + typeof ( + streamCompletionSpan?.row.metadata as { model?: unknown } | undefined + )?.model, + ).toBe("string"); + expect(responsesSpan?.row.metadata).toMatchObject({ + provider: "openai", + }); + expect( + typeof (responsesSpan?.row.metadata as { model?: unknown } | undefined) + ?.model, + ).toBe("string"); + + expect(chatOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(streamOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(responsesOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(chatCompletionSpan?.span.parentIds).toEqual([ + chatOperation?.span.id ?? "", + ]); + expect(streamCompletionSpan?.span.parentIds).toEqual([ + streamOperation?.span.id ?? "", + ]); + expect(responsesSpan?.span.parentIds).toEqual([ + responsesOperation?.span.id ?? "", + ]); + expect(chatCompletionSpan?.input).toBeDefined(); + expect(chatCompletionSpan?.output).toBeDefined(); + expect(streamCompletionSpan?.output).toBeDefined(); + expect(streamCompletionSpan?.metrics).toBeDefined(); + expect(responsesSpan?.output).toBeDefined(); + + expect( + normalizeForSnapshot( + [ + root, + chatOperation, + chatCompletionSpan, + streamOperation, + streamCompletionSpan, + responsesOperation, + responsesSpan, + ].map((event) => summarizeOpenAIContract(event!)) as Json, + ), + ).toMatchSnapshot("span-events"); + }); + }, +); diff --git a/e2e/tsconfig.json b/e2e/tsconfig.json new file mode 100644 index 000000000..fefc39848 --- /dev/null +++ b/e2e/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "lib": ["es2022"], + "module": "es2022", + "target": "es2022", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "types": ["node", "vitest/globals"] + }, + "include": ["."], + "exclude": ["node_modules/**", "**/dist/**"] +} diff --git a/e2e/vitest.config.mts b/e2e/vitest.config.mts new file mode 100644 index 000000000..e41a1d0ec --- /dev/null +++ b/e2e/vitest.config.mts @@ -0,0 +1,9 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + hookTimeout: 20_000, + include: ["scenarios/**/*.test.ts"], + testTimeout: 20_000, + }, +}); diff --git a/js/src/auto-instrumentations/configs/openai.test.ts b/js/src/auto-instrumentations/configs/openai.test.ts index f43fdbdf1..824f9608d 100644 --- a/js/src/auto-instrumentations/configs/openai.test.ts +++ b/js/src/auto-instrumentations/configs/openai.test.ts @@ -2,6 +2,10 @@ import { describe, it, expect } from "vitest"; import { openaiConfigs } from "./openai"; describe("OpenAI Instrumentation Configs", () => { + function configsForChannel(channelName: string) { + return openaiConfigs.filter((config) => config.channelName === channelName); + } + it("should have valid configs", () => { expect(openaiConfigs).toBeDefined(); expect(Array.isArray(openaiConfigs)).toBe(true); @@ -9,17 +13,49 @@ describe("OpenAI Instrumentation Configs", () => { }); it("should have chat.completions.create config", () => { - const config = openaiConfigs.find( - (c) => c.channelName === "chat.completions.create", + const configs = configsForChannel("chat.completions.create"); + + expect(configs).toHaveLength(3); + expect(configs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + module: expect.objectContaining({ + name: "openai", + versionRange: ">=4.0.0 <5.0.0", + filePath: "resources/chat/completions.mjs", + }), + functionQuery: expect.objectContaining({ + className: "Completions", + methodName: "create", + kind: "Async", + }), + }), + expect.objectContaining({ + module: expect.objectContaining({ + name: "openai", + versionRange: ">=4.0.0 <5.0.0", + filePath: "resources/chat/completions/completions.mjs", + }), + functionQuery: expect.objectContaining({ + className: "Completions", + methodName: "create", + kind: "Async", + }), + }), + expect.objectContaining({ + module: expect.objectContaining({ + name: "openai", + versionRange: ">=5.0.0", + filePath: "resources/chat/completions/completions.mjs", + }), + functionQuery: expect.objectContaining({ + className: "Completions", + methodName: "create", + kind: "Async", + }), + }), + ]), ); - - expect(config).toBeDefined(); - expect(config?.module.name).toBe("openai"); - expect(config?.module.versionRange).toBe(">=4.0.0"); - expect(config?.module.filePath).toBe("resources/chat/completions.mjs"); - expect((config?.functionQuery as any).className).toBe("Completions"); - expect((config?.functionQuery as any).methodName).toBe("create"); - expect((config?.functionQuery as any).kind).toBe("Async"); }); it("should have embeddings.create config", () => { @@ -47,16 +83,37 @@ describe("OpenAI Instrumentation Configs", () => { }); it("should have beta.chat.completions.parse config", () => { - const config = openaiConfigs.find( - (c) => c.channelName === "beta.chat.completions.parse", + const configs = configsForChannel("beta.chat.completions.parse"); + + expect(configs).toHaveLength(2); + expect(configs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + module: expect.objectContaining({ + name: "openai", + versionRange: ">=4.0.0 <5.0.0", + filePath: "resources/beta/chat/completions.mjs", + }), + functionQuery: expect.objectContaining({ + className: "Completions", + methodName: "parse", + kind: "Async", + }), + }), + expect.objectContaining({ + module: expect.objectContaining({ + name: "openai", + versionRange: ">=5.0.0", + filePath: "resources/chat/completions/completions.mjs", + }), + functionQuery: expect.objectContaining({ + className: "Completions", + methodName: "parse", + kind: "Async", + }), + }), + ]), ); - - expect(config).toBeDefined(); - expect(config?.module.name).toBe("openai"); - expect(config?.module.filePath).toBe("resources/beta/chat/completions.mjs"); - expect((config?.functionQuery as any).className).toBe("Completions"); - expect((config?.functionQuery as any).methodName).toBe("parse"); - expect((config?.functionQuery as any).kind).toBe("Async"); }); it("should NOT include braintrust: prefix (code-transformer adds orchestrion:openai: prefix)", () => { @@ -74,7 +131,9 @@ describe("OpenAI Instrumentation Configs", () => { it("should have valid version ranges", () => { for (const config of openaiConfigs) { - expect(config.module.versionRange).toMatch(/^>=\d+\.\d+\.\d+$/); + expect(config.module.versionRange).toMatch( + /^>=\d+\.\d+\.\d+( <\d+\.\d+\.\d+)?$/, + ); } }); @@ -86,16 +145,37 @@ describe("OpenAI Instrumentation Configs", () => { }); it("should have beta.chat.completions.stream config with Sync kind", () => { - const config = openaiConfigs.find( - (c) => c.channelName === "beta.chat.completions.stream", + const configs = configsForChannel("beta.chat.completions.stream"); + + expect(configs).toHaveLength(2); + expect(configs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + module: expect.objectContaining({ + name: "openai", + versionRange: ">=4.0.0 <5.0.0", + filePath: "resources/beta/chat/completions.mjs", + }), + functionQuery: expect.objectContaining({ + className: "Completions", + methodName: "stream", + kind: "Sync", + }), + }), + expect.objectContaining({ + module: expect.objectContaining({ + name: "openai", + versionRange: ">=5.0.0", + filePath: "resources/chat/completions/completions.mjs", + }), + functionQuery: expect.objectContaining({ + className: "Completions", + methodName: "stream", + kind: "Sync", + }), + }), + ]), ); - - expect(config).toBeDefined(); - expect(config?.module.name).toBe("openai"); - expect(config?.module.filePath).toBe("resources/beta/chat/completions.mjs"); - expect((config?.functionQuery as any).className).toBe("Completions"); - expect((config?.functionQuery as any).methodName).toBe("stream"); - expect((config?.functionQuery as any).kind).toBe("Sync"); }); it("should have responses.create config with version >=4.87.0", () => { diff --git a/js/src/auto-instrumentations/configs/openai.ts b/js/src/auto-instrumentations/configs/openai.ts index ec293c7d2..785e7a495 100644 --- a/js/src/auto-instrumentations/configs/openai.ts +++ b/js/src/auto-instrumentations/configs/openai.ts @@ -18,7 +18,7 @@ export const openaiConfigs: InstrumentationConfig[] = [ channelName: OPENAI_CHANNEL_SUFFIX.CHAT_COMPLETIONS_CREATE, module: { name: "openai", - versionRange: ">=4.0.0", + versionRange: ">=4.0.0 <5.0.0", filePath: "resources/chat/completions.mjs", }, functionQuery: { @@ -28,6 +28,34 @@ export const openaiConfigs: InstrumentationConfig[] = [ }, }, + { + channelName: OPENAI_CHANNEL_SUFFIX.CHAT_COMPLETIONS_CREATE, + module: { + name: "openai", + versionRange: ">=4.0.0 <5.0.0", + filePath: "resources/chat/completions/completions.mjs", + }, + functionQuery: { + className: "Completions", + methodName: "create", + kind: "Async", + }, + }, + + { + channelName: OPENAI_CHANNEL_SUFFIX.CHAT_COMPLETIONS_CREATE, + module: { + name: "openai", + versionRange: ">=5.0.0", + filePath: "resources/chat/completions/completions.mjs", + }, + functionQuery: { + className: "Completions", + methodName: "create", + kind: "Async", + }, + }, + // Embeddings { channelName: OPENAI_CHANNEL_SUFFIX.EMBEDDINGS_CREATE, @@ -48,7 +76,7 @@ export const openaiConfigs: InstrumentationConfig[] = [ channelName: OPENAI_CHANNEL_SUFFIX.BETA_CHAT_COMPLETIONS_PARSE, module: { name: "openai", - versionRange: ">=4.0.0", + versionRange: ">=4.0.0 <5.0.0", filePath: "resources/beta/chat/completions.mjs", }, functionQuery: { @@ -58,6 +86,20 @@ export const openaiConfigs: InstrumentationConfig[] = [ }, }, + { + channelName: OPENAI_CHANNEL_SUFFIX.BETA_CHAT_COMPLETIONS_PARSE, + module: { + name: "openai", + versionRange: ">=5.0.0", + filePath: "resources/chat/completions/completions.mjs", + }, + functionQuery: { + className: "Completions", + methodName: "parse", + kind: "Async", + }, + }, + // Moderations { channelName: OPENAI_CHANNEL_SUFFIX.MODERATIONS_CREATE, @@ -78,7 +120,7 @@ export const openaiConfigs: InstrumentationConfig[] = [ channelName: OPENAI_CHANNEL_SUFFIX.BETA_CHAT_COMPLETIONS_STREAM, module: { name: "openai", - versionRange: ">=4.0.0", + versionRange: ">=4.0.0 <5.0.0", filePath: "resources/beta/chat/completions.mjs", }, functionQuery: { @@ -88,6 +130,20 @@ export const openaiConfigs: InstrumentationConfig[] = [ }, }, + { + channelName: OPENAI_CHANNEL_SUFFIX.BETA_CHAT_COMPLETIONS_STREAM, + module: { + name: "openai", + versionRange: ">=5.0.0", + filePath: "resources/chat/completions/completions.mjs", + }, + functionQuery: { + className: "Completions", + methodName: "stream", + kind: "Sync", + }, + }, + // Responses API (v4.87.0+) { channelName: OPENAI_CHANNEL_SUFFIX.RESPONSES_CREATE, diff --git a/js/tests/auto-instrumentations/fixtures/openai-e2e-test.mjs b/js/tests/auto-instrumentations/fixtures/openai-e2e-test.mjs new file mode 100644 index 000000000..7dd0d778f --- /dev/null +++ b/js/tests/auto-instrumentations/fixtures/openai-e2e-test.mjs @@ -0,0 +1,61 @@ +import OpenAI from "openai"; +import { initLogger, _exportsForTestingOnly } from "../../../dist/index.mjs"; + +const backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); +await _exportsForTestingOnly.simulateLoginForTests(); + +const logger = initLogger({ + projectName: "auto-instrumentation-test", + projectId: "test-project-id", +}); + +// Create OpenAI client with mocked fetch +const mockFetch = async (url, options) => { + return { + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + json: async () => ({ + id: "chatcmpl-test123", + object: "chat.completion", + created: Date.now(), + model: "gpt-4", + choices: [ + { + index: 0, + message: { role: "assistant", content: "Test response" }, + finish_reason: "stop", + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 5, + total_tokens: 15, + }, + }), + }; +}; + +const client = new OpenAI({ + apiKey: "test-key", + fetch: mockFetch, +}); + +try { + const completion = await client.chat.completions.create({ + model: "gpt-4", + messages: [{ role: "user", content: "Hello!" }], + }); + + const spans = await backgroundLogger.drain(); + + for (const span of spans) { + console.log("SPAN_DATA:", JSON.stringify(span)); + } + + console.log("SUCCESS: API call completed"); + process.exit(0); +} catch (error) { + console.error("ERROR:", error.message); + process.exit(1); +} diff --git a/knip.jsonc b/knip.jsonc index 7f8b7b0e4..e7d364a45 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -10,6 +10,33 @@ "**/generated_types.ts": ["exports", "types"], }, "workspaces": { + "e2e": { + "entry": [ + "package.json", + "vitest.config.mts", + "helpers/**/*.ts", + "scenarios/**/*.test.ts", + "scenarios/**/package.json", + "scenarios/**/*.ts", + "scenarios/**/*.mjs", + ], + "project": [ + "helpers/**/*.ts", + "scenarios/**/*.test.ts", + "scenarios/**/package.json", + "scenarios/**/*.ts", + "scenarios/**/*.mjs", + ], + "ignore": ["runScenarioDir", "runNodeScenarioDir"], + "ignoreFiles": [ + "helpers/scenario-runtime.ts", + "scenarios/**/scenario.ts", + "scenarios/**/scenario.mjs", + "scenarios/**/scenario.*.ts", + "scenarios/**/scenario.*.mjs", + ], + "ignoreDependencies": ["openai", "openai-v4", "openai-v5"], + }, "js": { "entry": [ "src/auto-instrumentations/bundler/*.ts", diff --git a/package.json b/package.json index ad067e9c9..d043057b4 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,8 @@ "clean": "turbo run clean", "knip": "knip --config knip.jsonc --no-config-hints", "test": "turbo run test --filter=\"!@braintrust/otel\"", + "test:e2e": "turbo run test:e2e", + "test:e2e:update": "turbo run test:e2e:update", "playground": "turbo run playground --filter=\"braintrust\"", "prepare": "husky || true", "lint:prettier": "prettier --check .", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4282e3739..6cad4d451 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -27,6 +27,36 @@ importers: specifier: ^2.5.6 version: 2.5.6 + e2e: + devDependencies: + '@braintrust/otel': + specifier: workspace:^ + version: link:../integrations/otel-js + '@opentelemetry/api': + specifier: '>=1.9.0' + version: 1.9.0 + '@opentelemetry/context-async-hooks': + specifier: '>=1.9.0' + version: 2.6.0(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': + specifier: '>=1.9.0' + version: 2.2.0(@opentelemetry/api@1.9.0) + '@types/node': + specifier: ^20.10.5 + version: 20.19.16 + braintrust: + specifier: workspace:^ + version: link:../js + tsx: + specifier: ^3.14.0 + version: 3.14.0 + typescript: + specifier: 5.4.4 + version: 5.4.4 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@20.19.16)(msw@2.6.6(@types/node@20.19.16)(typescript@5.4.4))(terser@5.44.1) + integrations/browser-js: dependencies: als-browser: @@ -2048,6 +2078,12 @@ packages: resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} engines: {node: '>=8.0.0'} + '@opentelemetry/context-async-hooks@2.6.0': + resolution: {integrity: sha512-L8UyDwqpTcbkIK5cgwDRDYDoEhQoj8wp8BwsO19w3LB1Z41yEQm2VJyNfAi9DrLP/YTqXqWpKHyZfR9/tFYo1Q==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/core@2.2.0': resolution: {integrity: sha512-FuabnnUm8LflnieVxs6eP7Z383hgQU4W1e3KJS6aOG3RxWxcHyBxH8fDMHNgu/gFx/M2jvTOW/4/PHhLz6bjWw==} engines: {node: ^18.19.0 || >=20.6.0} @@ -4749,18 +4785,6 @@ packages: zod: optional: true - openai@6.15.0: - resolution: {integrity: sha512-F1Lvs5BoVvmZtzkUEVyh8mDQPPFolq4F+xdsx/DO8Hee8YF3IGAlZqUIsF+DVGhqf4aU0a3bTghsxB6OIsRy1g==} - hasBin: true - peerDependencies: - ws: ^8.18.0 - zod: ^3.25 || ^4.0 - peerDependenciesMeta: - ws: - optional: true - zod: - optional: true - openai@6.25.0: resolution: {integrity: sha512-mEh6VZ2ds2AGGokWARo18aPISI1OhlgdEIC1ewhkZr8pSIT31dec0ecr9Nhxx0JlybyOgoAT1sWeKtwPZzJyww==} hasBin: true @@ -5736,6 +5760,10 @@ packages: engines: {node: '>=0.8.0'} hasBin: true + unbash@2.2.0: + resolution: {integrity: sha512-X2wH19RAPZE3+ldGicOkoj/SIA83OIxcJ6Cuaw23hf8Xc6fQpvZXY0SftE2JgS0QhYLUG4uwodSI3R53keyh7w==} + engines: {node: '>=14'} + undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} @@ -7723,7 +7751,7 @@ snapshots: dependencies: '@langchain/core': 1.1.10(@opentelemetry/api@1.9.0)(@opentelemetry/sdk-trace-base@2.2.0(@opentelemetry/api@1.9.0))(openai@6.25.0(ws@8.18.3)(zod@3.25.76)) js-tiktoken: 1.0.21 - openai: 6.15.0(ws@8.18.3)(zod@3.25.76) + openai: 6.25.0(ws@8.18.3)(zod@3.25.76) zod: 3.25.76 transitivePeerDependencies: - ws @@ -7896,6 +7924,10 @@ snapshots: '@opentelemetry/api@1.9.0': {} + '@opentelemetry/context-async-hooks@2.6.0(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core@2.2.0(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -11300,11 +11332,6 @@ snapshots: ws: 8.18.3 zod: 3.25.76 - openai@6.15.0(ws@8.18.3)(zod@3.25.76): - optionalDependencies: - ws: 8.18.3 - zod: 3.25.76 - openai@6.25.0(ws@8.18.3)(zod@3.25.76): optionalDependencies: ws: 8.18.3 @@ -12378,6 +12405,8 @@ snapshots: uglify-js@3.19.3: optional: true + unbash@2.2.0: {} + undici-types@5.26.5: {} undici-types@6.21.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index aa39e5413..fc34687e1 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,5 @@ packages: + - e2e - js - integrations/* - internal/golden diff --git a/turbo.json b/turbo.json index 01a3e7447..24b5bb89c 100644 --- a/turbo.json +++ b/turbo.json @@ -1,6 +1,10 @@ { "$schema": "https://turbo.build/schema.json", - "globalPassThroughEnv": ["OPENAI_API_KEY", "ANTHROPIC_API_KEY"], + "globalPassThroughEnv": [ + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "ANTHROPIC_API_KEY" + ], "tasks": { "build": { "dependsOn": ["^build"], @@ -12,7 +16,32 @@ "outputs": [] }, "test": { - "env": ["ANTHROPIC_API_KEY", "BRAINTRUST_API_KEY", "OPENAI_API_KEY"], + "env": [ + "ANTHROPIC_API_KEY", + "BRAINTRUST_API_KEY", + "OPENAI_API_KEY", + "OPENAI_BASE_URL" + ], + "dependsOn": ["^build"], + "outputs": [] + }, + "test:e2e": { + "env": [ + "ANTHROPIC_API_KEY", + "BRAINTRUST_API_KEY", + "OPENAI_API_KEY", + "OPENAI_BASE_URL" + ], + "dependsOn": ["^build"], + "outputs": [] + }, + "test:e2e:update": { + "env": [ + "ANTHROPIC_API_KEY", + "BRAINTRUST_API_KEY", + "OPENAI_API_KEY", + "OPENAI_BASE_URL" + ], "dependsOn": ["^build"], "outputs": [] },