diff --git a/.github/workflows/js.yaml b/.github/workflows/js.yaml index ba40b3cad..81aa19881 100644 --- a/.github/workflows/js.yaml +++ b/.github/workflows/js.yaml @@ -57,6 +57,14 @@ jobs: PACKED_TARBALL=$(npm pack --pack-destination artifacts) echo "packed_tarball=$PACKED_TARBALL" >> "$GITHUB_OUTPUT" + - name: Pack @braintrust/browser + id: prepare_browser_artifact + working-directory: integrations/browser-js + shell: bash + run: | + PACKED_BROWSER_TARBALL=$(npm pack --pack-destination ../../js/artifacts) + echo "packed_browser_tarball=$PACKED_BROWSER_TARBALL" >> "$GITHUB_OUTPUT" + - name: Build and pack @braintrust/otel id: prepare_otel_artifact shell: bash @@ -89,11 +97,22 @@ jobs: PACKED_OTEL_TARBALL=$(npm pack --pack-destination ../../js/artifacts) echo "packed_otel_tarball=$PACKED_OTEL_TARBALL" >> "$GITHUB_OUTPUT" + - name: Build and pack @braintrust/templates-nunjucks + id: prepare_templates_nunjucks_artifact + shell: bash + run: | + cd integrations/templates-nunjucks + pnpm run build + PACKED_NUNJUCKS_TARBALL=$(npm pack --pack-destination ../../js/artifacts) + echo "packed_nunjucks_tarball=$PACKED_NUNJUCKS_TARBALL" >> "$GITHUB_OUTPUT" + - name: List artifacts before upload shell: bash run: | echo "Braintrust tarball: ${{ steps.prepare_artifact.outputs.packed_tarball }}" + echo "Browser tarball: ${{ steps.prepare_browser_artifact.outputs.packed_browser_tarball }}" echo "Otel tarball: ${{ steps.prepare_otel_artifact.outputs.packed_otel_tarball }}" + echo "Templates-nunjucks tarball: ${{ steps.prepare_templates_nunjucks_artifact.outputs.packed_nunjucks_tarball }}" ls -la js/artifacts/ - name: Upload build artifacts @@ -102,7 +121,9 @@ jobs: name: ${{ steps.artifact.outputs.name }}-${{ matrix.node-version }}-dist path: | js/artifacts/${{ steps.prepare_artifact.outputs.packed_tarball }} + js/artifacts/${{ steps.prepare_browser_artifact.outputs.packed_browser_tarball }} js/artifacts/${{ steps.prepare_otel_artifact.outputs.packed_otel_tarball }} + js/artifacts/${{ steps.prepare_templates_nunjucks_artifact.outputs.packed_nunjucks_tarball }} retention-days: 1 api-compatibility: @@ -210,8 +231,15 @@ jobs: working-directory: js/smoke shell: bash run: | - # Use the same auto-discovery logic as the Makefile - SCENARIOS=$(command find scenarios -mindepth 1 -maxdepth 1 -type d -exec test -f {}/Makefile \; -print | sed 's|scenarios/||' | sort | jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "Discovering local and integration scenarios..." + SCRIPTDIR="../../integrations" + # Local scenarios (under js/smoke/scenarios): emit their directory names + LOCAL=$(find scenarios -mindepth 1 -maxdepth 1 -type d -exec test -f {}/Makefile \; -print | sed 's|scenarios/||' | tr '\n' ' ') + # Integration scenarios: emit as integration/name + INTEGRATIONS=$(find "$SCRIPTDIR" -type f -path '*/smoke/scenarios/*/Makefile' 2>/dev/null | sed "s|^$SCRIPTDIR/||" | sed 's|/Makefile$||' | sed 's|/smoke/scenarios/|/|' | tr '\n' ' ') + ALL="${LOCAL} ${INTEGRATIONS}" + # Convert space-separated list to JSON array + SCENARIOS=$(printf "%s\n" $ALL | jq -R -s -c 'split("\n") | map(select(length>0))') echo "scenarios=$SCENARIOS" >> "$GITHUB_OUTPUT" echo "Discovered scenarios: $SCENARIOS" @@ -255,6 +283,15 @@ jobs: fi done + # Copy browser tarball to well-known path + for f in braintrust-browser-[0-9]*.tgz; do + if [ -f "$f" ]; then + cp "$f" braintrust-browser-latest.tgz + echo "Copied $f to braintrust-browser-latest.tgz" + break + fi + done + # Copy otel tarball to well-known path for f in braintrust-otel-[0-9]*.tgz; do if [ -f "$f" ]; then @@ -264,6 +301,15 @@ jobs: fi done + # Copy templates-nunjucks-js tarball to well-known path + for f in braintrust-templates-nunjucks-js-[0-9]*.tgz; do + if [ -f "$f" ]; then + cp "$f" braintrust-templates-nunjucks-js-latest.tgz + echo "Copied $f to braintrust-templates-nunjucks-js-latest.tgz" + break + fi + done + - name: Build shared test package (once for all scenarios) working-directory: js/smoke/shared shell: bash @@ -278,7 +324,9 @@ jobs: BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} CI: true BRAINTRUST_TAR: ../artifacts/braintrust-latest.tgz + BRAINTRUST_BROWSER_TAR: ../artifacts/braintrust-browser-latest.tgz BRAINTRUST_OTEL_TAR: ../artifacts/braintrust-otel-latest.tgz + BRAINTRUST_TEMPLATES_NUNJUCKS_JS_TAR: ../artifacts/braintrust-templates-nunjucks-js-latest.tgz SMOKE_V2_SHARED_DIST: shared/dist run: | make test ${{ matrix.scenario }} diff --git a/.github/workflows/templates-nunjucks-build-test.yaml b/.github/workflows/templates-nunjucks-build-test.yaml new file mode 100644 index 000000000..ea48d56b3 --- /dev/null +++ b/.github/workflows/templates-nunjucks-build-test.yaml @@ -0,0 +1,46 @@ +name: templates-nunjucks-js + +on: + pull_request: + paths: + - "integrations/templates-nunjucks/**" + - "js/**" + - ".github/workflows/templates-nunjucks-build-test.yaml" + push: + branches: [main] + paths: + - "integrations/templates-nunjucks/**" + - "js/**" + +jobs: + build-and-test: + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + node-version: [20, 22] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + + - uses: pnpm/action-setup@v4 + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build braintrust + working-directory: js + run: pnpm run build + + - name: Build @braintrust/templates-nunjucks-js + working-directory: integrations/templates-nunjucks + run: pnpm run build + + - name: Run @braintrust/templates-nunjucks-js tests + working-directory: integrations/templates-nunjucks + run: pnpm run test diff --git a/Makefile b/Makefile index 71a4182f6..162b8871e 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,8 @@ js-build: pnpm run build js-test: js-build - pnpm run test + # Run tests only for the JS workspace packages and exclude integration scenario tests + pnpm --filter ./js... run test cd js && make test js-docs: js-build diff --git a/integrations/browser-js/README.md b/integrations/browser-js/README.md new file mode 100644 index 000000000..929c68553 --- /dev/null +++ b/integrations/browser-js/README.md @@ -0,0 +1,75 @@ +# Braintrust Browser SDK + +Official browser-only SDK for [Braintrust](https://braintrust.dev). + +This is an integration package that provides browser-optimized builds of the Braintrust SDK with AsyncLocalStorage polyfill support for standard browsers. + +## Installation + +```bash +npm install @braintrust/browser braintrust +# or +pnpm add @braintrust/browser braintrust +# or +yarn add @braintrust/browser braintrust +``` + +Note: `braintrust` is a peer dependency and must be installed alongside `@braintrust/browser`. + +## Usage + +```typescript +import * as braintrust from "@braintrust/browser"; + +const experiment = await braintrust.init("BrowserExperiment", { + apiKey: "YOUR_API_KEY", +}); + +// Use tracing in browser +const result = await braintrust.traced( + async () => { + // Your code here + return "result"; + }, + { name: "myOperation" }, +); +``` + +## Differences from Main Package + +This package: + +- **Includes** `als-browser` polyfill for AsyncLocalStorage (bundled) +- **Requires** `braintrust` as a peer dependency + +## When to Use + +Use `@braintrust/browser` when: + +- Building browser-only applications +- Need AsyncLocalStorage support in standard browsers + +Use `braintrust` directly when: + +- Building Node.js applications +- Using in Next.js or other full-stack frameworks (with proper module resolution) +- Need CLI tools or filesystem access + +## Features + +All browser-compatible features from the main SDK: + +- Logging and tracing +- Experiments and datasets +- Prompt management +- AI provider wrappers (OpenAI, Anthropic, Google) +- Evaluation framework +- OpenTelemetry integration + +## Documentation + +For full documentation, visit [https://www.braintrust.dev/docs](https://www.braintrust.dev/docs) + +## License + +Apache-2.0 diff --git a/integrations/browser-js/package.json b/integrations/browser-js/package.json new file mode 100644 index 000000000..aa4485367 --- /dev/null +++ b/integrations/browser-js/package.json @@ -0,0 +1,58 @@ +{ + "name": "@braintrust/browser", + "version": "0.0.2", + "description": "Braintrust SDK for browser environments with AsyncLocalStorage polyfill", + "type": "module", + "module": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + } + }, + "files": [ + "dist", + "README.md" + ], + "scripts": { + "build": "tsup", + "watch": "tsup --watch", + "clean": "rm -rf dist", + "test": "vitest run" + }, + "dependencies": { + "als-browser": "^1.0.1" + }, + "devDependencies": { + "@types/node": "^20.10.5", + "braintrust": "workspace:*", + "tsup": "^8.3.5", + "typescript": "^5.3.3", + "vitest": "^2.1.9" + }, + "peerDependencies": { + "braintrust": ">=3.0.0", + "zod": "^3.25.34 || ^4.0" + }, + "peerDependenciesMeta": { + "braintrust": { + "optional": false + }, + "zod": { + "optional": false + } + }, + "repository": { + "type": "git", + "url": "git+https://github.com/braintrustdata/braintrust-sdk.git", + "directory": "sdk/integrations/browser-js" + }, + "homepage": "https://www.braintrust.dev/docs", + "license": "Apache-2.0", + "publishConfig": { + "access": "public", + "registry": "https://registry.npmjs.org/" + } +} diff --git a/integrations/browser-js/src/browser-config.ts b/integrations/browser-js/src/browser-config.ts new file mode 100644 index 000000000..139cc3d90 --- /dev/null +++ b/integrations/browser-js/src/browser-config.ts @@ -0,0 +1,48 @@ +import { _internalIso as iso, _internalSetInitialState } from "braintrust"; +import { AsyncLocalStorage as BrowserAsyncLocalStorage } from "als-browser"; + +let browserConfigured = false; + +export function configureBrowser() { + if (browserConfigured) { + return; + } + + // Set build type indicator + iso.buildType = "browser"; + + iso.newAsyncLocalStorage = () => new BrowserAsyncLocalStorage(); + + iso.getEnv = (name: string) => { + if (typeof process === "undefined" || typeof process.env === "undefined") { + return undefined; + } + return process.env[name]; + }; + + // noop implementations for git config + iso.getRepoInfo = async () => ({ + commit: null, + branch: null, + tag: null, + dirty: false, + }); + iso.getCallerLocation = () => undefined; + + // Implement browser-compatible hash function using a simple hash algorithm + iso.hash = (data: string): string => { + // Simple hash function for browser compatibility + let hash = 0; + for (let i = 0; i < data.length; i++) { + const char = data.charCodeAt(i); + hash = (hash << 5) - hash + char; + hash = hash & hash; // Convert to 32bit integer + } + // Convert to hex string + const hashHex = (hash >>> 0).toString(16).padStart(8, "0"); + return hashHex.repeat(8).substring(0, 64); // Make it look like a SHA-256 hash length + }; + + _internalSetInitialState(); + browserConfigured = true; +} diff --git a/integrations/browser-js/src/index.ts b/integrations/browser-js/src/index.ts new file mode 100644 index 000000000..70b350745 --- /dev/null +++ b/integrations/browser-js/src/index.ts @@ -0,0 +1,5 @@ +import { configureBrowser } from "./browser-config"; + +configureBrowser(); + +export * from "braintrust"; diff --git a/integrations/browser-js/tsconfig.json b/integrations/browser-js/tsconfig.json new file mode 100644 index 000000000..94649b1d8 --- /dev/null +++ b/integrations/browser-js/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "declaration": true, + "lib": ["es2022", "dom"], + "module": "esnext", + "target": "es2022", + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "resolveJsonModule": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/integrations/browser-js/tsup.config.ts b/integrations/browser-js/tsup.config.ts new file mode 100644 index 000000000..b532d4d5c --- /dev/null +++ b/integrations/browser-js/tsup.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + format: ["esm"], + dts: true, + sourcemap: true, + clean: true, + external: ["braintrust", "zod"], + target: "es2022", + platform: "browser", + outDir: "./dist", + treeshake: true, +}); diff --git a/integrations/browser-js/turbo.json b/integrations/browser-js/turbo.json new file mode 100644 index 000000000..6b90853c5 --- /dev/null +++ b/integrations/browser-js/turbo.json @@ -0,0 +1,10 @@ +{ + "$schema": "https://turbo.build/schema.json", + "extends": ["//"], + "tasks": { + "build": { + "dependsOn": ["braintrust#build"], + "outputs": ["dist/**"] + } + } +} diff --git a/integrations/browser-js/vitest.config.ts b/integrations/browser-js/vitest.config.ts new file mode 100644 index 000000000..73a26a648 --- /dev/null +++ b/integrations/browser-js/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["tests/**/*.test.ts", "src/**/*.test.ts"], + }, +}); diff --git a/integrations/langchain-js/vitest.config.ts b/integrations/langchain-js/vitest.config.ts index 197422ec0..d16bcb8cc 100644 --- a/integrations/langchain-js/vitest.config.ts +++ b/integrations/langchain-js/vitest.config.ts @@ -3,5 +3,6 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { setupFiles: ["./src/test/setup.ts"], + include: ["tests/**/*.test.ts", "src/**/*.test.ts"], }, }); diff --git a/integrations/openai-agents-js/src/index.ts b/integrations/openai-agents-js/src/index.ts index 95a18edd5..4a6a565d3 100644 --- a/integrations/openai-agents-js/src/index.ts +++ b/integrations/openai-agents-js/src/index.ts @@ -382,7 +382,8 @@ export class OpenAIAgentsTraceProcessor { if (!data.metrics.completion_tokens && usage.completionTokens) data.metrics.completion_tokens = usage.completionTokens; if (usage.input_tokens_details?.cached_tokens != null) - data.metrics.prompt_cached_tokens = usage.input_tokens_details.cached_tokens; + data.metrics.prompt_cached_tokens = + usage.input_tokens_details.cached_tokens; } return data; diff --git a/integrations/openai-agents-js/src/openai-agents-integration.test.ts b/integrations/openai-agents-js/src/openai-agents-integration.test.ts index 618da167b..75c954942 100644 --- a/integrations/openai-agents-js/src/openai-agents-integration.test.ts +++ b/integrations/openai-agents-js/src/openai-agents-integration.test.ts @@ -908,7 +908,7 @@ describe( output_tokens: 50, total_tokens: 150, input_tokens_details: { - cached_tokens: 80, // check for this later + cached_tokens: 80, // check for this later }, }, }, @@ -934,9 +934,17 @@ describe( const metrics = (responseSpanLog as any).metrics; assert.ok(metrics, "Response span should have metrics"); assert.equal(metrics.prompt_tokens, 100, "Should have prompt_tokens"); - assert.equal(metrics.completion_tokens, 50, "Should have completion_tokens"); + assert.equal( + metrics.completion_tokens, + 50, + "Should have completion_tokens", + ); assert.equal(metrics.tokens, 150, "Should have total tokens"); - assert.equal(metrics.prompt_cached_tokens, 80, "Should extract cached_tokens to prompt_cached_tokens"); + assert.equal( + metrics.prompt_cached_tokens, + 80, + "Should extract cached_tokens to prompt_cached_tokens", + ); }); test("Response span handles zero cached tokens correctly", async () => { @@ -965,7 +973,7 @@ describe( input_tokens: 100, output_tokens: 50, input_tokens_details: { - cached_tokens: 0, // Zero is a valid value + cached_tokens: 0, // Zero is a valid value }, }, }, @@ -977,7 +985,9 @@ describe( await processor.onSpanEnd(responseSpan); const spans = await backgroundLogger.drain(); - const responseSpanLog = spans.find((s: any) => s.span_attributes?.type === "llm"); + const responseSpanLog = spans.find( + (s: any) => s.span_attributes?.type === "llm", + ); const metrics = (responseSpanLog as any).metrics; // Zero should be logged, not skipped @@ -1024,11 +1034,16 @@ describe( await processor.onSpanEnd(responseSpan); const spans = await backgroundLogger.drain(); - const responseSpanLog = spans.find((s: any) => s.span_attributes?.type === "llm"); + const responseSpanLog = spans.find( + (s: any) => s.span_attributes?.type === "llm", + ); const metrics = (responseSpanLog as any).metrics; // Should not have prompt_cached_tokens if not present in usage - assert.isUndefined(metrics.prompt_cached_tokens, "Should not add prompt_cached_tokens if not in usage"); + assert.isUndefined( + metrics.prompt_cached_tokens, + "Should not add prompt_cached_tokens if not in usage", + ); }); test("Generation span extracts cached tokens from usage", async () => { @@ -1060,7 +1075,7 @@ describe( output_tokens: 75, total_tokens: 275, input_tokens_details: { - cached_tokens: 150, // Test Generation span extraction + cached_tokens: 150, // Test Generation span extraction }, }, }, @@ -1080,8 +1095,16 @@ describe( const metrics = (generationSpanLog as any).metrics; assert.ok(metrics, "Generation span should have metrics"); assert.equal(metrics.prompt_tokens, 200, "Should have prompt_tokens"); - assert.equal(metrics.completion_tokens, 75, "Should have completion_tokens"); - assert.equal(metrics.prompt_cached_tokens, 150, "Should extract cached_tokens from Generation span"); + assert.equal( + metrics.completion_tokens, + 75, + "Should have completion_tokens", + ); + assert.equal( + metrics.prompt_cached_tokens, + 150, + "Should extract cached_tokens from Generation span", + ); }); }, ); diff --git a/integrations/openai-agents-js/vitest.config.ts b/integrations/openai-agents-js/vitest.config.ts index be2185492..8e30c5974 100644 --- a/integrations/openai-agents-js/vitest.config.ts +++ b/integrations/openai-agents-js/vitest.config.ts @@ -2,6 +2,7 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { + include: ["tests/**/*.test.ts", "src/**/*.test.ts"], // Add any specific test configuration if needed }, }); diff --git a/integrations/otel-js/otel-v1/vitest.config.ts b/integrations/otel-js/otel-v1/vitest.config.ts index 8c8114c32..fd400cc49 100644 --- a/integrations/otel-js/otel-v1/vitest.config.ts +++ b/integrations/otel-js/otel-v1/vitest.config.ts @@ -17,4 +17,7 @@ export default defineConfig({ alias: createOtelAliases(cwd), } : {}, + test: { + include: ["tests/**/*.test.ts", "src/**/*.test.ts"], + }, }); diff --git a/integrations/otel-js/otel-v2/vitest.config.ts b/integrations/otel-js/otel-v2/vitest.config.ts index 8c8114c32..fd400cc49 100644 --- a/integrations/otel-js/otel-v2/vitest.config.ts +++ b/integrations/otel-js/otel-v2/vitest.config.ts @@ -17,4 +17,7 @@ export default defineConfig({ alias: createOtelAliases(cwd), } : {}, + test: { + include: ["tests/**/*.test.ts", "src/**/*.test.ts"], + }, }); diff --git a/js/smoke/scenarios/otel-v1/.gitignore b/integrations/otel-js/smoke/scenarios/otel-v1/.gitignore similarity index 100% rename from js/smoke/scenarios/otel-v1/.gitignore rename to integrations/otel-js/smoke/scenarios/otel-v1/.gitignore diff --git a/js/smoke/scenarios/otel-v1/Makefile b/integrations/otel-js/smoke/scenarios/otel-v1/Makefile similarity index 78% rename from js/smoke/scenarios/otel-v1/Makefile rename to integrations/otel-js/smoke/scenarios/otel-v1/Makefile index 0b72d0207..83e3338f5 100644 --- a/js/smoke/scenarios/otel-v1/Makefile +++ b/integrations/otel-js/smoke/scenarios/otel-v1/Makefile @@ -13,7 +13,7 @@ setup: echo "==> Using BRAINTRUST_TAR: $(BRAINTRUST_TAR)"; \ else \ echo "==> Building SDK"; \ - cd ../../.. && pnpm exec turbo build --filter=braintrust && mkdir -p artifacts && pnpm pack --pack-destination artifacts; \ + cd ../../../.. && pnpm exec turbo build --filter=braintrust && mkdir -p artifacts && pnpm pack --pack-destination artifacts; \ \ for f in artifacts/braintrust-*.tgz; do \ if [ "$$(basename $$f)" != "braintrust-latest.tgz" ] && \ @@ -27,7 +27,7 @@ setup: @# Build shared package (if not running from parent) @if [ -z "$(SMOKE_V2_SHARED_DIST)" ]; then \ echo "==> Building shared package"; \ - cd ../../shared && npm ci && npm run build; \ + cd ../../../shared && npm ci && npm run build; \ fi @# Check if BRAINTRUST_OTEL_TAR is set (from parent or CI), otherwise build @@ -35,11 +35,11 @@ setup: echo "==> Using BRAINTRUST_OTEL_TAR: $(BRAINTRUST_OTEL_TAR)"; \ else \ echo "==> Building @braintrust/otel package"; \ - cd ../../../../integrations/otel-js && pnpm exec turbo build --filter=@braintrust/otel && pnpm pack --pack-destination ../../js/artifacts; \ + cd ../../../../../integrations/otel-js && pnpm exec turbo build --filter=@braintrust/otel && pnpm pack --pack-destination ../../js/artifacts; \ \ - for f in ../../js/artifacts/braintrust-otel-*.tgz; do \ + for f in ../../../js/artifacts/braintrust-otel-*.tgz; do \ if [ "$$(basename $$f)" != "braintrust-otel-latest.tgz" ]; then \ - cp "$$f" ../../js/artifacts/braintrust-otel-latest.tgz; \ + cp "$$f" ../../../js/artifacts/braintrust-otel-latest.tgz; \ break; \ fi; \ done; \ diff --git a/js/smoke/scenarios/otel-v1/README.md b/integrations/otel-js/smoke/scenarios/otel-v1/README.md similarity index 100% rename from js/smoke/scenarios/otel-v1/README.md rename to integrations/otel-js/smoke/scenarios/otel-v1/README.md diff --git a/js/smoke/scenarios/otel-v1/mise.toml b/integrations/otel-js/smoke/scenarios/otel-v1/mise.toml similarity index 100% rename from js/smoke/scenarios/otel-v1/mise.toml rename to integrations/otel-js/smoke/scenarios/otel-v1/mise.toml diff --git a/js/smoke/scenarios/otel-v1/package.json b/integrations/otel-js/smoke/scenarios/otel-v1/package.json similarity index 73% rename from js/smoke/scenarios/otel-v1/package.json rename to integrations/otel-js/smoke/scenarios/otel-v1/package.json index 335a0b33b..50b0a8700 100644 --- a/js/smoke/scenarios/otel-v1/package.json +++ b/integrations/otel-js/smoke/scenarios/otel-v1/package.json @@ -3,8 +3,8 @@ "private": true, "type": "module", "dependencies": { - "braintrust": "file:../../../artifacts/braintrust-latest.tgz", - "@braintrust/otel": "file:../../../artifacts/braintrust-otel-latest.tgz", + "braintrust": "file:../../../../../js/artifacts/braintrust-latest.tgz", + "@braintrust/otel": "file:../../../../../js/artifacts/braintrust-otel-latest.tgz", "@opentelemetry/api": "^1.9.0", "@opentelemetry/core": "^1.9.0", "@opentelemetry/exporter-trace-otlp-http": "^0.53.0", diff --git a/js/smoke/scenarios/otel-v1/src/test-helpers.ts b/integrations/otel-js/smoke/scenarios/otel-v1/src/test-helpers.ts similarity index 100% rename from js/smoke/scenarios/otel-v1/src/test-helpers.ts rename to integrations/otel-js/smoke/scenarios/otel-v1/src/test-helpers.ts diff --git a/js/smoke/scenarios/otel-v1/tests/basic.test.ts b/integrations/otel-js/smoke/scenarios/otel-v1/tests/basic.test.ts similarity index 97% rename from js/smoke/scenarios/otel-v1/tests/basic.test.ts rename to integrations/otel-js/smoke/scenarios/otel-v1/tests/basic.test.ts index 2031df34d..8ae35e369 100644 --- a/js/smoke/scenarios/otel-v1/tests/basic.test.ts +++ b/integrations/otel-js/smoke/scenarios/otel-v1/tests/basic.test.ts @@ -8,7 +8,7 @@ import { displayTestResults, hasFailures, type TestResult, -} from "../../../shared/dist/index.mjs"; +} from "../../../../../../js/smoke/shared/dist/index.mjs"; async function main() { const results: TestResult[] = []; diff --git a/js/smoke/scenarios/otel-v1/tests/filtering.test.ts b/integrations/otel-js/smoke/scenarios/otel-v1/tests/filtering.test.ts similarity index 98% rename from js/smoke/scenarios/otel-v1/tests/filtering.test.ts rename to integrations/otel-js/smoke/scenarios/otel-v1/tests/filtering.test.ts index 3fdfab0a4..0a7cf10a9 100644 --- a/js/smoke/scenarios/otel-v1/tests/filtering.test.ts +++ b/integrations/otel-js/smoke/scenarios/otel-v1/tests/filtering.test.ts @@ -8,7 +8,7 @@ import { displayTestResults, hasFailures, type TestResult, -} from "../../../shared/dist/index.mjs"; +} from "../../../../../../js/smoke/shared/dist/index.mjs"; type OtelPayload = { resourceSpans?: Array<{ diff --git a/js/smoke/scenarios/otel-v1/tests/shared-suite.test.ts b/integrations/otel-js/smoke/scenarios/otel-v1/tests/shared-suite.test.ts similarity index 80% rename from js/smoke/scenarios/otel-v1/tests/shared-suite.test.ts rename to integrations/otel-js/smoke/scenarios/otel-v1/tests/shared-suite.test.ts index 5559e89ac..83fba976f 100644 --- a/js/smoke/scenarios/otel-v1/tests/shared-suite.test.ts +++ b/integrations/otel-js/smoke/scenarios/otel-v1/tests/shared-suite.test.ts @@ -4,6 +4,7 @@ import { runTests, + expectFailure, testBasicSpanLogging, testMultipleSpans, testDirectLogging, @@ -25,7 +26,10 @@ import { testTestingExports, testStateManagementExports, testBuildResolution, -} from "../../../shared/dist/index.js"; + testMustacheTemplate, + testNunjucksTemplate, + testEvalSmoke, +} from "../../../../../../js/smoke/shared/dist/index.js"; async function runSharedTestSuites() { const braintrust = await import("braintrust"); @@ -55,6 +59,14 @@ async function runSharedTestSuites() { testAsyncLocalStorageTraced, testNestedTraced, testCurrentSpan, + testEvalSmoke, + testMustacheTemplate, + expectFailure( + testNunjucksTemplate, + (e: { message: string }) => + e.message.includes("requires @braintrust/template-nunjucks"), + "Nunjucks requires separate package", + ), ], }); diff --git a/js/smoke/scenarios/otel-v1/tsconfig.json b/integrations/otel-js/smoke/scenarios/otel-v1/tsconfig.json similarity index 100% rename from js/smoke/scenarios/otel-v1/tsconfig.json rename to integrations/otel-js/smoke/scenarios/otel-v1/tsconfig.json diff --git a/integrations/otel-js/vitest.config.ts b/integrations/otel-js/vitest.config.ts index 8fb6f2dcf..73a26a648 100644 --- a/integrations/otel-js/vitest.config.ts +++ b/integrations/otel-js/vitest.config.ts @@ -1,3 +1,7 @@ import { defineConfig } from "vitest/config"; -export default defineConfig({}); +export default defineConfig({ + test: { + include: ["tests/**/*.test.ts", "src/**/*.test.ts"], + }, +}); diff --git a/integrations/templates-nunjucks/package.json b/integrations/templates-nunjucks/package.json new file mode 100644 index 000000000..71c0a490b --- /dev/null +++ b/integrations/templates-nunjucks/package.json @@ -0,0 +1,42 @@ +{ + "name": "@braintrust/templates-nunjucks-js", + "version": "0.0.1", + "description": "Nunjucks templating support for the Braintrust JS SDK", + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "exports": { + "./package.json": "./package.json", + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.mjs", + "module": "./dist/index.mjs", + "require": "./dist/index.js" + } + }, + "files": [ + "dist/**/*" + ], + "scripts": { + "build": "tsup", + "watch": "tsup --watch", + "clean": "rm -r dist/*", + "test": "vitest run --exclude 'smoke/scenarios/**'" + }, + "author": "Braintrust Data Inc.", + "license": "Apache-2.0", + "dependencies": { + "nunjucks": "^3.2.4" + }, + "peerDependencies": { + "braintrust": ">=3.0.0" + }, + "devDependencies": { + "@types/nunjucks": "^3.2.6", + "@types/node": "^20.10.5", + "braintrust": "workspace:*", + "tsup": "^8.5.1", + "typescript": "5.5.4", + "vitest": "^2.1.9" + } +} diff --git a/integrations/templates-nunjucks/smoke/scenarios/deno-node/Makefile b/integrations/templates-nunjucks/smoke/scenarios/deno-node/Makefile new file mode 100644 index 000000000..92a53564b --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/deno-node/Makefile @@ -0,0 +1,37 @@ +.PHONY: setup test + +setup: + @echo "==> Setting up templates-nunjucks deno scenario" + mise install + + @# Build or extract braintrust SDK (workspace link will use dist/) + @if [ -n "$(BRAINTRUST_TAR)" ]; then \ + echo "==> Extracting SDK tarball for workspace link"; \ + TARBALL_PATH="$$(cd ../../../../../js/smoke && pwd)/$(BRAINTRUST_TAR)"; \ + echo "==> Resolved tarball path: $$TARBALL_PATH"; \ + cd ../../../../../js && tar -xzf "$$TARBALL_PATH" --strip-components=1 package/dist; \ + else \ + echo "==> Building SDK"; \ + cd ../../../../../js && pnpm build; \ + fi + + @# Build or extract @braintrust/templates-nunjucks-js (workspace link will use dist/) + @if [ -n "$(BRAINTRUST_TEMPLATES_NUNJUCKS_JS_TAR)" ]; then \ + echo "==> Extracting templates-nunjucks tarball for workspace link"; \ + TARBALL_PATH="$$(cd ../../../../../js/smoke && pwd)/$(BRAINTRUST_TEMPLATES_NUNJUCKS_JS_TAR)"; \ + echo "==> Resolved tarball path: $$TARBALL_PATH"; \ + cd ../../.. && tar -xzf "$$TARBALL_PATH" --strip-components=1 package/dist; \ + else \ + echo "==> Building @braintrust/templates-nunjucks-js package"; \ + cd ../../../../.. && pnpm install && pnpm --filter=@braintrust/templates-nunjucks-js build; \ + fi + + @# Build shared package (if not running from parent) + @if [ -z "$(SMOKE_V2_SHARED_DIST)" ]; then \ + echo "==> Building shared package"; \ + cd ../../../../../js/smoke/shared && npm install && npm run build; \ + fi + +test: setup + @echo "==> Running deno tests" + mise exec -- deno test --no-check --sloppy-imports --allow-all tests/*.test.ts diff --git a/integrations/templates-nunjucks/smoke/scenarios/deno-node/deno.json b/integrations/templates-nunjucks/smoke/scenarios/deno-node/deno.json new file mode 100644 index 000000000..deba541b0 --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/deno-node/deno.json @@ -0,0 +1,16 @@ +{ + "imports": { + "@std/assert": "jsr:@std/assert@^1.0.14", + "@braintrust/smoke-test-shared": "jsr:@braintrust/smoke-test-shared", + "@braintrust/templates-nunjucks-js": "npm:@braintrust/templates-nunjucks-js", + "braintrust": "npm:braintrust", + "zod": "npm:zod@^4.2.1", + "zod/": "npm:/zod@^4.2.1/" + }, + "nodeModulesDir": "auto", + "links": [ + "../../../../../js", + "../../../", + "../../../../../js/smoke/shared" + ] +} diff --git a/integrations/templates-nunjucks/smoke/scenarios/deno-node/mise.toml b/integrations/templates-nunjucks/smoke/scenarios/deno-node/mise.toml new file mode 100644 index 000000000..dd563ce6f --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/deno-node/mise.toml @@ -0,0 +1,3 @@ +[tools] +deno = "latest" +pnpm = "10.26.2" diff --git a/integrations/templates-nunjucks/smoke/scenarios/deno-node/tests/shared-suite.test.ts b/integrations/templates-nunjucks/smoke/scenarios/deno-node/tests/shared-suite.test.ts new file mode 100644 index 000000000..5800d3e6b --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/deno-node/tests/shared-suite.test.ts @@ -0,0 +1,27 @@ +// @ts-nocheck +/** + * Templates-Nunjucks Deno smoke test - testing template rendering only + */ + +import { assertEquals } from "@std/assert"; +import { + runTests, + testMustacheTemplate, + testNunjucksTemplate, +} from "@braintrust/smoke-test-shared"; +import * as braintrust from "braintrust"; +import { nunjucksPlugin } from "@braintrust/templates-nunjucks-js"; + +Deno.test("Run template tests with Nunjucks", async () => { + // Register nunjucks plugin before running tests + braintrust.registerTemplatePlugin(nunjucksPlugin); + + const { failed } = await runTests({ + name: "templates-nunjucks-deno", + braintrust, + tests: [testMustacheTemplate, testNunjucksTemplate], + skipCoverage: true, + }); + + assertEquals(failed.length, 0, "All template tests should pass"); +}); diff --git a/integrations/templates-nunjucks/smoke/scenarios/jest/Makefile b/integrations/templates-nunjucks/smoke/scenarios/jest/Makefile new file mode 100644 index 000000000..29291218a --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/jest/Makefile @@ -0,0 +1,48 @@ +.PHONY: setup test + +setup: + @echo "==> Setting up templates-nunjucks jest scenario" + mise install + + @# Build SDK and shared package if not provided + @if [ -n "$(BRAINTRUST_TAR)" ]; then \ + echo "==> Using BRAINTRUST_TAR: $(BRAINTRUST_TAR)"; \ + else \ + echo "==> Building SDK"; \ + cd ../../../../.. && pnpm exec turbo build --filter=braintrust && mkdir -p js/artifacts && pnpm --filter=braintrust pack --pack-destination js/artifacts; \ + for f in js/artifacts/braintrust-*.tgz; do \ + if [ "$$(basename $$f)" != "braintrust-latest.tgz" ] && \ + [ "$$(basename $$f)" != "braintrust-otel-latest.tgz" ] && \ + [ "$$(basename $$f)" != "braintrust-templates-nunjucks-js-latest.tgz" ]; then \ + cp "$$f" js/artifacts/braintrust-latest.tgz; \ + break; \ + fi; \ + done; \ + fi + + @# Build shared package (if not running from parent) + @if [ -z "$(SMOKE_V2_SHARED_DIST)" ]; then \ + echo "==> Building shared package"; \ + cd ../../../../../js/smoke/shared && npm install && npm run build; \ + fi + + @# Build @braintrust/templates-nunjucks-js package + @if [ -n "$(BRAINTRUST_TEMPLATES_NUNJUCKS_JS_TAR)" ]; then \ + echo "==> Using BRAINTRUST_TEMPLATES_NUNJUCKS_JS_TAR: $(BRAINTRUST_TEMPLATES_NUNJUCKS_JS_TAR)"; \ + else \ + echo "==> Building @braintrust/templates-nunjucks-js package"; \ + cd ../../../../.. && pnpm exec turbo build --filter=@braintrust/templates-nunjucks-js && cd integrations/templates-nunjucks && pnpm pack --pack-destination ../js/artifacts; \ + for f in ../../js/artifacts/braintrust-templates-nunjucks-*.tgz; do \ + if [ "$$(basename $$f)" != "braintrust-templates-nunjucks-js-latest.tgz" ]; then \ + cp "$$f" ../../js/artifacts/braintrust-templates-nunjucks-js-latest.tgz; \ + break; \ + fi; \ + done; \ + fi + + npm install --no-package-lock + +test: setup + @echo "==> Running jest tests" + @# Run only tests under the local tests/ directory to avoid workspace-wide discovery + npx jest --passWithNoTests --roots ./tests diff --git a/integrations/templates-nunjucks/smoke/scenarios/jest/package.json b/integrations/templates-nunjucks/smoke/scenarios/jest/package.json new file mode 100644 index 000000000..703b21d4d --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/jest/package.json @@ -0,0 +1,16 @@ +{ + "name": "smoke-templates-nunjucks-jest", + "private": true, + "type": "commonjs", + "dependencies": { + "braintrust": "file:../../../../../js/artifacts/braintrust-latest.tgz", + "@braintrust/templates-nunjucks-js": "file:../../../../../js/artifacts/braintrust-templates-nunjucks-js-latest.tgz" + }, + "devDependencies": { + "@types/jest": "^29.5.0", + "@types/node": "^20.10.5", + "jest": "^29.7.0", + "ts-jest": "^29.1.0", + "typescript": "^5.4.4" + } +} diff --git a/integrations/templates-nunjucks/smoke/scenarios/jest/tests/jest-basic.test.js b/integrations/templates-nunjucks/smoke/scenarios/jest/tests/jest-basic.test.js new file mode 100644 index 000000000..98aa16d55 --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/jest/tests/jest-basic.test.js @@ -0,0 +1,26 @@ +const { + runTests, + testMustacheTemplate, + testNunjucksTemplate, +} = require("../../../../../../js/smoke/shared/dist/index.js"); +const braintrust = require("braintrust"); +const { nunjucksPlugin } = require("@braintrust/templates-nunjucks-js"); + +test("Templates-Nunjucks basic behavior", async () => { + // Register nunjucks plugin before running tests + braintrust.registerTemplatePlugin(nunjucksPlugin); + + const { failed } = await runTests({ + name: "templates-nunjucks-jest", + braintrust, + tests: [testMustacheTemplate, testNunjucksTemplate], + skipCoverage: true, + }); + + if (failed.length > 0) { + const msg = failed + .map((f) => `${f.name}: ${f.error ?? "failed"}`) + .join("\n"); + throw new Error(`Found failures:\n${msg}`); + } +}); diff --git a/integrations/templates-nunjucks/smoke/scenarios/nextjs/.gitignore b/integrations/templates-nunjucks/smoke/scenarios/nextjs/.gitignore new file mode 100644 index 000000000..a3f58edb0 --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/nextjs/.gitignore @@ -0,0 +1,6 @@ +node_modules/ +.next/ +test-results/ +*.log +.env.local +package-lock.json diff --git a/integrations/templates-nunjucks/smoke/scenarios/nextjs/Makefile b/integrations/templates-nunjucks/smoke/scenarios/nextjs/Makefile new file mode 100644 index 000000000..3191aeadf --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/nextjs/Makefile @@ -0,0 +1,56 @@ +.PHONY: setup test + +setup: + @echo "==> Setting up templates-nunjucks nextjs scenario" + mise install + + @# Build SDK and shared package if not provided + @if [ -n "$(BRAINTRUST_TAR)" ]; then \ + echo "==> Using BRAINTRUST_TAR: $(BRAINTRUST_TAR)"; \ + else \ + echo "==> Building SDK"; \ + cd ../../../../.. && pnpm exec turbo build --filter=braintrust && mkdir -p js/artifacts && pnpm --filter=braintrust pack --pack-destination js/artifacts; \ + for f in js/artifacts/braintrust-*.tgz; do \ + if [ "$$(basename $$f)" != "braintrust-latest.tgz" ] && \ + [ "$$(basename $$f)" != "braintrust-otel-latest.tgz" ] && \ + [ "$$(basename $$f)" != "braintrust-templates-nunjucks-js-latest.tgz" ]; then \ + cp "$$f" js/artifacts/braintrust-latest.tgz; \ + break; \ + fi; \ + done; \ + fi + + @# Build shared package (if not running from parent) + @if [ -z "$(SMOKE_V2_SHARED_DIST)" ]; then \ + echo "==> Building shared package"; \ + cd ../../../../../js/smoke/shared && npm install && npm run build; \ + fi + + @# Build @braintrust/templates-nunjucks-js package + @if [ -n "$(BRAINTRUST_TEMPLATES_NUNJUCKS_JS_TAR)" ]; then \ + echo "==> Using BRAINTRUST_TEMPLATES_NUNJUCKS_JS_TAR: $(BRAINTRUST_TEMPLATES_NUNJUCKS_JS_TAR)"; \ + else \ + echo "==> Building @braintrust/templates-nunjucks-js package"; \ + cd ../../../../.. && \ + pnpm exec turbo build --filter=@braintrust/templates-nunjucks-js && \ + cd integrations/templates-nunjucks && \ + pnpm pack --pack-destination ../../js/artifacts && \ + cd ../../js/artifacts && \ + for f in braintrust-templates-nunjucks-*.tgz; do \ + if [ "$$(basename $$f)" != "braintrust-templates-nunjucks-js-latest.tgz" ]; then \ + cp "$$f" braintrust-templates-nunjucks-js-latest.tgz; \ + break; \ + fi; \ + done; \ + fi + + @echo "==> Cleaning and reinstalling node_modules to pick up new tarballs" + rm -rf node_modules package-lock.json .next + npm install --no-package-lock + +test: setup + @echo "==> Running Next.js integration test" + @echo "==> Building Next.js app" + npx next build + @echo "==> Starting Next.js dev server and running Playwright tests" + npx playwright test diff --git a/integrations/templates-nunjucks/smoke/scenarios/nextjs/next-env.d.ts b/integrations/templates-nunjucks/smoke/scenarios/nextjs/next-env.d.ts new file mode 100644 index 000000000..830fb594c --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/nextjs/next-env.d.ts @@ -0,0 +1,6 @@ +/// +/// +/// + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/integrations/templates-nunjucks/smoke/scenarios/nextjs/next.config.mjs b/integrations/templates-nunjucks/smoke/scenarios/nextjs/next.config.mjs new file mode 100644 index 000000000..4678774e6 --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/nextjs/next.config.mjs @@ -0,0 +1,4 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = {}; + +export default nextConfig; diff --git a/integrations/templates-nunjucks/smoke/scenarios/nextjs/package.json b/integrations/templates-nunjucks/smoke/scenarios/nextjs/package.json new file mode 100644 index 000000000..22c862740 --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/nextjs/package.json @@ -0,0 +1,17 @@ +{ + "name": "smoke-templates-nunjucks-nextjs", + "private": true, + "dependencies": { + "braintrust": "file:../../../../../js/artifacts/braintrust-latest.tgz", + "@braintrust/templates-nunjucks-js": "file:../../../../../js/artifacts/braintrust-templates-nunjucks-js-latest.tgz", + "next": "^15.1.4", + "react": "^19.0.0", + "react-dom": "^19.0.0" + }, + "devDependencies": { + "@playwright/test": "^1.49.1", + "@types/node": "^20.11.5", + "@types/react": "^18.2.48", + "typescript": "^5.3.3" + } +} diff --git a/integrations/templates-nunjucks/smoke/scenarios/nextjs/playwright.config.mts b/integrations/templates-nunjucks/smoke/scenarios/nextjs/playwright.config.mts new file mode 100644 index 000000000..ec75bdd0d --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/nextjs/playwright.config.mts @@ -0,0 +1,26 @@ +import { defineConfig, devices } from "@playwright/test"; +import path from "path"; +import { fileURLToPath } from "url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +export default defineConfig({ + testDir: "./tests", + timeout: 30000, + webServer: { + command: "npx next dev --port 3456", + port: 3456, + cwd: __dirname, + timeout: 120000, + reuseExistingServer: !process.env.CI, + }, + use: { + baseURL: "http://localhost:3456", + }, + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + ], +}); diff --git a/integrations/templates-nunjucks/smoke/scenarios/nextjs/src/app/api/test/route.ts b/integrations/templates-nunjucks/smoke/scenarios/nextjs/src/app/api/test/route.ts new file mode 100644 index 000000000..27827c73a --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/nextjs/src/app/api/test/route.ts @@ -0,0 +1,128 @@ +import { NextResponse } from "next/server"; +import { registerTemplatePlugin, Prompt } from "braintrust"; +import { nunjucksPlugin } from "@braintrust/templates-nunjucks-js"; + +export const runtime = "nodejs"; + +export async function GET() { + console.log("API route called - starting dynamic imports"); + try { + registerTemplatePlugin(nunjucksPlugin); + + const results = []; + + // Test 1: Basic nunjucks template rendering + const prompt = new Prompt( + { + id: "test-prompt-1", + _xact_id: "test-xact", + project_id: "test-project", + name: "nunjucks-nextjs-test", + slug: "nunjucks-nextjs-test", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: + "Items: {% for item in items %}{{ item.name }}{% if not loop.last %}, {% endif %}{% endfor %}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result = prompt.build( + { items: [{ name: "apple" }, { name: "banana" }, { name: "cherry" }] }, + { templateFormat: "nunjucks" }, + ); + if (result.messages[0]?.content === "Items: apple, banana, cherry") { + results.push({ + status: "pass", + name: "Nunjucks template loop rendering", + }); + } else { + results.push({ + status: "fail", + name: "Nunjucks template loop rendering", + error: { + message: `Expected "Items: apple, banana, cherry", got "${result.messages[0]?.content}"`, + }, + }); + } + + // Test 2: Conditional rendering + const conditionalPrompt = new Prompt( + { + id: "test-prompt-2", + _xact_id: "test-xact", + project_id: "test-project", + name: "nunjucks-conditional-test", + slug: "nunjucks-conditional-test", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: + "{% if showGreeting %}Hello, {{ name }}!{% else %}Goodbye!{% endif %}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const conditionalResult = conditionalPrompt.build( + { showGreeting: true, name: "World" }, + { templateFormat: "nunjucks" }, + ); + + if (conditionalResult.messages[0]?.content === "Hello, World!") { + results.push({ + status: "pass", + name: "Nunjucks conditional rendering", + }); + } else { + results.push({ + status: "fail", + name: "Nunjucks conditional rendering", + error: { + message: `Expected "Hello, World!", got "${conditionalResult.messages[0]?.content}"`, + }, + }); + } + + const failures = results.filter((r) => r.status === "fail"); + + return NextResponse.json( + { + success: failures.length === 0, + message: + failures.length > 0 + ? `${failures.length} test(s) failed` + : "All tests passed", + content: result.messages[0]?.content, + totalTests: results.length, + passedTests: results.filter((r) => r.status === "pass").length, + failedTests: failures.length, + results, + }, + { status: failures.length === 0 ? 200 : 500 }, + ); + } catch (error) { + return NextResponse.json( + { success: false, error: String(error) }, + { status: 500 }, + ); + } +} diff --git a/integrations/templates-nunjucks/smoke/scenarios/nextjs/src/app/layout.tsx b/integrations/templates-nunjucks/smoke/scenarios/nextjs/src/app/layout.tsx new file mode 100644 index 000000000..225b6038d --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/nextjs/src/app/layout.tsx @@ -0,0 +1,11 @@ +export default function RootLayout({ + children, +}: { + children: React.ReactNode; +}) { + return ( + + {children} + + ); +} diff --git a/integrations/templates-nunjucks/smoke/scenarios/nextjs/src/app/page.tsx b/integrations/templates-nunjucks/smoke/scenarios/nextjs/src/app/page.tsx new file mode 100644 index 000000000..cfaa25088 --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/nextjs/src/app/page.tsx @@ -0,0 +1,3 @@ +export default function Home() { + return
Templates-Nunjucks Next.js Test
; +} diff --git a/integrations/templates-nunjucks/smoke/scenarios/nextjs/tests/nextjs.test.ts b/integrations/templates-nunjucks/smoke/scenarios/nextjs/tests/nextjs.test.ts new file mode 100644 index 000000000..e08fe5ee0 --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/nextjs/tests/nextjs.test.ts @@ -0,0 +1,14 @@ +import { test, expect } from "@playwright/test"; + +test("Nunjucks template rendering in Next.js API route", async ({ + request, +}) => { + const response = await request.get("/api/test"); + + const text = await response.text(); + const data = JSON.parse(text); + + expect(response.ok()).toBeTruthy(); + expect(data.success).toBe(true); + expect(data.content).toBe("Items: apple, banana, cherry"); +}); diff --git a/integrations/templates-nunjucks/smoke/scenarios/nextjs/tsconfig.json b/integrations/templates-nunjucks/smoke/scenarios/nextjs/tsconfig.json new file mode 100644 index 000000000..f619795e7 --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/nextjs/tsconfig.json @@ -0,0 +1,35 @@ +{ + "compilerOptions": { + "target": "ES2017", + "lib": [ + "dom", + "dom.iterable", + "esnext" + ], + "allowJs": true, + "skipLibCheck": true, + "strict": false, + "noEmit": true, + "incremental": true, + "module": "esnext", + "esModuleInterop": true, + "moduleResolution": "node", + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": "preserve", + "plugins": [ + { + "name": "next" + } + ] + }, + "include": [ + "next-env.d.ts", + ".next/types/**/*.ts", + "**/*.ts", + "**/*.tsx" + ], + "exclude": [ + "node_modules" + ] +} diff --git a/integrations/templates-nunjucks/smoke/scenarios/node-esm/Makefile b/integrations/templates-nunjucks/smoke/scenarios/node-esm/Makefile new file mode 100644 index 000000000..4c705d23d --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/node-esm/Makefile @@ -0,0 +1,55 @@ +.PHONY: setup test + +# ============================================================================ +# Setup +# ============================================================================ + +setup: + @echo "==> Setting up templates-nunjucks scenario" + mise install + + @# Check if BRAINTRUST_TAR is set (from parent or CI), otherwise build + @if [ -n "$(BRAINTRUST_TAR)" ]; then \ + echo "==> Using BRAINTRUST_TAR: $(BRAINTRUST_TAR)"; \ + else \ + echo "==> Building SDK"; \ + cd ../../../../.. && pnpm exec turbo build --filter=braintrust && mkdir -p js/artifacts && pnpm --filter=braintrust pack --pack-destination js/artifacts; \ + for f in js/artifacts/braintrust-*.tgz; do \ + if [ "$$(basename $$f)" != "braintrust-latest.tgz" ] && \ + [ "$$(basename $$f)" != "braintrust-otel-latest.tgz" ] && \ + [ "$$(basename $$f)" != "braintrust-templates-nunjucks-latest.tgz" ]; then \ + cp "$$f" js/artifacts/braintrust-latest.tgz; \ + break; \ + fi; \ + done; \ + fi + + @# Build shared package (if not running from parent) + @if [ -z "$(SMOKE_V2_SHARED_DIST)" ]; then \ + echo "==> Building shared package"; \ + cd ../../../../../js/smoke/shared && npm install && npm run build; \ + fi + + @# Build @braintrust/templates-nunjucks-js package into js/artifacts + @if [ -n "$(BRAINTRUST_TEMPLATES_NUNJUCKS_JS_TAR)" ]; then \ + echo "==> Using BRAINTRUST_TEMPLATES_NUNJUCKS_JS_TAR: $(BRAINTRUST_TEMPLATES_NUNJUCKS_JS_TAR)"; \ + else \ + echo "==> Building @braintrust/templates-nunjucks-js package"; \ + cd ../../../../.. && pnpm exec turbo build --filter=@braintrust/templates-nunjucks-js && cd integrations/templates-nunjucks && pnpm pack --pack-destination ../js/artifacts; \ + for f in ../../js/artifacts/braintrust-templates-nunjucks-*.tgz; do \ + if [ "$$(basename $$f)" != "braintrust-templates-nunjucks-js-latest.tgz" ]; then \ + cp "$$f" ../../js/artifacts/braintrust-templates-nunjucks-js-latest.tgz; \ + break; \ + fi; \ + done; \ + fi + + npm install --no-package-lock + +# ============================================================================ +# Test +# ============================================================================ + +test: setup + @echo "==> Running templates-nunjucks tests" + npx tsx tests/basic.test.ts diff --git a/integrations/templates-nunjucks/smoke/scenarios/node-esm/package.json b/integrations/templates-nunjucks/smoke/scenarios/node-esm/package.json new file mode 100644 index 000000000..942dfbbf5 --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/node-esm/package.json @@ -0,0 +1,14 @@ +{ + "name": "smoke-templates-nunjucks-integration", + "private": true, + "type": "module", + "dependencies": { + "braintrust": "file:../../../../../js/artifacts/braintrust-latest.tgz", + "@braintrust/templates-nunjucks-js": "file:../../../../../js/artifacts/braintrust-templates-nunjucks-js-latest.tgz" + }, + "devDependencies": { + "@types/node": "^20.10.5", + "tsx": "^4.19.2", + "typescript": "^5.4.4" + } +} diff --git a/integrations/templates-nunjucks/smoke/scenarios/node-esm/tests/basic.test.ts b/integrations/templates-nunjucks/smoke/scenarios/node-esm/tests/basic.test.ts new file mode 100644 index 000000000..0e53dce22 --- /dev/null +++ b/integrations/templates-nunjucks/smoke/scenarios/node-esm/tests/basic.test.ts @@ -0,0 +1,28 @@ +import { + runTests, + testMustacheTemplate, + testNunjucksTemplate, +} from "../../../../../../js/smoke/shared/dist/index.mjs"; +import * as braintrust from "braintrust"; +import { nunjucksPlugin } from "@braintrust/templates-nunjucks-js"; + +async function main() { + // Register nunjucks plugin before running tests + braintrust.registerTemplatePlugin(nunjucksPlugin); + + const { failed } = await runTests({ + name: "templates-nunjucks-basic-render", + braintrust, + tests: [testMustacheTemplate, testNunjucksTemplate], + skipCoverage: true, + }); + + if (failed.length > 0) { + process.exit(1); + } +} + +main().catch((err) => { + console.error("Fatal error:", err); + process.exit(1); +}); diff --git a/integrations/templates-nunjucks/src/index.test.ts b/integrations/templates-nunjucks/src/index.test.ts new file mode 100644 index 000000000..60bee39c1 --- /dev/null +++ b/integrations/templates-nunjucks/src/index.test.ts @@ -0,0 +1,728 @@ +import { describe, test, expect, beforeAll } from "vitest"; +import { registerTemplatePlugin, Prompt } from "braintrust"; +import { nunjucksPlugin } from "./index"; + +// Register and activate the plugin for all tests +beforeAll(() => { + // registerTemplatePlugin will auto-activate using the plugin's + // `defaultOptions` when available. + registerTemplatePlugin(nunjucksPlugin); +}); + +describe("nunjucks rendering via Prompt", () => { + test("renders variable and control structures", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: + "Hello {{ name | upper }} {% if age > 18 %}Adult{% else %}Minor{% endif %}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result = prompt.build( + { name: "alice", age: 30 }, + { templateFormat: "nunjucks" }, + ); + expect(result.messages[0]?.content).toBe("Hello ALICE Adult"); + }); + + test("loops render", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: `{% for item in items %}{{ item }},{% endfor %}`, + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result = prompt.build( + { items: ["a", "b"] }, + { templateFormat: "nunjucks" }, + ); + expect(result.messages[0]?.content).toBe("a,b,"); + }); + + test("strict mode throws for missing top-level variable", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: "Hello {{ name }}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + expect(() => + prompt.build({ user: "x" }, { templateFormat: "nunjucks", strict: true }), + ).toThrow(); + }); + + test("strict mode passes for defined variable and filters", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: "Hello {{ name | upper }}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + expect(() => + prompt.build( + { name: "alice" }, + { templateFormat: "nunjucks", strict: true }, + ), + ).not.toThrow(); + }); + + test("strict mode: for over undefined is empty (does not throw)", () => { + const prompt1 = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: `{% for item in items %}{{ item }}{% endfor %}`, + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + expect(() => + prompt1.build( + { items: [1, 2, 3] }, + { templateFormat: "nunjucks", strict: true }, + ), + ).not.toThrow(); + + expect(() => + prompt1.build({}, { templateFormat: "nunjucks", strict: true }), + ).not.toThrow(); + }); + + test("strict mode: nested path with numeric index using brackets", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: `{{ user.addresses[2].city }}`, + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const ok = { user: { addresses: [{}, {}, { city: "SF" }] } }; + expect(() => + prompt.build(ok, { templateFormat: "nunjucks", strict: true }), + ).not.toThrow(); + + const bad = { user: {} }; + expect(() => + prompt.build(bad, { templateFormat: "nunjucks", strict: true }), + ).toThrow(); + }); + + test("renders nested object properties", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: "{{ user.profile.name }}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result = prompt.build( + { user: { profile: { name: "Alice" } } }, + { templateFormat: "nunjucks" }, + ); + expect(result.messages[0]?.content).toBe("Alice"); + }); + + test("renders multiple variables with context", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: + "{{ firstName }} {{ lastName }} is {{ age }} years old", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result = prompt.build( + { firstName: "Bob", lastName: "Smith", age: 25 }, + { templateFormat: "nunjucks" }, + ); + expect(result.messages[0]?.content).toBe("Bob Smith is 25 years old"); + }); + + test("renders with string concatenation", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: "{{ greeting ~ ' ' ~ name }}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result = prompt.build( + { greeting: "Hello", name: "World" }, + { templateFormat: "nunjucks" }, + ); + expect(result.messages[0]?.content).toBe("Hello World"); + }); + + test("renders numeric operations", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: "Total: {{ price * quantity }}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result = prompt.build( + { price: 10, quantity: 3 }, + { templateFormat: "nunjucks" }, + ); + expect(result.messages[0]?.content).toBe("Total: 30"); + }); + + test("renders with filters and context", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: "{{ message | upper | trim }}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result = prompt.build( + { message: " hello world " }, + { templateFormat: "nunjucks" }, + ); + expect(result.messages[0]?.content).toBe("HELLO WORLD"); + }); + + test("renders array elements with index", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: "First: {{ items[0] }}, Last: {{ items[2] }}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result = prompt.build( + { items: ["apple", "banana", "cherry"] }, + { templateFormat: "nunjucks" }, + ); + expect(result.messages[0]?.content).toBe("First: apple, Last: cherry"); + }); + + test("renders nested arrays and objects", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: "{{ users[0].name }} from {{ users[0].city }}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result = prompt.build( + { + users: [ + { name: "John", city: "NYC" }, + { name: "Jane", city: "LA" }, + ], + }, + { templateFormat: "nunjucks" }, + ); + expect(result.messages[0]?.content).toBe("John from NYC"); + }); + + test("renders with default filter", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: "{{ name | default('Guest') }}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result1 = prompt.build( + { name: "Alice" }, + { templateFormat: "nunjucks" }, + ); + expect(result1.messages[0]?.content).toBe("Alice"); + + const result2 = prompt.build({}, { templateFormat: "nunjucks" }); + expect(result2.messages[0]?.content).toBe("Guest"); + }); + + test("renders ternary expressions", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: "{{ user.name if user else 'Anonymous' }}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result1 = prompt.build( + { user: { name: "Alice" } }, + { templateFormat: "nunjucks" }, + ); + expect(result1.messages[0]?.content).toBe("Alice"); + + const result2 = prompt.build({}, { templateFormat: "nunjucks" }); + expect(result2.messages[0]?.content).toBe("Anonymous"); + }); + + test("renders with multiple filters chained", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: "{{ text | lower | trim }}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result = prompt.build( + { text: " HELLO WORLD " }, + { templateFormat: "nunjucks" }, + ); + expect(result.messages[0]?.content).toBe("hello world"); + }); + + test("renders complex nested context", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: + "{{ order.customer.name }} ordered {{ order.items[0].name }} for ${{ order.total }}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result = prompt.build( + { + order: { + customer: { name: "Alice" }, + items: [{ name: "Widget", price: 10 }], + total: 10, + }, + }, + { templateFormat: "nunjucks" }, + ); + expect(result.messages[0]?.content).toBe("Alice ordered Widget for $10"); + }); + + test("renders with length filter on arrays", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: "You have {{ items | length }} items", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result = prompt.build( + { items: ["a", "b", "c", "d"] }, + { templateFormat: "nunjucks" }, + ); + expect(result.messages[0]?.content).toBe("You have 4 items"); + }); + + test("renders with join filter", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: "{{ tags | join(', ') }}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result = prompt.build( + { tags: ["red", "green", "blue"] }, + { templateFormat: "nunjucks" }, + ); + expect(result.messages[0]?.content).toBe("red, green, blue"); + }); +}); + +describe("nunjucks linting", () => { + test("lint throws for missing variable", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: "Hello {{ user.name }}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + expect(() => + prompt.build({}, { templateFormat: "nunjucks", strict: true }), + ).toThrow(); + }); + + test("lint passes for valid template with loops", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: `{% for item in items %}{{ item }}{% endfor %}`, + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + expect(() => + prompt.build({}, { templateFormat: "nunjucks", strict: true }), + ).not.toThrow(); + }); + + test("lint passes for valid template with conditionals", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: `{% if user %}{{ user.name }}{% endif %}`, + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + expect(() => + prompt.build({}, { templateFormat: "nunjucks", strict: true }), + ).not.toThrow(); + }); + + test("lint throws for invalid template syntax", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: "{{ unclosed", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + expect(() => + prompt.build({}, { templateFormat: "nunjucks", strict: true }), + ).toThrow(); + }); + + test("lint throws for mismatched tags", () => { + const prompt = new Prompt( + { + name: "test-prompt", + slug: "test-prompt", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: "{% if x %}{% endfor %}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + expect(() => + prompt.build({}, { templateFormat: "nunjucks", strict: true }), + ).toThrow(); + }); +}); diff --git a/integrations/templates-nunjucks/src/index.ts b/integrations/templates-nunjucks/src/index.ts new file mode 100644 index 000000000..e88e1cb1e --- /dev/null +++ b/integrations/templates-nunjucks/src/index.ts @@ -0,0 +1,142 @@ +import * as nunjucks from "nunjucks"; + +import type { TemplateRendererPlugin } from "braintrust"; + +/** + * Configuration options for the Nunjucks template renderer. + * + * @example + * ```typescript + * import { registerTemplatePlugin } from "braintrust"; + * import { nunjucksPlugin, type NunjucksOptions } from "@braintrust/templates-nunjucks-js"; + * + * // Configure options before registering + * nunjucksPlugin.defaultOptions = { + * autoescape: true, + * throwOnUndefined: false + * } as NunjucksOptions; + * + * registerTemplatePlugin(nunjucksPlugin); + * ``` + */ +export interface NunjucksOptions { + /** + * Controls whether HTML escaping is enabled for template variables. + * + * When `true` (default), variables are automatically HTML-escaped to prevent XSS attacks. + * When `false`, variables are rendered as-is without escaping. + * + * @default true + * @example + * ```typescript + * // With autoescape enabled (default) + * nunjucksPlugin.defaultOptions = { autoescape: true }; + * // Template: "{{ html }}" + * // Variables: { html: "
Test
" } + * // Output: "<div>Test</div>" + * + * // With autoescape disabled + * nunjucksPlugin.defaultOptions = { autoescape: false }; + * // Output: "
Test
" + * ``` + */ + autoescape?: boolean; + + /** + * Controls whether undefined variables throw errors. + * + * When `true`, accessing undefined variables throws an error, making it easier to catch typos. + * When `false` (default), undefined variables render as empty strings. + * + * Note: When using `prompt.build()` with `strict: true`, this is always enabled for linting. + * + * @default false + * @example + * ```typescript + * // Lenient mode (default) - typos render as empty + * nunjucksPlugin.defaultOptions = { throwOnUndefined: false }; + * // Template: "Hello {{ userName }}" + * // Variables: { userName: "Alice" } + * // Output: "Hello Alice" + * + * // Strict mode - typos throw errors + * nunjucksPlugin.defaultOptions = { throwOnUndefined: true }; + * // Output: Error: Variable 'userName' is undefined + * ``` + */ + throwOnUndefined?: boolean; +} + +/** + * Nunjucks template renderer plugin for Braintrust. + * + * Provides support for Nunjucks/Jinja2-style templating in Braintrust prompts, + * including loops, conditionals, filters, and more. + * + * @example + * ```typescript + * import { registerTemplatePlugin, Prompt } from "braintrust"; + * import { nunjucksPlugin } from "@braintrust/templates-nunjucks-js"; + * + * // Register the plugin + * registerTemplatePlugin(nunjucksPlugin); + * + * // Use in prompts + * const prompt = new Prompt({ + * name: "example", + * slug: "example", + * prompt_data: { + * prompt: { + * type: "chat", + * messages: [{ + * role: "user", + * content: "{% for item in items %}{{ item }}{% if not loop.last %}, {% endif %}{% endfor %}" + * }] + * }, + * options: { model: "gpt-4" } + * } + * }, {}, false); + * + * const result = prompt.build( + * { items: ["apple", "banana", "cherry"] }, + * { templateFormat: "nunjucks" } + * ); + * // Output: "apple, banana, cherry" + * ``` + */ +export const nunjucksPlugin: TemplateRendererPlugin = { + name: "nunjucks", + defaultOptions: { + autoescape: true, + throwOnUndefined: false, + } as NunjucksOptions, + createRenderer() { + const opts = (this.defaultOptions ?? {}) as NunjucksOptions; + const autoescape = opts.autoescape ?? true; + const throwOnUndefined = opts.throwOnUndefined ?? false; + + const env = new nunjucks.Environment(null, { + autoescape, + throwOnUndefined, + }); + + const strictEnv = new nunjucks.Environment(null, { + autoescape: true, + throwOnUndefined: true, + }); + + return { + render( + template: string, + variables: Record, + _escape: (v: unknown) => string, + strict: boolean, + ) { + return (strict ? strictEnv : env).renderString(template, variables); + }, + lint(template: string, variables: Record) { + strictEnv.renderString(template, variables); + }, + }; + }, +}; diff --git a/integrations/templates-nunjucks/tsconfig.json b/integrations/templates-nunjucks/tsconfig.json new file mode 100644 index 000000000..47119abe2 --- /dev/null +++ b/integrations/templates-nunjucks/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "declaration": true, + "lib": ["es2022"], + "module": "esnext", + "target": "es2022", + "moduleResolution": "node", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true + }, + "include": ["src"] +} diff --git a/integrations/templates-nunjucks/tsup.config.ts b/integrations/templates-nunjucks/tsup.config.ts new file mode 100644 index 000000000..41099bc7a --- /dev/null +++ b/integrations/templates-nunjucks/tsup.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + format: ["cjs", "esm"], + outDir: "dist", + dts: true, + splitting: true, + clean: true, +}); diff --git a/integrations/templates-nunjucks/vitest.config.ts b/integrations/templates-nunjucks/vitest.config.ts new file mode 100644 index 000000000..73a26a648 --- /dev/null +++ b/integrations/templates-nunjucks/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["tests/**/*.test.ts", "src/**/*.test.ts"], + }, +}); diff --git a/integrations/temporal-js/vitest.config.ts b/integrations/temporal-js/vitest.config.ts index 77a73cf2e..73a26a648 100644 --- a/integrations/temporal-js/vitest.config.ts +++ b/integrations/temporal-js/vitest.config.ts @@ -1,5 +1,7 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ - test: {}, + test: { + include: ["tests/**/*.test.ts", "src/**/*.test.ts"], + }, }); diff --git a/integrations/vercel-ai-sdk/vitest.config.ts b/integrations/vercel-ai-sdk/vitest.config.ts index e69de29bb..73a26a648 100644 --- a/integrations/vercel-ai-sdk/vitest.config.ts +++ b/integrations/vercel-ai-sdk/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["tests/**/*.test.ts", "src/**/*.test.ts"], + }, +}); diff --git a/js/Makefile b/js/Makefile index 125beef2b..42ee5a23e 100644 --- a/js/Makefile +++ b/js/Makefile @@ -163,23 +163,10 @@ clean: SMOKE_DIR := smoke -# Run smoke tests (always prepares - relies on turbo/tsup caching for build efficiency) +# Run smoke tests - delegates to smoke/Makefile which handles auto-discovery and tarball creation # Usage: make test-smoke [TEST_NAME...] test-smoke: - @TESTS="$(filter-out test-smoke,$(MAKECMDGOALS))"; \ - SMOKE_ABS="$$(pwd)/$(SMOKE_DIR)"; \ - cd $(SMOKE_DIR) && ./prepare-tests.sh $$TESTS; \ - EXIT_CODE=0; \ - cd $$SMOKE_ABS && ./run-tests.sh $$TESTS || EXIT_CODE=$$?; \ - echo ""; \ - echo "Restoring package files..."; \ - cd $$SMOKE_ABS && \ - for dir in tests/*/; do \ - if [ -f "$$dir/package.json" ] && grep -q '"restore"' "$$dir/package.json" 2>/dev/null; then \ - (cd "$$dir" && npm run restore >/dev/null 2>&1 && echo " ✓ Restored $$(basename $$dir)") || true; \ - fi; \ - done; \ - exit $$EXIT_CODE + @cd $(SMOKE_DIR) && $(MAKE) test $(filter-out test-smoke,$(MAKECMDGOALS)) # Allow passing test names as make targets (they become no-ops) cloudflare-worker deno nextjs-instrumentation otel-v1 span span-jest: diff --git a/js/README.md b/js/README.md index 1b09f9b4b..e09fc1f9e 100644 --- a/js/README.md +++ b/js/README.md @@ -42,3 +42,22 @@ async function main() { main().catch(console.error); ``` + +### Browser Support + +**For browser-only applications, use the dedicated browser package:** + +```bash +npm install @braintrust/browser +``` + +The `@braintrust/browser` package is optimized for browser environments and includes the `als-browser` polyfill for AsyncLocalStorage support. It's a standalone package with no peer dependencies. + +**When to use each package:** + +- **`braintrust`** (this package) - For Node.js applications, full-stack frameworks (Next.js, etc.), and edge runtimes with native AsyncLocalStorage (Cloudflare Workers, Vercel Edge) +- **`@braintrust/browser`** - For browser-only applications that need AsyncLocalStorage support in standard browsers + +See the [@braintrust/browser README](../integrations/browser-js/README.md) for more details. + +**Breaking change in v3.0.0:** The `braintrust/browser` subpath export has been deprecated. Browser users should migrate to the `@braintrust/browser` package. diff --git a/js/deno.json b/js/deno.json new file mode 100644 index 000000000..353175f94 --- /dev/null +++ b/js/deno.json @@ -0,0 +1,6 @@ +{ + "name": "braintrust", + "exports": { + ".": "./dist/index.mjs" + } +} diff --git a/js/package.json b/js/package.json index b211d7cfb..b9d5c9f9a 100644 --- a/js/package.json +++ b/js/package.json @@ -1,7 +1,7 @@ { "name": "braintrust", - "version": "2.1.0", - "description": "SDK for integrating Braintrust", + "version": "3.0.0", + "description": "SDK for integrating Braintrust.", "repository": { "type": "git", "url": "git+https://github.com/braintrustdata/braintrust-sdk.git", @@ -33,12 +33,12 @@ "browser": "./dist/browser.mjs", "import": "./dist/index.mjs", "require": "./dist/index.js", - "default": "./dist/browser.mjs" + "default": "./dist/index.mjs" }, "./browser": { "import": "./dist/browser.mjs", - "module": "./dist/browser.mjs", - "require": "./dist/browser.js" + "require": "./dist/browser.js", + "default": "./dist/browser.mjs" }, "./node": { "types": "./dist/index.d.ts", @@ -132,7 +132,6 @@ "dependencies": { "@ai-sdk/provider": "^1.1.3", "@next/env": "^14.2.3", - "@types/nunjucks": "^3.2.6", "@vercel/functions": "^1.0.2", "argparse": "^2.0.1", "boxen": "^8.0.1", @@ -148,7 +147,6 @@ "http-errors": "^2.0.0", "minimatch": "^9.0.3", "mustache": "^4.2.0", - "nunjucks": "^3.2.4", "pluralize": "^8.0.0", "simple-git": "^3.21.0", "source-map": "^0.7.4", diff --git a/js/smoke/Makefile b/js/smoke/Makefile index effaa15e5..bcd11e472 100644 --- a/js/smoke/Makefile +++ b/js/smoke/Makefile @@ -1,36 +1,52 @@ -# Auto-discover scenarios (any folder in scenarios/ with a Makefile) -SCENARIOS := $(shell find scenarios -mindepth 1 -maxdepth 1 -type d -exec test -f {}/Makefile \; -print | sed 's|scenarios/||' | sort) +## Discover integration scenarios under ../../integrations +SCRIPTDIR := ../../integrations +LOCAL_MAKEFILES := $(shell find scenarios -type f -path 'scenarios/*/Makefile' -not -path '*/node_modules/*' 2>/dev/null) +INTEGRATION_MAKEFILES := $(shell find $(SCRIPTDIR) -type f -path '*/smoke/scenarios/*/Makefile' -not -path '*/node_modules/*' 2>/dev/null) +SCENARIO_MAKEFILES := $(LOCAL_MAKEFILES) $(INTEGRATION_MAKEFILES) +LOCAL_SCENARIOS := $(patsubst scenarios/%/Makefile,%,$(LOCAL_MAKEFILES)) +INTEGRATION_SCENARIOS := $(shell echo "$(INTEGRATION_MAKEFILES)" | tr ' ' '\n' | sed 's|$(SCRIPTDIR)/||' | sed 's|/smoke/scenarios/|/|' | sed 's|/Makefile$$||' | tr '\n' ' ') +SCENARIOS := $(LOCAL_SCENARIOS) $(INTEGRATION_SCENARIOS) -.PHONY: help setup test list clean +.PHONY: help setup test list clean discover -# ============================================================================= -# Help -# ============================================================================= +# Declare all scenarios as phony targets with empty recipes +# This prevents "No rule to make target" errors when running: make test scenario-name +.PHONY: $(LOCAL_SCENARIOS) $(INTEGRATION_SCENARIOS) +$(LOCAL_SCENARIOS) $(INTEGRATION_SCENARIOS): + @: # Empty recipe - actual test logic is in the test target + +## ============================================================================ +## Help +## ============================================================================ help: @echo "Smoke Test Infrastructure" @echo "" @echo "Available targets:" - @echo " make test - Run all scenarios (builds SDK once if needed)" - @echo " make test - Run specific scenario (e.g., make test otel-v1)" - @echo " make setup - Ensure SDK artifacts + shared package are ready" - @echo " make clean - Remove all build artifacts (force rebuild)" - @echo " make list - List discovered scenarios" + @echo " make test - Run all discovered integration scenarios" + @echo " make test - Run specific scenario (e.g., make test templates-nunjucks/basic-render)" + @echo " make setup - Ensure SDK artifacts + shared package are ready" + @echo " make clean - Remove all build artifacts (force rebuild)" + @echo " make list - List discovered scenarios" @echo "" @echo "Environment variables:" @echo " BRAINTRUST_TAR - Path to braintrust tarball (auto-set by parent when running locally)" + @echo " BRAINTRUST_BROWSER_TAR - Path to browser tarball (auto-set by parent when running locally)" @echo " BRAINTRUST_OTEL_TAR - Path to otel tarball (auto-set by parent when running locally)" @echo " SMOKE_V2_SHARED_DIST - Path to shared package dist (auto-set by parent when running locally)" @echo "" - @echo "Running individual scenarios:" - @echo " cd scenarios/otel-v1 && make test - Runs standalone, builds if needed" - @echo "" - @echo "Discovered scenarios:" - @for scenario in $(SCENARIOS); do echo " - $$scenario"; done + @echo "Discovered local scenarios:"; \ + for s in $(LOCAL_SCENARIOS); do \ + if [ -n "$$s" ]; then echo " - $$s"; fi; \ + done; \ + echo "Discovered integration scenarios:"; \ + for s in $(INTEGRATION_SCENARIOS); do \ + if [ -n "$$s" ]; then echo " - $$s"; fi; \ + done -# ============================================================================= -# Setup - Build SDK + shared package -# ============================================================================= +## ============================================================================ +## Setup - Build SDK + shared package +## ============================================================================ setup: @echo "==> Setting up smoke" @@ -42,16 +58,28 @@ setup: else \ echo "==> Building SDK and creating tarball"; \ cd ../../.. && pnpm exec turbo build --filter=braintrust && pnpm --filter=braintrust pack --pack-destination sdk/js/artifacts; \ - \ for f in sdk/js/artifacts/braintrust-*.tgz; do \ - if [ "$$(basename $$f)" != "braintrust-latest.tgz" ] && \ - [ "$$(basename $$f)" != "braintrust-otel-latest.tgz" ]; then \ + if [ "$$(basename $$f)" != "braintrust-latest.tgz" ] && [ "$$(basename $$f)" != "braintrust-otel-latest.tgz" ] && [ "$$(basename $$f)" != "braintrust-browser-latest.tgz" ]; then \ cp "$$f" sdk/js/artifacts/braintrust-latest.tgz; \ break; \ fi; \ done; \ fi + @# Build browser package and create tarball if not provided + @if [ -n "$(BRAINTRUST_BROWSER_TAR)" ] && [ -f "$(BRAINTRUST_BROWSER_TAR)" ]; then \ + echo "==> Using provided BRAINTRUST_BROWSER_TAR: $(BRAINTRUST_BROWSER_TAR)"; \ + else \ + echo "==> Building browser package and creating tarball"; \ + cd ../../.. && pnpm exec turbo build --filter=@braintrust/browser && pnpm --filter=@braintrust/browser pack --pack-destination sdk/js/artifacts; \ + for f in sdk/js/artifacts/braintrust-browser-*.tgz; do \ + if [ "$$(basename $$f)" != "braintrust-browser-latest.tgz" ]; then \ + cp "$$f" sdk/js/artifacts/braintrust-browser-latest.tgz; \ + break; \ + fi; \ + done; \ + fi + @# Build shared package if not provided @if [ -n "$(SMOKE_V2_SHARED_DIST)" ] && [ -d "$(SMOKE_V2_SHARED_DIST)" ]; then \ echo "==> Using provided SMOKE_V2_SHARED_DIST: $(SMOKE_V2_SHARED_DIST)"; \ @@ -64,52 +92,72 @@ setup: fi; \ fi -# ============================================================================= -# Clean - Remove all build artifacts -# ============================================================================= +## ============================================================================ +## Clean - Remove all build artifacts +## ============================================================================ clean: @echo "==> Cleaning smoke artifacts" rm -rf ../artifacts/*.tgz rm -rf shared/dist shared/node_modules -# ============================================================================= -# Test - Run scenarios -# ============================================================================= +## ============================================================================ +## Test - Run scenarios +## ============================================================================ + +# Support positional scenario argument: make test integration/name + test: - @REQUESTED="$(filter-out test,$(MAKECMDGOALS))"; \ - \ + @# Collect requested scenario names (strip any trailing /Makefile if provided by user) + @REQUESTED_RAW="$(filter-out test,$(MAKECMDGOALS))"; \ + REQUESTED="$$(echo $$REQUESTED_RAW | tr ' ' '\n' | sed 's|/Makefile$$||' | tr '\n' ' ' | xargs)"; \ if [ -z "$$REQUESTED" ]; then \ SCENARIOS_TO_RUN="$(SCENARIOS)"; \ - echo "==> Running all scenarios"; \ + echo "==> Running all discovered scenarios"; \ else \ SCENARIOS_TO_RUN=""; \ for scenario in $$REQUESTED; do \ - if [ ! -d "scenarios/$$scenario" ]; then \ - echo "Error: Scenario '$$scenario' not found"; \ + if echo "$$scenario" | grep -q '/'; then \ + integration=$$(echo $$scenario | cut -d/ -f1); \ + name=$$(echo $$scenario | cut -d/ -f2-); \ + dir="$(SCRIPTDIR)/$$integration/smoke/scenarios/$$name"; \ + else \ + dir="scenarios/$$scenario"; \ + fi; \ + if [ ! -d "$$dir" ]; then \ + echo "Error: Scenario '$$scenario' not found at $$dir"; \ echo "Available scenarios:"; \ - for s in $(SCENARIOS); do echo " - $$s"; done; \ + for s in $(LOCAL_SCENARIOS) $(INTEGRATION_SCENARIOS); do \ + echo " - $$s"; \ + done; \ exit 1; \ fi; \ - SCENARIOS_TO_RUN="$$SCENARIOS_TO_RUN $$scenario"; \ + SCENARIOS_TO_RUN="$$SCENARIOS_TO_RUN $$dir"; \ done; \ - echo "==> Running scenarios:$$SCENARIOS_TO_RUN"; \ + echo "==> Running scenarios: $$REQUESTED"; \ fi; \ \ $(MAKE) setup; \ \ : $${BRAINTRUST_TAR:=../artifacts/braintrust-latest.tgz}; \ + : $${BRAINTRUST_BROWSER_TAR:=../artifacts/braintrust-browser-latest.tgz}; \ : $${BRAINTRUST_OTEL_TAR:=../artifacts/braintrust-otel-latest.tgz}; \ : $${SMOKE_V2_SHARED_DIST:=shared/dist}; \ - export BRAINTRUST_TAR BRAINTRUST_OTEL_TAR SMOKE_V2_SHARED_DIST; \ + export BRAINTRUST_TAR BRAINTRUST_BROWSER_TAR BRAINTRUST_OTEL_TAR SMOKE_V2_SHARED_DIST; \ \ FAILED_SCENARIOS=""; \ for scenario in $$SCENARIOS_TO_RUN; do \ echo ""; \ echo "=== Testing $$scenario ==="; \ - if ! $(MAKE) -C scenarios/$$scenario test; then \ - FAILED_SCENARIOS="$$FAILED_SCENARIOS $$scenario"; \ + if [ -f "$$scenario/Makefile" ]; then \ + if ! $(MAKE) -C $$scenario test; then \ + FAILED_SCENARIOS="$$FAILED_SCENARIOS $$(echo $$scenario | sed 's|$(SCRIPTDIR)/||' | sed 's|/scenarios/|/|')"; \ + fi; \ + elif [ -f "$$scenario/test.js" ]; then \ + ( cd $$scenario && node test.js ) || FAILED_SCENARIOS="$$FAILED_SCENARIOS $$(echo $$scenario | sed 's|$(SCRIPTDIR)/||' | sed 's|/scenarios/|/|')"; \ + else \ + FAILED_SCENARIOS="$$FAILED_SCENARIOS $$(echo $$scenario | sed 's|$(SCRIPTDIR)/||' | sed 's|/scenarios/|/|')"; \ fi; \ done; \ \ @@ -121,17 +169,11 @@ test: echo "✓ All requested scenarios passed"; \ fi -# ============================================================================= -# List - Show discovered scenarios -# ============================================================================= +## ============================================================================ +## List - Show discovered scenarios +## ============================================================================ list: - @echo "Discovered scenarios:" - @for scenario in $(SCENARIOS); do echo " - $$scenario"; done - -# ============================================================================= -# Make scenario names valid targets (no-ops to support "make test otel-v1") -# ============================================================================= - -$(SCENARIOS): - @: + @echo "Discovered scenarios:"; \ + for s in $(SCENARIOS); do echo " - $$s"; done + @echo '' diff --git a/js/smoke/README.md b/js/smoke/README.md index f0b52cec5..89b725452 100644 --- a/js/smoke/README.md +++ b/js/smoke/README.md @@ -5,9 +5,18 @@ Smoke test infrastructure verifying SDK installation across different runtimes a ## Quick Reference ```bash -make test # Run all scenarios (doesn't exit early on failures) -make test otel-v1 # Run specific scenario -make list # List available scenarios +# From sdk/js/: +make test-smoke # Run all scenarios + +# From sdk/js/smoke/: +make test # Run all scenarios (local + integration) +make test deno-node # Run specific local scenario +make test templates-nunjucks/jest # Run integration scenario +make list # List all discovered scenarios + +# From a specific scenario: +cd scenarios/deno-node +make test # Auto-creates tarball if needed ``` ## Output Standardization (REQUIRED) @@ -182,6 +191,72 @@ test: setup This ensures all tests run and display their results, but the suite still fails if any test failed. +## Environment Variables & Tarball Creation + +### Automatic Tarball Creation + +Scenarios automatically create tarballs if they're not provided via environment variables. This happens during the `setup` target: + +1. **Check for existing tarball**: If `BRAINTRUST_TAR` env var is set and points to a valid file, use it +2. **Build from source**: Otherwise, build SDK and create tarball: + + ```bash + cd ../../.. && pnpm exec turbo build --filter=braintrust && \ + mkdir-p artifacts && pnpm pack --pack-destination artifacts + ``` + +3. **Rename to well-known path**: Copy to `braintrust-latest.tgz` for consistent references + +### Environment Variables + +- **`BRAINTRUST_TAR`**: Path to braintrust tarball (auto-created if not set) + + - Example: `../artifacts/braintrust-latest.tgz` + +- **`BRAINTRUST_OTEL_TAR`**: Path to @braintrust/otel tarball (auto-created for scenarios that need it) + + - Example: `../artifacts/braintrust-otel-latest.tgz` + +- **`BRAINTRUST_TEMPLATES_NUNJUCKS_JS_TAR`**: Path to @braintrust/templates-nunjucks-js tarball + + - Example: `../artifacts/braintrust-templates-nunjucks-js-latest.tgz` + +- **`SMOKE_V2_SHARED_DIST`**: Path to shared test utilities (auto-built if not set) + - Example: `shared/dist` + +### Local vs CI Execution + +**Locally:** + +- Tarballs auto-created on first run +- Cached for subsequent runs (fast) +- Run from any level: `sdk/js/`, `sdk/js/smoke/`, or scenario directory + +**In CI** (`.github/workflows/js.yaml`): + +1. **Build job**: Creates all tarballs (braintrust, otel, templates-nunjucks), uploads as artifacts +2. **Smoke-discover job**: Auto-discovers all scenarios (local + integration) +3. **Smoke-test job**: Downloads tarballs, runs all scenarios in parallel with `fail-fast: false` + +## Integration Scenarios + +Integration scenarios test SDK integrations and are located under `../../integrations/*/scenarios/`: + +### Templates-Nunjucks + +- **templates-nunjucks/basic-render**: Basic Nunjucks template rendering +- **templates-nunjucks/deno**: Deno runtime with Nunjucks +- **templates-nunjucks/jest**: Jest test runner with Nunjucks +- **templates-nunjucks/nextjs**: Next.js integration with Nunjucks + +### OpenTelemetry (OTel) + +- **otel-js/otel-v1**: OpenTelemetry smoke test scenario (using shared test suite) + +Note: The `otel-js/otel-v1` and `otel-js/otel-v2` directories at the root of `integrations/otel-js/` are integration test environments (not smoke tests) and use a different test runner (vitest). + +All integration scenarios are automatically discovered and run alongside local scenarios. + ## Design Principles ### Well-Known Tarball Paths diff --git a/js/smoke/scenarios/browser-main-package/.gitignore b/js/smoke/scenarios/browser-main-package/.gitignore new file mode 100644 index 000000000..2bbe28c25 --- /dev/null +++ b/js/smoke/scenarios/browser-main-package/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +dist/ +package-lock.json +test-results/ +playwright-report/ diff --git a/js/smoke/scenarios/browser-main-package/Makefile b/js/smoke/scenarios/browser-main-package/Makefile new file mode 100644 index 000000000..704f7aa07 --- /dev/null +++ b/js/smoke/scenarios/browser-main-package/Makefile @@ -0,0 +1,14 @@ +.PHONY: install build test clean + +install: + npm install + npx playwright install --with-deps chromium + +build: install + node esbuild.config.mjs + +test: build + npx playwright test + +clean: + rm -rf node_modules dist package-lock.json diff --git a/js/smoke/scenarios/browser-main-package/README.md b/js/smoke/scenarios/browser-main-package/README.md new file mode 100644 index 000000000..581f3fbe7 --- /dev/null +++ b/js/smoke/scenarios/browser-main-package/README.md @@ -0,0 +1,64 @@ +# Browser Main Package Smoke Test + +This smoke test verifies that the informational message appears when using the browser build from the main `braintrust` package. + +## What This Tests + +When a user imports from the main `braintrust` package in a browser environment: + +```typescript +import * as braintrust from "braintrust"; +``` + +The bundler (via the `"browser"` field in package.json) will resolve to the browser build (`dist/browser.mjs`), which should: + +1. Show an informational console message suggesting `@braintrust/browser` for optimal use +2. Provide working browser-safe implementations +3. Not include Node.js modules + +## Test Structure + +- **src/browser-message-test.ts** - Browser test script that imports from main package +- **pages/browser-message-test.html** - HTML page to run the test +- **tests/browser-message.test.ts** - Playwright test that verifies the message + +## Running the Test + +```bash +make test +``` + +Or step by step: + +```bash +# Install dependencies +make install + +# Build the test bundle +make build + +# Run Playwright tests +npx playwright test +``` + +## What Gets Verified + +✓ Import from main package works in browser +✓ Basic functions are available (init, newId, traceable) +✓ Informational message appears in console +✓ Message mentions "@braintrust/browser" package +✓ No Node.js module errors + +## Expected Console Output + +When the test runs, you should see: + +``` +Braintrust SDK Browser Build +You are using a browser-compatible build from the main package. +For optimal browser support consider: + npm install @braintrust/browser + import * as braintrust from "@braintrust/browser" +``` + +This message guides users toward the optimized `@braintrust/browser` package while ensuring the main package works correctly in browsers. diff --git a/js/smoke/scenarios/browser-main-package/esbuild.config.mjs b/js/smoke/scenarios/browser-main-package/esbuild.config.mjs new file mode 100644 index 000000000..681330c31 --- /dev/null +++ b/js/smoke/scenarios/browser-main-package/esbuild.config.mjs @@ -0,0 +1,18 @@ +import esbuild from "esbuild"; +import { rmSync } from "node:fs"; + +rmSync("dist", { recursive: true, force: true }); + +await esbuild.build({ + entryPoints: ["src/browser-message-test.ts"], + bundle: true, + format: "esm", + outdir: "dist", + platform: "browser", + target: "es2022", + sourcemap: true, + mainFields: ["browser", "module", "main"], + external: [], +}); + +console.log("Build complete!"); diff --git a/js/smoke/scenarios/browser-main-package/mise.toml b/js/smoke/scenarios/browser-main-package/mise.toml new file mode 100644 index 000000000..6a0493c69 --- /dev/null +++ b/js/smoke/scenarios/browser-main-package/mise.toml @@ -0,0 +1,2 @@ +[tools] +node = "22" diff --git a/js/smoke/scenarios/browser-main-package/package.json b/js/smoke/scenarios/browser-main-package/package.json new file mode 100644 index 000000000..bd3bc537a --- /dev/null +++ b/js/smoke/scenarios/browser-main-package/package.json @@ -0,0 +1,18 @@ +{ + "name": "smoke-browser-main-package", + "version": "1.0.0", + "private": true, + "type": "module", + "description": "Smoke test for main package browser build with informational message", + "dependencies": { + "braintrust": "file:../../../artifacts/braintrust-latest.tgz", + "zod": "^4.3.5" + }, + "devDependencies": { + "@playwright/test": "^1.51.1", + "@types/node": "^20.10.5", + "esbuild": "^0.27.2", + "http-server": "^14.1.1", + "typescript": "^5.4.4" + } +} diff --git a/js/smoke/scenarios/browser-main-package/pages/browser-message-test.html b/js/smoke/scenarios/browser-main-package/pages/browser-message-test.html new file mode 100644 index 000000000..3f23e1f2e --- /dev/null +++ b/js/smoke/scenarios/browser-main-package/pages/browser-message-test.html @@ -0,0 +1,17 @@ + + + + + + Braintrust Main Package Browser Build Test + + +

Braintrust Main Package Browser Build Test

+

+ This test verifies the informational message appears when using the + browser build from the main package. +

+
+ + + diff --git a/js/smoke/scenarios/browser-main-package/playwright.config.ts b/js/smoke/scenarios/browser-main-package/playwright.config.ts new file mode 100644 index 000000000..3cce5c28d --- /dev/null +++ b/js/smoke/scenarios/browser-main-package/playwright.config.ts @@ -0,0 +1,26 @@ +import { defineConfig, devices } from "@playwright/test"; + +export default defineConfig({ + testDir: "./tests", + fullyParallel: false, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: 1, + reporter: "list", + use: { + baseURL: "http://localhost:8765", + trace: "on-first-retry", + }, + projects: [ + { + name: "chromium", + use: { ...devices["Desktop Chrome"] }, + }, + ], + webServer: { + command: "npx http-server -p 8765", + port: 8765, + reuseExistingServer: !process.env.CI, + timeout: 10000, + }, +}); diff --git a/js/smoke/scenarios/browser-main-package/src/browser-message-test.ts b/js/smoke/scenarios/browser-main-package/src/browser-message-test.ts new file mode 100644 index 000000000..dd728e468 --- /dev/null +++ b/js/smoke/scenarios/browser-main-package/src/browser-message-test.ts @@ -0,0 +1,58 @@ +declare global { + interface Window { + __btBrowserMessageTest?: { + completed: boolean; + consoleMessages: string[]; + importSuccessful: boolean; + hasInit: boolean; + hasNewId: boolean; + hasTraceable: boolean; + }; + } +} + +// Capture console.info messages BEFORE importing braintrust +const capturedMessages: string[] = []; +const originalConsoleInfo = console.info; +console.info = (...args: any[]) => { + const message = args.join(" "); + capturedMessages.push(message); + originalConsoleInfo.apply(console, args); +}; + +// Import from main package browser export AFTER setting up console capture +// This must be done dynamically to ensure console.info is overridden first +const braintrust = await import("braintrust/browser"); + +// Test that imports work +const importSuccessful = true; +const hasInit = typeof braintrust.init === "function"; +const hasNewId = typeof braintrust.newId === "function"; +const hasTraceable = typeof braintrust.traceable === "function"; + +// Store results +window.__btBrowserMessageTest = { + completed: true, + consoleMessages: capturedMessages, + importSuccessful, + hasInit, + hasNewId, + hasTraceable, +}; + +// Display results +const output = document.getElementById("output"); +if (output) { + output.innerHTML = ` +

Test Results

+
    +
  • Import successful: ${importSuccessful ? "✓" : "✗"}
  • +
  • Has init function: ${hasInit ? "✓" : "✗"}
  • +
  • Has newId function: ${hasNewId ? "✓" : "✗"}
  • +
  • Has traceable function: ${hasTraceable ? "✓" : "✗"}
  • +
  • Console messages captured: ${capturedMessages.length}
  • +
+

Console Messages:

+
${capturedMessages.join("\n")}
+ `; +} diff --git a/js/smoke/scenarios/browser-main-package/tests/browser-message.test.ts b/js/smoke/scenarios/browser-main-package/tests/browser-message.test.ts new file mode 100644 index 000000000..d815136e3 --- /dev/null +++ b/js/smoke/scenarios/browser-main-package/tests/browser-message.test.ts @@ -0,0 +1,88 @@ +import { test, expect } from "@playwright/test"; + +test.describe("Braintrust Main Package Browser Build", () => { + test("should display informational message when using browser build", async ({ + page, + baseURL, + }) => { + const consoleMessages: string[] = []; + + // Capture console.info messages + page.on("console", (msg) => { + if (msg.type() === "info") { + consoleMessages.push(msg.text()); + } else if (msg.type() === "error") { + console.error(`[Browser Console Error] ${msg.text()}`); + } + }); + + page.on("pageerror", (error) => { + console.error("[Browser Page Error]", error.message); + }); + + // Load the test page + const response = await page.goto( + `${baseURL}/pages/browser-message-test.html`, + { + waitUntil: "domcontentloaded", + timeout: 10000, + }, + ); + + if (!response || !response.ok()) { + throw new Error(`Failed to load page: ${response?.status()}`); + } + + // Wait for test to complete + await page.waitForFunction( + () => { + return (window as any).__btBrowserMessageTest?.completed === true; + }, + { timeout: 10000 }, + ); + + // Get test results + const testResults = await page.evaluate( + () => (window as any).__btBrowserMessageTest, + ); + + // Log results for debugging + console.log("Test Results:", testResults); + console.log("Captured Console Messages:", consoleMessages); + + // Assertions + expect(testResults).toBeTruthy(); + expect(testResults.completed).toBe(true); + expect(testResults.importSuccessful).toBe(true); + expect(testResults.hasInit).toBe(true); + expect(testResults.hasNewId).toBe(true); + expect(testResults.hasTraceable).toBe(true); + + // Expected message from browser-config.ts + const expectedMessage = + "This entrypoint is no longer supported.\n\n" + + "You should be using entrypoints:\n\n" + + "- `/workerd` (cloudflare envs)\n" + + "- `/edge-light` (next-js or other edge envs)\n\n" + + "If you'd like to use braintrust in the browser use the dedicated package: @braintrust/browser\n"; + + // Verify the full informational message appears in console + const hasInformationalMessage = consoleMessages.some( + (msg) => msg === expectedMessage, + ); + + expect(hasInformationalMessage).toBe(true); + + // Also verify from captured messages in the page + const hasCapturedMessage = testResults.consoleMessages.some( + (msg: string) => msg === expectedMessage, + ); + + expect(hasCapturedMessage).toBe(true); + + console.log("✓ Complete informational message verified"); + console.log( + "✓ Main package browser build working correctly with full message", + ); + }); +}); diff --git a/js/smoke/scenarios/browser-main-package/tsconfig.json b/js/smoke/scenarios/browser-main-package/tsconfig.json new file mode 100644 index 000000000..d1677290d --- /dev/null +++ b/js/smoke/scenarios/browser-main-package/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "lib": ["ES2022", "DOM"], + "moduleResolution": "bundler", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true + }, + "include": ["src/**/*", "tests/**/*"] +} diff --git a/js/smoke/scenarios/cloudflare-vite-hono-vite-dev-node-esm/src/worker-node-esm.ts b/js/smoke/scenarios/cloudflare-vite-hono-vite-dev-node-esm/src/worker-node-esm.ts index 90f59075c..bfe08e49b 100644 --- a/js/smoke/scenarios/cloudflare-vite-hono-vite-dev-node-esm/src/worker-node-esm.ts +++ b/js/smoke/scenarios/cloudflare-vite-hono-vite-dev-node-esm/src/worker-node-esm.ts @@ -78,8 +78,9 @@ app.get("/api/test", async (c) => { testMustacheTemplate, expectFailure( testNunjucksTemplate, - (e) => e.message.includes("Disallowed in this environment"), - "Cloudflare Workers blocks dynamic code generation (eval/Function)", + (e: { message: string }) => + e.message.includes("requires @braintrust/template-nunjucks"), + "Nunjucks requires separate package", ), ], }); diff --git a/js/smoke/scenarios/cloudflare-vite-hono-vite-dev/package.json b/js/smoke/scenarios/cloudflare-vite-hono-vite-dev/package.json index 272211cad..8b0168d16 100644 --- a/js/smoke/scenarios/cloudflare-vite-hono-vite-dev/package.json +++ b/js/smoke/scenarios/cloudflare-vite-hono-vite-dev/package.json @@ -4,6 +4,7 @@ "type": "module", "dependencies": { "braintrust": "file:../../../artifacts/braintrust-latest.tgz", + "@braintrust/browser": "file:../../../artifacts/braintrust-browser-latest.tgz", "hono": "^4.11.1", "react": "^19.2.1", "react-dom": "^19.2.1", diff --git a/js/smoke/scenarios/cloudflare-vite-hono-vite-dev/src/worker.ts b/js/smoke/scenarios/cloudflare-vite-hono-vite-dev/src/worker.ts index 29fbbbd41..887aa1d0d 100644 --- a/js/smoke/scenarios/cloudflare-vite-hono-vite-dev/src/worker.ts +++ b/js/smoke/scenarios/cloudflare-vite-hono-vite-dev/src/worker.ts @@ -28,7 +28,7 @@ import { testEvalSmoke, } from "../../../shared"; -import * as braintrust from "braintrust"; +import * as braintrust from "@braintrust/browser"; const app = new Hono<{ Bindings: Env }>(); @@ -73,8 +73,9 @@ app.get("/api/test", async (c) => { testMustacheTemplate, expectFailure( testNunjucksTemplate, - (e) => e.message.includes("Nunjucks templating is not supported"), - "Nunjucks not supported in browser build", + (e: { message: string }) => + e.message.includes("requires @braintrust/template-nunjucks"), + "Nunjucks requires separate package", ), ], }); diff --git a/js/smoke/scenarios/cloudflare-vite-hono-wrangler-dev/package.json b/js/smoke/scenarios/cloudflare-vite-hono-wrangler-dev/package.json index c0edc8625..9a8899a7a 100644 --- a/js/smoke/scenarios/cloudflare-vite-hono-wrangler-dev/package.json +++ b/js/smoke/scenarios/cloudflare-vite-hono-wrangler-dev/package.json @@ -4,6 +4,7 @@ "type": "module", "dependencies": { "braintrust": "file:../../../artifacts/braintrust-latest.tgz", + "@braintrust/browser": "file:../../../artifacts/braintrust-browser-latest.tgz", "hono": "^4.11.1", "zod": "^3.25.76" }, diff --git a/js/smoke/scenarios/cloudflare-vite-hono-wrangler-dev/src/worker.ts b/js/smoke/scenarios/cloudflare-vite-hono-wrangler-dev/src/worker.ts index c440196ba..33b9cfe33 100644 --- a/js/smoke/scenarios/cloudflare-vite-hono-wrangler-dev/src/worker.ts +++ b/js/smoke/scenarios/cloudflare-vite-hono-wrangler-dev/src/worker.ts @@ -28,7 +28,7 @@ import { testEvalSmoke, } from "../../../shared"; -import * as braintrust from "braintrust"; +import * as braintrust from "@braintrust/browser"; const app = new Hono<{ Bindings: Env }>(); @@ -73,8 +73,9 @@ app.get("/api/test", async (c) => { testMustacheTemplate, expectFailure( testNunjucksTemplate, - (e) => e.message.includes("Nunjucks templating is not supported"), - "Nunjucks not supported in browser build", + (e: { message: string }) => + e.message.includes("requires @braintrust/template-nunjucks"), + "Nunjucks requires separate package", ), ], }); diff --git a/js/smoke/scenarios/cloudflare-worker-browser-compat/package.json b/js/smoke/scenarios/cloudflare-worker-browser-compat/package.json index c39db0404..65ba9854f 100644 --- a/js/smoke/scenarios/cloudflare-worker-browser-compat/package.json +++ b/js/smoke/scenarios/cloudflare-worker-browser-compat/package.json @@ -4,6 +4,7 @@ "type": "module", "dependencies": { "braintrust": "file:../../../artifacts/braintrust-latest.tgz", + "@braintrust/browser": "file:../../../artifacts/braintrust-browser-latest.tgz", "zod": "^3.25.76" }, "devDependencies": { diff --git a/js/smoke/scenarios/cloudflare-worker-browser-compat/src/worker.ts b/js/smoke/scenarios/cloudflare-worker-browser-compat/src/worker.ts index a80533a28..28d7d810a 100644 --- a/js/smoke/scenarios/cloudflare-worker-browser-compat/src/worker.ts +++ b/js/smoke/scenarios/cloudflare-worker-browser-compat/src/worker.ts @@ -1,4 +1,4 @@ -import * as braintrust from "braintrust"; +import * as braintrust from "@braintrust/browser"; import { runTests, expectFailure, @@ -64,8 +64,9 @@ export default { testMustacheTemplate, expectFailure( testNunjucksTemplate, - (e) => e.message.includes("String template rendering. Disallowed"), - "Nunjucks evals not supported", + (e: { message: string }) => + e.message.includes("requires @braintrust/template-nunjucks"), + "Nunjucks requires separate package", ), ], }); diff --git a/js/smoke/scenarios/cloudflare-worker-browser-no-compat/package.json b/js/smoke/scenarios/cloudflare-worker-browser-no-compat/package.json index 5e8ae393f..438b44a00 100644 --- a/js/smoke/scenarios/cloudflare-worker-browser-no-compat/package.json +++ b/js/smoke/scenarios/cloudflare-worker-browser-no-compat/package.json @@ -4,6 +4,7 @@ "type": "module", "dependencies": { "braintrust": "file:../../../artifacts/braintrust-latest.tgz", + "@braintrust/browser": "file:../../../artifacts/braintrust-browser-latest.tgz", "zod": "^3.25.76" }, "devDependencies": { diff --git a/js/smoke/scenarios/cloudflare-worker-browser-no-compat/src/worker.ts b/js/smoke/scenarios/cloudflare-worker-browser-no-compat/src/worker.ts index 56e8c1cc1..328496ebb 100644 --- a/js/smoke/scenarios/cloudflare-worker-browser-no-compat/src/worker.ts +++ b/js/smoke/scenarios/cloudflare-worker-browser-no-compat/src/worker.ts @@ -1,4 +1,4 @@ -import * as braintrust from "braintrust"; +import * as braintrust from "@braintrust/browser"; import { runTests, expectFailure, @@ -64,8 +64,9 @@ export default { testMustacheTemplate, expectFailure( testNunjucksTemplate, - (e) => e.message.includes("Nunjucks templating is not supported"), - "Nunjucks not supported in browser build", + (e: { message: string }) => + e.message.includes("requires @braintrust/template-nunjucks"), + "Nunjucks requires separate package", ), ], }); diff --git a/js/smoke/scenarios/cloudflare-worker-node-compat/src/worker.ts b/js/smoke/scenarios/cloudflare-worker-node-compat/src/worker.ts index 17c8e186e..6fd2f7f01 100644 --- a/js/smoke/scenarios/cloudflare-worker-node-compat/src/worker.ts +++ b/js/smoke/scenarios/cloudflare-worker-node-compat/src/worker.ts @@ -64,8 +64,9 @@ export default { testMustacheTemplate, expectFailure( testNunjucksTemplate, - (e) => e.message.includes("Disallowed in this environment"), - "Cloudflare Workers blocks dynamic code generation (eval/Function)", + (e: { message: string }) => + e.message.includes("requires @braintrust/template-nunjucks"), + "Nunjucks requires separate package", ), ], }); diff --git a/js/smoke/scenarios/cloudflare-worker-node-no-compat/src/worker.ts b/js/smoke/scenarios/cloudflare-worker-node-no-compat/src/worker.ts index 8090a2fce..86a08c0cf 100644 --- a/js/smoke/scenarios/cloudflare-worker-node-no-compat/src/worker.ts +++ b/js/smoke/scenarios/cloudflare-worker-node-no-compat/src/worker.ts @@ -1,6 +1,7 @@ import * as braintrust from "braintrust/node"; import { runTests, + expectFailure, testBasicSpanLogging, testMultipleSpans, testDirectLogging, @@ -61,7 +62,12 @@ export default { testCurrentSpan, testEvalSmoke, testMustacheTemplate, - testNunjucksTemplate, + expectFailure( + testNunjucksTemplate, + (e: { message: string }) => + e.message.includes("requires @braintrust/template-nunjucks"), + "Nunjucks requires separate package", + ), ], }); diff --git a/js/smoke/scenarios/deno-browser/Makefile b/js/smoke/scenarios/deno-browser/Makefile index 09ad4910d..8c95c70db 100644 --- a/js/smoke/scenarios/deno-browser/Makefile +++ b/js/smoke/scenarios/deno-browser/Makefile @@ -15,4 +15,5 @@ setup: fi; \ test: setup - deno test --sloppy-imports --allow-all tests/*.test.ts + @# Run only tests under the local tests/ directory + mise exec -- deno test --sloppy-imports --allow-all tests/*.test.ts diff --git a/js/smoke/scenarios/deno-browser/deno.json b/js/smoke/scenarios/deno-browser/deno.json index ab612b4d0..db0573ac5 100644 --- a/js/smoke/scenarios/deno-browser/deno.json +++ b/js/smoke/scenarios/deno-browser/deno.json @@ -2,7 +2,7 @@ "imports": { "@std/assert": "jsr:@std/assert@^1.0.14", "@braintrust/smoke-test-shared": "jsr:@braintrust/smoke-test-shared", - "braintrust": "npm:braintrust/browser" + "@braintrust/browser": "npm:@braintrust/browser@^0.0.2-rc.0" }, "nodeModulesDir": "auto", "links": [ diff --git a/js/smoke/scenarios/deno-browser/tests/shared-suite.test.ts b/js/smoke/scenarios/deno-browser/tests/shared-suite.test.ts index 2c37b4f98..eb3dd0884 100644 --- a/js/smoke/scenarios/deno-browser/tests/shared-suite.test.ts +++ b/js/smoke/scenarios/deno-browser/tests/shared-suite.test.ts @@ -32,7 +32,7 @@ import { testNunjucksTemplate, testEvalSmoke, } from "@braintrust/smoke-test-shared"; -import * as braintrust from "braintrust"; +import * as braintrust from "@braintrust/browser"; Deno.test("Run shared test suites (browser build)", async () => { const { failed } = await runTests({ @@ -65,8 +65,8 @@ Deno.test("Run shared test suites (browser build)", async () => { expectFailure( testNunjucksTemplate, (e: { message: string }) => - e.message.includes("Nunjucks templating is not supported"), - "Nunjucks not supported in browser build", + e.message.includes("requires @braintrust/template-nunjucks"), + "Nunjucks requires separate package", ), ], }); diff --git a/js/smoke/scenarios/deno-node/Makefile b/js/smoke/scenarios/deno-node/Makefile index 3a7c730a7..05babbc0f 100644 --- a/js/smoke/scenarios/deno-node/Makefile +++ b/js/smoke/scenarios/deno-node/Makefile @@ -15,4 +15,5 @@ setup: fi; \ test: setup - deno test --sloppy-imports --allow-all tests/*.test.ts + @# Run only tests under the local tests/ directory + mise exec -- deno test --sloppy-imports --allow-all tests/*.test.ts diff --git a/js/smoke/scenarios/deno-node/tests/shared-suite.test.ts b/js/smoke/scenarios/deno-node/tests/shared-suite.test.ts index 543312576..3286585cb 100644 --- a/js/smoke/scenarios/deno-node/tests/shared-suite.test.ts +++ b/js/smoke/scenarios/deno-node/tests/shared-suite.test.ts @@ -6,6 +6,7 @@ import { assertEquals } from "@std/assert"; import { runTests, + expectFailure, testBasicSpanLogging, testMultipleSpans, testDirectLogging, @@ -61,7 +62,12 @@ Deno.test("Run shared test suites", async () => { testCurrentSpan, testEvalSmoke, testMustacheTemplate, - testNunjucksTemplate, + expectFailure( + testNunjucksTemplate, + (e: { message: string }) => + e.message.includes("requires @braintrust/template-nunjucks"), + "Nunjucks requires separate package", + ), ], }); diff --git a/js/smoke/scenarios/jest-node/Makefile b/js/smoke/scenarios/jest-node/Makefile index 22f859f8c..ccdff2cf0 100644 --- a/js/smoke/scenarios/jest-node/Makefile +++ b/js/smoke/scenarios/jest-node/Makefile @@ -36,4 +36,5 @@ setup: test: setup @echo "==> Running jest-node tests" - npx jest + @# Run only tests under the local tests/ directory to avoid workspace-wide discovery + npx jest --passWithNoTests --roots ./tests diff --git a/js/smoke/scenarios/jest-node/tests/shared-suite.test.js b/js/smoke/scenarios/jest-node/tests/shared-suite.test.js index 33c2d24a4..13aeffdf4 100644 --- a/js/smoke/scenarios/jest-node/tests/shared-suite.test.js +++ b/js/smoke/scenarios/jest-node/tests/shared-suite.test.js @@ -1,5 +1,6 @@ const { runTests, + expectFailure, testBasicSpanLogging, testMultipleSpans, testDirectLogging, @@ -23,6 +24,7 @@ const { testBuildResolution, testMustacheTemplate, testNunjucksTemplate, + testEvalSmoke, } = require("../../../shared/dist/index.js"); const braintrust = require("braintrust"); @@ -53,8 +55,13 @@ test("shared test suites pass in Jest", async () => { testAsyncLocalStorageTraced, testNestedTraced, testCurrentSpan, + testEvalSmoke, testMustacheTemplate, - testNunjucksTemplate, + expectFailure( + testNunjucksTemplate, + (e) => e.message.includes("requires @braintrust/template-nunjucks"), + "Nunjucks requires separate package", + ), ], }); diff --git a/js/smoke/scenarios/nextjs-instrumentation/package.json b/js/smoke/scenarios/nextjs-instrumentation/package.json index e8faeef10..19528dbd4 100644 --- a/js/smoke/scenarios/nextjs-instrumentation/package.json +++ b/js/smoke/scenarios/nextjs-instrumentation/package.json @@ -8,6 +8,7 @@ }, "dependencies": { "braintrust": "file:../../../artifacts/braintrust-latest.tgz", + "@braintrust/browser": "file:../../../artifacts/braintrust-browser-latest.tgz", "@braintrust/otel": "file:../../../artifacts/braintrust-otel-latest.tgz", "@opentelemetry/api": "^1.9.0", "@opentelemetry/api-logs": "^0.200.0", diff --git a/js/smoke/scenarios/nextjs-instrumentation/src/app/api/smoke-test/edge/route.ts b/js/smoke/scenarios/nextjs-instrumentation/src/app/api/smoke-test/edge/route.ts index c0bf8769c..79beb7dff 100644 --- a/js/smoke/scenarios/nextjs-instrumentation/src/app/api/smoke-test/edge/route.ts +++ b/js/smoke/scenarios/nextjs-instrumentation/src/app/api/smoke-test/edge/route.ts @@ -32,7 +32,7 @@ import { testEvalSmoke, } from "../../../../../../../shared"; -import * as braintrust from "braintrust"; +import * as braintrust from "@braintrust/browser"; export const runtime = "edge"; @@ -68,8 +68,9 @@ export async function GET() { testMustacheTemplate, expectFailure( testNunjucksTemplate, - (e) => e.message.includes("Nunjucks templating is not supported"), - "Nunjucks not supported in Edge Runtime", + (e: { message: string }) => + e.message.includes("requires @braintrust/template-nunjucks"), + "Nunjucks requires separate package", ), ], }); diff --git a/js/smoke/scenarios/nextjs-instrumentation/src/app/api/smoke-test/node/route.ts b/js/smoke/scenarios/nextjs-instrumentation/src/app/api/smoke-test/node/route.ts index 1730617da..7c86e8440 100644 --- a/js/smoke/scenarios/nextjs-instrumentation/src/app/api/smoke-test/node/route.ts +++ b/js/smoke/scenarios/nextjs-instrumentation/src/app/api/smoke-test/node/route.ts @@ -5,6 +5,7 @@ import { NextResponse } from "next/server"; import { runTests, + expectFailure, testBasicSpanLogging, testMultipleSpans, testDirectLogging, @@ -65,7 +66,12 @@ export async function GET() { testCurrentSpan, testEvalSmoke, testMustacheTemplate, - testNunjucksTemplate, + expectFailure( + testNunjucksTemplate, + (e: { message: string }) => + e.message.includes("requires @braintrust/template-nunjucks"), + "Nunjucks requires separate package", + ), ], }); diff --git a/js/smoke/scenarios/playwright-browser/package.json b/js/smoke/scenarios/playwright-browser/package.json index 49818a121..e703f5b2d 100644 --- a/js/smoke/scenarios/playwright-browser/package.json +++ b/js/smoke/scenarios/playwright-browser/package.json @@ -6,6 +6,7 @@ "description": "Playwright browser smoke test for Braintrust SDK", "dependencies": { "braintrust": "file:../../../artifacts/braintrust-latest.tgz", + "@braintrust/browser": "file:../../../artifacts/braintrust-browser-latest.tgz", "zod": "^4.3.5" }, "devDependencies": { diff --git a/js/smoke/scenarios/playwright-browser/src/browser-tests.ts b/js/smoke/scenarios/playwright-browser/src/browser-tests.ts index e56a78d0a..f28a9ada7 100644 --- a/js/smoke/scenarios/playwright-browser/src/browser-tests.ts +++ b/js/smoke/scenarios/playwright-browser/src/browser-tests.ts @@ -1,4 +1,4 @@ -import * as braintrust from "braintrust"; +import * as braintrust from "@braintrust/browser"; import { runTests, expectFailure, @@ -84,8 +84,9 @@ async function runAllTestSuites() { testMustacheTemplate, expectFailure( testNunjucksTemplate, - (e) => e.message.includes("Nunjucks templating is not supported"), - "Nunjucks not supported in browser build", + (e: { message: string }) => + e.message.includes("requires @braintrust/template-nunjucks"), + "Nunjucks requires separate package", ), ], }); diff --git a/js/smoke/scenarios/playwright-browser/tests/browser.test.ts b/js/smoke/scenarios/playwright-browser/tests/browser.test.ts index fec302726..42dd1a9c5 100644 --- a/js/smoke/scenarios/playwright-browser/tests/browser.test.ts +++ b/js/smoke/scenarios/playwright-browser/tests/browser.test.ts @@ -110,11 +110,19 @@ test.describe("Braintrust SDK Browser Tests", () => { name: `All tests (${smoke.sections.tests.passed} passed)`, }); } else { + // Extract individual test failures for better error reporting + const failureDetails = smoke.sections.tests.failures + .map( + (f) => + ` • ${f.testName}: ${f.error}${f.message ? ` (${f.message})` : ""}`, + ) + .join("\n"); + results.push({ status: "fail", name: "All tests", error: { - message: `Completed: ${smoke.sections.tests.completed}, Passed: ${smoke.sections.tests.passed}, Failed: ${smoke.sections.tests.failed}`, + message: `Completed: ${smoke.sections.tests.completed}, Passed: ${smoke.sections.tests.passed}, Failed: ${smoke.sections.tests.failed}\n\nFailed tests:\n${failureDetails}`, }, }); } diff --git a/js/smoke/shared/package-lock.json b/js/smoke/shared/package-lock.json index f67d631b2..32ad0c26b 100644 --- a/js/smoke/shared/package-lock.json +++ b/js/smoke/shared/package-lock.json @@ -933,6 +933,7 @@ "dev": true, "hasInstallScript": true, "license": "MIT", + "peer": true, "bin": { "esbuild": "bin/esbuild" }, @@ -1125,6 +1126,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -1419,6 +1421,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" diff --git a/js/smoke/shared/src/helpers/register.ts b/js/smoke/shared/src/helpers/register.ts index 663481b5e..9b45d0126 100644 --- a/js/smoke/shared/src/helpers/register.ts +++ b/js/smoke/shared/src/helpers/register.ts @@ -120,6 +120,7 @@ export interface RunTestsOptions { name: string; braintrust: BraintrustModule; tests: TestFn[]; + skipCoverage?: boolean; } export interface TestRunResults { @@ -142,6 +143,7 @@ export async function runTests({ name, braintrust, tests, + skipCoverage = false, }: RunTestsOptions): Promise { const results: TestResult[] = []; @@ -149,7 +151,9 @@ export async function runTests({ results.push(await test(braintrust)); } - results.push(validateCoverage(results)); + if (!skipCoverage) { + results.push(validateCoverage(results)); + } displayTestResults({ scenarioName: name, results }); diff --git a/js/src/browser-config.ts b/js/src/browser-config.ts index 2607a4ec1..718ec83cb 100644 --- a/js/src/browser-config.ts +++ b/js/src/browser-config.ts @@ -1,3 +1,6 @@ +// Browser-safe isomorph that noops Node.js features +// This file is only used for the /browser, /edge-light, /workerd exports + import iso from "./isomorph"; import { _internalSetInitialState } from "./logger"; @@ -10,13 +13,30 @@ declare global { } // End copied code +let messageShown = false; let browserConfigured = false; -export function configureBrowser() { + +/** + * Configure the isomorph for browser environments. + */ +export function configureBrowser(): void { if (browserConfigured) { return; } - // Set build type indicator + // Show informational message once + if (!messageShown && typeof console !== "undefined") { + console.info( + "This entrypoint is no longer supported.\n\n" + + "You should be using entrypoints:\n\n" + + "- `/workerd` (cloudflare envs)\n" + + "- `/edge-light` (next-js or other edge envs)\n\n" + + "If you'd like to use braintrust in the browser use the dedicated package: @braintrust/browser\n", + ); + messageShown = true; + } + + // Configure browser-safe implementations iso.buildType = "browser"; try { @@ -34,12 +54,6 @@ export function configureBrowser() { return process.env[name]; }; - iso.renderNunjucksString = () => { - throw new Error( - "Nunjucks templating is not supported in this build. Use templateFormat: 'mustache' (or omit templateFormat).", - ); - }; - // Implement browser-compatible hash function using a simple hash algorithm iso.hash = (data: string): string => { // Simple hash function for browser compatibility diff --git a/js/src/browser.ts b/js/src/browser.ts index 706b2bb4b..9f4f756d1 100644 --- a/js/src/browser.ts +++ b/js/src/browser.ts @@ -1,7 +1,16 @@ +/** + * Browser-compatible build of the Braintrust SDK. + * + * This build uses a noop isomorph that provides browser-safe implementations + * for Node.js-specific features. + * + * For optimal browser support with AsyncLocalStorage polyfill, consider: + * npm install @braintrust/browser + * import * as braintrust from '@braintrust/browser'; + */ + import { configureBrowser } from "./browser-config"; configureBrowser(); -// eslint-disable-next-line no-restricted-syntax -- already enforced in exports export * from "./exports"; -export * as default from "./exports"; diff --git a/js/src/exports.ts b/js/src/exports.ts index fee5d4b43..420e1bd03 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -103,12 +103,26 @@ export { registerOtelFlush, } from "./logger"; +// Internal isomorph layer for platform-specific implementations +import _internalIso from "./isomorph"; +export { _internalIso }; + export { isTemplateFormat, parseTemplateFormat, renderTemplateContent, } from "./template/renderer"; -export type { TemplateFormat } from "./template/renderer"; +export type { TemplateFormat } from "./template/registry"; + +export type { + TemplateRenderer, + TemplateRendererPlugin, +} from "./template/registry"; +export { + registerTemplatePlugin, + getTemplateRenderer, + templateRegistry, +} from "./template/registry"; export type { InvokeFunctionArgs, InvokeReturn } from "./functions/invoke"; export { initFunction, invoke } from "./functions/invoke"; diff --git a/js/src/framework.test.ts b/js/src/framework.test.ts index 62129fe17..0b3c7f7d8 100644 --- a/js/src/framework.test.ts +++ b/js/src/framework.test.ts @@ -1075,9 +1075,10 @@ describe("framework2 metadata support", () => { // Check that template_format is stored at the top level of prompt data expect(prompt.templateFormat).toBe("nunjucks"); - // Verify it renders correctly - const result = prompt.build({ name: "World" }); - expect(result.messages[0].content).toBe("Hello World"); + // Verify it requires the addon to render + expect(() => prompt.build({ name: "World" })).toThrow( + "Nunjucks templating requires @braintrust/template-nunjucks. Install and import it to enable templateFormat: 'nunjucks'.", + ); }); }); }); diff --git a/js/src/isomorph.ts b/js/src/isomorph.ts index 690fdcb38..eef739889 100644 --- a/js/src/isomorph.ts +++ b/js/src/isomorph.ts @@ -75,13 +75,6 @@ export interface Common { // zlib (promisified and type-erased). gunzip?: (data: any) => Promise; gzip?: (data: any) => Promise; - - // Nunjucks template rendering (lints if strict is true) - renderNunjucksString: ( - template: string, - variables: Record, - options?: { strict?: boolean }, - ) => string; } const iso: Common = { @@ -92,11 +85,6 @@ const iso: Common = { getCallerLocation: () => undefined, newAsyncLocalStorage: () => new DefaultAsyncLocalStorage(), processOn: (_0, _1) => {}, - renderNunjucksString: () => { - throw new Error( - "Nunjucks templating is not supported in this build. Use templateFormat: 'mustache' (or omit templateFormat).", - ); - }, basename: (filepath: string) => filepath.split(/[\\/]/).pop() || filepath, writeln: (text: string) => console.log(text), }; diff --git a/js/src/logger.test.ts b/js/src/logger.test.ts index 7123985ed..a9ad24bce 100644 --- a/js/src/logger.test.ts +++ b/js/src/logger.test.ts @@ -209,30 +209,16 @@ describe("prompt.build structured output templating", () => { false, ); - const result = prompt.build( - { - user: { name: "ada" }, - }, - { templateFormat: "nunjucks" }, - ); - - expect(result).toMatchObject({ - response_format: { - type: "json_schema", - json_schema: { - name: "schema", - schema: { - type: "object", - properties: { - greeting: { - type: "string", - description: "Hello ADA", - }, - }, - }, + expect(() => + prompt.build( + { + user: { name: "ada" }, }, - }, - }); + { templateFormat: "nunjucks" }, + ), + ).toThrow( + "Nunjucks templating requires @braintrust/template-nunjucks. Install and import it to enable templateFormat: 'nunjucks'.", + ); }); test("prompt.build with structured output templating", () => { diff --git a/js/src/logger.ts b/js/src/logger.ts index 9d1838dde..b1beb1548 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -75,8 +75,8 @@ import Mustache from "mustache"; import { parseTemplateFormat, renderTemplateContent, - type TemplateFormat, } from "./template/renderer"; +import type { TemplateFormat } from "./template/registry"; import { z, ZodError } from "zod/v3"; import { @@ -6497,25 +6497,15 @@ function renderTemplatedObject( options: { strict?: boolean; templateFormat: TemplateFormat }, ): unknown { if (typeof obj === "string") { - const strict = !!options.strict; - if (options.templateFormat === "nunjucks") { - return iso.renderNunjucksString(obj, args, { strict }); - } - if (options.templateFormat === "mustache") { - if (strict) { - lintMustacheTemplate(obj, args); - } - return Mustache.render(obj, args, undefined, { - escape: (value) => { - if (typeof value === "string") { - return value; - } else { - return JSON.stringify(value); - } - }, - }); - } - return obj; + return renderTemplateContent( + obj, + args, + (value) => (typeof value === "string" ? value : JSON.stringify(value)), + { + strict: options.strict, + templateFormat: options.templateFormat, + }, + ); } else if (isArray(obj)) { return obj.map((item) => renderTemplatedObject(item, args, options)); } else if (isObject(obj)) { diff --git a/js/src/node.ts b/js/src/node.ts index df453a3a8..2351c6ca4 100644 --- a/js/src/node.ts +++ b/js/src/node.ts @@ -9,8 +9,6 @@ import iso from "./isomorph"; import { getRepoInfo, getPastNAncestors } from "./gitutil"; import { getCallerLocation } from "./stackutil"; import { _internalSetInitialState } from "./logger"; -import { renderNunjucksString as nunjucksRender } from "./template/nunjucks-env"; -import { lintTemplate as nunjucksLint } from "./template/nunjucks-utils"; import { promisify } from "node:util"; import * as zlib from "node:zlib"; @@ -26,15 +24,6 @@ export function configureNode() { iso.processOn = (event: string, handler: (code: unknown) => void) => { process.on(event, handler); }; - iso.renderNunjucksString = (template, variables, options = {}) => { - const strict = options.strict ?? false; - - if (strict) { - nunjucksLint(template, variables); - } - - return nunjucksRender(template, variables, { strict }); - }; iso.basename = path.basename; iso.writeln = (text: string) => process.stdout.write(text + "\n"); iso.pathJoin = path.join; diff --git a/js/src/prompt.test.ts b/js/src/prompt.test.ts index c822a39d7..849158cc2 100644 --- a/js/src/prompt.test.ts +++ b/js/src/prompt.test.ts @@ -168,8 +168,9 @@ describe("prompt template_format", () => { true, ); - const result = prompt.build({ name: "World" }); - expect(result.messages[0].content).toBe("Hello World"); + expect(() => prompt.build({ name: "World" })).toThrow( + "Nunjucks templating requires @braintrust/template-nunjucks. Install and import it to enable templateFormat: 'nunjucks'.", + ); }); test("defaults to mustache when no templateFormat specified", () => { @@ -258,8 +259,11 @@ describe("prompt template_format", () => { true, ); - const result = prompt.build({ text: "Hello" }, { flavor: "completion" }); - expect(result.prompt).toBe("Complete this: Hello"); + expect(() => + prompt.build({ text: "Hello" }, { flavor: "completion" }), + ).toThrow( + "Nunjucks templating requires @braintrust/template-nunjucks. Install and import it to enable templateFormat: 'nunjucks'.", + ); }); }); @@ -298,10 +302,9 @@ describe("prompt template_format (unconfigured/browser-like)", () => { ); expect(() => prompt.build({ name: "World" })).toThrowError( - /Nunjucks templating is not supported in this build/, + /Nunjucks templating requires @braintrust\/template-nunjucks/, ); }); - test("throws unsupported error after configureBrowser()", async () => { vi.resetModules(); const { configureBrowser } = await import("./browser-config"); @@ -339,7 +342,7 @@ describe("prompt template_format (unconfigured/browser-like)", () => { ); expect(() => prompt.build({ name: "World" })).toThrowError( - /Nunjucks templating is not supported in this build/, + /Nunjucks templating requires @braintrust\/template-nunjucks/, ); }); }); diff --git a/js/src/template/nunjucks-env.ts b/js/src/template/nunjucks-env.ts deleted file mode 100644 index 16232f72b..000000000 --- a/js/src/template/nunjucks-env.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { nunjucks } from "./nunjucks"; -import type { Environment as NunjucksEnvironment } from "nunjucks"; -import { SyncLazyValue } from "../util"; - -const createNunjucksEnv = (throwOnUndefined: boolean): NunjucksEnvironment => { - return new nunjucks.Environment(null, { - autoescape: true, - throwOnUndefined, - }); -}; - -const nunjucksEnv = new SyncLazyValue(() => - createNunjucksEnv(false), -); - -const nunjucksStrictEnv = new SyncLazyValue(() => - createNunjucksEnv(true), -); - -export function getNunjucksEnv(options?: { - strict?: boolean; -}): NunjucksEnvironment { - const strict = options?.strict ?? false; - return strict ? nunjucksStrictEnv.get() : nunjucksEnv.get(); -} - -export function renderNunjucksString( - template: string, - variables: Record, - options?: { strict?: boolean }, -): string { - const strict = options?.strict ?? false; - try { - return getNunjucksEnv({ strict }).renderString(template, variables); - } catch (error) { - if ( - error instanceof Error && - error.message.includes( - "Code generation from strings disallowed for this context", - ) - ) { - throw new Error( - `String template rendering. Disallowed in this environment for security reasons. Try a different template renderer. Original error: ${error.message}`, - ); - } - throw error; - } -} diff --git a/js/src/template/nunjucks-render.test.ts b/js/src/template/nunjucks-render.test.ts deleted file mode 100644 index e6dd85ae1..000000000 --- a/js/src/template/nunjucks-render.test.ts +++ /dev/null @@ -1,215 +0,0 @@ -import { describe, test, expect } from "vitest"; -import { lintTemplate } from "./nunjucks-utils"; -import { getNunjucksEnv } from "./nunjucks-env"; - -function getEnv() { - return getNunjucksEnv({ strict: false }); -} - -function getStrictEnv() { - return getNunjucksEnv({ strict: true }); -} - -describe("nunjucks rendering", () => { - test("renders variable and control structures", () => { - const env = getEnv(); - const out = env.renderString( - "Hello {{ name | upper }} {% if age > 18 %}Adult{% else %}Minor{% endif %}", - { name: "alice", age: 30 }, - ); - expect(out).toBe("Hello ALICE Adult"); - }); - - test("loops render", () => { - const env = getEnv(); - const out = env.renderString( - `{% for item in items %}{{ item }},{% endfor %}`, - { items: ["a", "b"] }, - ); - expect(out).toBe("a,b,"); - }); - - test("strict mode throws for missing top-level variable", () => { - const env = getStrictEnv(); - const tpl = "Hello {{ name }}"; - expect(() => env.renderString(tpl, { user: "x" })).toThrow(); - }); - - test("strict mode passes for defined variable and filters", () => { - const env = getStrictEnv(); - const tpl = "Hello {{ name | upper }}"; - expect(() => env.renderString(tpl, { name: "alice" })).not.toThrow(); - }); - - test("strict mode: for over undefined is empty (does not throw)", () => { - const env = getStrictEnv(); - const tpl = `{% for item in items %}{{ item }}{% endfor %}`; - expect(() => env.renderString(tpl, { items: [1, 2, 3] })).not.toThrow(); - expect(() => env.renderString(tpl, {})).not.toThrow(); - }); - - test("strict mode: nested path with numeric index using brackets", () => { - const env = getStrictEnv(); - const tpl = `{{ user.addresses[2].city }}`; - const ok = { user: { addresses: [{}, {}, { city: "SF" }] } }; - expect(() => env.renderString(tpl, ok)).not.toThrow(); - const bad = { user: {} }; - expect(() => env.renderString(tpl, bad)).toThrow(); - }); - - test("renders nested object properties", () => { - const env = getEnv(); - const out = env.renderString("{{ user.profile.name }}", { - user: { profile: { name: "Alice" } }, - }); - expect(out).toBe("Alice"); - }); - - test("renders multiple variables with context", () => { - const env = getEnv(); - const out = env.renderString( - "{{ firstName }} {{ lastName }} is {{ age }} years old", - { firstName: "Bob", lastName: "Smith", age: 25 }, - ); - expect(out).toBe("Bob Smith is 25 years old"); - }); - - test("renders with string concatenation", () => { - const env = getEnv(); - const out = env.renderString("{{ greeting ~ ' ' ~ name }}", { - greeting: "Hello", - name: "World", - }); - expect(out).toBe("Hello World"); - }); - - test("renders numeric operations", () => { - const env = getEnv(); - const out = env.renderString("Total: {{ price * quantity }}", { - price: 10, - quantity: 3, - }); - expect(out).toBe("Total: 30"); - }); - - test("renders with filters and context", () => { - const env = getEnv(); - const out = env.renderString("{{ message | upper | trim }}", { - message: " hello world ", - }); - expect(out).toBe("HELLO WORLD"); - }); - - test("renders array elements with index", () => { - const env = getEnv(); - const out = env.renderString( - "First: {{ items[0] }}, Last: {{ items[2] }}", - { - items: ["apple", "banana", "cherry"], - }, - ); - expect(out).toBe("First: apple, Last: cherry"); - }); - - test("renders nested arrays and objects", () => { - const env = getEnv(); - const out = env.renderString( - "{{ users[0].name }} from {{ users[0].city }}", - { - users: [ - { name: "John", city: "NYC" }, - { name: "Jane", city: "LA" }, - ], - }, - ); - expect(out).toBe("John from NYC"); - }); - - test("renders with default filter", () => { - const env = getEnv(); - const out1 = env.renderString("{{ name | default('Guest') }}", { - name: "Alice", - }); - expect(out1).toBe("Alice"); - const out2 = env.renderString("{{ name | default('Guest') }}", {}); - expect(out2).toBe("Guest"); - }); - - test("renders ternary expressions", () => { - const env = getEnv(); - const out1 = env.renderString("{{ user.name if user else 'Anonymous' }}", { - user: { name: "Alice" }, - }); - expect(out1).toBe("Alice"); - const out2 = env.renderString( - "{{ user.name if user else 'Anonymous' }}", - {}, - ); - expect(out2).toBe("Anonymous"); - }); - - test("renders with multiple filters chained", () => { - const env = getEnv(); - const out = env.renderString("{{ text | lower | trim }}", { - text: " HELLO WORLD ", - }); - expect(out).toBe("hello world"); - }); - - test("renders complex nested context", () => { - const env = getEnv(); - const out = env.renderString( - "{{ order.customer.name }} ordered {{ order.items[0].name }} for ${{ order.total }}", - { - order: { - customer: { name: "Alice" }, - items: [{ name: "Widget", price: 10 }], - total: 10, - }, - }, - ); - expect(out).toBe("Alice ordered Widget for $10"); - }); - - test("renders with length filter on arrays", () => { - const env = getEnv(); - const out = env.renderString("You have {{ items | length }} items", { - items: ["a", "b", "c", "d"], - }); - expect(out).toBe("You have 4 items"); - }); - - test("renders with join filter", () => { - const env = getEnv(); - const out = env.renderString("{{ tags | join(', ') }}", { - tags: ["red", "green", "blue"], - }); - expect(out).toBe("red, green, blue"); - }); -}); - -describe("nunjucks lintTemplate", () => { - test("passes for valid template syntax", () => { - expect(() => lintTemplate("Hello {{ user.name }}", {})).toThrow(); - }); - - test("passes for valid template with loops", () => { - expect(() => - lintTemplate(`{% for item in items %}{{ item }}{% endfor %}`, {}), - ).not.toThrow(); - }); - - test("passes for valid template with conditionals", () => { - expect(() => - lintTemplate(`{% if user %}{{ user.name }}{% endif %}`, {}), - ).not.toThrow(); - }); - - test("throws for invalid template syntax", () => { - expect(() => lintTemplate("{{ unclosed", {})).toThrow(); - }); - - test("throws for mismatched tags", () => { - expect(() => lintTemplate("{% if x %}{% endfor %}", {})).toThrow(); - }); -}); diff --git a/js/src/template/nunjucks-utils.ts b/js/src/template/nunjucks-utils.ts deleted file mode 100644 index 7afd682cd..000000000 --- a/js/src/template/nunjucks-utils.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { getNunjucksEnv } from "./nunjucks-env"; - -export function lintTemplate(template: string, context: any): void { - const env = getNunjucksEnv({ strict: true }); - env.renderString(template, context); -} diff --git a/js/src/template/nunjucks.ts b/js/src/template/nunjucks.ts deleted file mode 100644 index a7a60ffa0..000000000 --- a/js/src/template/nunjucks.ts +++ /dev/null @@ -1,3 +0,0 @@ -import * as nunjucksNodeModule from "nunjucks"; - -export const nunjucks = nunjucksNodeModule; diff --git a/js/src/template/registry.ts b/js/src/template/registry.ts new file mode 100644 index 000000000..a71817c7e --- /dev/null +++ b/js/src/template/registry.ts @@ -0,0 +1,161 @@ +import Mustache from "mustache"; + +import { lintTemplate as lintMustacheTemplate } from "./mustache-utils"; + +export type TemplateFormat = "mustache" | "nunjucks" | "none"; + +export interface TemplateRenderer { + render: ( + template: string, + variables: Record, + escape: (v: unknown) => string, + strict: boolean, + ) => string; + lint?: (template: string, variables: Record) => void; +} + +/** + * A template renderer plugin that can be registered with Braintrust. + * + * Plugins provide support for different template engines (e.g., Nunjucks). + * They use a factory pattern where the plugin is registered once, then activated with specific + * configuration options when needed. + * + * @example + * ```typescript + * import type { TemplateRendererPlugin } from "braintrust"; + * + * export const myPlugin: TemplateRendererPlugin = { + * name: "my-template-engine", + * version: "1.0.0", + * defaultOptions: { strict: false }, + * createRenderer(options?: unknown) { + * const opts = options ?? this.defaultOptions; + * return { + * render(template, variables, escape, strict) { + * // Your rendering logic here + * } + * }; + * } + * }; + * ``` + */ +export interface TemplateRendererPlugin { + /** + * Unique identifier for this plugin. + * Must match the format string used in `templateFormat` option. + */ + name: string; + + /** + * Factory function that creates a renderer instance. + * + * @param options - If not provided, `defaultOptions` is used. + * @returns A configured TemplateRenderer instance + */ + createRenderer: () => TemplateRenderer; + + /** + * Default configuration options for this plugin. + */ + defaultOptions?: unknown; +} + +class TemplatePluginRegistry { + private plugins = new Map< + string, + { plugin: TemplateRendererPlugin; renderer?: TemplateRenderer } + >(); + + register(plugin: TemplateRendererPlugin): void { + if (this.plugins.has(plugin.name)) { + console.warn( + `Template plugin '${plugin.name}' already registered, overwriting`, + ); + } + + const entry = { + plugin, + renderer: + plugin.defaultOptions !== undefined + ? plugin.createRenderer() + : undefined, + }; + + this.plugins.set(plugin.name, entry); + } + + getAvailable(): string[] { + return Array.from(this.plugins.keys()); + } + + get(name: string): TemplateRenderer | undefined { + return this.plugins.get(name)?.renderer; + } + + isRegistered(name: string): boolean { + return this.plugins.has(name); + } +} + +export const templateRegistry = new TemplatePluginRegistry(); + +/** + * Register a template plugin and optionally activate it + * + * If `options` is provided it will be used to create the active renderer. + * If `options` is omitted but the plugin defines `defaultOptions`, the + * registry will activate the renderer using those defaults. + */ +export const registerTemplatePlugin = + templateRegistry.register.bind(templateRegistry); + +/** + * Gets an active template renderer by name. + * + * Returns `undefined` if the renderer is not active. + * + * @param name - Name of the renderer to retrieve + * @returns The active renderer, or undefined if not activated + * + * @example + * ```typescript + * import { getTemplateRenderer } from "braintrust"; + * + * const renderer = getTemplateRenderer("nunjucks"); + * if (renderer) { + * const output = renderer.render(template, variables, escape, strict); + * } + * ``` + */ +export const getTemplateRenderer = templateRegistry.get.bind(templateRegistry); + +// Built-in mustache plugin +const jsonEscape = (v: unknown) => + typeof v === "string" ? v : JSON.stringify(v); + +const mustachePlugin: TemplateRendererPlugin = { + name: "mustache", + defaultOptions: { strict: true, escape: jsonEscape }, + createRenderer() { + const opts = (this.defaultOptions ?? {}) as any; + const escapeFn: (v: unknown) => string = opts?.escape ?? jsonEscape; + const strictDefault: boolean = + typeof opts?.strict === "boolean" ? opts.strict : true; + + return { + render(template, variables, escape, strict) { + const esc = escape ?? escapeFn; + const strictMode = typeof strict === "boolean" ? strict : strictDefault; + if (strictMode) lintMustacheTemplate(template, variables); + return Mustache.render(template, variables, undefined, { escape: esc }); + }, + lint(template, variables) { + lintMustacheTemplate(template, variables); + }, + }; + }, +}; + +// Auto-register built-in mustache plugin. +registerTemplatePlugin(mustachePlugin); diff --git a/js/src/template/renderer.ts b/js/src/template/renderer.ts index 2d002cde7..4fc34203c 100644 --- a/js/src/template/renderer.ts +++ b/js/src/template/renderer.ts @@ -1,8 +1,6 @@ -import Mustache from "mustache"; -import { lintTemplate as lintMustacheTemplate } from "./mustache-utils"; -import iso from "../isomorph"; +import { getTemplateRenderer, type TemplateFormat } from "./registry"; -export type TemplateFormat = "mustache" | "nunjucks" | "none"; +export type { TemplateFormat } from "./registry"; export function isTemplateFormat(v: unknown): v is TemplateFormat { return v === "mustache" || v === "nunjucks" || v === "none"; @@ -23,15 +21,22 @@ export function renderTemplateContent( ): string { const strict = !!options.strict; const templateFormat = parseTemplateFormat(options.templateFormat); - if (templateFormat === "nunjucks") { - return iso.renderNunjucksString(template, variables, { strict }); - } else if (templateFormat === "mustache") { - if (strict) { - lintMustacheTemplate(template, variables); + if (templateFormat === "none") { + return template; + } + + const renderer = getTemplateRenderer(templateFormat); + if (!renderer) { + if (templateFormat === "nunjucks") { + throw new Error( + "Nunjucks templating requires @braintrust/template-nunjucks. Install and import it to enable templateFormat: 'nunjucks'.", + ); } - return Mustache.render(template, variables, undefined, { - escape, - }); + throw new Error(`No template renderer registered for ${templateFormat}`); + } + + if (strict && renderer.lint) { + renderer.lint(template, variables); } - return template; + return renderer.render(template, variables, escape, strict); } diff --git a/js/tsup.config.ts b/js/tsup.config.ts index f5100c6cb..875d4ec22 100644 --- a/js/tsup.config.ts +++ b/js/tsup.config.ts @@ -18,21 +18,6 @@ export default defineConfig([ splitting: true, clean: true, }, - { - entry: ["src/browser.ts"], - format: ["cjs", "esm"], - outDir: "dist", - removeNodeProtocol: false, - external: ["zod"], - dts: { - // Split DTS generation to reduce memory usage - compilerOptions: { - skipLibCheck: true, - }, - }, - splitting: true, - clean: false, - }, { entry: { cli: "src/cli/index.ts" }, format: ["cjs"], @@ -73,4 +58,18 @@ export default defineConfig([ splitting: true, clean: true, }, + // Browser/edge exports - single build used by browser, edge-light, and workerd + { + entry: { + browser: "src/browser.ts", + }, + format: ["cjs", "esm"], + outDir: "dist", + external: ["zod"], + removeNodeProtocol: false, + platform: "browser", + splitting: false, + dts: true, + clean: false, + }, ]); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b3d52aaae..37220265d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,6 +15,31 @@ importers: specifier: ^2.5.6 version: 2.5.6 + integrations/browser-js: + dependencies: + als-browser: + specifier: ^1.0.1 + version: 1.0.1 + zod: + specifier: ^3.25.34 || ^4.0 + version: 3.25.76 + devDependencies: + '@types/node': + specifier: ^20.10.5 + version: 20.19.16 + braintrust: + specifier: workspace:* + version: link:../../js + tsup: + specifier: ^8.3.5 + version: 8.5.1(@swc/core@1.15.8)(jiti@2.6.1)(postcss@8.5.6)(typescript@5.5.4)(yaml@2.8.2) + typescript: + specifier: ^5.3.3 + version: 5.5.4 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@20.19.16)(msw@2.6.6(@types/node@20.19.16)(typescript@5.5.4))(terser@5.44.1) + integrations/langchain-js: devDependencies: '@langchain/anthropic': @@ -118,6 +143,31 @@ importers: specifier: 5.5.4 version: 5.5.4 + integrations/templates-nunjucks: + dependencies: + nunjucks: + specifier: ^3.2.4 + version: 3.2.4 + devDependencies: + '@types/node': + specifier: ^20.10.5 + version: 20.19.16 + '@types/nunjucks': + specifier: ^3.2.6 + version: 3.2.6 + braintrust: + specifier: workspace:* + version: link:../../js + tsup: + specifier: ^8.5.1 + version: 8.5.1(@swc/core@1.15.8)(jiti@2.6.1)(postcss@8.5.6)(typescript@5.5.4)(yaml@2.8.2) + typescript: + specifier: 5.5.4 + version: 5.5.4 + vitest: + specifier: ^2.1.9 + version: 2.1.9(@types/node@20.19.16)(msw@2.6.6(@types/node@20.19.16)(typescript@5.5.4))(terser@5.44.1) + integrations/temporal-js: devDependencies: '@temporalio/activity': @@ -196,9 +246,6 @@ importers: '@next/env': specifier: ^14.2.3 version: 14.2.3 - '@types/nunjucks': - specifier: ^3.2.6 - version: 3.2.6 '@vercel/functions': specifier: ^1.0.2 version: 1.0.2 @@ -244,9 +291,6 @@ importers: mustache: specifier: ^4.2.0 version: 4.2.0 - nunjucks: - specifier: ^3.2.4 - version: 3.2.4 pluralize: specifier: ^8.0.0 version: 8.0.0 @@ -2649,6 +2693,9 @@ packages: ajv@8.17.1: resolution: {integrity: sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==} + als-browser@1.0.1: + resolution: {integrity: sha512-DjavKf6zf4DFPdEmgsEM474MBjFcZG/1amv2/+WHGf61kVQWqf7XEn4jvpjFS4ssQbh/pkmYThaPfQK1ERC+3g==} + ansi-align@3.0.1: resolution: {integrity: sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==} @@ -8275,6 +8322,15 @@ snapshots: msw: 2.6.6(@types/node@20.19.16)(typescript@5.4.4) vite: 5.4.14(@types/node@20.19.16)(terser@5.44.1) + '@vitest/mocker@2.1.9(msw@2.6.6(@types/node@20.19.16)(typescript@5.5.4))(vite@5.4.14(@types/node@20.19.16)(terser@5.44.1))': + dependencies: + '@vitest/spy': 2.1.9 + estree-walker: 3.0.3 + magic-string: 0.30.19 + optionalDependencies: + msw: 2.6.6(@types/node@20.19.16)(typescript@5.5.4) + vite: 5.4.14(@types/node@20.19.16)(terser@5.44.1) + '@vitest/mocker@2.1.9(msw@2.6.6(@types/node@20.19.9)(typescript@5.4.4))(vite@5.4.14(@types/node@20.19.9)(terser@5.44.1))': dependencies: '@vitest/spy': 2.1.9 @@ -8555,6 +8611,8 @@ snapshots: json-schema-traverse: 1.0.0 require-from-string: 2.0.2 + als-browser@1.0.1: {} + ansi-align@3.0.1: dependencies: string-width: 4.2.3 @@ -10831,6 +10889,32 @@ snapshots: - '@types/node' optional: true + msw@2.6.6(@types/node@20.19.16)(typescript@5.5.4): + dependencies: + '@bundled-es-modules/cookie': 2.0.1 + '@bundled-es-modules/statuses': 1.0.1 + '@bundled-es-modules/tough-cookie': 0.1.6 + '@inquirer/confirm': 5.0.2(@types/node@20.19.16) + '@mswjs/interceptors': 0.37.3 + '@open-draft/deferred-promise': 2.2.0 + '@open-draft/until': 2.1.0 + '@types/cookie': 0.6.0 + '@types/statuses': 2.0.5 + chalk: 4.1.2 + graphql: 16.9.0 + headers-polyfill: 4.0.3 + is-node-process: 1.2.0 + outvariant: 1.4.3 + path-to-regexp: 6.3.0 + strict-event-emitter: 0.5.1 + type-fest: 4.30.0 + yargs: 17.7.2 + optionalDependencies: + typescript: 5.5.4 + transitivePeerDependencies: + - '@types/node' + optional: true + msw@2.6.6(@types/node@20.19.9)(typescript@5.4.4): dependencies: '@bundled-es-modules/cookie': 2.0.1 @@ -12468,6 +12552,41 @@ snapshots: - supports-color - terser + vitest@2.1.9(@types/node@20.19.16)(msw@2.6.6(@types/node@20.19.16)(typescript@5.5.4))(terser@5.44.1): + dependencies: + '@vitest/expect': 2.1.9 + '@vitest/mocker': 2.1.9(msw@2.6.6(@types/node@20.19.16)(typescript@5.5.4))(vite@5.4.14(@types/node@20.19.16)(terser@5.44.1)) + '@vitest/pretty-format': 2.1.9 + '@vitest/runner': 2.1.9 + '@vitest/snapshot': 2.1.9 + '@vitest/spy': 2.1.9 + '@vitest/utils': 2.1.9 + chai: 5.2.0 + debug: 4.4.3 + expect-type: 1.2.0 + magic-string: 0.30.19 + pathe: 1.1.2 + std-env: 3.8.1 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinypool: 1.0.2 + tinyrainbow: 1.2.0 + vite: 5.4.14(@types/node@20.19.16)(terser@5.44.1) + vite-node: 2.1.9(@types/node@20.19.16)(terser@5.44.1) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 20.19.16 + transitivePeerDependencies: + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + vitest@2.1.9(@types/node@20.19.9)(msw@2.6.6(@types/node@20.19.9)(typescript@5.4.4))(terser@5.44.1): dependencies: '@vitest/expect': 2.1.9