diff --git a/js/src/logger.test.ts b/js/src/logger.test.ts index ca9b22ed5..d51d8f01a 100644 --- a/js/src/logger.test.ts +++ b/js/src/logger.test.ts @@ -18,6 +18,7 @@ import { Attachment, deepCopyEvent, renderMessage, + renderMessageImpl, } from "./logger"; import { parseTemplateFormat, @@ -62,7 +63,15 @@ test("renderMessage with file content parts", () => { ], }; - const rendered = renderMessage( + const variables = { + item: "document", + image_url: "https://example.com/image.png", + file_data: "base64data", + file_id: "file-456", + filename: "report.pdf", + }; + + const rendered = renderMessageImpl( (template) => template .replace("{{item}}", "document") @@ -71,6 +80,7 @@ test("renderMessage with file content parts", () => { .replace("{{file_id}}", "file-456") .replace("{{filename}}", "report.pdf"), message, + variables, ); expect(rendered.content).toEqual([ @@ -95,6 +105,293 @@ test("renderMessage with file content parts", () => { ]); }); +test("renderMessage expands attachment array in image_url parts", () => { + const message = { + role: "user" as const, + content: [ + { + type: "image_url" as const, + image_url: { url: "{{images}}" }, + }, + ], + }; + + const variables = { + images: ["https://example.com/img1.jpg", "https://example.com/img2.jpg"], + }; + + const rendered = renderMessageImpl( + (template) => template, // Template rendering shouldn't happen for attachment arrays + message, + variables, + ); + + expect(rendered.content).toEqual([ + { + type: "image_url", + image_url: { + url: "https://example.com/img1.jpg", + }, + }, + { + type: "image_url", + image_url: { + url: "https://example.com/img2.jpg", + }, + }, + ]); +}); + +test("renderMessage expands inline attachment array in image_url parts", () => { + const message = { + role: "user" as const, + content: [ + { + type: "image_url" as const, + image_url: { url: "{{images}}" }, + }, + ], + }; + + const variables = { + images: [ + { + type: "inline_attachment", + src: "data:image/png;base64,abc", + content_type: "image/png", + }, + { + type: "inline_attachment", + src: "data:image/jpeg;base64,def", + content_type: "image/jpeg", + }, + ], + }; + + const rendered = renderMessageImpl( + (template) => template, + message, + variables, + ); + + expect(rendered.content).toEqual([ + { + type: "image_url", + image_url: { + url: { + type: "inline_attachment", + src: "data:image/png;base64,abc", + content_type: "image/png", + }, + }, + }, + { + type: "image_url", + image_url: { + url: { + type: "inline_attachment", + src: "data:image/jpeg;base64,def", + content_type: "image/jpeg", + }, + }, + }, + ]); +}); + +test("renderMessage does NOT expand mixed content (text + variable)", () => { + const message = { + role: "user" as const, + content: "Look at {{images}}", + }; + + const variables = { + images: ["https://example.com/img1.jpg", "https://example.com/img2.jpg"], + }; + + const rendered = renderMessageImpl( + (template) => template.replace("{{images}}", "[array]"), + message, + variables, + ); + + // Mixed content is not expanded - just rendered normally + expect(rendered.content).toBe("Look at [array]"); +}); + +test("renderMessage expands nested attachment arrays in image_url parts", () => { + const message = { + role: "user" as const, + content: [ + { + type: "image_url" as const, + image_url: { url: "{{data.images}}" }, + }, + ], + }; + + const variables = { + data: { + images: ["https://example.com/img1.jpg", "https://example.com/img2.jpg"], + }, + }; + + const rendered = renderMessageImpl( + (template) => template, // Template rendering shouldn't happen + message, + variables, + ); + + expect(rendered.content).toEqual([ + { + type: "image_url", + image_url: { + url: "https://example.com/img1.jpg", + }, + }, + { + type: "image_url", + image_url: { + url: "https://example.com/img2.jpg", + }, + }, + ]); +}); + +test("renderMessage expands deeply nested attachment arrays in image_url parts", () => { + const message = { + role: "user" as const, + content: [ + { + type: "image_url" as const, + image_url: { url: "{{user.profile.images}}" }, + }, + ], + }; + + const variables = { + user: { + profile: { + images: [ + { + type: "inline_attachment", + src: "data:image/png;base64,abc", + content_type: "image/png", + }, + { + type: "inline_attachment", + src: "data:image/jpeg;base64,def", + content_type: "image/jpeg", + }, + ], + }, + }, + }; + + const rendered = renderMessageImpl( + (template) => template, + message, + variables, + ); + + expect(rendered.content).toEqual([ + { + type: "image_url", + image_url: { + url: { + type: "inline_attachment", + src: "data:image/png;base64,abc", + content_type: "image/png", + }, + }, + }, + { + type: "image_url", + image_url: { + url: { + type: "inline_attachment", + src: "data:image/jpeg;base64,def", + content_type: "image/jpeg", + }, + }, + }, + ]); +}); + +test("renderMessage handles single image_url (no array)", () => { + const message = { + role: "user" as const, + content: [ + { + type: "image_url" as const, + image_url: { + url: "{{image}}", + }, + }, + ], + }; + + const variables = { + image: "https://example.com/single.jpg", + }; + + const rendered = renderMessageImpl( + (template) => + template.replace("{{image}}", "https://example.com/single.jpg"), + message, + variables, + ); + + expect(rendered.content).toEqual([ + { + type: "image_url", + image_url: { + url: "https://example.com/single.jpg", + }, + }, + ]); +}); + +test("renderMessage expands attachment arrays in structured content", () => { + // This tests the case where content is already an array with structured parts, + // and one part has a template variable for an attachment array + const message = { + role: "user" as const, + content: [ + { type: "text" as const, text: "Describe these images" }, + { type: "image_url" as const, image_url: { url: "{{attachments}}" } }, + ], + }; + + const variables = { + attachments: [ + "https://example.com/img1.jpg", + "https://example.com/img2.jpg", + ], + }; + + const rendered = renderMessageImpl( + (template) => template, + message, + variables, + ); + + // Should expand {{attachments}} into multiple image_url parts + expect(rendered.content).toEqual([ + { + type: "text", + text: "Describe these images", + }, + { + type: "image_url", + image_url: { url: "https://example.com/img1.jpg" }, + }, + { + type: "image_url", + image_url: { url: "https://example.com/img2.jpg" }, + }, + ]); +}); + test("verify MemoryBackgroundLogger intercepts logs", async () => { // Log to memory for the tests. _exportsForTestingOnly.simulateLoginForTests(); diff --git a/js/src/logger.ts b/js/src/logger.ts index db0887a79..c5da5106d 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -38,6 +38,7 @@ import { VALID_SOURCES, isArray, isObject, + getObjValueByPath, } from "./util"; import { type AnyModelParamsType as AnyModelParam, @@ -126,6 +127,18 @@ import { prettifyXact } from "../util/index"; import { SpanCache, CachedSpan } from "./span-cache"; import type { EvalParameters, InferParameters } from "./eval-parameters"; +// Manual type definition for inline attachments (not in generated_types) +const InlineAttachmentReferenceSchema = z.object({ + type: z.literal("inline_attachment"), + src: z.string().min(1), + content_type: z.string().optional(), + filename: z.string().optional(), +}); + +type InlineAttachmentReference = z.infer< + typeof InlineAttachmentReferenceSchema +>; + // Context management interfaces export interface ContextParentSpanIds { rootSpanId: string; @@ -6946,9 +6959,80 @@ export type DefaultPromptArgs = Partial< CompiledPromptParams & AnyModelParam & ChatPrompt & CompletionPrompt >; +function isAttachmentObject(value: unknown): boolean { + return ( + BraintrustAttachmentReferenceSchema.safeParse(value).success || + InlineAttachmentReferenceSchema.safeParse(value).success || + ExternalAttachmentReferenceSchema.safeParse(value).success + ); +} + +// Simple URL validation all braintrust attachment urls or generic links +function isURL(url: string): boolean { + try { + const parsedUrl = new URL(url.trim()); + return parsedUrl.protocol === "http:" || parsedUrl.protocol === "https:"; + } catch { + return false; + } +} + +/** + * Detect and expand attachment array variables before template rendering. + * Supports both simple variables ({{images}}) and nested paths ({{data.images}}). + * @param content The message content (should be a template variable like "{{images}}" or "{{data.images}}") + * @param variables The variables object with array values + * @returns Array of image_url parts if expansion occurred, null otherwise + */ +function expandAttachmentArrayPreTemplate( + content: unknown, + variables: Record, +): unknown[] | null { + if (typeof content !== "string") return null; + + // Detect {{varName}} or {{nested.path}} pattern (exact match only - no mixed content) + // Supports: {{images}}, {{data.images}}, {{user.profile.images}}, etc. + const match = content.match(/^\{\{\s*([\w.]+)\s*\}\}$/); + if (!match) return null; + + const varPath = match[1]; + + const value = varPath.includes(".") + ? getObjValueByPath(variables, varPath.split(".")) + : variables[varPath]; + + if (!Array.isArray(value)) return null; + + // Check if array contains attachments or image URLs + const allValid = value.every( + (v) => isAttachmentObject(v) || (typeof v === "string" && isURL(v)), + ); + + if (!allValid) return null; + + // Expand directly to image_url parts + return value.map((item) => ({ + type: "image_url" as const, + image_url: { url: item }, + })); +} + +export function renderMessage( + render: (template: string) => string, + message: T, +): T; + export function renderMessage( render: (template: string) => string, message: T, +): T { + return renderMessageImpl(render, message, {}); +} + +export function renderMessageImpl( + render: (template: string) => string, + message: T, + variables?: Record, ): T { return { ...message, @@ -6958,36 +7042,54 @@ export function renderMessage( ? undefined : typeof message.content === "string" ? render(message.content) - : message.content.map((c) => { + : message.content.flatMap((c) => { switch (c.type) { case "text": - return { ...c, text: render(c.text) }; + return [{ ...c, text: render(c.text) }]; case "image_url": if (isObject(c.image_url.url)) { throw new Error( "Attachments must be replaced with URLs before calling `build()`", ); } - return { - ...c, - image_url: { - ...c.image_url, - url: render(c.image_url.url), + + if (variables) { + const expanded = expandAttachmentArrayPreTemplate( + c.image_url.url, + variables, + ); + if (expanded) { + return expanded; + } + } + + // Otherwise render the URL template normally + return [ + { + ...c, + image_url: { + ...c.image_url, + url: render(c.image_url.url), + }, }, - }; + ]; case "file": - return { - ...c, - file: { - file_data: render(c.file.file_data || ""), - ...(c.file.file_id && { - file_id: render(c.file.file_id), - }), - ...(c.file.filename && { - filename: render(c.file.filename), - }), + return [ + { + ...c, + file: { + ...(c.file.file_data && { + file_data: render(c.file.file_data), + }), + ...(c.file.file_id && { + file_id: render(c.file.file_id), + }), + ...(c.file.filename && { + filename: render(c.file.filename), + }), + }, }, - }; + ]; default: const _exhaustiveCheck: never = c; return _exhaustiveCheck; @@ -7391,7 +7493,7 @@ export class Prompt< }); const baseMessages = (prompt.messages || []).map((m) => - renderMessage(render, m), + renderMessageImpl(render, m, variables), ); const hasSystemPrompt = baseMessages.some((m) => m.role === "system"); diff --git a/js/src/template/renderer.test.ts b/js/src/template/renderer.test.ts index 32adc01b6..3e7a8ed64 100644 --- a/js/src/template/renderer.test.ts +++ b/js/src/template/renderer.test.ts @@ -52,13 +52,6 @@ describe("renderTemplateContent", () => { return JSON.stringify(v); }; - test("renders mustache templates", () => { - const result = renderTemplateContent("Hello {{name}}!", variables, escape, { - templateFormat: "mustache", - }); - expect(result).toBe("Hello World!"); - }); - test("renders with none format (no templating)", () => { const result = renderTemplateContent("Hello {{name}}!", variables, escape, { templateFormat: "none", @@ -76,10 +69,202 @@ describe("renderTemplateContent", () => { expect(result).toBe("Value: 42"); }); - test("escapes non-string values in mustache", () => { - const result = renderTemplateContent("Data: {{value}}", variables, escape, { - templateFormat: "mustache", + describe("mustache", () => { + test("renders basic template", () => { + const result = renderTemplateContent( + "Hello {{name}}!", + variables, + escape, + { + templateFormat: "mustache", + }, + ); + expect(result).toBe("Hello World!"); + }); + + test("escapes non-string values", () => { + const result = renderTemplateContent( + "Data: {{value}}", + variables, + escape, + { + templateFormat: "mustache", + }, + ); + expect(result).toBe("Data: 42"); + }); + + test("renders objects as JSON strings (not [object Object])", () => { + const varsWithObject = { + user: { name: "Alice", age: 30 }, + items: ["a", "b", "c"], + }; + const result = renderTemplateContent( + "User: {{user}}, Items: {{items}}", + varsWithObject, + escape, + { templateFormat: "mustache" }, + ); + expect(result).toBe( + 'User: {"name":"Alice","age":30}, Items: ["a","b","c"]', + ); + expect(result).not.toContain("[object Object]"); + }); + + test("renders nested objects as JSON strings", () => { + const varsWithNested = { + data: { + nested: { + value: 123, + items: ["x", "y"], + }, + }, + }; + const result = renderTemplateContent( + "Data: {{data}}", + varsWithNested, + escape, + { templateFormat: "mustache" }, + ); + expect(result).toBe('Data: {"nested":{"value":123,"items":["x","y"]}}'); + expect(result).not.toContain("[object Object]"); + }); + + test("renders array of strings", () => { + const varsWithStrings = { + tags: ["javascript", "typescript", "nodejs"], + }; + const result = renderTemplateContent( + "Tags: {{tags}}", + varsWithStrings, + escape, + { templateFormat: "mustache" }, + ); + expect(result).toBe('Tags: ["javascript","typescript","nodejs"]'); + }); + + test("renders array of numbers", () => { + const varsWithNumbers = { + scores: [95, 87, 92, 100], + }; + const result = renderTemplateContent( + "Scores: {{scores}}", + varsWithNumbers, + escape, + { templateFormat: "mustache" }, + ); + expect(result).toBe("Scores: [95,87,92,100]"); + }); + + test("renders array of objects", () => { + const varsWithObjects = { + users: [ + { id: 1, name: "Alice" }, + { id: 2, name: "Bob" }, + { id: 3, name: "Charlie" }, + ], + }; + const result = renderTemplateContent( + "Users: {{users}}", + varsWithObjects, + escape, + { templateFormat: "mustache" }, + ); + expect(result).toBe( + 'Users: [{"id":1,"name":"Alice"},{"id":2,"name":"Bob"},{"id":3,"name":"Charlie"}]', + ); + }); + + test("renders mixed array with strings, numbers, and objects", () => { + const varsWithMixed = { + data: ["hello", 42, { type: "object", value: true }, null, "world"], + }; + const result = renderTemplateContent( + "Mixed: {{data}}", + varsWithMixed, + escape, + { templateFormat: "mustache" }, + ); + expect(result).toBe( + 'Mixed: ["hello",42,{"type":"object","value":true},null,"world"]', + ); + }); + + test("renders comma-separated image URLs as string", () => { + const varsWithUrls = { + images: + "https://example.com/image1.jpg, https://example.com/image2.jpg", + }; + const result = renderTemplateContent("{{images}}", varsWithUrls, escape, { + templateFormat: "mustache", + }); + expect(result).toBe( + "https://example.com/image1.jpg, https://example.com/image2.jpg", + ); + }); + + test("renders braintrust_attachment object", () => { + const varsWithAttachment = { + images: { + type: "braintrust_attachment", + filename: "deep.txt", + content_type: "text/plain", + key: "attachments/deep/deep.txt", + }, + }; + const result = renderTemplateContent( + "{{images}}", + varsWithAttachment, + escape, + { templateFormat: "mustache" }, + ); + expect(result).toBe( + '{"type":"braintrust_attachment","filename":"deep.txt","content_type":"text/plain","key":"attachments/deep/deep.txt"}', + ); + }); + + test("renders array of braintrust_attachment objects", () => { + const varsWithAttachments = { + images: [ + { + type: "braintrust_attachment", + filename: "image1.jpg", + content_type: "image/jpeg", + key: "attachments/image1.jpg", + }, + { + type: "braintrust_attachment", + filename: "image2.jpg", + content_type: "image/jpeg", + key: "attachments/image2.jpg", + }, + ], + }; + const result = renderTemplateContent( + "{{images}}", + varsWithAttachments, + escape, + { templateFormat: "mustache" }, + ); + expect(result).toBe( + '[{"type":"braintrust_attachment","filename":"image1.jpg","content_type":"image/jpeg","key":"attachments/image1.jpg"},{"type":"braintrust_attachment","filename":"image2.jpg","content_type":"image/jpeg","key":"attachments/image2.jpg"}]', + ); + }); + + test("renders comma-separated braintrust_attachment objects as string", () => { + const varsWithAttachments = { + images: + '{"type":"braintrust_attachment","filename":"image1.jpg","key":"attachments/image1.jpg"}, {"type":"braintrust_attachment","filename":"image2.jpg","key":"attachments/image2.jpg"}', + }; + const result = renderTemplateContent( + "{{images}}", + varsWithAttachments, + escape, + { templateFormat: "mustache" }, + ); + expect(result).toBe( + '{"type":"braintrust_attachment","filename":"image1.jpg","key":"attachments/image1.jpg"}, {"type":"braintrust_attachment","filename":"image2.jpg","key":"attachments/image2.jpg"}', + ); }); - expect(result).toBe("Data: 42"); }); });