From ebbeae3684b41d263f4b6c59aec5caefdc2d7721 Mon Sep 17 00:00:00 2001 From: Roblmvp Date: Sat, 6 Jun 2026 11:23:44 -0500 Subject: [PATCH 01/17] Update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6ba722675f0..bf053cdad63 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,5 @@ next-env.d.ts # Sentry Config File .env.sentry-build-plugin .env.docker +.env.production.vyaxis + .env* From 1a66257c0c69550021202f1029ad5adecc95ea8e Mon Sep 17 00:00:00 2001 From: Roblmvp Date: Sat, 6 Jun 2026 11:25:23 -0500 Subject: [PATCH 02/17] Update .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index bf053cdad63..e0f8a4a958d 100644 --- a/.gitignore +++ b/.gitignore @@ -50,4 +50,4 @@ next-env.d.ts .env.sentry-build-plugin .env.docker .env.production.vyaxis - .env* +.env* From 2d359978c77a5c918379d6f5543ab2a442429698 Mon Sep 17 00:00:00 2001 From: Roblmvp Date: Sat, 6 Jun 2026 11:26:25 -0500 Subject: [PATCH 03/17] Update prompts.config.ts --- prompts.config.ts | 73 ++++++++++++++--------------------------------- 1 file changed, 21 insertions(+), 52 deletions(-) diff --git a/prompts.config.ts b/prompts.config.ts index 7c8a9e6c7f1..e0938b5cecc 100644 --- a/prompts.config.ts +++ b/prompts.config.ts @@ -1,92 +1,61 @@ import { defineConfig } from "@/lib/config"; -// Set to true to use clone branding (hide prompts.chat repo branding) -const useCloneBranding = false; +// White-label mode ON: hide all prompts.chat repo branding, +// achievements (Forbes/GitHub stars), and sponsor links. +const useCloneBranding = true; export default defineConfig({ - // Branding - customize for white-label + // Branding — swap logo/favicon for your design-system assets in /public branding: { - name: "prompts.chat", + name: "Vyaxis Prompt Library", logo: "/logo.svg", logoDark: "/logo-dark.svg", favicon: "/logo.svg", - description: "Collect, organize, and share AI prompts", - - // Delete this if useCloneBranding is true - appStoreUrl: "https://apps.apple.com/tr/app/prompts-chat/id6756895736", - chromeExtensionUrl: "https://chromewebstore.google.com/detail/promptschat/eemdohkhbaifiocagjlhibfbhamlbeej", + description: "Governed prompt catalog for Vyaxis agents, BidIQ, and the acquisition stack.", + // NOTE: appStoreUrl / chromeExtensionUrl intentionally removed (clone branding). }, - // Theme - design system configuration + // Theme — primary is a placeholder; set from your Vyaxis design-system token. theme: { - // Border radius: "none" | "sm" | "md" | "lg" radius: "sm", - // UI style: "flat" | "default" | "brutal" variant: "default", - // Spacing density: "compact" | "default" | "comfortable" density: "default", - // Colors (hex or oklch) colors: { - primary: "#6366f1", // Indigo + primary: "#6366f1", // TODO: replace with Vyaxis accent hex }, }, - // Authentication plugins + // Auth — GitHub (your Vyaxis-LLC / RoblImvp org) + credentials for staff + // without GitHub accounts. Registration locked; you seed the admin. auth: { - // Available: "credentials" | "google" | "azure" | "github" | "apple" | "oidc" | "oauth" | custom - // Use `providers` array to enable multiple auth providers - providers: ["github", "google", "apple"], - // Allow public registration (only applies to credentials provider) + providers: ["github", "credentials"], allowRegistration: false, }, - // Internationalization + // English only for internal use (add "es" if dealership staff need it). i18n: { - locales: ["en", "tr", "es", "zh", "ja", "ar", "pt", "fr", "it", "de", "nl", "ko", "ru", "he", "el", "az", "fa"], + locales: ["en"], defaultLocale: "en", }, // Features features: { - // Allow users to create private prompts privatePrompts: true, - // Enable change request system for versioning changeRequests: true, - // Enable categories categories: true, - // Enable tags tags: true, - // Enable AI-powered semantic search (requires OPENAI_API_KEY) - aiSearch: true, - // Enable AI-powered generation features (requires OPENAI_API_KEY) - aiGeneration: true, - // Enable MCP (Model Context Protocol) features including API key generation + // AI search/gen need an OpenAI(-compatible) key + embedding model. + aiSearch: false, + aiGeneration: false, + // MCP on: API-key generation so agents can pull prompts programmatically. mcp: true, - // Enable comments on prompts comments: true, }, - // Homepage customization + // Homepage — clone branding hides repo achievements + sponsors. homepage: { - // Set to true to hide prompts.chat repo branding and use your own branding useCloneBranding, - achievements: { - enabled: !useCloneBranding, - }, - sponsors: { - enabled: !useCloneBranding, - items: [ - // Add sponsors here - { name: "Neon", className: 'py-1', logo: '/sponsors/neon.svg', darkLogo: '/sponsors/neon-dark.svg', url: "https://get.neon.com/VqfnMo4" }, - { name: "Clemta", logo: '/sponsors/clemta.webp', url: "https://clemta.com/?utm_source=prompts.chat" }, - { name: "Wiro.ai", className: 'py-1', darkLogo: '/sponsors/wiro.png', logo: '/sponsors/wiro.png', url: "https://wiro.ai/?utm_source=prompts.chat" }, - { name: "Cognition", logo: "/sponsors/cognition.svg", url: "https://wind.surf/prompts-chat" }, - { name: "CodeRabbit", className: 'py-1', logo: '/sponsors/coderabbit.svg', darkLogo: '/sponsors/coderabbit-dark.svg', url: "https://coderabbit.link/fatih" }, - { name: "Sentry", className: 'py-1', logo: '/sponsors/sentry.svg', darkLogo: '/sponsors/sentry-dark.svg', url: "https://sentry.io/?utm_source=prompts.chat" }, - - { name: "eachlabs", className: 'py-[6px]', logo: '/sponsors/eachlabs.png', darkLogo: '/sponsors/eachlabs-dark.png', url: "https://www.eachlabs.ai/?utm_source=promptschat&utm_medium=referral" }, - { name: "CommandCode", className: 'py-1', logo: '/sponsors/commandcode.svg', darkLogo: '/sponsors/commandcode-dark.svg', url: "https://commandcode.ai/?utm_source=prompts.chat" }, - ], - }, + achievements: { enabled: !useCloneBranding }, + sponsors: { enabled: !useCloneBranding }, }, }); From eee0c1523e47865b88c3f1c46b8ae7971db8fcc2 Mon Sep 17 00:00:00 2001 From: Roblmvp Date: Sat, 6 Jun 2026 12:02:03 -0500 Subject: [PATCH 04/17] Update prompts.config.ts --- prompts.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prompts.config.ts b/prompts.config.ts index e0938b5cecc..6df7a68b5b3 100644 --- a/prompts.config.ts +++ b/prompts.config.ts @@ -56,6 +56,6 @@ export default defineConfig({ homepage: { useCloneBranding, achievements: { enabled: !useCloneBranding }, - sponsors: { enabled: !useCloneBranding }, +sponsors: { enabled: !useCloneBranding, items: [] }, }, }); From 04cfa1be094a5025db28880a64a5f64f37a75129 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 6 Jun 2026 18:53:04 +0000 Subject: [PATCH 05/17] feat(db): add multi-tenancy and execution trace tables (Step 1) Adds the Organization model and nullable organizationId FKs on User and Prompt, plus the ExecutionRecord and StageTrace tables and the ExecutionStatus / AdaptationMode enums for the Composition Engine execution traces. Scope is intentionally atomic (Step 1 only): organizationId is nullable; backfill and the NOT NULL flip are deferred to Step 1b. compositionId is a plain indexed column with no FK (Composition model lands in Phase 2). https://claude.ai/code/session_01HhSSACKMJYh92GqDAakXHT --- .../migration.sql | 101 ++++++++ prisma/schema.prisma | 224 ++++++++++++------ 2 files changed, 255 insertions(+), 70 deletions(-) create mode 100644 prisma/migrations/20260606185240_add_multitenancy_and_execution_traces/migration.sql diff --git a/prisma/migrations/20260606185240_add_multitenancy_and_execution_traces/migration.sql b/prisma/migrations/20260606185240_add_multitenancy_and_execution_traces/migration.sql new file mode 100644 index 00000000000..7b4d6869bc9 --- /dev/null +++ b/prisma/migrations/20260606185240_add_multitenancy_and_execution_traces/migration.sql @@ -0,0 +1,101 @@ +-- CreateEnum +CREATE TYPE "ExecutionStatus" AS ENUM ('SUCCESS', 'PARTIAL', 'FAILED'); + +-- CreateEnum +CREATE TYPE "AdaptationMode" AS ENUM ('CONFIG', 'LLM'); + +-- AlterTable +ALTER TABLE "users" ADD COLUMN "organizationId" TEXT; + +-- AlterTable +ALTER TABLE "prompts" ADD COLUMN "organizationId" TEXT; + +-- CreateTable +CREATE TABLE "organizations" ( + "id" TEXT NOT NULL, + "name" TEXT NOT NULL, + "slug" TEXT NOT NULL, + "tier" TEXT NOT NULL DEFAULT 'internal', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "organizations_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "execution_records" ( + "id" TEXT NOT NULL, + "compositionId" TEXT NOT NULL, + "organizationId" TEXT NOT NULL, + "status" "ExecutionStatus" NOT NULL, + "input" TEXT NOT NULL, + "finalOutput" TEXT, + "totalTokens" INTEGER NOT NULL DEFAULT 0, + "totalDurationMs" INTEGER NOT NULL DEFAULT 0, + "overallScore" DOUBLE PRECISION, + "startedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "completedAt" TIMESTAMP(3), + + CONSTRAINT "execution_records_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "stage_traces" ( + "id" TEXT NOT NULL, + "executionId" TEXT NOT NULL, + "stageOrder" INTEGER NOT NULL, + "promptId" TEXT NOT NULL, + "promptVersion" INTEGER NOT NULL, + "rawInput" TEXT NOT NULL, + "adaptedInput" TEXT NOT NULL, + "output" TEXT, + "adaptationMode" "AdaptationMode" NOT NULL DEFAULT 'CONFIG', + "rulesApplied" TEXT[], + "tokensUsed" INTEGER NOT NULL DEFAULT 0, + "durationMs" INTEGER NOT NULL DEFAULT 0, + "model" TEXT, + "score" DOUBLE PRECISION, + "thresholdMet" BOOLEAN, + "fallbackApplied" BOOLEAN NOT NULL DEFAULT false, + "fallbackPromptId" TEXT, + "error" TEXT, + + CONSTRAINT "stage_traces_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "organizations_slug_key" ON "organizations"("slug"); + +-- CreateIndex +CREATE INDEX "execution_records_organizationId_idx" ON "execution_records"("organizationId"); + +-- CreateIndex +CREATE INDEX "execution_records_compositionId_idx" ON "execution_records"("compositionId"); + +-- CreateIndex +CREATE INDEX "stage_traces_executionId_idx" ON "stage_traces"("executionId"); + +-- CreateIndex +CREATE INDEX "stage_traces_promptId_idx" ON "stage_traces"("promptId"); + +-- CreateIndex +CREATE UNIQUE INDEX "stage_traces_executionId_stageOrder_key" ON "stage_traces"("executionId", "stageOrder"); + +-- CreateIndex +CREATE INDEX "users_organizationId_idx" ON "users"("organizationId"); + +-- CreateIndex +CREATE INDEX "prompts_organizationId_idx" ON "prompts"("organizationId"); + +-- AddForeignKey +ALTER TABLE "users" ADD CONSTRAINT "users_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "organizations"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "prompts" ADD CONSTRAINT "prompts_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "organizations"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "execution_records" ADD CONSTRAINT "execution_records_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "organizations"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "stage_traces" ADD CONSTRAINT "stage_traces_executionId_fkey" FOREIGN KEY ("executionId") REFERENCES "execution_records"("id") ON DELETE CASCADE ON UPDATE CASCADE; + diff --git a/prisma/schema.prisma b/prisma/schema.prisma index b58dfe55795..c56aa9a30e3 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -9,46 +9,49 @@ datasource db { } model User { - id String @id @default(cuid()) - email String @unique - username String @unique - name String? - password String? - avatar String? - bio String? @db.VarChar(250) - customLinks Json? // Array of {type: string, url: string, label?: string} - role UserRole @default(USER) - locale String @default("en") - emailVerified DateTime? - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - verified Boolean @default(false) - githubUsername String? - apiKey String? @unique - mcpPromptsPublicByDefault Boolean @default(false) - flagged Boolean @default(false) - flaggedAt DateTime? - flaggedReason String? - dailyGenerationLimit Int @default(3) - generationCreditsRemaining Int @default(3) - generationCreditsResetAt DateTime? - accounts Account[] - subscriptions CategorySubscription[] - changeRequests ChangeRequest[] - pinnedPrompts PinnedPrompt[] - reports PromptReport[] - promptVersions PromptVersion[] - votes PromptVote[] - prompts Prompt[] @relation("PromptAuthor") - sessions Session[] - contributions Prompt[] @relation("PromptContributors") - comments Comment[] @relation("CommentAuthor") - commentVotes CommentVote[] - notifications Notification[] @relation("NotificationRecipient") - notificationsActed Notification[] @relation("NotificationActor") - collections Collection[] - userPromptExamples UserPromptExample[] - + id String @id @default(cuid()) + email String @unique + username String @unique + name String? + password String? + avatar String? + bio String? @db.VarChar(250) + customLinks Json? // Array of {type: string, url: string, label?: string} + role UserRole @default(USER) + locale String @default("en") + emailVerified DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + verified Boolean @default(false) + githubUsername String? + apiKey String? @unique + mcpPromptsPublicByDefault Boolean @default(false) + flagged Boolean @default(false) + flaggedAt DateTime? + flaggedReason String? + dailyGenerationLimit Int @default(3) + generationCreditsRemaining Int @default(3) + generationCreditsResetAt DateTime? + accounts Account[] + subscriptions CategorySubscription[] + changeRequests ChangeRequest[] + pinnedPrompts PinnedPrompt[] + reports PromptReport[] + promptVersions PromptVersion[] + votes PromptVote[] + prompts Prompt[] @relation("PromptAuthor") + sessions Session[] + contributions Prompt[] @relation("PromptContributors") + comments Comment[] @relation("CommentAuthor") + commentVotes CommentVote[] + notifications Notification[] @relation("NotificationRecipient") + notificationsActed Notification[] @relation("NotificationActor") + collections Collection[] + userPromptExamples UserPromptExample[] + organizationId String? + organization Organization? @relation(fields: [organizationId], references: [id]) + + @@index([organizationId]) @@map("users") } @@ -91,27 +94,27 @@ model VerificationToken { } model Prompt { - id String @id @default(cuid()) + id String @id @default(cuid()) title String slug String? description String? content String - type PromptType @default(TEXT) - isPrivate Boolean @default(false) + type PromptType @default(TEXT) + isPrivate Boolean @default(false) mediaUrl String? - viewCount Int @default(0) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt + viewCount Int @default(0) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt authorId String categoryId String? embedding Json? requiredMediaCount Int? requiredMediaType RequiredMediaType? - requiresMediaUpload Boolean @default(false) + requiresMediaUpload Boolean @default(false) structuredFormat StructuredFormat? featuredAt DateTime? - isFeatured Boolean @default(false) - isUnlisted Boolean @default(false) + isFeatured Boolean @default(false) + isUnlisted Boolean @default(false) unlistedAt DateTime? delistReason DelistReason? deletedAt DateTime? @@ -122,16 +125,18 @@ model Prompt { versions PromptVersion[] votes PromptVote[] comments Comment[] - author User @relation("PromptAuthor", fields: [authorId], references: [id], onDelete: Cascade) - category Category? @relation(fields: [categoryId], references: [id]) - contributors User[] @relation("PromptContributors") - outgoingConnections PromptConnection[] @relation("ConnectionSource") - incomingConnections PromptConnection[] @relation("ConnectionTarget") + author User @relation("PromptAuthor", fields: [authorId], references: [id], onDelete: Cascade) + category Category? @relation(fields: [categoryId], references: [id]) + contributors User[] @relation("PromptContributors") + outgoingConnections PromptConnection[] @relation("ConnectionSource") + incomingConnections PromptConnection[] @relation("ConnectionTarget") collectedBy Collection[] userExamples UserPromptExample[] - bestWithModels String[] // Model slugs this prompt works best with (max 3), e.g. ["gpt-4o", "claude-3-5-sonnet"] - bestWithMCP Json? // MCP configs array, e.g. [{command: "npx -y @mcp/server", tools: ["tool1"]}] - workflowLink String? // URL to test/demo the workflow when prompt has previous/next connections + bestWithModels String[] // Model slugs this prompt works best with (max 3), e.g. ["gpt-4o", "claude-3-5-sonnet"] + bestWithMCP Json? // MCP configs array, e.g. [{command: "npx -y @mcp/server", tools: ["tool1"]}] + workflowLink String? // URL to test/demo the workflow when prompt has previous/next connections + organizationId String? + organization Organization? @relation(fields: [organizationId], references: [id]) @@index([authorId]) @@index([categoryId]) @@ -141,6 +146,7 @@ model Prompt { @@index([isUnlisted]) @@index([slug]) @@index([isPrivate, isUnlisted, deletedAt, createdAt(sort: Desc)]) + @@index([organizationId]) @@map("prompts") } @@ -349,10 +355,10 @@ model Comment { model CommentVote { userId String commentId String - value Int // 1 for upvote, -1 for downvote - createdAt DateTime @default(now()) - comment Comment @relation(fields: [commentId], references: [id], onDelete: Cascade) - user User @relation(fields: [userId], references: [id], onDelete: Cascade) + value Int // 1 for upvote, -1 for downvote + createdAt DateTime @default(now()) + comment Comment @relation(fields: [commentId], references: [id], onDelete: Cascade) + user User @relation(fields: [userId], references: [id], onDelete: Cascade) @@id([userId, commentId]) @@index([userId]) @@ -378,15 +384,15 @@ model Notification { } model PromptConnection { - id String @id @default(cuid()) - sourceId String - targetId String - label String - order Int @default(0) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - source Prompt @relation("ConnectionSource", fields: [sourceId], references: [id], onDelete: Cascade) - target Prompt @relation("ConnectionTarget", fields: [targetId], references: [id], onDelete: Cascade) + id String @id @default(cuid()) + sourceId String + targetId String + label String + order Int @default(0) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + source Prompt @relation("ConnectionSource", fields: [sourceId], references: [id], onDelete: Cascade) + target Prompt @relation("ConnectionTarget", fields: [targetId], references: [id], onDelete: Cascade) @@unique([sourceId, targetId]) @@index([sourceId]) @@ -399,6 +405,84 @@ enum NotificationType { REPLY } +// --------------------------------------------------------------------------- +// Phase 1: Multi-tenancy + Composition Engine execution traces +// Added by service-extraction Day-1 migration. See SERVICE-EXTRACTION-PLAN.md +// and EXECUTION-TRACE-SCORING-SPEC.md for rationale. +// --------------------------------------------------------------------------- + +model Organization { + id String @id @default(cuid()) + name String + slug String @unique + tier String @default("internal") // internal | pro | enterprise (field only; no enforcement in v1 per Q4) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + users User[] + prompts Prompt[] + executionRecords ExecutionRecord[] + + @@map("organizations") +} + +model ExecutionRecord { + id String @id @default(cuid()) + compositionId String // opaque id; FK added in Phase 2 when Composition model lands (no orphan FK now) + organizationId String + status ExecutionStatus + input String + finalOutput String? + totalTokens Int @default(0) + totalDurationMs Int @default(0) + overallScore Float? // null in v1 (manual review); populated v1.1 per Q2 + startedAt DateTime @default(now()) + completedAt DateTime? + organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade) + stageTraces StageTrace[] + + @@index([organizationId]) + @@index([compositionId]) + @@map("execution_records") +} + +model StageTrace { + id String @id @default(cuid()) + executionId String + stageOrder Int + promptId String // references prompts.id by value; no FK on the hot prompts table by design + promptVersion Int + rawInput String // input BEFORE adaptation + adaptedInput String // input AFTER adaptation rules + output String? + adaptationMode AdaptationMode @default(CONFIG) // CONFIG only in v1; LLM dark-shipped per Q1 + rulesApplied String[] + tokensUsed Int @default(0) + durationMs Int @default(0) + model String? + score Float? // null in v1 per Q2 + thresholdMet Boolean? // null in v1 per Q2 + fallbackApplied Boolean @default(false) + fallbackPromptId String? + error String? + execution ExecutionRecord @relation(fields: [executionId], references: [id], onDelete: Cascade) + + @@unique([executionId, stageOrder]) + @@index([executionId]) + @@index([promptId]) + @@map("stage_traces") +} + +enum ExecutionStatus { + SUCCESS + PARTIAL + FAILED +} + +enum AdaptationMode { + CONFIG + LLM +} + enum UserRole { ADMIN USER From d1f357e5078c02742fbafeffaab25dc17f0bb7ed Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 6 Jun 2026 19:25:37 +0000 Subject: [PATCH 06/17] feat(db): backfill default internal organization (Step 1b) Data-only migration: creates the single default "Vyaxis (Internal)" org (slug "internal") and assigns all existing org-less users and prompts to it. Idempotent (ON CONFLICT on slug; UPDATEs scoped to NULLs). organizationId remains nullable and the FK is unchanged; the NOT NULL flip and FK on-delete change are deferred to Step 1c, after the app write-paths populate organizationId. https://claude.ai/code/session_01HhSSACKMJYh92GqDAakXHT --- .../migration.sql | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 prisma/migrations/20260606192213_backfill_default_org/migration.sql diff --git a/prisma/migrations/20260606192213_backfill_default_org/migration.sql b/prisma/migrations/20260606192213_backfill_default_org/migration.sql new file mode 100644 index 00000000000..243af9e8c4b --- /dev/null +++ b/prisma/migrations/20260606192213_backfill_default_org/migration.sql @@ -0,0 +1,23 @@ +-- Step 1b: Backfill default organization for existing rows. +-- DATA-ONLY migration. No schema changes, no constraints altered. +-- Idempotent: safe to run more than once (ON CONFLICT on slug; UPDATEs scoped to NULLs). +-- The NOT NULL flip on organizationId is intentionally DEFERRED to a later migration +-- (Step 1c), after application write-paths set organizationId. See HANDOFF-STEP1B.md. + +-- 1. Create the single default internal organization. +-- id is supplied explicitly because cuid() is a Prisma application-level default, +-- not a database default; raw SQL must provide it. updatedAt likewise has no DB +-- default (@updatedAt is app-level), so it is set explicitly. +INSERT INTO "organizations" ("id", "name", "slug", "tier", "createdAt", "updatedAt") +VALUES (gen_random_uuid()::text, 'Vyaxis (Internal)', 'internal', 'internal', now(), now()) +ON CONFLICT ("slug") DO NOTHING; + +-- 2. Assign all existing org-less users to the internal org. +UPDATE "users" +SET "organizationId" = (SELECT "id" FROM "organizations" WHERE "slug" = 'internal') +WHERE "organizationId" IS NULL; + +-- 3. Assign all existing org-less prompts to the internal org. +UPDATE "prompts" +SET "organizationId" = (SELECT "id" FROM "organizations" WHERE "slug" = 'internal') +WHERE "organizationId" IS NULL; From ad9af0756e2db3568c9ee5db911e013ddf2a473b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 6 Jun 2026 19:56:32 +0000 Subject: [PATCH 07/17] feat(observability): structured logging + request correlation (Step 2) Adds a Pino structured logger (src/lib/logger.ts) emitting JSON to stdout with service/env base fields and defensive redaction of password/apiKey/token/authorization. proxy.ts now propagates an x-request-id correlation id (reuses an inbound one, else mints a UUID), which route handlers read via requestLogger(). Adopted in the collection route (add/remove) as the first consumer. Path 1: complements the existing Sentry stack, does not replace it. No @vercel/otel (would conflict with Sentry's OTel). Sentry trace correlation deferred (requestId-only already satisfies the core need). https://claude.ai/code/session_01HhSSACKMJYh92GqDAakXHT --- src/app/api/collection/route.ts | 15 ++++++++--- src/lib/logger.ts | 47 +++++++++++++++++++++++++++++++++ src/proxy.ts | 7 +++++ 3 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 src/lib/logger.ts diff --git a/src/app/api/collection/route.ts b/src/app/api/collection/route.ts index e7216c6b79d..a9682993fb9 100644 --- a/src/app/api/collection/route.ts +++ b/src/app/api/collection/route.ts @@ -1,6 +1,7 @@ import { NextRequest, NextResponse } from "next/server"; import { auth } from "@/lib/auth"; import { db } from "@/lib/db"; +import { requestLogger } from "@/lib/logger"; import { z } from "zod"; const addToCollectionSchema = z.object({ @@ -53,8 +54,10 @@ export async function GET() { } export async function POST(req: NextRequest) { + const log = requestLogger(req.headers.get("x-request-id")); + const start = Date.now(); const session = await auth(); - + if (!session?.user) { return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); } @@ -96,19 +99,22 @@ export async function POST(req: NextRequest) { }, }); + log.info({ op: "collection.add", promptId, durationMs: Date.now() - start }, "added to collection"); return NextResponse.json({ collection, added: true }); } catch (error) { if (error instanceof z.ZodError) { return NextResponse.json({ error: "Invalid input" }, { status: 400 }); } - console.error("Failed to add to collection:", error); + log.error({ op: "collection.add", err: error, durationMs: Date.now() - start }, "failed to add to collection"); return NextResponse.json({ error: "Failed to add to collection" }, { status: 500 }); } } export async function DELETE(req: NextRequest) { + const log = requestLogger(req.headers.get("x-request-id")); + const start = Date.now(); const session = await auth(); - + if (!session?.user) { return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); } @@ -130,9 +136,10 @@ export async function DELETE(req: NextRequest) { }, }); + log.info({ op: "collection.remove", promptId, durationMs: Date.now() - start }, "removed from collection"); return NextResponse.json({ removed: true }); } catch (error) { - console.error("Failed to remove from collection:", error); + log.error({ op: "collection.remove", err: error, durationMs: Date.now() - start }, "failed to remove from collection"); return NextResponse.json({ error: "Failed to remove from collection" }, { status: 500 }); } } diff --git a/src/lib/logger.ts b/src/lib/logger.ts new file mode 100644 index 00000000000..a912c4d94e1 --- /dev/null +++ b/src/lib/logger.ts @@ -0,0 +1,47 @@ +import pino, { type Logger } from "pino"; + +// Structured logger for Vyaxis Prompts. +// +// Design (Step 2, Path 1 — complements the existing Sentry stack, does not replace it): +// - Emits JSON to stdout. Vercel auto-captures stdout into runtime logs, and Vercel +// Drains (Pro) can export it to any OTel-compatible backend. +// - Pino is also Fastify's native logger, so this module ports to the extracted +// Prompts Service with no rewrite. +// - Redacts known-sensitive fields defensively (the User model carries password/apiKey). +// +// What this adds that Sentry does not: request-scoped correlation IDs for queryable, +// structured operation logs (recommendation latency, chain execution timing, etc.). + +const isProd = process.env.NODE_ENV === "production"; + +export const logger: Logger = pino({ + level: process.env.LOG_LEVEL ?? (isProd ? "info" : "debug"), + base: { + service: "vyaxis-prompts", + env: process.env.VERCEL_ENV ?? process.env.NODE_ENV ?? "development", + }, + formatters: { + // Emit the level as a string label ("info") instead of the numeric code. + level: (label: string) => ({ level: label }), + }, + redact: { + paths: [ + "password", + "*.password", + "apiKey", + "*.apiKey", + "token", + "*.token", + "authorization", + "*.authorization", + ], + censor: "[redacted]", + }, +}); + +// Create a request-scoped child logger bound to a correlation id. +// The id is set on the inbound request by src/proxy.ts (header `x-request-id`) +// and read back in route handlers via request.headers.get("x-request-id"). +export function requestLogger(requestId: string | null | undefined): Logger { + return logger.child({ requestId: requestId ?? "unknown" }); +} diff --git a/src/proxy.ts b/src/proxy.ts index e8daea8d1a2..0681f7d483a 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -15,6 +15,13 @@ export function proxy(request: NextRequest) { const requestHeaders = new Headers(request.headers); requestHeaders.set("x-pathname", pathname); + // Propagate a correlation id for request-scoped logging (Step 2). + // Reuse an inbound id if one is present (e.g. from an upstream proxy), else mint one. + // Read back in route handlers via request.headers.get("x-request-id"). + if (!requestHeaders.get("x-request-id")) { + requestHeaders.set("x-request-id", crypto.randomUUID()); + } + return NextResponse.next({ request: { headers: requestHeaders, From b40fe708837a7a3d21db6cf7a9733afbd5282fec Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 6 Jun 2026 19:56:32 +0000 Subject: [PATCH 08/17] fix(sentry): source DSN from env and disable PII (Step 2a) The hardcoded DSN was the upstream prompts.chat project's (org "promptschat"); with enabled-in-production + sendDefaultPii:true a fork deploy would send users' errors/traces/logs and PII to the upstream maintainer's Sentry. Move the DSN to env (SENTRY_DSN for server/edge, NEXT_PUBLIC_SENTRY_DSN for the browser client) defaulting to empty, which disables sending until a self-owned project is configured, and set sendDefaultPii:false everywhere. Documented the vars in .env.example. https://claude.ai/code/session_01HhSSACKMJYh92GqDAakXHT --- .env.example | 7 +++++++ sentry.edge.config.ts | 8 +++++--- sentry.server.config.ts | 8 +++++--- src/instrumentation-client.ts | 9 ++++++--- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.env.example b/.env.example index 30579bceffa..32346563390 100644 --- a/.env.example +++ b/.env.example @@ -63,6 +63,13 @@ CRON_SECRET="your-secret-key-here" # FAL_VIDEO_MODELS="fal-ai/veo3,fal-ai/kling-video/v2/master/text-to-video" # Comma-separated list of video models # FAL_IMAGE_MODELS="fal-ai/flux-pro/v1.1-ultra,fal-ai/flux/dev" # Comma-separated list of image models +# Sentry (optional). Leave the DSNs unset to disable error/trace reporting. +# Set them to YOUR OWN Sentry project's DSN to enable it — do NOT reuse the +# upstream project's DSN. SENTRY_DSN is used by the server/edge runtimes; +# NEXT_PUBLIC_SENTRY_DSN is used by the browser client (and is typically the +# same DSN value). SENTRY_AUTH_TOKEN is only needed for source-map uploads. +# SENTRY_DSN= +# NEXT_PUBLIC_SENTRY_DSN= # SENTRY_AUTH_TOKEN=sentry-auth-token # GOOGLE_ADSENSE_ACCOUNT=ca-pub-xxxxxxxxxxxxxxxx diff --git a/sentry.edge.config.ts b/sentry.edge.config.ts index 44b49909cac..ea66435b7f6 100644 --- a/sentry.edge.config.ts +++ b/sentry.edge.config.ts @@ -6,7 +6,9 @@ import * as Sentry from "@sentry/nextjs"; Sentry.init({ - dsn: "https://9c2eb3b4441745efad28a908001c30bf@o4510673866063872.ingest.de.sentry.io/4510673871306832", + // DSN comes from the environment so this fork does not ship telemetry to the + // upstream project's Sentry. Empty/unset disables sending (set your own to enable). + dsn: process.env.SENTRY_DSN ?? "", // Disable Sentry in development enabled: process.env.NODE_ENV === "production", @@ -17,7 +19,7 @@ Sentry.init({ // Enable logs to be sent to Sentry enableLogs: true, - // Enable sending user PII (Personally Identifiable Information) + // Do not attach user PII (IP, headers, etc.) to events by default. // https://docs.sentry.io/platforms/javascript/guides/nextjs/configuration/options/#sendDefaultPii - sendDefaultPii: true, + sendDefaultPii: false, }); diff --git a/sentry.server.config.ts b/sentry.server.config.ts index 2014e601a39..2f4b2dd6aa2 100644 --- a/sentry.server.config.ts +++ b/sentry.server.config.ts @@ -5,7 +5,9 @@ import * as Sentry from "@sentry/nextjs"; Sentry.init({ - dsn: "https://9c2eb3b4441745efad28a908001c30bf@o4510673866063872.ingest.de.sentry.io/4510673871306832", + // DSN comes from the environment so this fork does not ship telemetry to the + // upstream project's Sentry. Empty/unset disables sending (set your own to enable). + dsn: process.env.SENTRY_DSN ?? "", // Disable Sentry in development enabled: process.env.NODE_ENV === "production", @@ -16,7 +18,7 @@ Sentry.init({ // Enable logs to be sent to Sentry enableLogs: true, - // Enable sending user PII (Personally Identifiable Information) + // Do not attach user PII (IP, headers, etc.) to events by default. // https://docs.sentry.io/platforms/javascript/guides/nextjs/configuration/options/#sendDefaultPii - sendDefaultPii: true, + sendDefaultPii: false, }); diff --git a/src/instrumentation-client.ts b/src/instrumentation-client.ts index 1419df73d68..4705602661e 100644 --- a/src/instrumentation-client.ts +++ b/src/instrumentation-client.ts @@ -23,7 +23,10 @@ const ignoreErrors = [ ]; Sentry.init({ - dsn: "https://9c2eb3b4441745efad28a908001c30bf@o4510673866063872.ingest.de.sentry.io/4510673871306832", + // Client DSN must be NEXT_PUBLIC_ to be inlined into the browser bundle. Sourced + // from the environment so this fork does not ship telemetry to the upstream + // project's Sentry. Empty/unset disables sending (set your own to enable). + dsn: process.env.NEXT_PUBLIC_SENTRY_DSN ?? "", // Disable Sentry in development enabled: process.env.NODE_ENV === "production", @@ -44,9 +47,9 @@ Sentry.init({ // Define how likely Replay events are sampled when an error occurs. replaysOnErrorSampleRate: 1.0, - // Enable sending user PII (Personally Identifiable Information) + // Do not attach user PII (IP, headers, etc.) to events by default. // https://docs.sentry.io/platforms/javascript/guides/nextjs/configuration/options/#sendDefaultPii - sendDefaultPii: true, + sendDefaultPii: false, // Filter out browser extension and third-party script errors beforeSend(event) { From d865273a08a2fb06bec3382a1ed387174489622b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 6 Jun 2026 20:19:57 +0000 Subject: [PATCH 09/17] fix(security): revoke Data-API grants + enable RLS on public tables Closes an active exposure: anon/authenticated held full DML grants on all public tables with RLS disabled, so the public anon key could read/write users.password, accounts OAuth tokens, verification_tokens, etc. via PostgREST over HTTPS (confirmed: anon GET on users returned a password hash). The app is pure-Prisma and connects as the bypass `postgres` role, so this is app-safe. - REVOKE ALL from anon/authenticated on all 26 public tables - ENABLE ROW LEVEL SECURITY (deny-by-default, no policies) on all 26 - ALTER DEFAULT PRIVILEGES FOR ROLE postgres to stop future tables auto-granting anon/authenticated Verified: anon REST read now 401 permission denied; 0 anon grants remain; RLS enabled on 26/26 tables. https://claude.ai/code/session_01HhSSACKMJYh92GqDAakXHT --- .../migration.sql | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 prisma/migrations/20260606200000_secure_public_tables_revoke_and_rls/migration.sql diff --git a/prisma/migrations/20260606200000_secure_public_tables_revoke_and_rls/migration.sql b/prisma/migrations/20260606200000_secure_public_tables_revoke_and_rls/migration.sql new file mode 100644 index 00000000000..df6859cb804 --- /dev/null +++ b/prisma/migrations/20260606200000_secure_public_tables_revoke_and_rls/migration.sql @@ -0,0 +1,62 @@ +-- Security fix: close the Supabase Data API (PostgREST) exposure on public tables. +-- +-- Investigation (RLS review) proved that anon/authenticated held full DML grants on +-- every public table while RLS was disabled, making users.password, accounts OAuth +-- tokens, verification_tokens, etc. readable AND writable via the public anon key +-- over HTTPS. Empirically confirmed: anon GET on /rest/v1/users returned 200 with a +-- password hash. +-- +-- The application is pure-Prisma and connects as the bypass role `postgres` +-- (rolbypassrls = true); it does NOT use the Supabase Data API (no @supabase/* dep, +-- no anon key usage). Therefore revoking these grants and enabling RLS does not +-- affect the app. +-- +-- 1) Revoke all Data-API grants from anon/authenticated on every public table. +-- 2) Enable RLS deny-by-default (no permissive policies) as defense-in-depth. +-- 3) Reverse the postgres default privileges so FUTURE postgres-created tables do +-- not auto-grant anon/authenticated (prevents silent regression). +-- +-- NOTE: the supabase_admin default privileges are intentionally left as Supabase +-- manages them; they govern Supabase-internal table creation, not app tables. + +-- 1) Revoke existing grants on all 26 public tables +REVOKE ALL PRIVILEGES ON TABLE + "_PromptContributors", "_prisma_migrations", "accounts", "categories", + "category_subscriptions", "change_requests", "collections", "comment_votes", + "comments", "execution_records", "notifications", "organizations", "pinned_prompts", + "prompt_connections", "prompt_reports", "prompt_tags", "prompt_versions", + "prompt_votes", "prompts", "sessions", "stage_traces", "tags", + "user_prompt_examples", "users", "verification_tokens", "webhook_configs" + FROM anon, authenticated; + +-- 2) Enable RLS deny-by-default on all 26 public tables +ALTER TABLE "_PromptContributors" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "_prisma_migrations" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "accounts" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "categories" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "category_subscriptions" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "change_requests" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "collections" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "comment_votes" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "comments" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "execution_records" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "notifications" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "organizations" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "pinned_prompts" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "prompt_connections" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "prompt_reports" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "prompt_tags" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "prompt_versions" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "prompt_votes" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "prompts" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "sessions" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "stage_traces" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "tags" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "user_prompt_examples" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "users" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "verification_tokens" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "webhook_configs" ENABLE ROW LEVEL SECURITY; + +-- 3) Prevent future postgres-created tables from auto-granting anon/authenticated +ALTER DEFAULT PRIVILEGES FOR ROLE postgres IN SCHEMA public + REVOKE ALL ON TABLES FROM anon, authenticated; From 9fe7f47eb9220e3f9e7f6e7ee29ec29b60eb7c54 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 6 Jun 2026 20:34:46 +0000 Subject: [PATCH 10/17] feat(db): add Composition + Stage models for Composition Engine (Phase 2) Superset/non-destructive schema change: adds Composition and Stage models and the compositionId -> Composition FK (deferred from Step 1). Reuses the existing audited ExecutionRecord/StageTrace tables as-is, adding only ExecutionRecord.userId (NOT NULL) and sessionId (nullable); all Level-3 scoring columns are preserved. Schema only. The additive migration has NOT been applied to production yet (awaiting approval); no execution/trace columns are dropped. https://claude.ai/code/session_01HhSSACKMJYh92GqDAakXHT --- prisma/schema.prisma | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/prisma/schema.prisma b/prisma/schema.prisma index c56aa9a30e3..394b5cb26b6 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -409,8 +409,45 @@ enum NotificationType { // Phase 1: Multi-tenancy + Composition Engine execution traces // Added by service-extraction Day-1 migration. See SERVICE-EXTRACTION-PLAN.md // and EXECUTION-TRACE-SCORING-SPEC.md for rationale. +// +// Phase 2 (Composition Engine, Level 2): Composition + Stage models added. +// ExecutionRecord/StageTrace are reused as-is (superset) — the audited trace +// columns (rawInput/adaptedInput/promptVersion/score/thresholdMet/...) are +// preserved so Level-3 scoring stays migration-free. // --------------------------------------------------------------------------- +model Composition { + id String @id @default(cuid()) + organizationId String + name String + description String? + createdBy String + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + stages Stage[] + executions ExecutionRecord[] + + @@index([organizationId]) + @@map("compositions") +} + +model Stage { + id String @id @default(cuid()) + compositionId String + composition Composition @relation(fields: [compositionId], references: [id], onDelete: Cascade) + order Int + promptId String + fallbackPromptId String? + successThreshold Float @default(0.5) + extractFields String[] + transformRules Json? + createdAt DateTime @default(now()) + + @@unique([compositionId, order]) + @@index([compositionId]) + @@map("stages") +} + model Organization { id String @id @default(cuid()) name String @@ -427,8 +464,10 @@ model Organization { model ExecutionRecord { id String @id @default(cuid()) - compositionId String // opaque id; FK added in Phase 2 when Composition model lands (no orphan FK now) + compositionId String // Phase 2: now FK-backed by Composition (see `composition` relation) organizationId String + userId String // Phase 2: user who ran the execution + sessionId String? // Phase 2: optional client/session correlation id status ExecutionStatus input String finalOutput String? @@ -438,6 +477,7 @@ model ExecutionRecord { startedAt DateTime @default(now()) completedAt DateTime? organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade) + composition Composition @relation(fields: [compositionId], references: [id], onDelete: Cascade) stageTraces StageTrace[] @@index([organizationId]) From 35797877d3f473eb21aaf0bc3302e6b96cbcff57 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 6 Jun 2026 20:47:25 +0000 Subject: [PATCH 11/17] feat(db): apply composition engine migration + org FK + RLS Adds the org FK on compositions.organizationId -> organizations(id) with ON DELETE RESTRICT (deliberate deletion, no cascade), and ships the two migrations applied to prod: the additive composition-engine schema (compositions, stages, execution_records.userId/sessionId, composition FKs) and RLS enablement on the two new tables. https://claude.ai/code/session_01HhSSACKMJYh92GqDAakXHT --- .../migration.sql | 49 +++++++++++++++++++ .../migration.sql | 6 +++ prisma/schema.prisma | 2 + 3 files changed, 57 insertions(+) create mode 100644 prisma/migrations/20260606201500_add_composition_engine/migration.sql create mode 100644 prisma/migrations/20260606201600_enable_rls_compositions_stages/migration.sql diff --git a/prisma/migrations/20260606201500_add_composition_engine/migration.sql b/prisma/migrations/20260606201500_add_composition_engine/migration.sql new file mode 100644 index 00000000000..8de10f1ddb5 --- /dev/null +++ b/prisma/migrations/20260606201500_add_composition_engine/migration.sql @@ -0,0 +1,49 @@ +-- AlterTable +ALTER TABLE "execution_records" ADD COLUMN "sessionId" TEXT, +ADD COLUMN "userId" TEXT NOT NULL; + +-- CreateTable +CREATE TABLE "compositions" ( + "id" TEXT NOT NULL, + "organizationId" TEXT NOT NULL, + "name" TEXT NOT NULL, + "description" TEXT, + "createdBy" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "compositions_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "stages" ( + "id" TEXT NOT NULL, + "compositionId" TEXT NOT NULL, + "order" INTEGER NOT NULL, + "promptId" TEXT NOT NULL, + "fallbackPromptId" TEXT, + "successThreshold" DOUBLE PRECISION NOT NULL DEFAULT 0.5, + "extractFields" TEXT[], + "transformRules" JSONB, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "stages_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE INDEX "compositions_organizationId_idx" ON "compositions"("organizationId"); + +-- CreateIndex +CREATE INDEX "stages_compositionId_idx" ON "stages"("compositionId"); + +-- CreateIndex +CREATE UNIQUE INDEX "stages_compositionId_order_key" ON "stages"("compositionId", "order"); + +-- AddForeignKey +ALTER TABLE "compositions" ADD CONSTRAINT "compositions_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "organizations"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "stages" ADD CONSTRAINT "stages_compositionId_fkey" FOREIGN KEY ("compositionId") REFERENCES "compositions"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "execution_records" ADD CONSTRAINT "execution_records_compositionId_fkey" FOREIGN KEY ("compositionId") REFERENCES "compositions"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/prisma/migrations/20260606201600_enable_rls_compositions_stages/migration.sql b/prisma/migrations/20260606201600_enable_rls_compositions_stages/migration.sql new file mode 100644 index 00000000000..4f941e5c7f1 --- /dev/null +++ b/prisma/migrations/20260606201600_enable_rls_compositions_stages/migration.sql @@ -0,0 +1,6 @@ +-- Security consistency: the Composition Engine tables were created after the +-- RLS hardening migration, so enable RLS deny-by-default on them too (no anon +-- grants exist on them thanks to the default-privileges revoke). App is +-- pure-Prisma on the bypass `postgres` role, so this does not affect it. +ALTER TABLE "compositions" ENABLE ROW LEVEL SECURITY; +ALTER TABLE "stages" ENABLE ROW LEVEL SECURITY; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 394b5cb26b6..2a8c0456145 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -424,6 +424,7 @@ model Composition { createdBy String createdAt DateTime @default(now()) updatedAt DateTime @updatedAt + organization Organization @relation(fields: [organizationId], references: [id], onDelete: Restrict) stages Stage[] executions ExecutionRecord[] @@ -458,6 +459,7 @@ model Organization { users User[] prompts Prompt[] executionRecords ExecutionRecord[] + compositions Composition[] @@map("organizations") } From fe276a434570a18d1f36cbf2045e6b6dd47e9367 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 6 Jun 2026 20:47:25 +0000 Subject: [PATCH 12/17] feat(composition): Composition Engine orchestrator + API (Level 2) Multi-step chain orchestrator: dagResolver (order stages) -> stageExecutor (LLM via a new provider-agnostic abstraction over the existing OpenAI integration) -> adaptationLayer (extractFields + transformRules) -> thresholdCheck (deterministic v1; Level-3 scoring later) -> traceRecorder (atomic ExecutionRecord + StageTrace persistence). Falls back to fallbackPromptId on below-threshold/error; status success|partial|failed. Engine maps onto the audited trace columns (input->adaptedInput, duration->durationMs, success->thresholdMet, errorMessage->error, status->ExecutionStatus enum), preserving Level-3 scoring fields. Adds 5 routes under /api/v1/compositions (create/list, get, execute, history, test) and an e2e orchestrator test (stub LLM + mocked Prisma) covering the 3-stage chain, fallback, adaptation, and all status paths. https://claude.ai/code/session_01HhSSACKMJYh92GqDAakXHT --- .../composition/orchestrator.test.ts | 201 ++++++++++++++++++ .../api/v1/compositions/[id]/execute/route.ts | 57 +++++ .../api/v1/compositions/[id]/history/route.ts | 45 ++++ src/app/api/v1/compositions/[id]/route.ts | 33 +++ .../api/v1/compositions/[id]/test/route.ts | 80 +++++++ src/app/api/v1/compositions/route.ts | 115 ++++++++++ src/lib/ai/llm-provider.ts | 89 ++++++++ src/lib/composition/adaptationLayer.ts | 68 ++++++ src/lib/composition/dagResolver.ts | 14 ++ src/lib/composition/index.ts | 193 +++++++++++++++++ src/lib/composition/stageExecutor.ts | 58 +++++ src/lib/composition/thresholdCheck.ts | 17 ++ src/lib/composition/traceRecorder.ts | 78 +++++++ src/lib/composition/types.ts | 53 +++++ 14 files changed, 1101 insertions(+) create mode 100644 src/__tests__/composition/orchestrator.test.ts create mode 100644 src/app/api/v1/compositions/[id]/execute/route.ts create mode 100644 src/app/api/v1/compositions/[id]/history/route.ts create mode 100644 src/app/api/v1/compositions/[id]/route.ts create mode 100644 src/app/api/v1/compositions/[id]/test/route.ts create mode 100644 src/app/api/v1/compositions/route.ts create mode 100644 src/lib/ai/llm-provider.ts create mode 100644 src/lib/composition/adaptationLayer.ts create mode 100644 src/lib/composition/dagResolver.ts create mode 100644 src/lib/composition/index.ts create mode 100644 src/lib/composition/stageExecutor.ts create mode 100644 src/lib/composition/thresholdCheck.ts create mode 100644 src/lib/composition/traceRecorder.ts create mode 100644 src/lib/composition/types.ts diff --git a/src/__tests__/composition/orchestrator.test.ts b/src/__tests__/composition/orchestrator.test.ts new file mode 100644 index 00000000000..8fa3441e235 --- /dev/null +++ b/src/__tests__/composition/orchestrator.test.ts @@ -0,0 +1,201 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import type { LLMProvider } from "@/lib/ai/llm-provider"; + +// --- Mock the Prisma client (sandbox has no DB egress). Capture what gets written. --- +const { mockDb, captured } = vi.hoisted(() => { + const captured: { + traces: Array>; + record: Record | null; + } = { traces: [], record: null }; + + const tx = { + executionRecord: { + create: async ({ data }: { data: Record }) => { + captured.record = data; + return { id: "exec_test_1", ...data }; + }, + }, + stageTrace: { + createMany: async ({ data }: { data: Array> }) => { + captured.traces = data; + return { count: data.length }; + }, + }, + }; + + const mockDb = { + composition: { findUnique: vi.fn() }, + prompt: { findUnique: vi.fn() }, + $transaction: async (cb: (t: typeof tx) => unknown) => cb(tx), + }; + + return { mockDb, captured }; +}); + +vi.mock("@/lib/db", () => ({ db: mockDb })); + +import { executeComposition } from "@/lib/composition"; +import { setLLMProvider } from "@/lib/ai/llm-provider"; + +// --- Stub LLM provider: deterministic outputs keyed off the prompt content. --- +const stub: LLMProvider = { + name: "stub", + async complete(req) { + const id = (req.system ?? "").replace("SYS:", ""); + if (id === "p1") { + // structured output so the adaptation layer can extract a field + return { + output: JSON.stringify({ summary: "S1-summary", topic: "weather" }), + inputTokens: 10, + outputTokens: 20, + model: "stub", + }; + } + if (id === "p2") { + // empty output -> scores below threshold -> triggers fallback + return { output: "", inputTokens: 5, outputTokens: 0, model: "stub" }; + } + return { + output: `out:${id}:${req.prompt.slice(0, 24)}`, + inputTokens: 7, + outputTokens: 8, + model: "stub", + }; + }, +}; + +interface StageSeed { + order: number; + promptId: string; + fallbackPromptId: string | null; + successThreshold: number; + extractFields: string[]; + transformRules: unknown; +} + +function stage(seed: StageSeed) { + return { + id: `stage_${seed.order}`, + compositionId: "comp1", + createdAt: new Date(), + ...seed, + }; +} + +beforeEach(() => { + captured.traces = []; + captured.record = null; + mockDb.prompt.findUnique.mockImplementation( + async ({ where }: { where: { id: string } }) => ({ + content: `SYS:${where.id}`, + versions: [{ version: 1 }], + }), + ); + setLLMProvider(stub); +}); + +afterEach(() => { + setLLMProvider(null); + vi.clearAllMocks(); +}); + +describe("executeComposition", () => { + it("runs a 3-stage chain, adapts between stages, and applies fallback (partial)", async () => { + mockDb.composition.findUnique.mockResolvedValue({ + id: "comp1", + organizationId: "org1", + name: "Test chain", + // intentionally out of order to prove dagResolver sorts by `order` + stages: [ + stage({ order: 2, promptId: "p2", fallbackPromptId: "p2fb", successThreshold: 0.5, extractFields: [], transformRules: null }), + stage({ order: 1, promptId: "p1", fallbackPromptId: null, successThreshold: 0.5, extractFields: ["summary"], transformRules: { summary: "Use this summary as context" } }), + stage({ order: 3, promptId: "p3", fallbackPromptId: null, successThreshold: 0.5, extractFields: [], transformRules: null }), + ], + }); + + const result = await executeComposition("comp1", "hello", { + organizationId: "org1", + userId: "u1", + sessionId: "sess1", + }); + + // status + per-stage results + expect(result.status).toBe("partial"); + expect(result.stages).toHaveLength(3); + expect(result.stages[0].fallbackApplied).toBe(false); // stage 1 ok + expect(result.stages[1].fallbackApplied).toBe(true); // stage 2 fell back + expect(result.stages[2].fallbackApplied).toBe(false); // stage 3 ok + expect(result.finalOutput).toContain("out:p3"); + expect(result.totalTokens).toBeGreaterThan(0); + expect(result.totalDuration).toBeGreaterThanOrEqual(0); + + // one StageTrace row per stage + expect(captured.traces).toHaveLength(3); + + // adaptation: stage 2's adaptedInput is built from stage 1 output via transformRules + const t2 = captured.traces.find((t) => t.stageOrder === 2)!; + expect(t2.adaptedInput).toContain("Use this summary as context"); + expect(t2.adaptedInput).toContain("S1-summary"); + expect(t2.fallbackApplied).toBe(true); + expect(t2.promptId).toBe("p2fb"); // the prompt actually used is recorded + expect(t2.fallbackPromptId).toBe("p2fb"); + expect(t2.thresholdMet).toBe(true); // fallback output passed + + // stage 1 trace records which adaptation rules fired + const t1 = captured.traces.find((t) => t.stageOrder === 1)!; + expect(t1.rulesApplied).toEqual(["summary"]); + + // ExecutionRecord persisted with mapped enum status + context + expect(captured.record).toMatchObject({ + compositionId: "comp1", + organizationId: "org1", + userId: "u1", + sessionId: "sess1", + status: "PARTIAL", + }); + expect(result.executionId).toBe("exec_test_1"); + }); + + it("returns success when all stages pass with no fallback", async () => { + mockDb.composition.findUnique.mockResolvedValue({ + id: "comp1", + organizationId: "org1", + name: "Happy path", + stages: [ + stage({ order: 1, promptId: "p1", fallbackPromptId: null, successThreshold: 0.5, extractFields: ["summary"], transformRules: null }), + stage({ order: 2, promptId: "p3", fallbackPromptId: null, successThreshold: 0.5, extractFields: [], transformRules: null }), + ], + }); + + const result = await executeComposition("comp1", "hi", { + organizationId: "org1", + userId: "u1", + }); + + expect(result.status).toBe("success"); + expect(result.stages.every((s) => s.success && !s.fallbackApplied)).toBe(true); + expect(captured.record).toMatchObject({ status: "SUCCESS", sessionId: null }); + }); + + it("returns failed when a stage fails and has no fallback", async () => { + mockDb.composition.findUnique.mockResolvedValue({ + id: "comp1", + organizationId: "org1", + name: "Failing chain", + stages: [ + stage({ order: 1, promptId: "p2", fallbackPromptId: null, successThreshold: 0.5, extractFields: [], transformRules: null }), + ], + }); + + const result = await executeComposition("comp1", "hi", { + organizationId: "org1", + userId: "u1", + }); + + expect(result.status).toBe("failed"); + expect(result.stages[0].success).toBe(false); + expect(captured.traces).toHaveLength(1); + expect(captured.traces[0].thresholdMet).toBe(false); + expect(captured.record).toMatchObject({ status: "FAILED" }); + }); +}); diff --git a/src/app/api/v1/compositions/[id]/execute/route.ts b/src/app/api/v1/compositions/[id]/execute/route.ts new file mode 100644 index 00000000000..4f7602e70b5 --- /dev/null +++ b/src/app/api/v1/compositions/[id]/execute/route.ts @@ -0,0 +1,57 @@ +import { NextRequest, NextResponse } from "next/server"; +import { z } from "zod"; +import { auth } from "@/lib/auth"; +import { requestLogger } from "@/lib/logger"; +import { executeComposition } from "@/lib/composition"; + +const executeSchema = z.object({ + input: z.string().min(1), + context: z.object({ + organizationId: z.string().min(1), + userId: z.string().min(1), + sessionId: z.string().nullable().optional(), + }), +}); + +// POST /api/v1/compositions/[id]/execute — run the chain end-to-end +export async function POST( + request: NextRequest, + { params }: { params: Promise<{ id: string }> }, +) { + const log = requestLogger(request.headers.get("x-request-id")); + try { + const session = await auth(); + if (!session?.user) { + return NextResponse.json({ error: "unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const body = await request.json(); + const { input, context } = executeSchema.parse(body); + + const result = await executeComposition(id, input, context); + + log.info( + { + op: "composition.execute", + compositionId: id, + executionId: result.executionId, + status: result.status, + totalTokens: result.totalTokens, + totalDuration: result.totalDuration, + }, + "composition executed", + ); + + return NextResponse.json({ ...result, executionTrace: [] }); + } catch (error) { + if (error instanceof z.ZodError) { + return NextResponse.json({ error: "invalid input", details: error.issues }, { status: 400 }); + } + if (error instanceof Error && error.message.startsWith("Composition not found")) { + return NextResponse.json({ error: "not_found" }, { status: 404 }); + } + log.error({ op: "composition.execute", err: error }, "composition execution failed"); + return NextResponse.json({ error: "server_error" }, { status: 500 }); + } +} diff --git a/src/app/api/v1/compositions/[id]/history/route.ts b/src/app/api/v1/compositions/[id]/history/route.ts new file mode 100644 index 00000000000..2ad7e47794b --- /dev/null +++ b/src/app/api/v1/compositions/[id]/history/route.ts @@ -0,0 +1,45 @@ +import { NextRequest, NextResponse } from "next/server"; +import { auth } from "@/lib/auth"; +import { db } from "@/lib/db"; +import { requestLogger } from "@/lib/logger"; + +// GET /api/v1/compositions/[id]/history — paginated execution history +export async function GET( + request: NextRequest, + { params }: { params: Promise<{ id: string }> }, +) { + const log = requestLogger(request.headers.get("x-request-id")); + try { + const session = await auth(); + if (!session?.user) { + return NextResponse.json({ error: "unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const { searchParams } = new URL(request.url); + const page = Math.max(1, parseInt(searchParams.get("page") ?? "1", 10) || 1); + const limit = Math.min(100, Math.max(1, parseInt(searchParams.get("limit") ?? "20", 10) || 20)); + + const [total, executions] = await Promise.all([ + db.executionRecord.count({ where: { compositionId: id } }), + db.executionRecord.findMany({ + where: { compositionId: id }, + orderBy: { startedAt: "desc" }, + skip: (page - 1) * limit, + take: limit, + include: { _count: { select: { stageTraces: true } } }, + }), + ]); + + return NextResponse.json({ + executions, + page, + limit, + total, + totalPages: Math.ceil(total / limit), + }); + } catch (error) { + log.error({ op: "composition.history", err: error }, "failed to fetch history"); + return NextResponse.json({ error: "server_error" }, { status: 500 }); + } +} diff --git a/src/app/api/v1/compositions/[id]/route.ts b/src/app/api/v1/compositions/[id]/route.ts new file mode 100644 index 00000000000..7e724b3b01a --- /dev/null +++ b/src/app/api/v1/compositions/[id]/route.ts @@ -0,0 +1,33 @@ +import { NextRequest, NextResponse } from "next/server"; +import { auth } from "@/lib/auth"; +import { db } from "@/lib/db"; +import { requestLogger } from "@/lib/logger"; + +// GET /api/v1/compositions/[id] — fetch a single composition with its stages +export async function GET( + request: NextRequest, + { params }: { params: Promise<{ id: string }> }, +) { + const log = requestLogger(request.headers.get("x-request-id")); + try { + const session = await auth(); + if (!session?.user) { + return NextResponse.json({ error: "unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const composition = await db.composition.findUnique({ + where: { id }, + include: { stages: { orderBy: { order: "asc" } } }, + }); + + if (!composition) { + return NextResponse.json({ error: "not_found" }, { status: 404 }); + } + + return NextResponse.json(composition); + } catch (error) { + log.error({ op: "composition.get", err: error }, "failed to fetch composition"); + return NextResponse.json({ error: "server_error" }, { status: 500 }); + } +} diff --git a/src/app/api/v1/compositions/[id]/test/route.ts b/src/app/api/v1/compositions/[id]/test/route.ts new file mode 100644 index 00000000000..149d43d7678 --- /dev/null +++ b/src/app/api/v1/compositions/[id]/test/route.ts @@ -0,0 +1,80 @@ +import { NextRequest, NextResponse } from "next/server"; +import { z } from "zod"; +import { auth } from "@/lib/auth"; +import { requestLogger } from "@/lib/logger"; +import { executeComposition } from "@/lib/composition"; + +const testSchema = z.object({ + sampleInputs: z.array(z.string().min(1)).min(1), + context: z.object({ + organizationId: z.string().min(1), + userId: z.string().min(1), + sessionId: z.string().nullable().optional(), + }), +}); + +// POST /api/v1/compositions/[id]/test — run the chain against multiple sample +// inputs and return per-sample + aggregated results. +export async function POST( + request: NextRequest, + { params }: { params: Promise<{ id: string }> }, +) { + const log = requestLogger(request.headers.get("x-request-id")); + try { + const session = await auth(); + if (!session?.user) { + return NextResponse.json({ error: "unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const body = await request.json(); + const { sampleInputs, context } = testSchema.parse(body); + + const results: Array<{ + input: string; + executionId: string; + status: string; + finalOutput: string; + totalTokens: number; + totalDuration: number; + stageCount: number; + }> = []; + + // Sequential to avoid hammering the LLM provider with parallel chains. + for (const input of sampleInputs) { + const r = await executeComposition(id, input, context); + results.push({ + input, + executionId: r.executionId, + status: r.status, + finalOutput: r.finalOutput, + totalTokens: r.totalTokens, + totalDuration: r.totalDuration, + stageCount: r.stages.length, + }); + } + + const count = results.length; + const summary = { + total: count, + success: results.filter((r) => r.status === "success").length, + partial: results.filter((r) => r.status === "partial").length, + failed: results.filter((r) => r.status === "failed").length, + avgTokens: Math.round(results.reduce((a, r) => a + r.totalTokens, 0) / count), + avgDuration: Math.round(results.reduce((a, r) => a + r.totalDuration, 0) / count), + }; + + log.info({ op: "composition.test", compositionId: id, ...summary }, "composition test run"); + + return NextResponse.json({ summary, results }); + } catch (error) { + if (error instanceof z.ZodError) { + return NextResponse.json({ error: "invalid input", details: error.issues }, { status: 400 }); + } + if (error instanceof Error && error.message.startsWith("Composition not found")) { + return NextResponse.json({ error: "not_found" }, { status: 404 }); + } + log.error({ op: "composition.test", err: error }, "composition test failed"); + return NextResponse.json({ error: "server_error" }, { status: 500 }); + } +} diff --git a/src/app/api/v1/compositions/route.ts b/src/app/api/v1/compositions/route.ts new file mode 100644 index 00000000000..6213b1d892f --- /dev/null +++ b/src/app/api/v1/compositions/route.ts @@ -0,0 +1,115 @@ +import { NextRequest, NextResponse } from "next/server"; +import { z } from "zod"; +import { Prisma } from "@prisma/client"; +import { auth } from "@/lib/auth"; +import { db } from "@/lib/db"; +import { requestLogger } from "@/lib/logger"; + +const stageSchema = z.object({ + order: z.number().int(), + promptId: z.string().min(1), + fallbackPromptId: z.string().nullable().optional(), + successThreshold: z.number().min(0).max(1).optional(), + extractFields: z.array(z.string()).optional(), + transformRules: z.record(z.string(), z.unknown()).nullable().optional(), +}); + +const createSchema = z.object({ + name: z.string().min(1), + description: z.string().optional(), + organizationId: z.string().min(1), + stages: z.array(stageSchema).min(1), +}); + +// POST /api/v1/compositions — create a composition with N stages +export async function POST(request: NextRequest) { + const log = requestLogger(request.headers.get("x-request-id")); + try { + const session = await auth(); + if (!session?.user) { + return NextResponse.json({ error: "unauthorized" }, { status: 401 }); + } + + const body = await request.json(); + const data = createSchema.parse(body); + + const orders = data.stages.map((s) => s.order); + if (new Set(orders).size !== orders.length) { + return NextResponse.json({ error: "duplicate stage order" }, { status: 400 }); + } + + const composition = await db.composition.create({ + data: { + name: data.name, + description: data.description, + organizationId: data.organizationId, + createdBy: session.user.id, + stages: { + create: data.stages.map((s) => ({ + order: s.order, + promptId: s.promptId, + fallbackPromptId: s.fallbackPromptId ?? null, + successThreshold: s.successThreshold ?? 0.5, + extractFields: s.extractFields ?? [], + transformRules: + s.transformRules == null + ? Prisma.DbNull + : (s.transformRules as Prisma.InputJsonValue), + })), + }, + }, + include: { stages: { orderBy: { order: "asc" } } }, + }); + + log.info( + { op: "composition.create", compositionId: composition.id, stages: composition.stages.length }, + "composition created", + ); + + return NextResponse.json( + { + id: composition.id, + name: composition.name, + description: composition.description, + stages: composition.stages, + createdAt: composition.createdAt, + createdBy: composition.createdBy, + }, + { status: 201 }, + ); + } catch (error) { + if (error instanceof z.ZodError) { + return NextResponse.json({ error: "invalid input", details: error.issues }, { status: 400 }); + } + log.error({ op: "composition.create", err: error }, "failed to create composition"); + return NextResponse.json({ error: "server_error" }, { status: 500 }); + } +} + +// GET /api/v1/compositions — list compositions (optionally by organizationId) +export async function GET(request: NextRequest) { + const log = requestLogger(request.headers.get("x-request-id")); + try { + const session = await auth(); + if (!session?.user) { + return NextResponse.json({ error: "unauthorized" }, { status: 401 }); + } + + const { searchParams } = new URL(request.url); + const organizationId = searchParams.get("organizationId") ?? undefined; + + const compositions = await db.composition.findMany({ + where: organizationId ? { organizationId } : undefined, + orderBy: { createdAt: "desc" }, + include: { + stages: { orderBy: { order: "asc" } }, + _count: { select: { executions: true } }, + }, + }); + + return NextResponse.json({ compositions }); + } catch (error) { + log.error({ op: "composition.list", err: error }, "failed to list compositions"); + return NextResponse.json({ error: "server_error" }, { status: 500 }); + } +} diff --git a/src/lib/ai/llm-provider.ts b/src/lib/ai/llm-provider.ts new file mode 100644 index 00000000000..327b4bef725 --- /dev/null +++ b/src/lib/ai/llm-provider.ts @@ -0,0 +1,89 @@ +import OpenAI from "openai"; + +// Provider-agnostic LLM completion abstraction. +// +// stageExecutor (and any future caller) depends ONLY on this interface, so the +// underlying provider can be swapped without touching call sites. v1 ships an +// OpenAI-backed implementation that reuses the existing src/lib/ai setup +// (OPENAI_API_KEY / OPENAI_BASE_URL / OPENAI_GENERATIVE_MODEL). Claude can be +// added later as an alternate provider (e.g. when ANTHROPIC_API_KEY exists) +// by registering a new LLMProvider here — no engine changes required. + +export interface LLMCompletionRequest { + /** Optional system / instruction prompt (the stage's prompt content). */ + system?: string; + /** User input for this turn (the stage input). */ + prompt: string; + temperature?: number; + maxTokens?: number; +} + +export interface LLMCompletionResult { + output: string; + inputTokens: number; + outputTokens: number; + model: string; +} + +export interface LLMProvider { + readonly name: string; + complete(req: LLMCompletionRequest): Promise; +} + +class OpenAIProvider implements LLMProvider { + readonly name = "openai"; + private client: OpenAI | null = null; + private readonly model = process.env.OPENAI_GENERATIVE_MODEL || "gpt-4o-mini"; + + private getClient(): OpenAI { + if (!this.client) { + const apiKey = process.env.OPENAI_API_KEY; + if (!apiKey) { + throw new Error("OPENAI_API_KEY is not set"); + } + this.client = new OpenAI({ + apiKey, + baseURL: process.env.OPENAI_BASE_URL || undefined, + }); + } + return this.client; + } + + async complete(req: LLMCompletionRequest): Promise { + const client = this.getClient(); + const messages: OpenAI.Chat.ChatCompletionMessageParam[] = []; + if (req.system) { + messages.push({ role: "system", content: req.system }); + } + messages.push({ role: "user", content: req.prompt }); + + const response = await client.chat.completions.create({ + model: this.model, + messages, + temperature: req.temperature ?? 0.7, + max_tokens: req.maxTokens ?? 4000, + }); + + return { + output: response.choices[0]?.message?.content?.trim() ?? "", + inputTokens: response.usage?.prompt_tokens ?? 0, + outputTokens: response.usage?.completion_tokens ?? 0, + model: response.model ?? this.model, + }; + } +} + +let activeProvider: LLMProvider | null = null; + +/** Returns the active LLM provider (OpenAI-backed by default). */ +export function getLLMProvider(): LLMProvider { + if (!activeProvider) { + activeProvider = new OpenAIProvider(); + } + return activeProvider; +} + +/** Override the active provider (used by tests and future provider swaps). */ +export function setLLMProvider(provider: LLMProvider | null): void { + activeProvider = provider; +} diff --git a/src/lib/composition/adaptationLayer.ts b/src/lib/composition/adaptationLayer.ts new file mode 100644 index 00000000000..17281873b0c --- /dev/null +++ b/src/lib/composition/adaptationLayer.ts @@ -0,0 +1,68 @@ +// Adaptation layer: transform stage N output -> stage N+1 input. +// +// Config-based (deterministic, debuggable) per EXECUTION-TRACE-SCORING-SPEC Q1. +// If the stage defines `extractFields`, we pull those fields out of the prior +// output (parsed as JSON when possible) and frame each with its `transformRules` +// entry. With no `extractFields`, the raw output is threaded straight through. + +type TransformRules = Record; + +export interface AdaptationResult { + adaptedInput: string; + /** Names of fields whose transformRules entry actually fired (for the trace). */ + rulesApplied: string[]; +} + +function tryParseJsonObject(text: string): Record | null { + try { + const parsed: unknown = JSON.parse(text); + return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as Record) + : null; + } catch { + return null; + } +} + +function stringifyValue(value: unknown): string { + if (value === null || value === undefined) return ""; + if (typeof value === "string") return value; + return JSON.stringify(value); +} + +export function adaptOutput( + output: string, + extractFields: string[], + transformRules: unknown, +): AdaptationResult { + if (!extractFields || extractFields.length === 0) { + return { adaptedInput: output, rulesApplied: [] }; + } + + const rules: TransformRules = + transformRules !== null && + typeof transformRules === "object" && + !Array.isArray(transformRules) + ? (transformRules as TransformRules) + : {}; + + const parsed = tryParseJsonObject(output); + const parts: string[] = []; + const rulesApplied: string[] = []; + + for (const field of extractFields) { + // Prefer the extracted field from structured output; otherwise fall back to + // the whole output so the chain still threads information forward. + const value = + parsed && field in parsed ? stringifyValue(parsed[field]) : output; + const rule = rules[field]; + if (typeof rule === "string" && rule.length > 0) { + parts.push(`${rule}: ${value}`); + rulesApplied.push(field); + } else { + parts.push(`${field}: ${value}`); + } + } + + return { adaptedInput: parts.join("\n\n"), rulesApplied }; +} diff --git a/src/lib/composition/dagResolver.ts b/src/lib/composition/dagResolver.ts new file mode 100644 index 00000000000..08661560e12 --- /dev/null +++ b/src/lib/composition/dagResolver.ts @@ -0,0 +1,14 @@ +import type { Stage } from "@prisma/client"; + +/** + * Resolve a composition's stages into an ordered execution plan. + * + * v1 chains are strictly linear (Prompt A -> B -> C), so this sorts by `order` + * ascending. Non-mutating (returns a sorted copy). Throws if there are no stages. + */ +export function resolveDag(stages: Stage[]): Stage[] { + if (stages.length === 0) { + throw new Error("Composition has no stages to execute"); + } + return [...stages].sort((a, b) => a.order - b.order); +} diff --git a/src/lib/composition/index.ts b/src/lib/composition/index.ts new file mode 100644 index 00000000000..71c8d5f76e6 --- /dev/null +++ b/src/lib/composition/index.ts @@ -0,0 +1,193 @@ +import { db } from "@/lib/db"; +import { resolveDag } from "./dagResolver"; +import { executeStage } from "./stageExecutor"; +import { adaptOutput } from "./adaptationLayer"; +import { checkThreshold } from "./thresholdCheck"; +import { persistExecution } from "./traceRecorder"; +import type { + ExecutionContext, + ExecutionResult, + ExecutionStatusLabel, + StageResult, + StageTraceInput, +} from "./types"; + +export type { + ExecutionContext, + ExecutionResult, + StageResult, +} from "./types"; + +/** + * Orchestrate a composition execution end-to-end: + * resolve -> for each stage { execute -> threshold -> fallback? } -> adapt -> trace + * then persist one ExecutionRecord with its StageTrace rows. + * + * Status: "success" = all stages passed, no fallback. "partial" = completed but at + * least one stage used fallback. "failed" = a stage produced no usable output and + * had no working fallback (chain stops). + */ +export async function executeComposition( + compositionId: string, + input: string, + context: ExecutionContext, +): Promise { + const composition = await db.composition.findUnique({ + where: { id: compositionId }, + include: { stages: true }, + }); + + if (!composition) { + throw new Error(`Composition not found: ${compositionId}`); + } + + const stages = resolveDag(composition.stages); + + const stageResults: StageResult[] = []; + const traces: StageTraceInput[] = []; + + // rawInput = what the current stage received BEFORE adaptation + // adaptedInput = what it received AFTER the previous stage's adaptation rules + let rawInput = input; + let adaptedInput = input; + let lastOutput = ""; + let totalTokens = 0; + let totalDurationMs = 0; + let anyFallback = false; + let chainFailed = false; + + for (let i = 0; i < stages.length; i++) { + const stage = stages[i]; + + let output = ""; + let tokensUsed = 0; + let durationMs = 0; + let model: string | null = null; + let promptVersion = 1; + let usedPromptId = stage.promptId; + let fallbackApplied = false; + let success = false; + let score: number | null = null; + let errorMessage: string | null = null; + + // --- primary attempt --- + try { + const r = await executeStage({ promptId: stage.promptId, input: adaptedInput }); + output = r.output; + tokensUsed = r.tokensUsed; + durationMs = r.duration; + model = r.model; + promptVersion = r.promptVersion; + const tc = checkThreshold(output, stage.successThreshold); + score = tc.score; + success = tc.passed; + } catch (err) { + errorMessage = err instanceof Error ? err.message : String(err); + success = false; + } + + // --- fallback when the primary failed or scored below threshold --- + if (!success && stage.fallbackPromptId) { + try { + const fb = await executeStage({ + promptId: stage.fallbackPromptId, + input: adaptedInput, + }); + output = fb.output; + tokensUsed += fb.tokensUsed; + durationMs += fb.duration; + model = fb.model; + promptVersion = fb.promptVersion; + usedPromptId = stage.fallbackPromptId; + fallbackApplied = true; + anyFallback = true; + const tc = checkThreshold(output, stage.successThreshold); + score = tc.score; + success = tc.passed; + } catch (err) { + const fbErr = err instanceof Error ? err.message : String(err); + errorMessage = errorMessage + ? `${errorMessage}; fallback failed: ${fbErr}` + : `fallback failed: ${fbErr}`; + success = false; + } + } + + totalTokens += tokensUsed; + totalDurationMs += durationMs; + + const trace: StageTraceInput = { + stageOrder: stage.order, + promptId: usedPromptId, + promptVersion, + rawInput, + adaptedInput, + output: output || null, + tokensUsed, + durationMs, + score, + thresholdMet: success, + fallbackApplied, + fallbackPromptId: stage.fallbackPromptId, + model, + rulesApplied: [], + error: errorMessage, + }; + traces.push(trace); + + stageResults.push({ + stageOrder: stage.order, + promptId: usedPromptId, + input: adaptedInput, + output, + tokensUsed, + duration: durationMs, + success, + fallbackApplied, + }); + + if (!success) { + // No usable output and no working fallback -> the chain cannot continue. + chainFailed = true; + break; + } + + lastOutput = output; + + // --- adapt this stage's output into the next stage's input --- + if (i < stages.length - 1) { + const adaptation = adaptOutput(output, stage.extractFields, stage.transformRules); + rawInput = output; + adaptedInput = adaptation.adaptedInput; + trace.rulesApplied = adaptation.rulesApplied; + } + } + + const status: ExecutionStatusLabel = chainFailed + ? "failed" + : anyFallback + ? "partial" + : "success"; + + const executionId = await persistExecution( + { + compositionId, + context, + status, + input, + finalOutput: lastOutput, + totalTokens, + totalDurationMs, + }, + traces, + ); + + return { + executionId, + status, + stages: stageResults, + finalOutput: lastOutput, + totalTokens, + totalDuration: totalDurationMs, + }; +} diff --git a/src/lib/composition/stageExecutor.ts b/src/lib/composition/stageExecutor.ts new file mode 100644 index 00000000000..e0b12eab297 --- /dev/null +++ b/src/lib/composition/stageExecutor.ts @@ -0,0 +1,58 @@ +import { db } from "@/lib/db"; +import { getLLMProvider } from "@/lib/ai/llm-provider"; + +export interface StageExecutionResult { + output: string; + tokensUsed: number; + duration: number; + model: string; + promptVersion: number; +} + +/** + * Execute a single stage: load the prompt content (pinning the executed version), + * invoke the active LLM provider with the stage input, and return the output plus + * token/duration metrics. + * + * Throws on a missing prompt or a provider error — the orchestrator catches and + * applies the stage's fallback. + */ +export async function executeStage(params: { + promptId: string; + input: string; +}): Promise { + const { promptId, input } = params; + + const prompt = await db.prompt.findUnique({ + where: { id: promptId }, + select: { + content: true, + versions: { + orderBy: { version: "desc" }, + take: 1, + select: { version: true }, + }, + }, + }); + + if (!prompt) { + throw new Error(`Prompt not found: ${promptId}`); + } + + const promptVersion = prompt.versions[0]?.version ?? 1; + + const start = Date.now(); + const result = await getLLMProvider().complete({ + system: prompt.content, + prompt: input, + }); + const duration = Date.now() - start; + + return { + output: result.output, + tokensUsed: result.inputTokens + result.outputTokens, + duration, + model: result.model, + promptVersion, + }; +} diff --git a/src/lib/composition/thresholdCheck.ts b/src/lib/composition/thresholdCheck.ts new file mode 100644 index 00000000000..d9fa5aed8ee --- /dev/null +++ b/src/lib/composition/thresholdCheck.ts @@ -0,0 +1,17 @@ +// Threshold check: score a stage output and decide whether it passes. +// +// Level 2 uses a minimal deterministic scorer: a non-empty, non-whitespace +// output scores 1.0, an empty output scores 0.0. This intentionally defers real +// scoring to Level 3 (heuristic -> model-based per EXECUTION-TRACE-SCORING-SPEC +// Q2) while still giving the orchestrator a real signal that can drop below the +// configured successThreshold (e.g. an empty/failed output) to drive fallback. + +export interface ThresholdResult { + score: number; + passed: boolean; +} + +export function checkThreshold(output: string, successThreshold: number): ThresholdResult { + const score = output && output.trim().length > 0 ? 1 : 0; + return { score, passed: score >= successThreshold }; +} diff --git a/src/lib/composition/traceRecorder.ts b/src/lib/composition/traceRecorder.ts new file mode 100644 index 00000000000..33770c1e098 --- /dev/null +++ b/src/lib/composition/traceRecorder.ts @@ -0,0 +1,78 @@ +import type { PrismaClient } from "@prisma/client"; +import { db } from "@/lib/db"; +import type { + ExecutionContext, + ExecutionStatusLabel, + StageTraceInput, +} from "./types"; + +export interface ExecutionRecordInput { + compositionId: string; + context: ExecutionContext; + status: ExecutionStatusLabel; + input: string; + finalOutput: string; + totalTokens: number; + totalDurationMs: number; +} + +const STATUS_MAP = { + success: "SUCCESS", + partial: "PARTIAL", + failed: "FAILED", +} as const; + +/** + * Persist the ExecutionRecord and all of its StageTrace rows atomically. + * All LLM work is already done by the time this runs, so the transaction is short. + * Returns the new executionId. + */ +export async function persistExecution( + record: ExecutionRecordInput, + traces: StageTraceInput[], + client: PrismaClient = db, +): Promise { + const exec = await client.$transaction(async (tx) => { + const created = await tx.executionRecord.create({ + data: { + compositionId: record.compositionId, + organizationId: record.context.organizationId, + userId: record.context.userId, + sessionId: record.context.sessionId ?? null, + status: STATUS_MAP[record.status], + input: record.input, + finalOutput: record.finalOutput, + totalTokens: record.totalTokens, + totalDurationMs: record.totalDurationMs, + completedAt: new Date(), + }, + }); + + if (traces.length > 0) { + await tx.stageTrace.createMany({ + data: traces.map((t) => ({ + executionId: created.id, + stageOrder: t.stageOrder, + promptId: t.promptId, + promptVersion: t.promptVersion, + rawInput: t.rawInput, + adaptedInput: t.adaptedInput, + output: t.output, + tokensUsed: t.tokensUsed, + durationMs: t.durationMs, + score: t.score, + thresholdMet: t.thresholdMet, + fallbackApplied: t.fallbackApplied, + fallbackPromptId: t.fallbackPromptId, + model: t.model, + rulesApplied: t.rulesApplied, + error: t.error, + })), + }); + } + + return created; + }); + + return exec.id; +} diff --git a/src/lib/composition/types.ts b/src/lib/composition/types.ts new file mode 100644 index 00000000000..78cc7af5282 --- /dev/null +++ b/src/lib/composition/types.ts @@ -0,0 +1,53 @@ +// Shared types for the Composition Engine (Level 2). + +export type ExecutionStatusLabel = "success" | "partial" | "failed"; + +export interface ExecutionContext { + organizationId: string; + userId: string; + sessionId?: string | null; +} + +/** Per-stage result returned to API callers (matches the execute response schema). */ +export interface StageResult { + stageOrder: number; + promptId: string; + input: string; + output: string; + tokensUsed: number; + duration: number; + success: boolean; + fallbackApplied: boolean; +} + +export interface ExecutionResult { + executionId: string; + status: ExecutionStatusLabel; + stages: StageResult[]; + finalOutput: string; + totalTokens: number; + totalDuration: number; +} + +/** + * Richer per-stage trace captured internally. Maps onto the audited StageTrace + * columns (rawInput/adaptedInput/promptVersion/score/thresholdMet/...), preserved + * so Level-3 scoring stays migration-free. `thresholdMet` is the engine's `success`. + */ +export interface StageTraceInput { + stageOrder: number; + promptId: string; + promptVersion: number; + rawInput: string; + adaptedInput: string; + output: string | null; + tokensUsed: number; + durationMs: number; + score: number | null; + thresholdMet: boolean | null; + fallbackApplied: boolean; + fallbackPromptId: string | null; + model: string | null; + rulesApplied: string[]; + error: string | null; +} From 1cf5ee9388ac5425bd878f683339ea42bf7ca8ad Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 6 Jun 2026 20:57:52 +0000 Subject: [PATCH 13/17] test(composition): add live smoke script for /execute over HTTP Drives one real 3-stage chain through the deployed HTTP routes (create + execute) against a live DB, then asserts the ExecutionRecord and the 3 StageTrace rows landed correctly (incl. adaptation between stages and real token usage). Mints a NextAuth v5 JWT session cookie for a seeded user. Run with: npm run smoke:composition (see header for required env). https://claude.ai/code/session_01HhSSACKMJYh92GqDAakXHT --- package.json | 1 + scripts/smoke-composition.ts | 257 +++++++++++++++++++++++++++++++++++ 2 files changed, 258 insertions(+) create mode 100644 scripts/smoke-composition.ts diff --git a/package.json b/package.json index 2f654c986d4..57a3b0cf4ca 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "db:setup": "prisma generate && prisma migrate dev && prisma db seed", "setup": "node scripts/setup.js", "generate:examples": "npx tsx scripts/generate-examples.ts", + "smoke:composition": "tsx scripts/smoke-composition.ts", "postinstall": "prisma generate", "test": "vitest run", "test:watch": "vitest", diff --git a/scripts/smoke-composition.ts b/scripts/smoke-composition.ts new file mode 100644 index 00000000000..d729c8fd15c --- /dev/null +++ b/scripts/smoke-composition.ts @@ -0,0 +1,257 @@ +/** + * Composition Engine — live smoke test (Level 2). + * + * Drives ONE real 3-stage chain through the actual HTTP routes against a + * DEPLOYED server (staging), using a real OPENAI_API_KEY on the server side, + * then asserts the ExecutionRecord + StageTrace rows landed in the real DB. + * + * It: + * 1. Seeds prerequisites in the DB (org, user, 3 prompts) via Prisma. + * 2. Mints a valid NextAuth (v5, JWT) session cookie for the seeded user. + * 3. POST /api/v1/compositions -> create a 3-stage composition (HTTP) + * 4. POST /api/v1/compositions/:id/execute -> run the chain (HTTP) + * 5. Reads ExecutionRecord + StageTrace back from the DB and asserts them. + * 6. Cleans up (unless SMOKE_KEEP=1). + * + * REQUIRED ENV: + * SMOKE_BASE_URL Base URL of the deployed app, e.g. https://staging.example.com + * DATABASE_URL The SAME database the deployed app writes to (Prisma uses this) + * AUTH secret One of SMOKE_AUTH_SECRET | AUTH_SECRET | NEXTAUTH_SECRET, + * matching the deployment's secret (so the cookie verifies). + * + * OPTIONAL ENV: + * SMOKE_SESSION_COOKIE Override the session cookie name (default derived from + * the URL scheme: __Secure-authjs.session-token on https, + * authjs.session-token on http). + * SMOKE_KEEP=1 Keep the seeded composition/prompts (default: clean up). + * + * RUN: + * SMOKE_BASE_URL=https://staging... DATABASE_URL=postgres://... \ + * NEXTAUTH_SECRET= npx tsx scripts/smoke-composition.ts + */ + +import { PrismaClient } from "@prisma/client"; +import { encode } from "next-auth/jwt"; + +const prisma = new PrismaClient(); + +// ---- env ---- +const BASE_URL = (process.env.SMOKE_BASE_URL || process.env.BASE_URL || "").replace(/\/$/, ""); +const AUTH_SECRET = + process.env.SMOKE_AUTH_SECRET || process.env.AUTH_SECRET || process.env.NEXTAUTH_SECRET || ""; +const KEEP = process.env.SMOKE_KEEP === "1"; + +function fail(msg: string): never { + console.error(`\n❌ ASSERT FAILED: ${msg}`); + process.exit(1); +} +function assert(cond: unknown, msg: string): asserts cond { + if (!cond) fail(msg); + console.log(` ✓ ${msg}`); +} + +interface ExecuteResponse { + executionId: string; + status: "success" | "partial" | "failed"; + stages: Array<{ + stageOrder: number; + promptId: string; + input: string; + output: string; + tokensUsed: number; + duration: number; + success: boolean; + fallbackApplied: boolean; + }>; + finalOutput: string; + totalTokens: number; + totalDuration: number; + executionTrace: unknown[]; +} + +async function main(): Promise { + // ---- preconditions ---- + if (!BASE_URL) fail("SMOKE_BASE_URL (or BASE_URL) is required, e.g. https://staging.example.com"); + if (!process.env.DATABASE_URL) fail("DATABASE_URL is required (same DB the deployed app uses)"); + if (!AUTH_SECRET) fail("AUTH secret required: set SMOKE_AUTH_SECRET / AUTH_SECRET / NEXTAUTH_SECRET to the deployment's value"); + + const isHttps = BASE_URL.startsWith("https://"); + const cookieName = + process.env.SMOKE_SESSION_COOKIE || (isHttps ? "__Secure-authjs.session-token" : "authjs.session-token"); + + console.log(`\n🔧 Smoke test against ${BASE_URL}`); + console.log(` session cookie: ${cookieName}\n`); + + // ---- 1. seed org + user + 3 prompts ---- + console.log("1) Seeding org, user, prompts…"); + const org = await prisma.organization.upsert({ + where: { slug: "smoke-composition" }, + update: {}, + create: { name: "Smoke Test Org", slug: "smoke-composition", tier: "internal" }, + }); + const user = await prisma.user.upsert({ + where: { email: "smoke-composition@prompts.chat" }, + update: { organizationId: org.id }, + create: { + email: "smoke-composition@prompts.chat", + username: "smoke_composition", + name: "Smoke Composition Bot", + role: "USER", + locale: "en", + organizationId: org.id, + }, + }); + + const mkPrompt = (title: string, content: string) => + prisma.prompt.create({ + data: { title, content, type: "TEXT", authorId: user.id, organizationId: org.id }, + }); + const pA = await mkPrompt( + "Smoke Stage 1", + "You are stage 1. Given the user's topic, reply with ONE short factual sentence about it. No preamble.", + ); + const pB = await mkPrompt( + "Smoke Stage 2", + "You are stage 2. Expand the provided context into exactly two concise bullet points. No preamble.", + ); + const pC = await mkPrompt( + "Smoke Stage 3", + "You are stage 3. Summarize the provided text in exactly ONE sentence. No preamble.", + ); + console.log(` org=${org.id} user=${user.id} prompts=[${pA.id}, ${pB.id}, ${pC.id}]`); + + // ---- 2. mint a session cookie for the seeded user ---- + const token = await encode({ + salt: cookieName, + secret: AUTH_SECRET, + maxAge: 60 * 60, + token: { + id: user.id, + sub: user.id, + email: user.email, + name: user.name, + role: user.role, + username: user.username, + locale: user.locale, + picture: null, + }, + }); + const cookie = `${cookieName}=${token}`; + + const postJson = async (path: string, body: unknown): Promise<{ status: number; json: unknown }> => { + const res = await fetch(`${BASE_URL}${path}`, { + method: "POST", + headers: { "content-type": "application/json", cookie }, + body: JSON.stringify(body), + }); + let json: unknown = null; + try { + json = await res.json(); + } catch { + /* non-JSON body */ + } + return { status: res.status, json }; + }; + + // ---- 3. create the composition over HTTP ---- + console.log("\n2) Creating composition via POST /api/v1/compositions…"); + const createRes = await postJson("/api/v1/compositions", { + name: "Smoke 3-stage chain", + description: "Live smoke test composition", + organizationId: org.id, + stages: [ + { + order: 1, + promptId: pA.id, + successThreshold: 0.3, + extractFields: ["sentence"], + transformRules: { sentence: "Context from stage 1" }, + }, + { + order: 2, + promptId: pB.id, + fallbackPromptId: pC.id, + successThreshold: 0.3, + extractFields: ["points"], + transformRules: { points: "Bullet points from stage 2" }, + }, + { order: 3, promptId: pC.id, successThreshold: 0.3, extractFields: [] }, + ], + }); + if (createRes.status !== 201) { + fail(`create returned ${createRes.status} (expected 201). Body: ${JSON.stringify(createRes.json)} — check the session cookie/secret and that the deploy is up.`); + } + const compositionId = (createRes.json as { id: string }).id; + assert(typeof compositionId === "string" && compositionId.length > 0, "composition created, got an id"); + + // ---- 4. execute the chain over HTTP ---- + console.log("\n3) Executing chain via POST /api/v1/compositions/:id/execute…"); + const execRes = await postJson(`/api/v1/compositions/${compositionId}/execute`, { + input: "The water cycle", + context: { organizationId: org.id, userId: user.id, sessionId: "smoke-1" }, + }); + if (execRes.status !== 200) { + fail(`execute returned ${execRes.status} (expected 200). Body: ${JSON.stringify(execRes.json)}`); + } + const result = execRes.json as ExecuteResponse; + console.log(` status=${result.status} executionId=${result.executionId} totalTokens=${result.totalTokens} totalDuration=${result.totalDuration}ms`); + + // ---- 5a. assert the HTTP response shape ---- + console.log("\n4) Asserting HTTP response…"); + assert(typeof result.executionId === "string" && result.executionId.length > 0, "response has executionId"); + assert(["success", "partial", "failed"].includes(result.status), `response.status is valid (${result.status})`); + assert(Array.isArray(result.stages) && result.stages.length === 3, "response has 3 stage results"); + assert(typeof result.finalOutput === "string" && result.finalOutput.length > 0, "response finalOutput is non-empty"); + assert(result.totalTokens > 0, `response totalTokens > 0 (real LLM usage: ${result.totalTokens})`); + + // ---- 5b. assert the rows landed in the real DB ---- + console.log("\n5) Asserting rows in the live database…"); + const exec = await prisma.executionRecord.findUnique({ + where: { id: result.executionId }, + include: { stageTraces: { orderBy: { stageOrder: "asc" } } }, + }); + assert(exec !== null, "ExecutionRecord row exists in the DB"); + if (!exec) return; // type narrowing + assert(exec.compositionId === compositionId, "ExecutionRecord.compositionId matches the composition"); + assert(exec.organizationId === org.id, "ExecutionRecord.organizationId matches the org"); + assert(exec.userId === user.id, "ExecutionRecord.userId matches the context user"); + assert(exec.sessionId === "smoke-1", "ExecutionRecord.sessionId persisted"); + assert(["SUCCESS", "PARTIAL", "FAILED"].includes(exec.status), `ExecutionRecord.status is a valid enum (${exec.status})`); + assert(exec.totalTokens > 0, `ExecutionRecord.totalTokens > 0 (${exec.totalTokens})`); + assert((exec.finalOutput ?? "").length > 0, "ExecutionRecord.finalOutput persisted"); + + assert(exec.stageTraces.length === 3, `exactly 3 StageTrace rows persisted (got ${exec.stageTraces.length})`); + for (const t of exec.stageTraces) { + assert(t.output !== null && t.output.length > 0, `stage ${t.stageOrder}: output persisted`); + assert(t.tokensUsed > 0, `stage ${t.stageOrder}: tokensUsed > 0 (${t.tokensUsed})`); + assert(t.promptVersion >= 1, `stage ${t.stageOrder}: promptVersion pinned`); + assert(typeof t.thresholdMet === "boolean", `stage ${t.stageOrder}: thresholdMet (success) recorded`); + } + + // adaptation proof: each stage's adaptedInput carries the prior stage's transformRule label + const t2 = exec.stageTraces.find((t) => t.stageOrder === 2); + const t3 = exec.stageTraces.find((t) => t.stageOrder === 3); + assert(!!t2 && t2.adaptedInput.includes("Context from stage 1"), "stage 2 adaptedInput shows stage-1 adaptation"); + assert(!!t3 && t3.adaptedInput.includes("Bullet points from stage 2"), "stage 3 adaptedInput shows stage-2 adaptation"); + + // ---- 6. cleanup ---- + if (KEEP) { + console.log(`\n6) SMOKE_KEEP=1 → leaving data. compositionId=${compositionId}, executionId=${result.executionId}`); + } else { + console.log("\n6) Cleaning up (composition cascades execution_records + stage_traces)…"); + await prisma.composition.delete({ where: { id: compositionId } }); + await prisma.prompt.deleteMany({ where: { id: { in: [pA.id, pB.id, pC.id] } } }); + console.log(" cleaned up composition + prompts (org/user retained for reuse)."); + } + + console.log("\n✅ SMOKE TEST PASSED — chain executed over HTTP and traces verified in the live DB.\n"); +} + +main() + .catch((err) => { + console.error("\n❌ SMOKE TEST ERROR:", err instanceof Error ? err.message : err); + process.exitCode = 1; + }) + .finally(async () => { + await prisma.$disconnect(); + }); From 6516df94e4054294cfa840b8e6f0827ce86c9df2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 13 Jun 2026 01:44:08 +0000 Subject: [PATCH 14/17] fix(security): derive identity server-side in composition routes The five /api/v1/compositions routes trusted organizationId and userId from the request body and only checked that a session existed, letting any authenticated user read, execute, or attribute work to another org. - userId now always comes from the authenticated session - organizationId is resolved server-side from the user's org, falling back to the default internal org (new src/lib/organization.ts) - client-supplied organizationId/userId are optional and rejected with 403 on mismatch; they are never used as the source of truth - list is scoped to the caller's org; all [id] routes return 404 for another org's composition so ids don't leak - 15 new route-level authz tests cover the rejection paths scripts/smoke-composition.ts is unchanged: its payloads supply the seeded user's real org/user ids, which now pass the mismatch checks. https://claude.ai/code/session_01GP2YigWVnAGR3xGQ59N4SK --- src/__tests__/composition/authz.test.ts | 277 ++++++++++++++++++ .../api/v1/compositions/[id]/execute/route.ts | 43 ++- .../api/v1/compositions/[id]/history/route.ts | 15 + src/app/api/v1/compositions/[id]/route.ts | 9 +- .../api/v1/compositions/[id]/test/route.ts | 45 ++- src/app/api/v1/compositions/route.ts | 30 +- src/lib/organization.ts | 28 ++ 7 files changed, 429 insertions(+), 18 deletions(-) create mode 100644 src/__tests__/composition/authz.test.ts create mode 100644 src/lib/organization.ts diff --git a/src/__tests__/composition/authz.test.ts b/src/__tests__/composition/authz.test.ts new file mode 100644 index 00000000000..9e9a65768ac --- /dev/null +++ b/src/__tests__/composition/authz.test.ts @@ -0,0 +1,277 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import type { NextRequest } from "next/server"; + +// Route-level authz tests for /api/v1/compositions: userId comes from the +// session, organizationId is resolved server-side, and cross-org access is +// rejected on every route. + +vi.mock("@/lib/db", () => ({ + db: { + user: { findUnique: vi.fn() }, + organization: { findUnique: vi.fn() }, + composition: { create: vi.fn(), findMany: vi.fn(), findUnique: vi.fn() }, + executionRecord: { count: vi.fn(), findMany: vi.fn() }, + }, +})); + +vi.mock("@/lib/auth", () => ({ + auth: vi.fn(), +})); + +vi.mock("@/lib/composition", () => ({ + executeComposition: vi.fn(), +})); + +import { db } from "@/lib/db"; +import { auth } from "@/lib/auth"; +import { executeComposition } from "@/lib/composition"; +import { POST as createComposition, GET as listCompositions } from "@/app/api/v1/compositions/route"; +import { GET as getComposition } from "@/app/api/v1/compositions/[id]/route"; +import { GET as getHistory } from "@/app/api/v1/compositions/[id]/history/route"; +import { POST as executeRoute } from "@/app/api/v1/compositions/[id]/execute/route"; +import { POST as testRoute } from "@/app/api/v1/compositions/[id]/test/route"; + +const BASE = "http://localhost:3000/api/v1/compositions"; + +function jsonRequest(url: string, body?: unknown): NextRequest { + return new Request(url, { + method: body === undefined ? "GET" : "POST", + headers: { "content-type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), + }) as unknown as NextRequest; +} + +const params = (id: string) => ({ params: Promise.resolve({ id }) }); + +// next-auth's `auth` export is overloaded (session getter + middleware wrapper), +// which breaks vi.mocked's inference — cast once to a plain mock. +const mockedAuth = auth as unknown as ReturnType; + +const mockSession = (userId: string) => mockedAuth.mockResolvedValue({ user: { id: userId } }); + +const mockUserOrg = (organizationId: string | null) => + // eslint-disable-next-line @typescript-eslint/no-explicit-any + vi.mocked(db.user.findUnique).mockResolvedValue({ organizationId } as any); + +const validStages = [{ order: 1, promptId: "p1" }]; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("POST /api/v1/compositions (create)", () => { + it("returns 401 without a session", async () => { + mockedAuth.mockResolvedValue(null); + const res = await createComposition(jsonRequest(BASE, { name: "x", stages: validStages })); + expect(res.status).toBe(401); + }); + + it("rejects a client-supplied organizationId that mismatches the user's org", async () => { + mockSession("u1"); + mockUserOrg("org1"); + const res = await createComposition( + jsonRequest(BASE, { name: "x", organizationId: "org2", stages: validStages }), + ); + expect(res.status).toBe(403); + expect((await res.json()).error).toBe("organization_mismatch"); + expect(db.composition.create).not.toHaveBeenCalled(); + }); + + it("derives organizationId and createdBy server-side, ignoring the body for identity", async () => { + mockSession("u1"); + mockUserOrg("org1"); + vi.mocked(db.composition.create).mockResolvedValue({ + id: "c1", + name: "x", + description: null, + stages: [], + createdAt: new Date(), + createdBy: "u1", + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any); + + const res = await createComposition(jsonRequest(BASE, { name: "x", stages: validStages })); + expect(res.status).toBe(201); + const createArgs = vi.mocked(db.composition.create).mock.calls[0][0]; + expect(createArgs.data.organizationId).toBe("org1"); + expect(createArgs.data.createdBy).toBe("u1"); + }); + + it("falls back to the default internal org for org-less users", async () => { + mockSession("u1"); + mockUserOrg(null); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + vi.mocked(db.organization.findUnique).mockResolvedValue({ id: "org_default" } as any); + vi.mocked(db.composition.create).mockResolvedValue({ + id: "c1", + name: "x", + description: null, + stages: [], + createdAt: new Date(), + createdBy: "u1", + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any); + + const res = await createComposition(jsonRequest(BASE, { name: "x", stages: validStages })); + expect(res.status).toBe(201); + expect(db.organization.findUnique).toHaveBeenCalledWith( + expect.objectContaining({ where: { slug: "internal" } }), + ); + expect(vi.mocked(db.composition.create).mock.calls[0][0].data.organizationId).toBe( + "org_default", + ); + }); + + it("returns 403 when the user has no org and the default org is missing", async () => { + mockSession("u1"); + mockUserOrg(null); + vi.mocked(db.organization.findUnique).mockResolvedValue(null); + const res = await createComposition(jsonRequest(BASE, { name: "x", stages: validStages })); + expect(res.status).toBe(403); + expect((await res.json()).error).toBe("no_organization"); + }); +}); + +describe("GET /api/v1/compositions (list)", () => { + it("always scopes the query to the caller's organization", async () => { + mockSession("u1"); + mockUserOrg("org1"); + vi.mocked(db.composition.findMany).mockResolvedValue([]); + + const res = await listCompositions(jsonRequest(BASE)); + expect(res.status).toBe(200); + expect(vi.mocked(db.composition.findMany).mock.calls[0][0]).toMatchObject({ + where: { organizationId: "org1" }, + }); + }); + + it("rejects an organizationId query param for another org", async () => { + mockSession("u1"); + mockUserOrg("org1"); + const res = await listCompositions(jsonRequest(`${BASE}?organizationId=org2`)); + expect(res.status).toBe(403); + expect(db.composition.findMany).not.toHaveBeenCalled(); + }); +}); + +describe("GET /api/v1/compositions/[id]", () => { + it("returns 404 for another org's composition", async () => { + mockSession("u1"); + mockUserOrg("org1"); + vi.mocked(db.composition.findUnique).mockResolvedValue({ + id: "c1", + organizationId: "org2", + stages: [], + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any); + + const res = await getComposition(jsonRequest(`${BASE}/c1`), params("c1")); + expect(res.status).toBe(404); + }); +}); + +describe("GET /api/v1/compositions/[id]/history", () => { + it("returns 404 for another org's composition without querying records", async () => { + mockSession("u1"); + mockUserOrg("org1"); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + vi.mocked(db.composition.findUnique).mockResolvedValue({ organizationId: "org2" } as any); + + const res = await getHistory(jsonRequest(`${BASE}/c1/history`), params("c1")); + expect(res.status).toBe(404); + expect(db.executionRecord.findMany).not.toHaveBeenCalled(); + }); +}); + +describe("POST /api/v1/compositions/[id]/execute", () => { + it("returns 404 for another org's composition and never executes", async () => { + mockSession("u1"); + mockUserOrg("org1"); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + vi.mocked(db.composition.findUnique).mockResolvedValue({ organizationId: "org2" } as any); + + const res = await executeRoute( + jsonRequest(`${BASE}/c1/execute`, { input: "hi" }), + params("c1"), + ); + expect(res.status).toBe(404); + expect(executeComposition).not.toHaveBeenCalled(); + }); + + it("rejects a context.userId that is not the session user", async () => { + mockSession("u1"); + mockUserOrg("org1"); + const res = await executeRoute( + jsonRequest(`${BASE}/c1/execute`, { input: "hi", context: { userId: "u2" } }), + params("c1"), + ); + expect(res.status).toBe(403); + expect((await res.json()).error).toBe("user_mismatch"); + expect(executeComposition).not.toHaveBeenCalled(); + }); + + it("rejects a context.organizationId for another org", async () => { + mockSession("u1"); + mockUserOrg("org1"); + const res = await executeRoute( + jsonRequest(`${BASE}/c1/execute`, { input: "hi", context: { organizationId: "org2" } }), + params("c1"), + ); + expect(res.status).toBe(403); + expect((await res.json()).error).toBe("organization_mismatch"); + expect(executeComposition).not.toHaveBeenCalled(); + }); + + it("builds the execution context from the session, not the body", async () => { + mockSession("u1"); + mockUserOrg("org1"); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + vi.mocked(db.composition.findUnique).mockResolvedValue({ organizationId: "org1" } as any); + vi.mocked(executeComposition).mockResolvedValue({ + executionId: "e1", + status: "success", + stages: [], + finalOutput: "out", + totalTokens: 1, + totalDuration: 1, + }); + + const res = await executeRoute( + jsonRequest(`${BASE}/c1/execute`, { input: "hi", context: { sessionId: "s1" } }), + params("c1"), + ); + expect(res.status).toBe(200); + expect(executeComposition).toHaveBeenCalledWith("c1", "hi", { + organizationId: "org1", + userId: "u1", + sessionId: "s1", + }); + }); +}); + +describe("POST /api/v1/compositions/[id]/test", () => { + it("returns 404 for another org's composition and never executes", async () => { + mockSession("u1"); + mockUserOrg("org1"); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + vi.mocked(db.composition.findUnique).mockResolvedValue({ organizationId: "org2" } as any); + + const res = await testRoute( + jsonRequest(`${BASE}/c1/test`, { sampleInputs: ["hi"] }), + params("c1"), + ); + expect(res.status).toBe(404); + expect(executeComposition).not.toHaveBeenCalled(); + }); + + it("rejects a context.userId that is not the session user", async () => { + mockSession("u1"); + mockUserOrg("org1"); + const res = await testRoute( + jsonRequest(`${BASE}/c1/test`, { sampleInputs: ["hi"], context: { userId: "u2" } }), + params("c1"), + ); + expect(res.status).toBe(403); + expect(executeComposition).not.toHaveBeenCalled(); + }); +}); diff --git a/src/app/api/v1/compositions/[id]/execute/route.ts b/src/app/api/v1/compositions/[id]/execute/route.ts index 4f7602e70b5..fa8a083cbe8 100644 --- a/src/app/api/v1/compositions/[id]/execute/route.ts +++ b/src/app/api/v1/compositions/[id]/execute/route.ts @@ -1,16 +1,22 @@ import { NextRequest, NextResponse } from "next/server"; import { z } from "zod"; import { auth } from "@/lib/auth"; +import { db } from "@/lib/db"; import { requestLogger } from "@/lib/logger"; import { executeComposition } from "@/lib/composition"; +import { resolveUserOrganizationId } from "@/lib/organization"; const executeSchema = z.object({ input: z.string().min(1), - context: z.object({ - organizationId: z.string().min(1), - userId: z.string().min(1), - sessionId: z.string().nullable().optional(), - }), + // organizationId / userId are optional and only ever validated against the + // session-derived values — the execution context is built server-side. + context: z + .object({ + organizationId: z.string().min(1).optional(), + userId: z.string().min(1).optional(), + sessionId: z.string().nullable().optional(), + }) + .optional(), }); // POST /api/v1/compositions/[id]/execute — run the chain end-to-end @@ -25,11 +31,36 @@ export async function POST( return NextResponse.json({ error: "unauthorized" }, { status: 401 }); } + const organizationId = await resolveUserOrganizationId(session.user.id); + if (!organizationId) { + return NextResponse.json({ error: "no_organization" }, { status: 403 }); + } + const { id } = await params; const body = await request.json(); const { input, context } = executeSchema.parse(body); - const result = await executeComposition(id, input, context); + if (context?.organizationId && context.organizationId !== organizationId) { + return NextResponse.json({ error: "organization_mismatch" }, { status: 403 }); + } + if (context?.userId && context.userId !== session.user.id) { + return NextResponse.json({ error: "user_mismatch" }, { status: 403 }); + } + + const composition = await db.composition.findUnique({ + where: { id }, + select: { organizationId: true }, + }); + // Cross-org executes return 404 (not 403) so composition ids don't leak. + if (!composition || composition.organizationId !== organizationId) { + return NextResponse.json({ error: "not_found" }, { status: 404 }); + } + + const result = await executeComposition(id, input, { + organizationId, + userId: session.user.id, + sessionId: context?.sessionId ?? null, + }); log.info( { diff --git a/src/app/api/v1/compositions/[id]/history/route.ts b/src/app/api/v1/compositions/[id]/history/route.ts index 2ad7e47794b..476bbe96ce4 100644 --- a/src/app/api/v1/compositions/[id]/history/route.ts +++ b/src/app/api/v1/compositions/[id]/history/route.ts @@ -2,6 +2,7 @@ import { NextRequest, NextResponse } from "next/server"; import { auth } from "@/lib/auth"; import { db } from "@/lib/db"; import { requestLogger } from "@/lib/logger"; +import { resolveUserOrganizationId } from "@/lib/organization"; // GET /api/v1/compositions/[id]/history — paginated execution history export async function GET( @@ -15,7 +16,21 @@ export async function GET( return NextResponse.json({ error: "unauthorized" }, { status: 401 }); } + const organizationId = await resolveUserOrganizationId(session.user.id); + if (!organizationId) { + return NextResponse.json({ error: "no_organization" }, { status: 403 }); + } + const { id } = await params; + const composition = await db.composition.findUnique({ + where: { id }, + select: { organizationId: true }, + }); + // Cross-org reads return 404 (not 403) so composition ids don't leak. + if (!composition || composition.organizationId !== organizationId) { + return NextResponse.json({ error: "not_found" }, { status: 404 }); + } + const { searchParams } = new URL(request.url); const page = Math.max(1, parseInt(searchParams.get("page") ?? "1", 10) || 1); const limit = Math.min(100, Math.max(1, parseInt(searchParams.get("limit") ?? "20", 10) || 20)); diff --git a/src/app/api/v1/compositions/[id]/route.ts b/src/app/api/v1/compositions/[id]/route.ts index 7e724b3b01a..0dac48fc941 100644 --- a/src/app/api/v1/compositions/[id]/route.ts +++ b/src/app/api/v1/compositions/[id]/route.ts @@ -2,6 +2,7 @@ import { NextRequest, NextResponse } from "next/server"; import { auth } from "@/lib/auth"; import { db } from "@/lib/db"; import { requestLogger } from "@/lib/logger"; +import { resolveUserOrganizationId } from "@/lib/organization"; // GET /api/v1/compositions/[id] — fetch a single composition with its stages export async function GET( @@ -15,13 +16,19 @@ export async function GET( return NextResponse.json({ error: "unauthorized" }, { status: 401 }); } + const organizationId = await resolveUserOrganizationId(session.user.id); + if (!organizationId) { + return NextResponse.json({ error: "no_organization" }, { status: 403 }); + } + const { id } = await params; const composition = await db.composition.findUnique({ where: { id }, include: { stages: { orderBy: { order: "asc" } } }, }); - if (!composition) { + // Cross-org reads return 404 (not 403) so composition ids don't leak. + if (!composition || composition.organizationId !== organizationId) { return NextResponse.json({ error: "not_found" }, { status: 404 }); } diff --git a/src/app/api/v1/compositions/[id]/test/route.ts b/src/app/api/v1/compositions/[id]/test/route.ts index 149d43d7678..b40c41ed66e 100644 --- a/src/app/api/v1/compositions/[id]/test/route.ts +++ b/src/app/api/v1/compositions/[id]/test/route.ts @@ -1,16 +1,22 @@ import { NextRequest, NextResponse } from "next/server"; import { z } from "zod"; import { auth } from "@/lib/auth"; +import { db } from "@/lib/db"; import { requestLogger } from "@/lib/logger"; import { executeComposition } from "@/lib/composition"; +import { resolveUserOrganizationId } from "@/lib/organization"; const testSchema = z.object({ sampleInputs: z.array(z.string().min(1)).min(1), - context: z.object({ - organizationId: z.string().min(1), - userId: z.string().min(1), - sessionId: z.string().nullable().optional(), - }), + // organizationId / userId are optional and only ever validated against the + // session-derived values — the execution context is built server-side. + context: z + .object({ + organizationId: z.string().min(1).optional(), + userId: z.string().min(1).optional(), + sessionId: z.string().nullable().optional(), + }) + .optional(), }); // POST /api/v1/compositions/[id]/test — run the chain against multiple sample @@ -26,10 +32,37 @@ export async function POST( return NextResponse.json({ error: "unauthorized" }, { status: 401 }); } + const organizationId = await resolveUserOrganizationId(session.user.id); + if (!organizationId) { + return NextResponse.json({ error: "no_organization" }, { status: 403 }); + } + const { id } = await params; const body = await request.json(); const { sampleInputs, context } = testSchema.parse(body); + if (context?.organizationId && context.organizationId !== organizationId) { + return NextResponse.json({ error: "organization_mismatch" }, { status: 403 }); + } + if (context?.userId && context.userId !== session.user.id) { + return NextResponse.json({ error: "user_mismatch" }, { status: 403 }); + } + + const composition = await db.composition.findUnique({ + where: { id }, + select: { organizationId: true }, + }); + // Cross-org test runs return 404 (not 403) so composition ids don't leak. + if (!composition || composition.organizationId !== organizationId) { + return NextResponse.json({ error: "not_found" }, { status: 404 }); + } + + const executionContext = { + organizationId, + userId: session.user.id, + sessionId: context?.sessionId ?? null, + }; + const results: Array<{ input: string; executionId: string; @@ -42,7 +75,7 @@ export async function POST( // Sequential to avoid hammering the LLM provider with parallel chains. for (const input of sampleInputs) { - const r = await executeComposition(id, input, context); + const r = await executeComposition(id, input, executionContext); results.push({ input, executionId: r.executionId, diff --git a/src/app/api/v1/compositions/route.ts b/src/app/api/v1/compositions/route.ts index 6213b1d892f..759368fd3e7 100644 --- a/src/app/api/v1/compositions/route.ts +++ b/src/app/api/v1/compositions/route.ts @@ -4,6 +4,7 @@ import { Prisma } from "@prisma/client"; import { auth } from "@/lib/auth"; import { db } from "@/lib/db"; import { requestLogger } from "@/lib/logger"; +import { resolveUserOrganizationId } from "@/lib/organization"; const stageSchema = z.object({ order: z.number().int(), @@ -17,7 +18,9 @@ const stageSchema = z.object({ const createSchema = z.object({ name: z.string().min(1), description: z.string().optional(), - organizationId: z.string().min(1), + // Optional and only ever validated against the server-resolved org — never + // used as the source of truth. + organizationId: z.string().min(1).optional(), stages: z.array(stageSchema).min(1), }); @@ -30,9 +33,18 @@ export async function POST(request: NextRequest) { return NextResponse.json({ error: "unauthorized" }, { status: 401 }); } + const organizationId = await resolveUserOrganizationId(session.user.id); + if (!organizationId) { + return NextResponse.json({ error: "no_organization" }, { status: 403 }); + } + const body = await request.json(); const data = createSchema.parse(body); + if (data.organizationId && data.organizationId !== organizationId) { + return NextResponse.json({ error: "organization_mismatch" }, { status: 403 }); + } + const orders = data.stages.map((s) => s.order); if (new Set(orders).size !== orders.length) { return NextResponse.json({ error: "duplicate stage order" }, { status: 400 }); @@ -42,7 +54,7 @@ export async function POST(request: NextRequest) { data: { name: data.name, description: data.description, - organizationId: data.organizationId, + organizationId, createdBy: session.user.id, stages: { create: data.stages.map((s) => ({ @@ -86,7 +98,7 @@ export async function POST(request: NextRequest) { } } -// GET /api/v1/compositions — list compositions (optionally by organizationId) +// GET /api/v1/compositions — list compositions in the caller's organization export async function GET(request: NextRequest) { const log = requestLogger(request.headers.get("x-request-id")); try { @@ -95,11 +107,19 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: "unauthorized" }, { status: 401 }); } + const organizationId = await resolveUserOrganizationId(session.user.id); + if (!organizationId) { + return NextResponse.json({ error: "no_organization" }, { status: 403 }); + } + const { searchParams } = new URL(request.url); - const organizationId = searchParams.get("organizationId") ?? undefined; + const requestedOrgId = searchParams.get("organizationId"); + if (requestedOrgId && requestedOrgId !== organizationId) { + return NextResponse.json({ error: "organization_mismatch" }, { status: 403 }); + } const compositions = await db.composition.findMany({ - where: organizationId ? { organizationId } : undefined, + where: { organizationId }, orderBy: { createdAt: "desc" }, include: { stages: { orderBy: { order: "asc" } }, diff --git a/src/lib/organization.ts b/src/lib/organization.ts new file mode 100644 index 00000000000..6c8fa158e2e --- /dev/null +++ b/src/lib/organization.ts @@ -0,0 +1,28 @@ +import { db } from "@/lib/db"; + +/** Slug of the default internal organization (created by the backfill migration). */ +export const DEFAULT_ORG_SLUG = "internal"; + +/** + * Resolve the organization an authenticated user acts within, server-side. + * Client-supplied organization ids must never be trusted — they are only ever + * compared against this value. Users that predate multi-tenancy (organizationId + * still NULL) fall back to the default internal org. + * + * Returns null only if the user has no org AND the default org is missing, + * which callers should treat as a 403. + */ +export async function resolveUserOrganizationId(userId: string): Promise { + const user = await db.user.findUnique({ + where: { id: userId }, + select: { organizationId: true }, + }); + if (user?.organizationId) { + return user.organizationId; + } + const fallback = await db.organization.findUnique({ + where: { slug: DEFAULT_ORG_SLUG }, + select: { id: true }, + }); + return fallback?.id ?? null; +} From 5bf493da15ef2ab69bb35dab2644f32ef243d2eb Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 13 Jun 2026 01:47:17 +0000 Subject: [PATCH 15/17] feat(ai): add Anthropic provider with prompt caching (default-off) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AnthropicProvider implements the existing LLMProvider interface — no changes to stageExecutor or any call site. Selection is opt-in: the provider is used only when LLM_PROVIDER=anthropic AND ANTHROPIC_API_KEY are both set; otherwise the OpenAI provider runs exactly as before. - model from ANTHROPIC_GENERATIVE_MODEL, default claude-opus-4-8 - prompt caching v1: the stage's system prompt is a stable prefix re-sent on every chain execution, so it carries cache_control (ephemeral) and repeated executions hit the cache - cache read/write tokens fold into inputTokens so StageTrace token accounting stays comparable across providers - temperature is not forwarded (current Claude models reject sampling parameters) - all three env vars documented in .env.example - 9 unit tests with a mocked SDK: completion mapping, token accounting, cache flag, no-sampling-params, and selection logic Citations support is deferred to the Level 3 spec. https://claude.ai/code/session_01GP2YigWVnAGR3xGQ59N4SK --- .env.example | 7 + package-lock.json | 64 +++++++++- package.json | 1 + src/__tests__/lib/ai/llm-provider.test.ts | 148 ++++++++++++++++++++++ src/lib/ai/llm-provider.ts | 86 ++++++++++++- 5 files changed, 298 insertions(+), 8 deletions(-) create mode 100644 src/__tests__/lib/ai/llm-provider.test.ts diff --git a/.env.example b/.env.example index 32346563390..e8db491c81e 100644 --- a/.env.example +++ b/.env.example @@ -45,6 +45,13 @@ NEXTAUTH_SECRET="your-super-secret-key-change-in-production" # OPENAI_GENERATIVE_MODEL=gpt-4o-mini # Optional: generative model for AI generation # OPENAI_TRANSLATION_MODEL=gpt-4o-mini # Optional: model for translating non-English search queries +# Composition Engine LLM provider (optional - default is OpenAI) +# The Anthropic provider is used only when LLM_PROVIDER=anthropic AND +# ANTHROPIC_API_KEY are both set; otherwise behavior is unchanged. +# LLM_PROVIDER=anthropic +# ANTHROPIC_API_KEY=your_anthropic_api_key +# ANTHROPIC_GENERATIVE_MODEL=claude-opus-4-8 # Optional: model for composition stage execution + # GOOGLE_ANALYTICS_ID="G-XXXXXXXXX" # Logging (optional) diff --git a/package-lock.json b/package-lock.json index 77a8eb3064b..e72c522666e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "hasInstallScript": true, "dependencies": { + "@anthropic-ai/sdk": "^0.104.1", "@auth/prisma-adapter": "^2.11.1", "@aws-sdk/client-s3": "^3.948.0", "@hookform/resolvers": "^5.2.2", @@ -127,6 +128,27 @@ "node": ">=6.0.0" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.104.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.104.1.tgz", + "integrity": "sha512-gGACa/+IaiXzRRmF96aOhamoBgapKRBiFWbmmTFP8aMkpaEcuStF+Q61bjo4vPxBM7gqWJNZqsngslRdnLHv0Q==", + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1", + "standardwebhooks": "^1.0.0" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, "node_modules/@apm-js-collab/code-transformer": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/@apm-js-collab/code-transformer/-/code-transformer-0.8.2.tgz", @@ -1315,7 +1337,6 @@ "version": "7.28.4", "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.4.tgz", "integrity": "sha512-Q/N6JNWvIvPnLDvjlE1OUBLPQHH6l3CltCEsHIujp45zQUSSh8K+gHnaEX45yAT1nyngnINhvWtzN+Nb9D8RAQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -7204,6 +7225,12 @@ "node": ">=18.0.0" } }, + "node_modules/@stablelib/base64": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", + "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", + "license": "MIT" + }, "node_modules/@standard-schema/spec": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", @@ -12601,6 +12628,12 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-sha256": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", + "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", + "license": "Unlicense" + }, "node_modules/fast-uri": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", @@ -14334,6 +14367,19 @@ "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "license": "MIT" }, + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -19140,6 +19186,16 @@ "node": ">=6" } }, + "node_modules/standardwebhooks": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.0.0.tgz", + "integrity": "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==", + "license": "MIT", + "dependencies": { + "@stablelib/base64": "^1.0.0", + "fast-sha256": "^1.3.0" + } + }, "node_modules/state-local": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/state-local/-/state-local-1.0.7.tgz", @@ -19910,6 +19966,12 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" + }, "node_modules/ts-api-utils": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz", diff --git a/package.json b/package.json index 57a3b0cf4ca..d6b5402cfbf 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "book:pdf:print:all": "npx tsx scripts/generate-book-pdf.ts --all --print && npx tsx scripts/html-to-pdf.ts --all --print" }, "dependencies": { + "@anthropic-ai/sdk": "^0.104.1", "@auth/prisma-adapter": "^2.11.1", "@aws-sdk/client-s3": "^3.948.0", "@hookform/resolvers": "^5.2.2", diff --git a/src/__tests__/lib/ai/llm-provider.test.ts b/src/__tests__/lib/ai/llm-provider.test.ts new file mode 100644 index 00000000000..34065e00d19 --- /dev/null +++ b/src/__tests__/lib/ai/llm-provider.test.ts @@ -0,0 +1,148 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +// Unit tests for the LLM provider abstraction: Anthropic completion mapping, +// token accounting (cache tokens folded into inputTokens), the prompt-caching +// flag on the system prompt, and env-driven provider selection. + +const { mockCreate } = vi.hoisted(() => ({ mockCreate: vi.fn() })); + +vi.mock("@anthropic-ai/sdk", () => ({ + default: class MockAnthropic { + messages = { create: mockCreate }; + constructor(public opts: { apiKey: string }) {} + }, +})); + +import { getLLMProvider, setLLMProvider } from "@/lib/ai/llm-provider"; + +function anthropicResponse(overrides: Record = {}) { + return { + content: [{ type: "text", text: " hello from claude " }], + usage: { + input_tokens: 10, + output_tokens: 5, + cache_creation_input_tokens: 100, + cache_read_input_tokens: 200, + }, + model: "claude-opus-4-8", + ...overrides, + }; +} + +beforeEach(() => { + setLLMProvider(null); // clear the cached provider so env changes take effect + mockCreate.mockReset(); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + setLLMProvider(null); +}); + +describe("provider selection", () => { + it("defaults to OpenAI when no provider env is set", () => { + vi.stubEnv("LLM_PROVIDER", ""); + vi.stubEnv("ANTHROPIC_API_KEY", ""); + expect(getLLMProvider().name).toBe("openai"); + }); + + it("stays on OpenAI when LLM_PROVIDER=anthropic but no API key is set", () => { + vi.stubEnv("LLM_PROVIDER", "anthropic"); + vi.stubEnv("ANTHROPIC_API_KEY", ""); + expect(getLLMProvider().name).toBe("openai"); + }); + + it("stays on OpenAI when only ANTHROPIC_API_KEY is set", () => { + vi.stubEnv("LLM_PROVIDER", ""); + vi.stubEnv("ANTHROPIC_API_KEY", "sk-test"); + expect(getLLMProvider().name).toBe("openai"); + }); + + it("selects Anthropic only when LLM_PROVIDER=anthropic and ANTHROPIC_API_KEY are both set", () => { + vi.stubEnv("LLM_PROVIDER", "anthropic"); + vi.stubEnv("ANTHROPIC_API_KEY", "sk-test"); + expect(getLLMProvider().name).toBe("anthropic"); + }); +}); + +describe("AnthropicProvider.complete", () => { + beforeEach(() => { + vi.stubEnv("LLM_PROVIDER", "anthropic"); + vi.stubEnv("ANTHROPIC_API_KEY", "sk-test"); + }); + + it("maps text output, token usage, and model onto LLMCompletionResult", async () => { + mockCreate.mockResolvedValue(anthropicResponse()); + + const result = await getLLMProvider().complete({ + system: "You are stage 1.", + prompt: "The water cycle", + }); + + expect(result.output).toBe("hello from claude"); + // cache write (100) + cache read (200) are input tokens too + expect(result.inputTokens).toBe(310); + expect(result.outputTokens).toBe(5); + expect(result.model).toBe("claude-opus-4-8"); + }); + + it("concatenates only text blocks from mixed content", async () => { + mockCreate.mockResolvedValue( + anthropicResponse({ + content: [ + { type: "text", text: "part one" }, + { type: "tool_use", id: "t1", name: "x", input: {} }, + { type: "text", text: " part two" }, + ], + }), + ); + + const result = await getLLMProvider().complete({ prompt: "hi" }); + expect(result.output).toBe("part one part two"); + }); + + it("marks the system prompt with cache_control and never sends sampling params", async () => { + mockCreate.mockResolvedValue(anthropicResponse()); + + await getLLMProvider().complete({ + system: "Stable stage prompt", + prompt: "input", + temperature: 0.9, + maxTokens: 1234, + }); + + const request = mockCreate.mock.calls[0][0]; + expect(request.system).toEqual([ + { + type: "text", + text: "Stable stage prompt", + cache_control: { type: "ephemeral" }, + }, + ]); + expect(request.messages).toEqual([{ role: "user", content: "input" }]); + expect(request.max_tokens).toBe(1234); + // Opus 4.7+ rejects sampling parameters — temperature must not be forwarded. + expect(request).not.toHaveProperty("temperature"); + }); + + it("omits the system field entirely when no system prompt is given", async () => { + mockCreate.mockResolvedValue(anthropicResponse()); + + await getLLMProvider().complete({ prompt: "input" }); + expect(mockCreate.mock.calls[0][0]).not.toHaveProperty("system"); + }); + + it("uses ANTHROPIC_GENERATIVE_MODEL when set, defaulting to claude-opus-4-8", async () => { + mockCreate.mockResolvedValue(anthropicResponse({ model: undefined })); + + await getLLMProvider().complete({ prompt: "hi" }); + expect(mockCreate.mock.calls[0][0].model).toBe("claude-opus-4-8"); + + setLLMProvider(null); + vi.stubEnv("ANTHROPIC_GENERATIVE_MODEL", "claude-sonnet-4-6"); + mockCreate.mockResolvedValue(anthropicResponse()); + + await getLLMProvider().complete({ prompt: "hi" }); + expect(mockCreate.mock.calls[1][0].model).toBe("claude-sonnet-4-6"); + }); +}); diff --git a/src/lib/ai/llm-provider.ts b/src/lib/ai/llm-provider.ts index 327b4bef725..98f09bedede 100644 --- a/src/lib/ai/llm-provider.ts +++ b/src/lib/ai/llm-provider.ts @@ -1,13 +1,14 @@ import OpenAI from "openai"; +import Anthropic from "@anthropic-ai/sdk"; // Provider-agnostic LLM completion abstraction. // // stageExecutor (and any future caller) depends ONLY on this interface, so the -// underlying provider can be swapped without touching call sites. v1 ships an -// OpenAI-backed implementation that reuses the existing src/lib/ai setup -// (OPENAI_API_KEY / OPENAI_BASE_URL / OPENAI_GENERATIVE_MODEL). Claude can be -// added later as an alternate provider (e.g. when ANTHROPIC_API_KEY exists) -// by registering a new LLMProvider here — no engine changes required. +// underlying provider can be swapped without touching call sites. The default +// is the OpenAI-backed implementation that reuses the existing src/lib/ai +// setup (OPENAI_API_KEY / OPENAI_BASE_URL / OPENAI_GENERATIVE_MODEL). The +// Anthropic-backed provider is selected only when BOTH LLM_PROVIDER=anthropic +// and ANTHROPIC_API_KEY are set — existing behavior is otherwise unchanged. export interface LLMCompletionRequest { /** Optional system / instruction prompt (the stage's prompt content). */ @@ -73,12 +74,83 @@ class OpenAIProvider implements LLMProvider { } } +class AnthropicProvider implements LLMProvider { + readonly name = "anthropic"; + private client: Anthropic | null = null; + private readonly model = process.env.ANTHROPIC_GENERATIVE_MODEL || "claude-opus-4-8"; + + private getClient(): Anthropic { + if (!this.client) { + const apiKey = process.env.ANTHROPIC_API_KEY; + if (!apiKey) { + throw new Error("ANTHROPIC_API_KEY is not set"); + } + this.client = new Anthropic({ apiKey }); + } + return this.client; + } + + async complete(req: LLMCompletionRequest): Promise { + const client = this.getClient(); + + const response = await client.messages.create({ + model: this.model, + max_tokens: req.maxTokens ?? 4000, + // The stage's system prompt is a stable prefix re-sent on every chain + // execution — cache_control marks it so repeated executions hit the + // prompt cache instead of re-processing it at full price. + ...(req.system + ? { + system: [ + { + type: "text" as const, + text: req.system, + cache_control: { type: "ephemeral" as const }, + }, + ], + } + : {}), + messages: [{ role: "user", content: req.prompt }], + // req.temperature is intentionally not forwarded: current Claude models + // (Opus 4.7+) reject sampling parameters with a 400. + }); + + const output = response.content + .map((block) => (block.type === "text" ? block.text : "")) + .join("") + .trim(); + + const usage = response.usage; + return { + output, + // Cache reads/writes are still input tokens — fold them in so + // StageTrace.tokensUsed stays comparable across providers. + inputTokens: + (usage?.input_tokens ?? 0) + + (usage?.cache_creation_input_tokens ?? 0) + + (usage?.cache_read_input_tokens ?? 0), + outputTokens: usage?.output_tokens ?? 0, + model: response.model ?? this.model, + }; + } +} + let activeProvider: LLMProvider | null = null; -/** Returns the active LLM provider (OpenAI-backed by default). */ +function createDefaultProvider(): LLMProvider { + if (process.env.LLM_PROVIDER === "anthropic" && process.env.ANTHROPIC_API_KEY) { + return new AnthropicProvider(); + } + return new OpenAIProvider(); +} + +/** + * Returns the active LLM provider. OpenAI-backed by default; Anthropic-backed + * only when LLM_PROVIDER=anthropic and ANTHROPIC_API_KEY are both set. + */ export function getLLMProvider(): LLMProvider { if (!activeProvider) { - activeProvider = new OpenAIProvider(); + activeProvider = createDefaultProvider(); } return activeProvider; } From b08826ee4e0e83bd4825af084e609b6e2c376178 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 13 Jun 2026 01:49:52 +0000 Subject: [PATCH 16/17] =?UTF-8?q?docs:=20Level=203=20blueprint=20=E2=80=94?= =?UTF-8?q?=20validation,=20governed=20promotion,=20routing,=20MCP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Build-ready spec for the next session. Six sections, each with API sketches and data-flow diagrams: 1. Validation framework: Zod-enforced evaluator contract with UNKNOWN / BLOCKED_BY_MISSING_EVIDENCE states; non-conforming model output is rejected, never coerced; results persist into the reserved StageTrace scoring columns. 2. Governed promotion: candidate -> evaluation -> ChangeRequest -> PromptVersion promotion -> rollback, reusing the existing models; machine improvements never mutate production prompts in place. 3. Computed routing hints: nightly StageTrace rollup by prompt x model into bestWithModels + a new PromptModelStats table. 4. MCP tools: recommend_composition (pgvector over the existing embedding column, org-scoped, ranked by routing hints — with the explicit post-seed embedding backfill prerequisite) and execute_composition. 5. Provider phase 2: citations mode on AnthropicProvider for evidence-backed stages. 6. Two-week sequencing with dependencies, plus explicit non-goals (no swarm, no auto-publishing, nothing bypasses ChangeRequest review). All Level 3 schema changes live in this spec; none land in Level 2 code. https://claude.ai/code/session_01GP2YigWVnAGR3xGQ59N4SK --- docs/level3-plan.md | 430 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 430 insertions(+) create mode 100644 docs/level3-plan.md diff --git a/docs/level3-plan.md b/docs/level3-plan.md new file mode 100644 index 00000000000..abc93ff588b --- /dev/null +++ b/docs/level3-plan.md @@ -0,0 +1,430 @@ +# Level 3 Blueprint — Evaluated, Governed, Self-Improving Compositions + +> **Status:** build-ready spec. Nothing in this document is implemented yet. +> **Scope rule:** every schema change Level 3 needs lands here, not in Level 2 code. +> **Verified against:** `prisma/schema.prisma` @ branch `claude/wizardly-dirac-mhzb9v` +> (PromptVersion, ChangeRequest, `Prompt.embedding`, `Prompt.bestWithModels`, +> `StageTrace.score` / `thresholdMet` / `overallScore` all already exist in the +> production schema — Level 3 populates them, it does not create them). + +Level 2 shipped a deterministic Composition Engine: DAG resolution, stage +execution through the `LLMProvider` abstraction, config-driven adaptation, +`thresholdCheck` gating, fallbacks, and a full audit trail +(`ExecutionRecord` + `StageTrace`). Level 3 turns that audited trace into a +feedback loop — **evaluation, governed improvement, evaluation-refreshed +routing, and MCP exposure** — without ever letting a machine mutate a +production prompt in place. + +--- + +## 1. Validation Framework + +### Problem + +`thresholdCheck` (src/lib/composition/thresholdCheck.ts) is a deterministic +length/structure heuristic. It answers "did the stage emit *something* +plausible", not "is the output *correct, grounded, and safe to feed forward*". +Level 3 replaces it with an LLM evaluator whose output is a **Zod-enforced +structured contract**. Non-conforming evaluator output is **rejected, never +coerced** — a malformed evaluation is itself a failed evaluation. + +### The contract + +```ts +// src/lib/composition/evaluation/contract.ts +import { z } from "zod"; + +export const EvalStateSchema = z.enum([ + "PASS", + "FAIL", + "UNKNOWN", // evaluator could not determine quality + "BLOCKED_BY_MISSING_EVIDENCE", // evaluation impossible without facts the stage never had +]); + +export const StageEvaluationSchema = z.object({ + state: EvalStateSchema, + scores: z.object({ + relevance: z.number().min(0).max(1), + completeness: z.number().min(0).max(1), + grounding: z.number().min(0).max(1), // claims traceable to stage input + formatFit: z.number().min(0).max(1), // matches what the next stage expects + }), + facts: z.array(z.string()), // claims in the output supported by the input + inferences: z.array(z.string()), // claims derived but not directly stated + assumptions: z.array(z.string()), // claims with no support in the input + risks: z.array(z.string()), // ways this output could mislead stage N+1 + blockers: z.array(z.string()), // required only when state != PASS; must be non-empty then +}); +export type StageEvaluation = z.infer; +``` + +Hard rules: + +- **Reject, never coerce.** `StageEvaluationSchema.safeParse` on the raw model + output. On failure: one re-ask with the Zod error appended; on second + failure the evaluation is recorded as `UNKNOWN` with + `blockers: ["evaluator_nonconforming_output"]`. We never `.catch()`-default + scores, never clamp, never strip unknown keys silently. +- **Explicit ignorance is a first-class state.** `UNKNOWN` and + `BLOCKED_BY_MISSING_EVIDENCE` flow through the engine as "not passed" but + are distinguishable in traces and in the routing aggregation (§3), which + excludes them from pass-rate denominators. +- The evaluator runs through the existing `LLMProvider` interface (so it can + run on the Anthropic provider with the stage rubric as a cached system + prefix), with `output` requested as JSON. + +### Data flow + +``` +stage N executes + └─> output ──────────────────────────────┐ + v + evaluator prompt = rubric (stable, cached) + + stage input + stage output + v + LLMProvider.complete() + v + StageEvaluationSchema.safeParse(raw) + │ ok │ fail + v v + StageEvaluation re-ask once with Zod issues + │ │ fail again + │ v + │ { state: UNKNOWN, + │ blockers: [evaluator_nonconforming_output] } + v + persist into StageTrace reserved columns: + score <- mean(scores) (Float?, exists today) + thresholdMet <- state == "PASS" (Boolean?, exists today) + + new column StageTrace.evaluation Json? (full contract object) + ExecutionRecord.overallScore <- mean of stage scores (Float?, exists today) + v + engine gate: PASS -> continue | else -> fallback path (unchanged semantics) +``` + +### Schema delta (migration lands in Level 3, not before) + +```prisma +model StageTrace { + // existing reserved columns get populated: score, thresholdMet + evaluation Json? // full StageEvaluation contract; null = legacy thresholdCheck run +} +``` + +### API sketch + +No new routes. The execute/test responses gain an optional per-stage +`evaluation` object (the parsed contract), and `GET /history` exposes it via +the existing StageTrace include. Engine config: `Stage.successThreshold` is +reinterpreted as the minimum mean score for `PASS` at evaluation time; +`EVALUATOR_ENABLED=true` env-gates the rollout (default off, falls back to +`thresholdCheck` — same default-off discipline as the Anthropic provider). + +--- + +## 2. Governed Promotion Pipeline + +### Problem + +Self-improvement must never mutate production prompts in place. The schema +already has the full governance surface: `PromptVersion` (append-only version +history, `@@unique([promptId, version])`) and `ChangeRequest` +(PENDING → APPROVED/REJECTED review flow with `originalContent` / +`proposedContent` / `reviewNote`). Level 3 reuses both, unchanged. + +### Flow + +``` +candidate generation (machine) review (human) production +───────────────────────────── ─────────────── ────────── +StageTrace rows where state in +(FAIL, UNKNOWN) for prompt P + │ (improvement job batches the failing + │ inputs/outputs/blockers as evidence) + v +LLMProvider: "propose improved prompt" + v +candidate evaluation: re-run the SAME +evaluator (§1) over a frozen sample of +recent inputs, candidate vs current + │ candidate mean score <= current ──> discard, log, stop + │ candidate wins + v +db.changeRequest.create({ + promptId: P, + authorId: SYSTEM_IMPROVER_USER_ID, // dedicated bot user, exists in users table + originalContent / originalTitle, // snapshot at proposal time + proposedContent, + reason: "", + status: PENDING, +}) + v + human reviews in the existing ChangeRequest UI + (features.changeRequests is already a config flag) + │ REJECTED ──> reviewNote recorded; improver backs off for that prompt + │ APPROVED + v + promotion (existing approval path): + db.promptVersion.create({ version: n+1, + content, changeNote: reason, createdBy }) + + prompt.content updated transactionally + v +rollback = promote version n again as n+2 (append-only; no destructive revert) +``` + +### Invariants + +- Machine-generated improvements arrive **only** as `ChangeRequest` rows. + There is no code path from the improver to `prompt.update()`. +- The improver bot is a normal `User` row scoped to the organization — RLS + and the Phase-B authz rules apply to it like anyone else. +- `StageTrace.promptVersion` already pins which version each execution used, + so before/after comparison across a promotion is a pure query. +- Rollback is a forward promotion of an old version — version history is + append-only. + +### API sketch + +``` +POST /api/v1/prompts/:id/improvement-runs (admin/system) trigger candidate generation +GET /api/v1/prompts/:id/improvement-runs list runs + candidate eval scores +-- review/approve/reject: existing ChangeRequest routes, unchanged +``` + +Schema delta: none required. Optional nicety (decide at build time): a +`source` enum column on ChangeRequest (`USER` | `SYSTEM_IMPROVER`) so the UI +can badge machine proposals; until then the bot author id serves that role. + +--- + +## 3. Computed Routing Hints + +### Problem + +`Prompt.bestWithModels` (`String[]`, the "works best with" metadata surfaced +in the prompt UI) is hand-maintained opinion. Level 3 converts it into +**evaluation-refreshed routing data** computed from StageTrace outcomes. + +### Aggregation job + +``` +nightly job (Vercel cron, 03:00 UTC) read path +────────────────────────────────── ───────── +SELECT promptId, model, + count(*) AS runs, + avg(CASE WHEN thresholdMet THEN 1 ELSE 0 END + FILTER (state not in UNKNOWN/BLOCKED) AS passRate, + avg(tokensUsed) AS avgTokens, + avg(durationMs) AS avgDurationMs +FROM stage_traces +WHERE createdAt > now() - interval '30 days' -- rolling window +GROUP BY promptId, model +HAVING count(*) >= 20 -- min sample size; below it, keep prior value + │ + v +rank models per prompt: passRate DESC, then avgTokens ASC (cost tiebreak) + │ + v +write top 3 slugs -> Prompt.bestWithModels (existing String[] column) +write full rows -> PromptModelStats (new table, see delta) + │ + v +read path: composition engine + MCP recommend tool (§4) read PromptModelStats + (full numbers); the prompt page keeps reading bestWithModels (slugs) +``` + +- **Refresh cadence:** nightly, plus an immediate re-aggregation for a prompt + when a promotion (§2) lands, so routing reflects the new version quickly. +- `UNKNOWN` / `BLOCKED_BY_MISSING_EVIDENCE` traces are excluded from the + pass-rate denominator (they measure the evaluator, not the prompt). +- Model names come from `StageTrace.model` (already recorded per stage by + both providers). + +### Schema delta + +```prisma +model PromptModelStats { + id String @id @default(cuid()) + promptId String + model String + windowDays Int @default(30) + runs Int + passRate Float + avgTokens Float + avgDurationMs Float + computedAt DateTime @default(now()) + + @@unique([promptId, model]) + @@index([promptId]) + @@map("prompt_model_stats") +} +``` + +### API sketch + +``` +GET /api/v1/prompts/:id/routing-hints + -> { promptId, computedAt, hints: [{ model, passRate, avgTokens, avgDurationMs, runs }] } +``` + +--- + +## 4. MCP Tools + +Two tools added to the existing MCP server (`src/pages/api/mcp.ts`, which +already exposes prompt search/save via `server.registerTool`). Both are +org-scoped through the same session/API-key auth the server already performs, +combined with the Phase-B rule: organization resolved server-side, never from +tool input. + +### `recommend_composition` + +```ts +server.registerTool("recommend_composition", { + description: "Recommend prompts/compositions for a task, ranked by measured performance", + inputSchema: { + task: z.string().min(1), // natural-language description + limit: z.number().int().max(10).default(3), + }, +}, handler); +``` + +Data flow: + +``` +task text + └─> embed via existing src/lib/ai/embeddings.ts (same model that fills Prompt.embedding) + v + pgvector similarity over prompts.embedding + WHERE organizationId = -- org scoping is in the SQL, not post-filter + ORDER BY embedding <=> $task_embedding + LIMIT 25 candidates + v + re-rank: similarity x routing hints (§3) + score = sim * w1 + passRate * w2 - normalizedCost * w3 + (prompts with no PromptModelStats rows rank on similarity alone, flagged "unmeasured") + v + return [{ promptId, title, model: bestModel, passRate, similarity, rationale }] +``` + +**Stated prerequisite — embedding backfill.** `Prompt.embedding` is currently +`Json?` and **unpopulated**: the prompts table is empty until content seeding +completes. The Level 3 plan therefore includes, in order: + +1. content seeding completes (prompts > 0); +2. **post-seed embedding backfill job** — batch job that walks all prompts + with `embedding IS NULL` and fills them via the existing embedding path + (`generatePromptEmbedding`), rate-limited, resumable by cursor; +3. migration of `embedding Json?` → a real `vector` column + HNSW index + (pgvector is available on Supabase; the Json column was a placeholder). + +`recommend_composition` ships behind a flag until (1)–(3) are done. + +### `execute_composition` + +```ts +server.registerTool("execute_composition", { + description: "Execute a saved composition chain and return the final output + trace summary", + inputSchema: { + compositionId: z.string().min(1), + input: z.string().min(1), + }, +}, handler); +``` + +Data flow: resolve caller → org (server-side) → verify the composition belongs +to that org (404 otherwise, identical to the Phase-B HTTP rule) → call +`executeComposition()` directly (same engine the HTTP route uses) → return +`{ executionId, status, finalOutput, totalTokens, totalDuration, stages: [...summaries] }`. +Execution context is `{ organizationId: , userId: , sessionId: "mcp" }` — +the MCP surface gets no identity inputs at all. + +--- + +## 5. Provider Phase 2 — Citations Mode + +Builds directly on the Phase-C `AnthropicProvider`. For **evidence-backed +stages** (stages whose output feeds the §1 evaluator's `facts[]`/`grounding` +checks), the provider accepts source documents and returns cited spans. + +### Interface extension (additive, optional fields only) + +```ts +export interface LLMCompletionRequest { + system?: string; + prompt: string; + temperature?: number; + maxTokens?: number; + // Level 3: evidence documents for citation-grounded stages + documents?: Array<{ title: string; content: string }>; +} + +export interface LLMCompletionResult { + output: string; + inputTokens: number; + outputTokens: number; + model: string; + // Level 3: populated only by providers/stages that support citations + citations?: Array<{ + text: string; // the cited span in the output + documentTitle: string; + start: number; // char offsets into the source document + end: number; + }>; +} +``` + +OpenAIProvider ignores `documents` (passes content inline) and returns no +`citations` — callers must treat the field as optional. AnthropicProvider maps +`documents` to the Messages API `document` content blocks with +`citations: { enabled: true }` and maps the returned citation deltas onto the +result shape above. The stage rubric stays a cached system prefix (Phase C +caching v1 carries over unchanged). + +### Where citations land + +``` +stage executes with documents + └─> result.citations + └─> evaluator (§1) receives them as ground truth: + facts[] entries that match a cited span score grounding=1, + uncited claims fall to inferences[]/assumptions[] + └─> StageTrace.evaluation Json captures the citation map +``` + +Schema delta: none (rides in the `evaluation Json` column from §1). + +--- + +## 6. Sequencing and Non-Goals + +### Two-week build order + +| Days | Work | Depends on | +|------|------|-----------| +| 1–2 | §1 evaluation contract + evaluator module behind `EVALUATOR_ENABLED`, unit tests with mocked provider (conforming, non-conforming, re-ask, UNKNOWN paths) | nothing | +| 3 | §1 migration (`StageTrace.evaluation Json?`) + engine wiring + trace persistence | days 1–2 | +| 4–5 | §2 improvement job: failure batching, candidate generation, candidate-vs-current eval, ChangeRequest creation; improver bot user + backoff | §1 (evaluator is the judge) | +| 6 | §2 promotion/rollback verification on the existing ChangeRequest approval path; before/after query on `StageTrace.promptVersion` | days 4–5 | +| 7–8 | §3 `PromptModelStats` migration + nightly aggregation job + `bestWithModels` writer + routing-hints route | §1 (needs real scores flowing) | +| 9 | Content seeding completes → **post-seed embedding backfill job** → pgvector column + index migration | external (seeding) | +| 10–11| §4 MCP tools: `execute_composition` first (no embedding dependency), then `recommend_composition` behind a flag | §3 (ranker), day 9 (embeddings) | +| 12–13| §5 citations mode on AnthropicProvider + evaluator grounding integration, mocked-SDK tests | §1, Phase C | +| 14 | End-to-end smoke (extend `scripts/smoke-composition.ts` with an evaluated run), docs, flag-flip checklist | all | + +Dependency spine: **§1 → §2 → §3 → §4**, with §5 parallelizable after §1. +If seeding slips, day 9–11's `recommend_composition` slips with it; everything +else is unaffected. + +### Explicit non-goals + +- **No continuous-learning swarm.** One improvement job, batch cadence, + per-prompt backoff. No agent fleet mutating prompts concurrently. +- **No auto-publishing loops.** Nothing promotes without a human approving + the ChangeRequest. The improver cannot approve its own proposals, and there + is no "auto-approve above score X" rule in Level 3. +- **Nothing bypasses ChangeRequest review.** No direct `prompt.update()` from + any machine path, no shadow prompts, no per-execution prompt rewriting. +- No cross-organization learning (traces never inform another org's prompts). +- No evaluator-of-the-evaluator recursion; evaluator quality is monitored by + humans via the `UNKNOWN`-rate metric, not by another model. From a655a27ca9dd67f78e361ac768a941a207fe4e14 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 13 Jun 2026 02:16:45 +0000 Subject: [PATCH 17/17] fix(build): let prisma generate run without DATABASE_URL The Vercel Preview build failed in postinstall: prisma/config's env() helper throws PrismaConfigEnvError at config-load time when DATABASE_URL is unset, so 'prisma generate' (pure codegen, no DB connection) never ran. Read the var directly with a non-functional placeholder fallback so codegen works without the secret; migrate/db push/seed and all runtime code run only where DATABASE_URL is set, so the placeholder is never dialed. Verified: 'env -u DATABASE_URL prisma generate' now exits 0. https://claude.ai/code/session_01GP2YigWVnAGR3xGQ59N4SK --- prisma.config.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/prisma.config.ts b/prisma.config.ts index 8ded1a58d52..a54bb8aa762 100644 --- a/prisma.config.ts +++ b/prisma.config.ts @@ -2,7 +2,19 @@ // This file was generated by Prisma and assumes you have installed the following: // npm install --save-dev prisma dotenv import "dotenv/config"; -import { defineConfig, env } from "prisma/config"; +import { defineConfig } from "prisma/config"; + +// `prisma generate` is pure client codegen and never opens a connection, but +// prisma/config's env() helper throws at config-load time when DATABASE_URL is +// unset — which breaks codegen in environments that legitimately don't carry +// the secret (e.g. Vercel Preview builds, where postinstall runs generate). +// Read the variable directly and fall back to a non-functional placeholder so +// generate works without a database. Every command that actually connects +// (migrate, db push, seed) and all runtime code run only where DATABASE_URL is +// set, so the placeholder is never dialed. +const databaseUrl = + process.env.DATABASE_URL ?? + "postgresql://placeholder:placeholder@localhost:5432/placeholder"; export default defineConfig({ schema: "prisma/schema.prisma", @@ -11,6 +23,6 @@ export default defineConfig({ }, engine: "classic", datasource: { - url: env("DATABASE_URL"), + url: databaseUrl, }, });