diff --git a/backend/.env.example b/backend/.env.example index 6b4d56150..63db31750 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -1,12 +1,12 @@ PORT=3001 FRONTEND_URL=http://localhost:3000 - -# HMAC key used to sign /download/:token URLs. Required at startup. -# Generate with: openssl rand -hex 32 -# Use a dedicated secret distinct from SUPABASE_SECRET_KEY. -DOWNLOAD_SIGNING_SECRET=replace-with-a-random-32-byte-hex-string SUPABASE_URL=https://your-project.supabase.co SUPABASE_SECRET_KEY=your-supabase-service-role-key +# Required for cross-tenant test suite (npm run test:cross-tenant) to sign test +# users into anon-key sessions and obtain real JWTs. Optional for runtime. +SUPABASE_ANON_KEY= + +DOWNLOAD_SIGNING_SECRET=your-random-signing-secret-min-32-chars R2_ENDPOINT_URL=https://your-account-id.r2.cloudflarestorage.com R2_ACCESS_KEY_ID=your-r2-access-key @@ -15,6 +15,25 @@ R2_BUCKET_NAME=mike GEMINI_API_KEY=your-gemini-key ANTHROPIC_API_KEY=your-anthropic-key -OPENAI_API_KEY=your-openai-key + +# Optional — when set, enables raw LLM stream console logging (debug only; remove in production) +LLM_STREAM_DEBUG= + +OPENROUTER_API_KEY=your-openrouter-key RESEND_API_KEY=your-resend-key -USER_API_KEYS_ENCRYPTION_SECRET=your-long-random-secret + +# Migration runner — Supabase direct connection, NOT the pgBouncer pooler. +# Format: postgresql://postgres:@db..supabase.co:5432/postgres +DATABASE_URL= + +# LLM rate limiting (per-user, applies to all LLM-spending routes) +RATE_LIMIT_WINDOW_MS=60000 # Sliding window in milliseconds (default: 60000 = 1 minute) +RATE_LIMIT_MAX=20 # Max LLM requests per user per window (default: 20) + +# CLEAN-05 — at-rest encryption of user LLM API keys (AES-256-GCM) +# Generate with: openssl rand -hex 32 +HUGO_MASTER_KEY= + +# CLEAN-44 — HMAC secret for account-restore tokens (30-day soft-delete window) +# Generate with: openssl rand -base64 48 +HUGO_RESTORE_TOKEN_SECRET= diff --git a/backend/.gitignore b/backend/.gitignore index 6b319f760..bcbd2ce0b 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -3,5 +3,4 @@ dist .env* !.env.example *.log -logs/ .DS_Store diff --git a/backend/bun.lock b/backend/bun.lock index 90061e1af..eb64b683d 100644 --- a/backend/bun.lock +++ b/backend/bun.lock @@ -14,10 +14,8 @@ "docx": "^9.5.0", "dotenv": "^17.4.1", "express": "^4.21.2", - "express-rate-limit": "^8.5.1", "fast-diff": "^1.3.0", "fast-xml-parser": "^5.7.1", - "helmet": "^8.1.0", "jszip": "^3.10.1", "libreoffice-convert": "^1.6.0", "mammoth": "^1.9.0", @@ -473,8 +471,6 @@ "express": ["express@4.22.1", "", { "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", "body-parser": "~1.20.3", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", "cookie-signature": "~1.0.6", "debug": "2.6.9", "depd": "2.0.0", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", "finalhandler": "~1.3.1", "fresh": "~0.5.2", "http-errors": "~2.0.0", "merge-descriptors": "1.0.3", "methods": "~1.1.2", "on-finished": "~2.4.1", "parseurl": "~1.3.3", "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", "qs": "~6.14.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", "serve-static": "~1.16.2", "setprototypeof": "1.2.0", "statuses": "~2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" } }, "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g=="], - "express-rate-limit": ["express-rate-limit@8.5.1", "", { "dependencies": { "ip-address": "^10.2.0" }, "peerDependencies": { "express": ">= 4.11" } }, "sha512-5O6KYmyJEpuPJV5hNTXKbAHWRqrzyu+OI3vUnSd2kXFubIVpG7ezpgxQy76Zo5GQZtrQBg86hF+CM/NX+cioiQ=="], - "extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="], "fast-deep-equal": ["fast-deep-equal@2.0.1", "", {}, "sha512-bCK/2Z4zLidyB4ReuIsvALH6w31YfAQDmXMqMx6FyfHqvBxtjC0eRumeSu4Bs3XtXwpyIywtSTrVT99BxY1f9w=="], @@ -521,8 +517,6 @@ "hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="], - "helmet": ["helmet@8.1.0", "", {}, "sha512-jOiHyAZsmnr8LqoPGmCjYAaiuWwjAPLgY8ZX2XrmHawt99/u1y6RgrZMTeoPfpUbV96HOalYgz1qzkRbw54Pmg=="], - "html-to-text": ["html-to-text@9.0.5", "", { "dependencies": { "@selderee/plugin-htmlparser2": "^0.11.0", "deepmerge": "^4.3.1", "dom-serializer": "^2.0.0", "htmlparser2": "^8.0.2", "selderee": "^0.11.0" } }, "sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg=="], "htmlparser2": ["htmlparser2@8.0.2", "", { "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", "domutils": "^3.0.1", "entities": "^4.4.0" } }, "sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA=="], @@ -539,8 +533,6 @@ "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], - "ip-address": ["ip-address@10.2.0", "", {}, "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA=="], - "ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], "isarray": ["isarray@1.0.0", "", {}, "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="], diff --git a/backend/nixpacks.toml b/backend/nixpacks.toml index 9f20b0d2b..4d89cbf9a 100644 --- a/backend/nixpacks.toml +++ b/backend/nixpacks.toml @@ -1,2 +1,2 @@ [phases.setup] -nixPkgs = ["...", "libreoffice"] +nixPkgs = ["libreoffice"] diff --git a/backend/package-lock.json b/backend/package-lock.json index effa2adef..0025b4a12 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -7,6 +7,7 @@ "": { "name": "mike-backend", "version": "1.0.0", + "license": "AGPL-3.0-only", "dependencies": { "@anthropic-ai/sdk": "^0.90.0", "@aws-sdk/client-s3": "^3.787.0", @@ -20,22 +21,34 @@ "express-rate-limit": "^8.5.1", "fast-diff": "^1.3.0", "fast-xml-parser": "^5.7.1", - "helmet": "^8.1.0", "jszip": "^3.10.1", "libreoffice-convert": "^1.6.0", + "lru-cache": "^11.3.5", "mammoth": "^1.9.0", - "multer": "^1.4.5-lts.2", + "multer": "^2.1.1", + "node-pg-migrate": "^8.0.4", + "p-limit": "^7.3.0", + "p-queue": "^9.2.0", "pdfjs-dist": "^4.10.38", - "resend": "^4.5.1" + "pg": "^8.20.0", + "pino": "^10.3.1", + "pino-http": "^11.0.0", + "resend": "^4.5.1", + "zod": "^4.4.2" }, "devDependencies": { "@types/cors": "^2.8.17", "@types/express": "^4.17.21", - "@types/multer": "^1.4.12", + "@types/multer": "^2.1.0", "@types/node": "^22.14.1", + "@types/pg": "^8.20.0", + "@types/supertest": "^7.2.0", + "pino-pretty": "^13.1.3", "prettier": "^3.8.1", + "supertest": "^7.2.2", "tsx": "^4.19.3", - "typescript": "^5.8.3" + "typescript": "^5.8.3", + "vitest": "^4.1.5" } }, "node_modules/@anthropic-ai/sdk": { @@ -972,6 +985,40 @@ "node": ">=6.9.0" } }, + "node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.27.7", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", @@ -1437,6 +1484,22 @@ } } }, + "node_modules/@isaacs/cliui": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-9.0.0.tgz", + "integrity": "sha512-AokJm4tuBHillT+FpMtxQ60n8ObyXBatq7jD2/JA9dxbDDokKQm8KMht5ibGzLVU9IJDIKK4TPKgMHEYMn3lMg==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, "node_modules/@napi-rs/canvas": { "version": "0.1.97", "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-0.1.97.tgz", @@ -1687,6 +1750,38 @@ "url": "https://github.com/sponsors/Brooooooklyn" } }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", + "integrity": "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@noble/hashes": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", + "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@nodable/entities": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-2.1.0.tgz", @@ -1699,6 +1794,32 @@ ], "license": "MIT" }, + "node_modules/@oxc-project/types": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.127.0.tgz", + "integrity": "sha512-aIYXQBo4lCbO4z0R3FHeucQHpF46l2LbMdxRvqvuRuW2OxdnSkcng5B8+K12spgLDj93rtN3+J2Vac/TIO+ciQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@paralleldrive/cuid2": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/@paralleldrive/cuid2/-/cuid2-2.3.1.tgz", + "integrity": "sha512-XO7cAxhnTZl0Yggq6jOgjiOHhbgcO4NqFqwSmQpjK3b6TEE6Uj/jfSk6wzYyemh3+I0sHirKSetjQwn5cZktFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@noble/hashes": "^1.1.5" + } + }, + "node_modules/@pinojs/redact": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", + "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", + "license": "MIT" + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -1781,6 +1902,270 @@ "react-dom": "^18.0 || ^19.0 || ^19.0.0-rc" } }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.17.tgz", + "integrity": "sha512-s70pVGhw4zqGeFnXWvAzJDlvxhlRollagdCCKRgOsgUOH3N1l0LIxf83AtGzmb5SiVM4Hjl5HyarMRfdfj3DaQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.17.tgz", + "integrity": "sha512-4ksWc9n0mhlZpZ9PMZgTGjeOPRu8MB1Z3Tz0Mo02eWfWCHMW1zN82Qz/pL/rC+yQa+8ZnutMF0JjJe7PjwasYw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.17.tgz", + "integrity": "sha512-SUSDOI6WwUVNcWxd02QEBjLdY1VPHvlEkw6T/8nYG322iYWCTxRb1vzk4E+mWWYehTp7ERibq54LSJGjmouOsw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.17.tgz", + "integrity": "sha512-hwnz3nw9dbJ05EDO/PvcjaaewqqDy7Y1rn1UO81l8iIK1GjenME75dl16ajbvSSMfv66WXSRCYKIqfgq2KCfxw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.17.tgz", + "integrity": "sha512-IS+W7epTcwANmFSQFrS1SivEXHtl1JtuQA9wlxrZTcNi6mx+FDOYrakGevvvTwgj2JvWiK8B29/qD9BELZPyXQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.17.tgz", + "integrity": "sha512-e6usGaHKW5BMNZOymS1UcEYGowQMWcgZ71Z17Sl/h2+ZziNJ1a9n3Zvcz6LdRyIW5572wBCTH/Z+bKuZouGk9Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.17.tgz", + "integrity": "sha512-b/CgbwAJpmrRLp02RPfhbudf5tZnN9nsPWK82znefso832etkem8H7FSZwxrOI9djcdTP7U6YfNhbRnh7djErg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.17.tgz", + "integrity": "sha512-4EII1iNGRUN5WwGbF/kOh/EIkoDN9HsupgLQoXfY+D1oyJm7/F4t5PYU5n8SWZgG0FEwakyM8pGgwcBYruGTlA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.17.tgz", + "integrity": "sha512-AH8oq3XqQo4IibpVXvPeLDI5pzkpYn0WiZAfT05kFzoJ6tQNzwRdDYQ45M8I/gslbodRZwW8uxLhbSBbkv96rA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.17.tgz", + "integrity": "sha512-cLnjV3xfo7KslbU41Z7z8BH/E1y5mzUYzAqih1d1MDaIGZRCMqTijqLv76/P7fyHuvUcfGsIpqCdddbxLLK9rA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.17.tgz", + "integrity": "sha512-0phclDw1spsL7dUB37sIARuis2tAgomCJXAHZlpt8PXZ4Ba0dRP1e+66lsRqrfhISeN9bEGNjQs+T/Fbd7oYGw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.17.tgz", + "integrity": "sha512-0ag/hEgXOwgw4t8QyQvUCxvEg+V0KBcA6YuOx9g0r02MprutRF5dyljgm3EmR02O292UX7UeS6HzWHAl6KgyhA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.17.tgz", + "integrity": "sha512-LEXei6vo0E5wTGwpkJ4KoT3OZJRnglwldt5ziLzOlc6qqb55z4tWNq2A+PFqCJuvWWdP53CVhG1Z9NtToDPJrA==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.10.0", + "@emnapi/runtime": "1.10.0", + "@napi-rs/wasm-runtime": "^1.1.4" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.17.tgz", + "integrity": "sha512-gUmyzBl3SPMa6hrqFUth9sVfcLBlYsbMzBx5PlexMroZStgzGqlZ26pYG89rBb45Mnia+oil6YAIFeEWGWhoZA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.17.tgz", + "integrity": "sha512-3hkiolcUAvPB9FLb3UZdfjVVNWherN1f/skkGWJP/fgSQhYUZpSIRr0/I8ZK9TkF3F7kxvJAk0+IcKvPHk9qQg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.17.tgz", + "integrity": "sha512-n8iosDOt6Ig1UhJ2AYqoIhHWh/isz0xpicHTzpKBeotdVsTEcxsSA/i3EVM7gQAj0rU27OLAxCjzlj15IWY7bg==", + "dev": true, + "license": "MIT" + }, "node_modules/@selderee/plugin-htmlparser2": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.11.0.tgz", @@ -2513,6 +2898,13 @@ "node": ">=18.0.0" } }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, "node_modules/@supabase/auth-js": { "version": "2.102.1", "resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.102.1.tgz", @@ -2599,6 +2991,17 @@ "node": ">=20.0.0" } }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", + "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@types/body-parser": { "version": "1.19.6", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz", @@ -2610,6 +3013,17 @@ "@types/node": "*" } }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, "node_modules/@types/connect": { "version": "3.4.38", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", @@ -2620,7 +3034,14 @@ "@types/node": "*" } }, - "node_modules/@types/cors": { + "node_modules/@types/cookiejar": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.5.tgz", + "integrity": "sha512-he+DHOWReW0nghN24E1WUqM0efK4kI9oTqDm6XmK8ZPe2djZ90BSNdGnIyCLzCPw7/pogPlGbzI2wHGGmi4O/Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/cors": { "version": "2.8.19", "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.19.tgz", "integrity": "sha512-mFNylyeyqN93lfe/9CSxOGREz8cpzAhH+E93xJ4xWQf62V8sQ/24reV2nyzUWM6H6Xji+GGHpkbLe7pVoUEskg==", @@ -2630,6 +3051,20 @@ "@types/node": "*" } }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/express": { "version": "4.17.25", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz", @@ -2663,6 +3098,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/methods": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@types/methods/-/methods-1.1.4.tgz", + "integrity": "sha512-ymXWVrDiCxTBE3+RIrrP533E70eA+9qu7zdWoHuOmGujkYtzf4HQF96b8nwHLqhuf4ykX61IGRIB38CC6/sImQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/mime": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", @@ -2671,9 +3113,9 @@ "license": "MIT" }, "node_modules/@types/multer": { - "version": "1.4.13", - "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.13.tgz", - "integrity": "sha512-bhhdtPw7JqCiEfC9Jimx5LqX9BDIPJEh2q/fQ4bqbBPtyEZYr3cvF22NwG0DmPZNYA0CAf2CnqDB4KIGGpJcaw==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/multer/-/multer-2.1.0.tgz", + "integrity": "sha512-zYZb0+nJhOHtPpGDb3vqPjwpdeGlGC157VpkqNQL+UU2qwoacoQ7MpsAmUptI/0Oa127X32JzWDqQVEXp2RcIA==", "dev": true, "license": "MIT", "dependencies": { @@ -2689,6 +3131,18 @@ "undici-types": "~6.21.0" } }, + "node_modules/@types/pg": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.20.0.tgz", + "integrity": "sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, "node_modules/@types/qs": { "version": "6.15.0", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.15.0.tgz", @@ -2742,6 +3196,30 @@ "@types/node": "*" } }, + "node_modules/@types/superagent": { + "version": "8.1.9", + "resolved": "https://registry.npmjs.org/@types/superagent/-/superagent-8.1.9.tgz", + "integrity": "sha512-pTVjI73witn+9ILmoJdajHGW2jkSaOzhiFYF1Rd3EQ94kymLqB9PjD9ISg7WaALC7+dCHT0FGe9T2LktLq/3GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/cookiejar": "^2.1.5", + "@types/methods": "^1.1.4", + "@types/node": "*", + "form-data": "^4.0.0" + } + }, + "node_modules/@types/supertest": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@types/supertest/-/supertest-7.2.0.tgz", + "integrity": "sha512-uh2Lv57xvggst6lCqNdFAmDSvoMG7M/HDtX4iUCquxQ5EGPtaPM5PL5Hmi7LCvOG8db7YaCPNJEeoI8s/WzIQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/methods": "^1.1.4", + "@types/superagent": "^8.1.0" + } + }, "node_modules/@types/ws": { "version": "8.18.1", "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", @@ -2751,6 +3229,119 @@ "@types/node": "*" } }, + "node_modules/@vitest/expect": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.5.tgz", + "integrity": "sha512-PWBaRY5JoKuRnHlUHfpV/KohFylaDZTupcXN1H9vYryNLOnitSw60Mw9IAE2r67NbwwzBw/Cc/8q9BK3kIX8Kw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.5", + "@vitest/utils": "4.1.5", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.5.tgz", + "integrity": "sha512-/x2EmFC4mT4NNzqvC3fmesuV97w5FC903KPmey4gsnJiMQ3Be1IlDKVaDaG8iqaLFHqJ2FVEkxZk5VmeLjIItw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.5", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.5.tgz", + "integrity": "sha512-7I3q6l5qr03dVfMX2wCo9FxwSJbPdwKjy2uu/YPpU3wfHvIL4QHwVRp57OfGrDFeUJ8/8QdfBKIV12FTtLn00g==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.5.tgz", + "integrity": "sha512-2D+o7Pr82IEO46YPpoA/YU0neeyr6FTerQb5Ro7BUnBuv6NQtT/kmVnczngiMEBhzgqz2UZYl5gArejsyERDSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.1.5", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.5.tgz", + "integrity": "sha512-zypXEt4KH/XgKGPUz4eC2AvErYx0My5hfL8oDb1HzGFpEk1P62bxSohdyOmvz+d9UJwanI68MKwr2EquOaOgMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.5", + "@vitest/utils": "4.1.5", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.5.tgz", + "integrity": "sha512-2lNOsh6+R2Idnf1TCZqSwYlKN2E/iDlD8sgU59kYVl+OMDmvldO1VDk39smRfpUNwYpNRVn3w4YfuC7KfbBnkQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.5.tgz", + "integrity": "sha512-76wdkrmfXfqGjueGgnb45ITPyUi1ycZ4IHgC2bhPDUfWHklY/q3MdLOAB+TF1e6xfl8NxNY0ZYaPCFNWSsw3Ug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.5", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, "node_modules/@xmldom/xmldom": { "version": "0.8.12", "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.12.tgz", @@ -2782,6 +3373,30 @@ "node": ">= 14" } }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, "node_modules/append-field": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", @@ -2803,12 +3418,54 @@ "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", "license": "MIT" }, + "node_modules/asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", + "dev": true, + "license": "MIT" + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, "node_modules/async": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", "license": "MIT" }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -2874,6 +3531,18 @@ "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", "license": "MIT" }, + "node_modules/brace-expansion": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -2935,21 +3604,107 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", + "dev": true, + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/component-emitter": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.1.tgz", + "integrity": "sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/concat-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", "engines": [ - "node >= 0.8" + "node >= 6.0" ], "license": "MIT", "dependencies": { "buffer-from": "^1.0.0", "inherits": "^2.0.3", - "readable-stream": "^2.2.2", + "readable-stream": "^3.0.2", "typedarray": "^0.0.6" } }, + "node_modules/concat-stream/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -2971,6 +3726,13 @@ "node": ">= 0.6" } }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, "node_modules/cookie": { "version": "0.7.2", "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", @@ -2986,6 +3748,13 @@ "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", "license": "MIT" }, + "node_modules/cookiejar": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/cookiejar/-/cookiejar-2.1.4.tgz", + "integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==", + "dev": true, + "license": "MIT" + }, "node_modules/core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", @@ -3009,6 +3778,20 @@ "url": "https://opencollective.com/express" } }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", @@ -3018,6 +3801,16 @@ "node": ">= 12" } }, + "node_modules/dateformat": { + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", + "integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -3036,6 +3829,16 @@ "node": ">=0.10.0" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -3055,6 +3858,27 @@ "npm": "1.2.8000 || >= 1.4.16" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/dezalgo": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/dezalgo/-/dezalgo-1.0.4.tgz", + "integrity": "sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==", + "dev": true, + "license": "ISC", + "dependencies": { + "asap": "^2.0.0", + "wrappy": "1" + } + }, "node_modules/dingbat-to-unicode": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz", @@ -3198,6 +4022,12 @@ "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", "license": "MIT" }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, "node_modules/encodeurl": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", @@ -3207,6 +4037,16 @@ "node": ">= 0.8" } }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, "node_modules/entities": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", @@ -3237,6 +4077,13 @@ "node": ">= 0.4" } }, + "node_modules/es-module-lexer": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.1.0.tgz", + "integrity": "sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==", + "dev": true, + "license": "MIT" + }, "node_modules/es-object-atoms": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", @@ -3249,6 +4096,22 @@ "node": ">= 0.4" } }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/esbuild": { "version": "0.27.7", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", @@ -3291,12 +4154,31 @@ "@esbuild/win32-x64": "0.27.7" } }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/escape-html": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", "license": "MIT" }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, "node_modules/etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -3306,14 +4188,30 @@ "node": ">= 0.6" } }, - "node_modules/express": { - "version": "4.22.1", - "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", - "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", - "license": "MIT", - "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", + "node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, + "node_modules/expect-type": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", + "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/express": { + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", "body-parser": "~1.20.3", "content-disposition": "~0.5.4", "content-type": "~1.0.4", @@ -3376,6 +4274,13 @@ "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", "license": "MIT" }, + "node_modules/fast-copy": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-4.0.3.tgz", + "integrity": "sha512-58apWr0GUiDFM8+3afrO6eYwJBn9ZAhDOzG3L+/9llab/haCARS2UIfffmOurYLwbgDRs8n0rfr6qAAPEAuAQw==", + "dev": true, + "license": "MIT" + }, "node_modules/fast-deep-equal": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", @@ -3388,6 +4293,13 @@ "integrity": "sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==", "license": "Apache-2.0" }, + "node_modules/fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==", + "dev": true, + "license": "MIT" + }, "node_modules/fast-xml-builder": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.1.5.tgz", @@ -3424,6 +4336,24 @@ "fxparser": "src/cli/cli.js" } }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, "node_modules/fetch-blob": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", @@ -3465,6 +4395,39 @@ "node": ">= 0.8" } }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/form-data": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "dev": true, + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -3477,6 +4440,24 @@ "node": ">=12.20.0" } }, + "node_modules/formidable": { + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.4.tgz", + "integrity": "sha512-YikH+7CUTOtP44ZTnUhR7Ic2UASBPOqmaRkRKxRbywPTe5VxF7RRCck4af9wutiZ/QKM5nME9Bie2fFaPz5Gug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@paralleldrive/cuid2": "^2.2.2", + "dezalgo": "^1.0.4", + "once": "^1.4.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "url": "https://ko-fi.com/tunnckoCore/commissions" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -3547,6 +4528,15 @@ "node": ">=18" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-intrinsic": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", @@ -3597,6 +4587,30 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/glob": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-11.1.0.tgz", + "integrity": "sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "BlueOak-1.0.0", + "dependencies": { + "foreground-child": "^3.3.1", + "jackspeak": "^4.1.1", + "minimatch": "^10.1.1", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^2.0.0" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/google-auth-library": { "version": "10.6.2", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.6.2.tgz", @@ -3647,6 +4661,22 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/hash.js": { "version": "1.1.7", "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", @@ -3669,14 +4699,12 @@ "node": ">= 0.4" } }, - "node_modules/helmet": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/helmet/-/helmet-8.1.0.tgz", - "integrity": "sha512-jOiHyAZsmnr8LqoPGmCjYAaiuWwjAPLgY8ZX2XrmHawt99/u1y6RgrZMTeoPfpUbV96HOalYgz1qzkRbw54Pmg==", - "license": "MIT", - "engines": { - "node": ">=18.0.0" - } + "node_modules/help-me": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz", + "integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==", + "dev": true, + "license": "MIT" }, "node_modules/html-to-text": { "version": "9.0.5", @@ -3820,12 +4848,52 @@ "node": ">= 0.10" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", "license": "MIT" }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jackspeak": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.2.3.tgz", + "integrity": "sha512-ykkVRwrYvFm1nb2AJfKKYPr0emF6IiXDYUaFx4Zn9ZuIH7MrzEZ3sD5RlqGXNRpHtvUHJyOnCEFxOlNDtGo7wg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^9.0.0" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/joycon": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", + "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, "node_modules/json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -3912,106 +4980,386 @@ "immediate": "~3.0.5" } }, - "node_modules/long": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", - "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", - "license": "Apache-2.0" - }, - "node_modules/lop": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/lop/-/lop-0.4.2.tgz", - "integrity": "sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw==", - "license": "BSD-2-Clause", - "dependencies": { - "duck": "^0.1.12", - "option": "~0.2.1", - "underscore": "^1.13.1" - } - }, - "node_modules/mammoth": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/mammoth/-/mammoth-1.12.0.tgz", - "integrity": "sha512-cwnK1RIcRdDMi2HRx2EXGYlxqIEh0Oo3bLhorgnsVJi2UkbX1+jKxuBNR9PC5+JaX7EkmJxFPmo6mjLpqShI2w==", - "license": "BSD-2-Clause", + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "dev": true, + "license": "MPL-2.0", "dependencies": { - "@xmldom/xmldom": "^0.8.6", - "argparse": "~1.0.3", - "base64-js": "^1.5.1", - "bluebird": "~3.4.0", - "dingbat-to-unicode": "^1.0.1", - "jszip": "^3.7.1", - "lop": "^0.4.2", - "path-is-absolute": "^1.0.0", - "underscore": "^1.13.1", - "xmlbuilder": "^10.0.0" + "detect-libc": "^2.0.3" }, - "bin": { - "mammoth": "bin/mammoth" + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], "engines": { - "node": ">=12.0.0" + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" } }, - "node_modules/math-intrinsics": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", - "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "license": "MIT", + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], "engines": { - "node": ">= 0.4" + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" } }, - "node_modules/media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", - "license": "MIT", + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], "engines": { - "node": ">= 0.6" - } - }, - "node_modules/merge-descriptors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", - "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", - "license": "MIT", + "node": ">= 12.0.0" + }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "type": "opencollective", + "url": "https://opencollective.com/parcel" } }, - "node_modules/methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", - "license": "MIT", + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", - "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", - "license": "MIT", - "bin": { - "mime": "cli.js" + "node": ">= 12.0.0" }, - "engines": { - "node": ">=4" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" } }, - "node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], "engines": { - "node": ">= 0.6" + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" } }, - "node_modules/mime-types": { - "version": "2.1.35", + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/lop": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/lop/-/lop-0.4.2.tgz", + "integrity": "sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw==", + "license": "BSD-2-Clause", + "dependencies": { + "duck": "^0.1.12", + "option": "~0.2.1", + "underscore": "^1.13.1" + } + }, + "node_modules/lru-cache": { + "version": "11.3.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.3.5.tgz", + "integrity": "sha512-NxVFwLAnrd9i7KUBxC4DrUhmgjzOs+1Qm50D3oF1/oL+r1NpZ4gA7xvG0/zJ8evR7zIKn4vLf7qTNduWFtCrRw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/mammoth": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/mammoth/-/mammoth-1.12.0.tgz", + "integrity": "sha512-cwnK1RIcRdDMi2HRx2EXGYlxqIEh0Oo3bLhorgnsVJi2UkbX1+jKxuBNR9PC5+JaX7EkmJxFPmo6mjLpqShI2w==", + "license": "BSD-2-Clause", + "dependencies": { + "@xmldom/xmldom": "^0.8.6", + "argparse": "~1.0.3", + "base64-js": "^1.5.1", + "bluebird": "~3.4.0", + "dingbat-to-unicode": "^1.0.1", + "jszip": "^3.7.1", + "lop": "^0.4.2", + "path-is-absolute": "^1.0.0", + "underscore": "^1.13.1", + "xmlbuilder": "^10.0.0" + }, + "bin": { + "mammoth": "bin/mammoth" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "license": "MIT", @@ -4028,25 +5376,38 @@ "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", "license": "ISC" }, + "node_modules/minimatch": { + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.5" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/mkdirp": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", - "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", - "license": "MIT", - "dependencies": { - "minimist": "^1.2.6" - }, - "bin": { - "mkdirp": "bin/cmd.js" + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" } }, "node_modules/ms": { @@ -4056,22 +5417,22 @@ "license": "MIT" }, "node_modules/multer": { - "version": "1.4.5-lts.2", - "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.5-lts.2.tgz", - "integrity": "sha512-VzGiVigcG9zUAoCNU+xShztrlr1auZOlurXynNvO9GiWD1/mTBbUljOKY+qMeazBqXgRnjzeEgJI/wyjJUHg9A==", - "deprecated": "Multer 1.x is impacted by a number of vulnerabilities, which have been patched in 2.x. You should upgrade to the latest 2.x version.", + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/multer/-/multer-2.1.1.tgz", + "integrity": "sha512-mo+QTzKlx8R7E5ylSXxWzGoXoZbOsRMpyitcht8By2KHvMbf3tjwosZ/Mu/XYU6UuJ3VZnODIrak5ZrPiPyB6A==", "license": "MIT", "dependencies": { "append-field": "^1.0.0", - "busboy": "^1.0.0", - "concat-stream": "^1.5.2", - "mkdirp": "^0.5.4", - "object-assign": "^4.1.1", - "type-is": "^1.6.4", - "xtend": "^4.0.0" + "busboy": "^1.6.0", + "concat-stream": "^2.0.0", + "type-is": "^1.6.18" }, "engines": { - "node": ">= 6.0.0" + "node": ">= 10.16.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/nanoid": { @@ -4139,6 +5500,31 @@ "url": "https://opencollective.com/node-fetch" } }, + "node_modules/node-pg-migrate": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/node-pg-migrate/-/node-pg-migrate-8.0.4.tgz", + "integrity": "sha512-HTlJ6fOT/2xHhAUtsqSN85PGMAqSbfGJNRwQF8+ZwQ1+sVGNUTl/ZGEshPsOI3yV22tPIyHXrKXr3S0JxeYLrg==", + "license": "MIT", + "dependencies": { + "glob": "~11.1.0", + "yargs": "~17.7.0" + }, + "bin": { + "node-pg-migrate": "bin/node-pg-migrate.js" + }, + "engines": { + "node": ">=20.11.0" + }, + "peerDependencies": { + "@types/pg": ">=6.0.0 <9.0.0", + "pg": ">=4.3.0 <9.0.0" + }, + "peerDependenciesMeta": { + "@types/pg": { + "optional": true + } + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -4160,6 +5546,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/obug": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", + "integrity": "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT" + }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -4172,12 +5578,53 @@ "node": ">= 0.8" } }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/option": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/option/-/option-0.2.4.tgz", "integrity": "sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A==", "license": "BSD-2-Clause" }, + "node_modules/p-limit": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-7.3.0.tgz", + "integrity": "sha512-7cIXg/Z0M5WZRblrsOla88S4wAK+zOQQWeBYfV3qJuJXMr+LnbYjaadrFaS0JILfEDPVqHyKnZ1Z/1d6J9VVUw==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^1.2.1" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-queue": { + "version": "9.2.0", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.2.0.tgz", + "integrity": "sha512-dWgLE8AH0HjQ9fe74pUkKkvzzYT18Inp4zra3lKHnnwqGvcfcUBrvF2EAVX+envufDNBOzpPq/IBUONDbI7+3g==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^5.0.4", + "p-timeout": "^7.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-retry": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", @@ -4191,6 +5638,24 @@ "node": ">=8" } }, + "node_modules/p-timeout": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.1.tgz", + "integrity": "sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "license": "BlueOak-1.0.0" + }, "node_modules/pako": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", @@ -4243,12 +5708,44 @@ "node": ">=0.10.0" } }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/path-to-regexp": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", "integrity": "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==", "license": "MIT" }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, "node_modules/pdfjs-dist": { "version": "4.10.38", "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-4.10.38.tgz", @@ -4270,6 +5767,276 @@ "url": "https://ko-fi.com/killymxi" } }, + "node_modules/pg": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz", + "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.12.0", + "pg-pool": "^3.13.0", + "pg-protocol": "^1.13.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.3.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz", + "integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.12.0.tgz", + "integrity": "sha512-U7qg+bpswf3Cs5xLzRqbXbQl85ng0mfSV/J0nnA31MCLgvEaAo7CIhmeyrmJpOr7o+zm0rXK+hNnT5l9RHkCkQ==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.13.0.tgz", + "integrity": "sha512-gB+R+Xud1gLFuRD/QgOIgGOBE2KCQPaPwkzBBGC9oG69pHTkhQeIuejVIk3/cnDyX39av2AxomQiyPT13WKHQA==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.13.0.tgz", + "integrity": "sha512-zzdvXfS6v89r6v7OcFCHfHlyG/wvry1ALxZo4LqgUoy7W9xhBDMaqOuMiF3qEV45VqsN6rdlcehHrfDtlCPc8w==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pino": { + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz", + "integrity": "sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==", + "license": "MIT", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^3.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^4.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz", + "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/pino-http": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/pino-http/-/pino-http-11.0.0.tgz", + "integrity": "sha512-wqg5XIAGRRIWtTk8qPGxkbrfiwEWz1lgedVLvhLALudKXvg1/L2lTFgTGPJ4Z2e3qcRmxoFxDuSdMdMGNM6I1g==", + "license": "MIT", + "dependencies": { + "get-caller-file": "^2.0.5", + "pino": "^10.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0" + } + }, + "node_modules/pino-pretty": { + "version": "13.1.3", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-13.1.3.tgz", + "integrity": "sha512-ttXRkkOz6WWC95KeY9+xxWL6AtImwbyMHrL1mSwqwW9u+vLp/WIElvHvCSDg0xO/Dzrggz1zv3rN5ovTRVowKg==", + "dev": true, + "license": "MIT", + "dependencies": { + "colorette": "^2.0.7", + "dateformat": "^4.6.3", + "fast-copy": "^4.0.0", + "fast-safe-stringify": "^2.1.1", + "help-me": "^5.0.0", + "joycon": "^3.1.1", + "minimist": "^1.2.6", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^3.0.0", + "pump": "^3.0.0", + "secure-json-parse": "^4.0.0", + "sonic-boom": "^4.0.1", + "strip-json-comments": "^5.0.2" + }, + "bin": { + "pino-pretty": "bin.js" + } + }, + "node_modules/pino-std-serializers": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.1.0.tgz", + "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==", + "license": "MIT" + }, + "node_modules/postcss": { + "version": "8.5.14", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.14.tgz", + "integrity": "sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss/node_modules/nanoid": { + "version": "3.3.12", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", + "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prettier": { "version": "3.8.1", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.1.tgz", @@ -4291,6 +6058,22 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "license": "MIT" }, + "node_modules/process-warning": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", + "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, "node_modules/protobufjs": { "version": "7.5.5", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.5.tgz", @@ -4328,6 +6111,17 @@ "node": ">= 0.10" } }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/qs": { "version": "6.14.2", "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", @@ -4343,6 +6137,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", + "license": "MIT" + }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -4420,6 +6220,24 @@ "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "license": "MIT" }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "license": "MIT", + "engines": { + "node": ">= 12.13.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/resend": { "version": "4.8.0", "resolved": "https://registry.npmjs.org/resend/-/resend-4.8.0.tgz", @@ -4451,6 +6269,40 @@ "node": ">= 4" } }, + "node_modules/rolldown": { + "version": "1.0.0-rc.17", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.17.tgz", + "integrity": "sha512-ZrT53oAKrtA4+YtBWPQbtPOxIbVDbxT0orcYERKd63VJTF13zPcgXTvD4843L8pcsI7M6MErt8QtON6lrB9tyA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.127.0", + "@rolldown/pluginutils": "1.0.0-rc.17" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.0.0-rc.17", + "@rolldown/binding-darwin-arm64": "1.0.0-rc.17", + "@rolldown/binding-darwin-x64": "1.0.0-rc.17", + "@rolldown/binding-freebsd-x64": "1.0.0-rc.17", + "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.17", + "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.17", + "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.17", + "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.17", + "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.17", + "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.17", + "@rolldown/binding-linux-x64-musl": "1.0.0-rc.17", + "@rolldown/binding-openharmony-arm64": "1.0.0-rc.17", + "@rolldown/binding-wasm32-wasi": "1.0.0-rc.17", + "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.17", + "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.17" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -4471,6 +6323,15 @@ ], "license": "MIT" }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -4493,6 +6354,23 @@ "license": "MIT", "peer": true }, + "node_modules/secure-json-parse": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-4.1.0.tgz", + "integrity": "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/selderee": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/selderee/-/selderee-0.11.0.tgz", @@ -4562,6 +6440,27 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", "license": "ISC" }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/side-channel": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", @@ -4628,10 +6527,57 @@ "side-channel-map": "^1.0.1" }, "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/sonic-boom": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", + "integrity": "sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" } }, "node_modules/sprintf-js": { @@ -4640,6 +6586,13 @@ "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "license": "BSD-3-Clause" }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -4649,6 +6602,13 @@ "node": ">= 0.8" } }, + "node_modules/std-env": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.1.0.tgz", + "integrity": "sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==", + "dev": true, + "license": "MIT" + }, "node_modules/streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", @@ -4672,6 +6632,45 @@ "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "license": "MIT" }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-5.0.3.tgz", + "integrity": "sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/strnum": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.2.3.tgz", @@ -4684,6 +6683,146 @@ ], "license": "MIT" }, + "node_modules/superagent": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/superagent/-/superagent-10.3.0.tgz", + "integrity": "sha512-B+4Ik7ROgVKrQsXTV0Jwp2u+PXYLSlqtDAhYnkkD+zn3yg8s/zjA2MeGayPoY/KICrbitwneDHrjSotxKL+0XQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "component-emitter": "^1.3.1", + "cookiejar": "^2.1.4", + "debug": "^4.3.7", + "fast-safe-stringify": "^2.1.1", + "form-data": "^4.0.5", + "formidable": "^3.5.4", + "methods": "^1.1.2", + "mime": "2.6.0", + "qs": "^6.14.1" + }, + "engines": { + "node": ">=14.18.0" + } + }, + "node_modules/superagent/node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/superagent/node_modules/mime": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.6.0.tgz", + "integrity": "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==", + "dev": true, + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/superagent/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/supertest": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/supertest/-/supertest-7.2.2.tgz", + "integrity": "sha512-oK8WG9diS3DlhdUkcFn4tkNIiIbBx9lI2ClF8K+b2/m8Eyv47LSawxUzZQSNKUrVb2KsqeTDCcjAAVPYaSLVTA==", + "dev": true, + "license": "MIT", + "dependencies": { + "cookie-signature": "^1.2.2", + "methods": "^1.1.2", + "superagent": "^10.3.0" + }, + "engines": { + "node": ">=14.18.0" + } + }, + "node_modules/supertest/node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/thread-stream": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.0.0.tgz", + "integrity": "sha512-4iMVL6HAINXWf1ZKZjIPcz5wYaOdPhtO8ATvZ+Xqp3BTdaqtAwQkNmKORqcIo5YkQqGXq5cwfswDwMqqQNrpJA==", + "license": "MIT", + "dependencies": { + "real-require": "^0.2.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.1.2.tgz", + "integrity": "sha512-dAqSqE/RabpBKI8+h26GfLq6Vb3JVXs30XYQjdMjaj/c2tS8IYYMbIzP599KtRj7c57/wYApb3QjgRgXmrCukA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.16", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", + "integrity": "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyrainbow": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", + "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/tmp": { "version": "0.2.5", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", @@ -4812,6 +6951,174 @@ "node": ">= 0.8" } }, + "node_modules/vite": { + "version": "8.0.10", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.10.tgz", + "integrity": "sha512-rZuUu9j6J5uotLDs+cAA4O5H4K1SfPliUlQwqa6YEwSrWDZzP4rhm00oJR5snMewjxF5V/K3D4kctsUTsIU9Mw==", + "dev": true, + "license": "MIT", + "dependencies": { + "lightningcss": "^1.32.0", + "picomatch": "^4.0.4", + "postcss": "^8.5.10", + "rolldown": "1.0.0-rc.17", + "tinyglobby": "^0.2.16" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.1.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.5.tgz", + "integrity": "sha512-9Xx1v3/ih3m9hN+SbfkUyy0JAs72ap3r7joc87XL6jwF0jGg6mFBvQ1SrwaX+h8BlkX6Hz9shdd1uo6AF+ZGpg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.5", + "@vitest/mocker": "4.1.5", + "@vitest/pretty-format": "4.1.5", + "@vitest/runner": "4.1.5", + "@vitest/snapshot": "4.1.5", + "@vitest/spy": "4.1.5", + "@vitest/utils": "4.1.5", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.5", + "@vitest/browser-preview": "4.1.5", + "@vitest/browser-webdriverio": "4.1.5", + "@vitest/coverage-istanbul": "4.1.5", + "@vitest/coverage-v8": "4.1.5", + "@vitest/ui": "4.1.5", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, "node_modules/web-streams-polyfill": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", @@ -4821,6 +7128,62 @@ "node": ">= 8" } }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, "node_modules/ws": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz", @@ -4877,6 +7240,63 @@ "engines": { "node": ">=0.4" } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yocto-queue": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", + "license": "MIT", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.2.tgz", + "integrity": "sha512-IynmDyxsEsb9RKzO3J9+4SxXnl2FTFSzNBaKKaMV6tsSk0rw9gYw9gs+JFCq/qk2LCZ78KDwyj+Z289TijSkUw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/backend/package.json b/backend/package.json index 8451ab8b7..94626309d 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,11 +1,22 @@ { "name": "mike-backend", "version": "1.0.0", + "license": "AGPL-3.0-only", "private": true, "scripts": { "dev": "tsx watch src/index.ts", "build": "tsc", - "start": "node dist/index.js" + "prestart": "npm run db:migrate", + "start": "node dist/index.js", + "db:migrate": "node-pg-migrate -m migrations -j ts up", + "db:migrate-down": "node-pg-migrate -m migrations -j ts down 1", + "db:migrate-create": "node-pg-migrate -m migrations -j ts create", + "test:cross-tenant": "vitest run --config vitest.config.ts", + "test:no-db": "vitest run --config vitest.no-db.config.ts", + "test:golden-log": "vitest run --config vitest.golden-log.config.ts", + "test:docx": "vitest run --config vitest.docx.config.ts", + "test:auth-hardening": "vitest run --config vitest.auth-hardening.config.ts", + "test:saga": "vitest run --config vitest.saga.config.ts" }, "dependencies": { "@anthropic-ai/sdk": "^0.90.0", @@ -20,22 +31,33 @@ "express-rate-limit": "^8.5.1", "fast-diff": "^1.3.0", "fast-xml-parser": "^5.7.1", - "helmet": "^8.1.0", "jszip": "^3.10.1", "libreoffice-convert": "^1.6.0", + "lru-cache": "^11.3.5", "mammoth": "^1.9.0", - "multer": "^1.4.5-lts.2", + "multer": "^2.1.1", + "node-pg-migrate": "^8.0.4", + "p-limit": "^7.3.0", + "p-queue": "^9.2.0", "pdfjs-dist": "^4.10.38", - "resend": "^4.5.1" + "pg": "^8.20.0", + "pino": "^10.3.1", + "pino-http": "^11.0.0", + "resend": "^4.5.1", + "zod": "^4.4.2" }, "devDependencies": { "@types/cors": "^2.8.17", "@types/express": "^4.17.21", - "@types/multer": "^1.4.12", + "@types/multer": "^2.1.0", "@types/node": "^22.14.1", + "@types/pg": "^8.20.0", + "@types/supertest": "^7.2.0", + "pino-pretty": "^13.1.3", "prettier": "^3.8.1", + "supertest": "^7.2.2", "tsx": "^4.19.3", - "typescript": "^5.8.3" - }, - "license": "AGPL-3.0-only" + "typescript": "^5.8.3", + "vitest": "^4.1.5" + } } diff --git a/backend/schema.sql b/backend/schema.sql deleted file mode 100644 index cc9b9cef9..000000000 --- a/backend/schema.sql +++ /dev/null @@ -1,367 +0,0 @@ --- Mike Supabase schema --- Based on supabase-migration.sql plus the later backend/migrations/*.sql files. --- Use this for a fresh Supabase database. Existing deployments should continue --- to apply the incremental migration files instead. - -create extension if not exists "pgcrypto"; - --- --------------------------------------------------------------------------- --- User profiles --- --------------------------------------------------------------------------- - -create table if not exists public.user_profiles ( - id uuid primary key default gen_random_uuid(), - user_id uuid not null unique references auth.users(id) on delete cascade, - display_name text, - organisation text, - tier text not null default 'Free', - message_credits_used integer not null default 0, - credits_reset_date timestamptz not null default (now() + interval '30 days'), - tabular_model text not null default 'gemini-3-flash-preview', - created_at timestamptz not null default now(), - updated_at timestamptz not null default now() -); - -create index if not exists idx_user_profiles_user - on public.user_profiles(user_id); - -create or replace function public.handle_new_user() -returns trigger -language plpgsql -security definer -set search_path = public -as $$ -begin - insert into public.user_profiles (user_id) - values (new.id) - on conflict (user_id) do nothing; - return new; -exception when others then - -- Never block signup if the profile insert fails. - return new; -end; -$$; - -drop trigger if exists on_auth_user_created on auth.users; -create trigger on_auth_user_created - after insert on auth.users - for each row execute procedure public.handle_new_user(); - -create table if not exists public.user_api_keys ( - id uuid primary key default gen_random_uuid(), - user_id uuid not null references auth.users(id) on delete cascade, - provider text not null check (provider in ('claude', 'gemini', 'openai')), - encrypted_key text not null, - iv text not null, - auth_tag text not null, - created_at timestamptz not null default now(), - updated_at timestamptz not null default now(), - unique(user_id, provider) -); - -create index if not exists idx_user_api_keys_user - on public.user_api_keys(user_id); - --- --------------------------------------------------------------------------- --- Projects and documents --- --------------------------------------------------------------------------- - -create table if not exists public.projects ( - id uuid primary key default gen_random_uuid(), - user_id text not null, - name text not null, - cm_number text, - visibility text not null default 'private', - shared_with jsonb not null default '[]'::jsonb, - created_at timestamptz not null default now(), - updated_at timestamptz not null default now() -); - -create index if not exists idx_projects_user - on public.projects(user_id); - -create index if not exists projects_shared_with_idx - on public.projects using gin (shared_with); - -create table if not exists public.project_subfolders ( - id uuid primary key default gen_random_uuid(), - project_id uuid not null references public.projects(id) on delete cascade, - user_id text not null, - name text not null, - parent_folder_id uuid references public.project_subfolders(id) on delete cascade, - created_at timestamptz not null default now(), - updated_at timestamptz not null default now() -); - -create index if not exists idx_project_subfolders_project - on public.project_subfolders(project_id); - -create table if not exists public.documents ( - id uuid primary key default gen_random_uuid(), - project_id uuid references public.projects(id) on delete cascade, - user_id text not null, - filename text not null, - file_type text, - size_bytes integer not null default 0, - page_count integer, - structure_tree jsonb, - status text not null default 'pending', - folder_id uuid references public.project_subfolders(id) on delete set null, - created_at timestamptz not null default now(), - updated_at timestamptz not null default now() -); - -create index if not exists idx_documents_user_project - on public.documents(user_id, project_id); - -create index if not exists idx_documents_project_folder - on public.documents(project_id, folder_id); - -create table if not exists public.document_versions ( - id uuid primary key default gen_random_uuid(), - document_id uuid not null references public.documents(id) on delete cascade, - storage_path text not null, - pdf_storage_path text, - source text not null default 'upload', - version_number integer, - display_name text, - created_at timestamptz not null default now(), - constraint document_versions_source_check - check (source = any (array[ - 'upload'::text, - 'user_upload'::text, - 'assistant_edit'::text, - 'user_accept'::text, - 'user_reject'::text, - 'generated'::text - ])) -); - -create index if not exists document_versions_document_id_idx - on public.document_versions(document_id, created_at desc); - -create index if not exists document_versions_doc_vnum_idx - on public.document_versions(document_id, version_number); - -alter table public.documents - add column if not exists current_version_id uuid - references public.document_versions(id) on delete set null; - -create table if not exists public.document_edits ( - id uuid primary key default gen_random_uuid(), - document_id uuid not null references public.documents(id) on delete cascade, - chat_message_id uuid, - version_id uuid not null references public.document_versions(id) on delete cascade, - change_id text not null, - del_w_id text, - ins_w_id text, - deleted_text text not null default '', - inserted_text text not null default '', - context_before text, - context_after text, - status text not null default 'pending' - check (status = any (array[ - 'pending'::text, - 'accepted'::text, - 'rejected'::text - ])), - created_at timestamptz not null default now(), - resolved_at timestamptz -); - -create index if not exists document_edits_document_id_idx - on public.document_edits(document_id, created_at desc); - -create index if not exists document_edits_message_id_idx - on public.document_edits(chat_message_id); - -create index if not exists document_edits_version_id_idx - on public.document_edits(version_id); - --- --------------------------------------------------------------------------- --- Workflows --- --------------------------------------------------------------------------- - -create table if not exists public.workflows ( - id uuid primary key default gen_random_uuid(), - user_id text, - title text not null, - type text not null, - prompt_md text, - columns_config jsonb, - practice text, - is_system boolean not null default false, - created_at timestamptz not null default now() -); - -create index if not exists idx_workflows_user - on public.workflows(user_id); - -create table if not exists public.hidden_workflows ( - id uuid primary key default gen_random_uuid(), - user_id text not null, - workflow_id text not null, - created_at timestamptz not null default now(), - unique(user_id, workflow_id) -); - -create index if not exists idx_hidden_workflows_user - on public.hidden_workflows(user_id); - -create table if not exists public.workflow_shares ( - id uuid primary key default gen_random_uuid(), - workflow_id uuid not null references public.workflows(id) on delete cascade, - shared_by_user_id text not null, - shared_with_email text not null, - allow_edit boolean not null default false, - created_at timestamptz not null default now(), - constraint workflow_shares_workflow_email_unique - unique(workflow_id, shared_with_email) -); - -create index if not exists workflow_shares_workflow_id_idx - on public.workflow_shares(workflow_id); - -create index if not exists workflow_shares_email_idx - on public.workflow_shares(shared_with_email); - --- --------------------------------------------------------------------------- --- Assistant chats --- --------------------------------------------------------------------------- - -create table if not exists public.chats ( - id uuid primary key default gen_random_uuid(), - project_id uuid references public.projects(id) on delete cascade, - user_id text not null, - title text, - created_at timestamptz not null default now() -); - -create index if not exists idx_chats_user - on public.chats(user_id); - -create index if not exists idx_chats_project - on public.chats(project_id); - -create table if not exists public.chat_messages ( - id uuid primary key default gen_random_uuid(), - chat_id uuid not null references public.chats(id) on delete cascade, - role text not null, - content jsonb, - files jsonb, - annotations jsonb, - created_at timestamptz not null default now() -); - -create index if not exists idx_chat_messages_chat - on public.chat_messages(chat_id); - -do $$ -begin - if not exists ( - select 1 - from pg_constraint - where conname = 'document_edits_chat_message_id_fkey' - and conrelid = 'public.document_edits'::regclass - ) then - alter table public.document_edits - add constraint document_edits_chat_message_id_fkey - foreign key (chat_message_id) - references public.chat_messages(id) - on delete set null; - end if; -end; -$$; - --- --------------------------------------------------------------------------- --- Tabular reviews --- --------------------------------------------------------------------------- - -create table if not exists public.tabular_reviews ( - id uuid primary key default gen_random_uuid(), - project_id uuid references public.projects(id) on delete cascade, - user_id text not null, - title text, - columns_config jsonb, - workflow_id uuid references public.workflows(id) on delete set null, - practice text, - shared_with jsonb not null default '[]'::jsonb, - created_at timestamptz not null default now(), - updated_at timestamptz not null default now() -); - -create index if not exists idx_tabular_reviews_user - on public.tabular_reviews(user_id); - -create index if not exists idx_tabular_reviews_project - on public.tabular_reviews(project_id); - -create index if not exists tabular_reviews_shared_with_idx - on public.tabular_reviews using gin (shared_with); - -create table if not exists public.tabular_cells ( - id uuid primary key default gen_random_uuid(), - review_id uuid not null references public.tabular_reviews(id) on delete cascade, - document_id uuid not null references public.documents(id) on delete cascade, - column_index integer not null, - content text, - citations jsonb, - status text not null default 'pending', - created_at timestamptz not null default now() -); - -create index if not exists idx_tabular_cells_review - on public.tabular_cells(review_id, document_id, column_index); - -create table if not exists public.tabular_review_chats ( - id uuid primary key default gen_random_uuid(), - review_id uuid not null references public.tabular_reviews(id) on delete cascade, - user_id text not null, - title text, - created_at timestamptz not null default now(), - updated_at timestamptz not null default now() -); - -create index if not exists tabular_review_chats_review_idx - on public.tabular_review_chats(review_id, updated_at desc); - -create index if not exists tabular_review_chats_user_idx - on public.tabular_review_chats(user_id); - -create table if not exists public.tabular_review_chat_messages ( - id uuid primary key default gen_random_uuid(), - chat_id uuid not null references public.tabular_review_chats(id) on delete cascade, - role text not null, - content jsonb, - annotations jsonb, - created_at timestamptz not null default now() -); - -create index if not exists tabular_review_chat_messages_chat_idx - on public.tabular_review_chat_messages(chat_id, created_at); - --- --------------------------------------------------------------------------- --- Direct client grant hardening --- --------------------------------------------------------------------------- --- --- The frontend uses Supabase directly only for authentication. Application --- data access goes through the backend API with the service role after the --- backend verifies the user's JWT. Do not grant the browser anon/authenticated --- roles direct table privileges for backend-owned data. - -revoke all on public.user_profiles from anon, authenticated; -revoke all on public.projects from anon, authenticated; -revoke all on public.project_subfolders from anon, authenticated; -revoke all on public.documents from anon, authenticated; -revoke all on public.document_versions from anon, authenticated; -revoke all on public.document_edits from anon, authenticated; -revoke all on public.workflows from anon, authenticated; -revoke all on public.hidden_workflows from anon, authenticated; -revoke all on public.workflow_shares from anon, authenticated; -revoke all on public.chats from anon, authenticated; -revoke all on public.chat_messages from anon, authenticated; -revoke all on public.tabular_reviews from anon, authenticated; -revoke all on public.tabular_cells from anon, authenticated; -revoke all on public.tabular_review_chats from anon, authenticated; -revoke all on public.tabular_review_chat_messages from anon, authenticated; -revoke all on public.user_api_keys from anon, authenticated; diff --git a/backend/src/app.ts b/backend/src/app.ts new file mode 100644 index 000000000..2232a4903 --- /dev/null +++ b/backend/src/app.ts @@ -0,0 +1,41 @@ +import "dotenv/config"; +import "./env"; +import express from "express"; +import cors from "cors"; +import { httpLogger } from "./lib/logger"; +import { chatRouter } from "./routes/chat"; +import { projectsRouter } from "./routes/projects"; +import { projectChatRouter } from "./routes/projectChat"; +import { documentsRouter } from "./routes/documents"; +import { tabularRouter } from "./routes/tabular"; +import { workflowsRouter } from "./routes/workflows"; +import { userRouter } from "./routes/user"; +import { downloadsRouter } from "./routes/downloads"; +import { modelsRouter } from "./routes/models"; + +export const app = express(); + +app.use( + cors({ + origin: process.env.FRONTEND_URL ?? "http://localhost:3000", + credentials: true, + exposedHeaders: ["X-Docs-Skipped"], + }), +); + +app.use(httpLogger); + +app.use(express.json({ limit: "1mb" })); + +app.use("/chat", chatRouter); +app.use("/projects", projectsRouter); +app.use("/projects/:projectId/chat", projectChatRouter); +app.use("/single-documents", documentsRouter); +app.use("/tabular-review", tabularRouter); +app.use("/workflows", workflowsRouter); +app.use("/user", userRouter); +app.use("/users", userRouter); +app.use("/download", downloadsRouter); +app.use("/models", modelsRouter); + +app.get("/health", (_req, res) => res.json({ ok: true })); diff --git a/backend/src/env.ts b/backend/src/env.ts new file mode 100644 index 000000000..425983241 --- /dev/null +++ b/backend/src/env.ts @@ -0,0 +1,60 @@ +/** + * Centralized, validated environment variables for the backend. + * + * Required env vars (process throws on missing): + * SUPABASE_URL — Supabase project URL + * SUPABASE_SECRET_KEY — Supabase service role key (never exposed to clients) + * DOWNLOAD_SIGNING_SECRET — HMAC secret for signed download tokens (CLEAN-07) + * FRONTEND_URL — CORS allow-list origin + * R2_ENDPOINT_URL — Cloudflare R2 endpoint, https://.r2.cloudflarestorage.com + * R2_ACCESS_KEY_ID — R2 API token (Access Key ID) + * R2_SECRET_ACCESS_KEY — R2 API token (Secret Access Key) + * R2_BUCKET_NAME — R2 bucket name + * HUGO_MASTER_KEY — AES-256-GCM master key for at-rest encryption of user LLM API keys (CLEAN-05). + * Must be exactly 64 hex characters (32 bytes). Generate with: openssl rand -hex 32 + * HUGO_RESTORE_TOKEN_SECRET — HMAC secret for account-restore tokens (CLEAN-44). + * Must be at least 32 characters. Generate with: openssl rand -base64 48 + * + * Optional: + * ANTHROPIC_API_KEY — Claude provider key (operators may configure one provider only) + * GEMINI_API_KEY — Gemini provider key (operators may configure one provider only) + * PORT — HTTP port (default 3001 in index.ts) + * LLM_STREAM_DEBUG — when set, enables raw LLM stream console logging (CLEAN-06) + * + * Note: The 30-day account deletion grace window is a hardcoded constant + * (`DELETE_GRACE_DAYS` in lib/accountDeletion.ts), not an env var (D-04). + * + * Importing this module at startup validates process.env and throws with + * a helpful, aggregated error if any required var is missing. + */ +import { z } from "zod"; + +export const envSchema = z.object({ + SUPABASE_URL: z.string().min(1), + SUPABASE_SECRET_KEY: z.string().min(1), + DOWNLOAD_SIGNING_SECRET: z.string().min(1), + FRONTEND_URL: z.string().min(1), + R2_ENDPOINT_URL: z.string().min(1), + R2_ACCESS_KEY_ID: z.string().min(1), + R2_SECRET_ACCESS_KEY: z.string().min(1), + R2_BUCKET_NAME: z.string().min(1), + HUGO_MASTER_KEY: z.string().regex(/^[0-9a-fA-F]{64}$/, "HUGO_MASTER_KEY must be exactly 64 hex characters (32 bytes). Generate with: openssl rand -hex 32"), + HUGO_RESTORE_TOKEN_SECRET: z.string().min(32, "HUGO_RESTORE_TOKEN_SECRET must be at least 32 characters. Generate with: openssl rand -base64 48"), + ANTHROPIC_API_KEY: z.string().min(1).optional(), + GEMINI_API_KEY: z.string().min(1).optional(), + PORT: z.string().optional(), + LLM_STREAM_DEBUG: z.string().optional(), +}); + +const result = envSchema.safeParse(process.env); +if (!result.success) { + const issues = result.error.issues + .map((i) => ` ${i.path.join(".")}: ${i.message}`) + .join("\n"); + throw new Error( + `[env] Server cannot start — missing or invalid environment variables:\n${issues}\n` + + `See backend/.env.example for required variables.`, + ); +} + +export const env = result.data; diff --git a/backend/src/index.ts b/backend/src/index.ts index 07b3b8490..ab21ec4d0 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -1,126 +1,14 @@ -import "dotenv/config"; -import express from "express"; -import cors from "cors"; -import helmet from "helmet"; -import rateLimit from "express-rate-limit"; -import { chatRouter } from "./routes/chat"; -import { projectsRouter } from "./routes/projects"; -import { projectChatRouter } from "./routes/projectChat"; -import { documentsRouter } from "./routes/documents"; -import { tabularRouter } from "./routes/tabular"; -import { workflowsRouter } from "./routes/workflows"; -import { userRouter } from "./routes/user"; -import { downloadsRouter } from "./routes/downloads"; +import { app } from "./app"; +import { resetStuckPendingConversions } from "./lib/pdfQueue"; +import { resetStuckRunningJobs, startAccountDeletionWorker } from "./lib/accountDeletionWorker"; +import { logger } from "./lib/logger"; -const app = express(); const PORT = process.env.PORT ?? 3001; -const isProduction = process.env.NODE_ENV === "production"; -function envInt(name: string, fallback: number): number { - const raw = process.env[name]; - if (!raw) return fallback; - const parsed = Number.parseInt(raw, 10); - return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; -} - -function minutes(value: number): number { - return value * 60 * 1000; -} - -function hours(value: number): number { - return minutes(value * 60); -} - -function makeLimiter(options: { - windowMs: number; - max: number; - message?: string; -}) { - return rateLimit({ - windowMs: options.windowMs, - max: options.max, - standardHeaders: true, - legacyHeaders: false, - skip: (req) => req.method === "OPTIONS", - message: { - detail: - options.message ?? "Too many requests. Please try again later.", - }, - }); -} - -const generalLimiter = makeLimiter({ - windowMs: minutes(envInt("RATE_LIMIT_GENERAL_WINDOW_MINUTES", 15)), - max: envInt("RATE_LIMIT_GENERAL_MAX", 300), -}); - -const chatLimiter = makeLimiter({ - windowMs: minutes(envInt("RATE_LIMIT_CHAT_WINDOW_MINUTES", 15)), - max: envInt("RATE_LIMIT_CHAT_MAX", 30), - message: "Too many chat requests. Please try again later.", -}); - -const chatCreateLimiter = makeLimiter({ - windowMs: minutes(envInt("RATE_LIMIT_CHAT_CREATE_WINDOW_MINUTES", 15)), - max: envInt("RATE_LIMIT_CHAT_CREATE_MAX", 60), -}); - -const uploadLimiter = makeLimiter({ - windowMs: hours(envInt("RATE_LIMIT_UPLOAD_WINDOW_HOURS", 1)), - max: envInt("RATE_LIMIT_UPLOAD_MAX", 50), - message: "Too many upload requests. Please try again later.", -}); - -app.disable("x-powered-by"); -app.set("trust proxy", envInt("TRUST_PROXY_HOPS", 1)); - -app.use( - helmet({ - contentSecurityPolicy: false, - crossOriginEmbedderPolicy: false, - hsts: isProduction - ? { - maxAge: 15552000, - includeSubDomains: true, - } - : false, - referrerPolicy: { policy: "no-referrer" }, - }), -); - -app.use( - cors({ - origin: process.env.FRONTEND_URL ?? "http://localhost:3000", - credentials: true, - }), -); - -app.use(generalLimiter); - -app.use(express.json({ limit: "50mb" })); - -app.post("/chat", chatLimiter); -app.post("/projects/:projectId/chat", chatLimiter); -app.post("/tabular-review/:reviewId/chat", chatLimiter); -app.post("/tabular-review/:reviewId/generate", chatLimiter); -app.post("/chat/create", chatCreateLimiter); -app.post("/chat/:chatId/generate-title", chatCreateLimiter); -app.post("/single-documents", uploadLimiter); -app.post("/single-documents/:documentId/versions", uploadLimiter); -app.post("/projects/:projectId/documents", uploadLimiter); - -app.use("/chat", chatRouter); -app.use("/projects", projectsRouter); -app.use("/projects/:projectId/chat", projectChatRouter); -app.use("/single-documents", documentsRouter); -app.use("/tabular-review", tabularRouter); -app.use("/workflows", workflowsRouter); -app.use("/user", userRouter); -app.use("/users", userRouter); -app.use("/download", downloadsRouter); - -app.get("/health", (_req, res) => res.json({ ok: true })); +void resetStuckPendingConversions(); +void resetStuckRunningJobs(); +startAccountDeletionWorker(); app.listen(PORT, () => { - console.log(`Mike backend running on port ${PORT}`); + logger.info({ port: PORT }, "Hugo backend running"); }); diff --git a/backend/src/lib/access.ts b/backend/src/lib/access.ts index 5964578ae..c52d6fc03 100644 --- a/backend/src/lib/access.ts +++ b/backend/src/lib/access.ts @@ -119,48 +119,6 @@ export async function ensureReviewAccess( return { ok: false }; } -/** - * Filter user-supplied document IDs down to documents the caller can read. - * - * Tabular review routes accept document IDs from request bodies. Without this - * check, a caller with access to any review could attach arbitrary document - * UUIDs and later cause /generate or /regenerate-cell to extract those bytes. - */ -export async function filterAccessibleDocumentIds( - documentIds: string[], - userId: string, - userEmail: string | null | undefined, - db: Db, -): Promise { - if (documentIds.length === 0) return []; - const { data: docs } = await db - .from("documents") - .select("id, user_id, project_id") - .in("id", documentIds); - const rows = (docs ?? []) as { - id: string; - user_id: string; - project_id: string | null; - }[]; - if (rows.length === 0) return []; - - const accessibleProjectIds = new Set( - await listAccessibleProjectIds(userId, userEmail, db), - ); - const allowed: string[] = []; - for (const doc of rows) { - if (doc.user_id === userId) { - allowed.push(doc.id); - } else if ( - doc.project_id && - accessibleProjectIds.has(doc.project_id) - ) { - allowed.push(doc.id); - } - } - return allowed; -} - /** * Returns the set of project IDs the user can access — own projects plus * any project where their email is in `shared_with`. Used to scope chat @@ -171,18 +129,26 @@ export async function listAccessibleProjectIds( userEmail: string | null | undefined, db: Db, ): Promise { - const [{ data: own }, { data: shared }] = await Promise.all([ + const [{ data: own }, { data: sharedCandidates }] = await Promise.all([ db.from("projects").select("id").eq("user_id", userId), userEmail ? db - .from("projects") - .select("id") - .filter("shared_with", "cs", JSON.stringify([userEmail])) - .neq("user_id", userId) + .from("projects") + .select("id, shared_with") + .neq("user_id", userId) : Promise.resolve({ data: [] as { id: string }[] }), ]); const ids = new Set(); for (const p of (own ?? []) as { id: string }[]) ids.add(p.id); - for (const p of (shared ?? []) as { id: string }[]) ids.add(p.id); + const email = (userEmail ?? "").toLowerCase(); + for (const p of (sharedCandidates ?? []) as { + id: string; + shared_with?: string[] | null; + }[]) { + const sharedWith = Array.isArray(p.shared_with) ? p.shared_with : []; + if (email && sharedWith.some((e) => (e ?? "").toLowerCase() === email)) { + ids.add(p.id); + } + } return [...ids]; } diff --git a/backend/src/lib/accountDeletion.ts b/backend/src/lib/accountDeletion.ts new file mode 100644 index 000000000..b71a3f967 --- /dev/null +++ b/backend/src/lib/accountDeletion.ts @@ -0,0 +1,531 @@ +/** + * Account soft-delete and restore helpers for CLEAN-44. + * + * These helpers are called by: + * - `routes/user.ts` (DELETE /user/account + POST /user/account/restore) + * - `lib/accountDeletionWorker.ts` (Plan 09 — hard-delete after grace window) + * + * Supabase Auth ban API verified at 2026-05-10: AdminUserAttributes uses `ban_duration`. + * Source: backend/node_modules/@supabase/auth-js/dist/module/lib/types.d.ts:446 + * Field signature: `ban_duration?: string | 'none'` + * Ban: set to e.g. "8760h" (1 year). Unban: set to "none". + * + * Design decisions (per CONTEXT.md): + * - D-04: DELETE_GRACE_DAYS is a hardcoded constant, NOT an env var. + * "The 30-day window is not operator-configurable in v1." + * - D-05: Restore path is token-authenticated (HMAC). No email sent. + * - D-06: Hard-delete is the worker's job (Plan 09), not this module. + * + * All helpers follow CLAUDE.md "Errors in libs: return null on failure, do not throw." + */ + +import { createServerSupabase } from "./supabase"; +import { logger } from "./logger"; + +// ── Constants ───────────────────────────────────────────────────────────────── + +/** + * Grace period between soft-delete and hard-delete, in days. + * + * Per CONTEXT.md D-04: ship as a hardcoded constant, NOT as an env var. + * "The 30-day window is not operator-configurable in v1." M3 may revisit + * if operators ask for a different default. RESEARCH.md Open Question 2 + * (RESOLVED) confirms this is the locked decision. + * + * Plan 11 smoke C temporarily edits this constant to 0 for the worker + * fast-path verification, then reverts before commit. + */ +export const DELETE_GRACE_DAYS = 30; + +/** + * Ban duration passed to Supabase Auth admin API on soft-delete. + * + * 1 year (8760h) — long enough that the worker hard-deletes within the + * 30-day grace window, but the ban must outlast a multi-week worker outage. + * On restore, pass "none" to lift the ban. + */ +export const BAN_DURATION_FOR_SOFT_DELETE = "8760h"; + +/** + * R2 prefix roots scanned per user during hard-delete (Plan 09 worker). + * Each is joined with `//` to form the full prefix. + */ +export const DELETION_PREFIXES = [ + "documents", + "generated", + "converted-pdfs", +] as const; + +// ── Types ───────────────────────────────────────────────────────────────────── + +type DbClient = ReturnType; + +/** + * State persisted to `account_deletion_jobs.last_continuation_token` to make + * the worker resumable mid-walk. Plan 09 worker reads and writes this shape. + */ +export type ContinuationState = { + currentPrefix: string; + token: string | null; + completedPrefixes: string[]; +}; + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +/** + * Mark a user's profile as soft-deleted. + * + * Idempotent: if `deleted_at` is already set, fetches and returns the + * existing timestamp rather than failing. This supports the re-DELETE + * flow (RESEARCH.md Open Q3) where re-issuing DELETE doesn't change + * the schedule but does re-issue a new restore token. + * + * Returns `{ deletedAt }` on success (new or existing), `null` on error. + */ +export async function markSoftDelete( + userId: string, + db?: DbClient, +): Promise<{ deletedAt: Date } | null> { + const client = db ?? createServerSupabase(); + try { + const now = new Date().toISOString(); + const { data, error } = await client + .from("user_profiles") + .update({ deleted_at: now, updated_at: now }) + .eq("user_id", userId) + .is("deleted_at", null) + .select("deleted_at") + .single(); + + if (error) { + if (error.code === "PGRST116") { + // PGRST116 = "The result contains 0 rows" — row already has deleted_at set. + // Fetch the existing deleted_at. + const { data: existing, error: fetchError } = await client + .from("user_profiles") + .select("deleted_at") + .eq("user_id", userId) + .single(); + if (fetchError || !existing?.deleted_at) { + logger.error({ err: fetchError, userId }, "[accountDeletion] markSoftDelete: refetch failed"); + return null; + } + logger.info({ userId, deletedAt: existing.deleted_at }, "[accountDeletion] markSoftDelete: already deleted, returning existing"); + return { deletedAt: new Date(existing.deleted_at as string) }; + } + logger.error({ err: error, userId }, "[accountDeletion] markSoftDelete failed"); + return null; + } + + if (!data?.deleted_at) { + logger.error({ userId }, "[accountDeletion] markSoftDelete: no deleted_at in response"); + return null; + } + + logger.info({ userId, deletedAt: data.deleted_at }, "[accountDeletion] markSoftDelete"); + return { deletedAt: new Date(data.deleted_at as string) }; + } catch (err) { + logger.error({ err, userId }, "[accountDeletion] markSoftDelete threw"); + return null; + } +} + +/** + * Clear the soft-delete flag on a user's profile (restore path). + * + * Only updates rows where `deleted_at IS NOT NULL` so a stray call against a + * non-deleted user is a no-op (WR-04). Returns `true` on success, `false` on + * error. + */ +export async function clearSoftDelete( + userId: string, + db?: DbClient, +): Promise { + const client = db ?? createServerSupabase(); + try { + const { error } = await client + .from("user_profiles") + .update({ deleted_at: null, updated_at: new Date().toISOString() }) + .eq("user_id", userId) + .not("deleted_at", "is", null); + if (error) { + logger.error({ err: error, userId }, "[accountDeletion] clearSoftDelete failed"); + return false; + } + logger.info({ userId }, "[accountDeletion] clearSoftDelete"); + return true; + } catch (err) { + logger.error({ err, userId }, "[accountDeletion] clearSoftDelete threw"); + return false; + } +} + +/** + * Ban a user in Supabase Auth via the admin API. + * + * Uses `ban_duration: BAN_DURATION_FOR_SOFT_DELETE` (1 year). + * Returns `true` on success, `false` on error. + */ +export async function banUser( + userId: string, + db?: DbClient, +): Promise { + const client = db ?? createServerSupabase(); + try { + const { error } = await client.auth.admin.updateUserById(userId, { + ban_duration: BAN_DURATION_FOR_SOFT_DELETE, + }); + if (error) { + logger.error({ err: error, userId }, "[accountDeletion] banUser failed"); + return false; + } + logger.info({ userId }, "[accountDeletion] banUser"); + return true; + } catch (err) { + logger.error({ err, userId }, "[accountDeletion] banUser threw"); + return false; + } +} + +/** + * Unban a user in Supabase Auth via the admin API. + * + * Uses `ban_duration: "none"` to lift the ban. + * Returns `true` on success, `false` on error. + */ +export async function unbanUser( + userId: string, + db?: DbClient, +): Promise { + const client = db ?? createServerSupabase(); + try { + const { error } = await client.auth.admin.updateUserById(userId, { + ban_duration: "none", + }); + if (error) { + logger.error({ err: error, userId }, "[accountDeletion] unbanUser failed"); + return false; + } + logger.info({ userId }, "[accountDeletion] unbanUser"); + return true; + } catch (err) { + logger.error({ err, userId }, "[accountDeletion] unbanUser threw"); + return false; + } +} + +/** + * Insert a row into `account_deletion_jobs` for the given user. + * + * ON CONFLICT (user_id) DO NOTHING — idempotent per RESEARCH.md Open Q3. + * Re-issuing DELETE on an already-deleted account does NOT change the + * scheduled hard-delete date. + * + * Returns `{ existed: false }` for a new insert, `{ existed: true }` if + * the row was already present. Returns `null` on error. + */ +export async function enqueueDeletionJob( + userId: string, + scheduledFor: Date, + db?: DbClient, +): Promise<{ existed: boolean } | null> { + const client = db ?? createServerSupabase(); + try { + // INSERT ... ON CONFLICT (user_id) DO NOTHING — idempotent. + // We use upsert with ignoreDuplicates: true which translates to ON CONFLICT DO NOTHING. + // Returns the inserted row on new insert, empty array if the row already existed. + const { data, error } = await client + .from("account_deletion_jobs") + .upsert( + { + user_id: userId, + scheduled_for: scheduledFor.toISOString(), + status: "pending", + }, + { onConflict: "user_id", ignoreDuplicates: true }, + ) + .select("user_id"); + + if (error) { + logger.error({ err: error, userId }, "[accountDeletion] enqueueDeletionJob failed"); + return null; + } + + if (!data || data.length === 0) { + // ON CONFLICT (user_id) DO NOTHING — row already existed, schedule unchanged + logger.info({ userId }, "[accountDeletion] enqueueDeletionJob: row already existed (idempotent)"); + return { existed: true }; + } + + logger.info({ userId, scheduledFor: scheduledFor.toISOString() }, "[accountDeletion] enqueueDeletionJob: new row inserted"); + return { existed: false }; + } catch (err) { + logger.error({ err, userId }, "[accountDeletion] enqueueDeletionJob threw"); + return null; + } +} + +/** + * Fetch the pending deletion job for a user. + * + * Returns the row on success, `null` if no row exists or on error. + */ +export async function getDeletionJob( + userId: string, + db?: DbClient, +): Promise<{ user_id: string; scheduled_for: string; status: string; restore_token_used_at: string | null } | null> { + const client = db ?? createServerSupabase(); + try { + const { data, error } = await client + .from("account_deletion_jobs") + .select("user_id, scheduled_for, status, restore_token_used_at") + .eq("user_id", userId) + .single(); + + if (error) { + if (error.code !== "PGRST116") { + logger.error({ err: error, userId }, "[accountDeletion] getDeletionJob failed"); + } + return null; + } + + return data as { user_id: string; scheduled_for: string; status: string; restore_token_used_at: string | null }; + } catch (err) { + logger.error({ err, userId }, "[accountDeletion] getDeletionJob threw"); + return null; + } +} + +/** + * Atomically consume the restore token for a user's deletion job. + * + * Stamps `restore_token_used_at = now()` and sets `status = 'cancelled'` + * in a single UPDATE with a WHERE clause that enforces single-use semantics: + * `WHERE user_id = $1 AND restore_token_used_at IS NULL AND status IN ('pending', 'running')` + * + * Returns: + * - `{ ok: true }` if the row was updated (token consumed) + * - `{ ok: false, reason: "already_used" }` if the row exists but `restore_token_used_at IS NOT NULL` or status is not pending/running + * - `{ ok: false, reason: "no_job" }` if no row exists for the user + */ +export async function consumeRestoreToken( + userId: string, + db?: DbClient, +): Promise<{ ok: true } | { ok: false; reason: "no_job" | "already_used" }> { + const client = db ?? createServerSupabase(); + try { + const now = new Date().toISOString(); + + // Atomic single-use enforcement: update only if token has not been consumed yet + const { data, error } = await client + .from("account_deletion_jobs") + .update({ restore_token_used_at: now, status: "cancelled" }) + .eq("user_id", userId) + .is("restore_token_used_at", null) + .in("status", ["pending", "running"]) + .select("user_id"); + + if (error) { + logger.error({ err: error, userId }, "[accountDeletion] consumeRestoreToken failed"); + // Treat DB errors as a "no_job" to avoid leaking internal details + return { ok: false, reason: "no_job" }; + } + + if (data && data.length > 0) { + logger.info({ userId }, "[accountDeletion] consumeRestoreToken: token consumed"); + return { ok: true }; + } + + // No rows updated — check whether the row exists at all + const { data: existing, error: checkError } = await client + .from("account_deletion_jobs") + .select("user_id, restore_token_used_at, status") + .eq("user_id", userId) + .single(); + + if (checkError || !existing) { + // PGRST116 (no rows) or other error — no job row + logger.info({ userId }, "[accountDeletion] consumeRestoreToken: no_job"); + return { ok: false, reason: "no_job" }; + } + + // Row exists but didn't match the WHERE — already consumed (or wrong status) + logger.info({ userId, status: (existing as { status: string }).status }, "[accountDeletion] consumeRestoreToken: already_used"); + return { ok: false, reason: "already_used" }; + } catch (err) { + logger.error({ err, userId }, "[accountDeletion] consumeRestoreToken threw"); + return { ok: false, reason: "no_job" }; + } +} + +// ── Plan 09 worker helpers ──────────────────────────────────────────────────── + +/** + * Atomically claim a due deletion job for processing. + * + * UPDATE account_deletion_jobs + * SET status = 'running', claimed_by = $2, claimed_at = now(), attempts = attempts + 1 + * WHERE user_id = $1 AND status = 'pending' AND scheduled_for <= now() + * + * The `WHERE status = 'pending'` predicate is the SOLE de-dup gate (RESEARCH.md + * Open Q4 / T-12-09-03): if two replicas race, only one UPDATE matches the row. + * The other receives 0 rows back and returns `{ ok: false }`. + * + * On success returns `{ ok: true, lastToken }` so the worker can resume from + * persisted continuation state. `lastToken` is `null` when the job has never + * been touched (fresh claim, no resume needed). + */ +export async function claimJob( + userId: string, + claimedBy: string, + db?: DbClient, +): Promise<{ ok: true; lastToken: ContinuationState | null } | { ok: false }> { + const client = db ?? createServerSupabase(); + try { + const nowIso = new Date().toISOString(); + // Read current attempts to increment atomically inside this UPDATE. + // PostgREST does not support raw expressions in UPDATE; the read+write + // pair below is safe because the WHERE clause prevents two writers from + // matching at the same time — the second writer sees status='running'. + const { data: current, error: readError } = await client + .from("account_deletion_jobs") + .select("attempts") + .eq("user_id", userId) + .eq("status", "pending") + .lte("scheduled_for", nowIso) + .maybeSingle(); + + if (readError) { + logger.error({ err: readError, userId }, "[accountDeletion] claimJob read failed"); + return { ok: false }; + } + if (!current) { + // No matching pending row (either already running, done, or not due yet) + return { ok: false }; + } + + const nextAttempts = ((current as { attempts?: number }).attempts ?? 0) + 1; + const { data, error } = await client + .from("account_deletion_jobs") + .update({ + status: "running", + claimed_by: claimedBy, + claimed_at: nowIso, + attempts: nextAttempts, + }) + .eq("user_id", userId) + .eq("status", "pending") + .lte("scheduled_for", nowIso) + .select("last_continuation_token"); + + if (error) { + logger.error({ err: error, userId }, "[accountDeletion] claimJob update failed"); + return { ok: false }; + } + if (!data || data.length === 0) { + // Lost the race: another claimer flipped status between our read and write + return { ok: false }; + } + + const raw = (data[0] as { last_continuation_token: unknown }).last_continuation_token; + const lastToken = + raw && typeof raw === "object" ? (raw as ContinuationState) : null; + logger.info({ userId, claimedBy }, "[accountDeletion] claimJob"); + return { ok: true, lastToken }; + } catch (err) { + logger.error({ err, userId }, "[accountDeletion] claimJob threw"); + return { ok: false }; + } +} + +/** + * Persist (or clear) the worker's continuation state on the job row. + * + * Pass `null` after every prefix completes (state-cleared between prefixes). + * Returns `false` on error (logger.error); never throws. + */ +export async function persistContinuationToken( + userId: string, + state: ContinuationState | null, + db?: DbClient, +): Promise { + const client = db ?? createServerSupabase(); + try { + const { error } = await client + .from("account_deletion_jobs") + .update({ last_continuation_token: state }) + .eq("user_id", userId); + if (error) { + logger.error({ err: error, userId }, "[accountDeletion] persistContinuationToken failed"); + return false; + } + return true; + } catch (err) { + logger.error({ err, userId }, "[accountDeletion] persistContinuationToken threw"); + return false; + } +} + +/** + * Mark a job row as `done` or `failed` after the worker finishes processing. + * + * SUCCESS PATH: this function never runs successfully on success because + * `hardDeleteUser` cascades the row away first. It DOES run on hardDeleteUser + * failure to mark the row as `failed` with the error text for operator review. + * + * Returns `false` on error; row-already-gone (success path) returns `true`. + */ +export async function finalizeJob( + userId: string, + result: { rows: number; objects: number; errors: string[] }, + db?: DbClient, +): Promise { + const client = db ?? createServerSupabase(); + try { + const status = result.errors.length > 0 ? "failed" : "done"; + const lastError = result.errors.length > 0 ? result.errors.join("\n") : null; + const { error } = await client + .from("account_deletion_jobs") + .update({ status, last_error: lastError }) + .eq("user_id", userId); + if (error) { + // PGRST116-style "no rows" is not an error here — the row was already + // cascaded away by hardDeleteUser. Treat as success. + if (error.code === "PGRST116") return true; + logger.error({ err: error, userId }, "[accountDeletion] finalizeJob failed"); + return false; + } + return true; + } catch (err) { + logger.error({ err, userId }, "[accountDeletion] finalizeJob threw"); + return false; + } +} + +/** + * Hard-delete the auth user via the admin API. + * + * This is the LAST step of the worker pipeline. FK CASCADE (Phase 5 schema) + * wipes user_profiles + every user-owned table in one shot, including the + * `account_deletion_jobs` row itself. + * + * Returns `false` on error; never throws. + */ +export async function hardDeleteUser( + userId: string, + db?: DbClient, +): Promise { + const client = db ?? createServerSupabase(); + try { + const { error } = await client.auth.admin.deleteUser(userId); + if (error) { + logger.error({ err: error, userId }, "[accountDeletion] hardDeleteUser failed"); + return false; + } + logger.info({ userId }, "[accountDeletion] hardDeleteUser"); + return true; + } catch (err) { + logger.error({ err, userId }, "[accountDeletion] hardDeleteUser threw"); + return false; + } +} diff --git a/backend/src/lib/accountDeletionWorker.ts b/backend/src/lib/accountDeletionWorker.ts new file mode 100644 index 000000000..771709ac4 --- /dev/null +++ b/backend/src/lib/accountDeletionWorker.ts @@ -0,0 +1,264 @@ +/** + * Account-deletion worker (CLEAN-44). + * + * Polls `account_deletion_jobs` every minute, atomically claims due rows, + * walks each user's R2 prefixes deleting in 1000-key batches, then calls + * `auth.admin.deleteUser` LAST so FK CASCADE wipes every dependent table + * in one shot. + * + * Resumable: `last_continuation_token` is persisted after every page so a + * crash mid-walk resumes from the same R2 marker. The atomic `claimJob` + * UPDATE on `status = 'pending'` is the SOLE de-dup gate (T-12-09-03); + * concurrent claims short-circuit before any R2 work begins. + * + * Mirrors `pdfQueue.ts`: lazy p-queue singleton + setInterval loop + + * startup-fixup helper (`resetStuckRunningJobs`). + */ + +import os from "os"; +import { createServerSupabase } from "./supabase"; +import { logger } from "./logger"; +import { listObjectsByPrefix, deleteObjectsBatch, storageEnabled } from "./storage"; +import { + claimJob, + persistContinuationToken, + finalizeJob, + hardDeleteUser, + DELETION_PREFIXES, + type ContinuationState, +} from "./accountDeletion"; + +const POLL_INTERVAL_MS = 60_000; +const POLL_BATCH_SIZE = 5; +const WORKER_ID = `${os.hostname()}-${process.pid}`; + +type DbClient = ReturnType; +type ListObjectsFn = typeof listObjectsByPrefix; +type DeleteObjectsFn = typeof deleteObjectsBatch; + +export type ProcessJobResult = { + rows: number; + objects: number; + errors: string[]; +}; + +export type ProcessJobDeps = { + listObjects?: ListObjectsFn; + deleteObjects?: DeleteObjectsFn; + db?: DbClient; + hardDelete?: typeof hardDeleteUser; +}; + +let _queue: import("p-queue").default | null = null; +let _interval: NodeJS.Timeout | null = null; + +async function getQueue(): Promise { + if (!_queue) { + const { default: PQueue } = await import("p-queue"); + _queue = new PQueue({ concurrency: 1 }); + } + return _queue; +} + +/** + * Process a single job end-to-end: claim → walk R2 prefixes → hardDeleteUser. + * + * Exported via `_processJobForTesting` so integration tests can drive a + * single job synchronously without waiting for the setInterval tick. + */ +async function processJob( + userId: string, + deps: ProcessJobDeps = {}, +): Promise { + const db = deps.db ?? createServerSupabase(); + const listObjects = deps.listObjects ?? listObjectsByPrefix; + const deleteObjects = deps.deleteObjects ?? deleteObjectsBatch; + const hardDelete = deps.hardDelete ?? hardDeleteUser; + + // Refuse to proceed without R2 credentials. Otherwise listObjects no-ops + // silently and hardDeleteUser cascades the DB rows away while leaving every + // user-owned R2 object orphaned. The injected mock deps in tests opt out by + // passing custom listObjects/deleteObjects. (WR-05) + const hasInjectedR2 = + deps.listObjects !== undefined && deps.deleteObjects !== undefined; + if (!storageEnabled && !hasInjectedR2) { + const msg = + "storageEnabled is false — refusing to hard-delete without R2 cleanup"; + logger.error({ userId }, `[accountDeletionWorker] ${msg}`); + return { rows: 0, objects: 0, errors: [msg] }; + } + + // claimJob FIRST — atomic UPDATE WHERE status='pending' is the sole de-dup gate. + // No R2 calls happen before this returns ok (B1 invariant). + const claim = await claimJob(userId, WORKER_ID, db); + if (!claim.ok) { + logger.info({ userId }, "[accountDeletionWorker] job already claimed or not due"); + return { rows: 0, objects: 0, errors: [] }; + } + + // Idempotency invariant: re-walking a completedPrefixes entry on crash recovery is + // acceptable because R2 DeleteObjects on a missing key is a no-op (S3 API). + const startState: ContinuationState = claim.lastToken ?? { + currentPrefix: `${DELETION_PREFIXES[0]}/${userId}/`, + token: null, + completedPrefixes: [], + }; + let totalDeleted = 0; + const errors: string[] = []; + + try { + for (const prefixRoot of DELETION_PREFIXES) { + const fullPrefix = `${prefixRoot}/${userId}/`; + if (startState.completedPrefixes.includes(fullPrefix)) continue; + + const startToken = + startState.currentPrefix === fullPrefix + ? startState.token ?? undefined + : undefined; + + let nextToken: string | undefined; + for await (const batch of listObjects(fullPrefix, startToken)) { + if (batch.keys.length > 0) { + const result = await deleteObjects(batch.keys); + totalDeleted += result.deleted; + errors.push(...result.errors); + } + nextToken = batch.nextToken; + await persistContinuationToken( + userId, + { + currentPrefix: fullPrefix, + token: nextToken ?? null, + completedPrefixes: startState.completedPrefixes, + }, + db, + ); + if (!nextToken) break; + } + + startState.completedPrefixes.push(fullPrefix); + await persistContinuationToken( + userId, + { + currentPrefix: fullPrefix, + token: null, + completedPrefixes: startState.completedPrefixes, + }, + db, + ); + } + + // hardDeleteUser LAST — FK CASCADE wipes the job row and all user tables. + const ok = await hardDelete(userId, db); + if (!ok) { + await finalizeJob( + userId, + { rows: 0, objects: totalDeleted, errors: ["hardDeleteUser returned false"] }, + db, + ); + logger.error({ userId, totalDeleted }, "[accountDeletionWorker] hardDeleteUser failed"); + return { rows: 0, objects: totalDeleted, errors: [...errors, "hardDeleteUser returned false"] }; + } + + const result: ProcessJobResult = { rows: 1, objects: totalDeleted, errors }; + logger.info( + { + event: "account_deletion_complete", + user_id: userId, + rows: result.rows, + objects: result.objects, + errors: result.errors, + }, + "[accountDeletionWorker] account_deletion_complete", + ); + return result; + } catch (err) { + const errStr = String(err); + await finalizeJob(userId, { rows: 0, objects: totalDeleted, errors: [errStr] }, db); + logger.error({ err, userId, totalDeleted }, "[accountDeletionWorker] processJob threw"); + return { rows: 0, objects: totalDeleted, errors: [errStr] }; + } +} + +async function tick(): Promise { + const db = createServerSupabase(); + const nowIso = new Date().toISOString(); + const { data: jobs, error } = await db + .from("account_deletion_jobs") + .select("user_id") + .lte("scheduled_for", nowIso) + .eq("status", "pending") + .limit(POLL_BATCH_SIZE); + + if (error) { + logger.error({ err: error }, "[accountDeletionWorker] tick select failed"); + return; + } + if (!jobs || jobs.length === 0) return; + + const queue = await getQueue(); + for (const job of jobs as { user_id: string }[]) { + void queue.add(() => processJob(job.user_id)); + } +} + +/** + * Wire the polling setInterval. Idempotent — calling twice is a no-op. + * Returns immediately; the interval drives further work in the background. + */ +export function startAccountDeletionWorker(): void { + if (_interval) return; + _interval = setInterval(() => { + void tick().catch((err) => + logger.error({ err }, "[accountDeletionWorker] tick failed"), + ); + }, POLL_INTERVAL_MS); + logger.info( + { pollIntervalMs: POLL_INTERVAL_MS, workerId: WORKER_ID }, + "[accountDeletionWorker] started", + ); +} + +/** + * Crash-recovery: flip orphaned `running` rows back to `pending` at boot. + * Mirrors `pdfQueue.resetStuckPendingConversions`. + */ +export async function resetStuckRunningJobs(): Promise { + try { + const db = createServerSupabase(); + const { data, error } = await db + .from("account_deletion_jobs") + .update({ status: "pending", claimed_by: null, claimed_at: null }) + .eq("status", "running") + .select("user_id"); + if (error) { + logger.error({ err: error }, "[accountDeletionWorker] resetStuckRunningJobs failed"); + return; + } + const count = data?.length ?? 0; + if (count > 0) { + logger.info({ count }, "[accountDeletionWorker] startup fixup: reset stuck running rows to pending"); + } + } catch (err) { + logger.error({ err }, "[accountDeletionWorker] resetStuckRunningJobs threw"); + } +} + +/** + * Test-only export: drives a single job synchronously without the setInterval. + * Accepts optional dep-injection for R2 client + DB to enable mock-based tests. + */ +export async function _processJobForTesting( + userId: string, + deps?: ProcessJobDeps, +): Promise { + return processJob(userId, deps); +} + +/** + * Test-only export: drives a single tick synchronously (used by polling tests + * that don't want to wait for the setInterval). + */ +export async function _tickForTesting(): Promise { + return tick(); +} diff --git a/backend/src/lib/chatTools.ts b/backend/src/lib/chatTools.ts deleted file mode 100644 index 6d85c6aaa..000000000 --- a/backend/src/lib/chatTools.ts +++ /dev/null @@ -1,3284 +0,0 @@ -import path from "path"; -import { - downloadFile, - generatedDocKey, - storageKey, - uploadFile, -} from "./storage"; -import { convertedPdfKey } from "./convert"; -import { createServerSupabase } from "./supabase"; -import { - applyTrackedEdits, - extractDocxBodyText, - type EditInput, -} from "./docxTrackedChanges"; -import { buildDownloadUrl } from "./downloadTokens"; -import { - attachActiveVersionPaths, - loadActiveVersion, -} from "./documentVersions"; -import { - streamChatWithTools, - resolveModel, - DEFAULT_MAIN_MODEL, - type LlmMessage, - type OpenAIToolSchema, -} from "./llm"; - -const STANDARD_FONT_DATA_URL = (() => { - try { - const pkgPath = require.resolve("pdfjs-dist/package.json"); - return path.join(path.dirname(pkgPath), "standard_fonts") + path.sep; - } catch { - return undefined; - } -})(); - -// --------------------------------------------------------------------------- -// Types -// --------------------------------------------------------------------------- - -export type DocStore = Map< - string, - { storage_path: string; file_type: string; filename: string } ->; - -export type WorkflowStore = Map; - -export type DocIndex = Record< - string, - { - document_id: string; - filename: string; - version_id?: string | null; - version_number?: number | null; - } ->; - -export type TabularCellStore = { - columns: { index: number; name: string }[]; - documents: { id: string; filename: string }[]; - /** key: `${colIndex}:${docId}` */ - cells: Map< - string, - { summary: string; flag?: string; reasoning?: string } | null - >; -}; - -export type ToolCall = { - id: string; - function: { name: string; arguments: string }; -}; - -export type ChatMessage = { - role: string; - content: string | null; - files?: { filename: string; document_id?: string }[]; - workflow?: { id: string; title: string }; -}; - -// --------------------------------------------------------------------------- -// Constants -// --------------------------------------------------------------------------- - -export const SYSTEM_PROMPT = `You are Mike, an AI legal assistant that helps lawyers and legal professionals analyze documents, answer legal questions, and draft legal documents. - -DOCUMENT CITATION INSTRUCTIONS: -When you reference specific content from a document, place a numbered marker [1], [2], etc. inline in your prose at the point of reference. - -After your complete response, append a block containing a JSON array with one entry per marker: - - -[ - {"ref": 1, "doc_id": "doc-0", "page": 3, "quote": "exact verbatim text from the document"}, - {"ref": 2, "doc_id": "doc-1", "page": "41-42", "quote": "Section 4.2 describes the procedure [[PAGE_BREAK]] in all material respects."} -] - - -CRITICAL: The number inside the [N] marker in your prose is the "ref" value of a citation entry in the block — it is NOT a page number, footnote number, section number, or any other number that appears in the document. The marker [1] refers to the entry with "ref": 1 in the JSON block; [2] refers to "ref": 2; and so on. Refs are simple sequential integers you assign (1, 2, 3, …) in the order citations appear in your prose. Never use a page number or a document's own numbering as the marker number. Every [N] you write in prose MUST have a matching {"ref": N, ...} entry in the JSON block. - -Rules: -- Only cite text that appears verbatim in the provided documents -- In every entry, "doc_id" MUST be the exact chat-local document label you were given (for example "doc-0"). Never use a filename, document UUID, or any other identifier in "doc_id" -- Keep quotes short (ideally ≤ 25 words) and narrowly scoped to the specific claim. Don't reuse one quote to support multiple different claims — give each its own citation -- "page" refers to the sequential [Page N] marker in the text you were given (1-indexed from the first page). IGNORE any page numbers printed inside the document itself (footers, roman numerals, etc.) -- For a single-page quote, set "page" to an integer. If a quote is one continuous sentence that spans two pages, set "page" to "N-M" and insert [[PAGE_BREAK]] in the quote at the page break. Otherwise, use separate citations for text on different pages -- Put the block at the very end of the response. Omit it entirely if there are no citations - -DOCX GENERATION: -If asked to draft or generate a document, use the generate_docx tool to produce a downloadable Word document. Always use this tool rather than just displaying the document content inline when the user asks for a document to be created. -If the user follows up on a document you just generated and asks for changes (e.g. "make section 3 longer", "add a termination clause", "change the parties"), default to calling edit_document on that newly generated document — do NOT call generate_docx again to regenerate the whole document. Only fall back to generate_docx if the user explicitly asks for a brand-new document or the change is so sweeping that an edit would not be coherent. -After calling generate_docx, do NOT include any download links, URLs, or markdown links to the document in your prose response — the download card is presented automatically by the UI. Do not describe formatting choices such as orientation or layout. -After calling generate_docx, you MUST call read_document on the returned doc_id before writing your prose response. Base your description on the generated document's actual text, not on memory of what you intended to generate. -Your prose response MUST include a short description of the generated document: what it is, its structure (key sections/clauses), and — if the draft was informed by any provided source documents — which sources you drew from and how. Keep it concise (typically 3–8 sentences or a short bulleted list). Refer to the document by filename, never by a download link. -When the description makes factual claims about the contents of the newly generated document, cite the generated document with [N] markers and a block exactly as specified in the DOCUMENT CITATION INSTRUCTIONS above. If you also make factual claims about provided source documents, cite those source documents separately. In every citation entry, use the exact chat-local doc_id label for the cited document. Omit the block if the description makes no such claims. -Heading hierarchy: always use Heading 1 before introducing Heading 2, Heading 2 before Heading 3, and so on. Never skip levels (e.g. do not jump from Heading 1 to Heading 3). -Numbering: all numbering MUST start from 1, never 0. This applies at every level of the hierarchy. Legal clause numbering is applied automatically by the document generator: top-level operative headings render as 1., 2., 3.; the first numbered body clause under a top-level heading renders as 1.1; nested body clauses under that render as (a), (b), (c); deeper nested clauses render as (i), (ii), (iii), then (A), (B), (C). Do NOT use 1.1.1 for legal body clauses when (a) is the expected next level. Never produce 0., 0.1, 1.0, 1.0.1, or any other sequence that begins a level with 0. -Never duplicate the numbering prefix in heading text. The heading's own numbering is applied automatically by the document generator, so the heading text must contain the title only — do NOT prepend "1.", "1.1", "2.", etc. into the heading text itself. For example, a Heading 1 titled "Introduction" must be passed as "Introduction", never as "1. Introduction" (which would render as "1. 1. Introduction"). The same rule applies at every level. -Do not repeat the document title as the first section heading. The document generator already renders the title as a centered title paragraph. Put any opening preamble text directly in the first section's content, without a duplicate heading such as "Agreement", "Contract", "Mutual Non-Disclosure Agreement", or another shortened form of the title. -Contracts: when generating a contract or agreement, always include a signatures block at the very end of the document on its own page. Set pageBreak: true on that final section so it starts on a fresh page, and include a signature line for each party — typically the party name followed by lines for "By:", "Name:", "Title:", and "Date:". The entire signature block must be plain unnumbered text: do NOT number the signatures heading, do NOT number or letter the introductory signature sentence, party names, "By:", "Name:", "Title:", or "Date:" lines, and do NOT place the signature block inside a numbered clause. Put the signature block in the section's content rather than as a numbered heading. -Contract preambles: the preamble of a contract (the opening recitals, parties block, "WHEREAS" clauses, and any introductory narrative before the first operative clause) must NOT be numbered. Render these as unnumbered content (plain paragraphs or an unnumbered heading), and begin numbering only at the first operative clause/section. - -DOCUMENT EDITING: -When using edit_document, any edit that adds, removes, or reorders a numbered clause, section, sub-clause, schedule, exhibit, or list item shifts every downstream number. You MUST update all affected numbering AND every cross-reference to those numbers in the same edit_document call: -- Renumber the sibling clauses/sections/sub-clauses that follow the change so the sequence stays contiguous (e.g. if you insert a new Section 4, existing Sections 4, 5, 6… become 5, 6, 7…). -- Find every in-document reference to the shifted numbers — e.g. "see Section 5", "pursuant to Clause 4.2(b)", "as set out in Schedule 3", "defined in Section 2.1" — and update them to the new numbers. Include defined-term blocks, cross-references in recitals, schedules, and exhibits. -- Before issuing the edits, scan the full document (use read_document or find_in_document) to enumerate affected cross-references; do not assume references only appear near the change site. -- If you are uncertain whether a reference points to the shifted number or an unrelated number, err on the side of including it as an edit and explain in the reason field. -- When deleting square brackets, delete both the opening \`[\` and the closing \`]\`. Never leave behind an unmatched square bracket after an edit. - -WORKFLOWS: -When a user message begins with a [Workflow: (id: <id>)] marker, the user has selected a workflow and you MUST apply it. Immediately call the read_workflow tool with that exact id to load the workflow's full prompt, then follow those instructions for the current turn. Do this before producing any other output or calling any other tools (aside from any document reads the workflow requires). Do not ask the user to confirm — the selection itself is the instruction to apply the workflow. - -DOCUMENT NAMING IN PROSE: -The chat-local labels ("doc-0", "doc-1", "doc-N", …) are internal handles for tool calls and citation JSON ONLY. NEVER write them in your prose response or in any text the user reads — not in body text, not in headings, not in lists, not in tool-activity descriptions. The user does not know what "doc-0" means and seeing it is jarring. When referring to a document in prose, always use its filename (e.g. "the NDA draft" or "nda_v1.docx"). This rule applies to every word streamed back to the user; the only places "doc-N" identifiers are allowed are inside tool-call arguments and inside the <CITATIONS> JSON block's "doc_id" field. - -GENERAL GUIDANCE: -- Be precise and professional -- Cite the specific document and quote when making claims about document content -- When no documents are provided, answer based on your legal knowledge -- Do not fabricate document content -- Do not use emojis in your responses. -`; - -export const PROJECT_EXTRA_TOOLS = [ - { - type: "function", - function: { - name: "list_documents", - description: - "List all documents available in the project. Returns each document's ID, filename, and file type. Call this to discover what documents are available before deciding which ones to read.", - parameters: { type: "object", properties: {} }, - }, - }, - { - type: "function", - function: { - name: "fetch_documents", - description: - "Read the full text content of multiple documents in a single call. Use this instead of calling read_document repeatedly when you need to read several documents at once.", - parameters: { - type: "object", - properties: { - doc_ids: { - type: "array", - items: { type: "string" }, - description: - "Array of document IDs to read (e.g. ['doc-0', 'doc-2'])", - }, - }, - required: ["doc_ids"], - }, - }, - }, - { - type: "function", - function: { - name: "replicate_document", - description: - "Make byte-for-byte copies of an existing project document as new project documents. Use when the user wants standalone copies to edit (e.g. 'use this NDA as a template', 'give me three drafts I can adapt') without modifying the original. Pass `count` to create multiple copies in a single call rather than calling the tool repeatedly. Returns the new doc_id slugs so you can immediately call edit_document / read_document on them.", - parameters: { - type: "object", - properties: { - doc_id: { - type: "string", - description: - "ID of the source document to copy (e.g. 'doc-0').", - }, - count: { - type: "integer", - description: - "How many copies to create. Defaults to 1. Maximum 20.", - minimum: 1, - maximum: 20, - }, - new_filename: { - type: "string", - description: - "Optional base filename. With count > 1, copies are suffixed (e.g. 'Foo (1).docx', 'Foo (2).docx'). Extension is forced to match the source.", - }, - }, - required: ["doc_id"], - }, - }, - }, -]; - -export const TABULAR_TOOLS = [ - { - type: "function", - function: { - name: "read_table_cells", - description: - "Read the extracted cell content from the tabular review. Each cell contains the value extracted for a specific column from a specific document. Pass col_indices and/or row_indices (0-based) to read a subset; omit either to read all columns or all rows.", - parameters: { - type: "object", - properties: { - col_indices: { - type: "array", - items: { type: "integer" }, - description: - "0-based column indices to read (e.g. [0, 2]). Omit to read all columns.", - }, - row_indices: { - type: "array", - items: { type: "integer" }, - description: - "0-based document (row) indices to read (e.g. [0, 1]). Omit to read all rows.", - }, - }, - }, - }, - }, -]; - -export const WORKFLOW_TOOLS = [ - { - type: "function", - function: { - name: "list_workflows", - description: - "List all workflows available to the user. Returns each workflow's ID and title. Call this when the user asks to run a workflow, apply a template, or you need to discover what workflows exist.", - parameters: { type: "object", properties: {} }, - }, - }, - { - type: "function", - function: { - name: "read_workflow", - description: - "Read the full instructions (prompt) of a workflow by its ID. Call this after list_workflows to load a specific workflow's prompt, then follow those instructions.", - parameters: { - type: "object", - properties: { - workflow_id: { - type: "string", - description: "The workflow ID to read", - }, - }, - required: ["workflow_id"], - }, - }, - }, -]; - -export const TOOLS = [ - { - type: "function", - function: { - name: "read_document", - description: - "Read the full text content of a document attached by the user. Always call this before answering questions about, summarising, or citing from a document.", - parameters: { - type: "object", - properties: { - doc_id: { - type: "string", - description: - "The document ID to read (e.g. 'doc-0', 'doc-1')", - }, - }, - required: ["doc_id"], - }, - }, - }, - { - type: "function", - function: { - name: "find_in_document", - description: - "Search for specific strings inside a document — a Ctrl+F equivalent. Returns each match with surrounding context so you can locate and quote the exact text without reading the whole document. Matching is case-insensitive and whitespace-tolerant. Use this for targeted lookups (e.g. finding a clause title, party name, or a specific phrase) rather than reading the whole document.", - parameters: { - type: "object", - properties: { - doc_id: { - type: "string", - description: - "The document ID to search (e.g. 'doc-0').", - }, - query: { - type: "string", - description: - "The string to search for. Matching is case-insensitive and collapses runs of whitespace, so 'Section 4.2' matches 'section 4.2'.", - }, - max_results: { - type: "integer", - description: - "Maximum number of matches to return (default 20). Use a smaller value for common terms.", - }, - context_chars: { - type: "integer", - description: - "Characters of surrounding context to include on each side of a match (default 80).", - }, - }, - required: ["doc_id", "query"], - }, - }, - }, - { - type: "function", - function: { - name: "generate_docx", - description: - "Generate a Word (.docx) document from structured content. Use this when the user asks you to draft, create, or produce a legal document. Returns a download URL for the generated file.", - parameters: { - type: "object", - properties: { - title: { - type: "string", - description: - "Document title (used as filename and heading)", - }, - landscape: { - type: "boolean", - description: - "Set to true for landscape page orientation. Default is portrait.", - }, - sections: { - type: "array", - description: - "List of document sections. Each section may contain a heading, prose content, or a table.", - items: { - type: "object", - properties: { - heading: { - type: "string", - description: "Optional section heading", - }, - level: { - type: "integer", - description: "Heading level: 1, 2, or 3", - }, - content: { - type: "string", - description: - "Prose text content (paragraphs separated by double newlines)", - }, - pageBreak: { - type: "boolean", - description: - "Set to true to start this section on a new page. Use for contract signature pages.", - }, - table: { - type: "object", - description: - "Optional table to render in this section", - properties: { - headers: { - type: "array", - items: { type: "string" }, - description: "Column header labels", - }, - rows: { - type: "array", - items: { - type: "array", - items: { type: "string" }, - }, - description: - "Array of rows, each row is an array of cell strings matching the headers order", - }, - }, - required: ["headers", "rows"], - }, - }, - }, - }, - }, - required: ["title", "sections"], - }, - }, - }, - { - type: "function", - function: { - name: "edit_document", - description: - "Propose edits to a user-attached .docx as tracked changes. Each edit is a precise, minimal substitution of specific words/characters, NOT a whole-line or paragraph replacement. Use read_document first. Anchor each edit with short before/after context so it can be located unambiguously. Returns per-edit annotations the UI will render as Accept/Reject cards and a download link to the edited document.", - parameters: { - type: "object", - properties: { - doc_id: { - type: "string", - description: "Document slug (e.g. 'doc-0').", - }, - edits: { - type: "array", - description: "List of precise substitutions.", - items: { - type: "object", - properties: { - find: { - type: "string", - description: - "Exact substring to replace (keep it as short as possible — ideally just the words/chars being changed).", - }, - replace: { - type: "string", - description: - "Replacement text. Empty string = pure deletion.", - }, - context_before: { - type: "string", - description: - "~40 chars immediately preceding `find`, used to disambiguate.", - }, - context_after: { - type: "string", - description: - "~40 chars immediately following `find`.", - }, - reason: { - type: "string", - description: - "Short explanation shown to the user on the card.", - }, - }, - required: [ - "find", - "replace", - "context_before", - "context_after", - ], - }, - }, - }, - required: ["doc_id", "edits"], - }, - }, - }, -]; - -type ParsedCitation = { - ref: number; - doc_id: string; - page: number | string; - quote: string; -}; - -function normalizeCitation(raw: unknown): ParsedCitation | null { - if (!raw || typeof raw !== "object") return null; - const c = raw as Record<string, unknown>; - const markerRef = - typeof c.marker === "string" - ? Number(c.marker.match(/^\[(\d+)\]$/)?.[1]) - : NaN; - const ref = - typeof c.ref === "number" - ? c.ref - : Number.isFinite(markerRef) - ? markerRef - : null; - if (typeof ref !== "number" || typeof c.doc_id !== "string") return null; - const quote = typeof c.quote === "string" ? c.quote : c.text; - if (typeof quote !== "string" || !quote) return null; - let page: number | string; - if (typeof c.page === "number") { - page = c.page; - } else if (typeof c.page === "string" && /^\d+\s*-\s*\d+$/.test(c.page)) { - page = c.page; - } else { - const n = parseInt(String(c.page ?? ""), 10); - if (!Number.isFinite(n)) page = 1; - else page = n; - } - return { ref, doc_id: c.doc_id, page, quote }; -} - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -export function resolveDoc(rawId: string, docIndex: DocIndex) { - return docIndex[rawId]; -} - -/** - * Resolve whatever identifier the model passed (`doc-N` slug, filename, or - * document UUID) back to a chat-local doc label. Generated docs surface in - * tool results with both `doc_id` (slug) and `document_id` (UUID), so the - * model often picks the wrong one — without this fallback `read_document` - * silently returns "not found" and the model gives up and re-generates. - */ -export function resolveDocLabel( - rawId: string, - docStore: DocStore, - docIndex?: DocIndex, -): string | null { - if (docStore.has(rawId)) return rawId; - for (const [label, info] of docStore.entries()) { - if (info.filename === rawId) return label; - } - if (docIndex) { - for (const [label, info] of Object.entries(docIndex)) { - if (info.document_id === rawId) return label; - } - } - return null; -} - -function citationReminder(docLabel: string, filename: string): string { - return [ - `[Citation requirement for ${docLabel} ("${filename}")]:`, - `If your final answer makes any factual claim from this document, include inline [N] markers and append a final <CITATIONS> JSON block.`, - `Every citation entry for this document MUST use "doc_id": "${docLabel}".`, - `Use this exact citation object shape: {"ref": 1, "doc_id": "${docLabel}", "page": 1, "quote": "exact verbatim text from the document"}.`, - `Do not use "marker" or "text" keys in the citation block; use "ref" and "quote".`, - ].join("\n"); -} - -/** - * Append a tool-activity summary to the most recent assistant message so - * the model can see what it just did (read / create / edit / workflow - * applied) in the prior turn — otherwise it only sees its own prose and - * forgets which docs it touched, which leads to e.g. re-generating a doc - * that already exists. - * - * Doc references use the *current-turn* `doc_id` slug (looked up by - * matching the event's stored `document_id` against this turn's freshly - * built `docIndex`), since slugs are reassigned every turn and the old - * slug from the prior turn would be meaningless. Falls back to filename - * only if the doc is no longer in the index (deleted, scope changed). - */ -export async function enrichWithPriorEvents( - messages: ChatMessage[], - chatId: string | null | undefined, - db: ReturnType<typeof createServerSupabase>, - docIndex: DocIndex, -): Promise<ChatMessage[]> { - if (!chatId) return messages; - const { data: rows } = await db - .from("chat_messages") - .select("content, created_at") - .eq("chat_id", chatId) - .eq("role", "assistant") - .order("created_at", { ascending: false }) - .limit(1); - - const lastRow = rows?.[0] as { content?: unknown } | undefined; - const content = lastRow?.content; - if (!Array.isArray(content)) return messages; - - const slugByDocumentId = new Map<string, string>(); - for (const [slug, info] of Object.entries(docIndex)) { - if (info.document_id) slugByDocumentId.set(info.document_id, slug); - } - const refFor = (documentId: unknown, filename: unknown) => { - const slug = - typeof documentId === "string" - ? slugByDocumentId.get(documentId) - : undefined; - return slug ? `${slug} ("${filename}")` : `"${filename}"`; - }; - - const lines: string[] = []; - for (const ev of content as Record<string, unknown>[]) { - if (ev?.type === "doc_created") { - lines.push( - `- generate_docx → ${refFor(ev.document_id, ev.filename)}`, - ); - } else if (ev?.type === "doc_edited") { - lines.push( - `- edit_document → ${refFor(ev.document_id, ev.filename)}`, - ); - } else if (ev?.type === "doc_read") { - lines.push( - `- read_document → ${refFor(ev.document_id, ev.filename)}`, - ); - } else if (ev?.type === "doc_replicated") { - // The model needs to know what each copy resolved to so it - // can call edit_document / read_document on them. Emit one - // line per copy, all attributed back to the same source. - const srcLabel = - typeof ev.filename === "string" ? `"${ev.filename}"` : ""; - const copies = Array.isArray(ev.copies) - ? (ev.copies as { - new_filename?: unknown; - document_id?: unknown; - }[]) - : []; - for (const c of copies) { - const ref = refFor(c.document_id, c.new_filename); - lines.push( - srcLabel - ? `- replicate_document → ${ref} (copy of ${srcLabel})` - : `- replicate_document → ${ref}`, - ); - } - } else if (ev?.type === "workflow_applied") { - lines.push(`- applied workflow: "${ev.title}"`); - } - } - if (lines.length === 0) return messages; - const summary = `\n\n[Tool activity in your previous turn]\n${lines.join("\n")}`; - - // Find the index of the last assistant message and attach the - // summary there only. - let lastAssistantIdx = -1; - for (let i = messages.length - 1; i >= 0; i--) { - if (messages[i].role === "assistant") { - lastAssistantIdx = i; - break; - } - } - if (lastAssistantIdx < 0) return messages; - const enriched = messages.slice(); - const target = enriched[lastAssistantIdx]; - enriched[lastAssistantIdx] = { - ...target, - content: (target.content ?? "") + summary, - }; - return enriched; -} - -export function buildMessages( - messages: ChatMessage[], - docAvailability: { - doc_id: string; - filename: string; - folder_path?: string; - }[], - systemPromptExtra?: string, - docIndex?: DocIndex, -) { - const formatted: unknown[] = []; - let systemContent = SYSTEM_PROMPT; - - if (systemPromptExtra) { - systemContent += `\n\n${systemPromptExtra.trim()}`; - } - - if (docAvailability.length) { - systemContent += "\n\n---\nAVAILABLE DOCUMENTS:\n"; - for (const doc of docAvailability) { - const label = doc.folder_path - ? `${doc.folder_path} / ${doc.filename}` - : doc.filename; - systemContent += `- ${doc.doc_id}: ${label}\n`; - } - systemContent += - "\nYou do NOT retain document content between conversation turns. You MUST call read_document (or fetch_documents) at the start of every response that involves a document's content, even if you have read it in a previous turn. Failure to do so will result in hallucinated or stale content.\n---\n"; - } - formatted.push({ role: "system", content: systemContent }); - - // Map document_id (UUID) → current-turn doc_id slug, so when we - // inline a user attachment we hand the model the same handle it - // would use to call read_document / fetch_documents. - const slugByDocumentId = new Map<string, string>(); - if (docIndex) { - for (const [slug, info] of Object.entries(docIndex)) { - if (info.document_id) slugByDocumentId.set(info.document_id, slug); - } - } - - for (const msg of messages) { - let content = msg.content ?? ""; - if (msg.role === "user" && msg.workflow) { - content = `[Workflow: ${msg.workflow.title} (id: ${msg.workflow.id})]\n\n${content}`; - } - if (msg.role === "user" && msg.files?.length) { - const lines = msg.files.map((f) => { - const slug = f.document_id - ? slugByDocumentId.get(f.document_id) - : undefined; - return slug ? `- ${slug}: ${f.filename}` : `- ${f.filename}`; - }); - content = `[The user attached the following document(s) to this message:\n${lines.join("\n")}]\n\n${content}`; - } - formatted.push({ role: msg.role, content }); - } - return formatted; -} - -export async function extractPdfText(buf: ArrayBuffer): Promise<string> { - try { - const pdfjsLib = await import( - "pdfjs-dist/legacy/build/pdf.mjs" as string - ); - const pdf = await ( - pdfjsLib as unknown as { - getDocument: (opts: unknown) => { - promise: Promise<{ - numPages: number; - getPage: (n: number) => Promise<{ - getTextContent: () => Promise<{ - items: { str?: string }[]; - }>; - }>; - }>; - }; - } - ).getDocument({ - data: new Uint8Array(buf), - standardFontDataUrl: STANDARD_FONT_DATA_URL, - }).promise; - const parts: string[] = []; - for (let i = 1; i <= pdf.numPages; i++) { - const page = await pdf.getPage(i); - const textContent = await page.getTextContent(); - parts.push( - `[Page ${i}]\n${textContent.items.map((it) => it.str ?? "").join(" ")}`, - ); - } - return parts.join("\n\n"); - } catch { - return ""; - } -} - -export async function generateDocx( - title: string, - sections: unknown[], - userId: string, - db: ReturnType<typeof createServerSupabase>, - options?: { landscape?: boolean; projectId?: string | null }, -) { - try { - const { - Document, - Paragraph, - HeadingLevel, - Packer, - Table, - TableRow, - TableCell, - WidthType, - BorderStyle, - TextRun, - AlignmentType, - LevelFormat, - LevelSuffix, - PageOrientation, - PageBreak, - } = await import("docx"); - - const FONT = "Times New Roman"; - const SIZE = 22; // 11pt in half-points - - type DocChild = - | InstanceType<typeof Paragraph> - | InstanceType<typeof Table>; - const children: DocChild[] = []; - children.push( - new Paragraph({ - heading: HeadingLevel.TITLE, - spacing: { after: 200 }, - alignment: AlignmentType.CENTER, - children: [ - new TextRun({ - text: title.toUpperCase(), - color: "000000", - font: FONT, - size: SIZE, - bold: true, - }), - ], - }), - ); - - const cellBorder = { - top: { style: BorderStyle.SINGLE, size: 1, color: "CCCCCC" }, - bottom: { style: BorderStyle.SINGLE, size: 1, color: "CCCCCC" }, - left: { style: BorderStyle.SINGLE, size: 1, color: "CCCCCC" }, - right: { style: BorderStyle.SINGLE, size: 1, color: "CCCCCC" }, - }; - - const headingLevels = [ - HeadingLevel.HEADING_1, - HeadingLevel.HEADING_2, - HeadingLevel.HEADING_3, - HeadingLevel.HEADING_4, - ]; - const LEGAL_NUMBERING_REF = "legal-clause-numbering"; - const legalNumbering = (level: number) => ({ - reference: LEGAL_NUMBERING_REF, - level: Math.max(0, Math.min(level, 4)), - }); - const legalNumberingLevels = [ - { - level: 0, - format: LevelFormat.DECIMAL, - text: "%1.", - alignment: AlignmentType.START, - suffix: LevelSuffix.TAB, - isLegalNumberingStyle: true, - style: { - paragraph: { indent: { left: 720, hanging: 720 } }, - run: { - bold: true, - color: "000000", - font: FONT, - size: SIZE, - }, - }, - }, - { - level: 1, - format: LevelFormat.DECIMAL, - text: "%1.%2", - alignment: AlignmentType.START, - suffix: LevelSuffix.TAB, - isLegalNumberingStyle: true, - style: { - paragraph: { indent: { left: 720, hanging: 720 } }, - run: { color: "000000", font: FONT, size: SIZE }, - }, - }, - { - level: 2, - format: LevelFormat.LOWER_LETTER, - text: "(%3)", - alignment: AlignmentType.START, - suffix: LevelSuffix.TAB, - style: { - paragraph: { indent: { left: 1440, hanging: 720 } }, - run: { color: "000000", font: FONT, size: SIZE }, - }, - }, - { - level: 3, - format: LevelFormat.LOWER_ROMAN, - text: "(%4)", - alignment: AlignmentType.START, - suffix: LevelSuffix.TAB, - style: { - paragraph: { indent: { left: 1440, hanging: 720 } }, - run: { color: "000000", font: FONT, size: SIZE }, - }, - }, - { - level: 4, - format: LevelFormat.UPPER_LETTER, - text: "(%5)", - alignment: AlignmentType.START, - suffix: LevelSuffix.TAB, - style: { - paragraph: { indent: { left: 2520, hanging: 720 } }, - run: { color: "000000", font: FONT, size: SIZE }, - }, - }, - ]; - const normalizeTable = ( - table: unknown, - ): { headers: string[]; rows: string[][] } | null => { - if (!table || typeof table !== "object") return null; - const raw = table as { headers?: unknown; rows?: unknown }; - const headers = Array.isArray(raw.headers) - ? raw.headers - .map((header) => - typeof header === "string" ? header.trim() : "", - ) - .filter(Boolean) - : []; - if (headers.length === 0) return null; - - const rawRows = Array.isArray(raw.rows) ? raw.rows : []; - const rows = rawRows - .filter((row): row is unknown[] => Array.isArray(row)) - .map((row) => - headers.map((_, i) => - typeof row[i] === "string" ? row[i] : "", - ), - ); - - return { headers, rows }; - }; - const stripManualNumbering = ( - value: string, - ): { text: string; levelFromPrefix: number | null } => { - const match = value - .trim() - .match(/^(\d+(?:\.\d+)*)(?:[.)])?\s+(.+)$/); - if (!match) return { text: value.trim(), levelFromPrefix: null }; - return { - text: match[2].trim(), - levelFromPrefix: match[1].split(".").length - 1, - }; - }; - const parseManualListMarker = ( - value: string, - ): { text: string; levelOffset: number | null } => { - const trimmed = value.trim(); - const match = trimmed.match(/^(\(([a-z]+)\)|([a-z]+)[.)])\s+(.+)$/i); - if (!match) return { text: trimmed, levelOffset: null }; - const marker = (match[2] ?? match[3] ?? "").toLowerCase(); - const isRoman = - marker === "i" || - (marker.length > 1 && - /^(?:m{0,4}(?:cm|cd|d?c{0,3})(?:xc|xl|l?x{0,3})(?:ix|iv|v?i{0,3}))$/i.test( - marker, - )); - return { text: match[4].trim(), levelOffset: isRoman ? 3 : 2 }; - }; - const normalizeHeadingText = (value: string) => - value - .trim() - .replace(/[^a-zA-Z0-9]+/g, " ") - .trim() - .toLowerCase(); - - const isTitleLikeFirstHeading = ( - heading: string, - sectionIndex: number, - ) => { - if (sectionIndex !== 0) return false; - const normalized = normalizeHeadingText(heading); - const titleNormalized = normalizeHeadingText(title); - if (!normalized || !titleNormalized) return false; - if (normalized === titleNormalized) return true; - return ( - titleNormalized.includes(normalized) && - /\b(agreement|contract|deed|terms|policy|notice|nda|disclosure)\b/.test( - normalized, - ) - ); - }; - - const isUnnumberedHeading = (heading: string, sectionIndex: number) => { - const normalized = normalizeHeadingText(heading); - if (!normalized) return true; - if (normalized === "signatures" || normalized === "signature") { - return true; - } - if (isTitleLikeFirstHeading(heading, sectionIndex)) { - return true; - } - if ( - sectionIndex === 0 && - /^(agreement|contract|mutual non disclosure agreement|non disclosure agreement|employment agreement|service level agreement)$/.test( - normalized, - ) - ) { - return true; - } - return false; - }; - const isSignatureLine = (value: string) => - /^(?:by|name|title|date):\s*/i.test(value.trim()); - const looksLikeSignatureBlock = (value: string) => { - const lines = value - .split("\n") - .map((line) => line.trim()) - .filter(Boolean); - if (lines.length === 0) return false; - const signatureLineCount = lines.filter(isSignatureLine).length; - return signatureLineCount >= 2; - }; - let currentClauseLevel: number | null = null; - - for (const [sectionIndex, section] of (sections as { - heading?: string; - content?: string; - level?: number; - pageBreak?: boolean; - table?: { headers: string[]; rows: string[][] }; - }[]).entries()) { - if (section.pageBreak) { - children.push(new Paragraph({ children: [new PageBreak()] })); - } - if (section.heading) { - const stripped = stripManualNumbering(section.heading); - const isUnnumbered = isUnnumberedHeading( - stripped.text, - sectionIndex, - ); - const skipHeading = isTitleLikeFirstHeading( - stripped.text, - sectionIndex, - ); - const idx = Math.min( - stripped.levelFromPrefix ?? (section.level ?? 1) - 1, - 3, - ); - currentClauseLevel = isUnnumbered || skipHeading ? null : idx; - const headingText = - idx === 0 && !isUnnumbered - ? stripped.text.toUpperCase() - : stripped.text; - if (!skipHeading) { - children.push( - new Paragraph({ - heading: headingLevels[idx], - numbering: isUnnumbered - ? undefined - : legalNumbering(idx), - spacing: { after: 160 }, - children: [ - new TextRun({ - text: headingText, - color: "000000", - font: FONT, - size: SIZE, - bold: true, - }), - ], - }), - ); - } - } - const normalizedTable = normalizeTable(section.table); - if (normalizedTable) { - const { headers, rows } = normalizedTable; - const colCount = headers.length; - const tableRows: InstanceType<typeof TableRow>[] = []; - // Header row - tableRows.push( - new TableRow({ - tableHeader: true, - children: headers.map( - (h) => - new TableCell({ - borders: cellBorder, - shading: { fill: "F2F2F2" }, - children: [ - new Paragraph({ - children: [ - new TextRun({ - text: h, - bold: true, - font: FONT, - size: SIZE, - }), - ], - alignment: AlignmentType.LEFT, - }), - ], - }), - ), - }), - ); - // Data rows — normalize each row to exactly colCount cells. - // LLMs occasionally emit malformed rows (extra fragments from - // stray delimiters, or short rows); padding/truncating here - // keeps the rendered table aligned to the headers. - for (const normalized of rows) { - tableRows.push( - new TableRow({ - children: normalized.map( - (cell) => - new TableCell({ - borders: cellBorder, - children: [ - new Paragraph({ - children: [ - new TextRun({ - text: cell, - font: FONT, - size: SIZE, - }), - ], - }), - ], - }), - ), - }), - ); - } - children.push( - new Table({ - width: { size: 100, type: WidthType.PERCENTAGE }, - rows: tableRows, - }), - ); - children.push(new Paragraph({ text: "" })); - } - if (section.content) { - let numberedBodyParagraphs = 0; - const contentIsSignatureBlock = - section.heading && - normalizeHeadingText(section.heading).includes("signature") - ? true - : looksLikeSignatureBlock(section.content); - for (const line of section.content.split("\n")) { - const trimmed = line.trim(); - if (!trimmed) continue; - const bulletMatch = trimmed.match(/^[-•*]\s+(.+)/); - const rawText = bulletMatch - ? bulletMatch[1].trim() - : trimmed; - const manualList = parseManualListMarker(rawText); - const numeric = stripManualNumbering(rawText); - const text = bulletMatch - ? rawText - : manualList.levelOffset !== null - ? manualList.text - : numeric.text; - const inferredLevel = - currentClauseLevel === null || contentIsSignatureBlock - ? undefined - : bulletMatch - ? currentClauseLevel + 2 - : manualList.levelOffset !== null - ? currentClauseLevel + manualList.levelOffset - : numeric.levelFromPrefix !== null - ? numeric.levelFromPrefix - : numberedBodyParagraphs === 0 - ? currentClauseLevel + 1 - : currentClauseLevel + 2; - if (currentClauseLevel !== null) numberedBodyParagraphs++; - children.push( - new Paragraph({ - numbering: - inferredLevel === undefined - ? undefined - : legalNumbering(inferredLevel), - spacing: { after: 120 }, - children: [ - new TextRun({ - text, - font: FONT, - size: SIZE, - }), - ], - }), - ); - } - } - } - - const pageSetup = options?.landscape - ? { page: { size: { orientation: PageOrientation.LANDSCAPE } } } - : {}; - - const doc = new Document({ - numbering: { - config: [ - { - reference: LEGAL_NUMBERING_REF, - levels: legalNumberingLevels, - }, - ], - }, - sections: [{ properties: pageSetup, children }], - }); - const buf = await Packer.toBuffer(doc); - const zip = await import("jszip"); - const packageZip = await zip.default.loadAsync(buf); - for (const requiredPath of [ - "[Content_Types].xml", - "word/document.xml", - "word/_rels/document.xml.rels", - ]) { - if (!packageZip.file(requiredPath)) { - return { - error: `Generated DOCX is missing required package part: ${requiredPath}`, - }; - } - } - const docId = crypto.randomUUID().replace(/-/g, ""); - const safeTitle = - title - .replace(/[^a-zA-Z0-9 -]/g, "") - .trim() - .slice(0, 64) || "document"; - const filename = `${safeTitle}.docx`; - const key = generatedDocKey(userId, docId, filename); - - await uploadFile( - key, - buf.buffer as ArrayBuffer, - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - ); - const downloadUrl = buildDownloadUrl(key, filename); - - // Persist to DB so generated docs are first-class documents: - // openable in the DocPanel and editable via edit_document. In - // project chats we attach to the project so it appears in the - // sidebar; in the general chat we leave project_id null and it - // stays a standalone document. - const { data: docRow, error: docErr } = await db - .from("documents") - .insert({ - project_id: options?.projectId ?? null, - user_id: userId, - filename, - file_type: "docx", - size_bytes: buf.byteLength, - status: "ready", - }) - .select("id") - .single(); - if (docErr || !docRow) { - return { - error: `Failed to record generated document: ${docErr?.message ?? "unknown"}`, - }; - } - const documentId = docRow.id as string; - - const { data: versionRow, error: verErr } = await db - .from("document_versions") - .insert({ - document_id: documentId, - storage_path: key, - source: "generated", - version_number: 1, - display_name: filename, - }) - .select("id") - .single(); - if (verErr || !versionRow) { - return { - error: `Failed to record generated document version: ${verErr?.message ?? "unknown"}`, - }; - } - const versionId = versionRow.id as string; - - await db - .from("documents") - .update({ current_version_id: versionId }) - .eq("id", documentId); - - return { - filename, - download_url: downloadUrl, - document_id: documentId, - version_id: versionId, - version_number: 1, - storage_path: key, - message: `Document '${filename}' has been generated successfully.`, - }; - } catch (e) { - return { error: String(e) }; - } -} - -// --------------------------------------------------------------------------- -// Document version helpers (DOCX tracked-change editing) -// --------------------------------------------------------------------------- - -/** - * Resolve the current .docx bytes for a document, preferring the active - * tracked-changes version if one exists, else the original upload. - */ -export async function loadCurrentVersionBytes( - documentId: string, - db: ReturnType<typeof createServerSupabase>, -): Promise<{ bytes: Buffer; storage_path: string } | null> { - const active = await loadActiveVersion(documentId, db); - if (!active) return null; - const raw = await downloadFile(active.storage_path); - if (!raw) return null; - return { bytes: Buffer.from(raw), storage_path: active.storage_path }; -} - -/** - * Ensure the document has a document_versions row for the current upload. - * Called before writing the first 'assistant_edit' row so the history is - * complete. Idempotent. - */ -export async function runEditDocument(params: { - documentId: string; - userId: string; - edits: EditInput[]; - db: ReturnType<typeof createServerSupabase>; - /** - * If provided, append these edits to the existing turn-scoped version - * (overwrites the file at storagePath and reuses the document_versions - * row) instead of creating a new version. Used to collapse multiple - * edit_document tool calls within a single assistant turn into one - * version. - */ - reuseVersion?: { - versionId: string; - versionNumber: number; - storagePath: string; - }; -}): Promise< - | { - ok: true; - version_id: string; - version_number: number; - storage_path: string; - download_url: string; - annotations: EditAnnotation[]; - errors: { index: number; reason: string }[]; - } - | { ok: false; error: string } -> { - const { documentId, userId, edits, db, reuseVersion } = params; - - const { data: doc } = await db - .from("documents") - .select("id, filename") - .eq("id", documentId) - .single(); - if (!doc) return { ok: false, error: "Document not found." }; - - const current = await loadCurrentVersionBytes(documentId, db); - if (!current) return { ok: false, error: "Could not load document bytes." }; - - const { - bytes: editedBytes, - changes, - errors, - } = await applyTrackedEdits(current.bytes, edits, { author: "Mike" }); - - if (changes.length === 0) { - return { - ok: false, - error: - errors[0]?.reason ?? - "No edits could be applied. Refine context_before/context_after and retry.", - }; - } - - const ab = editedBytes.buffer.slice( - editedBytes.byteOffset, - editedBytes.byteOffset + editedBytes.byteLength, - ) as ArrayBuffer; - - let versionRowId: string; - let newPath: string; - let nextVersionNumber: number; - - if (reuseVersion) { - // Overwrite the existing turn version's file in place. The version - // row, version_number, and current_version_id all already point here. - newPath = reuseVersion.storagePath; - versionRowId = reuseVersion.versionId; - nextVersionNumber = reuseVersion.versionNumber; - await uploadFile( - newPath, - ab, - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - ); - } else { - const versionId = crypto.randomUUID().replace(/-/g, ""); - newPath = `documents/${userId}/${documentId}/edits/${versionId}.docx`; - await uploadFile( - newPath, - ab, - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - ); - - // Per-document sequential number for the new assistant_edit - // version. The counter spans upload + user_upload + assistant_edit - // so the original upload is V1 and the first assistant edit is V2. - const { data: maxRow } = await db - .from("document_versions") - .select("version_number") - .eq("document_id", documentId) - .in("source", ["upload", "user_upload", "assistant_edit"]) - .order("version_number", { ascending: false, nullsFirst: false }) - .limit(1) - .maybeSingle(); - nextVersionNumber = - ((maxRow?.version_number as number | null) ?? 1) + 1; - - // Inherit the display name from the most recent prior version so - // user-applied renames carry forward through further edits. Falls - // back to the parent document's filename when no prior version has - // a display name (e.g. the first assistant edit of a pre-existing - // doc). We intentionally do NOT append "[Edited Vn]" — the version - // number is surfaced separately as a tag in the UI. - const { data: prevRow } = await db - .from("document_versions") - .select("display_name, created_at") - .eq("document_id", documentId) - .order("created_at", { ascending: false }) - .limit(1) - .maybeSingle(); - const inheritedDisplayName = - (prevRow?.display_name as string | null) ?? - (doc.filename as string | null) ?? - null; - - const { data: versionRow, error: verErr } = await db - .from("document_versions") - .insert({ - document_id: documentId, - storage_path: newPath, - source: "assistant_edit", - version_number: nextVersionNumber, - display_name: inheritedDisplayName, - }) - .select("id") - .single(); - if (verErr || !versionRow) { - return { ok: false, error: "Failed to record document version." }; - } - versionRowId = versionRow.id as string; - } - - // Insert one row per change - const editRows = changes.map((c) => ({ - document_id: documentId, - version_id: versionRowId, - change_id: c.id, - del_w_id: c.delId ?? null, - ins_w_id: c.insId ?? null, - deleted_text: c.deletedText, - inserted_text: c.insertedText, - context_before: c.contextBefore ?? "", - context_after: c.contextAfter ?? "", - status: "pending" as const, - })); - const { data: insertedEdits, error: editsErr } = await db - .from("document_edits") - .insert(editRows) - .select( - "id, change_id, del_w_id, ins_w_id, deleted_text, inserted_text, context_before, context_after", - ); - - if (editsErr || !insertedEdits) { - return { ok: false, error: "Failed to record edits." }; - } - - await db - .from("documents") - .update({ current_version_id: versionRowId }) - .eq("id", documentId); - - const annotations: EditAnnotation[] = insertedEdits.map( - (r: { - id: string; - change_id: string; - deleted_text: string; - inserted_text: string; - context_before: string | null; - context_after: string | null; - }) => { - const src = changes.find((c) => c.id === r.change_id); - return { - kind: "edit", - edit_id: r.id, - document_id: documentId, - version_id: versionRowId, - version_number: nextVersionNumber, - change_id: r.change_id, - del_w_id: src?.delId, - ins_w_id: src?.insId, - deleted_text: r.deleted_text ?? "", - inserted_text: r.inserted_text ?? "", - context_before: r.context_before ?? "", - context_after: r.context_after ?? "", - reason: src?.reason, - status: "pending", - }; - }, - ); - - // Persistent, non-expiring permalink. The backend streams fresh bytes - // on each request, so this URL stays valid as long as the file exists. - const permalink = buildDownloadUrl(newPath, doc.filename as string); - - return { - ok: true, - version_id: versionRowId, - version_number: nextVersionNumber, - storage_path: newPath, - download_url: permalink, - annotations, - errors, - }; -} - -// --------------------------------------------------------------------------- -// Tool dispatch -// --------------------------------------------------------------------------- - -async function readDocumentContent( - docLabel: string, - docStore: DocStore, - write: (s: string) => void, - docIndex?: DocIndex, - db?: ReturnType<typeof createServerSupabase>, - opts?: { emitEvents?: boolean }, -): Promise<string> { - const emitEvents = opts?.emitEvents ?? true; - console.log(`[read_document] called with docLabel="${docLabel}"`); - const docInfo = docStore.get(docLabel); - if (!docInfo) { - console.log( - `[read_document] MISS — docLabel "${docLabel}" not in docStore. Known labels:`, - Array.from(docStore.keys()), - ); - return "Document not found."; - } - console.log( - `[read_document] docInfo: filename="${docInfo.filename}", file_type="${docInfo.file_type}", storage_path="${docInfo.storage_path}"`, - ); - - const documentId = docIndex?.[docLabel]?.document_id; - const emitDocRead = () => { - if (!emitEvents) return; - write( - `data: ${JSON.stringify({ - type: "doc_read", - filename: docInfo.filename, - document_id: documentId, - })}\n\n`, - ); - }; - if (emitEvents) - write( - `data: ${JSON.stringify({ - type: "doc_read_start", - filename: docInfo.filename, - document_id: documentId, - })}\n\n`, - ); - try { - // Prefer the current tracked-changes version (if any) so read_document - // reflects accepted/pending edits rather than the original upload. - let raw: ArrayBuffer | null = null; - let sourcePath = docInfo.storage_path; - if (documentId && db) { - const current = await loadCurrentVersionBytes(documentId, db); - if (current) { - raw = current.bytes.buffer.slice( - current.bytes.byteOffset, - current.bytes.byteOffset + current.bytes.byteLength, - ) as ArrayBuffer; - sourcePath = current.storage_path; - console.log( - `[read_document] using current version path="${sourcePath}" (bytes=${raw.byteLength})`, - ); - } else { - console.log( - `[read_document] loadCurrentVersionBytes returned null for documentId="${documentId}", falling back to original storage_path`, - ); - } - } - if (!raw) { - raw = await downloadFile(docInfo.storage_path); - if (raw) { - console.log( - `[read_document] fallback download from storage_path="${docInfo.storage_path}" (bytes=${raw.byteLength})`, - ); - } - } - if (!raw) { - console.log( - `[read_document] FAILED to download any bytes for docLabel="${docLabel}" (tried path="${sourcePath}")`, - ); - emitDocRead(); - return "Document could not be read."; - } - // Log the first 8 bytes so we can identify real file format regardless - // of the declared file_type. Valid .docx starts with "PK\x03\x04" - // (zip). Legacy .doc starts with "\xD0\xCF\x11\xE0" (OLE/CFB). - // %PDF-1 is a PDF even if mislabeled. Truncated uploads show as all-zero. - { - const head = Buffer.from(raw).subarray(0, 8); - const hex = head.toString("hex"); - const ascii = head.toString("binary").replace(/[^\x20-\x7e]/g, "."); - console.log( - `[read_document] magic bytes hex=${hex} ascii="${ascii}" for filename="${docInfo.filename}"`, - ); - } - let text: string; - if (docInfo.file_type === "pdf") { - text = await extractPdfText(raw); - console.log( - `[read_document] pdf extracted length=${text.length} for filename="${docInfo.filename}"`, - ); - } else if (docInfo.file_type === "docx") { - // Use the same flattening as the edit_document matcher so the - // LLM sees exactly the characters it can anchor against. - text = await extractDocxBodyText(Buffer.from(raw)); - console.log( - `[read_document] docx extractDocxBodyText length=${text.length} for filename="${docInfo.filename}"`, - ); - if (!text) { - console.log( - `[read_document] docx accepted-view extractor returned empty, falling back to mammoth for filename="${docInfo.filename}"`, - ); - const mammoth = await import("mammoth"); - const result = await mammoth.extractRawText({ - buffer: Buffer.from(raw), - }); - text = result.value; - console.log( - `[read_document] docx mammoth fallback length=${text.length} for filename="${docInfo.filename}"`, - ); - } - } else { - console.log( - `[read_document] unknown file_type="${docInfo.file_type}" for filename="${docInfo.filename}", trying mammoth`, - ); - const mammoth = await import("mammoth"); - const result = await mammoth.extractRawText({ - buffer: Buffer.from(raw), - }); - text = result.value; - console.log( - `[read_document] mammoth length=${text.length} for filename="${docInfo.filename}"`, - ); - } - console.log( - `[read_document] DONE filename="${docInfo.filename}" finalTextLength=${text.length} firstChars=${JSON.stringify(text.slice(0, 120))}`, - ); - emitDocRead(); - return text; - } catch (err) { - console.log( - `[read_document] THREW for docLabel="${docLabel}" filename="${docInfo.filename}":`, - err, - ); - if (emitEvents) - write( - `data: ${JSON.stringify({ type: "doc_read", filename: docInfo.filename })}\n\n`, - ); - return "Document could not be read."; - } -} - -/** - * Build a whitespace-collapsed, lowercased copy of `text`, plus a map from - * each character index in the normalized form back to the corresponding - * index in the original text. Used by `findInDocumentContent` so matches - * are tolerant of case + whitespace variance but can still return the - * exact original excerpt. - */ -function normalizeWithMap(text: string): { norm: string; origIdx: number[] } { - const norm: string[] = []; - const origIdx: number[] = []; - let prevSpace = false; - for (let i = 0; i < text.length; i++) { - const ch = text[i]; - if (/\s/.test(ch)) { - if (!prevSpace) { - norm.push(" "); - origIdx.push(i); - prevSpace = true; - } - } else { - norm.push(ch.toLowerCase()); - origIdx.push(i); - prevSpace = false; - } - } - return { norm: norm.join(""), origIdx }; -} - -function normalizeQuery(q: string): string { - return q.trim().replace(/\s+/g, " ").toLowerCase(); -} - -/** - * Ctrl+F helper. Returns a JSON-serializable result with up to `maxResults` - * hits, each containing the original-text excerpt plus surrounding context. - */ -async function findInDocumentContent(params: { - docLabel: string; - query: string; - maxResults?: number; - contextChars?: number; - docStore: DocStore; - write: (s: string) => void; - docIndex?: DocIndex; - db?: ReturnType<typeof createServerSupabase>; -}): Promise<string> { - const { - docLabel, - query, - maxResults = 20, - contextChars = 80, - docStore, - write, - docIndex, - db, - } = params; - - if (!query || !query.trim()) { - return JSON.stringify({ ok: false, error: "Empty query." }); - } - - const docInfo = docStore.get(docLabel); - if (!docInfo) { - return JSON.stringify({ - ok: false, - error: `Document '${docLabel}' not found.`, - }); - } - - // Announce the search to the UI, then reuse readDocumentContent for its - // fallbacks — but suppress its own doc_read events so the user only sees - // the doc_find block (not a competing doc_read block for the same op). - write( - `data: ${JSON.stringify({ - type: "doc_find_start", - filename: docInfo.filename, - query, - })}\n\n`, - ); - - const text = await readDocumentContent( - docLabel, - docStore, - write, - docIndex, - db, - { emitEvents: false }, - ); - if (!text || text === "Document could not be read.") { - write( - `data: ${JSON.stringify({ - type: "doc_find", - filename: docInfo.filename, - query, - total_matches: 0, - })}\n\n`, - ); - return JSON.stringify({ - ok: false, - filename: docInfo.filename, - error: "Document could not be read.", - }); - } - - const { norm, origIdx } = normalizeWithMap(text); - const needle = normalizeQuery(query); - if (!needle) { - return JSON.stringify({ - ok: false, - error: "Empty query after normalization.", - }); - } - - type Hit = { - index: number; - excerpt: string; - context: string; - }; - const hits: Hit[] = []; - let from = 0; - while (from <= norm.length - needle.length && hits.length < maxResults) { - const pos = norm.indexOf(needle, from); - if (pos < 0) break; - const endNormPos = pos + needle.length; - const origStart = origIdx[pos] ?? 0; - const origEnd = - endNormPos - 1 < origIdx.length - ? origIdx[endNormPos - 1] + 1 - : text.length; - const ctxStart = Math.max(0, origStart - contextChars); - const ctxEnd = Math.min(text.length, origEnd + contextChars); - hits.push({ - index: hits.length, - excerpt: text.slice(origStart, origEnd), - context: - (ctxStart > 0 ? "…" : "") + - text.slice(ctxStart, ctxEnd).replace(/\s+/g, " ").trim() + - (ctxEnd < text.length ? "…" : ""), - }); - from = pos + Math.max(1, needle.length); - } - - // Count total occurrences beyond the cap so the model knows whether to narrow the query. - let totalMatches = hits.length; - if (hits.length >= maxResults) { - let probe = from; - while (probe <= norm.length - needle.length) { - const pos = norm.indexOf(needle, probe); - if (pos < 0) break; - totalMatches++; - probe = pos + Math.max(1, needle.length); - } - } - - write( - `data: ${JSON.stringify({ - type: "doc_find", - filename: docInfo.filename, - query, - total_matches: totalMatches, - })}\n\n`, - ); - - return JSON.stringify({ - ok: true, - filename: docInfo.filename, - query, - total_matches: totalMatches, - returned: hits.length, - truncated: totalMatches > hits.length, - hits, - }); -} - -export type DocEditedResult = { - filename: string; - document_id: string; - version_id: string; - version_number: number | null; - download_url: string; - annotations: EditAnnotation[]; -}; - -export type TurnEditState = Map< - string, - { versionId: string; versionNumber: number; storagePath: string } ->; - -export type DocCreatedResult = { - filename: string; - download_url: string; - document_id?: string; - version_id?: string; - version_number?: number | null; -}; - -export type DocReplicatedResult = { - /** Filename of the source document being copied. */ - filename: string; - /** How many copies were produced in this single tool call. */ - count: number; - /** One entry per new copy. */ - copies: { - new_filename: string; - document_id: string; - version_id: string; - }[]; -}; - -export async function runToolCalls( - toolCalls: ToolCall[], - docStore: DocStore, - userId: string, - db: ReturnType<typeof createServerSupabase>, - write: (s: string) => void, - workflowStore?: WorkflowStore, - tabularStore?: TabularCellStore, - docIndex?: DocIndex, - turnEditState?: TurnEditState, - projectId?: string | null, -): Promise<{ - toolResults: unknown[]; - docsRead: { filename: string; document_id?: string }[]; - docsFound: { filename: string; query: string; total_matches: number }[]; - docsCreated: DocCreatedResult[]; - docsReplicated: DocReplicatedResult[]; - workflowsApplied: { workflow_id: string; title: string }[]; - docsEdited: DocEditedResult[]; -}> { - const toolResults: unknown[] = []; - const docsRead: { filename: string; document_id?: string }[] = []; - const docsFound: { - filename: string; - query: string; - total_matches: number; - }[] = []; - const docsCreated: DocCreatedResult[] = []; - const docsReplicated: DocReplicatedResult[] = []; - const workflowsApplied: { workflow_id: string; title: string }[] = []; - const docsEdited: DocEditedResult[] = []; - - for (const tc of toolCalls) { - let args: Record<string, unknown> = {}; - try { - args = JSON.parse(tc.function.arguments || "{}"); - } catch { - /* ignore */ - } - - if (tc.function.name === "read_document") { - const rawDocId = args.doc_id as string; - const docId = - resolveDocLabel(rawDocId, docStore, docIndex) ?? rawDocId; - const content = await readDocumentContent( - docId, - docStore, - write, - docIndex, - db, - ); - const filename = docStore.get(docId)?.filename; - const documentId = docIndex?.[docId]?.document_id; - if (filename) docsRead.push({ filename, document_id: documentId }); - toolResults.push({ - role: "tool", - tool_call_id: tc.id, - content: filename - ? `${citationReminder(docId, filename)}\n\n${content}` - : content, - }); - } else if (tc.function.name === "find_in_document") { - const rawDocId = args.doc_id as string; - const docId = - resolveDocLabel(rawDocId, docStore, docIndex) ?? rawDocId; - const query = (args.query as string) ?? ""; - const maxResults = - typeof args.max_results === "number" - ? args.max_results - : undefined; - const contextChars = - typeof args.context_chars === "number" - ? args.context_chars - : undefined; - const content = await findInDocumentContent({ - docLabel: docId, - query, - maxResults, - contextChars, - docStore, - write, - docIndex, - db, - }); - const filename = docStore.get(docId)?.filename; - if (filename) { - let totalMatches = 0; - try { - const parsed = JSON.parse(content) as { - total_matches?: number; - }; - totalMatches = parsed.total_matches ?? 0; - } catch { - /* ignore — still record the find attempt */ - } - docsFound.push({ - filename, - query, - total_matches: totalMatches, - }); - } - toolResults.push({ role: "tool", tool_call_id: tc.id, content }); - } else if (tc.function.name === "list_documents") { - const list = Array.from(docStore.entries()).map( - ([doc_id, info]) => ({ - doc_id, - filename: info.filename, - file_type: info.file_type, - }), - ); - toolResults.push({ - role: "tool", - tool_call_id: tc.id, - content: JSON.stringify(list), - }); - } else if (tc.function.name === "fetch_documents") { - const rawDocIds = (args.doc_ids as string[]) ?? []; - const docIds = rawDocIds.map( - (id) => resolveDocLabel(id, docStore, docIndex) ?? id, - ); - const parts: string[] = []; - for (const docId of docIds) { - const content = await readDocumentContent( - docId, - docStore, - write, - docIndex, - db, - ); - const filename = docStore.get(docId)?.filename ?? docId; - parts.push( - `--- ${filename} (${docId}) ---\n${citationReminder(docId, filename)}\n\n${content}`, - ); - if (docStore.get(docId)) { - const documentId = docIndex?.[docId]?.document_id; - docsRead.push({ filename, document_id: documentId }); - } - } - toolResults.push({ - role: "tool", - tool_call_id: tc.id, - content: parts.join("\n\n"), - }); - } else if (tc.function.name === "list_workflows") { - const list = workflowStore - ? Array.from(workflowStore.entries()).map(([id, w]) => ({ - id, - title: w.title, - })) - : []; - toolResults.push({ - role: "tool", - tool_call_id: tc.id, - content: JSON.stringify(list), - }); - } else if (tc.function.name === "read_workflow") { - const wfId = args.workflow_id as string; - const wf = workflowStore?.get(wfId); - if (wf) { - write( - `data: ${JSON.stringify({ type: "workflow_applied", workflow_id: wfId, title: wf.title })}\n\n`, - ); - workflowsApplied.push({ workflow_id: wfId, title: wf.title }); - } - toolResults.push({ - role: "tool", - tool_call_id: tc.id, - content: wf ? wf.prompt_md : `Workflow '${wfId}' not found.`, - }); - } else if (tc.function.name === "read_table_cells" && tabularStore) { - const colIndices = args.col_indices as number[] | undefined; - const rowIndices = args.row_indices as number[] | undefined; - - const filteredCols = colIndices?.length - ? tabularStore.columns.filter((_, i) => colIndices.includes(i)) - : tabularStore.columns; - const filteredDocs = rowIndices?.length - ? tabularStore.documents.filter((_, i) => - rowIndices.includes(i), - ) - : tabularStore.documents; - - const label = `${filteredCols.length} ${filteredCols.length === 1 ? "column" : "columns"} × ${filteredDocs.length} ${filteredDocs.length === 1 ? "row" : "rows"}`; - write( - `data: ${JSON.stringify({ type: "doc_read_start", filename: label })}\n\n`, - ); - - const lines: string[] = []; - for (const col of filteredCols) { - const colPos = tabularStore.columns.findIndex( - (c) => c.index === col.index, - ); - for (const doc of filteredDocs) { - const rowPos = tabularStore.documents.findIndex( - (d) => d.id === doc.id, - ); - const cell = tabularStore.cells.get( - `${col.index}:${doc.id}`, - ); - lines.push( - `[COL:${colPos} "${col.name}" | ROW:${rowPos} "${doc.filename}"]`, - ); - if (cell?.summary) { - lines.push(`Summary: ${cell.summary}`); - if (cell.flag) lines.push(`Flag: ${cell.flag}`); - if (cell.reasoning) - lines.push(`Reasoning: ${cell.reasoning}`); - } else { - lines.push(`(not yet generated)`); - } - lines.push(""); - } - } - - write( - `data: ${JSON.stringify({ type: "doc_read", filename: label })}\n\n`, - ); - docsRead.push({ filename: label }); - toolResults.push({ - role: "tool", - tool_call_id: tc.id, - content: lines.join("\n") || "No cells found.", - }); - } else if (tc.function.name === "edit_document" && docIndex) { - const rawDocId = args.doc_id as string; - const editsRaw = args.edits as unknown[] | undefined; - const docId = - resolveDocLabel(rawDocId, docStore, docIndex) ?? rawDocId; - const docInfo = docStore.get(docId); - const indexed = docIndex?.[docId]; - - const emitEditError = ( - filename: string, - documentId: string, - error: string, - ) => { - // Surface the failure as a failed "Edited" block in the UI - // (start → done-with-error) so it matches the shape the - // success/late-failure paths already use. - write( - `data: ${JSON.stringify({ - type: "doc_edited_start", - filename, - })}\n\n`, - ); - write( - `data: ${JSON.stringify({ - type: "doc_edited", - filename, - document_id: documentId, - version_id: "", - download_url: "", - annotations: [], - error, - })}\n\n`, - ); - }; - - if (!docInfo || !indexed) { - const err = `Document '${docId}' not found in this chat's attachments.`; - emitEditError(docId, indexed?.document_id ?? "", err); - toolResults.push({ - role: "tool", - tool_call_id: tc.id, - content: JSON.stringify({ error: err }), - }); - } else if (!Array.isArray(editsRaw) || editsRaw.length === 0) { - const err = "edits array is required and must not be empty."; - emitEditError(docInfo.filename, indexed.document_id, err); - toolResults.push({ - role: "tool", - tool_call_id: tc.id, - content: JSON.stringify({ error: err }), - }); - } else if (docInfo.file_type !== "docx") { - const err = "edit_document only supports .docx files."; - emitEditError(docInfo.filename, indexed.document_id, err); - toolResults.push({ - role: "tool", - tool_call_id: tc.id, - content: JSON.stringify({ error: err }), - }); - } else { - write( - `data: ${JSON.stringify({ - type: "doc_edited_start", - filename: docInfo.filename, - })}\n\n`, - ); - const edits: EditInput[] = ( - editsRaw as Record<string, unknown>[] - ).map((e) => ({ - find: String(e.find ?? ""), - replace: String(e.replace ?? ""), - context_before: String(e.context_before ?? ""), - context_after: String(e.context_after ?? ""), - reason: e.reason ? String(e.reason) : undefined, - })); - const reuseVersion = turnEditState?.get(indexed.document_id); - const result = await runEditDocument({ - documentId: indexed.document_id, - userId, - edits, - db, - reuseVersion, - }); - - if (result.ok) { - turnEditState?.set(indexed.document_id, { - versionId: result.version_id, - versionNumber: result.version_number, - storagePath: result.storage_path, - }); - // Keep the chat-local doc label pointed at the latest - // edited version so any follow-up read_document call in - // the same assistant turn reads and cites the same bytes. - if (docIndex[docId]) { - docIndex[docId] = { - ...docIndex[docId], - version_id: result.version_id, - version_number: result.version_number, - }; - } - const currentDocStore = docStore.get(docId); - if (currentDocStore) { - docStore.set(docId, { - ...currentDocStore, - storage_path: result.storage_path, - }); - } - const payload: DocEditedResult = { - filename: docInfo.filename, - document_id: indexed.document_id, - version_id: result.version_id, - version_number: result.version_number, - download_url: result.download_url, - annotations: result.annotations, - }; - docsEdited.push(payload); - write( - `data: ${JSON.stringify({ - type: "doc_edited", - ...payload, - })}\n\n`, - ); - toolResults.push({ - role: "tool", - tool_call_id: tc.id, - content: JSON.stringify({ - ok: true, - doc_id: docId, - document_id: indexed.document_id, - version_id: result.version_id, - version_number: result.version_number, - applied: result.annotations.length, - errors: result.errors, - }), - }); - } else { - write( - `data: ${JSON.stringify({ - type: "doc_edited", - filename: docInfo.filename, - document_id: indexed.document_id, - version_id: "", - download_url: "", - annotations: [], - error: result.error, - })}\n\n`, - ); - toolResults.push({ - role: "tool", - tool_call_id: tc.id, - content: JSON.stringify({ - ok: false, - error: result.error, - }), - }); - } - } - } else if (tc.function.name === "replicate_document" && docIndex) { - const rawDocId = args.doc_id as string; - const requestedFilename = - typeof args.new_filename === "string" && - args.new_filename.trim() - ? args.new_filename.trim() - : null; - const requestedCount = - typeof args.count === "number" && Number.isFinite(args.count) - ? Math.max(1, Math.min(20, Math.floor(args.count))) - : 1; - const sourceLabel = - resolveDocLabel(rawDocId, docStore, docIndex) ?? rawDocId; - const sourceInfo = docStore.get(sourceLabel); - const sourceIndexed = docIndex[sourceLabel]; - const sourceFilename = sourceInfo?.filename ?? rawDocId; - - write( - `data: ${JSON.stringify({ - type: "doc_replicate_start", - filename: sourceFilename, - count: requestedCount, - })}\n\n`, - ); - - const fail = (error: string) => { - write( - `data: ${JSON.stringify({ - type: "doc_replicated", - filename: sourceFilename, - count: requestedCount, - copies: [], - error, - })}\n\n`, - ); - toolResults.push({ - role: "tool", - tool_call_id: tc.id, - content: JSON.stringify({ ok: false, error }), - }); - }; - - if (!sourceInfo || !sourceIndexed) { - fail(`Document '${rawDocId}' not found in this project.`); - } else if (!projectId) { - fail("replicate_document is only available in project chats."); - } else { - try { - // Pull the active version once — every copy gets the - // same starting bytes (with any accepted tracked - // changes rolled in), no point re-fetching per copy. - const active = await loadActiveVersion( - sourceIndexed.document_id, - db, - ); - const sourcePath = - active?.storage_path ?? sourceInfo.storage_path; - const sourcePdfPath = active?.pdf_storage_path ?? null; - const raw = await downloadFile(sourcePath); - const pdfBytes = sourcePdfPath - ? await downloadFile(sourcePdfPath) - : null; - if (!raw) { - fail( - "Could not read the source document's bytes from storage.", - ); - } else { - // Build N filenames. With count=1 keep the - // pre-existing "(copy)" suffix; with count>1 use - // numbered "(1)", "(2)" suffixes. - const srcExt = - sourceInfo.filename.match(/\.[^./\\]+$/)?.[0] ?? ""; - const baseStem = (() => { - if (requestedFilename) { - return requestedFilename.replace( - /\.[^./\\]+$/, - "", - ); - } - return sourceInfo.filename.replace( - /\.[^./\\]+$/, - "", - ); - })(); - const filenames: string[] = []; - for (let n = 1; n <= requestedCount; n++) { - const suffix = - requestedCount === 1 - ? requestedFilename - ? "" - : " (copy)" - : ` (${n})`; - filenames.push(`${baseStem}${suffix}${srcExt}`); - } - - // Bulk insert N documents in one round-trip. - const docRows = filenames.map((fn) => ({ - project_id: projectId, - user_id: userId, - filename: fn, - file_type: sourceInfo.file_type, - size_bytes: raw.byteLength, - status: "ready", - })); - const { data: insertedDocs, error: docErr } = await db - .from("documents") - .insert(docRows) - .select("id, filename"); - if ( - docErr || - !insertedDocs || - insertedDocs.length === 0 - ) { - fail( - `Failed to record replicated documents: ${docErr?.message ?? "unknown"}`, - ); - } else { - // Preserve the request order so each row pairs - // with the right filename. Supabase returns - // inserted rows in the same order as the - // payload. - const newDocs = insertedDocs as { - id: string; - filename: string; - }[]; - const contentType = - sourceInfo.file_type === "pdf" - ? "application/pdf" - : "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; - - // Parallel uploads: the doc bytes (and PDF - // rendition if any) for every new copy. - const uploadJobs: Promise<unknown>[] = []; - const newKeys: string[] = []; - const newPdfKeys: (string | null)[] = []; - for (const d of newDocs) { - const key = storageKey( - userId, - d.id, - d.filename, - ); - newKeys.push(key); - uploadJobs.push( - uploadFile(key, raw, contentType), - ); - if (pdfBytes) { - const pdfKey = convertedPdfKey( - userId, - d.id, - ); - newPdfKeys.push(pdfKey); - uploadJobs.push( - uploadFile( - pdfKey, - pdfBytes, - "application/pdf", - ), - ); - } else { - newPdfKeys.push(null); - } - } - await Promise.all(uploadJobs); - - // Bulk insert N versions in one round-trip. - const versionRows = newDocs.map((d, idx) => ({ - document_id: d.id, - storage_path: newKeys[idx], - pdf_storage_path: newPdfKeys[idx], - source: "upload", - version_number: 1, - display_name: d.filename, - })); - const { data: insertedVersions, error: verErr } = - await db - .from("document_versions") - .insert(versionRows) - .select("id, document_id"); - if ( - verErr || - !insertedVersions || - insertedVersions.length !== newDocs.length - ) { - fail( - `Failed to record replicated document versions: ${verErr?.message ?? "unknown"}`, - ); - } else { - const versionByDocId = new Map< - string, - string - >(); - for (const v of insertedVersions as { - id: string; - document_id: string; - }[]) { - versionByDocId.set(v.document_id, v.id); - } - - // current_version_id has to be a per-row - // value, so a single UPDATE statement - // can't cover all N. Fan out in parallel - // instead of sequential awaits. - await Promise.all( - newDocs.map((d) => - db - .from("documents") - .update({ - current_version_id: - versionByDocId.get(d.id), - }) - .eq("id", d.id), - ), - ); - - // Register every copy under a fresh doc-N - // slug so the model can edit/read any of - // them in the same turn. - const existingLabels = new Set( - Object.keys(docIndex), - ); - let nextLabelIdx = 0; - const copies: { - new_filename: string; - document_id: string; - version_id: string; - }[] = []; - const toolPayloadCopies: { - doc_id: string; - document_id: string; - version_id: string; - filename: string; - download_url: string; - }[] = []; - for (let idx = 0; idx < newDocs.length; idx++) { - const d = newDocs[idx]; - const newKey = newKeys[idx]; - const versionId = versionByDocId.get(d.id); - if (!versionId) continue; - while ( - existingLabels.has( - `doc-${nextLabelIdx}`, - ) - ) - nextLabelIdx++; - const slug = `doc-${nextLabelIdx}`; - existingLabels.add(slug); - docIndex[slug] = { - document_id: d.id, - filename: d.filename, - }; - docStore.set(slug, { - storage_path: newKey, - file_type: sourceInfo.file_type, - filename: d.filename, - }); - copies.push({ - new_filename: d.filename, - document_id: d.id, - version_id: versionId, - }); - toolPayloadCopies.push({ - doc_id: slug, - document_id: d.id, - version_id: versionId, - filename: d.filename, - download_url: buildDownloadUrl( - newKey, - d.filename, - ), - }); - } - - write( - `data: ${JSON.stringify({ - type: "doc_replicated", - filename: sourceFilename, - count: copies.length, - copies, - })}\n\n`, - ); - docsReplicated.push({ - filename: sourceFilename, - count: copies.length, - copies, - }); - toolResults.push({ - role: "tool", - tool_call_id: tc.id, - content: JSON.stringify({ - ok: true, - count: copies.length, - copies: toolPayloadCopies, - }), - }); - } - } - } - } catch (e) { - fail(`replicate_document failed: ${String(e)}`); - } - } - } else if (tc.function.name === "generate_docx") { - const title = args.title as string; - const landscape = !!args.landscape; - console.log( - `[generate_docx] title="${title}" landscape=${landscape} args.landscape=${args.landscape}`, - ); - const previewFilename = `${ - title - .replace(/[^a-zA-Z0-9 _-]/g, "") - .trim() - .slice(0, 64) || "document" - }.docx`; - write( - `data: ${JSON.stringify({ type: "doc_created_start", filename: previewFilename })}\n\n`, - ); - const result = await generateDocx( - title, - args.sections as unknown[], - userId, - db, - { landscape, projectId: projectId ?? null }, - ); - let newDocLabel: string | null = null; - if ("filename" in result && "download_url" in result) { - const dlFilename = result.filename as string; - const dlUrl = result.download_url as string; - const documentId = (result as { document_id?: string }) - .document_id; - const versionId = (result as { version_id?: string }) - .version_id; - const versionNumber = - (result as { version_number?: number }).version_number ?? - null; - const storagePath = (result as { storage_path?: string }) - .storage_path; - - // Register the generated doc in the chat context so - // edit_document (and read_document / find_in_document) - // can act on it within the same assistant turn. New label - // is the next free `doc-N` index. Subsequent turns pick - // it up via the normal attachment/project doc query. - if (documentId && storagePath && docIndex) { - const existingLabels = new Set(Object.keys(docIndex)); - let i = 0; - while (existingLabels.has(`doc-${i}`)) i++; - newDocLabel = `doc-${i}`; - docIndex[newDocLabel] = { - document_id: documentId, - filename: dlFilename, - }; - docStore.set(newDocLabel, { - storage_path: storagePath, - file_type: "docx", - filename: dlFilename, - }); - } - - write( - `data: ${JSON.stringify({ - type: "doc_created", - filename: dlFilename, - download_url: dlUrl, - document_id: documentId, - version_id: versionId, - version_number: versionNumber, - })}\n\n`, - ); - docsCreated.push({ - filename: dlFilename, - download_url: dlUrl, - document_id: documentId, - version_id: versionId, - version_number: versionNumber, - }); - } else { - write( - `data: ${JSON.stringify({ type: "doc_created", filename: previewFilename, download_url: "" })}\n\n`, - ); - } - // Surface the chat-local doc label in the tool result so the - // model can pass it as `doc_id` to edit_document / read_document - // / find_in_document in the same turn. Without this the model - // only sees the DB UUID, which isn't valid as a doc_id anchor. - const { download_url, storage_path, ...safeToolResult } = - result as Record<string, unknown>; - const toolResultPayload = newDocLabel - ? { - ...safeToolResult, - doc_id: newDocLabel, - next_required_action: `Before writing your final response, call read_document with doc_id "${newDocLabel}". Describe and cite the generated document using doc_id "${newDocLabel}", not the source/template document.`, - } - : safeToolResult; - toolResults.push({ - role: "tool", - tool_call_id: tc.id, - content: JSON.stringify(toolResultPayload), - }); - } - } - - return { - toolResults, - docsRead, - docsFound, - docsCreated, - docsReplicated, - workflowsApplied, - docsEdited, - }; -} - -// --------------------------------------------------------------------------- -// Citation parsing -// --------------------------------------------------------------------------- - -const CITATIONS_BLOCK_RE = /<CITATIONS>\s*([\s\S]*?)\s*<\/CITATIONS>/; -const CITATIONS_OPEN_TAG = "<CITATIONS>"; - -function parseCitations(text: string): ParsedCitation[] { - const match = text.match(CITATIONS_BLOCK_RE); - if (!match) return []; - try { - const raw = JSON.parse(match[1]); - if (!Array.isArray(raw)) return []; - return raw - .map(normalizeCitation) - .filter((c): c is ParsedCitation => c !== null); - } catch { - return []; - } -} - -// --------------------------------------------------------------------------- -// LLM streaming loop -// --------------------------------------------------------------------------- - -export type EditAnnotation = { - kind: "edit"; - edit_id: string; - document_id: string; - version_id: string; - version_number?: number | null; - change_id: string; - del_w_id?: string; - ins_w_id?: string; - deleted_text: string; - inserted_text: string; - context_before: string; - context_after: string; - reason?: string; - status: "pending" | "accepted" | "rejected"; -}; - -type AssistantEvent = - | { type: "reasoning"; text: string } - | { type: "doc_read"; filename: string; document_id?: string } - | { - type: "doc_find"; - filename: string; - query: string; - total_matches: number; - } - | { - type: "doc_created"; - filename: string; - download_url: string; - document_id?: string; - version_id?: string; - version_number?: number | null; - } - | { type: "doc_download"; filename: string; download_url: string } - | { - type: "doc_replicated"; - /** Source document being copied. */ - filename: string; - count: number; - copies: { - new_filename: string; - document_id: string; - version_id: string; - }[]; - } - | { type: "workflow_applied"; workflow_id: string; title: string } - | { - type: "doc_edited"; - filename: string; - document_id: string; - version_id: string; - /** Per-document monotonic Vn; null if backend couldn't determine it. */ - version_number: number | null; - download_url: string; - annotations: EditAnnotation[]; - } - | { type: "content"; text: string }; - -export async function runLLMStream(params: { - apiMessages: unknown[]; - docStore: DocStore; - docIndex: DocIndex; - userId: string; - db: ReturnType<typeof createServerSupabase>; - write: (s: string) => void; - extraTools?: unknown[]; - workflowStore?: WorkflowStore; - tabularStore?: TabularCellStore; - buildCitations?: (fullText: string) => unknown[]; - model?: string; - apiKeys?: import("./llm").UserApiKeys; - /** - * If set, generate_docx will attach created docs to this project so - * they appear in the project sidebar. Leave null for general chats — - * generated docs still get persisted, but as standalone documents. - */ - projectId?: string | null; -}): Promise<{ fullText: string; events: AssistantEvent[] }> { - const { - apiMessages, - docStore, - docIndex, - userId, - db, - write, - extraTools, - workflowStore, - tabularStore, - buildCitations, - model, - apiKeys, - projectId, - } = params; - const activeTools = extraTools?.length - ? [...TOOLS, ...WORKFLOW_TOOLS, ...extraTools] - : [...TOOLS, ...WORKFLOW_TOOLS]; - - // Extract system prompt; pass remaining turns to the adapter as - // plain user/assistant messages. - const rawMsgs = apiMessages as { role: string; content: string | null }[]; - const systemPrompt = - rawMsgs[0]?.role === "system" ? (rawMsgs[0].content ?? "") : ""; - const chatMessages: LlmMessage[] = rawMsgs - .filter((m) => m.role !== "system") - .map((m) => ({ - role: m.role === "assistant" ? "assistant" : "user", - content: m.content ?? "", - })); - - const events: AssistantEvent[] = []; - // One assistant turn produces at most one document_versions row per - // edited doc. `runToolCalls` fires once per tool-call batch; the model - // may emit multiple batches in a single turn, so this map persists - // across batches to let subsequent edit_document calls overwrite the - // turn's existing version instead of creating a new one. - const turnEditState: TurnEditState = new Map(); - let fullText = ""; - let iterText = ""; - let iterVisibleText = ""; - let iterReasoning = ""; - let visibleTailBuffer = ""; - let citationsOpenSeen = false; - - const streamVisibleContent = (delta: string) => { - if (!delta) return; - if (citationsOpenSeen) return; - - const combined = visibleTailBuffer + delta; - const markerIdx = combined.indexOf(CITATIONS_OPEN_TAG); - if (markerIdx >= 0) { - const visible = combined.slice(0, markerIdx); - if (visible) { - iterVisibleText += visible; - write( - `data: ${JSON.stringify({ type: "content_delta", text: visible })}\n\n`, - ); - } - visibleTailBuffer = ""; - citationsOpenSeen = true; - return; - } - - const keep = Math.min(CITATIONS_OPEN_TAG.length - 1, combined.length); - const visible = combined.slice(0, combined.length - keep); - visibleTailBuffer = combined.slice(combined.length - keep); - if (visible) { - iterVisibleText += visible; - write( - `data: ${JSON.stringify({ type: "content_delta", text: visible })}\n\n`, - ); - } - }; - - const flushVisibleTail = () => { - if (citationsOpenSeen || !visibleTailBuffer) { - visibleTailBuffer = ""; - return; - } - iterVisibleText += visibleTailBuffer; - write( - `data: ${JSON.stringify({ type: "content_delta", text: visibleTailBuffer })}\n\n`, - ); - visibleTailBuffer = ""; - }; - - const flushText = () => { - if (!iterText) return; - fullText += iterText; - flushVisibleTail(); - if (iterVisibleText) { - events.push({ type: "content", text: iterVisibleText }); - } - iterText = ""; - iterVisibleText = ""; - visibleTailBuffer = ""; - citationsOpenSeen = false; - }; - - const selectedModel = resolveModel(model, DEFAULT_MAIN_MODEL); - - await streamChatWithTools({ - model: selectedModel, - systemPrompt, - messages: chatMessages, - tools: activeTools as OpenAIToolSchema[], - maxIterations: 10, - apiKeys, - enableThinking: true, - callbacks: { - onContentDelta: (delta) => { - iterText += delta; - streamVisibleContent(delta); - }, - onReasoningDelta: (delta) => { - iterReasoning += delta; - write( - `data: ${JSON.stringify({ type: "reasoning_delta", text: delta })}\n\n`, - ); - }, - onReasoningBlockEnd: () => { - if (!iterReasoning) return; - events.push({ type: "reasoning", text: iterReasoning }); - write( - `data: ${JSON.stringify({ type: "reasoning_block_end" })}\n\n`, - ); - iterReasoning = ""; - }, - // Fires after Claude's turn ends with stop_reason=tool_use, before - // the tool actually runs. Flushes any buffered assistant text so - // it's emitted in chronological order, then signals the client so - // it can open a fresh PreResponseWrapper (shows "Working…") while - // the tool executes — avoids the dead gap between message_stop - // and the first tool-specific event. - onToolCallStart: (call) => { - flushText(); - write( - `data: ${JSON.stringify({ - type: "tool_call_start", - name: call.name, - })}\n\n`, - ); - }, - }, - runTools: async (calls) => { - // Emit any text the model produced before this tool turn so the - // UI sees it before the tool results stream in. - flushText(); - - const toolCalls: ToolCall[] = calls.map((c) => ({ - id: c.id, - function: { - name: c.name, - arguments: JSON.stringify(c.input), - }, - })); - const { - toolResults, - docsRead, - docsFound, - docsCreated, - docsReplicated, - workflowsApplied, - docsEdited, - } = await runToolCalls( - toolCalls, - docStore, - userId, - db, - write, - workflowStore, - tabularStore, - docIndex, - turnEditState, - projectId, - ); - for (const r of docsRead) { - events.push({ - type: "doc_read", - filename: r.filename, - document_id: r.document_id, - }); - } - for (const f of docsFound) { - events.push({ - type: "doc_find", - filename: f.filename, - query: f.query, - total_matches: f.total_matches, - }); - } - for (const dl of docsCreated) { - events.push({ - type: "doc_created", - filename: dl.filename, - download_url: dl.download_url, - document_id: dl.document_id, - version_id: dl.version_id, - version_number: dl.version_number ?? null, - }); - } - for (const r of docsReplicated) { - events.push({ - type: "doc_replicated", - filename: r.filename, - count: r.count, - copies: r.copies, - }); - } - for (const wf of workflowsApplied) { - events.push({ - type: "workflow_applied", - workflow_id: wf.workflow_id, - title: wf.title, - }); - } - for (const e of docsEdited) { - events.push({ - type: "doc_edited", - filename: e.filename, - document_id: e.document_id, - version_id: e.version_id, - version_number: e.version_number, - download_url: e.download_url, - annotations: e.annotations, - }); - } - - // Index alignment would break if any tool branch skips its - // push (unhandled tool name, disabled store, guard failure). - // Each tool_result already carries its tool_call_id, so key off - // that directly — and fall back to an error result for any - // tool_use that didn't produce one, so Claude's next request - // has a tool_result for every tool_use it sent. - const resultByCallId = new Map<string, string>(); - for (const r of toolResults) { - const row = r as { tool_call_id: string; content?: unknown }; - resultByCallId.set(row.tool_call_id, String(row.content ?? "")); - } - return toolCalls.map((c) => ({ - tool_use_id: c.id, - content: - resultByCallId.get(c.id) ?? - JSON.stringify({ - error: `Tool '${c.function.name}' is not available.`, - }), - })); - }, - }); - - flushText(); - - // Parse and emit citations from <CITATIONS> block - const citations = buildCitations - ? buildCitations(fullText) - : parseCitations(fullText).map((c) => { - const docInfo = resolveDoc(c.doc_id, docIndex); - return { - ref: c.ref, - doc_id: c.doc_id, - document_id: docInfo?.document_id, - version_id: docInfo?.version_id ?? null, - version_number: docInfo?.version_number ?? null, - filename: docInfo?.filename ?? c.doc_id, - page: c.page, - quote: c.quote, - }; - }); - write(`data: ${JSON.stringify({ type: "citations", citations })}\n\n`); - write("data: [DONE]\n\n"); - - return { fullText, events }; -} - -// --------------------------------------------------------------------------- -// Annotation extraction (for DB save) -// --------------------------------------------------------------------------- - -export function extractAnnotations( - fullText: string, - docIndex: DocIndex, - events?: ({ type: string } & Record<string, unknown>[]) | unknown[], -): unknown[] { - const out: unknown[] = parseCitations(fullText).map((c) => { - const docInfo = resolveDoc(c.doc_id, docIndex); - return { - type: "citation_data", - ref: c.ref, - doc_id: c.doc_id, - document_id: docInfo?.document_id, - version_id: docInfo?.version_id ?? null, - version_number: docInfo?.version_number ?? null, - filename: docInfo?.filename ?? c.doc_id, - page: c.page, - quote: c.quote, - }; - }); - if (Array.isArray(events)) { - for (const ev of events as { - type?: string; - annotations?: EditAnnotation[]; - }[]) { - if (ev?.type === "doc_edited" && Array.isArray(ev.annotations)) { - for (const a of ev.annotations) - out.push({ ...a, type: "edit_data" }); - } - } - } - return out; -} - -// --------------------------------------------------------------------------- -// Document context builder (from message file attachments) -// --------------------------------------------------------------------------- - -export async function buildDocContext( - messages: ChatMessage[], - userId: string, - db: ReturnType<typeof createServerSupabase>, - chatId?: string | null, -): Promise<{ docIndex: DocIndex; docStore: DocStore }> { - const docIndex: DocIndex = {}; - const docStore: DocStore = new Map(); - - const documentIds = new Set<string>(); - for (const m of messages) { - for (const f of m.files ?? []) { - if (f.document_id) documentIds.add(f.document_id); - } - } - - // Also pull in document_ids from prior assistant events in this chat — - // generated docs (generate_docx) and tracked-change edits (edit_document) - // aren't attached to user messages as files, so they only live in the - // assistant's `doc_created` / `doc_edited` events. Without this sweep - // the model loses access to generated docs after the turn that created - // them, and can't call edit_document / read_document on them. - if (chatId) { - const { data: rows } = await db - .from("chat_messages") - .select("content") - .eq("chat_id", chatId) - .eq("role", "assistant"); - for (const row of rows ?? []) { - const content = (row as { content?: unknown }).content; - if (!Array.isArray(content)) continue; - for (const ev of content as Record<string, unknown>[]) { - if ( - (ev?.type === "doc_created" || ev?.type === "doc_edited") && - typeof ev.document_id === "string" - ) { - documentIds.add(ev.document_id); - } - } - } - } - - const ids = [...documentIds]; - if (ids.length > 0) { - const { data: docs } = await db - .from("documents") - .select("id, filename, file_type, current_version_id, status") - .in("id", ids) - .eq("user_id", userId) - .eq("status", "ready"); - - const docList = (docs ?? []) as unknown as { - id: string; - filename: string; - file_type: string; - current_version_id?: string | null; - active_version_number?: number | null; - storage_path?: string | null; - }[]; - await attachActiveVersionPaths(db, docList); - for (let i = 0; i < docList.length; i++) { - const doc = docList[i]; - if (!doc.storage_path) continue; - const docLabel = `doc-${i}`; - docIndex[docLabel] = { - document_id: doc.id, - filename: doc.filename, - version_id: doc.current_version_id ?? null, - version_number: doc.active_version_number ?? null, - }; - docStore.set(docLabel, { - storage_path: doc.storage_path, - file_type: doc.file_type, - filename: doc.filename, - }); - } - } - - console.log( - "[buildDocContext] available docs:", - Object.entries(docIndex).map(([label, info]) => ({ - label, - filename: info.filename, - document_id: info.document_id, - })), - ); - return { docIndex, docStore }; -} - -export async function buildProjectDocContext( - projectId: string, - _userId: string, - db: ReturnType<typeof createServerSupabase>, -): Promise<{ - docIndex: DocIndex; - docStore: DocStore; - folderPaths: Map<string, string>; -}> { - const docIndex: DocIndex = {}; - const docStore: DocStore = new Map(); - - const [{ data: docs }, { data: folders }] = await Promise.all([ - db - .from("documents") - .select( - "id, filename, file_type, current_version_id, status, folder_id", - ) - .eq("project_id", projectId) - .eq("status", "ready") - .order("created_at", { ascending: true }), - db - .from("project_subfolders") - .select("id, name, parent_folder_id") - .eq("project_id", projectId), - ]); - const docList = (docs ?? []) as unknown as { - id: string; - filename: string; - file_type: string; - current_version_id?: string | null; - active_version_number?: number | null; - folder_id?: string | null; - storage_path?: string | null; - }[]; - await attachActiveVersionPaths(db, docList); - - // Build folder id → full path map - const folderMap = new Map< - string, - { name: string; parent_folder_id: string | null } - >(); - for (const f of folders ?? []) - folderMap.set(f.id, { - name: f.name, - parent_folder_id: f.parent_folder_id, - }); - - function resolvePath(folderId: string | null): string { - if (!folderId) return ""; - const parts: string[] = []; - let cur: string | null = folderId; - while (cur) { - const f = folderMap.get(cur); - if (!f) break; - parts.unshift(f.name); - cur = f.parent_folder_id; - } - return parts.join(" / "); - } - - const folderPaths = new Map<string, string>(); // doc label → folder path - - for (let i = 0; i < docList.length; i++) { - const doc = docList[i]; - if (!doc.storage_path) continue; - const docLabel = `doc-${i}`; - docIndex[docLabel] = { - document_id: doc.id, - filename: doc.filename, - version_id: doc.current_version_id ?? null, - version_number: doc.active_version_number ?? null, - }; - docStore.set(docLabel, { - storage_path: doc.storage_path, - file_type: doc.file_type, - filename: doc.filename, - }); - const path = resolvePath(doc.folder_id ?? null); - if (path) folderPaths.set(docLabel, path); - } - - console.log( - "[buildProjectDocContext] available docs:", - Object.entries(docIndex).map(([label, info]) => ({ - label, - filename: info.filename, - document_id: info.document_id, - folder: folderPaths.get(label) ?? null, - })), - ); - return { docIndex, docStore, folderPaths }; -} - -export async function buildWorkflowStore( - userId: string, - userEmail: string | null | undefined, - db: ReturnType<typeof createServerSupabase>, -): Promise<WorkflowStore> { - const { BUILTIN_WORKFLOWS } = await import("./builtinWorkflows"); - const store: WorkflowStore = new Map(); - const normalizedUserEmail = (userEmail ?? "").trim().toLowerCase(); - - // Seed built-ins first - for (const wf of BUILTIN_WORKFLOWS) { - store.set(wf.id, { title: wf.title, prompt_md: wf.prompt_md }); - } - - // Then overlay user-owned assistant workflows. - const { data: workflows } = await db - .from("workflows") - .select("id, title, prompt_md") - .eq("user_id", userId) - .eq("type", "assistant"); - for (const wf of workflows ?? []) { - if (wf.prompt_md) { - store.set(wf.id, { title: wf.title, prompt_md: wf.prompt_md }); - } - } - - // Shared assistant workflows must also be readable by workflow tools. - if (normalizedUserEmail) { - const { data: shares } = await db - .from("workflow_shares") - .select("workflow_id") - .eq("shared_with_email", normalizedUserEmail); - const sharedIds = [ - ...new Set((shares ?? []).map((share) => share.workflow_id)), - ]; - if (sharedIds.length > 0) { - const { data: sharedWorkflows } = await db - .from("workflows") - .select("id, title, prompt_md") - .in("id", sharedIds) - .eq("type", "assistant"); - for (const wf of sharedWorkflows ?? []) { - if (wf.prompt_md) { - store.set(wf.id, { - title: wf.title, - prompt_md: wf.prompt_md, - }); - } - } - } - } - return store; -} diff --git a/backend/src/lib/chatTools/citations.ts b/backend/src/lib/chatTools/citations.ts new file mode 100644 index 000000000..0d70c04c4 --- /dev/null +++ b/backend/src/lib/chatTools/citations.ts @@ -0,0 +1,93 @@ +/** + * <CITATIONS> block parser and post-stream annotation extractor. + * + * stream.ts uses CITATIONS_OPEN_TAG as the inline sentinel for visible- + * content stripping (the inline streamVisibleContent closure in + * runLLMStream). After the stream ends, extractAnnotations parses the + * complete fullText into EditAnnotation[] for the assistant message + * persistence path. + */ + +import type { DocIndex, EditAnnotation } from "./types"; +import { parseLlmJson } from "./parseLlmJson"; +import { CitationsArraySchema } from "./llm-schemas"; +import { logger } from "../logger"; + +export const CITATIONS_BLOCK_RE = /<CITATIONS>\s*([\s\S]*?)\s*<\/CITATIONS>/; +export const CITATIONS_OPEN_TAG = "<CITATIONS>"; + +export type ParsedCitation = { + ref: number; + doc_id: string; + page: number | string; + quote: string; +}; + +function normalizeCitation(raw: unknown): ParsedCitation | null { + if (!raw || typeof raw !== "object") return null; + const c = raw as Record<string, unknown>; + if (typeof c.ref !== "number" || typeof c.doc_id !== "string") return null; + if (typeof c.quote !== "string" || !c.quote) return null; + let page: number | string; + if (typeof c.page === "number") { + page = c.page; + } else if (typeof c.page === "string" && /^\d+\s*-\s*\d+$/.test(c.page)) { + page = c.page; + } else { + const n = parseInt(String(c.page ?? ""), 10); + if (!Number.isFinite(n)) return null; + page = n; + } + return { ref: c.ref, doc_id: c.doc_id, page, quote: c.quote }; +} + +export function parseCitations( + text: string, + write?: (s: string) => void, +): ParsedCitation[] { + const match = text.match(CITATIONS_BLOCK_RE); + if (!match) return []; + const result = parseLlmJson(match[1], CitationsArraySchema); + if (!result.ok) { + logger.warn({ err: result.error }, "[chatTools/citations] parse failed"); + if (write) { + write( + `data: ${JSON.stringify({ type: "citations_parse_error", error: result.error })}\n\n`, + ); + } + return []; + } + return result.data + .map(normalizeCitation) + .filter((c): c is ParsedCitation => c !== null); +} + +export function extractAnnotations( + fullText: string, + docIndex: DocIndex, + events?: ({ type: string } & Record<string, unknown>)[], + write?: (s: string) => void, +): unknown[] { + const out: unknown[] = parseCitations(fullText, write).map((c) => { + const docInfo = docIndex[c.doc_id]; + return { + type: "citation_data", + ref: c.ref, + doc_id: c.doc_id, + document_id: docInfo?.document_id, + version_id: docInfo?.version_id ?? null, + version_number: docInfo?.version_number ?? null, + filename: docInfo?.filename ?? c.doc_id, + page: c.page, + quote: c.quote, + }; + }); + if (Array.isArray(events)) { + for (const ev of events as { type?: string; annotations?: EditAnnotation[] }[]) { + if (ev?.type === "doc_edited" && Array.isArray(ev.annotations)) { + for (const a of ev.annotations) out.push({ ...a, type: "edit_data" }); + } + } + } + return out; +} diff --git a/backend/src/lib/chatTools/doc-context.ts b/backend/src/lib/chatTools/doc-context.ts new file mode 100644 index 000000000..9bd94a591 --- /dev/null +++ b/backend/src/lib/chatTools/doc-context.ts @@ -0,0 +1,382 @@ +/** + * Per-turn document index/store builders + chat-history formatting + + * doc-id resolvers. + * + * buildDocContext: chat-scoped doc set from user attachments + prior + * assistant-generated documents. + * buildProjectDocContext: project-scoped doc set with folder paths. + * buildMessages: formats history with the system prompt prepended. + * enrichWithPriorEvents: attaches a tool-activity summary to the last + * assistant message so the model has continuity across turns. + * resolveDoc: docIndex passthrough by raw id. + * resolveDocLabel: chat-local label lookup tolerating doc-N slugs, + * filenames, and document UUIDs (model often picks the wrong one). + * + * One Supabase round-trip per builder. + */ + +import { attachActiveVersionPaths } from "../documentVersions"; +import { createServerSupabase } from "../supabase"; +import { SYSTEM_PROMPT } from "./system-prompts/en"; +import type { DocStore, DocIndex, ChatMessage } from "./types"; +import { logger } from "../logger"; + +export function resolveDoc(rawId: string, docIndex: DocIndex) { + return docIndex[rawId]; +} + +/** + * Resolve whatever identifier the model passed (`doc-N` slug, filename, or + * document UUID) back to a chat-local doc label. Generated docs surface in + * tool results with both `doc_id` (slug) and `document_id` (UUID), so the + * model often picks the wrong one — without this fallback `read_document` + * silently returns "not found" and the model gives up and re-generates. + */ +export function resolveDocLabel( + rawId: string, + docStore: DocStore, + docIndex?: DocIndex, +): string | null { + if (docStore.has(rawId)) return rawId; + for (const [label, info] of docStore.entries()) { + if (info.filename === rawId) return label; + } + if (docIndex) { + for (const [label, info] of Object.entries(docIndex)) { + if (info.document_id === rawId) return label; + } + } + return null; +} + +/** + * Append a tool-activity summary to the most recent assistant message so + * the model can see what it just did (read / create / edit / workflow + * applied) in the prior turn — otherwise it only sees its own prose and + * forgets which docs it touched, which leads to e.g. re-generating a doc + * that already exists. + * + * Doc references use the *current-turn* `doc_id` slug (looked up by + * matching the event's stored `document_id` against this turn's freshly + * built `docIndex`), since slugs are reassigned every turn and the old + * slug from the prior turn would be meaningless. Falls back to filename + * only if the doc is no longer in the index (deleted, scope changed). + */ +export async function enrichWithPriorEvents( + messages: ChatMessage[], + chatId: string | null | undefined, + db: ReturnType<typeof createServerSupabase>, + docIndex: DocIndex, +): Promise<ChatMessage[]> { + if (!chatId) return messages; + const { data: rows } = await db + .from("chat_messages") + .select("content, created_at") + .eq("chat_id", chatId) + .eq("role", "assistant") + .order("created_at", { ascending: false }) + .limit(1); + + const lastRow = rows?.[0] as { content?: unknown } | undefined; + const content = lastRow?.content; + if (!Array.isArray(content)) return messages; + + const slugByDocumentId = new Map<string, string>(); + for (const [slug, info] of Object.entries(docIndex)) { + if (info.document_id) slugByDocumentId.set(info.document_id, slug); + } + const refFor = (documentId: unknown, filename: unknown) => { + const slug = + typeof documentId === "string" + ? slugByDocumentId.get(documentId) + : undefined; + return slug ? `${slug} ("${filename}")` : `"${filename}"`; + }; + + const lines: string[] = []; + for (const ev of content as Record<string, unknown>[]) { + if (ev?.type === "doc_created") { + lines.push( + `- generate_docx → ${refFor(ev.document_id, ev.filename)}`, + ); + } else if (ev?.type === "doc_edited") { + lines.push( + `- edit_document → ${refFor(ev.document_id, ev.filename)}`, + ); + } else if (ev?.type === "doc_read") { + lines.push( + `- read_document → ${refFor(ev.document_id, ev.filename)}`, + ); + } else if (ev?.type === "doc_replicated") { + // The model needs to know what each copy resolved to so it + // can call edit_document / read_document on them. Emit one + // line per copy, all attributed back to the same source. + const srcLabel = + typeof ev.filename === "string" ? `"${ev.filename}"` : ""; + const copies = Array.isArray(ev.copies) + ? (ev.copies as { + new_filename?: unknown; + document_id?: unknown; + }[]) + : []; + for (const c of copies) { + const ref = refFor(c.document_id, c.new_filename); + lines.push( + srcLabel + ? `- replicate_document → ${ref} (copy of ${srcLabel})` + : `- replicate_document → ${ref}`, + ); + } + } else if (ev?.type === "workflow_applied") { + lines.push(`- applied workflow: "${ev.title}"`); + } + } + if (lines.length === 0) return messages; + const summary = `\n\n[Tool activity in your previous turn]\n${lines.join("\n")}`; + + // Find the index of the last assistant message and attach the + // summary there only. + let lastAssistantIdx = -1; + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role === "assistant") { + lastAssistantIdx = i; + break; + } + } + if (lastAssistantIdx < 0) return messages; + const enriched = messages.slice(); + const target = enriched[lastAssistantIdx]; + enriched[lastAssistantIdx] = { + ...target, + content: (target.content ?? "") + summary, + }; + return enriched; +} + +export function buildMessages( + messages: ChatMessage[], + docAvailability: { doc_id: string; filename: string; folder_path?: string }[], + systemPromptExtra?: string, + docIndex?: DocIndex, +) { + const formatted: unknown[] = []; + let systemContent = SYSTEM_PROMPT; + + if (systemPromptExtra) { + systemContent += `\n\n${systemPromptExtra.trim()}`; + } + + if (docAvailability.length) { + // NOTE: this inline rendering intentionally diverges from + // `buildDocsSection` in `system-prompts/en.ts`. The hot path + // wraps the list in `---` delimiters and appends the "You do NOT + // retain..." retention reminder, which `buildDocsSection` does + // not. `buildDocsSection` / `buildSystemPrompt` are exported for + // forward use by BILING-03 (Dutch locale) but are not yet wired + // into the live request path; until they are, this block is the + // authoritative source for the AVAILABLE DOCUMENTS section. + systemContent += "\n\n---\nAVAILABLE DOCUMENTS:\n"; + for (const doc of docAvailability) { + const label = doc.folder_path ? `${doc.folder_path} / ${doc.filename}` : doc.filename; + systemContent += `- ${doc.doc_id}: ${label}\n`; + } + systemContent += + "\nYou do NOT retain document content between conversation turns. You MUST call read_document (or fetch_documents) at the start of every response that involves a document's content, even if you have read it in a previous turn. Failure to do so will result in hallucinated or stale content.\n---\n"; + } + formatted.push({ role: "system", content: systemContent }); + + // Map document_id (UUID) → current-turn doc_id slug, so when we + // inline a user attachment we hand the model the same handle it + // would use to call read_document / fetch_documents. + const slugByDocumentId = new Map<string, string>(); + if (docIndex) { + for (const [slug, info] of Object.entries(docIndex)) { + if (info.document_id) slugByDocumentId.set(info.document_id, slug); + } + } + + for (const msg of messages) { + let content = msg.content ?? ""; + if (msg.role === "user" && msg.workflow) { + content = `[Workflow: ${msg.workflow.title} (id: ${msg.workflow.id})]\n\n${content}`; + } + if (msg.role === "user" && msg.files?.length) { + const lines = msg.files.map((f) => { + const slug = f.document_id + ? slugByDocumentId.get(f.document_id) + : undefined; + return slug + ? `- ${slug}: ${f.filename}` + : `- ${f.filename}`; + }); + content = `[The user attached the following document(s) to this message:\n${lines.join("\n")}]\n\n${content}`; + } + formatted.push({ role: msg.role, content }); + } + return formatted; +} + +export async function buildDocContext( + messages: ChatMessage[], + userId: string, + db: ReturnType<typeof createServerSupabase>, + chatId?: string | null, +): Promise<{ docIndex: DocIndex; docStore: DocStore }> { + const docIndex: DocIndex = {}; + const docStore: DocStore = new Map(); + + const documentIds = new Set<string>(); + for (const m of messages) { + for (const f of m.files ?? []) { + if (f.document_id) documentIds.add(f.document_id); + } + } + + // Also pull in document_ids from prior assistant events in this chat — + // generated docs (generate_docx) and tracked-change edits (edit_document) + // aren't attached to user messages as files, so they only live in the + // assistant's `doc_created` / `doc_edited` events. Without this sweep + // the model loses access to generated docs after the turn that created + // them, and can't call edit_document / read_document on them. + if (chatId) { + const { data: rows } = await db + .from("chat_messages") + .select("content") + .eq("chat_id", chatId) + .eq("role", "assistant"); + for (const row of rows ?? []) { + const content = (row as { content?: unknown }).content; + if (!Array.isArray(content)) continue; + for (const ev of content as Record<string, unknown>[]) { + if (ev?.type === "doc_replicated" && Array.isArray(ev.copies)) { + for (const copy of ev.copies as { document_id?: unknown }[]) { + if (typeof copy.document_id === "string") { + documentIds.add(copy.document_id); + } + } + } else if ( + (ev?.type === "doc_created" || + ev?.type === "doc_edited") && + typeof ev.document_id === "string" + ) { + documentIds.add(ev.document_id); + } + } + } + } + + const ids = [...documentIds]; + if (ids.length > 0) { + const { data: docs } = await db + .from("documents") + .select("id, filename, file_type, current_version_id, status") + .in("id", ids) + .eq("user_id", userId) + .eq("status", "ready"); + + const docList = (docs ?? []) as unknown as { + id: string; + filename: string; + file_type: string; + current_version_id?: string | null; + active_version_number?: number | null; + storage_path?: string | null; + }[]; + await attachActiveVersionPaths(db, docList); + for (let i = 0; i < docList.length; i++) { + const doc = docList[i]; + if (!doc.storage_path) continue; + const docLabel = `doc-${i}`; + docIndex[docLabel] = { + document_id: doc.id, + filename: doc.filename, + version_id: doc.current_version_id ?? null, + version_number: doc.active_version_number ?? null, + }; + docStore.set(docLabel, { + storage_path: doc.storage_path, + file_type: doc.file_type, + filename: doc.filename, + }); + } + } + + logger.info( + { docs: Object.entries(docIndex).map(([label, info]) => ({ label, filename: info.filename, document_id: info.document_id })) }, + "[buildDocContext] available docs", + ); + return { docIndex, docStore }; +} + +export async function buildProjectDocContext( + projectId: string, + db: ReturnType<typeof createServerSupabase>, +): Promise<{ docIndex: DocIndex; docStore: DocStore; folderPaths: Map<string, string> }> { + const docIndex: DocIndex = {}; + const docStore: DocStore = new Map(); + + const [{ data: docs }, { data: folders }] = await Promise.all([ + db.from("documents") + .select("id, filename, file_type, current_version_id, status, folder_id") + .eq("project_id", projectId) + .eq("status", "ready") + .order("created_at", { ascending: true }), + db.from("project_subfolders") + .select("id, name, parent_folder_id") + .eq("project_id", projectId), + ]); + const docList = (docs ?? []) as unknown as { + id: string; + filename: string; + file_type: string; + current_version_id?: string | null; + active_version_number?: number | null; + folder_id?: string | null; + storage_path?: string | null; + }[]; + await attachActiveVersionPaths(db, docList); + + // Build folder id → full path map + const folderMap = new Map<string, { name: string; parent_folder_id: string | null }>(); + for (const f of folders ?? []) folderMap.set(f.id, { name: f.name, parent_folder_id: f.parent_folder_id }); + + function resolvePath(folderId: string | null): string { + if (!folderId) return ""; + const parts: string[] = []; + let cur: string | null = folderId; + while (cur) { + const f = folderMap.get(cur); + if (!f) break; + parts.unshift(f.name); + cur = f.parent_folder_id; + } + return parts.join(" / "); + } + + const folderPaths = new Map<string, string>(); // doc label → folder path + + for (let i = 0; i < docList.length; i++) { + const doc = docList[i]; + if (!doc.storage_path) continue; + const docLabel = `doc-${i}`; + docIndex[docLabel] = { + document_id: doc.id, + filename: doc.filename, + version_id: doc.current_version_id ?? null, + version_number: doc.active_version_number ?? null, + }; + docStore.set(docLabel, { + storage_path: doc.storage_path, + file_type: doc.file_type, + filename: doc.filename, + }); + const path = resolvePath(doc.folder_id ?? null); + if (path) folderPaths.set(docLabel, path); + } + + logger.info( + { docs: Object.entries(docIndex).map(([label, info]) => ({ label, filename: info.filename, document_id: info.document_id, folder: folderPaths.get(label) ?? null })) }, + "[buildProjectDocContext] available docs", + ); + return { docIndex, docStore, folderPaths }; +} diff --git a/backend/src/lib/chatTools/index.ts b/backend/src/lib/chatTools/index.ts new file mode 100644 index 000000000..77d8b2dbb --- /dev/null +++ b/backend/src/lib/chatTools/index.ts @@ -0,0 +1,65 @@ +/** + * Public façade for the chatTools module (Phase 8 / CLEAN-30 split). + * + * Single entry point routes import from. Node resolves + * `import "../lib/chatTools"` to this file now that chatTools.ts is deleted. + * + * Locale routing for the system prompt will land here in M2 BILING-03. + */ + +// NOTE: `buildSystemPrompt` and `buildDocsSection` are exported here as the +// canonical builders for the system prompt and AVAILABLE DOCUMENTS block, +// but the live hot path currently constructs the system message inline in +// `buildMessages` (doc-context.ts) to preserve byte-identical SSE behavior +// across the Phase 8 split. M2 BILING-03 will route the live path through +// `buildSystemPrompt` so locale switching has a single seam. Until then +// these exports are forward-facing API only and are not invoked at runtime. +// Do not assume that editing these is sufficient to change runtime output. +export { + SYSTEM_PROMPT, + IDENTITY, + OUTPUT_FORMAT, + TOOL_POLICY, + BEHAVIOR, + buildDocsSection, + buildSystemPrompt, +} from "./system-prompts/en"; + +export { + TOOLS, + PROJECT_EXTRA_TOOLS, + TABULAR_TOOLS, + WORKFLOW_TOOLS, +} from "./tool-schemas"; + +export { runLLMStream } from "./stream"; +export type { AssistantEvent } from "./stream"; + +export { runToolCalls } from "./tool-runner"; + +export { + buildDocContext, + buildProjectDocContext, + buildMessages, + enrichWithPriorEvents, + resolveDoc, + resolveDocLabel, +} from "./doc-context"; + +export { buildWorkflowStore } from "./workflow-store"; + +export { parseCitations, extractAnnotations, CITATIONS_OPEN_TAG } from "./citations"; + +export type { + DocStore, + WorkflowStore, + DocIndex, + TabularCellStore, + ToolCall, + ChatMessage, + EditAnnotation, + TurnEditState, + DocEditedResult, + DocCreatedResult, + DocReplicatedResult, +} from "./types"; diff --git a/backend/src/lib/chatTools/llm-schemas.ts b/backend/src/lib/chatTools/llm-schemas.ts new file mode 100644 index 000000000..ca84ca9d1 --- /dev/null +++ b/backend/src/lib/chatTools/llm-schemas.ts @@ -0,0 +1,114 @@ +/** + * Shared zod schemas for LLM-output validation. + * + * These schemas validate JSON produced BY the LLM (tool call arguments, + * citation blocks, tabular cell results). They are intentionally separate + * from HTTP request body schemas (backend/src/lib/validate.ts). + * + * Used by parseLlmJson in citations.ts, tool-runner.ts, and tabular.ts. + */ + +import { z } from "zod"; + +// --------------------------------------------------------------------------- +// Chat citations (<CITATIONS> block in assistant replies) +// --------------------------------------------------------------------------- + +export const CitationSchema = z.object({ + ref: z.number().int(), + doc_id: z.string().min(1), + page: z.union([ + z.number().int(), + z.string().regex(/^\d+\s*-\s*\d+$/), + ]), + quote: z.string().min(1), +}); + +export const CitationsArraySchema = z.array(CitationSchema); + +// --------------------------------------------------------------------------- +// Tabular citations (<CITATIONS> block in tabular chat replies) +// --------------------------------------------------------------------------- + +export const TabularCitationSchema = z.object({ + ref: z.number().int(), + col_index: z.number().int().nonnegative(), + row_index: z.number().int().nonnegative(), + quote: z.string().min(1), +}); + +export const TabularCitationsArraySchema = z.array(TabularCitationSchema); + +// --------------------------------------------------------------------------- +// Tabular cell results (per-cell and per-column LLM output) +// --------------------------------------------------------------------------- + +export const TabularCellSchema = z + .object({ + summary: z.string().optional(), + value: z.string().optional(), + flag: z.enum(["green", "grey", "yellow", "red"]).optional(), + reasoning: z.string().optional(), + }) + .refine( + (d) => d.summary !== undefined || d.value !== undefined, + { message: "Cell must have summary or value" }, + ); + +// TabularCellLineSchema: per-line output from queryGeminiAllColumns. +// Uses .and() because .refine() returns ZodEffects which has no .extend(). +export const TabularCellLineSchema = TabularCellSchema.and( + z.object({ + column_index: z.number().int().nonnegative(), + }), +); + +// --------------------------------------------------------------------------- +// Tool argument schemas (keyed by tool name) +// --------------------------------------------------------------------------- + +export const ToolArgSchemas = { + read_document: z.object({ + doc_id: z.string(), + }), + find_in_document: z.object({ + doc_id: z.string(), + query: z.string(), + max_results: z.number().int().optional(), + context_chars: z.number().int().optional(), + }), + fetch_documents: z.object({ + doc_ids: z.array(z.string()), + }), + list_documents: z.object({}), + list_workflows: z.object({}), + read_workflow: z.object({ + workflow_id: z.string(), + }), + read_table_cells: z.object({ + col_indices: z.array(z.number().int()).optional(), + row_indices: z.array(z.number().int()).optional(), + }), + replicate_document: z.object({ + doc_id: z.string(), + count: z.number().int().min(1).max(20).optional(), + new_filename: z.string().optional(), + }), + generate_docx: z.object({ + title: z.string(), + landscape: z.boolean().optional(), + sections: z.unknown(), + }), + edit_document: z.object({ + doc_id: z.string(), + edits: z.array( + z.object({ + find: z.string(), + replace: z.string(), + context_before: z.string(), + context_after: z.string(), + reason: z.string().optional(), + }), + ), + }), +} as const; diff --git a/backend/src/lib/chatTools/parseLlmJson.ts b/backend/src/lib/chatTools/parseLlmJson.ts new file mode 100644 index 000000000..27b7a9750 --- /dev/null +++ b/backend/src/lib/chatTools/parseLlmJson.ts @@ -0,0 +1,42 @@ +/** + * parseLlmJson — zod-validated JSON parse helper for LLM output. + * + * Returns a Result-shaped value and NEVER throws. Parse failures are + * categorised as either JSON syntax errors or schema validation errors. + * Callers emit typed SSE error events on failure; this helper is solely + * responsible for producing the Result. + * + * Separate from parseBody (backend/src/lib/validate.ts) which validates + * HTTP request bodies and returns 400 — a different concern. + */ + +import { z } from "zod"; + +export type ParseLlmJsonResult<T> = + | { ok: true; data: T } + | { ok: false; error: string; raw: string }; + +export function parseLlmJson<T>( + raw: string, + schema: z.ZodSchema<T>, +): ParseLlmJsonResult<T> { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (e) { + return { + ok: false, + error: `JSON syntax: ${(e as Error).message}`, + raw, + }; + } + const result = schema.safeParse(parsed); + if (!result.success) { + return { + ok: false, + error: result.error.message, + raw, + }; + } + return { ok: true, data: result.data }; +} diff --git a/backend/src/lib/chatTools/stream.ts b/backend/src/lib/chatTools/stream.ts new file mode 100644 index 000000000..aa78e4a73 --- /dev/null +++ b/backend/src/lib/chatTools/stream.ts @@ -0,0 +1,368 @@ +/** + * runLLMStream — the assistant's per-turn streaming loop. + * + * Calls streamChatWithTools (provider-agnostic), drives the inline + * streamVisibleContent / flushVisibleTail drip animator (which strips + * the <CITATIONS> JSON tail from visible content), invokes runToolCalls + * on each tool-call batch, and re-enters the LLM with the updated + * conversation until the model produces a final response. + * + * The streamVisibleContent helpers stay inline because they close over + * per-call state (citationsOpenSeen, visibleTailBuffer, iterVisibleText). + * Only the regex constants (CITATIONS_OPEN_TAG) and the post-stream + * parseCitations helper come from ./citations. + * + * System prompt prepending currently lives in buildMessages (doc-context.ts) + * — this preserves byte-identical SSE behavior for the Phase 8 split. + * M2 BILING-03 will move locale-aware buildSystemPrompt wiring here. + */ + +import { + streamChatWithTools, + resolveModel, + DEFAULT_MAIN_MODEL, + type LlmMessage, + type OpenAIToolSchema, +} from "../llm"; +import { createServerSupabase } from "../supabase"; +import { runToolCalls } from "./tool-runner"; +import { logger } from "../logger"; +import { TOOLS, WORKFLOW_TOOLS } from "./tool-schemas"; +import { CITATIONS_OPEN_TAG, parseCitations } from "./citations"; +import { resolveDoc } from "./doc-context"; +import type { + DocStore, + DocIndex, + WorkflowStore, + TabularCellStore, + ToolCall, + EditAnnotation, + TurnEditState, +} from "./types"; + +export type AssistantEvent = + | { type: "reasoning"; text: string } + | { type: "doc_read"; filename: string; document_id?: string } + | { + type: "doc_find"; + filename: string; + query: string; + total_matches: number; + } + | { + type: "doc_created"; + filename: string; + download_url: string; + document_id?: string; + version_id?: string; + version_number?: number | null; + } + | { type: "doc_download"; filename: string; download_url: string } + | { + type: "doc_replicated"; + /** Source document being copied. */ + filename: string; + count: number; + copies: { + new_filename: string; + document_id: string; + version_id: string; + }[]; + } + | { type: "workflow_applied"; workflow_id: string; title: string } + | { + type: "doc_edited"; + filename: string; + document_id: string; + version_id: string; + /** Per-document monotonic Vn; null if backend couldn't determine it. */ + version_number: number | null; + download_url: string; + annotations: EditAnnotation[]; + } + | { type: "content"; text: string } + | { type: "citations_parse_error"; error: string } + | { type: "tool_args_parse_error"; tool: string; error: string } + | { type: "tabular_cell_parse_error"; col_index: number; doc_id: string; error: string }; + +export async function runLLMStream(params: { + apiMessages: unknown[]; + docStore: DocStore; + docIndex: DocIndex; + userId: string; + db: ReturnType<typeof createServerSupabase>; + write: (s: string) => void; + extraTools?: unknown[]; + workflowStore?: WorkflowStore; + tabularStore?: TabularCellStore; + buildCitations?: (fullText: string) => unknown[]; + model?: string; + apiKeys?: import("../llm").UserApiKeys; + /** + * If set, generate_docx will attach created docs to this project so + * they appear in the project sidebar. Leave null for general chats — + * generated docs still get persisted, but as standalone documents. + */ + projectId?: string | null; +}): Promise<{ fullText: string; events: AssistantEvent[] }> { + const { apiMessages, docStore, docIndex, userId, db, write, extraTools, workflowStore, tabularStore, buildCitations, model, apiKeys, projectId } = params; + const activeTools = extraTools?.length + ? [...TOOLS, ...WORKFLOW_TOOLS, ...extraTools] + : [...TOOLS, ...WORKFLOW_TOOLS]; + + // Extract system prompt; pass remaining turns to the adapter as + // plain user/assistant messages. + const rawMsgs = apiMessages as { role: string; content: string | null }[]; + const systemPrompt = + rawMsgs[0]?.role === "system" ? (rawMsgs[0].content ?? "") : ""; + logger.info({ systemPromptLength: systemPrompt.length }, "[runLLMStream] system prompt length"); + const chatMessages: LlmMessage[] = rawMsgs + .filter((m) => m.role !== "system") + .map((m) => ({ + role: m.role === "assistant" ? "assistant" : "user", + content: m.content ?? "", + })); + + const events: AssistantEvent[] = []; + // One assistant turn produces at most one document_versions row per + // edited doc. `runToolCalls` fires once per tool-call batch; the model + // may emit multiple batches in a single turn, so this map persists + // across batches to let subsequent edit_document calls overwrite the + // turn's existing version instead of creating a new one. + const turnEditState: TurnEditState = new Map(); + let fullText = ""; + let iterText = ""; + let iterVisibleText = ""; + let iterReasoning = ""; + let visibleTailBuffer = ""; + let citationsOpenSeen = false; + + const streamVisibleContent = (delta: string) => { + if (!delta) return; + if (citationsOpenSeen) return; + + const combined = visibleTailBuffer + delta; + const markerIdx = combined.indexOf(CITATIONS_OPEN_TAG); + if (markerIdx >= 0) { + const visible = combined.slice(0, markerIdx); + if (visible) { + iterVisibleText += visible; + write( + `data: ${JSON.stringify({ type: "content_delta", text: visible })}\n\n`, + ); + } + visibleTailBuffer = ""; + citationsOpenSeen = true; + return; + } + + const keep = Math.min(CITATIONS_OPEN_TAG.length - 1, combined.length); + const visible = combined.slice(0, combined.length - keep); + visibleTailBuffer = combined.slice(combined.length - keep); + if (visible) { + iterVisibleText += visible; + write( + `data: ${JSON.stringify({ type: "content_delta", text: visible })}\n\n`, + ); + } + }; + + const flushVisibleTail = () => { + if (citationsOpenSeen || !visibleTailBuffer) { + visibleTailBuffer = ""; + return; + } + iterVisibleText += visibleTailBuffer; + write( + `data: ${JSON.stringify({ type: "content_delta", text: visibleTailBuffer })}\n\n`, + ); + visibleTailBuffer = ""; + }; + + const flushText = () => { + if (!iterText) return; + fullText += iterText; + flushVisibleTail(); + if (iterVisibleText) { + events.push({ type: "content", text: iterVisibleText }); + } + iterText = ""; + iterVisibleText = ""; + visibleTailBuffer = ""; + citationsOpenSeen = false; + }; + + const selectedModel = resolveModel(model, DEFAULT_MAIN_MODEL); + + await streamChatWithTools({ + model: selectedModel, + systemPrompt, + messages: chatMessages, + tools: activeTools as OpenAIToolSchema[], + maxIterations: 10, + apiKeys, + enableThinking: true, + callbacks: { + onContentDelta: (delta) => { + iterText += delta; + streamVisibleContent(delta); + }, + onReasoningDelta: (delta) => { + iterReasoning += delta; + write( + `data: ${JSON.stringify({ type: "reasoning_delta", text: delta })}\n\n`, + ); + }, + onReasoningBlockEnd: () => { + if (!iterReasoning) return; + events.push({ type: "reasoning", text: iterReasoning }); + write( + `data: ${JSON.stringify({ type: "reasoning_block_end" })}\n\n`, + ); + iterReasoning = ""; + }, + // Fires after Claude's turn ends with stop_reason=tool_use, before + // the tool actually runs. Flushes any buffered assistant text so + // it's emitted in chronological order, then signals the client so + // it can open a fresh PreResponseWrapper (shows "Working…") while + // the tool executes — avoids the dead gap between message_stop + // and the first tool-specific event. + onToolCallStart: (call) => { + flushText(); + write( + `data: ${JSON.stringify({ + type: "tool_call_start", + name: call.name, + })}\n\n`, + ); + }, + }, + runTools: async (calls) => { + // Emit any text the model produced before this tool turn so the + // UI sees it before the tool results stream in. + flushText(); + + const toolCalls: ToolCall[] = calls.map((c) => ({ + id: c.id, + function: { + name: c.name, + arguments: JSON.stringify(c.input), + }, + })); + const { + toolResults, + docsRead, + docsFound, + docsCreated, + docsReplicated, + workflowsApplied, + docsEdited, + } = await runToolCalls( + toolCalls, + docStore, + userId, + db, + write, + workflowStore, + tabularStore, + docIndex, + turnEditState, + projectId, + ); + for (const r of docsRead) { + events.push({ + type: "doc_read", + filename: r.filename, + document_id: r.document_id, + }); + } + for (const f of docsFound) { + events.push({ + type: "doc_find", + filename: f.filename, + query: f.query, + total_matches: f.total_matches, + }); + } + for (const dl of docsCreated) { + events.push({ + type: "doc_created", + filename: dl.filename, + download_url: dl.download_url, + document_id: dl.document_id, + version_id: dl.version_id, + version_number: dl.version_number ?? null, + }); + } + for (const r of docsReplicated) { + events.push({ + type: "doc_replicated", + filename: r.filename, + count: r.count, + copies: r.copies, + }); + } + for (const wf of workflowsApplied) { + events.push({ + type: "workflow_applied", + workflow_id: wf.workflow_id, + title: wf.title, + }); + } + for (const e of docsEdited) { + events.push({ + type: "doc_edited", + filename: e.filename, + document_id: e.document_id, + version_id: e.version_id, + version_number: e.version_number, + download_url: e.download_url, + annotations: e.annotations, + }); + } + + // Index alignment would break if any tool branch skips its + // push (unhandled tool name, disabled store, guard failure). + // Each tool_result already carries its tool_call_id, so key off + // that directly — and fall back to an error result for any + // tool_use that didn't produce one, so Claude's next request + // has a tool_result for every tool_use it sent. + const resultByCallId = new Map<string, string>(); + for (const r of toolResults) { + const row = r as { tool_call_id: string; content?: unknown }; + resultByCallId.set(row.tool_call_id, String(row.content ?? "")); + } + return toolCalls.map((c) => ({ + tool_use_id: c.id, + content: + resultByCallId.get(c.id) ?? + JSON.stringify({ + error: `Tool '${c.function.name}' is not available.`, + }), + })); + }, + }); + + flushText(); + + // Parse and emit citations from <CITATIONS> block + const citations = buildCitations + ? buildCitations(fullText) + : parseCitations(fullText, write).map((c) => { + const docInfo = resolveDoc(c.doc_id, docIndex); + return { + ref: c.ref, + doc_id: c.doc_id, + document_id: docInfo?.document_id, + version_id: docInfo?.version_id ?? null, + version_number: docInfo?.version_number ?? null, + filename: docInfo?.filename ?? c.doc_id, + page: c.page, + quote: c.quote, + }; + }); + write(`data: ${JSON.stringify({ type: "citations", citations })}\n\n`); + write("data: [DONE]\n\n"); + + return { fullText, events }; +} diff --git a/backend/src/lib/chatTools/system-prompts/en.ts b/backend/src/lib/chatTools/system-prompts/en.ts new file mode 100644 index 000000000..748c8d1b3 --- /dev/null +++ b/backend/src/lib/chatTools/system-prompts/en.ts @@ -0,0 +1,90 @@ +/** + * English system prompt for the Mike assistant. + * + * Composed-sections shape (Phase 8 D-01): named string constants per + * section + dynamic buildDocsSection. buildSystemPrompt joins them. + * + * M2 BILING-03 will add system-prompts/nl.ts with the same export shape. + * The "Mike" name stays in this file; M2 BILING-07 handles the rename. + */ + +export const IDENTITY = `You are Mike, an AI legal assistant that helps lawyers and legal professionals analyze documents, answer legal questions, and draft legal documents.`; + +export const OUTPUT_FORMAT = `DOCUMENT CITATION INSTRUCTIONS: +When you reference specific content from a document, place a numbered marker [1], [2], etc. inline in your prose at the point of reference. + +After your complete response, append a <CITATIONS> block containing a JSON array with one entry per marker: + +<CITATIONS> +[ + {"ref": 1, "doc_id": "doc-0", "page": 3, "quote": "exact verbatim text from the document"}, + {"ref": 2, "doc_id": "doc-1", "page": "41-42", "quote": "Section 4.2 describes the procedure [[PAGE_BREAK]] in all material respects."} +] +</CITATIONS> + +CRITICAL: The number inside the [N] marker in your prose is the "ref" value of a citation entry in the <CITATIONS> block — it is NOT a page number, footnote number, section number, or any other number that appears in the document. The marker [1] refers to the entry with "ref": 1 in the JSON block; [2] refers to "ref": 2; and so on. Refs are simple sequential integers you assign (1, 2, 3, …) in the order citations appear in your prose. Never use a page number or a document's own numbering as the marker number. Every [N] you write in prose MUST have a matching {"ref": N, ...} entry in the JSON block. + +Rules: +- Only cite text that appears verbatim in the provided documents +- In every <CITATIONS> entry, "doc_id" MUST be the exact chat-local document label you were given (for example "doc-0"). Never use a filename, document UUID, or any other identifier in "doc_id" +- Keep quotes short (ideally ≤ 25 words) and narrowly scoped to the specific claim. Don't reuse one quote to support multiple different claims — give each its own citation +- "page" refers to the sequential [Page N] marker in the text you were given (1-indexed from the first page). IGNORE any page numbers printed inside the document itself (footers, roman numerals, etc.) +- For a single-page quote, set "page" to an integer. If a quote is one continuous sentence that spans two pages, set "page" to "N-M" and insert [[PAGE_BREAK]] in the quote at the page break. Otherwise, use separate citations for text on different pages +- Put the <CITATIONS> block at the very end of the response. Omit it entirely if there are no citations`; + +export const TOOL_POLICY = `DOCX GENERATION: +If asked to draft or generate a document, use the generate_docx tool to produce a downloadable Word document. Always use this tool rather than just displaying the document content inline when the user asks for a document to be created. +If the user follows up on a document you just generated and asks for changes (e.g. "make section 3 longer", "add a termination clause", "change the parties"), default to calling edit_document on that newly generated document — do NOT call generate_docx again to regenerate the whole document. Only fall back to generate_docx if the user explicitly asks for a brand-new document or the change is so sweeping that an edit would not be coherent. +After calling generate_docx, do NOT include any download links, URLs, or markdown links to the document in your prose response — the download card is presented automatically by the UI. Do not describe formatting choices such as orientation or layout. +After calling generate_docx, you MUST call read_document on the returned doc_id before writing your prose response. Base your description on the generated document's actual text, not on memory of what you intended to generate. +Your prose response MUST include a short description of the generated document: what it is, its structure (key sections/clauses), and — if the draft was informed by any provided source documents — which sources you drew from and how. Keep it concise (typically 3–8 sentences or a short bulleted list). Refer to the document by filename, never by a download link. +When the description makes factual claims about the contents of the newly generated document, cite the generated document with [N] markers and a <CITATIONS> block exactly as specified in the DOCUMENT CITATION INSTRUCTIONS above. If you also make factual claims about provided source documents, cite those source documents separately. In every citation entry, use the exact chat-local doc_id label for the cited document. Omit the <CITATIONS> block if the description makes no such claims. +Heading hierarchy: always use Heading 1 before introducing Heading 2, Heading 2 before Heading 3, and so on. Never skip levels (e.g. do not jump from Heading 1 to Heading 3). +Numbering: all numbering MUST start from 1, never 0. This applies at every level of the hierarchy — use 1., 1.1, 1.1.1, 1.1.1.1, etc. Never produce 0., 0.1, 1.0, 1.0.1, or any other sequence that begins a level with 0. +Never duplicate the numbering prefix in heading text. The heading's own numbering is applied automatically by the document generator, so the heading text must contain the title only — do NOT prepend "1.", "1.1", "2.", etc. into the heading text itself. For example, a Heading 1 titled "Introduction" must be passed as "Introduction", never as "1. Introduction" (which would render as "1. 1. Introduction"). The same rule applies at every level. +Contracts: when generating a contract or agreement, always include a signatures block at the very end of the document on its own page. Set pageBreak: true on that final section so it starts on a fresh page, and include a signature line for each party — typically the party name followed by lines for "By:", "Name:", "Title:", and "Date:". Do not number the signatures heading; put the signature block in the section's content rather than as a numbered heading. +Contract preambles: the preamble of a contract (the opening recitals, parties block, "WHEREAS" clauses, and any introductory narrative before the first operative clause) must NOT be numbered. Render these as unnumbered content (plain paragraphs or an unnumbered heading), and begin numbering only at the first operative clause/section. + +DOCUMENT EDITING: +When using edit_document, any edit that adds, removes, or reorders a numbered clause, section, sub-clause, schedule, exhibit, or list item shifts every downstream number. You MUST update all affected numbering AND every cross-reference to those numbers in the same edit_document call: +- Renumber the sibling clauses/sections/sub-clauses that follow the change so the sequence stays contiguous (e.g. if you insert a new Section 4, existing Sections 4, 5, 6… become 5, 6, 7…). +- Find every in-document reference to the shifted numbers — e.g. "see Section 5", "pursuant to Clause 4.2(b)", "as set out in Schedule 3", "defined in Section 2.1" — and update them to the new numbers. Include defined-term blocks, cross-references in recitals, schedules, and exhibits. +- Before issuing the edits, scan the full document (use read_document or find_in_document) to enumerate affected cross-references; do not assume references only appear near the change site. +- If you are uncertain whether a reference points to the shifted number or an unrelated number, err on the side of including it as an edit and explain in the reason field. +- When deleting square brackets, delete both the opening \`[\` and the closing \`]\`. Never leave behind an unmatched square bracket after an edit. + +WORKFLOWS: +When a user message begins with a [Workflow: <title> (id: <id>)] marker, the user has selected a workflow and you MUST apply it. Immediately call the read_workflow tool with that exact id to load the workflow's full prompt, then follow those instructions for the current turn. Do this before producing any other output or calling any other tools (aside from any document reads the workflow requires). Do not ask the user to confirm — the selection itself is the instruction to apply the workflow.`; + +export const BEHAVIOR = `DOCUMENT NAMING IN PROSE: +The chat-local labels ("doc-0", "doc-1", "doc-N", …) are internal handles for tool calls and citation JSON ONLY. NEVER write them in your prose response or in any text the user reads — not in body text, not in headings, not in lists, not in tool-activity descriptions. The user does not know what "doc-0" means and seeing it is jarring. When referring to a document in prose, always use its filename (e.g. "the NDA draft" or "nda_v1.docx"). This rule applies to every word streamed back to the user; the only places "doc-N" identifiers are allowed are inside tool-call arguments and inside the <CITATIONS> JSON block's "doc_id" field. + +GENERAL GUIDANCE: +- Be precise and professional +- Cite the specific document and quote when making claims about document content +- When no documents are provided, answer based on your legal knowledge +- Do not fabricate document content +- Do not use emojis in your responses. +`; + +export function buildDocsSection( + docs: { doc_id: string; filename: string; folder_path?: string }[], +): string { + if (!docs.length) return ""; + const lines = docs.map((d) => `- ${d.doc_id}: ${d.folder_path ? d.folder_path + "/" : ""}${d.filename}`); + return `AVAILABLE DOCUMENTS:\n${lines.join("\n")}`; +} + +export function buildSystemPrompt(args: { + docs?: { doc_id: string; filename: string; folder_path?: string }[]; + systemPromptExtra?: string; +} = {}): string { + const sections = [IDENTITY, OUTPUT_FORMAT, TOOL_POLICY, BEHAVIOR]; + const docsSection = buildDocsSection(args.docs ?? []); + const parts = [...sections]; + if (docsSection) parts.push(docsSection); + if (args.systemPromptExtra) parts.push(args.systemPromptExtra); + return parts.join("\n\n"); +} + +export const SYSTEM_PROMPT = [IDENTITY, OUTPUT_FORMAT, TOOL_POLICY, BEHAVIOR].join("\n\n"); diff --git a/backend/src/lib/chatTools/tool-runner.ts b/backend/src/lib/chatTools/tool-runner.ts new file mode 100644 index 000000000..ae20de233 --- /dev/null +++ b/backend/src/lib/chatTools/tool-runner.ts @@ -0,0 +1,516 @@ +/** + * Tool-call dispatcher: parses tool-call arguments and routes to the + * matching tools/*.ts runner. Returns aggregated docsRead / docsFound / + * docsCreated / docsReplicated / docsEdited / workflowsApplied arrays + * for the streaming layer to attach to the next-iteration prompt. + */ + +import { z } from "zod"; +import { createServerSupabase } from "../supabase"; +import type { + DocStore, + DocIndex, + WorkflowStore, + TabularCellStore, + ToolCall, + TurnEditState, + DocCreatedResult, + DocReplicatedResult, + DocEditedResult, +} from "./types"; +import { resolveDocLabel } from "./doc-context"; +import { logger } from "../logger"; +import { parseLlmJson } from "./parseLlmJson"; +import { ToolArgSchemas } from "./llm-schemas"; +import { runReadDocument } from "./tools/read-document"; +import { runFindInDocument } from "./tools/find-in-document"; +import { runListDocuments } from "./tools/list-documents"; +import { runFetchDocuments } from "./tools/fetch-documents"; +import { runReadWorkflow } from "./tools/read-workflow"; +import { runReplicateDocument } from "./tools/replicate-document"; +import { runGenerateDocx } from "./tools/generate-docx"; +import { runEditDocument } from "./tools/edit-document"; + +export async function runToolCalls( + toolCalls: ToolCall[], + docStore: DocStore, + userId: string, + db: ReturnType<typeof createServerSupabase>, + write: (s: string) => void, + workflowStore?: WorkflowStore, + tabularStore?: TabularCellStore, + docIndex?: DocIndex, + turnEditState?: TurnEditState, + projectId?: string | null, +): Promise<{ + toolResults: unknown[]; + docsRead: { filename: string; document_id?: string }[]; + docsFound: { filename: string; query: string; total_matches: number }[]; + docsCreated: DocCreatedResult[]; + docsReplicated: DocReplicatedResult[]; + workflowsApplied: { workflow_id: string; title: string }[]; + docsEdited: DocEditedResult[]; +}> { + const toolResults: unknown[] = []; + const docsRead: { filename: string; document_id?: string }[] = []; + const docsFound: { + filename: string; + query: string; + total_matches: number; + }[] = []; + const docsCreated: DocCreatedResult[] = []; + const docsReplicated: DocReplicatedResult[] = []; + const workflowsApplied: { workflow_id: string; title: string }[] = []; + const docsEdited: DocEditedResult[] = []; + + for (const tc of toolCalls) { + let args: Record<string, unknown> = {}; + const knownSchema: z.ZodSchema<Record<string, unknown>> = + (ToolArgSchemas[tc.function.name as keyof typeof ToolArgSchemas] as z.ZodSchema<Record<string, unknown>> | undefined) ?? + z.object({}).passthrough(); + const argsResult = parseLlmJson( + tc.function.arguments || "{}", + knownSchema, + ); + if (argsResult.ok) { + args = argsResult.data as Record<string, unknown>; + } else { + write( + `data: ${JSON.stringify({ type: "tool_args_parse_error", tool: tc.function.name, error: argsResult.error })}\n\n`, + ); + logger.warn( + { err: argsResult.error, tool: tc.function.name }, + "[chatTools/tool-runner] tool args parse failed", + ); + // Skip this tool call entirely — do not execute with empty args + continue; + } + + if (tc.function.name === "read_document") { + const rawDocId = args.doc_id as string; + const docId = + resolveDocLabel(rawDocId, docStore, docIndex) ?? rawDocId; + const content = await runReadDocument({ docLabel: docId, docStore, write, docIndex, db }); + const filename = docStore.get(docId)?.filename; + const documentId = docIndex?.[docId]?.document_id; + if (filename) docsRead.push({ filename, document_id: documentId }); + toolResults.push({ role: "tool", tool_call_id: tc.id, content }); + + } else if (tc.function.name === "find_in_document") { + const rawDocId = args.doc_id as string; + const docId = + resolveDocLabel(rawDocId, docStore, docIndex) ?? rawDocId; + const query = (args.query as string) ?? ""; + const maxResults = typeof args.max_results === "number" ? args.max_results : undefined; + const contextChars = typeof args.context_chars === "number" ? args.context_chars : undefined; + const content = await runFindInDocument({ + docLabel: docId, + query, + maxResults, + contextChars, + docStore, + write, + docIndex, + db, + }); + const filename = docStore.get(docId)?.filename; + if (filename) { + let totalMatches = 0; + // NOTE: This parses OUR tool result, not LLM output. Per Phase 10 / CLEAN-23, + // this site is intentionally NOT wrapped with parseLlmJson — failure here + // would be an internal bug, not LLM misbehavior. + try { + const parsed = JSON.parse(content) as { + total_matches?: number; + }; + totalMatches = parsed.total_matches ?? 0; + } catch { + /* ignore — still record the find attempt */ + } + docsFound.push({ + filename, + query, + total_matches: totalMatches, + }); + } + toolResults.push({ role: "tool", tool_call_id: tc.id, content }); + + } else if (tc.function.name === "list_documents") { + const content = runListDocuments({ docStore }); + toolResults.push({ + role: "tool", + tool_call_id: tc.id, + content, + }); + + } else if (tc.function.name === "fetch_documents") { + const rawDocIds = (args.doc_ids as string[]) ?? []; + const docIds = rawDocIds.map( + (id) => resolveDocLabel(id, docStore, docIndex) ?? id, + ); + const { content, docsRead: fetched } = await runFetchDocuments({ + docIds, + docStore, + write, + docIndex, + db, + }); + for (const r of fetched) docsRead.push(r); + toolResults.push({ + role: "tool", + tool_call_id: tc.id, + content, + }); + + } else if (tc.function.name === "list_workflows") { + const list = workflowStore + ? Array.from(workflowStore.entries()).map(([id, w]) => ({ id, title: w.title })) + : []; + toolResults.push({ role: "tool", tool_call_id: tc.id, content: JSON.stringify(list) }); + + } else if (tc.function.name === "read_workflow") { + const wfId = args.workflow_id as string; + const { content, applied } = runReadWorkflow({ + workflowId: wfId, + workflowStore, + write, + }); + if (applied) workflowsApplied.push(applied); + toolResults.push({ + role: "tool", + tool_call_id: tc.id, + content, + }); + + } else if (tc.function.name === "read_table_cells" && tabularStore) { + const colIndices = args.col_indices as number[] | undefined; + const rowIndices = args.row_indices as number[] | undefined; + + const filteredCols = colIndices?.length + ? tabularStore.columns.filter((_, i) => colIndices.includes(i)) + : tabularStore.columns; + const filteredDocs = rowIndices?.length + ? tabularStore.documents.filter((_, i) => rowIndices.includes(i)) + : tabularStore.documents; + + const label = `${filteredCols.length} ${filteredCols.length === 1 ? "column" : "columns"} × ${filteredDocs.length} ${filteredDocs.length === 1 ? "row" : "rows"}`; + write(`data: ${JSON.stringify({ type: "doc_read_start", filename: label })}\n\n`); + + const lines: string[] = []; + for (const col of filteredCols) { + const colPos = tabularStore.columns.findIndex((c) => c.index === col.index); + for (const doc of filteredDocs) { + const rowPos = tabularStore.documents.findIndex((d) => d.id === doc.id); + const cell = tabularStore.cells.get(`${col.index}:${doc.id}`); + lines.push(`[COL:${colPos} "${col.name}" | ROW:${rowPos} "${doc.filename}"]`); + if (cell?.summary) { + lines.push(`Summary: ${cell.summary}`); + if (cell.flag) lines.push(`Flag: ${cell.flag}`); + if (cell.reasoning) lines.push(`Reasoning: ${cell.reasoning}`); + } else { + lines.push(`(not yet generated)`); + } + lines.push(""); + } + } + + write(`data: ${JSON.stringify({ type: "doc_read", filename: label })}\n\n`); + docsRead.push({ filename: label }); + toolResults.push({ + role: "tool", + tool_call_id: tc.id, + content: lines.join("\n") || "No cells found.", + }); + + } else if (tc.function.name === "edit_document" && docIndex) { + const rawDocId = args.doc_id as string; + const editsRaw = args.edits as unknown[] | undefined; + const docId = + resolveDocLabel(rawDocId, docStore, docIndex) ?? rawDocId; + const docInfo = docStore.get(docId); + const indexed = docIndex?.[docId]; + + const emitEditError = ( + filename: string, + documentId: string, + error: string, + ) => { + // Surface the failure as a failed "Edited" block in the UI + // (start → done-with-error) so it matches the shape the + // success/late-failure paths already use. + write( + `data: ${JSON.stringify({ + type: "doc_edited_start", + filename, + })}\n\n`, + ); + write( + `data: ${JSON.stringify({ + type: "doc_edited", + filename, + document_id: documentId, + version_id: "", + download_url: "", + annotations: [], + error, + })}\n\n`, + ); + }; + + if (!docInfo || !indexed) { + const err = `Document '${docId}' not found in this chat's attachments.`; + emitEditError(docId, indexed?.document_id ?? "", err); + toolResults.push({ + role: "tool", + tool_call_id: tc.id, + content: JSON.stringify({ error: err }), + }); + } else if ( + !Array.isArray(editsRaw) || + editsRaw.length === 0 + ) { + const err = "edits array is required and must not be empty."; + emitEditError(docInfo.filename, indexed.document_id, err); + toolResults.push({ + role: "tool", + tool_call_id: tc.id, + content: JSON.stringify({ error: err }), + }); + } else if (docInfo.file_type !== "docx") { + const err = "edit_document only supports .docx files."; + emitEditError(docInfo.filename, indexed.document_id, err); + toolResults.push({ + role: "tool", + tool_call_id: tc.id, + content: JSON.stringify({ error: err }), + }); + } else { + write( + `data: ${JSON.stringify({ + type: "doc_edited_start", + filename: docInfo.filename, + })}\n\n`, + ); + const edits = (editsRaw as Record<string, unknown>[]).map( + (e) => ({ + find: String(e.find ?? ""), + replace: String(e.replace ?? ""), + context_before: String(e.context_before ?? ""), + context_after: String(e.context_after ?? ""), + reason: e.reason ? String(e.reason) : undefined, + }), + ); + const reuseVersion = turnEditState?.get(indexed.document_id); + const result = await runEditDocument({ + documentId: indexed.document_id, + userId, + edits, + db, + reuseVersion, + }); + + if (result.ok) { + turnEditState?.set(indexed.document_id, { + versionId: result.version_id, + versionNumber: result.version_number, + storagePath: result.storage_path, + }); + // Keep the chat-local doc label pointed at the latest + // edited version so any follow-up read_document call in + // the same assistant turn reads and cites the same bytes. + if (docIndex[docId]) { + docIndex[docId] = { + ...docIndex[docId], + version_id: result.version_id, + version_number: result.version_number, + }; + } + const currentDocStore = docStore.get(docId); + if (currentDocStore) { + docStore.set(docId, { + ...currentDocStore, + storage_path: result.storage_path, + }); + } + const payload: DocEditedResult = { + filename: docInfo.filename, + document_id: indexed.document_id, + version_id: result.version_id, + version_number: result.version_number, + download_url: result.download_url, + annotations: result.annotations, + }; + docsEdited.push(payload); + write( + `data: ${JSON.stringify({ + type: "doc_edited", + ...payload, + })}\n\n`, + ); + toolResults.push({ + role: "tool", + tool_call_id: tc.id, + content: JSON.stringify({ + ok: true, + doc_id: docId, + document_id: indexed.document_id, + version_id: result.version_id, + version_number: result.version_number, + applied: result.annotations.length, + errors: result.errors, + }), + }); + } else { + write( + `data: ${JSON.stringify({ + type: "doc_edited", + filename: docInfo.filename, + document_id: indexed.document_id, + version_id: "", + download_url: "", + annotations: [], + error: result.error, + })}\n\n`, + ); + toolResults.push({ + role: "tool", + tool_call_id: tc.id, + content: JSON.stringify({ + ok: false, + error: result.error, + }), + }); + } + } + + } else if (tc.function.name === "replicate_document" && docIndex) { + const rawDocId = args.doc_id as string; + const requestedFilename = + typeof args.new_filename === "string" && + args.new_filename.trim() + ? args.new_filename.trim() + : null; + // CLEAN-51: hard-reject out-of-range count; model must retry. + const rawCount = + typeof args.count === "number" && Number.isFinite(args.count) + ? Math.floor(args.count) + : 1; + if (rawCount < 1 || rawCount > 20) { + toolResults.push({ + role: "tool", + tool_call_id: tc.id, + content: JSON.stringify({ + ok: false, + error: `count must be between 1 and 20 (got ${rawCount})`, + }), + }); + continue; + } + const requestedCount = rawCount; + const sourceLabel = + resolveDocLabel(rawDocId, docStore, docIndex) ?? rawDocId; + + const { toolResult, replicated } = await runReplicateDocument({ + rawDocId, + requestedFilename, + requestedCount, + sourceLabel, + docStore, + docIndex, + userId, + projectId, + db, + write, + toolCallId: tc.id, + }); + if (replicated) docsReplicated.push(replicated); + toolResults.push(toolResult); + + } else if (tc.function.name === "generate_docx") { + const title = args.title as string; + const landscape = !!(args.landscape); + logger.info({ title, landscape, landscapeArg: args.landscape }, "[generate_docx] tool args"); + const previewFilename = `${(title.replace(/[^a-zA-Z0-9 _-]/g, "").trim().slice(0, 64) || "document")}.docx`; + write(`data: ${JSON.stringify({ type: "doc_created_start", filename: previewFilename })}\n\n`); + const result = await runGenerateDocx({ + title, + sections: args.sections as unknown[], + userId, + db, + options: { landscape, projectId: projectId ?? null }, + }); + let newDocLabel: string | null = null; + if ("filename" in result && "download_url" in result) { + const dlFilename = result.filename as string; + const dlUrl = result.download_url as string; + const documentId = (result as { document_id?: string }).document_id; + const versionId = (result as { version_id?: string }).version_id; + const versionNumber = (result as { version_number?: number }).version_number ?? null; + const storagePath = (result as { storage_path?: string }).storage_path; + + // Register the generated doc in the chat context so + // edit_document (and read_document / find_in_document) + // can act on it within the same assistant turn. New label + // is the next free `doc-N` index. Subsequent turns pick + // it up via the normal attachment/project doc query. + if (documentId && storagePath && docIndex) { + const existingLabels = new Set(Object.keys(docIndex)); + let i = 0; + while (existingLabels.has(`doc-${i}`)) i++; + newDocLabel = `doc-${i}`; + docIndex[newDocLabel] = { + document_id: documentId, + filename: dlFilename, + }; + docStore.set(newDocLabel, { + storage_path: storagePath, + file_type: "docx", + filename: dlFilename, + }); + } + + write( + `data: ${JSON.stringify({ + type: "doc_created", + filename: dlFilename, + download_url: dlUrl, + document_id: documentId, + version_id: versionId, + version_number: versionNumber, + })}\n\n`, + ); + docsCreated.push({ + filename: dlFilename, + download_url: dlUrl, + document_id: documentId, + version_id: versionId, + version_number: versionNumber, + }); + } else { + write(`data: ${JSON.stringify({ type: "doc_created", filename: previewFilename, download_url: "" })}\n\n`); + } + // Surface the chat-local doc label in the tool result so the + // model can pass it as `doc_id` to edit_document / read_document + // / find_in_document in the same turn. Without this the model + // only sees the DB UUID, which isn't valid as a doc_id anchor. + const toolResultPayload = newDocLabel + ? { ...(result as Record<string, unknown>), doc_id: newDocLabel } + : result; + toolResults.push({ + role: "tool", + tool_call_id: tc.id, + content: JSON.stringify(toolResultPayload), + }); + } + } + + return { + toolResults, + docsRead, + docsFound, + docsCreated, + docsReplicated, + workflowsApplied, + docsEdited, + }; +} diff --git a/backend/src/lib/chatTools/tool-schemas.ts b/backend/src/lib/chatTools/tool-schemas.ts new file mode 100644 index 000000000..8c1a11cfe --- /dev/null +++ b/backend/src/lib/chatTools/tool-schemas.ts @@ -0,0 +1,287 @@ +/** + * OpenAI-style tool-schema arrays for the Mike assistant. + * + * Pure data — extracted verbatim from the original chatTools.ts during + * the Phase 8 (CLEAN-30) split. No runtime imports. + */ + +export const PROJECT_EXTRA_TOOLS = [ + { + type: "function", + function: { + name: "list_documents", + description: + "List all documents available in the project. Returns each document's ID, filename, and file type. Call this to discover what documents are available before deciding which ones to read.", + parameters: { type: "object", properties: {} }, + }, + }, + { + type: "function", + function: { + name: "fetch_documents", + description: + "Read the full text content of multiple documents in a single call. Use this instead of calling read_document repeatedly when you need to read several documents at once.", + parameters: { + type: "object", + properties: { + doc_ids: { + type: "array", + items: { type: "string" }, + description: + "Array of document IDs to read (e.g. ['doc-0', 'doc-2'])", + }, + }, + required: ["doc_ids"], + }, + }, + }, + { + type: "function", + function: { + name: "replicate_document", + description: + "Make byte-for-byte copies of an existing project document as new project documents. Use when the user wants standalone copies to edit (e.g. 'use this NDA as a template', 'give me three drafts I can adapt') without modifying the original. Pass `count` to create multiple copies in a single call rather than calling the tool repeatedly. Returns the new doc_id slugs so you can immediately call edit_document / read_document on them.", + parameters: { + type: "object", + properties: { + doc_id: { + type: "string", + description: + "ID of the source document to copy (e.g. 'doc-0').", + }, + count: { + type: "integer", + description: + "How many copies to create. Defaults to 1. Maximum 20.", + minimum: 1, + maximum: 20, + }, + new_filename: { + type: "string", + description: + "Optional base filename. With count > 1, copies are suffixed (e.g. 'Foo (1).docx', 'Foo (2).docx'). Extension is forced to match the source.", + }, + }, + required: ["doc_id"], + }, + }, + }, +]; + +export const TABULAR_TOOLS = [ + { + type: "function", + function: { + name: "read_table_cells", + description: + "Read the extracted cell content from the tabular review. Each cell contains the value extracted for a specific column from a specific document. Pass col_indices and/or row_indices (0-based) to read a subset; omit either to read all columns or all rows.", + parameters: { + type: "object", + properties: { + col_indices: { + type: "array", + items: { type: "integer" }, + description: + "0-based column indices to read (e.g. [0, 2]). Omit to read all columns.", + }, + row_indices: { + type: "array", + items: { type: "integer" }, + description: + "0-based document (row) indices to read (e.g. [0, 1]). Omit to read all rows.", + }, + }, + }, + }, + }, +]; + +export const WORKFLOW_TOOLS = [ + { + type: "function", + function: { + name: "list_workflows", + description: + "List all workflows available to the user. Returns each workflow's ID and title. Call this when the user asks to run a workflow, apply a template, or you need to discover what workflows exist.", + parameters: { type: "object", properties: {} }, + }, + }, + { + type: "function", + function: { + name: "read_workflow", + description: + "Read the full instructions (prompt) of a workflow by its ID. Call this after list_workflows to load a specific workflow's prompt, then follow those instructions.", + parameters: { + type: "object", + properties: { + workflow_id: { + type: "string", + description: "The workflow ID to read", + }, + }, + required: ["workflow_id"], + }, + }, + }, +]; + +export const TOOLS = [ + { + type: "function", + function: { + name: "read_document", + description: + "Read the full text content of a document attached by the user. Always call this before answering questions about, summarising, or citing from a document.", + parameters: { + type: "object", + properties: { + doc_id: { + type: "string", + description: + "The document ID to read (e.g. 'doc-0', 'doc-1')", + }, + }, + required: ["doc_id"], + }, + }, + }, + { + type: "function", + function: { + name: "find_in_document", + description: + "Search for specific strings inside a document — a Ctrl+F equivalent. Returns each match with surrounding context so you can locate and quote the exact text without reading the whole document. Matching is case-insensitive and whitespace-tolerant. Use this for targeted lookups (e.g. finding a clause title, party name, or a specific phrase) rather than reading the whole document.", + parameters: { + type: "object", + properties: { + doc_id: { + type: "string", + description: + "The document ID to search (e.g. 'doc-0').", + }, + query: { + type: "string", + description: + "The string to search for. Matching is case-insensitive and collapses runs of whitespace, so 'Section 4.2' matches 'section 4.2'.", + }, + max_results: { + type: "integer", + description: + "Maximum number of matches to return (default 20). Use a smaller value for common terms.", + }, + context_chars: { + type: "integer", + description: + "Characters of surrounding context to include on each side of a match (default 80).", + }, + }, + required: ["doc_id", "query"], + }, + }, + }, + { + type: "function", + function: { + name: "generate_docx", + description: + "Generate a Word (.docx) document from structured content. Use this when the user asks you to draft, create, or produce a legal document. Returns a download URL for the generated file.", + parameters: { + type: "object", + properties: { + title: { + type: "string", + description: "Document title (used as filename and heading)", + }, + landscape: { + type: "boolean", + description: "Set to true for landscape page orientation. Default is portrait.", + }, + sections: { + type: "array", + description: "List of document sections. Each section may contain a heading, prose content, or a table.", + items: { + type: "object", + properties: { + heading: { type: "string", description: "Optional section heading" }, + level: { type: "integer", description: "Heading level: 1, 2, or 3" }, + content: { type: "string", description: "Prose text content (paragraphs separated by double newlines)" }, + pageBreak: { type: "boolean", description: "Set to true to start this section on a new page. Use for contract signature pages." }, + table: { + type: "object", + description: "Optional table to render in this section", + properties: { + headers: { + type: "array", + items: { type: "string" }, + description: "Column header labels", + }, + rows: { + type: "array", + items: { + type: "array", + items: { type: "string" }, + }, + description: "Array of rows, each row is an array of cell strings matching the headers order", + }, + }, + required: ["headers", "rows"], + }, + }, + }, + }, + }, + required: ["title", "sections"], + }, + }, + }, + { + type: "function", + function: { + name: "edit_document", + description: + "Propose edits to a user-attached .docx as tracked changes. Each edit is a precise, minimal substitution of specific words/characters, NOT a whole-line or paragraph replacement. Use read_document first. Anchor each edit with short before/after context so it can be located unambiguously. Returns per-edit annotations the UI will render as Accept/Reject cards and a download link to the edited document.", + parameters: { + type: "object", + properties: { + doc_id: { + type: "string", + description: "Document slug (e.g. 'doc-0').", + }, + edits: { + type: "array", + description: "List of precise substitutions.", + items: { + type: "object", + properties: { + find: { + type: "string", + description: + "Exact substring to replace (keep it as short as possible — ideally just the words/chars being changed).", + }, + replace: { + type: "string", + description: "Replacement text. Empty string = pure deletion.", + }, + context_before: { + type: "string", + description: "~40 chars immediately preceding `find`, used to disambiguate.", + }, + context_after: { + type: "string", + description: "~40 chars immediately following `find`.", + }, + reason: { + type: "string", + description: "Short explanation shown to the user on the card.", + }, + }, + required: ["find", "replace", "context_before", "context_after"], + }, + }, + }, + required: ["doc_id", "edits"], + }, + }, + }, +]; diff --git a/backend/src/lib/chatTools/tools/_helpers.ts b/backend/src/lib/chatTools/tools/_helpers.ts new file mode 100644 index 000000000..40135b2c6 --- /dev/null +++ b/backend/src/lib/chatTools/tools/_helpers.ts @@ -0,0 +1,99 @@ +/** + * Shared helpers for tools/*.ts. Anything imported by exactly one tool + * stays in that tool's file; only cross-tool helpers live here. + * + * Doc-id resolvers (resolveDoc, resolveDocLabel) live in ../doc-context + * — tools import them from there. + */ + +import path from "path"; + +// --------------------------------------------------------------------------- +// PDF standard fonts path (used by extractPdfText + any future tool that +// needs headless PDF rendering) +// --------------------------------------------------------------------------- + +export const STANDARD_FONT_DATA_URL = (() => { + try { + const pkgPath = require.resolve("pdfjs-dist/package.json"); + return path.join(path.dirname(pkgPath), "standard_fonts") + path.sep; + } catch { + return undefined; + } +})(); + +// --------------------------------------------------------------------------- +// PDF text extraction — shared by read-document and find-in-document +// --------------------------------------------------------------------------- + +export async function extractPdfText(buf: ArrayBuffer): Promise<string> { + try { + const pdfjsLib = await import( + "pdfjs-dist/legacy/build/pdf.mjs" as string + ); + const pdf = await ( + pdfjsLib as unknown as { + getDocument: (opts: unknown) => { + promise: Promise<{ + numPages: number; + getPage: (n: number) => Promise<{ + getTextContent: () => Promise<{ + items: { str?: string }[]; + }>; + }>; + }>; + }; + } + ).getDocument({ + data: new Uint8Array(buf), + standardFontDataUrl: STANDARD_FONT_DATA_URL, + }).promise; + const parts: string[] = []; + for (let i = 1; i <= pdf.numPages; i++) { + const page = await pdf.getPage(i); + const textContent = await page.getTextContent(); + parts.push( + `[Page ${i}]\n${textContent.items.map((it) => it.str ?? "").join(" ")}`, + ); + } + return parts.join("\n\n"); + } catch { + return ""; + } +} + +// --------------------------------------------------------------------------- +// Whitespace-normalised search helpers — shared by find-in-document +// --------------------------------------------------------------------------- + +/** + * Build a whitespace-collapsed, lowercased copy of `text`, plus a map from + * each character index in the normalized form back to the corresponding + * index in the original text. Used by findInDocumentContent so matches + * are tolerant of case + whitespace variance but can still return the + * exact original excerpt. + */ +export function normalizeWithMap(text: string): { norm: string; origIdx: number[] } { + const norm: string[] = []; + const origIdx: number[] = []; + let prevSpace = false; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (/\s/.test(ch)) { + if (!prevSpace) { + norm.push(" "); + origIdx.push(i); + prevSpace = true; + } + } else { + norm.push(ch.toLowerCase()); + origIdx.push(i); + prevSpace = false; + } + } + return { norm: norm.join(""), origIdx }; +} + +export function normalizeQuery(q: string): string { + return q.trim().replace(/\s+/g, " ").toLowerCase(); +} diff --git a/backend/src/lib/chatTools/tools/edit-document.ts b/backend/src/lib/chatTools/tools/edit-document.ts new file mode 100644 index 000000000..ba76727a1 --- /dev/null +++ b/backend/src/lib/chatTools/tools/edit-document.ts @@ -0,0 +1,312 @@ +/** + * edit_document tool runner + supporting helpers. + * + * loadCurrentVersionBytes — resolves the active .docx bytes for a document, + * preferring the tracked-changes version if one exists. + * + * runEditDocument — applies tracked-change edits to a DOCX, writes the new + * version to R2, records the document_versions row, persists document_edits + * rows, and returns the result shape with EditAnnotation[] for the stream layer. + */ + +import { randomUUID } from "crypto"; +import { + deleteFile, + downloadFile, + uploadFile, +} from "../../storage"; +import { createServerSupabase } from "../../supabase"; +import { + applyTrackedEdits, + type EditInput, +} from "../../docxTrackedChanges"; +import { buildDownloadUrl } from "../../downloadTokens"; +import { loadActiveVersion } from "../../documentVersions"; +import type { EditAnnotation } from "../types"; +import { insertVersionWithRetry } from "../../../routes/documents"; +import { logger } from "../../logger"; + +// --------------------------------------------------------------------------- +// loadCurrentVersionBytes (also used by read-document.ts) +// --------------------------------------------------------------------------- + +/** + * Resolve the current .docx bytes for a document, preferring the active + * tracked-changes version if one exists, else the original upload. + */ +export async function loadCurrentVersionBytes( + documentId: string, + db: ReturnType<typeof createServerSupabase>, +): Promise<{ bytes: Buffer; storage_path: string } | null> { + const active = await loadActiveVersion(documentId, db); + if (!active) return null; + const raw = await downloadFile(active.storage_path); + if (!raw) return null; + return { bytes: Buffer.from(raw), storage_path: active.storage_path }; +} + +// --------------------------------------------------------------------------- +// runEditDocument +// --------------------------------------------------------------------------- + +export async function runEditDocument(params: { + documentId: string; + userId: string; + edits: EditInput[]; + db: ReturnType<typeof createServerSupabase>; + /** + * If provided, append these edits to the existing turn-scoped version + * (overwrites the file at storagePath and reuses the document_versions + * row) instead of creating a new version. Used to collapse multiple + * edit_document tool calls within a single assistant turn into one + * version. + */ + reuseVersion?: { + versionId: string; + versionNumber: number; + storagePath: string; + }; +}): Promise< + | { + ok: true; + version_id: string; + version_number: number; + storage_path: string; + download_url: string; + annotations: EditAnnotation[]; + errors: { index: number; reason: string }[]; + } + | { ok: false; error: string } +> { + const { documentId, userId, edits, db, reuseVersion } = params; + + const { data: doc } = await db + .from("documents") + .select("id, filename") + .eq("id", documentId) + .single(); + if (!doc) return { ok: false, error: "Document not found." }; + + const current = await loadCurrentVersionBytes(documentId, db); + if (!current) return { ok: false, error: "Could not load document bytes." }; + + const { bytes: editedBytes, changes, errors } = await applyTrackedEdits( + current.bytes, + edits, + { author: "Mike" }, + ); + + if (changes.length === 0) { + return { + ok: false, + error: + errors[0]?.reason ?? + "No edits could be applied. Refine context_before/context_after and retry.", + }; + } + + const ab = editedBytes.buffer.slice( + editedBytes.byteOffset, + editedBytes.byteOffset + editedBytes.byteLength, + ) as ArrayBuffer; + + let versionRowId: string; + let newPath: string; + let nextVersionNumber: number; + + if (reuseVersion) { + // Overwrite the existing turn version's file in place. The version + // row, version_number, and current_version_id all already point here. + // NOTE: `uploadFile` is intentionally deferred until after the + // `document_edits` insert below succeeds, so that a DB failure does + // not leave R2 holding bytes whose change history was never + // recorded (storage / DB divergence). + newPath = reuseVersion.storagePath; + versionRowId = reuseVersion.versionId; + nextVersionNumber = reuseVersion.versionNumber; + } else { + const versionId = randomUUID().replace(/-/g, ""); + newPath = `documents/${userId}/${documentId}/edits/${versionId}.docx`; + await uploadFile( + newPath, + ab, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ); + + // Inherit the display name from the most recent prior version so + // user-applied renames carry forward through further edits. Falls + // back to the parent document's filename when no prior version has + // a display name (e.g. the first assistant edit of a pre-existing + // doc). We intentionally do NOT append "[Edited Vn]" — the version + // number is surfaced separately as a tag in the UI. + const { data: prevRow } = await db + .from("document_versions") + .select("display_name, created_at") + .eq("document_id", documentId) + .order("created_at", { ascending: false }) + .limit(1) + .maybeSingle(); + const inheritedDisplayName = + (prevRow?.display_name as string | null) ?? + (doc.filename as string | null) ?? + null; + + // insertVersionWithRetry handles 23505 unique_violation races (CLEAN-08). + // It fetches MAX(version_number)+1 and retries once on collision so + // concurrent assistant_edit calls cannot assign the same version_number. + const { data: versionRow, error: verErr } = await insertVersionWithRetry(db, documentId, { + document_id: documentId, + storage_path: newPath, + source: "assistant_edit", + display_name: inheritedDisplayName, + }); + if (verErr || !versionRow) { + return { ok: false, error: "Failed to record document version." }; + } + versionRowId = versionRow.id as string; + nextVersionNumber = versionRow.version_number; + } + + // Insert one row per change + const editRows = changes.map((c) => ({ + document_id: documentId, + version_id: versionRowId, + change_id: c.id, + del_w_id: c.delId ?? null, + ins_w_id: c.insId ?? null, + deleted_text: c.deletedText, + inserted_text: c.insertedText, + context_before: c.contextBefore ?? "", + context_after: c.contextAfter ?? "", + status: "pending" as const, + })); + const { data: insertedEdits, error: editsErr } = await db + .from("document_edits") + .insert(editRows) + .select("id, change_id, del_w_id, ins_w_id, deleted_text, inserted_text, context_before, context_after"); + + if (editsErr || !insertedEdits) { + if (!reuseVersion) { + // Compensating cleanup: the document_edits insert failed after + // R2 write + document_versions insert succeeded. Delete both to + // prevent permanent orphans (CR-01). + await deleteFile(newPath).catch((e: unknown) => + logger.error({ err: e }, "[edit-document] compensating R2 delete failed"), + ); + const { error: vDelErr } = await db + .from("document_versions") + .delete() + .eq("id", versionRowId); + if (vDelErr) { + logger.error({ err: vDelErr }, "[edit-document] compensating version row delete failed"); + } + } + return { ok: false, error: "Failed to record edits." }; + } + + if (reuseVersion) { + // Deferred from above: only overwrite the in-place R2 bytes once + // we've successfully recorded the new edits. If the upload fails, + // applyReuseVersionSaga deletes the inserted document_edits rows + // (compensating rollback) so storage and the change history stay + // consistent on partial failure (CLEAN-16). + const insertedEditIds = (insertedEdits ?? []).map( + (r: { id: string }) => r.id, + ); + const sagaResult = await applyReuseVersionSaga({ + db, + newPath, + ab, + mime: "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + insertedEditIds, + }); + if (!sagaResult.ok) { + return { ok: false, error: sagaResult.error }; + } + } + + await db + .from("documents") + .update({ current_version_id: versionRowId }) + .eq("id", documentId); + + const annotations: EditAnnotation[] = insertedEdits.map((r: { id: string; change_id: string; deleted_text: string; inserted_text: string; context_before: string | null; context_after: string | null }) => { + const src = changes.find((c) => c.id === r.change_id); + return { + kind: "edit", + edit_id: r.id, + document_id: documentId, + version_id: versionRowId, + version_number: nextVersionNumber, + change_id: r.change_id, + del_w_id: src?.delId, + ins_w_id: src?.insId, + deleted_text: r.deleted_text ?? "", + inserted_text: r.inserted_text ?? "", + context_before: r.context_before ?? "", + context_after: r.context_after ?? "", + reason: src?.reason, + status: "pending", + }; + }); + + // Persistent, non-expiring permalink. The backend streams fresh bytes + // on each request, so this URL stays valid as long as the file exists. + const permalink = buildDownloadUrl(newPath, doc.filename as string); + + return { + ok: true, + version_id: versionRowId, + version_number: nextVersionNumber, + storage_path: newPath, + download_url: permalink, + annotations, + errors, + }; +} + +// --------------------------------------------------------------------------- +// CLEAN-16: reuseVersion compensating saga +// --------------------------------------------------------------------------- + +/** + * Deferred upload guard for the reuseVersion path of runEditDocument. + * + * After the document_edits rows have been inserted, this helper attempts the + * in-place R2 overwrite. On failure it deletes the inserted rows (compensating + * rollback) so the DB never carries document_edits that reference bytes which + * were never written. + * + * Returns `{ ok: true }` on success or `{ ok: false, error }` on any storage + * failure. The caller must NOT update documents.current_version_id on failure. + */ +export async function applyReuseVersionSaga(deps: { + db: ReturnType<typeof createServerSupabase>; + newPath: string; + ab: ArrayBuffer; + mime: string; + insertedEditIds: string[]; +}): Promise<{ ok: true } | { ok: false; error: string }> { + try { + await uploadFile(deps.newPath, deps.ab, deps.mime); + return { ok: true }; + } catch (uploadErr) { + logger.error({ err: uploadErr }, "[edit-document] reuseVersion upload failed after document_edits insert — compensating delete"); + if (deps.insertedEditIds.length > 0) { + const { error: delErr } = await deps.db + .from("document_edits") + .delete() + .in("id", deps.insertedEditIds); + if (delErr) { + logger.error({ err: delErr }, "[edit-document] CRITICAL: compensating delete of document_edits failed — DB may carry orphaned edits"); + } + } + return { + ok: false, + error: + uploadErr instanceof Error + ? `Storage write failed: ${uploadErr.message}` + : "Storage write failed.", + }; + } +} diff --git a/backend/src/lib/chatTools/tools/fetch-documents.ts b/backend/src/lib/chatTools/tools/fetch-documents.ts new file mode 100644 index 000000000..d06eba4f3 --- /dev/null +++ b/backend/src/lib/chatTools/tools/fetch-documents.ts @@ -0,0 +1,35 @@ +/** + * fetch_documents tool runner. + * + * Reads multiple documents in a single tool call by dispatching to + * runReadDocument for each requested doc id. Returns a concatenated + * text block with per-document separators. + */ + +import { createServerSupabase } from "../../supabase"; +import type { DocStore, DocIndex } from "../types"; +import { runReadDocument } from "./read-document"; + +export async function runFetchDocuments(args: { + docIds: string[]; + docStore: DocStore; + write: (s: string) => void; + docIndex?: DocIndex; + db?: ReturnType<typeof createServerSupabase>; +}): Promise<{ content: string; docsRead: { filename: string; document_id?: string }[] }> { + const { docIds, docStore, write, docIndex, db } = args; + const parts: string[] = []; + const docsRead: { filename: string; document_id?: string }[] = []; + + for (const docId of docIds) { + const content = await runReadDocument({ docLabel: docId, docStore, write, docIndex, db }); + const filename = docStore.get(docId)?.filename ?? docId; + parts.push(`--- ${filename} (${docId}) ---\n${content}`); + if (docStore.get(docId)) { + const documentId = docIndex?.[docId]?.document_id; + docsRead.push({ filename, document_id: documentId }); + } + } + + return { content: parts.join("\n\n"), docsRead }; +} diff --git a/backend/src/lib/chatTools/tools/find-in-document.ts b/backend/src/lib/chatTools/tools/find-in-document.ts new file mode 100644 index 000000000..871396500 --- /dev/null +++ b/backend/src/lib/chatTools/tools/find-in-document.ts @@ -0,0 +1,147 @@ +/** + * find_in_document tool runner. + * + * Ctrl+F style search over a document's text content. Returns up to maxResults + * hits with excerpt + surrounding context. Emits doc_find_start / doc_find SSE + * events. + */ + +import { createServerSupabase } from "../../supabase"; +import type { DocStore, DocIndex } from "../types"; +import { runReadDocument } from "./read-document"; +import { normalizeWithMap, normalizeQuery } from "./_helpers"; + +export async function runFindInDocument(args: { + docLabel: string; + query: string; + maxResults?: number; + contextChars?: number; + docStore: DocStore; + write: (s: string) => void; + docIndex?: DocIndex; + db?: ReturnType<typeof createServerSupabase>; +}): Promise<string> { + const { + docLabel, + query, + maxResults = 20, + contextChars = 80, + docStore, + write, + docIndex, + db, + } = args; + + if (!query || !query.trim()) { + return JSON.stringify({ ok: false, error: "Empty query." }); + } + + const docInfo = docStore.get(docLabel); + if (!docInfo) { + return JSON.stringify({ + ok: false, + error: `Document '${docLabel}' not found.`, + }); + } + + // Announce the search to the UI, then reuse runReadDocument for its + // fallbacks — but suppress its own doc_read events so the user only sees + // the doc_find block (not a competing doc_read block for the same op). + write( + `data: ${JSON.stringify({ + type: "doc_find_start", + filename: docInfo.filename, + query, + })}\n\n`, + ); + + const text = await runReadDocument({ + docLabel, + docStore, + write, + docIndex, + db, + opts: { emitEvents: false }, + }); + if (!text || text === "Document could not be read.") { + write( + `data: ${JSON.stringify({ + type: "doc_find", + filename: docInfo.filename, + query, + total_matches: 0, + })}\n\n`, + ); + return JSON.stringify({ + ok: false, + filename: docInfo.filename, + error: "Document could not be read.", + }); + } + + const { norm, origIdx } = normalizeWithMap(text); + const needle = normalizeQuery(query); + if (!needle) { + return JSON.stringify({ ok: false, error: "Empty query after normalization." }); + } + + type Hit = { + index: number; + excerpt: string; + context: string; + }; + const hits: Hit[] = []; + let from = 0; + while (from <= norm.length - needle.length && hits.length < maxResults) { + const pos = norm.indexOf(needle, from); + if (pos < 0) break; + const endNormPos = pos + needle.length; + const origStart = origIdx[pos] ?? 0; + const origEnd = + endNormPos - 1 < origIdx.length + ? origIdx[endNormPos - 1] + 1 + : text.length; + const ctxStart = Math.max(0, origStart - contextChars); + const ctxEnd = Math.min(text.length, origEnd + contextChars); + hits.push({ + index: hits.length, + excerpt: text.slice(origStart, origEnd), + context: + (ctxStart > 0 ? "…" : "") + + text.slice(ctxStart, ctxEnd).replace(/\s+/g, " ").trim() + + (ctxEnd < text.length ? "…" : ""), + }); + from = pos + Math.max(1, needle.length); + } + + // Count total occurrences beyond the cap so the model knows whether to narrow the query. + let totalMatches = hits.length; + if (hits.length >= maxResults) { + let probe = from; + while (probe <= norm.length - needle.length) { + const pos = norm.indexOf(needle, probe); + if (pos < 0) break; + totalMatches++; + probe = pos + Math.max(1, needle.length); + } + } + + write( + `data: ${JSON.stringify({ + type: "doc_find", + filename: docInfo.filename, + query, + total_matches: totalMatches, + })}\n\n`, + ); + + return JSON.stringify({ + ok: true, + filename: docInfo.filename, + query, + total_matches: totalMatches, + returned: hits.length, + truncated: totalMatches > hits.length, + hits, + }); +} diff --git a/backend/src/lib/chatTools/tools/generate-docx.ts b/backend/src/lib/chatTools/tools/generate-docx.ts new file mode 100644 index 000000000..44b3898e6 --- /dev/null +++ b/backend/src/lib/chatTools/tools/generate-docx.ts @@ -0,0 +1,260 @@ +/** + * generate_docx tool runner. + * + * Generates a new DOCX from LLM-provided title + sections, uploads it to R2, + * persists a documents + document_versions row, and returns a result shape + * including the download URL and DB ids. The `docx` import is lazy so the + * SDK isn't pulled at process start. + */ + +import { randomUUID } from "crypto"; +import { + generatedDocKey, + uploadFile, +} from "../../storage"; +import { createServerSupabase } from "../../supabase"; +import { buildDownloadUrl } from "../../downloadTokens"; +import { logger } from "../../logger"; + +export async function runGenerateDocx(args: { + title: string; + sections: unknown[]; + userId: string; + db: ReturnType<typeof createServerSupabase>; + options?: { landscape?: boolean; projectId?: string | null }; +}): Promise< + | { filename: string; download_url: string; document_id: string; version_id: string; version_number: number; storage_path: string; message: string } + | { error: string } +> { + const { title, sections, userId, db, options } = args; + try { + const { + Document, Paragraph, HeadingLevel, Packer, + Table, TableRow, TableCell, WidthType, BorderStyle, + TextRun, AlignmentType, PageOrientation, PageBreak, + } = await import("docx"); + + const FONT = "Times New Roman"; + const SIZE = 22; // 11pt in half-points + + type DocChild = InstanceType<typeof Paragraph> | InstanceType<typeof Table>; + const children: DocChild[] = []; + children.push( + new Paragraph({ + heading: HeadingLevel.TITLE, + spacing: { after: 200 }, + alignment: AlignmentType.CENTER, + children: [new TextRun({ text: title.toUpperCase(), color: "000000", font: FONT, size: SIZE, bold: true })], + }), + ); + + const cellBorder = { + top: { style: BorderStyle.SINGLE, size: 1, color: "CCCCCC" }, + bottom: { style: BorderStyle.SINGLE, size: 1, color: "CCCCCC" }, + left: { style: BorderStyle.SINGLE, size: 1, color: "CCCCCC" }, + right: { style: BorderStyle.SINGLE, size: 1, color: "CCCCCC" }, + }; + + const headingLevels = [ + HeadingLevel.HEADING_1, + HeadingLevel.HEADING_2, + HeadingLevel.HEADING_3, + HeadingLevel.HEADING_4, + ]; + const counters = [0, 0, 0, 0]; + + for (const section of sections as { + heading?: string; + content?: string; + level?: number; + pageBreak?: boolean; + table?: { headers: string[]; rows: string[][] }; + }[]) { + if (section.pageBreak) { + children.push( + new Paragraph({ children: [new PageBreak()] }), + ); + } + if (section.heading) { + const idx = Math.min((section.level ?? 1) - 1, 3); + counters[idx]++; + for (let i = idx + 1; i < 4; i++) counters[i] = 0; + const prefix = counters.slice(0, idx + 1).join("."); + const headingText = `${prefix}. ${idx === 0 ? section.heading.toUpperCase() : section.heading}`; + children.push( + new Paragraph({ + heading: headingLevels[idx], + spacing: { after: 160 }, + children: [new TextRun({ text: headingText, color: "000000", font: FONT, size: SIZE, bold: true })], + }), + ); + } + if (section.table) { + const { headers, rows } = section.table; + const colCount = headers.length; + const tableRows: InstanceType<typeof TableRow>[] = []; + // Header row + tableRows.push( + new TableRow({ + tableHeader: true, + children: headers.map( + (h) => + new TableCell({ + borders: cellBorder, + shading: { fill: "F2F2F2" }, + children: [ + new Paragraph({ + children: [new TextRun({ text: h, bold: true, font: FONT, size: SIZE })], + alignment: AlignmentType.LEFT, + }), + ], + }), + ), + }), + ); + // Data rows — normalize each row to exactly colCount cells. + // LLMs occasionally emit malformed rows (extra fragments from + // stray delimiters, or short rows); padding/truncating here + // keeps the rendered table aligned to the headers. + for (const rawRow of rows) { + const row = Array.isArray(rawRow) ? rawRow : []; + const normalized: string[] = []; + for (let i = 0; i < colCount; i++) { + normalized.push( + typeof row[i] === "string" ? row[i] : "", + ); + } + if (row.length !== colCount) { + logger.warn({ rowLength: row.length, colCount }, "[generate_docx] row length != headers; normalized"); + } + tableRows.push( + new TableRow({ + children: normalized.map( + (cell) => + new TableCell({ + borders: cellBorder, + children: [ + new Paragraph({ + children: [new TextRun({ text: cell, font: FONT, size: SIZE })], + }), + ], + }), + ), + }), + ); + } + children.push( + new Table({ + width: { size: 100, type: WidthType.PERCENTAGE }, + rows: tableRows, + }), + ); + children.push(new Paragraph({ text: "" })); + } + if (section.content) { + for (const line of section.content.split("\n")) { + const trimmed = line.trim(); + if (!trimmed) continue; + const bulletMatch = trimmed.match(/^[-•*]\s+(.+)/); + if (bulletMatch) { + children.push( + new Paragraph({ + bullet: { level: 0 }, + spacing: { after: 120 }, + children: [new TextRun({ text: bulletMatch[1], font: FONT, size: SIZE })], + }), + ); + } else { + children.push( + new Paragraph({ + spacing: { after: 120 }, + children: [new TextRun({ text: trimmed, font: FONT, size: SIZE })], + }), + ); + } + } + } + } + + const pageSetup = options?.landscape + ? { page: { size: { orientation: PageOrientation.LANDSCAPE } } } + : {}; + + const doc = new Document({ sections: [{ properties: pageSetup, children }] }); + const buf = await Packer.toBuffer(doc); + const docId = randomUUID().replace(/-/g, ""); + const safeTitle = + title + .replace(/[^a-zA-Z0-9 -]/g, "") + .trim() + .slice(0, 64) || "document"; + const filename = `${safeTitle}.docx`; + const key = generatedDocKey(userId, docId, filename); + + await uploadFile( + key, + buf.buffer as ArrayBuffer, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ); + const downloadUrl = buildDownloadUrl(key, filename); + + // Persist to DB so generated docs are first-class documents: + // openable in the DocPanel and editable via edit_document. In + // project chats we attach to the project so it appears in the + // sidebar; in the general chat we leave project_id null and it + // stays a standalone document. + const { data: docRow, error: docErr } = await db + .from("documents") + .insert({ + project_id: options?.projectId ?? null, + user_id: userId, + filename, + file_type: "docx", + size_bytes: buf.byteLength, + status: "ready", + }) + .select("id") + .single(); + if (docErr || !docRow) { + return { + error: `Failed to record generated document: ${docErr?.message ?? "unknown"}`, + }; + } + const documentId = docRow.id as string; + + const { data: versionRow, error: verErr } = await db + .from("document_versions") + .insert({ + document_id: documentId, + storage_path: key, + source: "generated", + version_number: 1, + display_name: filename, + }) + .select("id") + .single(); + if (verErr || !versionRow) { + return { + error: `Failed to record generated document version: ${verErr?.message ?? "unknown"}`, + }; + } + const versionId = versionRow.id as string; + + await db + .from("documents") + .update({ current_version_id: versionId }) + .eq("id", documentId); + + return { + filename, + download_url: downloadUrl, + document_id: documentId, + version_id: versionId, + version_number: 1, + storage_path: key, + message: `Document '${filename}' has been generated successfully.`, + }; + } catch (e) { + return { error: String(e) }; + } +} diff --git a/backend/src/lib/chatTools/tools/list-documents.ts b/backend/src/lib/chatTools/tools/list-documents.ts new file mode 100644 index 000000000..52453cf9d --- /dev/null +++ b/backend/src/lib/chatTools/tools/list-documents.ts @@ -0,0 +1,22 @@ +/** + * list_documents tool runner. + * + * Returns the current turn's available document labels, filenames, and + * file types as a JSON array. No SSE events emitted. + */ + +import type { DocStore } from "../types"; + +export function runListDocuments(args: { + docStore: DocStore; +}): string { + const { docStore } = args; + const list = Array.from(docStore.entries()).map( + ([doc_id, info]) => ({ + doc_id, + filename: info.filename, + file_type: info.file_type, + }), + ); + return JSON.stringify(list); +} diff --git a/backend/src/lib/chatTools/tools/read-document.ts b/backend/src/lib/chatTools/tools/read-document.ts new file mode 100644 index 000000000..1eb4fb7e2 --- /dev/null +++ b/backend/src/lib/chatTools/tools/read-document.ts @@ -0,0 +1,171 @@ +/** + * read_document tool runner. + * + * Reads a document from R2 (preferring the current tracked-changes version) + * and returns its text content. Emits doc_read_start / doc_read SSE events + * unless opts.emitEvents is false (used internally by find_in_document to + * suppress duplicate UI blocks). + * + * Verbose tracing (storage paths, magic bytes, intermediate extraction + * lengths) is gated behind the DEBUG_CHATTOOLS env var so production logs + * don't leak document metadata or content. Per CLAUDE.md privacy policy, + * document body text MUST NOT appear in logs — the DONE log emits only + * filename + final length. + */ + +import { downloadFile } from "../../storage"; +import { extractDocxBodyText } from "../../docxTrackedChanges"; +import { createServerSupabase } from "../../supabase"; +import type { DocStore, DocIndex } from "../types"; +import { extractPdfText } from "./_helpers"; +import { loadCurrentVersionBytes } from "./edit-document"; +import { logger } from "../../logger"; + +const DEBUG = process.env.DEBUG_CHATTOOLS === "1" || process.env.DEBUG_CHATTOOLS === "true"; +function dlog(msg: string, data?: Record<string, unknown>) { + if (DEBUG) logger.debug(data ?? {}, msg); +} + +export async function runReadDocument(args: { + docLabel: string; + docStore: DocStore; + write: (s: string) => void; + docIndex?: DocIndex; + db?: ReturnType<typeof createServerSupabase>; + opts?: { emitEvents?: boolean }; +}): Promise<string> { + const { docLabel, docStore, write, docIndex, db, opts } = args; + const emitEvents = opts?.emitEvents ?? true; + dlog(`[read_document] called with docLabel="${docLabel}"`); + const docInfo = docStore.get(docLabel); + if (!docInfo) { + dlog( + `[read_document] MISS — docLabel "${docLabel}" not in docStore`, + { knownLabels: Array.from(docStore.keys()) }, + ); + return "Document not found."; + } + dlog( + `[read_document] docInfo: filename="${docInfo.filename}", file_type="${docInfo.file_type}", storage_path="${docInfo.storage_path}"`, + ); + + const documentId = docIndex?.[docLabel]?.document_id; + const emitDocRead = () => { + if (!emitEvents) return; + write( + `data: ${JSON.stringify({ + type: "doc_read", + filename: docInfo.filename, + document_id: documentId, + })}\n\n`, + ); + }; + if (emitEvents) + write( + `data: ${JSON.stringify({ + type: "doc_read_start", + filename: docInfo.filename, + document_id: documentId, + })}\n\n`, + ); + try { + // Prefer the current tracked-changes version (if any) so read_document + // reflects accepted/pending edits rather than the original upload. + let raw: ArrayBuffer | null = null; + let sourcePath = docInfo.storage_path; + if (documentId && db) { + const current = await loadCurrentVersionBytes(documentId, db); + if (current) { + raw = current.bytes.buffer.slice( + current.bytes.byteOffset, + current.bytes.byteOffset + current.bytes.byteLength, + ) as ArrayBuffer; + sourcePath = current.storage_path; + dlog( + `[read_document] using current version path="${sourcePath}" (bytes=${raw.byteLength})`, + ); + } else { + dlog( + `[read_document] loadCurrentVersionBytes returned null for documentId="${documentId}", falling back to original storage_path`, + ); + } + } + if (!raw) { + raw = await downloadFile(docInfo.storage_path); + if (raw) { + dlog( + `[read_document] fallback download from storage_path="${docInfo.storage_path}" (bytes=${raw.byteLength})`, + ); + } + } + if (!raw) { + logger.error({ filename: docInfo.filename }, "[read_document] failed to download bytes"); + emitDocRead(); + return "Document could not be read."; + } + // Log the first 8 bytes so we can identify real file format regardless + // of the declared file_type. Valid .docx starts with "PK\x03\x04" + // (zip). Legacy .doc starts with "\xD0\xCF\x11\xE0" (OLE/CFB). + // %PDF-1 is a PDF even if mislabeled. Truncated uploads show as all-zero. + if (DEBUG) { + const head = Buffer.from(raw).subarray(0, 8); + const hex = head.toString("hex"); + const ascii = head + .toString("binary") + .replace(/[^\x20-\x7e]/g, "."); + dlog( + `[read_document] magic bytes hex=${hex} ascii="${ascii}" for filename="${docInfo.filename}"`, + ); + } + let text: string; + if (docInfo.file_type === "pdf") { + text = await extractPdfText(raw); + dlog( + `[read_document] pdf extracted length=${text.length} for filename="${docInfo.filename}"`, + ); + } else if (docInfo.file_type === "docx") { + // Use the same flattening as the edit_document matcher so the + // LLM sees exactly the characters it can anchor against. + text = await extractDocxBodyText(Buffer.from(raw)); + dlog( + `[read_document] docx extractDocxBodyText length=${text.length} for filename="${docInfo.filename}"`, + ); + if (!text) { + dlog( + `[read_document] docx accepted-view extractor returned empty, falling back to mammoth for filename="${docInfo.filename}"`, + ); + const mammoth = await import("mammoth"); + const result = await mammoth.extractRawText({ + buffer: Buffer.from(raw), + }); + text = result.value; + dlog( + `[read_document] docx mammoth fallback length=${text.length} for filename="${docInfo.filename}"`, + ); + } + } else { + dlog( + `[read_document] unknown file_type="${docInfo.file_type}" for filename="${docInfo.filename}", trying mammoth`, + ); + const mammoth = await import("mammoth"); + const result = await mammoth.extractRawText({ + buffer: Buffer.from(raw), + }); + text = result.value; + dlog( + `[read_document] mammoth length=${text.length} for filename="${docInfo.filename}"`, + ); + } + // Always-on completion log: filename + final length only. + // Body text (firstChars slice) is intentionally omitted per + // CLAUDE.md privacy policy. + logger.info({ filename: docInfo.filename, length: text.length }, "[read_document] done"); + emitDocRead(); + return text; + } catch (err) { + logger.error({ err, filename: docInfo.filename }, "[read_document] threw"); + if (emitEvents) + write(`data: ${JSON.stringify({ type: "doc_read", filename: docInfo.filename })}\n\n`); + return "Document could not be read."; + } +} diff --git a/backend/src/lib/chatTools/tools/read-workflow.ts b/backend/src/lib/chatTools/tools/read-workflow.ts new file mode 100644 index 000000000..7e6888122 --- /dev/null +++ b/backend/src/lib/chatTools/tools/read-workflow.ts @@ -0,0 +1,28 @@ +/** + * read_workflow tool runner. + * + * Looks up a workflow by id in the WorkflowStore, emits a workflow_applied + * SSE event, and returns the workflow's prompt_md as the tool result content. + */ + +import type { WorkflowStore } from "../types"; + +export function runReadWorkflow(args: { + workflowId: string; + workflowStore: WorkflowStore | undefined; + write: (s: string) => void; +}): { content: string; applied: { workflow_id: string; title: string } | null } { + const { workflowId, workflowStore, write } = args; + const wf = workflowStore?.get(workflowId); + if (wf) { + write(`data: ${JSON.stringify({ type: "workflow_applied", workflow_id: workflowId, title: wf.title })}\n\n`); + return { + content: wf.prompt_md, + applied: { workflow_id: workflowId, title: wf.title }, + }; + } + return { + content: `Workflow '${workflowId}' not found.`, + applied: null, + }; +} diff --git a/backend/src/lib/chatTools/tools/replicate-document.ts b/backend/src/lib/chatTools/tools/replicate-document.ts new file mode 100644 index 000000000..902afe80b --- /dev/null +++ b/backend/src/lib/chatTools/tools/replicate-document.ts @@ -0,0 +1,339 @@ +/** + * replicate_document tool runner. + * + * Creates N copies of a source document within a project. Each copy gets its + * own documents row, document_versions row, and storage key. The source bytes + * (and PDF rendition if any) are fetched once and written in parallel. New + * copies are registered in the live docIndex / docStore so they can be + * edited/read in the same assistant turn. + */ + +import { + downloadFile, + storageKey, + uploadFile, +} from "../../storage"; +import { convertedPdfKey } from "../../convert"; +import { createServerSupabase } from "../../supabase"; +import { buildDownloadUrl } from "../../downloadTokens"; +import { loadActiveVersion } from "../../documentVersions"; +import type { DocStore, DocIndex, DocReplicatedResult } from "../types"; + +export async function runReplicateDocument(args: { + rawDocId: string; + requestedFilename: string | null; + requestedCount: number; + sourceLabel: string; + docStore: DocStore; + docIndex: DocIndex; + userId: string; + projectId: string | null | undefined; + db: ReturnType<typeof createServerSupabase>; + write: (s: string) => void; + toolCallId: string; +}): Promise<{ toolResult: unknown; replicated: DocReplicatedResult | null }> { + const { + rawDocId, + requestedFilename, + requestedCount, + sourceLabel, + docStore, + docIndex, + userId, + projectId, + db, + write, + toolCallId, + } = args; + + const sourceInfo = docStore.get(sourceLabel); + const sourceIndexed = docIndex[sourceLabel]; + const sourceFilename = sourceInfo?.filename ?? rawDocId; + + write( + `data: ${JSON.stringify({ + type: "doc_replicate_start", + filename: sourceFilename, + count: requestedCount, + })}\n\n`, + ); + + const fail = (error: string): { toolResult: unknown; replicated: null } => { + write( + `data: ${JSON.stringify({ + type: "doc_replicated", + filename: sourceFilename, + count: requestedCount, + copies: [], + error, + })}\n\n`, + ); + return { + toolResult: { + role: "tool", + tool_call_id: toolCallId, + content: JSON.stringify({ ok: false, error }), + }, + replicated: null, + }; + }; + + if (!sourceInfo || !sourceIndexed) { + return fail(`Document '${rawDocId}' not found in this project.`); + } + if (!projectId) { + return fail("replicate_document is only available in project chats."); + } + + try { + // Pull the active version once — every copy gets the + // same starting bytes (with any accepted tracked + // changes rolled in), no point re-fetching per copy. + const active = await loadActiveVersion( + sourceIndexed.document_id, + db, + ); + const sourcePath = + active?.storage_path ?? sourceInfo.storage_path; + const sourcePdfPath = active?.pdf_storage_path ?? null; + const raw = await downloadFile(sourcePath); + const pdfBytes = sourcePdfPath + ? await downloadFile(sourcePdfPath) + : null; + if (!raw) { + return fail( + "Could not read the source document's bytes from storage.", + ); + } + + // Build N filenames. With count=1 keep the + // pre-existing "(copy)" suffix; with count>1 use + // numbered "(1)", "(2)" suffixes. + const srcExt = + sourceInfo.filename.match(/\.[^./\\]+$/)?.[0] ?? ""; + const baseStem = (() => { + if (requestedFilename) { + return requestedFilename.replace( + /\.[^./\\]+$/, + "", + ); + } + return sourceInfo.filename.replace( + /\.[^./\\]+$/, + "", + ); + })(); + const filenames: string[] = []; + for (let n = 1; n <= requestedCount; n++) { + const suffix = + requestedCount === 1 + ? requestedFilename + ? "" + : " (copy)" + : ` (${n})`; + filenames.push(`${baseStem}${suffix}${srcExt}`); + } + + // Bulk insert N documents in one round-trip. + const docRows = filenames.map((fn) => ({ + project_id: projectId, + user_id: userId, + filename: fn, + file_type: sourceInfo.file_type, + size_bytes: raw.byteLength, + status: "ready", + })); + const { data: insertedDocs, error: docErr } = await db + .from("documents") + .insert(docRows) + .select("id, filename"); + if (docErr || !insertedDocs || insertedDocs.length === 0) { + return fail( + `Failed to record replicated documents: ${docErr?.message ?? "unknown"}`, + ); + } + + // Preserve the request order so each row pairs + // with the right filename. Supabase returns + // inserted rows in the same order as the + // payload. + const newDocs = insertedDocs as { + id: string; + filename: string; + }[]; + const contentType = + sourceInfo.file_type === "pdf" + ? "application/pdf" + : "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; + + // Parallel uploads: the doc bytes (and PDF + // rendition if any) for every new copy. + const uploadJobs: Promise<unknown>[] = []; + const newKeys: string[] = []; + const newPdfKeys: (string | null)[] = []; + for (const d of newDocs) { + const key = storageKey( + userId, + d.id, + d.filename, + ); + newKeys.push(key); + uploadJobs.push( + uploadFile(key, raw, contentType), + ); + if (pdfBytes) { + const pdfKey = convertedPdfKey( + userId, + d.id, + ); + newPdfKeys.push(pdfKey); + uploadJobs.push( + uploadFile( + pdfKey, + pdfBytes, + "application/pdf", + ), + ); + } else { + newPdfKeys.push(null); + } + } + await Promise.all(uploadJobs); + + // Bulk insert N versions in one round-trip. + const versionRows = newDocs.map((d, idx) => ({ + document_id: d.id, + storage_path: newKeys[idx], + pdf_storage_path: newPdfKeys[idx], + source: "upload", + version_number: 1, + display_name: d.filename, + })); + const { data: insertedVersions, error: verErr } = + await db + .from("document_versions") + .insert(versionRows) + .select("id, document_id"); + if ( + verErr || + !insertedVersions || + insertedVersions.length !== newDocs.length + ) { + return fail( + `Failed to record replicated document versions: ${verErr?.message ?? "unknown"}`, + ); + } + + const versionByDocId = new Map<string, string>(); + for (const v of insertedVersions as { + id: string; + document_id: string; + }[]) { + versionByDocId.set(v.document_id, v.id); + } + + // current_version_id has to be a per-row + // value, so a single UPDATE statement + // can't cover all N. Fan out in parallel + // instead of sequential awaits. + await Promise.all( + newDocs.map((d) => + db + .from("documents") + .update({ + current_version_id: + versionByDocId.get(d.id), + }) + .eq("id", d.id), + ), + ); + + // Register every copy under a fresh doc-N + // slug so the model can edit/read any of + // them in the same turn. + const existingLabels = new Set( + Object.keys(docIndex), + ); + let nextLabelIdx = 0; + const copies: { + new_filename: string; + document_id: string; + version_id: string; + }[] = []; + const toolPayloadCopies: { + doc_id: string; + document_id: string; + version_id: string; + filename: string; + download_url: string; + }[] = []; + for (let idx = 0; idx < newDocs.length; idx++) { + const d = newDocs[idx]; + const newKey = newKeys[idx]; + const versionId = versionByDocId.get(d.id); + if (!versionId) continue; + while ( + existingLabels.has( + `doc-${nextLabelIdx}`, + ) + ) + nextLabelIdx++; + const slug = `doc-${nextLabelIdx}`; + existingLabels.add(slug); + docIndex[slug] = { + document_id: d.id, + filename: d.filename, + }; + docStore.set(slug, { + storage_path: newKey, + file_type: sourceInfo.file_type, + filename: d.filename, + }); + copies.push({ + new_filename: d.filename, + document_id: d.id, + version_id: versionId, + }); + toolPayloadCopies.push({ + doc_id: slug, + document_id: d.id, + version_id: versionId, + filename: d.filename, + download_url: buildDownloadUrl( + newKey, + d.filename, + ), + }); + } + + write( + `data: ${JSON.stringify({ + type: "doc_replicated", + filename: sourceFilename, + count: copies.length, + copies, + })}\n\n`, + ); + + const replicated: DocReplicatedResult = { + filename: sourceFilename, + count: copies.length, + copies, + }; + + return { + toolResult: { + role: "tool", + tool_call_id: toolCallId, + content: JSON.stringify({ + ok: true, + count: copies.length, + copies: toolPayloadCopies, + }), + }, + replicated, + }; + } catch (e) { + return fail(`replicate_document failed: ${String(e)}`); + } +} diff --git a/backend/src/lib/chatTools/types.ts b/backend/src/lib/chatTools/types.ts new file mode 100644 index 000000000..0958f14a7 --- /dev/null +++ b/backend/src/lib/chatTools/types.ts @@ -0,0 +1,96 @@ +/** + * Shared type aliases for the chatTools module. + * + * Consumed by stream.ts, tool-runner.ts, doc-context.ts, citations.ts, + * workflow-store.ts and tools/*. Hosted here (not in index.ts) so + * sibling modules can import via "./types" without circular imports + * through the façade. + */ + +export type DocStore = Map< + string, + { storage_path: string; file_type: string; filename: string } +>; + +export type WorkflowStore = Map<string, { title: string; prompt_md: string }>; + +export type DocIndex = Record< + string, + { + document_id: string; + filename: string; + version_id?: string | null; + version_number?: number | null; + } +>; + +export type TabularCellStore = { + columns: { index: number; name: string }[]; + documents: { id: string; filename: string }[]; + /** key: `${colIndex}:${docId}` */ + cells: Map<string, { summary: string; flag?: string; reasoning?: string } | null>; +}; + +export type ToolCall = { + id: string; + function: { name: string; arguments: string }; +}; + +export type ChatMessage = { + role: string; + content: string | null; + files?: { filename: string; document_id?: string }[]; + workflow?: { id: string; title: string }; +}; + +export type EditAnnotation = { + kind: "edit"; + edit_id: string; + document_id: string; + version_id: string; + version_number?: number | null; + change_id: string; + del_w_id?: string; + ins_w_id?: string; + deleted_text: string; + inserted_text: string; + context_before: string; + context_after: string; + reason?: string; + status: "pending" | "accepted" | "rejected"; +}; + +export type TurnEditState = Map< + string, + { versionId: string; versionNumber: number; storagePath: string } +>; + +export type DocEditedResult = { + filename: string; + document_id: string; + version_id: string; + version_number: number | null; + download_url: string; + annotations: EditAnnotation[]; +}; + +export type DocCreatedResult = { + filename: string; + download_url: string; + document_id?: string; + version_id?: string; + version_number?: number | null; +}; + +export type DocReplicatedResult = { + /** Filename of the source document being copied. */ + filename: string; + /** How many copies were produced in this single tool call. */ + count: number; + /** One entry per new copy. */ + copies: { + new_filename: string; + document_id: string; + version_id: string; + }[]; +}; diff --git a/backend/src/lib/chatTools/workflow-store.ts b/backend/src/lib/chatTools/workflow-store.ts new file mode 100644 index 000000000..e26fe2351 --- /dev/null +++ b/backend/src/lib/chatTools/workflow-store.ts @@ -0,0 +1,56 @@ +/** + * Build the per-turn workflow store: user-scoped + shared workflows from + * Supabase merged with BUILTIN_WORKFLOWS (lazy-imported). + */ + +import { createServerSupabase } from "../supabase"; +import type { WorkflowStore } from "./types"; + +export async function buildWorkflowStore( + userId: string, + userEmail: string | null | undefined, + db: ReturnType<typeof createServerSupabase>, +): Promise<WorkflowStore> { + const { BUILTIN_WORKFLOWS } = await import("../builtinWorkflows"); + const store: WorkflowStore = new Map(); + const normalizedUserEmail = (userEmail ?? "").trim().toLowerCase(); + + // Seed built-ins first + for (const wf of BUILTIN_WORKFLOWS) { + store.set(wf.id, { title: wf.title, prompt_md: wf.prompt_md }); + } + + // Then overlay user-owned assistant workflows. + const { data: workflows } = await db + .from("workflows") + .select("id, title, prompt_md") + .eq("user_id", userId) + .eq("type", "assistant"); + for (const wf of workflows ?? []) { + if (wf.prompt_md) { + store.set(wf.id, { title: wf.title, prompt_md: wf.prompt_md }); + } + } + + // Shared assistant workflows must also be readable by workflow tools. + if (normalizedUserEmail) { + const { data: shares } = await db + .from("workflow_shares") + .select("workflow_id") + .eq("shared_with_email", normalizedUserEmail); + const sharedIds = [...new Set((shares ?? []).map((share) => share.workflow_id))]; + if (sharedIds.length > 0) { + const { data: sharedWorkflows } = await db + .from("workflows") + .select("id, title, prompt_md") + .in("id", sharedIds) + .eq("type", "assistant"); + for (const wf of sharedWorkflows ?? []) { + if (wf.prompt_md) { + store.set(wf.id, { title: wf.title, prompt_md: wf.prompt_md }); + } + } + } + } + return store; +} diff --git a/backend/src/lib/crypto.ts b/backend/src/lib/crypto.ts new file mode 100644 index 000000000..9bdae1089 --- /dev/null +++ b/backend/src/lib/crypto.ts @@ -0,0 +1,65 @@ +import crypto from "crypto"; +import { env } from "../env"; + +/** + * AES-256-GCM envelope encryption for at-rest storage of user LLM API keys. + * + * CLEAN-05: user-supplied API keys (stored in user_profiles) are never held + * as plaintext in the database. Each key is encrypted with a fresh random IV + * under the operator-supplied HUGO_MASTER_KEY. + * + * Key design decisions (from 12-CONTEXT.md): + * - 96-bit (12-byte) IV per record, generated fresh by crypto.randomBytes each call. + * IV reuse under the same master key would break GCM's security guarantees. + * - 128-bit (16-byte) authentication tag — GCM default, provides integrity. + * - setAuthTag MUST be called before update/final on the decipher (Node.js requirement). + * - decryptApiKey returns null on any error (tampered ciphertext, wrong IV, wrong tag) + * rather than throwing — per CLAUDE.md "libs return null, do not throw". + */ + +const ALG = "aes-256-gcm" as const; +const KEY_LEN = 32; +const IV_LEN = 12; +const TAG_LEN = 16; + +const MASTER_KEY: Buffer = Buffer.from(env.HUGO_MASTER_KEY, "hex"); +if (MASTER_KEY.length !== KEY_LEN) { + throw new Error("[crypto] HUGO_MASTER_KEY must decode to 32 bytes (64 hex chars)"); +} + +export type Encrypted = { + ciphertext: Buffer; + iv: Buffer; // 12 bytes — GCM standard nonce length + authTag: Buffer; // 16 bytes — GCM authentication tag +}; + +/** + * Encrypts a plaintext API key string and returns the ciphertext + IV + authTag. + * + * A fresh random IV is generated per call — 1000 calls produce 1000 distinct IVs. + */ +export function encryptApiKey(plaintext: string): Encrypted { + const iv = crypto.randomBytes(IV_LEN); + const cipher = crypto.createCipheriv(ALG, MASTER_KEY, iv, { authTagLength: TAG_LEN }); + const ciphertext = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]); + const authTag = cipher.getAuthTag(); + return { ciphertext, iv, authTag }; +} + +/** + * Decrypts an encrypted API key. Returns null on any error (tampering, wrong key, + * invalid IV/tag) — never throws. + * + * Order is load-bearing: createDecipheriv → setAuthTag → update → final. + * Node.js requires setAuthTag before update/final for GCM mode. + */ +export function decryptApiKey(enc: Encrypted): string | null { + try { + const decipher = crypto.createDecipheriv(ALG, MASTER_KEY, enc.iv, { authTagLength: TAG_LEN }); + decipher.setAuthTag(enc.authTag); + const plain = Buffer.concat([decipher.update(enc.ciphertext), decipher.final()]); + return plain.toString("utf8"); + } catch { + return null; + } +} diff --git a/backend/src/lib/downloadTokens.ts b/backend/src/lib/downloadTokens.ts index de2240af1..13c47607b 100644 --- a/backend/src/lib/downloadTokens.ts +++ b/backend/src/lib/downloadTokens.ts @@ -1,4 +1,5 @@ import crypto from "crypto"; +import { env } from "../env"; /** * HMAC-signed, non-expiring download tokens. @@ -10,16 +11,7 @@ import crypto from "crypto"; */ function getSecret(): string { - const secret = - process.env.DOWNLOAD_SIGNING_SECRET ?? - process.env.SUPABASE_SECRET_KEY; - if (!secret) { - throw new Error( - "DOWNLOAD_SIGNING_SECRET (or SUPABASE_SECRET_KEY as a fallback) must be set. " + - "Generate a strong random value (e.g. `openssl rand -hex 32`) and set it in the environment.", - ); - } - return secret; + return env.DOWNLOAD_SIGNING_SECRET; } function b64urlEncode(buf: Buffer): string { diff --git a/backend/src/lib/llm/claude.ts b/backend/src/lib/llm/claude.ts index 0ecef37a8..7fd5919c7 100644 --- a/backend/src/lib/llm/claude.ts +++ b/backend/src/lib/llm/claude.ts @@ -7,6 +7,7 @@ import type { NormalizedToolResult, } from "./types"; import { toClaudeTools } from "./tools"; +import { logger } from "../logger"; type ContentBlock = | { type: "text"; text: string } @@ -73,6 +74,12 @@ export async function streamClaude( let sawThinking = false; + stream.on("streamEvent", (event) => { + if (process.env.LLM_STREAM_DEBUG) { + logger.debug({ event }, "[claude raw stream]"); + } + }); + stream.on("text", (delta) => { callbacks.onContentDelta?.(delta); }); diff --git a/backend/src/lib/llm/gemini.ts b/backend/src/lib/llm/gemini.ts index dd7c4d7b6..8702f3a23 100644 --- a/backend/src/lib/llm/gemini.ts +++ b/backend/src/lib/llm/gemini.ts @@ -5,6 +5,7 @@ import type { NormalizedToolCall, } from "./types"; import { toGeminiTools } from "./tools"; +import { logger } from "../logger"; type GeminiPart = { text?: string; @@ -28,64 +29,9 @@ type GeminiContent = { parts: GeminiPart[]; }; -const RETRYABLE_STATUSES = new Set([429, 500, 502, 503, 504]); -const MAX_GEMINI_ATTEMPTS = 3; - -function apiKey(override?: string | null): string { - const key = override?.trim() || process.env.GEMINI_API_KEY?.trim() || ""; - if (!key) { - throw new Error( - "Gemini API key is not configured. Set GEMINI_API_KEY or add a user Gemini key.", - ); - } - return key; -} - function client(override?: string | null): GoogleGenAI { - return new GoogleGenAI({ apiKey: apiKey(override) }); -} - -function geminiStatus(err: unknown): number | null { - const status = (err as { status?: unknown })?.status; - return typeof status === "number" ? status : null; -} - -function isRetryableGeminiError(err: unknown): boolean { - const status = geminiStatus(err); - if (status != null && RETRYABLE_STATUSES.has(status)) return true; - - const message = - err instanceof Error ? err.message : typeof err === "string" ? err : ""; - return /UNAVAILABLE|Service Unavailable|high demand|try again later/i.test( - message, - ); -} - -function retryDelayMs(attempt: number): number { - return 400 * 2 ** attempt; -} - -async function sleep(ms: number): Promise<void> { - await new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function withGeminiRetries<T>(operation: () => Promise<T>): Promise<T> { - let lastError: unknown; - for (let attempt = 0; attempt < MAX_GEMINI_ATTEMPTS; attempt++) { - try { - return await operation(); - } catch (err) { - lastError = err; - const isLastAttempt = attempt === MAX_GEMINI_ATTEMPTS - 1; - if (isLastAttempt || !isRetryableGeminiError(err)) throw err; - console.warn("[gemini] transient error; retrying", { - attempt: attempt + 1, - status: geminiStatus(err), - }); - await sleep(retryDelayMs(attempt)); - } - } - throw lastError; + const apiKey = override?.trim() || process.env.GEMINI_API_KEY || ""; + return new GoogleGenAI({ apiKey }); } function toNativeContents(messages: StreamChatParams["messages"]): GeminiContent[] { @@ -107,25 +53,23 @@ export async function streamGemini( let fullText = ""; for (let iter = 0; iter < maxIter; iter++) { - const stream = await withGeminiRetries(() => - ai.models.generateContentStream({ - model, - contents: contents as never, - config: { - systemInstruction: systemPrompt, - tools: functionDeclarations.length - ? [{ functionDeclarations } as never] - : undefined, - // When enabled, ask Gemini to surface thought summaries. - // When disabled, explicitly zero the thinking budget so the - // model skips thinking entirely (saves tokens and latency - // for bulk extraction jobs). - thinkingConfig: enableThinking - ? { includeThoughts: true } - : { thinkingBudget: 0 }, - }, - }), - ); + const stream = await ai.models.generateContentStream({ + model, + contents: contents as never, + config: { + systemInstruction: systemPrompt, + tools: functionDeclarations.length + ? [{ functionDeclarations } as never] + : undefined, + // When enabled, ask Gemini to surface thought summaries. + // When disabled, explicitly zero the thinking budget so the + // model skips thinking entirely (saves tokens and latency + // for bulk extraction jobs). + thinkingConfig: enableThinking + ? { includeThoughts: true } + : { thinkingBudget: 0 }, + }, + }); // Per-iteration accumulators. const textParts: string[] = []; @@ -134,6 +78,9 @@ export async function streamGemini( let sawThinking = false; for await (const chunk of stream) { + if (process.env.LLM_STREAM_DEBUG) { + logger.debug({ chunk }, "[gemini stream chunk]"); + } const parts = (chunk as { candidates?: { content?: { parts?: GeminiPart[] } }[] }) .candidates?.[0]?.content?.parts ?? []; @@ -207,14 +154,12 @@ export async function completeGeminiText(params: { apiKeys?: { gemini?: string | null }; }): Promise<string> { const ai = client(params.apiKeys?.gemini); - const resp = await withGeminiRetries(() => - ai.models.generateContent({ - model: params.model, - contents: [{ role: "user", parts: [{ text: params.user }] }], - config: params.systemPrompt - ? { systemInstruction: params.systemPrompt } - : undefined, - }), - ); + const resp = await ai.models.generateContent({ + model: params.model, + contents: [{ role: "user", parts: [{ text: params.user }] }], + config: params.systemPrompt + ? { systemInstruction: params.systemPrompt } + : undefined, + }); return resp.text ?? ""; } diff --git a/backend/src/lib/llm/index.ts b/backend/src/lib/llm/index.ts index 4b5e97936..518ddc015 100644 --- a/backend/src/lib/llm/index.ts +++ b/backend/src/lib/llm/index.ts @@ -1,6 +1,5 @@ import { streamClaude, completeClaudeText } from "./claude"; import { streamGemini, completeGeminiText } from "./gemini"; -import { streamOpenAI, completeOpenAIText } from "./openai"; import { providerForModel } from "./models"; import type { StreamChatParams, StreamChatResult, UserApiKeys } from "./types"; @@ -12,7 +11,6 @@ export async function streamChatWithTools( ): Promise<StreamChatResult> { const provider = providerForModel(params.model); if (provider === "claude") return streamClaude(params); - if (provider === "openai") return streamOpenAI(params); return streamGemini(params); } @@ -25,6 +23,5 @@ export async function completeText(params: { }): Promise<string> { const provider = providerForModel(params.model); if (provider === "claude") return completeClaudeText(params); - if (provider === "openai") return completeOpenAIText(params); return completeGeminiText(params); } diff --git a/backend/src/lib/llm/models.ts b/backend/src/lib/llm/models.ts index ed4872eff..52314007d 100644 --- a/backend/src/lib/llm/models.ts +++ b/backend/src/lib/llm/models.ts @@ -9,18 +9,15 @@ export const GEMINI_MAIN_MODELS = [ "gemini-3.1-pro-preview", "gemini-3-flash-preview", ] as const; -export const OPENAI_MAIN_MODELS = ["gpt-5.5", "gpt-5.4-mini"] as const; // Mid-tier (used for tabular review) — user picks one in account settings. export const CLAUDE_MID_MODELS = ["claude-sonnet-4-6"] as const; export const GEMINI_MID_MODELS = ["gemini-3-flash-preview"] as const; -export const OPENAI_MID_MODELS = ["gpt-5.4-mini"] as const; // Low-tier (used for title generation, lightweight extractions) — user picks // one in account settings. export const CLAUDE_LOW_MODELS = ["claude-haiku-4-5"] as const; export const GEMINI_LOW_MODELS = ["gemini-3.1-flash-lite-preview"] as const; -export const OPENAI_LOW_MODELS = ["gpt-5.4-nano"] as const; export const DEFAULT_MAIN_MODEL = "gemini-3-flash-preview"; export const DEFAULT_TITLE_MODEL = "gemini-3.1-flash-lite-preview"; @@ -29,13 +26,10 @@ export const DEFAULT_TABULAR_MODEL = "gemini-3-flash-preview"; const ALL_MODELS = new Set<string>([ ...CLAUDE_MAIN_MODELS, ...GEMINI_MAIN_MODELS, - ...OPENAI_MAIN_MODELS, ...CLAUDE_MID_MODELS, ...GEMINI_MID_MODELS, - ...OPENAI_MID_MODELS, ...CLAUDE_LOW_MODELS, ...GEMINI_LOW_MODELS, - ...OPENAI_LOW_MODELS, ]); // --------------------------------------------------------------------------- @@ -45,7 +39,6 @@ const ALL_MODELS = new Set<string>([ export function providerForModel(model: string): Provider { if (model.startsWith("claude")) return "claude"; if (model.startsWith("gemini")) return "gemini"; - if (model.startsWith("gpt-")) return "openai"; throw new Error(`Unknown model id: ${model}`); } diff --git a/backend/src/lib/llm/openai.ts b/backend/src/lib/llm/openai.ts deleted file mode 100644 index dbb7ef65b..000000000 --- a/backend/src/lib/llm/openai.ts +++ /dev/null @@ -1,291 +0,0 @@ -import type { - LlmMessage, - NormalizedToolCall, - NormalizedToolResult, - OpenAIToolSchema, - StreamChatParams, - StreamChatResult, -} from "./types"; - -const OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"; -const MAX_OUTPUT_TOKENS = 16384; - -type ResponseInputItem = - | { role: "user" | "assistant"; content: string } - | { type: "function_call_output"; call_id: string; output: string }; - -type ResponseFunctionTool = { - type: "function"; - name: string; - description?: string; - parameters: Record<string, unknown>; -}; - -type ResponseFunctionCallItem = { - type: "function_call"; - call_id?: string; - name?: string; - arguments?: string; -}; - -type ResponseStreamEvent = { - type?: string; - delta?: string; - response?: { id?: string; output_text?: string }; - item?: ResponseFunctionCallItem; -}; - -function apiKey(override?: string | null): string { - return override?.trim() || process.env.OPENAI_API_KEY?.trim() || ""; -} - -function toResponseTools(tools: OpenAIToolSchema[]): ResponseFunctionTool[] { - return tools.map((tool) => ({ - type: "function", - name: tool.function.name, - description: tool.function.description, - parameters: tool.function.parameters, - })); -} - -function toResponseInput(messages: LlmMessage[]): ResponseInputItem[] { - return messages.map((message) => ({ - role: message.role, - content: message.content, - })); -} - -function extractSseJson(buffer: string): { events: unknown[]; rest: string } { - const events: unknown[] = []; - const chunks = buffer.split(/\n\n/); - const rest = chunks.pop() ?? ""; - - for (const chunk of chunks) { - const dataLines = chunk - .split("\n") - .map((line) => line.trim()) - .filter((line) => line.startsWith("data:")) - .map((line) => line.slice(5).trim()); - - for (const data of dataLines) { - if (!data || data === "[DONE]") continue; - try { - events.push(JSON.parse(data)); - } catch { - // Incomplete events stay buffered until the next read. - } - } - } - - return { events, rest }; -} - -function parseFunctionCall(item: ResponseFunctionCallItem): NormalizedToolCall { - let input: Record<string, unknown> = {}; - try { - const parsed = JSON.parse(item.arguments || "{}"); - if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { - input = parsed as Record<string, unknown>; - } - } catch { - input = {}; - } - - return { - id: item.call_id ?? item.name ?? "function_call", - name: item.name ?? "", - input, - }; -} - -async function createResponse(params: { - model: string; - input: ResponseInputItem[]; - instructions?: string; - tools?: ResponseFunctionTool[]; - stream?: boolean; - maxTokens?: number; - previousResponseId?: string; - reasoningSummary?: boolean; - apiKey: string; -}): Promise<Response> { - const response = await fetch(OPENAI_RESPONSES_URL, { - method: "POST", - headers: { - Authorization: `Bearer ${params.apiKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - model: params.model, - instructions: params.instructions || undefined, - input: params.input, - tools: params.tools?.length ? params.tools : undefined, - stream: params.stream, - max_output_tokens: params.maxTokens ?? MAX_OUTPUT_TOKENS, - previous_response_id: params.previousResponseId, - reasoning: params.reasoningSummary - ? { summary: "auto" } - : undefined, - }), - }); - - if (!response.ok) { - const text = await response.text().catch(() => ""); - throw new Error( - `OpenAI request failed (${response.status}): ${text || response.statusText}`, - ); - } - - return response; -} - -export async function streamOpenAI( - params: StreamChatParams, -): Promise<StreamChatResult> { - const { - model, - systemPrompt, - tools = [], - callbacks = {}, - runTools, - apiKeys, - enableThinking, - } = params; - const maxIter = params.maxIterations ?? 10; - const key = apiKey(apiKeys?.openai); - const responseTools = toResponseTools(tools); - let input = toResponseInput(params.messages); - let previousResponseId: string | undefined; - let fullText = ""; - const hasTools = responseTools.length > 0; - - for (let iter = 0; iter < maxIter; iter++) { - const response = await createResponse({ - model, - instructions: iter === 0 ? systemPrompt : undefined, - input, - tools: responseTools, - stream: true, - previousResponseId, - reasoningSummary: !!enableThinking, - apiKey: key, - }); - if (!response.body) throw new Error("OpenAI response had no body"); - - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - const toolCalls: NormalizedToolCall[] = []; - const startedToolCallIds = new Set<string>(); - let buffer = ""; - let pendingText = ""; - let sawReasoning = false; - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - buffer += decoder.decode(value, { stream: true }); - const extracted = extractSseJson(buffer); - buffer = extracted.rest; - - for (const event of extracted.events as ResponseStreamEvent[]) { - if (event.response?.id) { - previousResponseId = event.response.id; - } - - if ( - event.type === "response.reasoning_summary_text.delta" && - typeof event.delta === "string" - ) { - sawReasoning = true; - callbacks.onReasoningDelta?.(event.delta); - } - - if ( - event.type === "response.output_text.delta" && - typeof event.delta === "string" - ) { - if (hasTools) { - pendingText += event.delta; - } else { - fullText += event.delta; - callbacks.onContentDelta?.(event.delta); - } - } - - if ( - event.type === "response.output_item.added" && - event.item?.type === "function_call" - ) { - const call = parseFunctionCall(event.item); - startedToolCallIds.add(call.id); - callbacks.onToolCallStart?.(call); - } - - if ( - event.type === "response.output_item.done" && - event.item?.type === "function_call" - ) { - const call = parseFunctionCall(event.item); - if (!startedToolCallIds.has(call.id)) { - callbacks.onToolCallStart?.(call); - } - toolCalls.push(call); - } - } - } - - if (sawReasoning) callbacks.onReasoningBlockEnd?.(); - - if (!toolCalls.length || !runTools) { - if (pendingText) { - fullText += pendingText; - callbacks.onContentDelta?.(pendingText); - } - break; - } - - const results = await runTools(toolCalls); - input = results.map((result) => ({ - type: "function_call_output", - call_id: result.tool_use_id, - output: result.content, - })); - } - - return { fullText }; -} - -export async function completeOpenAIText(params: { - model: string; - systemPrompt?: string; - user: string; - maxTokens?: number; - apiKeys?: { openai?: string | null }; -}): Promise<string> { - const response = await createResponse({ - model: params.model, - instructions: params.systemPrompt, - input: [{ role: "user", content: params.user }], - maxTokens: params.maxTokens ?? 512, - apiKey: apiKey(params.apiKeys?.openai), - }); - const json = (await response.json()) as { - output_text?: string; - output?: { - content?: { type?: string; text?: string }[]; - }[]; - }; - - if (typeof json.output_text === "string") return json.output_text; - - return ( - json.output - ?.flatMap((item) => item.content ?? []) - .filter((content) => content.type === "output_text") - .map((content) => content.text ?? "") - .join("") ?? "" - ); -} - -export type { NormalizedToolResult }; diff --git a/backend/src/lib/llm/types.ts b/backend/src/lib/llm/types.ts index a8409d80e..8cc411a79 100644 --- a/backend/src/lib/llm/types.ts +++ b/backend/src/lib/llm/types.ts @@ -2,7 +2,7 @@ // Callers always speak OpenAI-style tools + { role, content } messages; each // provider translates internally. -export type Provider = "claude" | "gemini" | "openai"; +export type Provider = "claude" | "gemini"; export type OpenAIToolSchema = { type: "function"; @@ -39,7 +39,6 @@ export type StreamCallbacks = { export type UserApiKeys = { claude?: string | null; gemini?: string | null; - openai?: string | null; }; export type StreamChatParams = { diff --git a/backend/src/lib/logger.ts b/backend/src/lib/logger.ts new file mode 100644 index 000000000..adb03563d --- /dev/null +++ b/backend/src/lib/logger.ts @@ -0,0 +1,83 @@ +/** + * Structured pino logger for the Hugo backend. + * + * Optional env vars: + * LOG_LEVEL — pino log level (default: "info") + * NODE_ENV — when !== "production", enables pino-pretty dev transport + * + * Redacted paths (never appear in logs): + * messages[*].content — legal document content in chat messages + * body.messages[*].content + * *.api_key — user-supplied LLM provider keys + * api_key + * apiKeys.claude — resolved per-request Claude provider key + * apiKeys.gemini — resolved per-request Gemini provider key + * *.apiKeys.claude — nested variants (e.g. req.body.apiKeys.claude) + * *.apiKeys.gemini — nested variants (e.g. ctx.apiKeys.gemini) + * req.headers.authorization + * req.headers.cookie — session cookies + * Authorization + * *.claude_api_key_ciphertext — CLEAN-05: bytea ciphertext column (user_profiles) + * *.claude_api_key_iv — CLEAN-05: bytea IV column + * *.claude_api_key_auth_tag — CLEAN-05: bytea auth tag column + * *.gemini_api_key_ciphertext — CLEAN-05: bytea ciphertext column (user_profiles) + * *.gemini_api_key_iv — CLEAN-05: bytea IV column + * *.gemini_api_key_auth_tag — CLEAN-05: bytea auth tag column + * *.plaintext — CLEAN-05: defensive guard against any "plaintext" variable + * plaintext — CLEAN-05: top-level plaintext guard + */ +import pino from "pino"; +import pinoHttp from "pino-http"; +import { randomUUID } from "crypto"; + +const isDev = process.env.NODE_ENV !== "production"; + +export const logger = pino({ + level: process.env.LOG_LEVEL ?? "info", + redact: { + paths: [ + "messages[*].content", + "body.messages[*].content", + "*.api_key", + "api_key", + "apiKeys.claude", + "apiKeys.gemini", + "*.apiKeys.claude", + "*.apiKeys.gemini", + "req.headers.authorization", + "req.headers.cookie", + "Authorization", + // CLEAN-05: Pino redaction misses new variable names (Pitfall 7). + // *.api_key matches a literal property named "api_key" — NOT "api_key_ciphertext". + // These paths must be listed explicitly. + "*.claude_api_key_ciphertext", + "*.claude_api_key_iv", + "*.claude_api_key_auth_tag", + "*.gemini_api_key_ciphertext", + "*.gemini_api_key_iv", + "*.gemini_api_key_auth_tag", + "*.plaintext", + "plaintext", + ], + censor: "[REDACTED]", + }, + transport: isDev + ? { target: "pino-pretty", options: { colorize: true } } + : undefined, +}); + +export const httpLogger = pinoHttp({ + logger, + genReqId: (req, res) => { + const existing = req.headers["x-request-id"]; + if (existing) return Array.isArray(existing) ? existing[0] : existing; + const id = randomUUID(); + res.setHeader("X-Request-Id", id); + return id; + }, + customLogLevel: (_req, res) => { + if (res.statusCode >= 500) return "error"; + if (res.statusCode >= 400) return "warn"; + return "info"; + }, +}); diff --git a/backend/src/lib/pdfQueue.ts b/backend/src/lib/pdfQueue.ts new file mode 100644 index 000000000..923f9f9e9 --- /dev/null +++ b/backend/src/lib/pdfQueue.ts @@ -0,0 +1,107 @@ +import { docxToPdf, convertedPdfKey } from "./convert"; +import { uploadFile, downloadFile } from "./storage"; +import { createServerSupabase } from "./supabase"; +import { logger } from "./logger"; + +let _queue: import("p-queue").default | null = null; + +async function getQueue(): Promise<import("p-queue").default> { + if (!_queue) { + const { default: PQueue } = await import("p-queue"); + _queue = new PQueue({ concurrency: 1 }); + } + return _queue; +} + +export async function enqueueConversionFromBuffer(params: { + documentId: string; + versionId: string; + userId: string; + docxBuffer: Buffer; +}): Promise<void> { + const queue = await getQueue(); + void queue.add(async () => { + const db = createServerSupabase(); + try { + const { documentId, versionId, userId, docxBuffer } = params; + const pdfBuf = await docxToPdf(docxBuffer); + const pdfKey = convertedPdfKey(userId, documentId); + const ab = pdfBuf.buffer.slice( + pdfBuf.byteOffset, + pdfBuf.byteOffset + pdfBuf.byteLength, + ) as ArrayBuffer; + await uploadFile(pdfKey, ab, "application/pdf"); + await db + .from("document_versions") + .update({ pdf_storage_path: pdfKey }) + .eq("id", versionId); + await db + .from("documents") + .update({ pdf_conversion_status: "ok" }) + .eq("id", documentId); + } catch (err) { + logger.error({ err, documentId: params.documentId }, "[pdfQueue] conversion failed"); + await db + .from("documents") + .update({ pdf_conversion_status: "failed" }) + .eq("id", params.documentId); + } + }); +} + +export async function enqueueConversionForVersion( + documentId: string, + version: { id: string; storage_path: string }, + db: ReturnType<typeof createServerSupabase>, +): Promise<void> { + const queue = await getQueue(); + void queue.add(async () => { + try { + const raw = await downloadFile(version.storage_path); + if (!raw) throw new Error("Source DOCX not found in R2"); + const docxBuf = Buffer.from(raw); + const pdfBuf = await docxToPdf(docxBuf); + const pdfKey = `${version.storage_path.replace(/\.[^.]+$/, "")}_rendered.pdf`; + const ab = pdfBuf.buffer.slice( + pdfBuf.byteOffset, + pdfBuf.byteOffset + pdfBuf.byteLength, + ) as ArrayBuffer; + await uploadFile(pdfKey, ab, "application/pdf"); + await db + .from("document_versions") + .update({ pdf_storage_path: pdfKey }) + .eq("id", version.id); + await db + .from("documents") + .update({ pdf_conversion_status: "ok" }) + .eq("id", documentId); + } catch (err) { + logger.error({ err, documentId }, "[pdfQueue] retry failed"); + await db + .from("documents") + .update({ pdf_conversion_status: "failed" }) + .eq("id", documentId); + } + }); +} + +export async function resetStuckPendingConversions(): Promise<void> { + try { + const db = createServerSupabase(); + const { data, error } = await db + .from("documents") + .update({ pdf_conversion_status: "failed" }) + .eq("pdf_conversion_status", "pending") + .select("id"); + if (error) { + logger.error({ err: error }, "[pdfQueue] resetStuckPendingConversions failed"); + return; + } + const count = data?.length ?? 0; + if (count > 0) { + logger.info({ count }, "[pdfQueue] startup fixup: reset stuck pending rows to failed"); + } + } catch (err) { + logger.error({ err }, "[pdfQueue] resetStuckPendingConversions threw"); + } +} diff --git a/backend/src/lib/rateLimiter.ts b/backend/src/lib/rateLimiter.ts new file mode 100644 index 000000000..4ad07417d --- /dev/null +++ b/backend/src/lib/rateLimiter.ts @@ -0,0 +1,25 @@ +import { rateLimit } from "express-rate-limit"; + +const WINDOW_MS = Number(process.env.RATE_LIMIT_WINDOW_MS ?? 60_000); +const MAX = Number(process.env.RATE_LIMIT_MAX ?? 20); + +/** + * Per-user LLM rate limiter. + * + * MUST run AFTER requireAuth in the middleware chain — reads res.locals.userId. + * + * Env vars (optional — defaults used if absent): + * RATE_LIMIT_WINDOW_MS — sliding window in milliseconds (default: 60000 = 1 minute) + * RATE_LIMIT_MAX — max requests per user per window (default: 20) + */ +export const llmRateLimiter = rateLimit({ + windowMs: WINDOW_MS, + limit: MAX, + standardHeaders: "draft-8", + legacyHeaders: false, + keyGenerator: (_req, res) => `user:${res.locals.userId as string}`, + handler: (_req, res) => { + res.setHeader("Retry-After", String(Math.ceil(WINDOW_MS / 1000))); + res.status(429).json({ detail: "Rate limit exceeded. Try again later." }); + }, +}); diff --git a/backend/src/lib/restoreTokens.ts b/backend/src/lib/restoreTokens.ts new file mode 100644 index 000000000..270cee736 --- /dev/null +++ b/backend/src/lib/restoreTokens.ts @@ -0,0 +1,86 @@ +import crypto from "crypto"; +import { env } from "../env"; + +/** + * HMAC-signed account-restore tokens (CLEAN-44). + * + * When `DELETE /user/account` soft-deletes a user, it generates a signed + * restore token and returns it in the response body. The user can call + * `POST /user/account/restore?token=<token>` within the 30-day grace window + * to reverse the deletion. + * + * Token format: `<b64url-encoded-payload>.<b64url-encoded-hmac-sha256-sig>` + * + * The payload encodes `{ user_id, action: "restore", exp }` where `exp` is a + * Unix-millisecond timestamp. Tokens are verified without a DB lookup; + * single-use enforcement (replay prevention) is the responsibility of Plan 07 + * via `account_deletion_jobs.restore_token_used_at`. + * + * Mirrors `backend/src/lib/downloadTokens.ts` line-for-line; the only + * differences are the payload shape, the secret variable, and the expiry check. + */ + +function getSecret(): string { + return env.HUGO_RESTORE_TOKEN_SECRET; +} + +function b64urlEncode(buf: Buffer): string { + return buf + .toString("base64") + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=+$/g, ""); +} + +function b64urlDecode(s: string): Buffer { + let t = s.replace(/-/g, "+").replace(/_/g, "/"); + while (t.length % 4) t += "="; + return Buffer.from(t, "base64"); +} + +export type RestorePayload = { user_id: string; action: "restore"; exp: number }; + +export function signRestoreToken(userId: string, expiresAt: Date): string { + const payload: RestorePayload = { + user_id: userId, + action: "restore", + exp: expiresAt.getTime(), + }; + const enc = b64urlEncode(Buffer.from(JSON.stringify(payload), "utf8")); + const sig = crypto + .createHmac("sha256", getSecret()) + .update(enc) + .digest(); + return `${enc}.${b64urlEncode(sig)}`; +} + +export function verifyRestoreToken(token: string): RestorePayload | null { + const parts = token.split("."); + if (parts.length !== 2) return null; + const [enc, sigEnc] = parts; + const expected = crypto + .createHmac("sha256", getSecret()) + .update(enc) + .digest(); + // Compare raw HMAC bytes via timingSafeEqual on Buffers — comparing + // base64url strings leaks length via early-return and uses a different + // bit-level comparison than the digest. (CLEAN-44 CR-03) + let provided: Buffer; + try { + provided = b64urlDecode(sigEnc); + } catch { + return null; + } + if (provided.length !== expected.length) return null; + if (!crypto.timingSafeEqual(provided, expected)) return null; + try { + const parsed = JSON.parse(b64urlDecode(enc).toString("utf8")) as RestorePayload; + if (!parsed?.user_id || typeof parsed.user_id !== "string") return null; + if (parsed.action !== "restore") return null; + if (typeof parsed.exp !== "number") return null; + if (parsed.exp <= Date.now()) return null; + return parsed; + } catch { + return null; + } +} diff --git a/backend/src/lib/storage.ts b/backend/src/lib/storage.ts index f5035a395..51255ed56 100644 --- a/backend/src/lib/storage.ts +++ b/backend/src/lib/storage.ts @@ -14,6 +14,8 @@ import { PutObjectCommand, GetObjectCommand, DeleteObjectCommand, + ListObjectsV2Command, + DeleteObjectsCommand, } from "@aws-sdk/client-s3"; import { getSignedUrl as awsGetSignedUrl } from "@aws-sdk/s3-request-presigner"; @@ -21,7 +23,6 @@ function getClient(): S3Client { return new S3Client({ region: "auto", endpoint: process.env.R2_ENDPOINT_URL!, - forcePathStyle: true, credentials: { accessKeyId: process.env.R2_ACCESS_KEY_ID!, secretAccessKey: process.env.R2_SECRET_ACCESS_KEY!, @@ -86,6 +87,75 @@ export async function deleteFile(key: string): Promise<void> { await client.send(new DeleteObjectCommand({ Bucket: BUCKET, Key: key })); } +// --------------------------------------------------------------------------- +// Enumerate (paginated listing, for deletion worker) +// --------------------------------------------------------------------------- + +/** + * Enumerate every object key under `prefix`, in batches of up to 1000. + * Resumable via `startToken` (the worker persists the last token in DB). + * Yields `{ keys, nextToken }` where `nextToken` is undefined on the last batch. + */ +export async function* listObjectsByPrefix( + prefix: string, + startToken?: string, +): AsyncGenerator<{ keys: string[]; nextToken: string | undefined }> { + if (!storageEnabled) return; + const client = getClient(); + let token: string | undefined = startToken; + do { + const out = await client.send( + new ListObjectsV2Command({ + Bucket: BUCKET, + Prefix: prefix, + MaxKeys: 1000, + ContinuationToken: token, + }), + ); + const keys = (out.Contents ?? []) + .map((o) => o.Key) + .filter((k): k is string => Boolean(k)); + token = out.IsTruncated ? out.NextContinuationToken : undefined; + yield { keys, nextToken: token }; + } while (token); +} + +// --------------------------------------------------------------------------- +// Batch delete (for deletion worker) +// --------------------------------------------------------------------------- + +/** + * Batch-delete up to 1000 R2 keys in a single DeleteObjects call. + * `Quiet: true` suppresses successful keys in the response; only errors come back. + * Returns the count of successfully-deleted keys + the list of error messages. + */ +export async function deleteObjectsBatch( + keys: string[], +): Promise<{ deleted: number; errors: string[] }> { + if (keys.length === 0) return { deleted: 0, errors: [] }; + if (keys.length > 1000) { + throw new Error("[storage] deleteObjectsBatch: max 1000 keys per call"); + } + if (!storageEnabled) return { deleted: 0, errors: [] }; + const client = getClient(); + const out = await client.send( + new DeleteObjectsCommand({ + Bucket: BUCKET, + Delete: { + Objects: keys.map((Key) => ({ Key })), + Quiet: true, + }, + }), + ); + const errors = (out.Errors ?? []).map( + (e) => `${e.Key}: ${e.Code ?? "unknown"} ${e.Message ?? ""}`.trim(), + ); + return { + deleted: keys.length - errors.length, + errors, + }; +} + // --------------------------------------------------------------------------- // Signed URL (pre-signed for temporary direct access) // --------------------------------------------------------------------------- @@ -123,9 +193,7 @@ export function normalizeDownloadFilename(name: string): string { } export function sanitizeDispositionFilename(name: string): string { - return normalizeDownloadFilename(name) - .replace(/["\\]/g, "_") - .replace(/[^\x20-\x7E]/g, "_"); + return normalizeDownloadFilename(name).replace(/["\\]/g, "_"); } export function encodeRFC5987(str: string): string { diff --git a/backend/src/lib/structureTree.ts b/backend/src/lib/structureTree.ts new file mode 100644 index 000000000..686756e04 --- /dev/null +++ b/backend/src/lib/structureTree.ts @@ -0,0 +1,100 @@ +export interface StructureNode { + id: string; + title: string; + level: number; + page_number: number | null; + children: StructureNode[]; +} + +export async function extractStructureTree( + content: ArrayBuffer | Buffer, + fileType: string, +): Promise<StructureNode[] | null> { + try { + const ft = fileType.toLowerCase(); + if (ft === "pdf") { + return await extractPdfOutline(content); + } else if (ft === "docx" || ft === "doc") { + return await extractDocxHeadings(content); + } + return null; + } catch { + return null; + } +} + +async function extractDocxHeadings( + content: ArrayBuffer | Buffer, +): Promise<StructureNode[] | null> { + try { + const mammoth = await import("mammoth"); + const { value: html } = await mammoth.convertToHtml({ + buffer: Buffer.isBuffer(content) ? content : Buffer.from(content), + }); + const headingRegex = /<(h[1-6])[^>]*>(.*?)<\/\1>/gi; + const nodes: StructureNode[] = []; + let match: RegExpExecArray | null; + let idx = 0; + while ((match = headingRegex.exec(html)) !== null) { + const level = parseInt(match[1].slice(1), 10); + const title = match[2].replace(/<[^>]+>/g, "").trim().slice(0, 120); + if (!title) continue; + nodes.push({ + id: `h${level}-${idx++}`, + title, + level, + page_number: null, + children: [], + }); + } + return nodes.length ? nodes : null; + } catch { + return null; + } +} + +async function extractPdfOutline( + content: ArrayBuffer | Buffer, +): Promise<StructureNode[] | null> { + try { + const buf = Buffer.isBuffer(content) + ? (content.buffer.slice( + content.byteOffset, + content.byteOffset + content.byteLength, + ) as ArrayBuffer) + : content; + const pdfjsLib = await import( + "pdfjs-dist/legacy/build/pdf.mjs" as string + ); + const pdf = await ( + pdfjsLib as unknown as { + getDocument: (opts: unknown) => { + promise: Promise<{ + numPages: number; + getOutline: () => Promise<{ title?: string }[]>; + }>; + }; + } + ).getDocument({ data: new Uint8Array(buf) }).promise; + if (pdf.numPages <= 5) return null; + const outline = await pdf.getOutline(); + if (outline?.length) { + return outline.map((item, i) => ({ + id: `h1-${i}`, + title: item.title ?? `Item ${i + 1}`, + level: 1, + page_number: null, + children: [], + })); + } + return Array.from({ length: pdf.numPages }, (_, i) => ({ + id: `page-${i + 1}`, + title: `Page ${i + 1}`, + level: 1, + page_number: i + 1, + children: [], + })); + } catch { + return null; + } +} diff --git a/backend/src/lib/supabase.ts b/backend/src/lib/supabase.ts index 02bf11894..9a26c8cdb 100644 --- a/backend/src/lib/supabase.ts +++ b/backend/src/lib/supabase.ts @@ -1,18 +1,298 @@ -import { createClient } from "@supabase/supabase-js"; +/** + * Supabase admin client + token-verification cache. + * + * Required env vars: + * SUPABASE_URL — project URL (e.g. https://xxx.supabase.co) + * SUPABASE_SECRET_KEY — service-role JWT (bypasses RLS) + * + * The admin client is constructed once at module-load. Repeated calls to + * `createServerSupabase()` return the same instance so callers that still use + * the factory function don't break. + * + * `verifyToken` verifies bearer tokens via JWKS (for ES256/RS256 asymmetric + * keys used by Supabase CLI v2+) or via HMAC when SUPABASE_JWT_SECRET is set + * (HS256, used by older Supabase versions). Results are cached in an LRU for + * 60 s so chatty request bursts don't fan out to GoTrue. Only successful + * lookups are cached — failures are never stored so revocation takes effect + * on the next request. + */ + +import { createClient, type SupabaseClient } from "@supabase/supabase-js"; +import { LRUCache } from "lru-cache"; +import { createHash, createHmac, webcrypto } from "crypto"; +import { logger } from "./logger"; + +// ── Module-scope singleton ──────────────────────────────────────────────────── + +/** + * Single admin client shared across the entire process lifetime. + * `persistSession: false` prevents the SDK from writing to disk. + * `autoRefreshToken: false` is a no-op for service-role keys but avoids + * background timers that complicate unit-test teardown. + */ +export const adminClient: SupabaseClient = createClient( + process.env.SUPABASE_URL ?? "", + process.env.SUPABASE_SECRET_KEY ?? "", + { auth: { persistSession: false, autoRefreshToken: false } }, +); + +/** + * Backward-compatible factory function. Callers that still use + * `createServerSupabase()` get the singleton without any refactor cost. + */ +export function createServerSupabase(): SupabaseClient { + return adminClient; +} + +// ── Token-verification cache ────────────────────────────────────────────────── + +/** The shape of a verified Supabase user as stored in the LRU cache. */ +export type CachedUser = { id: string; email: string }; + +/** + * LRU cache for verified token results. + * + * max 1000 — hard cap on number of concurrent sessions cached. + * ttl 60 s — entries expire 60 s after insertion (not after last access). + * updateAgeOnGet: false — reading an entry does NOT reset its TTL; expiry is + * always relative to the insertion time so that a + * revoked token expires predictably. + */ +const userCache = new LRUCache<string, CachedUser>({ + max: 1000, + ttl: 60_000, + updateAgeOnGet: false, +}); + +/** + * Derives a cache key from a bearer token without storing the raw token. + * sha256 hex is 64 chars and is effectively collision-free for this purpose. + */ +function tokenKey(token: string): string { + return createHash("sha256").update(token).digest("hex"); +} + +// ── JWKS-based JWT verification ─────────────────────────────────────────────── + +interface JwkKey { + kty: string; + alg?: string; + kid?: string; + use?: string; + [k: string]: unknown; +} + +let _jwksCache: JwkKey[] | null = null; +let _jwksCachedAt = 0; + +async function getJwks(): Promise<JwkKey[]> { + const now = Date.now(); + if (_jwksCache && now - _jwksCachedAt < 5 * 60_000) return _jwksCache; + + const url = `${process.env.SUPABASE_URL}/auth/v1/.well-known/jwks.json`; + const resp = await fetch(url); + if (!resp.ok) throw new Error(`JWKS fetch failed: ${resp.status}`); + const json = (await resp.json()) as { keys: JwkKey[] }; + _jwksCache = json.keys ?? []; + _jwksCachedAt = now; + return _jwksCache; +} + +async function verifyAsymmetricJwt( + parts: string[], + kid: string | undefined, + alg: string, +): Promise<boolean> { + const keys = await getJwks(); + const jwk = kid ? keys.find((k) => k.kid === kid) : keys[0]; + if (!jwk) return false; + + const namedCurve = alg === "ES384" ? "P-384" : "P-256"; + const hashName = alg === "ES384" ? "SHA-384" : "SHA-256"; + const keyAlg = + alg.startsWith("RS") ? { name: "RSASSA-PKCS1-v1_5", hash: hashName } + : { name: "ECDSA", namedCurve }; + const verifyAlg = + alg.startsWith("RS") ? { name: "RSASSA-PKCS1-v1_5" } + : { name: "ECDSA", hash: { name: hashName } }; + + const cryptoKey = await (webcrypto.subtle as SubtleCrypto).importKey( + "jwk", + jwk as JsonWebKey, + keyAlg as AlgorithmIdentifier, + false, + ["verify"], + ); + + const data = Buffer.from(`${parts[0]}.${parts[1]}`); + const sig = Buffer.from(parts[2], "base64url"); + + return (webcrypto.subtle as SubtleCrypto).verify( + verifyAlg as AlgorithmIdentifier, + cryptoKey, + sig, + data, + ); +} + +async function verifyJwtLocally(token: string): Promise<CachedUser | null> { + const parts = token.split("."); + if (parts.length !== 3) { + const { data } = await adminClient.auth.getUser(token); + if (!data.user) return null; + return { + id: data.user.id, + email: (data.user.email ?? "").toLowerCase(), + }; + } + + let header: { alg?: string; kid?: string }; + let payload: { sub?: string; email?: string; exp?: number }; + try { + header = JSON.parse(Buffer.from(parts[0], "base64url").toString("utf8")); + payload = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")); + } catch { + return null; + } + + // Expiry check + if (payload.exp && payload.exp < Math.floor(Date.now() / 1000)) return null; + if (!payload.sub) return null; + + const alg = header.alg ?? "HS256"; + + if (alg === "HS256") { + const secret = process.env.SUPABASE_JWT_SECRET; + if (!secret) { + // No local secret — fall back to GoTrue admin round-trip. + const { data } = await adminClient.auth.getUser(token); + if (!data.user) return null; + } else { + const expected = createHmac("sha256", secret) + .update(`${parts[0]}.${parts[1]}`) + .digest("base64url"); + if (expected !== parts[2]) return null; + } + } else if (alg.startsWith("ES") || alg.startsWith("RS")) { + const valid = await verifyAsymmetricJwt(parts, header.kid, alg); + if (!valid) return null; + } else { + logger.warn({ alg }, "[auth] verifyToken: unsupported JWT algorithm"); + return null; + } + + return { + id: payload.sub, + email: (payload.email ?? "").toLowerCase(), + }; +} + +/** + * Verifies a bearer token and returns the authenticated user. + * + * Supports ES256/RS256 (Supabase CLI v2+, asymmetric keys via JWKS) and HS256 + * (older Supabase, requires SUPABASE_JWT_SECRET in env or falls back to the + * GoTrue admin round-trip). + * + * Returns `null` on failure. Never caches failures. + */ +export async function verifyToken(token: string): Promise<CachedUser | null> { + const key = tokenKey(token); + const cached = userCache.get(key); + if (cached !== undefined) return cached; + + try { + const user = await verifyJwtLocally(token); + if (!user) return null; + userCache.set(key, user); + return user; + } catch (err) { + logger.error({ err }, "[auth] verifyToken threw unexpectedly"); + return null; + } +} + +/** + * Clears the token cache. Exposed for test isolation — call in `beforeEach`. + * NOT intended for production use. + */ +export function _resetAuthCache(): void { + userCache.clear(); +} + +// ── Legacy / unused helpers ─────────────────────────────────────────────────── + +// ── Auth user lookup helpers (RPC-backed, CLEAN-15) ────────────────────────── /** - * Server-side Supabase client using the service role key. - * Bypasses RLS — only use in API routes after verifying the user. + * Look up a single auth user by email via the `get_auth_user_by_email` RPC. + * + * The RPC is SECURITY DEFINER with `search_path = ''` and is only callable by + * service_role — so this helper is safe for backend use and does not expose + * the full auth.users table to callers. + * + * Returns null if the user is not found or if the RPC call fails. */ -export function createServerSupabase() { - const url = process.env.SUPABASE_URL || ""; - const key = process.env.SUPABASE_SECRET_KEY || ""; - return createClient(url, key, { auth: { persistSession: false } }); +export async function getUserByEmail( + email: string, +): Promise<{ id: string; email: string } | null> { + const { data, error } = await adminClient.rpc( + "get_auth_user_by_email", + { p_email: email }, + ); + if (error || !Array.isArray(data) || data.length === 0) return null; + const row = data[0] as { id: string; email: string }; + return { id: row.id, email: row.email }; } /** - * Extract and verify the Supabase JWT from the Authorization header. - * Returns the user's UUID string, or throws a Response with 401. + * Look up multiple auth users by email. Returns a Map keyed on the + * lowercased email so callers can do O(1) lookups after a single fan-out. + * + * Unknown emails are silently omitted from the result (matching the prior + * behaviour where unregistered shared_with entries were simply absent). + */ +export async function getUsersByEmails( + emails: string[], +): Promise<Map<string, { id: string; email: string }>> { + const map = new Map<string, { id: string; email: string }>(); + await Promise.all( + emails.map(async (e) => { + const u = await getUserByEmail(e); + if (u) map.set(e.toLowerCase(), u); + }), + ); + return map; +} + +/** + * Look up a single auth user by UUID via the `get_auth_user_by_id` RPC. + * + * Used for owner email resolution in /people endpoints — avoids the + * listUsers paging approach and resolves in O(log N) via the auth.users PK. + * + * Returns null if the user is not found or if the RPC call fails. + */ +export async function getUserById( + userId: string, +): Promise<{ id: string; email: string } | null> { + const { data, error } = await adminClient.rpc( + "get_auth_user_by_id", + { p_id: userId }, + ); + if (error || !Array.isArray(data) || data.length === 0) return null; + const row = data[0] as { id: string; email: string }; + return { id: row.id, email: row.email }; +} + +// ── Legacy / unused helpers ─────────────────────────────────────────────────── + +/** + * Extract and verify the Supabase JWT from a Next.js-style Request. + * + * @deprecated This function is unused in the backend (per CLAUDE.md "Dead + * Code"). It is kept here to avoid breaking any external consumer + * that may have referenced it. Do not call from new code. */ export async function getUserIdFromRequest(req: Request): Promise<string> { const auth = req.headers.get("authorization") ?? ""; @@ -22,20 +302,9 @@ export async function getUserIdFromRequest(req: Request): Promise<string> { }); } const token = auth.slice(7).trim(); - - const supabaseUrl = process.env.SUPABASE_URL || ""; - const serviceKey = process.env.SUPABASE_SECRET_KEY || ""; - - if (!supabaseUrl || !serviceKey) { - throw new Response("Server auth is not configured", { status: 500 }); - } - - const admin = createClient(supabaseUrl, serviceKey, { - auth: { persistSession: false }, - }); - const { data } = await admin.auth.getUser(token); - if (!data.user) { + const user = await verifyToken(token); + if (!user) { throw new Response("Invalid or expired token", { status: 401 }); } - return data.user.id; + return user.id; } diff --git a/backend/src/lib/upload.ts b/backend/src/lib/upload.ts index caa44dbf6..c13425222 100644 --- a/backend/src/lib/upload.ts +++ b/backend/src/lib/upload.ts @@ -1,13 +1,18 @@ import type { RequestHandler } from "express"; import multer from "multer"; +import { tmpdir } from "os"; +import { randomUUID } from "crypto"; +import { unlink } from "fs/promises"; +import { basename } from "path"; -export const MAX_UPLOAD_SIZE_BYTES = 100 * 1024 * 1024; -export const MAX_UPLOAD_SIZE_MB = Math.round( - MAX_UPLOAD_SIZE_BYTES / (1024 * 1024), -); +export const MAX_UPLOAD_SIZE_MB = 100; +export const MAX_UPLOAD_SIZE_BYTES = MAX_UPLOAD_SIZE_MB * 1024 * 1024; -const memoryUpload = multer({ - storage: multer.memoryStorage(), +const diskUpload = multer({ + storage: multer.diskStorage({ + destination: (_req, _file, cb) => cb(null, tmpdir()), + filename: (_req, _file, cb) => cb(null, randomUUID()), + }), limits: { fileSize: MAX_UPLOAD_SIZE_BYTES, files: 1, @@ -16,7 +21,7 @@ const memoryUpload = multer({ export function singleFileUpload(fieldName: string): RequestHandler { return (req, res, next) => { - memoryUpload.single(fieldName)(req, res, (err) => { + diskUpload.single(fieldName)(req, res, (err) => { if (!err) return next(); if (err instanceof multer.MulterError) { @@ -34,3 +39,19 @@ export function singleFileUpload(fieldName: string): RequestHandler { }); }; } + +export async function cleanupTempFile(filePath: string): Promise<void> { + await unlink(filePath).catch(() => {}); +} + +export function sanitizeFilename(raw: string): string { + // Step 1: basename strips any directory component (path traversal protection) + // "../../etc/passwd" -> "passwd"; "foo/bar.docx" -> "bar.docx" + let safe = basename(raw); + // Step 2: strip characters dangerous in HTML or on filesystems + // Keep: alphanumeric, space, hyphen, underscore, dot, parens, brackets + safe = safe.replace(/[^a-zA-Z0-9 ._\-()[\]]/g, "_"); + // Step 3: trim leading dots (hidden files on Unix) + safe = safe.replace(/^\.+/, ""); + return safe || "upload"; +} diff --git a/backend/src/lib/userApiKeys.ts b/backend/src/lib/userApiKeys.ts deleted file mode 100644 index 4355c939e..000000000 --- a/backend/src/lib/userApiKeys.ts +++ /dev/null @@ -1,186 +0,0 @@ -import crypto from "crypto"; -import { createServerSupabase } from "./supabase"; -import type { UserApiKeys } from "./llm"; - -type Db = ReturnType<typeof createServerSupabase>; -export type ApiKeyProvider = "claude" | "gemini" | "openai"; -export type ApiKeySource = "user" | "env" | null; -export type ApiKeyStatus = Record<ApiKeyProvider, boolean> & { - sources: Record<ApiKeyProvider, ApiKeySource>; -}; - -type EncryptedKeyRow = { - provider: ApiKeyProvider; - encrypted_key: string; - iv: string; - auth_tag: string; -}; - -const PROVIDERS: ApiKeyProvider[] = ["claude", "gemini", "openai"]; - -function envApiKey(provider: ApiKeyProvider): string | null { - if (provider === "claude") { - return ( - process.env.ANTHROPIC_API_KEY?.trim() || - process.env.CLAUDE_API_KEY?.trim() || - null - ); - } - if (provider === "openai") { - return process.env.OPENAI_API_KEY?.trim() || null; - } - return process.env.GEMINI_API_KEY?.trim() || null; -} - -export function hasEnvApiKey(provider: ApiKeyProvider): boolean { - return !!envApiKey(provider); -} - -function encryptionKey(): Buffer { - const secret = - process.env.USER_API_KEYS_ENCRYPTION_SECRET || - process.env.API_KEYS_ENCRYPTION_SECRET || - process.env.SUPABASE_SECRET_KEY; - if (!secret) { - throw new Error("API key encryption secret is not configured"); - } - return crypto.createHash("sha256").update(secret).digest(); -} - -function encrypt(value: string): Omit<EncryptedKeyRow, "provider"> { - const iv = crypto.randomBytes(12); - const cipher = crypto.createCipheriv("aes-256-gcm", encryptionKey(), iv); - const encrypted = Buffer.concat([ - cipher.update(value, "utf8"), - cipher.final(), - ]); - return { - encrypted_key: encrypted.toString("base64"), - iv: iv.toString("base64"), - auth_tag: cipher.getAuthTag().toString("base64"), - }; -} - -function decrypt(row: EncryptedKeyRow): string | null { - try { - const decipher = crypto.createDecipheriv( - "aes-256-gcm", - encryptionKey(), - Buffer.from(row.iv, "base64"), - ); - decipher.setAuthTag(Buffer.from(row.auth_tag, "base64")); - const decrypted = Buffer.concat([ - decipher.update(Buffer.from(row.encrypted_key, "base64")), - decipher.final(), - ]); - return decrypted.toString("utf8"); - } catch (err) { - console.error("[user-api-keys] failed to decrypt stored key", { - provider: row.provider, - error: err instanceof Error ? err.message : String(err), - }); - return null; - } -} - -function isProvider(value: string): value is ApiKeyProvider { - return (PROVIDERS as string[]).includes(value); -} - -export function normalizeApiKeyProvider(value: string): ApiKeyProvider | null { - return isProvider(value) ? value : null; -} - -export async function getUserApiKeyStatus( - userId: string, - db: Db = createServerSupabase(), -): Promise<ApiKeyStatus> { - const status: ApiKeyStatus = { - claude: false, - gemini: false, - openai: false, - sources: { - claude: null, - gemini: null, - openai: null, - }, - }; - - for (const provider of PROVIDERS) { - if (hasEnvApiKey(provider)) { - status[provider] = true; - status.sources[provider] = "env"; - } - } - - const { data, error } = await db - .from("user_api_keys") - .select("provider") - .eq("user_id", userId); - if (error) throw error; - - for (const row of data ?? []) { - const provider = normalizeApiKeyProvider(String(row.provider)); - if (provider && !status[provider]) { - status[provider] = true; - status.sources[provider] = "user"; - } - } - - return status; -} - -export async function getUserApiKeys( - userId: string, - db: Db = createServerSupabase(), -): Promise<UserApiKeys> { - const apiKeys: UserApiKeys = { - claude: envApiKey("claude"), - gemini: envApiKey("gemini"), - openai: envApiKey("openai"), - }; - - const { data, error } = await db - .from("user_api_keys") - .select("provider, encrypted_key, iv, auth_tag") - .eq("user_id", userId); - if (error) throw error; - - for (const row of (data ?? []) as EncryptedKeyRow[]) { - const provider = normalizeApiKeyProvider(row.provider); - if (!provider) continue; - if (apiKeys[provider]?.trim()) continue; - apiKeys[provider] = decrypt(row); - } - - return apiKeys; -} - -export async function saveUserApiKey( - userId: string, - provider: ApiKeyProvider, - value: string | null, - db: Db = createServerSupabase(), -): Promise<void> { - const normalized = value?.trim() || null; - if (!normalized) { - const { error } = await db - .from("user_api_keys") - .delete() - .eq("user_id", userId) - .eq("provider", provider); - if (error) throw error; - return; - } - - const { error } = await db.from("user_api_keys").upsert( - { - user_id: userId, - provider, - ...encrypt(normalized), - updated_at: new Date().toISOString(), - }, - { onConflict: "user_id,provider" }, - ); - if (error) throw error; -} diff --git a/backend/src/lib/userSettings.ts b/backend/src/lib/userSettings.ts index bfbeb0fd5..e98251d0f 100644 --- a/backend/src/lib/userSettings.ts +++ b/backend/src/lib/userSettings.ts @@ -3,10 +3,10 @@ import { resolveModel, DEFAULT_TITLE_MODEL, DEFAULT_TABULAR_MODEL, - OPENAI_LOW_MODELS, type UserApiKeys, } from "./llm"; -import { getUserApiKeys as getStoredUserApiKeys } from "./userApiKeys"; +import { decryptApiKey } from "./crypto"; +import { logger } from "./logger"; export type UserModelSettings = { title_model: string; @@ -16,30 +16,123 @@ export type UserModelSettings = { // Title generation is a lightweight task — always routed to the cheapest model // of whichever provider the user has keys for: Gemini Flash Lite if Gemini is -// available, otherwise OpenAI nano, otherwise Claude Haiku. With no user keys -// set, defaults to Gemini (the dev-mode env fallback). +// available, otherwise Claude Haiku. With no user keys set, defaults to Gemini +// (the dev-mode env fallback). function resolveTitleModel(apiKeys: UserApiKeys): string { if (apiKeys.gemini?.trim()) return DEFAULT_TITLE_MODEL; - if (apiKeys.openai?.trim()) return OPENAI_LOW_MODELS[0]; if (apiKeys.claude?.trim()) return "claude-haiku-4-5"; return DEFAULT_TITLE_MODEL; } +/** + * Decodes three base64 bytea columns from PostgREST into Buffers and decrypts + * the AES-GCM ciphertext. Returns null when: + * - any column value is missing (user hasn't set a key) + * - decryption fails (tampered ciphertext / wrong master key) + * + * Callers that need to distinguish "no key" from "decrypt failure" must check + * whether ciphertextB64 was non-null before calling. + */ +function decryptColumn( + ciphertextB64: string | null | undefined, + ivB64: string | null | undefined, + authTagB64: string | null | undefined, +): string | null { + if (!ciphertextB64 || !ivB64 || !authTagB64) return null; + return decryptApiKey({ + ciphertext: decodeBytea(ciphertextB64), + iv: decodeBytea(ivB64), + authTag: decodeBytea(authTagB64), + }); +} + +function decodeBytea(value: string): Buffer { + return value.startsWith("\\x") + ? Buffer.from(value.slice(2), "hex") + : Buffer.from(value, "base64"); +} + +type EncryptedKeyRow = { + tabular_model?: string | null; + claude_api_key_ciphertext?: string | null; + claude_api_key_iv?: string | null; + claude_api_key_auth_tag?: string | null; + gemini_api_key_ciphertext?: string | null; + gemini_api_key_iv?: string | null; + gemini_api_key_auth_tag?: string | null; +}; + export async function getUserModelSettings( userId: string, db?: ReturnType<typeof createServerSupabase>, + ctx?: { route?: string; requestId?: string | number | object }, ): Promise<UserModelSettings> { const client = db ?? createServerSupabase(); - const { data } = await client + const { data: rawData } = await client .from("user_profiles") - .select("tabular_model") + .select( + "tabular_model, " + + "claude_api_key_ciphertext, claude_api_key_iv, claude_api_key_auth_tag, " + + "gemini_api_key_ciphertext, gemini_api_key_iv, gemini_api_key_auth_tag", + ) .eq("user_id", userId) .single(); - const api_keys = await getStoredUserApiKeys(userId, client); + const data = rawData as unknown as EncryptedKeyRow | null; + + const claude = data?.claude_api_key_ciphertext + ? decryptColumn( + data.claude_api_key_ciphertext, + data.claude_api_key_iv, + data.claude_api_key_auth_tag, + ) + : null; + + if (data?.claude_api_key_ciphertext && claude === null) { + logger.error( + { user_id: userId, provider: "claude" }, + "[userSettings] decrypt failed — possible master key mismatch", + ); + } + + const gemini = data?.gemini_api_key_ciphertext + ? decryptColumn( + data.gemini_api_key_ciphertext, + data.gemini_api_key_iv, + data.gemini_api_key_auth_tag, + ) + : null; + + if (data?.gemini_api_key_ciphertext && gemini === null) { + logger.error( + { user_id: userId, provider: "gemini" }, + "[userSettings] decrypt failed — possible master key mismatch", + ); + } + + if (claude) { + logger.info({ + event: "api_key_read", + user_id: userId, + provider: "claude", + route: ctx?.route ?? "unknown", + request_id: ctx?.requestId, + }, "[userSettings] api_key_read"); + } + if (gemini) { + logger.info({ + event: "api_key_read", + user_id: userId, + provider: "gemini", + route: ctx?.route ?? "unknown", + request_id: ctx?.requestId, + }, "[userSettings] api_key_read"); + } + + const api_keys: UserApiKeys = { claude, gemini }; return { title_model: resolveTitleModel(api_keys), - tabular_model: resolveModel(data?.tabular_model, DEFAULT_TABULAR_MODEL), + tabular_model: resolveModel(data?.tabular_model ?? null, DEFAULT_TABULAR_MODEL), api_keys, }; } @@ -47,7 +140,67 @@ export async function getUserModelSettings( export async function getUserApiKeys( userId: string, db?: ReturnType<typeof createServerSupabase>, + ctx?: { route?: string; requestId?: string | number | object }, ): Promise<UserApiKeys> { const client = db ?? createServerSupabase(); - return getStoredUserApiKeys(userId, client); + const { data: rawData } = await client + .from("user_profiles") + .select( + "claude_api_key_ciphertext, claude_api_key_iv, claude_api_key_auth_tag, " + + "gemini_api_key_ciphertext, gemini_api_key_iv, gemini_api_key_auth_tag", + ) + .eq("user_id", userId) + .single(); + const data = rawData as unknown as EncryptedKeyRow | null; + + const claude = data?.claude_api_key_ciphertext + ? decryptColumn( + data.claude_api_key_ciphertext, + data.claude_api_key_iv, + data.claude_api_key_auth_tag, + ) + : null; + + if (data?.claude_api_key_ciphertext && claude === null) { + logger.error( + { user_id: userId, provider: "claude" }, + "[userSettings] decrypt failed — possible master key mismatch", + ); + } + + const gemini = data?.gemini_api_key_ciphertext + ? decryptColumn( + data.gemini_api_key_ciphertext, + data.gemini_api_key_iv, + data.gemini_api_key_auth_tag, + ) + : null; + + if (data?.gemini_api_key_ciphertext && gemini === null) { + logger.error( + { user_id: userId, provider: "gemini" }, + "[userSettings] decrypt failed — possible master key mismatch", + ); + } + + if (claude) { + logger.info({ + event: "api_key_read", + user_id: userId, + provider: "claude", + route: ctx?.route ?? "unknown", + request_id: ctx?.requestId, + }, "[userSettings] api_key_read"); + } + if (gemini) { + logger.info({ + event: "api_key_read", + user_id: userId, + provider: "gemini", + route: ctx?.route ?? "unknown", + request_id: ctx?.requestId, + }, "[userSettings] api_key_read"); + } + + return { claude, gemini }; } diff --git a/backend/src/lib/validate.ts b/backend/src/lib/validate.ts new file mode 100644 index 000000000..c74bf6797 --- /dev/null +++ b/backend/src/lib/validate.ts @@ -0,0 +1,27 @@ +import { ZodSchema } from "zod"; +import type { Request, Response } from "express"; + +/** + * Validates req.body against the given zod schema. + * On failure: sends 400 { detail, fields } and returns null. + * On success: returns the validated, stripped data. + * + * Usage: + * const body = parseBody(MySchema, req, res); + * if (!body) return; // 400 already sent + */ +export function parseBody<T>( + schema: ZodSchema<T>, + req: Request, + res: Response, +): T | null { + const result = schema.safeParse(req.body); + if (!result.success) { + const fields = Object.fromEntries( + result.error.issues.map((i) => [i.path.join("."), i.message]), + ); + res.status(400).json({ detail: "Validation failed", fields }); + return null; + } + return result.data; +} diff --git a/backend/src/middleware/auth.ts b/backend/src/middleware/auth.ts index f30fd136f..b708dd841 100644 --- a/backend/src/middleware/auth.ts +++ b/backend/src/middleware/auth.ts @@ -1,5 +1,8 @@ import { Request, Response, NextFunction } from "express"; -import { createClient } from "@supabase/supabase-js"; +import { verifyToken } from "../lib/supabase"; +import { createServerSupabase } from "../lib/supabase"; +import { logger } from "../lib/logger"; +import { DELETE_GRACE_DAYS } from "../lib/accountDeletion"; export async function requireAuth( req: Request, @@ -13,25 +16,61 @@ export async function requireAuth( } const token = auth.slice(7).trim(); - const supabaseUrl = process.env.SUPABASE_URL ?? ""; - const serviceKey = process.env.SUPABASE_SECRET_KEY ?? ""; - - if (!supabaseUrl || !serviceKey) { - res.status(500).json({ detail: "Server auth is not configured" }); + let user; + try { + user = await verifyToken(token); + } catch (err) { + logger.error({ err }, "[auth] verifyToken failed"); + res.status(500).json({ detail: "Auth check failed" }); return; } - const admin = createClient(supabaseUrl, serviceKey, { - auth: { persistSession: false }, - }); - const { data } = await admin.auth.getUser(token); - if (!data.user) { + if (!user) { res.status(401).json({ detail: "Invalid or expired token" }); return; } - res.locals.userId = data.user.id; - res.locals.userEmail = data.user.email?.toLowerCase() ?? ""; + if (!user.email) { + res.status(401).json({ detail: "Account email not set; contact your operator" }); + return; + } + + // Soft-delete gate (CLEAN-44): reject users with deleted_at IS NOT NULL. + // Fresh SELECT on every authenticated request — locked v1 perf trade per + // CONTEXT.md D-04 + RESEARCH Open Q1 RESOLVED. The partial index + // idx_user_profiles_deleted_at (Plan 03) keeps this O(log N) of deleted rows. + // M3 may extend Phase 4 userAuthCache to include deleted_at; not in scope here. + const db = createServerSupabase(); + const { data: profile, error: profileError } = await db + .from("user_profiles") + .select("deleted_at") + .eq("user_id", user.id) + .single(); + + if (profileError && profileError.code !== "PGRST116") { + // PGRST116 = "0 rows" — user has no profile yet (signup not finished); allow through + logger.error({ err: profileError, userId: user.id }, "[auth] deleted_at lookup failed"); + res.status(500).json({ detail: "Auth check failed" }); + return; + } + + if (profile?.deleted_at) { + const deletedAt = new Date(profile.deleted_at as string); + const scheduledHardDeleteAt = new Date( + deletedAt.getTime() + DELETE_GRACE_DAYS * 86_400_000, + ); + res.status(403).json({ + detail: "Account scheduled for deletion", + deleted: true, + deleted_at: profile.deleted_at, + scheduled_hard_delete_at: scheduledHardDeleteAt.toISOString(), + restore_path: "/user/account/restore", + }); + return; + } + + res.locals.userId = user.id; + res.locals.userEmail = user.email; res.locals.token = token; next(); } diff --git a/backend/src/routes/chat.ts b/backend/src/routes/chat.ts index fe272c671..386642ec6 100644 --- a/backend/src/routes/chat.ts +++ b/backend/src/routes/chat.ts @@ -1,6 +1,9 @@ import { Router } from "express"; +import { z } from "zod"; import { requireAuth } from "../middleware/auth"; +import { llmRateLimiter } from "../lib/rateLimiter"; import { createServerSupabase } from "../lib/supabase"; +import { logger } from "../lib/logger"; import { buildDocContext, buildMessages, @@ -13,124 +16,35 @@ import { import { completeText } from "../lib/llm"; import { getUserApiKeys, getUserModelSettings } from "../lib/userSettings"; import { checkProjectAccess } from "../lib/access"; +import { parseBody } from "../lib/validate"; -export const chatRouter = Router(); - -type Db = ReturnType<typeof createServerSupabase>; -const isDev = process.env.NODE_ENV !== "production"; -const devLog = (...args: Parameters<typeof console.log>) => { - if (isDev) console.log(...args); -}; - -type AccessibleChat = { - id: string; - title: string | null; - user_id: string; - project_id: string | null; -} & Record<string, unknown>; - -function parseOptionalProjectId(value: unknown): - | { ok: true; provided: boolean; projectId: string | null } - | { ok: false; detail: string } { - if (value === undefined) - return { ok: true, provided: false, projectId: null }; - if (value === null) return { ok: true, provided: true, projectId: null }; - if (typeof value !== "string" || !value.trim()) { - return { - ok: false, - detail: "project_id must be a non-empty string or null", - }; - } - return { ok: true, provided: true, projectId: value.trim() }; -} - -function parseOptionalChatId(value: unknown): - | { ok: true; chatId: string | null } - | { ok: false; detail: string } { - if (value === undefined || value === null) return { ok: true, chatId: null }; - if (typeof value !== "string" || !value.trim()) { - return { ok: false, detail: "chat_id must be a non-empty string" }; - } - return { ok: true, chatId: value.trim() }; -} - -function parseChatMessages(value: unknown): - | { ok: true; messages: ChatMessage[] } - | { ok: false; detail: string } { - if (!Array.isArray(value) || value.length === 0) { - return { ok: false, detail: "messages must be a non-empty array" }; - } - - for (const message of value) { - if (!message || typeof message !== "object" || Array.isArray(message)) { - return { ok: false, detail: "messages must contain objects" }; - } - const row = message as Record<string, unknown>; - if (typeof row.role !== "string") { - return { ok: false, detail: "message.role must be a string" }; - } - if (row.content !== null && typeof row.content !== "string") { - return { - ok: false, - detail: "message.content must be a string or null", - }; - } - } - - return { ok: true, messages: value as ChatMessage[] }; -} - -function parseOptionalModel(value: unknown): - | { ok: true; model: string | undefined } - | { ok: false; detail: string } { - if (value === undefined) return { ok: true, model: undefined }; - if (typeof value !== "string" || !value.trim()) { - return { ok: false, detail: "model must be a non-empty string" }; - } - return { ok: true, model: value.trim() }; -} - -async function validateAccessibleProjectId( - projectId: string | null, - userId: string, - userEmail: string | null | undefined, - db: Db, -): Promise<{ ok: true } | { ok: false; status: number; detail: string }> { - if (!projectId) return { ok: true }; - const access = await checkProjectAccess(projectId, userId, userEmail, db); - if (!access.ok) - return { ok: false, status: 404, detail: "Project not found" }; - return { ok: true }; -} +const CreateChatSchema = z.object({ + project_id: z.string().uuid().optional().nullable(), +}); -async function getAccessibleChat( - chatId: string, - userId: string, - userEmail: string | null | undefined, - db: Db, -): Promise<AccessibleChat | null> { - const { data: chat, error } = await db - .from("chats") - .select("*") - .eq("id", chatId) - .maybeSingle(); - if (error || !chat) return null; +const PatchChatSchema = z.object({ + title: z.string().min(1), +}); - const row = chat as AccessibleChat; - if (row.user_id === userId) return row; +const GenerateTitleSchema = z.object({ + message: z.string().min(1), +}); - if (row.project_id) { - const access = await checkProjectAccess( - row.project_id, - userId, - userEmail, - db, - ); - if (access.ok) return row; - } +const ChatStreamSchema = z.object({ + messages: z.array( + z.object({ + role: z.enum(["user", "assistant"]), + content: z.string(), + files: z.array(z.unknown()).optional().nullable(), + workflow: z.unknown().optional().nullable(), + }), + ).min(1), + chat_id: z.string().uuid().optional(), + project_id: z.string().uuid().optional().nullable(), + model: z.string().optional(), +}); - return null; -} +export const chatRouter = Router(); // GET /chat // Visible chats = the user's own chats + every chat under a project the @@ -142,53 +56,62 @@ chatRouter.get("/", requireAuth, async (req, res) => { const userId = res.locals.userId as string; const db = createServerSupabase(); - const { data: ownProjects, error: projErr } = await db - .from("projects") - .select("id") - .eq("user_id", userId); - if (projErr) return void res.status(500).json({ detail: projErr.message }); - const ownProjectIds = ((ownProjects ?? []) as { id: string }[]).map( - (p) => p.id, - ); + const [ownProjectsResult, ownChatsResult] = await Promise.all([ + db.from("projects").select("id").eq("user_id", userId), + db.from("chats").select("*").eq("user_id", userId), + ]); + if (ownProjectsResult.error) + return void res + .status(500) + .json({ detail: ownProjectsResult.error.message }); + if (ownChatsResult.error) + return void res + .status(500) + .json({ detail: ownChatsResult.error.message }); - const filter = - ownProjectIds.length > 0 - ? `user_id.eq.${userId},project_id.in.(${ownProjectIds.join(",")})` - : `user_id.eq.${userId}`; + const ownProjectIds = ( + (ownProjectsResult.data ?? []) as { id: string }[] + ).map((p) => p.id); - const { data, error } = await db - .from("chats") - .select("*") - .or(filter) - .order("created_at", { ascending: false }); - if (error) return void res.status(500).json({ detail: error.message }); - res.json(data ?? []); + let projectChats: Record<string, unknown>[] = []; + if (ownProjectIds.length > 0) { + const { data, error } = await db + .from("chats") + .select("*") + .in("project_id", ownProjectIds); + if (error) + return void res.status(500).json({ detail: error.message }); + projectChats = (data ?? []) as Record<string, unknown>[]; + } + + // Merge + dedupe on id, then sort by created_at desc to preserve + // prior server-side ordering semantics (RESEARCH.md Pitfall 4). + const byId = new Map<string, Record<string, unknown>>(); + for (const c of (ownChatsResult.data ?? []) as Record< + string, + unknown + >[]) { + byId.set(c.id as string, c); + } + for (const c of projectChats) byId.set(c.id as string, c); + const merged = [...byId.values()].sort((a, b) => { + const ta = String(a.created_at ?? ""); + const tb = String(b.created_at ?? ""); + return tb.localeCompare(ta); + }); + res.json(merged); }); // POST /chat/create chatRouter.post("/create", requireAuth, async (req, res) => { const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const parsedProjectId = parseOptionalProjectId(req.body?.project_id); - if (!parsedProjectId.ok) { - return void res.status(400).json({ detail: parsedProjectId.detail }); - } - const projectId = parsedProjectId.projectId; + const body = parseBody(CreateChatSchema, req, res); + if (!body) return; + const projectId: string | null = body.project_id ?? null; const db = createServerSupabase(); - const projectAccess = await validateAccessibleProjectId( - projectId, - userId, - userEmail, - db, - ); - if (!projectAccess.ok) - return void res - .status(projectAccess.status) - .json({ detail: projectAccess.detail }); - const { data, error } = await db .from("chats") - .insert({ user_id: userId, project_id: projectId ?? null }) + .insert({ user_id: userId, project_id: projectId ?? undefined }) .select("id") .single(); @@ -203,18 +126,67 @@ chatRouter.get("/:chatId", requireAuth, async (req, res) => { const { chatId } = req.params; const db = createServerSupabase(); - const chat = await getAccessibleChat(chatId, userId, userEmail, db); - if (!chat) + const { data: chat, error } = await db + .from("chats") + .select("*") + .eq("id", chatId) + .single(); + if (error || !chat) + return void res.status(404).json({ detail: "Chat not found" }); + // Owner of the chat OR a member of the chat's project can view it. + let canView = chat.user_id === userId; + if (!canView && chat.project_id) { + const access = await checkProjectAccess( + chat.project_id, + userId, + userEmail, + db, + ); + canView = access.ok; + } + if (!canView) return void res.status(404).json({ detail: "Chat not found" }); - const { data: messages } = await db + // CLEAN-27: paginate to limit+before; default 50 most-recent. Cap 200. + // Cursor is exclusive (lt) to avoid duplicate at page boundary. + const limitRaw = + typeof req.query.limit === "string" + ? parseInt(req.query.limit, 10) + : NaN; + const limit = Number.isFinite(limitRaw) + ? Math.min(Math.max(limitRaw, 1), 200) + : 50; + + // Validate `before` cursor BEFORE building the query. + // An unparseable cursor must 400 — silent fallback to no-cursor would + // mask client bugs and return a different page than the caller asked for. + const beforeRaw = req.query.before; + if (beforeRaw !== undefined) { + const ts = + typeof beforeRaw === "string" ? Date.parse(beforeRaw) : NaN; + if (Number.isNaN(ts)) { + return void res.status(400).json({ + detail: "invalid 'before' cursor — expected ISO 8601", + }); + } + } + const before = + typeof beforeRaw === "string" ? beforeRaw : null; + + let mq = db .from("chat_messages") .select("*") - .eq("chat_id", chatId) - .order("created_at", { ascending: true }); - - const hydrated = await hydrateEditStatuses(messages ?? [], db); - res.json({ chat, messages: hydrated }); + .eq("chat_id", chatId); + if (before) mq = mq.lt("created_at", before); + const { data: messagesDesc } = await mq + .order("created_at", { ascending: false }) + .limit(limit + 1); // fetch one extra to compute has_more + const hasMore = (messagesDesc ?? []).length > limit; + const pageDesc = (messagesDesc ?? []).slice(0, limit); + const messages = [...pageDesc].reverse(); // back to ASC for client rendering + + const hydrated = await hydrateEditStatuses(messages, db); + res.json({ chat, messages: hydrated, has_more: hasMore }); }); // Stored message annotations/events capture the `status` at the time the @@ -222,7 +194,7 @@ chatRouter.get("/:chatId", requireAuth, async (req, res) => { // or rejects, `document_edits.status` is updated but the stored message // annotation is not. On chat load we merge the current DB status in so // EditCards render with the real state. -async function hydrateEditStatuses( +export async function hydrateEditStatuses( messages: Record<string, unknown>[], db: ReturnType<typeof createServerSupabase>, ): Promise<Record<string, unknown>[]> { @@ -338,9 +310,9 @@ async function hydrateEditStatuses( chatRouter.patch("/:chatId", requireAuth, async (req, res) => { const userId = res.locals.userId as string; const { chatId } = req.params; - const title = (req.body.title ?? "").trim(); - if (!title) - return void res.status(400).json({ detail: "title is required" }); + const body = parseBody(PatchChatSchema, req, res); + if (!body) return; + const title = body.title.trim(); const db = createServerSupabase(); const { data, error } = await db @@ -361,13 +333,15 @@ chatRouter.delete("/:chatId", requireAuth, async (req, res) => { const userId = res.locals.userId as string; const { chatId } = req.params; const db = createServerSupabase(); - const { error } = await db + const { data, error } = await db .from("chats") .delete() .eq("id", chatId) - .eq("user_id", userId); - + .eq("user_id", userId) + .select("id"); if (error) return void res.status(500).json({ detail: error.message }); + if (!data || data.length === 0) + return void res.status(404).json({ detail: "Chat not found" }); res.status(204).send(); }); @@ -376,20 +350,37 @@ chatRouter.post("/:chatId/generate-title", requireAuth, async (req, res) => { const userId = res.locals.userId as string; const userEmail = res.locals.userEmail as string | undefined; const { chatId } = req.params; - const message = - typeof req.body?.message === "string" ? req.body.message.trim() : ""; - if (!message) - return void res.status(400).json({ detail: "message is required" }); + const body = parseBody(GenerateTitleSchema, req, res); + if (!body) return; + const message = body.message.trim(); const db = createServerSupabase(); - const chat = await getAccessibleChat(chatId, userId, userEmail, db); - if (!chat) + const { data: chat, error } = await db + .from("chats") + .select("id, user_id, project_id") + .eq("id", chatId) + .single(); + + if (error || !chat) + return void res.status(404).json({ detail: "Chat not found" }); + let canTitle = chat.user_id === userId; + if (!canTitle && chat.project_id) { + const access = await checkProjectAccess( + chat.project_id, + userId, + userEmail, + db, + ); + canTitle = access.ok; + } + if (!canTitle) return void res.status(404).json({ detail: "Chat not found" }); try { const { title_model, api_keys } = await getUserModelSettings( userId, db, + { route: req.path, requestId: req.id }, ); const titleText = await completeText({ model: title_model, @@ -399,6 +390,8 @@ chatRouter.post("/:chatId/generate-title", requireAuth, async (req, res) => { }); const title = titleText.trim() || message.slice(0, 60); + // CLEAN-24: access already validated via checkProjectAccess above (~lines 301-311); + // no user_id predicate here so shared-project members can persist titles. await db .from("chats") .update({ title }) @@ -406,93 +399,79 @@ chatRouter.post("/:chatId/generate-title", requireAuth, async (req, res) => { res.json({ title }); } catch (err) { - console.error("[generate-title]", err); + logger.error({ err }, "[generate-title] error"); res.status(500).json({ detail: "Failed to generate title" }); } }); // POST /chat — streaming -chatRouter.post("/", requireAuth, async (req, res) => { +chatRouter.post("/", requireAuth, llmRateLimiter, async (req, res) => { const userId = res.locals.userId as string; - const body = - req.body && typeof req.body === "object" && !Array.isArray(req.body) - ? (req.body as Record<string, unknown>) - : {}; - const parsedMessages = parseChatMessages(body.messages); - if (!parsedMessages.ok) { - return void res.status(400).json({ detail: parsedMessages.detail }); - } - const parsedChatId = parseOptionalChatId(body.chat_id); - if (!parsedChatId.ok) { - return void res.status(400).json({ detail: parsedChatId.detail }); - } - const parsedProjectId = parseOptionalProjectId(body.project_id); - if (!parsedProjectId.ok) { - return void res.status(400).json({ detail: parsedProjectId.detail }); - } - const parsedModel = parseOptionalModel(body.model); - if (!parsedModel.ok) { - return void res.status(400).json({ detail: parsedModel.detail }); - } - - const messages = parsedMessages.messages; - const chat_id = parsedChatId.chatId; - const project_id = parsedProjectId.projectId; - const model = parsedModel.model; + const body = parseBody(ChatStreamSchema, req, res); + if (!body) return; + const { messages, chat_id, project_id, model } = body as unknown as { + messages: ChatMessage[]; + chat_id?: string; + project_id?: string | null; + model?: string; + }; - devLog("[chat/stream] incoming request", { + logger.info({ userId, - chat_id, - project_id, + chatId: chat_id, + projectId: project_id, model, messageCount: messages?.length, - }); + }, "[chat/stream] incoming request"); const userEmail = res.locals.userEmail as string | undefined; const db = createServerSupabase(); let chatId = chat_id ?? null; let chatTitle: string | null = null; - let resolvedProjectId: string | null = parsedProjectId.projectId; if (chatId) { - const existing = await getAccessibleChat(chatId, userId, userEmail, db); - if (!existing) - return void res.status(404).json({ detail: "Chat not found" }); - - const existingProjectId = existing.project_id ?? null; - if ( - parsedProjectId.provided && - parsedProjectId.projectId !== existingProjectId - ) { - return void res - .status(400) - .json({ detail: "project_id does not match chat" }); + // Either chat owner OR a member of the chat's project can post. + const { data: existing } = await db + .from("chats") + .select("id, title, user_id, project_id") + .eq("id", chatId) + .single(); + let canUse = !!existing && existing.user_id === userId; + if (!canUse && existing?.project_id) { + const access = await checkProjectAccess( + existing.project_id, + userId, + userEmail, + db, + ); + canUse = access.ok; } - resolvedProjectId = existingProjectId; - chatTitle = existing.title; + if (!canUse || !existing) chatId = null; + else chatTitle = existing.title; } if (!chatId) { // If creating a chat tied to a project, the user must have access // to the project (own or shared). - const projectAccess = await validateAccessibleProjectId( - resolvedProjectId, - userId, - userEmail, - db, - ); - if (!projectAccess.ok) - return void res - .status(projectAccess.status) - .json({ detail: projectAccess.detail }); - + if (project_id) { + const access = await checkProjectAccess( + project_id, + userId, + userEmail, + db, + ); + if (!access.ok) + return void res + .status(404) + .json({ detail: "Project not found" }); + } const { data: newChat, error } = await db .from("chats") - .insert({ user_id: userId, project_id: resolvedProjectId }) + .insert({ user_id: userId, project_id: project_id ?? null }) .select("id, title") .single(); if (error || !newChat) { - console.error("[chat/stream] failed to create chat", error); + logger.error({ err: error }, "[chat/stream] failed to create chat"); return void res .status(500) .json({ detail: "Failed to create chat" }); @@ -501,7 +480,7 @@ chatRouter.post("/", requireAuth, async (req, res) => { chatTitle = newChat.title; } - devLog("[chat/stream] resolved chatId", chatId); + logger.info({ chatId }, "[chat/stream] resolved chatId"); const lastUser = [...messages].reverse().find((m) => m.role === "user"); if (lastUser) { @@ -534,11 +513,11 @@ chatRouter.post("/", requireAuth, async (req, res) => { const workflowStore = await buildWorkflowStore(userId, userEmail, db); - devLog("[chat/stream] starting LLM stream", { + logger.info({ apiMessageCount: apiMessages.length, docCount: Object.keys(docIndex).length, workflowCount: Object.keys(workflowStore).length, - }); + }, "[chat/stream] starting LLM stream"); res.setHeader("Content-Type", "text/event-stream"); res.setHeader("Cache-Control", "no-cache"); @@ -548,7 +527,7 @@ chatRouter.post("/", requireAuth, async (req, res) => { const write = (line: string) => res.write(line); - const apiKeys = await getUserApiKeys(userId, db); + const apiKeys = await getUserApiKeys(userId, db, { route: req.path, requestId: req.id }); try { write(`data: ${JSON.stringify({ type: "chat_id", chatId })}\n\n`); @@ -563,13 +542,13 @@ chatRouter.post("/", requireAuth, async (req, res) => { workflowStore, model, apiKeys, - projectId: resolvedProjectId, + projectId: project_id ?? null, }); - devLog("[chat/stream] LLM stream finished", { + logger.info({ fullTextLen: fullText?.length ?? 0, eventCount: events?.length ?? 0, - }); + }, "[chat/stream] LLM stream finished"); const annotations = extractAnnotations(fullText, docIndex, events); await db.from("chat_messages").insert({ @@ -586,7 +565,7 @@ chatRouter.post("/", requireAuth, async (req, res) => { .eq("id", chatId); } } catch (err) { - console.error("[chat/stream] error:", err); + logger.error({ err }, "[chat/stream] error"); try { write( `data: ${JSON.stringify({ type: "error", message: "Stream error" })}\n\n`, diff --git a/backend/src/routes/documents.ts b/backend/src/routes/documents.ts index 32f4b881a..7e51a92e6 100644 --- a/backend/src/routes/documents.ts +++ b/backend/src/routes/documents.ts @@ -1,6 +1,11 @@ +import { randomUUID } from "crypto"; +import { readFile } from "fs/promises"; import { Router } from "express"; +import { z } from "zod"; import { requireAuth } from "../middleware/auth"; import { createServerSupabase } from "../lib/supabase"; +import { logger } from "../lib/logger"; +import { parseBody } from "../lib/validate"; import { buildContentDisposition, downloadFile, @@ -10,7 +15,6 @@ import { uploadFile, versionStorageKey, } from "../lib/storage"; -import { docxToPdf, convertedPdfKey } from "../lib/convert"; import { extractTrackedChangeIds, resolveTrackedChange, @@ -22,11 +26,144 @@ import { loadActiveVersion, } from "../lib/documentVersions"; import { ensureDocAccess } from "../lib/access"; -import { singleFileUpload } from "../lib/upload"; +import { + cleanupTempFile, + sanitizeFilename, + singleFileUpload, +} from "../lib/upload"; +import { + enqueueConversionFromBuffer, + enqueueConversionForVersion, +} from "../lib/pdfQueue"; +import { extractStructureTree } from "../lib/structureTree"; + +const DownloadZipSchema = z.object({ + document_ids: z.array(z.string().uuid()).min(1), +}); export const documentsRouter = Router(); const ALLOWED_TYPES = new Set(["pdf", "docx", "doc"]); +// --------------------------------------------------------------------------- +// CLEAN-08: version-number uniqueness — retry-on-23505 helper +// --------------------------------------------------------------------------- + +const UNIQUE_VIOLATION = "23505"; + +/** + * Insert a new document_version row, retrying once if Postgres returns a + * 23505 unique_violation (TOCTOU race where two concurrent uploads both + * computed the same MAX+1). + * + * On first 23505: re-fetches MAX from DB and retries with MAX+1. + * On any other error, or on a second 23505: surfaces the error unchanged. + */ +export async function insertVersionWithRetry( + db: ReturnType<typeof createServerSupabase>, + documentId: string, + payload: Record<string, unknown>, +): Promise<{ data: { id: string; version_number: number } | null; error: unknown }> { + const fetchMax = async (): Promise<number> => { + const { data: maxRow } = await db + .from("document_versions") + .select("version_number") + .eq("document_id", documentId) + .in("source", ["upload", "user_upload", "assistant_edit"]) + .order("version_number", { ascending: false, nullsFirst: false }) + .limit(1) + .maybeSingle(); + return ((maxRow?.version_number as number | null) ?? 1) + 1; + }; + + const firstNum = await fetchMax(); + let result = await db + .from("document_versions") + .insert({ ...payload, version_number: firstNum }) + .select("id, version_number") + .single(); + + if ((result.error as { code?: string } | null)?.code === UNIQUE_VIOLATION) { + // Race detected: re-fetch MAX and retry once + const retryNum = await fetchMax(); + result = await db + .from("document_versions") + .insert({ ...payload, version_number: retryNum }) + .select("id, version_number") + .single(); + } + + return result as { data: { id: string; version_number: number } | null; error: unknown }; +} + +// --------------------------------------------------------------------------- +// CLEAN-09 + CLEAN-34: edit-resolution compensating saga +// --------------------------------------------------------------------------- + +const DOCX_MIME = + "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; + +export interface EditResolutionSagaResult { + ok: boolean; + status: number; + detail?: string; +} + +/** + * Apply edit-resolution bytes to storage and record DB status as a + * compensating saga: + * + * 1. Download prior bytes (for rollback). + * 2. Upload new bytes. + * 3. Update DB status. + * 4. If DB fails: re-upload prior bytes (compensating rollback). + * + * Returns `{ ok: true }` on success or `{ ok: false, status, detail }` on + * any failure. Callers are responsible for returning the HTTP response. + */ +export async function applyEditResolutionSaga(deps: { + latestPath: string; + newBytes: ArrayBuffer; + status: "accepted" | "rejected"; + editId: string; + uploadFn: (key: string, body: ArrayBuffer, mime: string) => Promise<void>; + downloadFn: (key: string) => Promise<ArrayBuffer | null>; + dbUpdateFn: ( + status: "accepted" | "rejected", + editId: string, + ) => Promise<{ error: unknown }>; +}): Promise<EditResolutionSagaResult> { + const { latestPath, newBytes, status, editId, uploadFn, downloadFn, dbUpdateFn } = deps; + + // Step 1: snapshot prior bytes for rollback + const priorBytes = await downloadFn(latestPath); + + // Step 2: upload new bytes + try { + await uploadFn(latestPath, newBytes, DOCX_MIME); + } catch (uploadErr) { + logger.error({ err: uploadErr }, "[edit-resolution] storage upload failed"); + return { ok: false, status: 500, detail: "Storage write failed during edit resolution." }; + } + + // Step 3: update DB status + const { error: statusErr } = await dbUpdateFn(status, editId); + + if (statusErr) { + logger.error({ err: statusErr }, "[edit-resolution] DB status update failed after storage write — compensating rollback"); + // Step 4: compensating rollback — restore prior bytes + if (priorBytes) { + try { + await uploadFn(latestPath, priorBytes, DOCX_MIME); + } catch (rollbackErr) { + logger.error({ err: rollbackErr }, "[edit-resolution] CRITICAL: compensating rollback failed — storage may be inconsistent"); + } + } + return { ok: false, status: 500, detail: "Status update failed during edit resolution." }; + } + + return { ok: true, status: 200 }; +} + // GET /single-documents documentsRouter.get("/", requireAuth, async (req, res) => { const userId = res.locals.userId as string; @@ -156,10 +293,9 @@ documentsRouter.get("/:documentId/display", requireAuth, async (req, res) => { documentsRouter.post("/download-zip", requireAuth, async (req, res) => { const userId = res.locals.userId as string; const userEmail = res.locals.userEmail as string | undefined; - const { document_ids } = req.body as { document_ids?: string[] }; - - if (!Array.isArray(document_ids) || document_ids.length === 0) - return void res.status(400).json({ detail: "document_ids is required" }); + const zipBody = parseBody(DownloadZipSchema, req, res); + if (!zipBody) return; + const { document_ids } = zipBody; const db = createServerSupabase(); const { data: rawDocs, error } = await db @@ -180,17 +316,24 @@ documentsRouter.post("/download-zip", requireAuth, async (req, res) => { ), })), ); - const docs = accessChecks + const accessibleDocs = accessChecks .filter((x) => x.access.ok) .map((x) => x.doc as { id: string; filename: string }); - if (!docs || docs.length === 0) + + // CLEAN-25: collect IDs the requester named that we DROPPED so the client + // can surface a partial-response toast. Only IDs from the original request + // body can appear here — no third-party ID disclosure. + const accessibleIdSet = new Set(accessibleDocs.map((d) => d.id)); + const skippedIds = document_ids.filter((id) => !accessibleIdSet.has(id)); + + if (accessibleDocs.length === 0) return void res.status(404).json({ detail: "No documents found" }); const JSZip = (await import("jszip")).default; const zip = new JSZip(); await Promise.all( - docs.map(async (doc) => { + accessibleDocs.map(async (doc) => { const active = await loadActiveVersion(doc.id, db); if (!active) return; const raw = await downloadFile(active.storage_path); @@ -202,6 +345,9 @@ documentsRouter.post("/download-zip", requireAuth, async (req, res) => { const content = await zip.generateAsync({ type: "nodebuffer", compression: "DEFLATE" }); res.setHeader("Content-Type", "application/zip"); res.setHeader("Content-Disposition", 'attachment; filename="documents.zip"'); + if (skippedIds.length > 0) { + res.setHeader("X-Docs-Skipped", skippedIds.join(",")); + } res.send(content); }); @@ -395,18 +541,24 @@ documentsRouter.post( .select("id, filename, file_type, user_id, project_id") .eq("id", documentId) .single(); - if (!doc) + if (!doc) { + await cleanupTempFile(file.path); return void res.status(404).json({ detail: "Document not found" }); + } const access = await ensureDocAccess(doc, userId, userEmail, db); - if (!access.ok) + if (!access.ok) { + await cleanupTempFile(file.path); return void res.status(404).json({ detail: "Document not found" }); + } // Reject if the uploaded file's extension doesn't match the document's // declared type — otherwise every downstream viewer/extractor breaks. - const suffix = file.originalname.includes(".") - ? file.originalname.split(".").pop()!.toLowerCase() + const safeVersionFilename = sanitizeFilename(file.originalname); + const suffix = safeVersionFilename.includes(".") + ? safeVersionFilename.split(".").pop()!.toLowerCase() : ""; if (doc.file_type && suffix && doc.file_type !== suffix) { + await cleanupTempFile(file.path); return void res.status(400).json({ detail: `Uploaded file type (${suffix}) does not match document type (${doc.file_type}).`, }); @@ -414,98 +566,94 @@ documentsRouter.post( // Peg the new version into a predictable /versions/:id path under the // existing document folder so ops can spot the history in storage. - const versionSlug = crypto.randomUUID().replace(/-/g, ""); + const versionSlug = randomUUID().replace(/-/g, ""); const key = versionStorageKey( userId, documentId, versionSlug, - file.originalname, + safeVersionFilename, ); const contentType = suffix === "pdf" ? "application/pdf" : "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; + + let versionContent: Buffer; + try { + versionContent = await readFile(file.path); + } catch (e) { + logger.error({ err: e }, "[versions/upload] could not read temp file"); + await cleanupTempFile(file.path); + return void res + .status(500) + .json({ detail: "Failed to read uploaded file." }); + } + try { await uploadFile( key, - file.buffer.buffer.slice( - file.buffer.byteOffset, - file.buffer.byteOffset + file.buffer.byteLength, + versionContent.buffer.slice( + versionContent.byteOffset, + versionContent.byteOffset + versionContent.byteLength, ) as ArrayBuffer, contentType, ); } catch (e) { - console.error("[versions/upload] storage write failed", e); + logger.error({ err: e }, "[versions/upload] storage write failed"); + await cleanupTempFile(file.path); return void res .status(500) .json({ detail: "Failed to upload new version." }); } - // Render this version's bytes to PDF up front so /display can show - // historical versions without on-demand conversion. Same logic as the - // initial-upload pipeline; failures don't block the version row. + // Enqueue DOCX→PDF conversion in background so /display can show + // PDF rendition without blocking the request. Failures are non-fatal + // and will set pdf_conversion_status to 'failed'. let pdfStoragePath: string | null = null; - if (suffix === "docx" || suffix === "doc") { - try { - const pdfBuf = await docxToPdf(file.buffer); - const pdfKey = `converted-pdfs/${userId}/${documentId}/${versionSlug}.pdf`; - await uploadFile( - pdfKey, - pdfBuf.buffer.slice( - pdfBuf.byteOffset, - pdfBuf.byteOffset + pdfBuf.byteLength, - ) as ArrayBuffer, - "application/pdf", - ); - pdfStoragePath = pdfKey; - } catch (err) { - console.error( - `[versions/upload] DOCX→PDF conversion failed for ${file.originalname}:`, - err, - ); - } - } else if (suffix === "pdf") { + if (suffix === "pdf") { // For PDF uploads, the uploaded bytes are themselves the PDF rendition. pdfStoragePath = key; } // Per-document sequential version_number — the upload is V1 and // user_upload + assistant_edit count forward from there. - const { data: maxRow } = await db - .from("document_versions") - .select("version_number") - .eq("document_id", documentId) - .in("source", ["upload", "user_upload", "assistant_edit"]) - .order("version_number", { ascending: false, nullsFirst: false }) - .limit(1) - .maybeSingle(); - const nextVersionNumber = - ((maxRow?.version_number as number | null) ?? 1) + 1; - + // insertVersionWithRetry handles 23505 unique_violation races (CLEAN-08). const defaultDisplayName = typeof req.body?.display_name === "string" && req.body.display_name.trim() ? req.body.display_name.trim().slice(0, 200) - : file.originalname; - - const { data: versionRow, error: verErr } = await db - .from("document_versions") - .insert({ - document_id: documentId, - storage_path: key, - pdf_storage_path: pdfStoragePath, - source: "user_upload", - version_number: nextVersionNumber, - display_name: defaultDisplayName, - }) - .select("id, version_number, source, created_at, display_name") - .single(); + : safeVersionFilename; + + const { data: versionRow, error: verErr } = await insertVersionWithRetry(db, documentId, { + document_id: documentId, + storage_path: key, + pdf_storage_path: pdfStoragePath, + source: "user_upload", + display_name: defaultDisplayName, + }); if (verErr || !versionRow) { - console.error("[versions/upload] insert failed", verErr); + logger.error({ err: verErr }, "[versions/upload] insert failed"); + await cleanupTempFile(file.path); return void res .status(500) .json({ detail: "Failed to record new version." }); } + // Re-fetch the full version row so we have all fields (insertVersionWithRetry + // returns only id + version_number from the select). + const { data: fullVersionRow } = await db + .from("document_versions") + .select("id, version_number, source, created_at, display_name, storage_path") + .eq("id", versionRow.id) + .single(); + + // Enqueue background DOCX→PDF conversion for the new version. + if (suffix === "docx" || suffix === "doc") { + void enqueueConversionForVersion( + documentId, + { id: versionRow.id as string, storage_path: key }, + db, + ); + } // Also propagate the user-provided display_name to the parent document's // filename so the document's display name stays in sync across the UI. @@ -533,7 +681,12 @@ documentsRouter.post( .update(documentsUpdate) .eq("id", documentId); - res.status(201).json(versionRow); + await cleanupTempFile(file.path); + // Use fullVersionRow (all fields) for response, falling back to versionRow if re-fetch failed. + // Exclude internal storage_path from the API response. + const responseRow = fullVersionRow ?? versionRow; + const { storage_path: _sp, ...versionRowPublic } = responseRow as typeof responseRow & { storage_path?: string }; + res.status(201).json(versionRowPublic); }, ); @@ -632,11 +785,7 @@ async function handleEditResolution( const { documentId, editId } = req.params; const db = createServerSupabase(); - console.log(`[edit-resolution] incoming ${mode}`, { - userId, - documentId, - editId, - }); + logger.info({ userId, documentId, editId, mode }, "[edit-resolution] incoming"); const { data: edit, error: editErr } = await db .from("document_edits") @@ -644,31 +793,28 @@ async function handleEditResolution( .eq("id", editId) .eq("document_id", documentId) .single(); - console.log(`[edit-resolution] fetched edit row`, { edit, editErr }); + logger.info({ edit, editErr }, "[edit-resolution] fetched edit row"); if (!edit) { - console.log(`[edit-resolution] edit not found, returning 404`); + logger.info({ editId }, "[edit-resolution] edit not found, returning 404"); return void res.status(404).json({ detail: "Edit not found" }); } // Idempotent: if the edit is already resolved, return the current doc // state so stale UI (e.g. an old chat reloaded in a new session) can // reconcile without throwing. if (edit.status !== "pending") { - console.log(`[edit-resolution] edit already resolved`, { - editId, - status: edit.status, - }); + logger.info({ editId, status: edit.status }, "[edit-resolution] edit already resolved"); const { data: doc } = await db .from("documents") .select("current_version_id, filename, user_id, project_id") .eq("id", documentId) .single(); if (!doc) { - console.log(`[edit-resolution] doc not found for resolved edit`); + logger.info({ documentId }, "[edit-resolution] doc not found for resolved edit"); return void res.status(404).json({ detail: "Document not found" }); } const accessResolved = await ensureDocAccess(doc, userId, userEmail, db); if (!accessResolved.ok) { - console.log(`[edit-resolution] doc access denied for resolved edit`); + logger.info({ documentId, userId }, "[edit-resolution] doc access denied for resolved edit"); return void res.status(404).json({ detail: "Document not found" }); } const activeForResolved = await loadActiveVersion(documentId, db); @@ -685,7 +831,7 @@ async function handleEditResolution( : null, remaining_pending: 0, }; - console.log(`[edit-resolution] returning already-resolved payload`, payload); + logger.info({ payload }, "[edit-resolution] returning already-resolved payload"); return void res.status(200).json(payload); } @@ -694,7 +840,7 @@ async function handleEditResolution( .select("id, current_version_id, user_id, project_id") .eq("id", documentId) .single(); - console.log(`[edit-resolution] fetched doc`, { doc, docErr }); + logger.info({ doc, docErr }, "[edit-resolution] fetched doc"); if (!doc) return void res.status(404).json({ detail: "Document not found" }); const access = await ensureDocAccess(doc, userId, userEmail, db); @@ -703,17 +849,12 @@ async function handleEditResolution( const active = await loadActiveVersion(documentId, db); const latestPath = active?.storage_path ?? null; - console.log(`[edit-resolution] resolved latestPath`, { - latestPath, - current_version_id: doc.current_version_id, - }); + logger.info({ latestPath, currentVersionId: doc.current_version_id }, "[edit-resolution] resolved latestPath"); if (!latestPath) return void res.status(404).json({ detail: "No file to edit" }); const raw = await downloadFile(latestPath); - console.log(`[edit-resolution] downloaded bytes`, { - byteLength: raw?.byteLength ?? 0, - }); + logger.info({ byteLength: raw?.byteLength ?? 0 }, "[edit-resolution] downloaded bytes"); if (!raw) return void res.status(404).json({ detail: "Document bytes not available" }); @@ -725,24 +866,16 @@ async function handleEditResolution( wIds, mode, ); - console.log(`[edit-resolution] resolveTrackedChange result`, { - mode, - change_id: edit.change_id, - wIds, - found, - resolvedByteLength: resolvedBytes?.byteLength ?? 0, - }); + logger.info({ mode, changeId: edit.change_id, wIds, found, resolvedByteLength: resolvedBytes?.byteLength ?? 0 }, "[edit-resolution] resolveTrackedChange result"); if (!found) { - console.log( - `[edit-resolution] change_id not found in docx — updating status only`, - ); + logger.info({ changeId: edit.change_id }, "[edit-resolution] change_id not found in docx — updating status only"); // Still update DB status so the UI reflects the decision — the change // may have been auto-consumed by a previous accept/reject pass. const { error: updErr } = await db .from("document_edits") .update({ status: mode === "accept" ? "accepted" : "rejected", resolved_at: new Date().toISOString() }) .eq("id", editId); - console.log(`[edit-resolution] status-only update`, { updErr }); + logger.info({ updErr }, "[edit-resolution] status-only update"); const { data: filenameRow } = await db .from("documents") .select("filename") @@ -757,7 +890,7 @@ async function handleEditResolution( ), remaining_pending: 0, }; - console.log(`[edit-resolution] returning not-found payload`, payload); + logger.info({ payload }, "[edit-resolution] returning not-found payload"); return void res.status(200).json(payload); } @@ -766,39 +899,47 @@ async function handleEditResolution( // new row. This keeps document_versions lean (one row per assistant // edit, not one per accept/reject click) and avoids the N-versions- // per-doc churn as users resolve pending changes. + // + // CLEAN-09 + CLEAN-34: applyEditResolutionSaga sequences download-prior → + // upload-new → DB-update with a compensating re-upload on DB failure so + // storage and DB stay consistent. const ab = resolvedBytes.buffer.slice( resolvedBytes.byteOffset, resolvedBytes.byteOffset + resolvedBytes.byteLength, ) as ArrayBuffer; - console.log(`[edit-resolution] overwriting bytes in place`, { - latestPath, - byteLength: ab.byteLength, - }); - await uploadFile( - latestPath, - ab, - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - ); + logger.info({ latestPath, byteLength: ab.byteLength }, "[edit-resolution] overwriting bytes in place via saga"); - const { error: statusErr } = await db - .from("document_edits") - .update({ - status: mode === "accept" ? "accepted" : "rejected", - resolved_at: new Date().toISOString(), - }) - .eq("id", editId); - console.log(`[edit-resolution] updated document_edits status`, { + const resolvedStatus = mode === "accept" ? "accepted" : "rejected"; + const sagaResult = await applyEditResolutionSaga({ + latestPath, + newBytes: ab, + status: resolvedStatus, editId, - newStatus: mode === "accept" ? "accepted" : "rejected", - statusErr, + uploadFn: uploadFile, + downloadFn: downloadFile, + dbUpdateFn: async (status, editId) => { + return db + .from("document_edits") + .update({ + status, + resolved_at: new Date().toISOString(), + }) + .eq("id", editId); + }, }); + logger.info({ editId, newStatus: resolvedStatus, ok: sagaResult.ok }, "[edit-resolution] saga result"); + + if (!sagaResult.ok) { + return void res.status(sagaResult.status).json({ detail: sagaResult.detail }); + } + const { count: remainingPending } = await db .from("document_edits") .select("id", { count: "exact", head: true }) .eq("document_id", documentId) .eq("status", "pending"); - console.log(`[edit-resolution] remaining pending count`, { remainingPending }); + logger.info({ remainingPending }, "[edit-resolution] remaining pending count"); const { data: filenameRow } = await db .from("documents") @@ -814,7 +955,7 @@ async function handleEditResolution( ), remaining_pending: remainingPending ?? 0, }; - console.log(`[edit-resolution] returning success payload`, payload); + logger.info({ payload }, "[edit-resolution] returning success payload"); res.json(payload); } @@ -830,6 +971,52 @@ documentsRouter.post( (req, res) => void handleEditResolution(req, res, "reject"), ); +// POST /single-documents/:documentId/regenerate-pdf +// Re-enqueues the DOCX→PDF conversion for an existing document. +// Returns 202 immediately with pdf_conversion_status: "pending". +// Rejects non-DOCX/DOC with 400; rejects missing/unauthorized with 404. +documentsRouter.post( + "/:documentId/regenerate-pdf", + requireAuth, + async (req, res) => { + const userId = res.locals.userId as string; + const userEmail = res.locals.userEmail as string | undefined; + const { documentId } = req.params; + const db = createServerSupabase(); + + const { data: doc } = await db + .from("documents") + .select("id, file_type, user_id, project_id, current_version_id") + .eq("id", documentId) + .single(); + if (!doc) + return void res.status(404).json({ detail: "Document not found" }); + + const access = await ensureDocAccess(doc, userId, userEmail ?? "", db); + if (!access.ok) + return void res.status(404).json({ detail: "Document not found" }); + + const fileType = doc.file_type as string; + if (fileType !== "docx" && fileType !== "doc") { + return void res + .status(400) + .json({ detail: "PDF regeneration only applies to DOCX/DOC documents." }); + } + + await db + .from("documents") + .update({ pdf_conversion_status: "pending" }) + .eq("id", documentId); + + const active = await loadActiveVersion(documentId, db); + if (active) { + void enqueueConversionForVersion(documentId, active, db); + } + + return void res.status(202).json({ pdf_conversion_status: "pending" }); + }, +); + async function handleDocumentUpload( req: import("express").Request, res: import("express").Response, @@ -840,18 +1027,29 @@ async function handleDocumentUpload( const file = req.file; if (!file) return void res.status(400).json({ detail: "file is required" }); - const filename = file.originalname; + // sanitizeFilename must run before the storage key is constructed (CLEAN-26). + const filename = sanitizeFilename(file.originalname); const suffix = filename.includes(".") ? filename.split(".").pop()!.toLowerCase() : ""; - if (!ALLOWED_TYPES.has(suffix)) - return void res - .status(400) - .json({ - detail: `Unsupported file type: ${suffix}. Allowed: pdf, docx, doc`, - }); + if (!ALLOWED_TYPES.has(suffix)) { + await cleanupTempFile(file.path); + return void res.status(400).json({ + detail: `Unsupported file type: ${suffix}. Allowed: pdf, docx, doc`, + }); + } + + // Read bytes from the temp file (diskStorage writes to path, not buffer). + // This must happen before the temp file is cleaned up in the finally block. + let content: Buffer; + try { + content = await readFile(file.path); + } catch (readErr) { + logger.error({ err: readErr }, "[upload] could not read temp file"); + await cleanupTempFile(file.path); + return void res.status(500).json({ detail: "Failed to read uploaded file" }); + } - const content = file.buffer; const { data: doc, error: insertErr } = await db .from("documents") .insert({ @@ -861,13 +1059,16 @@ async function handleDocumentUpload( file_type: suffix, size_bytes: content.byteLength, status: "processing", + pdf_conversion_status: "pending", }) .select("*") .single(); - if (insertErr || !doc) + if (insertErr || !doc) { + await cleanupTempFile(file.path); return void res .status(500) .json({ detail: "Failed to create document record" }); + } try { const docId = doc.id as string; @@ -889,31 +1090,13 @@ async function handleDocumentUpload( content.byteOffset, content.byteOffset + content.byteLength, ) as ArrayBuffer; - const tree = await extractStructureTree(rawBuf, suffix, filename); + const tree = await extractStructureTree(rawBuf, suffix); const pageCount = suffix === "pdf" ? await countPdfPages(rawBuf) : null; - // Convert DOCX/DOC → PDF for display. PDFs are their own rendition. + // For PDF uploads the file is its own rendition; DOCX/DOC conversion is + // enqueued in the background so the request can return immediately. let pdfStoragePath: string | null = null; - if (suffix === "docx" || suffix === "doc") { - try { - const pdfBuf = await docxToPdf(content); - const pdfKey = convertedPdfKey(userId, docId); - await uploadFile( - pdfKey, - pdfBuf.buffer.slice( - pdfBuf.byteOffset, - pdfBuf.byteOffset + pdfBuf.byteLength, - ) as ArrayBuffer, - "application/pdf", - ); - pdfStoragePath = pdfKey; - } catch (err) { - console.error( - `[upload] DOCX→PDF conversion failed for ${filename}:`, - err, - ); - } - } else if (suffix === "pdf") { + if (suffix === "pdf") { pdfStoragePath = key; } @@ -938,6 +1121,17 @@ async function handleDocumentUpload( ); } + // Enqueue background PDF conversion for DOCX/DOC uploads (CLEAN-20). + // Pass the in-memory buffer so it survives temp-file cleanup below. + if (suffix === "docx" || suffix === "doc") { + void enqueueConversionFromBuffer({ + documentId: docId, + versionId: versionRow.id, + userId, + docxBuffer: content, + }); + } + await db .from("documents") .update({ @@ -965,6 +1159,8 @@ async function handleDocumentUpload( return void res .status(500) .json({ detail: `Document processing failed: ${String(e)}` }); + } finally { + await cleanupTempFile(file.path); } } @@ -984,61 +1180,3 @@ async function countPdfPages(buf: ArrayBuffer): Promise<number | null> { } } -async function extractStructureTree( - content: ArrayBuffer, - fileType: string, - _filename: string, -): Promise<unknown[] | null> { - try { - if (fileType === "pdf") { - const pdfjsLib = await import( - "pdfjs-dist/legacy/build/pdf.mjs" as string - ); - const pdf = await ( - pdfjsLib as unknown as { - getDocument: (opts: unknown) => { - promise: Promise<{ - numPages: number; - getOutline: () => Promise<{ title?: string }[]>; - }>; - }; - } - ).getDocument({ data: new Uint8Array(content) }).promise; - if (pdf.numPages <= 5) return null; - const outline = await pdf.getOutline(); - if (outline?.length) - return outline.map((item, i) => ({ - id: `h1-${i}`, - title: item.title ?? `Item ${i + 1}`, - level: 1, - page_number: null, - children: [], - })); - return Array.from({ length: pdf.numPages }, (_, i) => ({ - id: `page-${i + 1}`, - title: `Page ${i + 1}`, - level: 1, - page_number: i + 1, - children: [], - })); - } else { - const mammoth = await import("mammoth"); - const result = await mammoth.extractRawText({ - buffer: Buffer.from(content), - }); - const lines = result.value.split("\n").filter((l) => l.trim()); - const nodes = lines - .slice(0, 30) - .map((line, i) => ({ - id: `h1-${i}`, - title: line.slice(0, 100), - level: 1, - page_number: null, - children: [], - })); - return nodes.length ? nodes : null; - } - } catch { - return null; - } -} diff --git a/backend/src/routes/models.ts b/backend/src/routes/models.ts new file mode 100644 index 000000000..995808607 --- /dev/null +++ b/backend/src/routes/models.ts @@ -0,0 +1,44 @@ +import { Router } from "express"; +import { requireAuth } from "../middleware/auth"; +import { + CLAUDE_MAIN_MODELS, + GEMINI_MAIN_MODELS, + CLAUDE_MID_MODELS, + GEMINI_MID_MODELS, + CLAUDE_LOW_MODELS, + GEMINI_LOW_MODELS, + DEFAULT_MAIN_MODEL, + DEFAULT_TITLE_MODEL, + DEFAULT_TABULAR_MODEL, + providerForModel, +} from "../lib/llm/models"; + +export const modelsRouter = Router(); + +function toEntry(id: string, group: string) { + return { id, provider: providerForModel(id), label: id, group }; +} + +// CLEAN-50: single source of truth — return full model catalog grouped by tier. +// Auth-gated: model IDs are internal implementation details. +modelsRouter.get("/", requireAuth, (_req, res) => { + res.json({ + main: [ + ...CLAUDE_MAIN_MODELS.map((id) => toEntry(id, "Anthropic")), + ...GEMINI_MAIN_MODELS.map((id) => toEntry(id, "Google")), + ], + mid: [ + ...CLAUDE_MID_MODELS.map((id) => toEntry(id, "Anthropic")), + ...GEMINI_MID_MODELS.map((id) => toEntry(id, "Google")), + ], + low: [ + ...CLAUDE_LOW_MODELS.map((id) => toEntry(id, "Anthropic")), + ...GEMINI_LOW_MODELS.map((id) => toEntry(id, "Google")), + ], + defaults: { + main: DEFAULT_MAIN_MODEL, + title: DEFAULT_TITLE_MODEL, + tabular: DEFAULT_TABULAR_MODEL, + }, + }); +}); diff --git a/backend/src/routes/projectChat.ts b/backend/src/routes/projectChat.ts index 5e2996152..47ef77220 100644 --- a/backend/src/routes/projectChat.ts +++ b/backend/src/routes/projectChat.ts @@ -1,6 +1,9 @@ import { Router } from "express"; +import { z } from "zod"; import { requireAuth } from "../middleware/auth"; +import { llmRateLimiter } from "../lib/rateLimiter"; import { createServerSupabase } from "../lib/supabase"; +import { logger } from "../lib/logger"; import { buildProjectDocContext, buildMessages, @@ -13,6 +16,7 @@ import { } from "../lib/chatTools"; import { getUserApiKeys } from "../lib/userSettings"; import { checkProjectAccess } from "../lib/access"; +import { parseBody } from "../lib/validate"; const PROJECT_SYSTEM_PROMPT_EXTRA = `PROJECT CONTEXT: You are operating within a project folder that contains a collection of legal documents the user has organised for a single matter. The user's questions will usually refer to one or more documents in this project — your job is to find the relevant files to work on. Use list_documents to see what is available and fetch_documents / read_document to pull in any documents you need before answering. @@ -22,15 +26,46 @@ A document may currently be displayed in the user's side panel; when provided, t REPLICATING A DOCUMENT: When the user wants to use an existing project document as a starting point for a new file (e.g. "use this NDA as a template", "make me a copy of the SOW so I can edit it", "duplicate this and adapt it for company X"), call the replicate_document tool with the source doc_id. This creates a byte-for-byte copy as a new project document, returns a fresh doc_id slug, and shows a download/open card in the UI. Then call edit_document on the returned slug to make the user's requested changes — do NOT call generate_docx for cases where the user clearly wants the existing document's structure and formatting preserved.`; +const ProjectChatStreamSchema = z.object({ + messages: z.array( + z.object({ + role: z.enum(["user", "assistant"]), + content: z.string(), + files: z.array(z.unknown()).optional().nullable(), + workflow: z.unknown().optional().nullable(), + }), + ).min(1), + chat_id: z.string().uuid().optional(), + model: z.string().optional(), + displayed_doc: z + .object({ + filename: z.string(), + document_id: z.string(), + }) + .optional() + .nullable(), + attached_documents: z + .array( + z.object({ + filename: z.string(), + document_id: z.string(), + }), + ) + .optional() + .nullable(), +}); + export const projectChatRouter = Router({ mergeParams: true }); // POST /projects/:projectId/chat — streaming -projectChatRouter.post("/", requireAuth, async (req, res) => { +projectChatRouter.post("/", requireAuth, llmRateLimiter, async (req, res) => { const userId = res.locals.userId as string; const userEmail = res.locals.userEmail as string | undefined; const { projectId } = req.params; + const body = parseBody(ProjectChatStreamSchema, req, res); + if (!body) return; const { messages, chat_id, model, displayed_doc, attached_documents } = - req.body as { + body as unknown as { messages: ChatMessage[]; chat_id?: string; model?: string; @@ -91,7 +126,6 @@ projectChatRouter.post("/", requireAuth, async (req, res) => { const { docIndex, docStore, folderPaths } = await buildProjectDocContext( projectId, - userId, db, ); const docAvailability = Object.entries(docIndex).map(([doc_id, info]) => ({ @@ -152,7 +186,7 @@ projectChatRouter.post("/", requireAuth, async (req, res) => { const write = (line: string) => res.write(line); - const apiKeys = await getUserApiKeys(userId, db); + const apiKeys = await getUserApiKeys(userId, db, { route: req.path, requestId: req.id }); try { write(`data: ${JSON.stringify({ type: "chat_id", chatId })}\n\n`); @@ -186,7 +220,7 @@ projectChatRouter.post("/", requireAuth, async (req, res) => { .eq("id", chatId); } } catch (err) { - console.error("[project-chat/stream] error:", err); + logger.error({ err }, "[project-chat/stream] error"); try { write( `data: ${JSON.stringify({ type: "error", message: "Stream error" })}\n\n`, diff --git a/backend/src/routes/projects.ts b/backend/src/routes/projects.ts index 58de3c083..a45bb88ab 100644 --- a/backend/src/routes/projects.ts +++ b/backend/src/routes/projects.ts @@ -1,28 +1,55 @@ +import { readFile } from "fs/promises"; import { Router } from "express"; +import { z } from "zod"; import { requireAuth } from "../middleware/auth"; -import { createServerSupabase } from "../lib/supabase"; +import { parseBody } from "../lib/validate"; +import { createServerSupabase, getUsersByEmails, getUserById } from "../lib/supabase"; +import { logger } from "../lib/logger"; import { createClient } from "@supabase/supabase-js"; import { attachActiveVersionPaths, attachLatestVersionNumbers, } from "../lib/documentVersions"; import { downloadFile, uploadFile, storageKey } from "../lib/storage"; -import { docxToPdf, convertedPdfKey } from "../lib/convert"; +import { convertedPdfKey } from "../lib/convert"; import { checkProjectAccess } from "../lib/access"; -import { singleFileUpload } from "../lib/upload"; +import { + cleanupTempFile, + sanitizeFilename, + singleFileUpload, +} from "../lib/upload"; +import { enqueueConversionFromBuffer } from "../lib/pdfQueue"; +import { extractStructureTree } from "../lib/structureTree"; + +const CreateProjectSchema = z.object({ + name: z.string().trim().min(1, "name is required"), + cm_number: z.string().trim().optional(), + shared_with: z.array(z.string().email()).optional(), +}); + +const UpdateProjectSchema = z.object({ + name: z.string().trim().min(1).optional(), + cm_number: z.string().trim().nullable().optional(), + shared_with: z.array(z.string().email()).optional(), +}).strict(); + +const CreateFolderSchema = z.object({ + name: z.string().trim().min(1, "name is required"), + parent_folder_id: z.string().uuid().nullable().optional(), +}); + +const UpdateFolderSchema = z.object({ + name: z.string().trim().min(1).optional(), + parent_folder_id: z.string().uuid().nullable().optional(), +}).strict(); + +const MoveDocumentToFolderSchema = z.object({ + folder_id: z.string().uuid().nullable(), +}); export const projectsRouter = Router(); const ALLOWED_TYPES = new Set(["pdf", "docx", "doc"]); -function normalizeDocumentFilename(nextName: unknown, currentName: string) { - if (typeof nextName !== "string") return null; - const trimmed = nextName.trim().slice(0, 200); - if (!trimmed) return null; - if (/\.[a-z0-9]{1,6}$/i.test(trimmed)) return trimmed; - const ext = currentName.match(/\.[a-z0-9]{1,6}$/i)?.[0] ?? ""; - return `${trimmed}${ext}`; -} - // GET /projects projectsRouter.get("/", requireAuth, async (req, res) => { const userId = res.locals.userId as string; @@ -40,7 +67,7 @@ projectsRouter.get("/", requireAuth, async (req, res) => { ? await db .from("projects") .select("*") - .filter("shared_with", "cs", JSON.stringify([userEmail])) + .contains("shared_with", JSON.stringify([userEmail])) .neq("user_id", userId) .order("created_at", { ascending: false }) : { data: [], error: null }; @@ -83,13 +110,9 @@ projectsRouter.get("/", requireAuth, async (req, res) => { // POST /projects projectsRouter.post("/", requireAuth, async (req, res) => { const userId = res.locals.userId as string; - const { name, cm_number, shared_with } = req.body as { - name: string; - cm_number?: string; - shared_with?: string[]; - }; - if (!name?.trim()) - return void res.status(400).json({ detail: "name is required" }); + const body = parseBody(CreateProjectSchema, req, res); + if (!body) return; + const { name, cm_number, shared_with } = body; const db = createServerSupabase(); const { data, error } = await db @@ -125,7 +148,9 @@ projectsRouter.get("/:projectId", requireAuth, async (req, res) => { project.user_id === userId || (userEmail && Array.isArray(project.shared_with) && - project.shared_with.includes(userEmail)); + project.shared_with.some( + (e: string) => (e ?? "").toLowerCase() === (userEmail ?? "").toLowerCase(), + )); if (!canAccess) return void res.status(404).json({ detail: "Project not found" }); @@ -175,19 +200,13 @@ projectsRouter.get("/:projectId/people", requireAuth, async (req, res) => { if (!isOwner && !isShared) return void res.status(404).json({ detail: "Project not found" }); - // Pull every auth user (matching the lookup endpoint's pattern). For - // larger deployments this should page or be replaced with a bulk-by-id - // RPC, but it keeps things simple while user counts are modest. - const { data: usersData } = await db.auth.admin.listUsers({ perPage: 1000 }); - const allUsers = usersData?.users ?? []; - const userByEmail = new Map<string, { id: string; email: string }>(); - const userById = new Map<string, { id: string; email: string }>(); - for (const u of allUsers) { - if (!u.email) continue; - const lower = u.email.toLowerCase(); - userByEmail.set(lower, { id: u.id, email: u.email }); - userById.set(u.id, { id: u.id, email: u.email }); - } + // Resolve shared-with emails to user records via RPC (CLEAN-15). + // get_auth_user_by_email is SECURITY DEFINER and O(log N) — replaces the + // listUsers({ perPage: 1000 }) walk that silently truncates above 1000. + const [userByEmail, ownerRecord] = await Promise.all([ + getUsersByEmails(sharedWith), + getUserById(project.user_id as string), + ]); const memberUserIds: string[] = []; for (const email of sharedWith) { @@ -217,20 +236,19 @@ projectsRouter.get("/:projectId/people", requireAuth, async (req, res) => { } } - const ownerInfo = userById.get(project.user_id as string); const owner = { user_id: project.user_id, - email: ownerInfo?.email ?? null, + email: ownerRecord?.email ?? null, display_name: profileByUserId.get(project.user_id as string)?.display_name ?? null, }; - const members = sharedWith.map((email) => { - const u = userByEmail.get(email); - const display_name = u - ? profileByUserId.get(u.id)?.display_name ?? null - : null; - return { email, display_name }; - }); + const members = sharedWith + .filter((email) => userByEmail.has(email)) + .map((email) => { + const u = userByEmail.get(email)!; + const display_name = profileByUserId.get(u.id)?.display_name ?? null; + return { email, display_name }; + }); res.json({ owner, members }); }); @@ -239,19 +257,20 @@ projectsRouter.get("/:projectId/people", requireAuth, async (req, res) => { projectsRouter.patch("/:projectId", requireAuth, async (req, res) => { const userId = res.locals.userId as string; const { projectId } = req.params; + const body = parseBody(UpdateProjectSchema, req, res); + if (!body) return; const updates: Record<string, unknown> = {}; - if (req.body.name != null) updates.name = req.body.name; - if (req.body.cm_number != null) updates.cm_number = req.body.cm_number; - if (Array.isArray(req.body.shared_with)) { + if (body.name != null) updates.name = body.name; + if (body.cm_number != null) updates.cm_number = body.cm_number; + if (Array.isArray(body.shared_with)) { // Normalise: lowercase + dedupe + drop empties. const seen = new Set<string>(); const cleaned: string[] = []; - for (const raw of req.body.shared_with) { - if (typeof raw !== "string") continue; - const e = raw.trim().toLowerCase(); - if (!e || seen.has(e)) continue; - seen.add(e); - cleaned.push(e); + for (const e of body.shared_with) { + const norm = e.trim().toLowerCase(); + if (!norm || seen.has(norm)) continue; + seen.add(norm); + cleaned.push(norm); } updates.shared_with = cleaned; } @@ -284,12 +303,15 @@ projectsRouter.delete("/:projectId", requireAuth, async (req, res) => { const userId = res.locals.userId as string; const { projectId } = req.params; const db = createServerSupabase(); - const { error } = await db + const { data, error } = await db .from("projects") .delete() .eq("id", projectId) - .eq("user_id", userId); + .eq("user_id", userId) + .select("id"); if (error) return void res.status(500).json({ detail: error.message }); + if (!data || data.length === 0) + return void res.status(404).json({ detail: "Project not found" }); res.status(204).send(); }); @@ -446,51 +468,6 @@ projectsRouter.post( }, ); -// PATCH /projects/:projectId/documents/:documentId — rename a project document -projectsRouter.patch("/:projectId/documents/:documentId", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const userEmail = res.locals.userEmail as string | undefined; - const { projectId, documentId } = req.params; - const db = createServerSupabase(); - - const access = await checkProjectAccess(projectId, userId, userEmail, db); - if (!access.ok) - return void res.status(404).json({ detail: "Project not found" }); - - const { data: doc } = await db - .from("documents") - .select("id, filename, current_version_id") - .eq("id", documentId) - .eq("project_id", projectId) - .single(); - if (!doc) - return void res.status(404).json({ detail: "Document not found" }); - - const filename = normalizeDocumentFilename(req.body?.filename, doc.filename as string); - if (!filename) - return void res.status(400).json({ detail: "filename is required" }); - - const { data: updated, error } = await db - .from("documents") - .update({ filename, updated_at: new Date().toISOString() }) - .eq("id", documentId) - .eq("project_id", projectId) - .select("*") - .single(); - if (error || !updated) - return void res.status(404).json({ detail: "Document not found" }); - - if (doc.current_version_id) { - await db - .from("document_versions") - .update({ display_name: filename }) - .eq("id", doc.current_version_id) - .eq("document_id", documentId); - } - - res.json(updated); -}); - // POST /projects/:projectId/documents projectsRouter.post( "/:projectId/documents", @@ -541,8 +518,9 @@ projectsRouter.post("/:projectId/folders", requireAuth, async (req, res) => { const userId = res.locals.userId as string; const userEmail = res.locals.userEmail as string | undefined; const { projectId } = req.params; - const { name, parent_folder_id } = req.body as { name: string; parent_folder_id?: string | null }; - if (!name?.trim()) return void res.status(400).json({ detail: "name is required" }); + const folderBody = parseBody(CreateFolderSchema, req, res); + if (!folderBody) return; + const { name, parent_folder_id } = folderBody; const db = createServerSupabase(); const access = await checkProjectAccess(projectId, userId, userEmail, db); @@ -569,7 +547,8 @@ projectsRouter.patch("/:projectId/folders/:folderId", requireAuth, async (req, r const userId = res.locals.userId as string; const userEmail = res.locals.userEmail as string | undefined; const { projectId, folderId } = req.params; - const body = req.body as { name?: string; parent_folder_id?: string | null }; + const body = parseBody(UpdateFolderSchema, req, res); + if (!body) return; const db = createServerSupabase(); const access = await checkProjectAccess(projectId, userId, userEmail, db); @@ -580,14 +559,11 @@ projectsRouter.patch("/:projectId/folders/:folderId", requireAuth, async (req, r if ("parent_folder_id" in body) { // Cycle check: walk up the tree from the proposed parent to ensure folderId is not an ancestor if (body.parent_folder_id) { - const parent = await loadProjectFolder(db, projectId, body.parent_folder_id); - if (!parent) return void res.status(404).json({ detail: "Parent folder not found" }); - let cur: string | null = body.parent_folder_id; while (cur) { if (cur === folderId) return void res.status(400).json({ detail: "Cannot move a folder into itself or a descendant" }); - const p = await loadProjectFolder(db, projectId, cur); - if (!p) return void res.status(404).json({ detail: "Parent folder not found" }); + const { data: p }: { data: { parent_folder_id: string | null } | null } = + await db.from("project_subfolders").select("parent_folder_id").eq("id", cur).single(); cur = p?.parent_folder_id ?? null; } } @@ -612,11 +588,8 @@ projectsRouter.delete("/:projectId/folders/:folderId", requireAuth, async (req, const access = await checkProjectAccess(projectId, userId, userEmail, db); if (!access.ok) return void res.status(404).json({ detail: "Project not found" }); - const folder = await loadProjectFolder(db, projectId, folderId); - if (!folder) return void res.status(404).json({ detail: "Folder not found" }); - // Move direct documents to root before cascade-deleting subfolders - await db.from("documents").update({ folder_id: null }).eq("folder_id", folderId).eq("project_id", projectId); + await db.from("documents").update({ folder_id: null }).eq("folder_id", folderId); const { error } = await db.from("project_subfolders") .delete().eq("id", folderId).eq("project_id", projectId); @@ -629,17 +602,14 @@ projectsRouter.patch("/:projectId/documents/:documentId/folder", requireAuth, as const userId = res.locals.userId as string; const userEmail = res.locals.userEmail as string | undefined; const { projectId, documentId } = req.params; - const { folder_id } = req.body as { folder_id: string | null }; + const moveBody = parseBody(MoveDocumentToFolderSchema, req, res); + if (!moveBody) return; + const { folder_id } = moveBody; const db = createServerSupabase(); const access = await checkProjectAccess(projectId, userId, userEmail, db); if (!access.ok) return void res.status(404).json({ detail: "Project not found" }); - if (folder_id) { - const folder = await loadProjectFolder(db, projectId, folder_id); - if (!folder) return void res.status(404).json({ detail: "Folder not found" }); - } - const { data, error } = await db.from("documents") .update({ folder_id: folder_id ?? null, updated_at: new Date().toISOString() }) .eq("id", documentId).eq("project_id", projectId) @@ -648,20 +618,6 @@ projectsRouter.patch("/:projectId/documents/:documentId/folder", requireAuth, as res.json(data); }); -async function loadProjectFolder( - db: ReturnType<typeof createServerSupabase>, - projectId: string, - folderId: string, -): Promise<{ id: string; parent_folder_id: string | null } | null> { - const { data } = await db - .from("project_subfolders") - .select("id, parent_folder_id") - .eq("id", folderId) - .eq("project_id", projectId) - .maybeSingle(); - return (data as { id: string; parent_folder_id: string | null } | null) ?? null; -} - export async function handleDocumentUpload( req: import("express").Request, res: import("express").Response, @@ -672,18 +628,29 @@ export async function handleDocumentUpload( const file = req.file; if (!file) return void res.status(400).json({ detail: "file is required" }); - const filename = file.originalname; + // sanitizeFilename must run before the storage key is constructed (CLEAN-26). + const filename = sanitizeFilename(file.originalname); const suffix = filename.includes(".") ? filename.split(".").pop()!.toLowerCase() : ""; - if (!ALLOWED_TYPES.has(suffix)) - return void res - .status(400) - .json({ - detail: `Unsupported file type: ${suffix}. Allowed: pdf, docx, doc`, - }); + if (!ALLOWED_TYPES.has(suffix)) { + await cleanupTempFile(file.path); + return void res.status(400).json({ + detail: `Unsupported file type: ${suffix}. Allowed: pdf, docx, doc`, + }); + } + + // Read bytes from the temp file (diskStorage writes to path, not buffer). + // This must happen before the temp file is cleaned up in the finally block. + let content: Buffer; + try { + content = await readFile(file.path); + } catch (readErr) { + logger.error({ err: readErr }, "[upload] could not read temp file"); + await cleanupTempFile(file.path); + return void res.status(500).json({ detail: "Failed to read uploaded file" }); + } - const content = file.buffer; const { data: doc, error: insertErr } = await db .from("documents") .insert({ @@ -693,14 +660,16 @@ export async function handleDocumentUpload( file_type: suffix, size_bytes: content.byteLength, status: "processing", + pdf_conversion_status: "pending", }) .select("*") .single(); - - if (insertErr || !doc) + if (insertErr || !doc) { + await cleanupTempFile(file.path); return void res .status(500) .json({ detail: "Failed to create document record" }); + } try { const docId = doc.id as string; @@ -722,31 +691,13 @@ export async function handleDocumentUpload( content.byteOffset, content.byteOffset + content.byteLength, ) as ArrayBuffer; - const tree = await extractStructureTree(rawBuf, suffix, filename); + const tree = await extractStructureTree(rawBuf, suffix); const pageCount = suffix === "pdf" ? await countPdfPages(rawBuf) : null; - // Convert DOCX/DOC → PDF for display. PDFs are their own rendition. + // For PDF uploads the file is its own rendition; DOCX/DOC conversion is + // enqueued in the background so the request can return immediately. let pdfStoragePath: string | null = null; - if (suffix === "docx" || suffix === "doc") { - try { - const pdfBuf = await docxToPdf(content); - const pdfKey = convertedPdfKey(userId, docId); - await uploadFile( - pdfKey, - pdfBuf.buffer.slice( - pdfBuf.byteOffset, - pdfBuf.byteOffset + pdfBuf.byteLength, - ) as ArrayBuffer, - "application/pdf", - ); - pdfStoragePath = pdfKey; - } catch (err) { - console.error( - `[upload] DOCX→PDF conversion failed for ${filename}:`, - err, - ); - } - } else if (suffix === "pdf") { + if (suffix === "pdf") { pdfStoragePath = key; } @@ -770,6 +721,17 @@ export async function handleDocumentUpload( ); } + // Enqueue background PDF conversion for DOCX/DOC uploads (CLEAN-20). + // Pass the in-memory buffer so it survives temp-file cleanup below. + if (suffix === "docx" || suffix === "doc") { + void enqueueConversionFromBuffer({ + documentId: docId, + versionId: versionRow.id, + userId, + docxBuffer: content, + }); + } + await db .from("documents") .update({ @@ -787,12 +749,9 @@ export async function handleDocumentUpload( .select("*") .eq("id", docId) .single(); + // Surface storage paths to the caller for backward compatibility. const responseDoc = updated - ? { - ...updated, - storage_path: key, - pdf_storage_path: pdfStoragePath, - } + ? { ...updated, storage_path: key, pdf_storage_path: pdfStoragePath } : updated; return void res.status(201).json(responseDoc); } catch (e) { @@ -800,6 +759,8 @@ export async function handleDocumentUpload( return void res .status(500) .json({ detail: `Document processing failed: ${String(e)}` }); + } finally { + await cleanupTempFile(file.path); } } @@ -818,63 +779,3 @@ async function countPdfPages(buf: ArrayBuffer): Promise<number | null> { return null; } } - -async function extractStructureTree( - content: ArrayBuffer, - fileType: string, - filename: string, -): Promise<unknown[] | null> { - try { - if (fileType === "pdf") { - const pdfjsLib = await import( - "pdfjs-dist/legacy/build/pdf.mjs" as string - ); - const pdf = await ( - pdfjsLib as unknown as { - getDocument: (opts: unknown) => { - promise: Promise<{ - numPages: number; - getOutline: () => Promise<{ title?: string }[]>; - }>; - }; - } - ).getDocument({ data: new Uint8Array(content) }).promise; - if (pdf.numPages <= 5) return null; - const outline = await pdf.getOutline(); - if (outline?.length) { - return outline.map((item, i) => ({ - id: `h1-${i}`, - title: item.title ?? `Item ${i + 1}`, - level: 1, - page_number: null, - children: [], - })); - } - return Array.from({ length: pdf.numPages }, (_, i) => ({ - id: `page-${i + 1}`, - title: `Page ${i + 1}`, - level: 1, - page_number: i + 1, - children: [], - })); - } else { - const mammoth = await import("mammoth"); - const result = await mammoth.extractRawText({ - buffer: Buffer.from(content), - }); - const lines = result.value.split("\n").filter((l) => l.trim()); - const nodes = lines - .slice(0, 30) - .map((line, i) => ({ - id: `h1-${i}`, - title: line.slice(0, 100), - level: 1, - page_number: null, - children: [], - })); - return nodes.length ? nodes : null; - } - } catch { - return null; - } -} diff --git a/backend/src/routes/tabular.ts b/backend/src/routes/tabular.ts index b7efff601..b37058db5 100644 --- a/backend/src/routes/tabular.ts +++ b/backend/src/routes/tabular.ts @@ -1,6 +1,10 @@ import { Router } from "express"; +import { z } from "zod"; import { requireAuth } from "../middleware/auth"; -import { createServerSupabase } from "../lib/supabase"; +import { llmRateLimiter } from "../lib/rateLimiter"; +import { createServerSupabase, getUsersByEmails, getUserById } from "../lib/supabase"; +import { logger } from "../lib/logger"; +import { parseBody } from "../lib/validate"; import { downloadFile } from "../lib/storage"; import { loadActiveVersion } from "../lib/documentVersions"; import { normalizeDocxZipPaths } from "../lib/convert"; @@ -10,20 +14,19 @@ import { type ChatMessage, type TabularCellStore, } from "../lib/chatTools"; -import { - completeText, - providerForModel, - streamChatWithTools, - type Provider, - type UserApiKeys, -} from "../lib/llm"; -import { getUserModelSettings } from "../lib/userSettings"; +import { completeText, streamChatWithTools } from "../lib/llm"; +import { getUserApiKeys, getUserModelSettings } from "../lib/userSettings"; import { checkProjectAccess, ensureReviewAccess, - filterAccessibleDocumentIds, listAccessibleProjectIds, } from "../lib/access"; +import { parseLlmJson } from "../lib/chatTools/parseLlmJson"; +import { + TabularCellSchema, + TabularCellLineSchema, + TabularCitationsArraySchema, +} from "../lib/chatTools/llm-schemas"; function formatPromptSuffix(format?: string, tags?: string[]): string { switch (format) { @@ -50,23 +53,102 @@ function formatPromptSuffix(format?: string, tags?: string[]): string { } } -export const tabularRouter = Router(); - -function providerLabel(provider: Provider): string { - if (provider === "claude") return "Anthropic"; - if (provider === "openai") return "OpenAI"; - return "Gemini"; +/** + * runBoundedFanOut — bounded concurrency fan-out for tabular generation. + * + * Rejects requests where docs × columns > cellCap (default 200) before + * any processFn is called (ASVS V5 input validation — DoS mitigation T-05-12). + * Caps concurrent processFn calls at `concurrency` (default 5) via p-limit + * (T-05-13). + * + * p-limit is ESM-only; we load it via dynamic import (mirrors pdfQueue.ts). + */ +export async function runBoundedFanOut<TDoc>(deps: { + docs: TDoc[]; + columnsCount: number; + cellCap?: number; + concurrency?: number; + processFn: (doc: TDoc) => Promise<void>; +}): Promise<{ ok: true } | { ok: false; code: number; detail: string }> { + const cap = deps.cellCap ?? 200; + const conc = deps.concurrency ?? 5; + const totalCells = deps.docs.length * deps.columnsCount; + if (totalCells > cap) { + return { + ok: false, + code: 400, + detail: `Request exceeds maximum of ${cap} cells (${deps.docs.length} docs × ${deps.columnsCount} columns = ${totalCells}). Reduce document count or column count.`, + }; + } + // p-limit is ESM-only; use dynamic import (mirrors pdfQueue.ts pattern) + const { default: pLimit } = await import("p-limit"); + const limit = pLimit(conc); + await Promise.all(deps.docs.map((doc) => limit(() => deps.processFn(doc)))); + return { ok: true }; } -function missingModelApiKey(model: string, apiKeys: UserApiKeys) { - const provider = providerForModel(model); - if (apiKeys[provider]?.trim()) return null; - return { - provider, - model, - detail: `${providerLabel(provider)} API key is required to use ${model}. Add an API key or select a different tabular review model.`, - }; -} +const CreateReviewSchema = z.object({ + title: z.string().optional(), + document_ids: z.array(z.string()).min(1), + columns_config: z.array( + z.object({ + index: z.number().int(), + name: z.string(), + prompt: z.string(), + }), + ), + workflow_id: z.string().uuid().optional(), + project_id: z.string().uuid().optional(), +}); + +const RegenerateCellSchema = z.object({ + document_id: z.string().min(1), + column_index: z.number().int(), +}); + +const TabularChatSchema = z.object({ + messages: z.array( + z.object({ + role: z.enum(["user", "assistant"]), + content: z.string(), + }), + ).min(1), + chat_id: z.string().uuid().optional(), + review_title: z.string().optional(), + project_name: z.string().optional(), +}); + +const PatchReviewSchema = z + .object({ + title: z.string().optional(), + columns_config: z.array(z.unknown()).optional(), + project_id: z.string().uuid().optional().nullable(), + shared_with: z.array(z.string().email()).optional(), + document_ids: z.array(z.string()).optional(), + }) + .partial(); + +const PromptSchema = z.object({ + title: z.string().trim().min(1, "title is required"), + format: z + .enum([ + "text", + "bulleted_list", + "number", + "percentage", + "monetary_amount", + "currency", + "yes_no", + "date", + "tag", + ]) + .optional() + .default("text"), + documentName: z.string().trim().optional().default(""), + tags: z.array(z.string()).optional().default([]), +}); + +export const tabularRouter = Router(); // GET /tabular-review tabularRouter.get("/", requireAuth, async (req, res) => { @@ -127,7 +209,7 @@ tabularRouter.get("/", requireAuth, async (req, res) => { ? db .from("tabular_reviews") .select("*") - .filter("shared_with", "cs", JSON.stringify([userEmail])) + .contains("shared_with", JSON.stringify([userEmail])) .neq("user_id", userId) .order("created_at", { ascending: false }) : Promise.resolve({ @@ -140,15 +222,9 @@ tabularRouter.get("/", requireAuth, async (req, res) => { // commonly the tabular_reviews.shared_with column hasn't been migrated // yet. Log and continue so the user still sees their own reviews. if (sharedErr) - console.warn( - "[tabular] shared-by-project query failed:", - sharedErr.message, - ); + logger.warn({ err: sharedErr }, "[tabular] shared-by-project query failed"); if (sharedDirectErr) - console.warn( - "[tabular] shared-by-email query failed:", - sharedDirectErr.message, - ); + logger.warn({ err: sharedDirectErr }, "[tabular] shared-by-email query failed"); const seen = new Set<string>(); const reviews: Record<string, unknown>[] = []; for (const r of [ @@ -162,23 +238,21 @@ tabularRouter.get("/", requireAuth, async (req, res) => { reviews.push(r as Record<string, unknown>); } - // Fetch distinct document counts per review + // CLEAN-28: aggregation via RPC (single query). EXPLAIN confirms + // idx_tabular_cells_review (review_id, document_id, column_index) leftmost-prefix + // index is used — no new index needed. const reviewIds = reviews.map((r) => (r as { id: string }).id); let docCounts: Record<string, number> = {}; if (reviewIds.length > 0) { - const { data: cells } = await db - .from("tabular_cells") - .select("review_id, document_id") - .in("review_id", reviewIds); - if (cells) { - const seen = new Set<string>(); - for (const cell of cells) { - const key = `${cell.review_id}:${cell.document_id}`; - if (!seen.has(key)) { - seen.add(key); - docCounts[cell.review_id] = - (docCounts[cell.review_id] ?? 0) + 1; - } + const { data: counts, error: cErr } = await db.rpc( + "select_review_doc_counts", + { review_ids: reviewIds }, + ); + if (cErr) { + logger.warn({ err: cErr }, "[tabular] doc-counts rpc failed"); + } else if (counts) { + for (const row of counts as { review_id: string; doc_count: number }[]) { + docCounts[row.review_id] = Number(row.doc_count); } } } @@ -195,8 +269,10 @@ tabularRouter.get("/", requireAuth, async (req, res) => { tabularRouter.post("/", requireAuth, async (req, res) => { const userId = res.locals.userId as string; const userEmail = res.locals.userEmail as string | undefined; + const body = parseBody(CreateReviewSchema, req, res); + if (!body) return; const { title, document_ids, columns_config, workflow_id, project_id } = - req.body as { + body as unknown as { title?: string; document_ids: string[]; columns_config: { index: number; name: string; prompt: string }[]; @@ -215,14 +291,6 @@ tabularRouter.post("/", requireAuth, async (req, res) => { if (!access.ok) return void res.status(404).json({ detail: "Project not found" }); } - const allowedDocumentIds = Array.isArray(document_ids) - ? await filterAccessibleDocumentIds( - document_ids, - userId, - userEmail, - db, - ) - : []; const { data: review, error } = await db .from("tabular_reviews") .insert({ @@ -239,7 +307,7 @@ tabularRouter.post("/", requireAuth, async (req, res) => { .status(500) .json({ detail: error?.message ?? "Failed to create review" }); - const cells = allowedDocumentIds.flatMap((docId) => + const cells = document_ids.flatMap((docId) => columns_config.map((col) => ({ review_id: review.id, document_id: docId, @@ -255,20 +323,9 @@ tabularRouter.post("/", requireAuth, async (req, res) => { // POST /tabular-review/prompt (must come before /:reviewId routes) tabularRouter.post("/prompt", requireAuth, async (req, res) => { const userId = res.locals.userId as string; - const title = - typeof req.body.title === "string" ? req.body.title.trim() : ""; - if (!title) - return void res.status(400).json({ detail: "title is required" }); - - const format: string = - typeof req.body.format === "string" ? req.body.format : "text"; - const documentName: string = - typeof req.body.documentName === "string" - ? req.body.documentName.trim() - : ""; - const tags: string[] = Array.isArray(req.body.tags) - ? req.body.tags.filter((t: unknown) => typeof t === "string") - : []; + const body = parseBody(PromptSchema, req, res); + if (!body) return; + const { title, format, documentName, tags } = body; const formatDescriptions: Record<string, string> = { text: "free-form text", @@ -298,7 +355,7 @@ tabularRouter.post("/prompt", requireAuth, async (req, res) => { `format handling is applied separately and must not be duplicated inside the prompt text.`; try { - const { title_model, api_keys } = await getUserModelSettings(userId); + const { title_model, api_keys } = await getUserModelSettings(userId, undefined, { route: req.path, requestId: req.id }); const raw = await completeText({ model: title_model, systemPrompt: @@ -307,17 +364,16 @@ tabularRouter.post("/prompt", requireAuth, async (req, res) => { maxTokens: 512, apiKeys: api_keys, }); - const parsed = JSON.parse( - raw - .replace(/^```(?:json)?\n?/i, "") - .replace(/\n?```$/, "") - .trim(), - ) as { prompt?: unknown }; - if (typeof parsed.prompt === "string" && parsed.prompt.trim()) { - res.json({ prompt: parsed.prompt.trim(), source: "llm" }); - } else { - res.status(502).json({ detail: "LLM returned an empty prompt" }); + const rawText = raw + .replace(/^```(?:json)?\n?/i, "") + .replace(/\n?```$/, "") + .trim(); + const promptResult = parseLlmJson(rawText, z.object({ prompt: z.string().min(1) })); + if (!promptResult.ok) { + logger.warn({ err: promptResult.error }, "[tabular] /prompt parse failed"); + return void res.status(502).json({ detail: "LLM returned malformed JSON" }); } + res.json({ prompt: promptResult.data.prompt.trim(), source: "llm" }); } catch { res.status(502).json({ detail: "Failed to generate prompt from LLM" }); } @@ -394,20 +450,13 @@ tabularRouter.get("/:reviewId/people", requireAuth, async (req, res) => { : [] ).map((e) => (e ?? "").toLowerCase()); - // Same pattern as /projects/:id/people: walk auth.users to map emails - // to user_ids, then pull display_names from user_profiles by user_id. - const { data: usersData } = await db.auth.admin.listUsers({ - perPage: 1000, - }); - const allUsers = usersData?.users ?? []; - const userByEmail = new Map<string, { id: string; email: string }>(); - const userById = new Map<string, { id: string; email: string }>(); - for (const u of allUsers) { - if (!u.email) continue; - const lower = u.email.toLowerCase(); - userByEmail.set(lower, { id: u.id, email: u.email }); - userById.set(u.id, { id: u.id, email: u.email }); - } + // Resolve shared-with emails to user records via RPC (CLEAN-15). + // get_auth_user_by_email is SECURITY DEFINER and O(log N) — replaces the + // listUsers({ perPage: 1000 }) walk that silently truncates above 1000. + const [userByEmail, ownerRecord] = await Promise.all([ + getUsersByEmails(sharedWith), + getUserById(review.user_id as string), + ]); const memberUserIds: string[] = []; for (const email of sharedWith) { @@ -433,18 +482,19 @@ tabularRouter.get("/:reviewId/people", requireAuth, async (req, res) => { } } - const ownerInfo = userById.get(review.user_id as string); res.json({ owner: { user_id: review.user_id, - email: ownerInfo?.email ?? null, + email: ownerRecord?.email ?? null, display_name: profileByUserId.get(review.user_id as string) ?? null, }, - members: sharedWith.map((email) => { - const u = userByEmail.get(email); - const display_name = u ? (profileByUserId.get(u.id) ?? null) : null; - return { email, display_name }; - }), + members: sharedWith + .filter((email) => userByEmail.has(email)) + .map((email) => { + const u = userByEmail.get(email)!; + const display_name = profileByUserId.get(u.id) ?? null; + return { email, display_name }; + }), }); }); @@ -453,20 +503,21 @@ tabularRouter.patch("/:reviewId", requireAuth, async (req, res) => { const userId = res.locals.userId as string; const userEmail = res.locals.userEmail as string | undefined; const { reviewId } = req.params; + const patchBody = parseBody(PatchReviewSchema, req, res); + if (!patchBody) return; const updates: Record<string, unknown> = {}; - if (req.body.title != null) updates.title = req.body.title; - if (req.body.columns_config != null) - updates.columns_config = req.body.columns_config; - if (req.body.project_id !== undefined) - updates.project_id = req.body.project_id; + if (patchBody.title != null) updates.title = patchBody.title; + if (patchBody.columns_config != null) + updates.columns_config = patchBody.columns_config; + if (patchBody.project_id !== undefined) + updates.project_id = patchBody.project_id; // shared_with edits are owner-only — gated below after we know who's // making the call. Normalize lowercase + dedupe + drop empties. let sharedWithUpdate: string[] | undefined; - if (Array.isArray(req.body.shared_with)) { + if (Array.isArray(patchBody.shared_with)) { const seen = new Set<string>(); const cleaned: string[] = []; - for (const raw of req.body.shared_with) { - if (typeof raw !== "string") continue; + for (const raw of patchBody.shared_with) { const e = raw.trim().toLowerCase(); if (!e || seen.has(e)) continue; seen.add(e); @@ -512,8 +563,8 @@ tabularRouter.patch("/:reviewId", requireAuth, async (req, res) => { }); if ( - Array.isArray(req.body.columns_config) || - Array.isArray(req.body.document_ids) + Array.isArray(patchBody.columns_config) || + Array.isArray(patchBody.document_ids) ) { const { data: existingCells } = await db .from("tabular_cells") @@ -527,26 +578,12 @@ tabularRouter.patch("/:reviewId", requireAuth, async (req, res) => { let documentIds: string[]; - if (Array.isArray(req.body.document_ids)) { + if (Array.isArray(patchBody.document_ids)) { // document_ids is the new source of truth — delete removed docs' cells - const requestedDocIds = req.body.document_ids as string[]; + const newDocIds = patchBody.document_ids as string[]; const existingDocIds = (existingCells ?? []).map( (cell) => cell.document_id, ); - const existingDocIdSet = new Set(existingDocIds); - const newDocCandidates = requestedDocIds.filter( - (id) => !existingDocIdSet.has(id), - ); - const newDocAllowed = await filterAccessibleDocumentIds( - newDocCandidates, - userId, - userEmail, - db, - ); - const newDocAllowedSet = new Set(newDocAllowed); - const newDocIds = requestedDocIds.filter( - (id) => existingDocIdSet.has(id) || newDocAllowedSet.has(id), - ); const removedDocIds = existingDocIds.filter( (id) => !newDocIds.includes(id), ); @@ -580,8 +617,8 @@ tabularRouter.patch("/:reviewId", requireAuth, async (req, res) => { } } - const activeColumns = Array.isArray(req.body.columns_config) - ? req.body.columns_config + const activeColumns = Array.isArray(patchBody.columns_config) + ? patchBody.columns_config : (updatedReview.columns_config ?? []); const newCells = documentIds.flatMap((documentId) => activeColumns @@ -616,28 +653,32 @@ tabularRouter.delete("/:reviewId", requireAuth, async (req, res) => { const userId = res.locals.userId as string; const { reviewId } = req.params; const db = createServerSupabase(); - const { error } = await db + const { data, error } = await db .from("tabular_reviews") .delete() .eq("id", reviewId) - .eq("user_id", userId); + .eq("user_id", userId) + .select("id"); if (error) return void res.status(500).json({ detail: error.message }); + if (!data || data.length === 0) + return void res.status(404).json({ detail: "Review not found" }); res.status(204).send(); }); // POST /tabular-review/:reviewId/clear-cells +const ClearCellsSchema = z.object({ + document_ids: z.array(z.string()).min(1), +}); + // Reset cells to an empty/pending state for the given document_ids. Does not // delete the rows — it blanks `content` and sets `status` back to "pending". tabularRouter.post("/:reviewId/clear-cells", requireAuth, async (req, res) => { const userId = res.locals.userId as string; const userEmail = res.locals.userEmail as string | undefined; const { reviewId } = req.params; - const { document_ids } = req.body as { document_ids?: string[] }; - - if (!Array.isArray(document_ids) || document_ids.length === 0) - return void res - .status(400) - .json({ detail: "document_ids is required" }); + const clearBody = parseBody(ClearCellsSchema, req, res); + if (!clearBody) return; + const { document_ids } = clearBody; const db = createServerSupabase(); const { data: review, error: reviewError } = await db @@ -664,19 +705,14 @@ tabularRouter.post("/:reviewId/clear-cells", requireAuth, async (req, res) => { tabularRouter.post( "/:reviewId/regenerate-cell", requireAuth, + llmRateLimiter, async (req, res) => { const userId = res.locals.userId as string; const userEmail = res.locals.userEmail as string | undefined; const { reviewId } = req.params; - const { document_id, column_index } = req.body as { - document_id: string; - column_index: number; - }; - - if (!document_id || column_index == null) - return void res - .status(400) - .json({ detail: "document_id and column_index are required" }); + const cellBody = parseBody(RegenerateCellSchema, req, res); + if (!cellBody) return; + const { document_id, column_index } = cellBody; const db = createServerSupabase(); const { data: review, error: reviewError } = await db @@ -702,14 +738,6 @@ tabularRouter.post( if (!column) return void res.status(400).json({ detail: "Column not found" }); - const docAllowed = await filterAccessibleDocumentIds( - [document_id], - userId, - userEmail, - db, - ); - if (docAllowed.length === 0) - return void res.status(404).json({ detail: "Document not found" }); const { data: doc } = await db .from("documents") .select("id, filename, file_type") @@ -719,18 +747,6 @@ tabularRouter.post( return void res.status(404).json({ detail: "Document not found" }); const docActive = await loadActiveVersion(document_id, db); - const { tabular_model, api_keys } = await getUserModelSettings( - userId, - db, - ); - const missingKey = missingModelApiKey(tabular_model, api_keys); - if (missingKey) { - return void res.status(422).json({ - code: "missing_api_key", - ...missingKey, - }); - } - await db .from("tabular_cells") .update({ status: "generating", content: null }) @@ -748,15 +764,17 @@ tabularRouter.post( ? await extractPdfMarkdown(buf) : await extractDocxMarkdown(buf); } catch (err) { - console.error( - `[regenerate-cell] extraction error doc=${document_id}`, - err, - ); + logger.error({ err, documentId: document_id }, "[regenerate-cell] extraction error"); } } } - const result = await queryTabularCell( + const { tabular_model, api_keys } = await getUserModelSettings( + userId, + db, + { route: req.path, requestId: req.id }, + ); + const result = await queryGemini( tabular_model, doc.filename as string, markdown, @@ -788,7 +806,7 @@ tabularRouter.post( ); // POST /tabular-review/:reviewId/generate -tabularRouter.post("/:reviewId/generate", requireAuth, async (req, res) => { +tabularRouter.post("/:reviewId/generate", requireAuth, llmRateLimiter, async (req, res) => { const userId = res.locals.userId as string; const userEmail = res.locals.userEmail as string | undefined; const { reviewId } = req.params; @@ -824,19 +842,12 @@ tabularRouter.post("/:reviewId/generate", requireAuth, async (req, res) => { cellMap.set(`${cell.document_id}:${cell.column_index}`, cell); const docIds = [...new Set((cells ?? []).map((c) => c.document_id))]; - const allowedDocIds = new Set( - await filterAccessibleDocumentIds(docIds, userId, userEmail, db), - ); let docs: Record<string, unknown>[] = []; if (docIds.length > 0) { - const filteredIds = docIds.filter((id) => allowedDocIds.has(id)); - const { data } = - filteredIds.length > 0 - ? await db - .from("documents") - .select("id, filename, file_type, page_count") - .in("id", filteredIds) - : { data: [] as Record<string, unknown>[] }; + const { data } = await db + .from("documents") + .select("id, filename, file_type, page_count") + .in("id", docIds); docs = data ?? []; } else if (review.project_id) { const { data } = await db @@ -847,12 +858,13 @@ tabularRouter.post("/:reviewId/generate", requireAuth, async (req, res) => { docs = data ?? []; } - const { tabular_model, api_keys } = await getUserModelSettings(userId, db); - const missingKey = missingModelApiKey(tabular_model, api_keys); - if (missingKey) { - return void res.status(422).json({ - code: "missing_api_key", - ...missingKey, + const { tabular_model, api_keys } = await getUserModelSettings(userId, db, { route: req.path, requestId: req.id }); + + // Cell-count guard: reject before SSE headers are flushed so a JSON 400 + // is still deliverable (T-05-12 — DoS mitigation, ASVS V5 input validation). + if (docs.length * columns.length > 200) { + return void res.status(400).json({ + detail: `Request exceeds maximum of 200 cells (${docs.length} docs × ${columns.length} columns = ${docs.length * columns.length}). Reduce document count or column count.`, }); } @@ -865,8 +877,10 @@ tabularRouter.post("/:reviewId/generate", requireAuth, async (req, res) => { const write = (line: string) => res.write(line); try { - await Promise.all( - docs.map(async (doc) => { + await runBoundedFanOut({ + docs, + columnsCount: columns.length, + processFn: async (doc) => { const docId = doc.id as string; const filename = doc.filename as string; let markdown = ""; @@ -881,10 +895,7 @@ tabularRouter.post("/:reviewId/generate", requireAuth, async (req, res) => { ? await extractPdfMarkdown(buf) : await extractDocxMarkdown(buf); } catch (err) { - console.error( - `[tabular/generate] extraction error doc=${docId}`, - err, - ); + logger.error({ err, docId }, "[tabular/generate] extraction error"); } } } @@ -920,7 +931,7 @@ tabularRouter.post("/:reviewId/generate", requireAuth, async (req, res) => { // Single LLM call for all columns, streaming one JSON line per column const receivedColumns = new Set<number>(); try { - await queryTabularAllColumns( + await queryGeminiAllColumns( tabular_model, filename, markdown, @@ -941,12 +952,11 @@ tabularRouter.post("/:reviewId/generate", requireAuth, async (req, res) => { ); }, api_keys, + write, + docId, ); } catch (err) { - console.error( - `[tabular/generate] queryTabularAllColumns error doc=${docId}`, - err, - ); + logger.error({ err, docId }, "[tabular/generate] queryGeminiAllColumns error"); } // Mark any columns the LLM didn't return as error @@ -963,12 +973,12 @@ tabularRouter.post("/:reviewId/generate", requireAuth, async (req, res) => { ); } } - }), - ); + }, + }); write("data: [DONE]\n\n"); } catch (err) { - console.error("[tabular/generate] stream error", err); + logger.error({ err }, "[tabular/generate] stream error"); try { write( `data: ${JSON.stringify({ type: "error", message: String(err) })}\n\ndata: [DONE]\n\n`, @@ -1083,21 +1093,31 @@ type TabularParsedCitation = { const TABULAR_CITATIONS_BLOCK_RE = /<CITATIONS>\s*([\s\S]*?)\s*<\/CITATIONS>/; -function parseTabularCitations(text: string): TabularParsedCitation[] { +function parseTabularCitations( + text: string, + write?: (s: string) => void, +): TabularParsedCitation[] { const match = text.match(TABULAR_CITATIONS_BLOCK_RE); if (!match) return []; - try { - return JSON.parse(match[1]) as TabularParsedCitation[]; - } catch { + const citationsResult = parseLlmJson(match[1], TabularCitationsArraySchema); + if (!citationsResult.ok) { + if (write) { + write( + `data: ${JSON.stringify({ type: "citations_parse_error", error: citationsResult.error })}\n\n`, + ); + } + logger.warn({ err: citationsResult.error }, "[tabular] citations parse failed"); return []; } + return citationsResult.data as TabularParsedCitation[]; } function extractTabularAnnotations( fullText: string, tabularStore: TabularCellStore, + write?: (s: string) => void, ) { - return parseTabularCitations(fullText).map((c) => ({ + return parseTabularCitations(fullText, write).map((c) => ({ type: "tabular_citation" as const, ref: c.ref, col_index: c.col_index, @@ -1170,16 +1190,18 @@ Rules: // --------------------------------------------------------------------------- // POST /tabular-review/:reviewId/chat -tabularRouter.post("/:reviewId/chat", requireAuth, async (req, res) => { +tabularRouter.post("/:reviewId/chat", requireAuth, llmRateLimiter, async (req, res) => { const userId = res.locals.userId as string; const userEmail = res.locals.userEmail as string | undefined; const { reviewId } = req.params; + const chatBody = parseBody(TabularChatSchema, req, res); + if (!chatBody) return; const { messages, chat_id: existingChatId, review_title: clientReviewTitle, project_name: clientProjectName, - } = req.body as { + } = chatBody as unknown as { messages: ChatMessage[]; chat_id?: string; review_title?: string; @@ -1246,15 +1268,6 @@ tabularRouter.post("/:reviewId/chat", requireAuth, async (req, res) => { ), }; - const { tabular_model, api_keys } = await getUserModelSettings(userId, db); - const missingKey = missingModelApiKey(tabular_model, api_keys); - if (missingKey) { - return void res.status(422).json({ - code: "missing_api_key", - ...missingKey, - }); - } - // Create or verify chat record let chatId = existingChatId ?? null; let chatTitle: string | null = null; @@ -1312,6 +1325,8 @@ tabularRouter.post("/:reviewId/chat", requireAuth, async (req, res) => { write(`data: ${JSON.stringify({ type: "chat_id", chatId })}\n\n`); } + const apiKeys = await getUserApiKeys(userId, db, { route: req.path, requestId: req.id }); + try { const { fullText, events } = await runLLMStream({ apiMessages, @@ -1323,9 +1338,8 @@ tabularRouter.post("/:reviewId/chat", requireAuth, async (req, res) => { extraTools: TABULAR_TOOLS, tabularStore, buildCitations: (text) => - extractTabularAnnotations(text, tabularStore), - model: tabular_model, - apiKeys: api_keys, + extractTabularAnnotations(text, tabularStore, write), + apiKeys, }); const annotations = extractTabularAnnotations(fullText, tabularStore); @@ -1345,7 +1359,7 @@ tabularRouter.post("/:reviewId/chat", requireAuth, async (req, res) => { // Generate title on first exchange if (chatId && isFirstExchange && !chatTitle && lastUser.content) { - const { title_model } = await getUserModelSettings(userId, db); + const { title_model } = await getUserModelSettings(userId, db, { route: req.path, requestId: req.id }); const title = await generateChatTitle( title_model, lastUser.content, @@ -1353,7 +1367,7 @@ tabularRouter.post("/:reviewId/chat", requireAuth, async (req, res) => { reviewTitle: clientReviewTitle ?? review.title ?? null, projectName: clientProjectName ?? null, }, - api_keys, + apiKeys, ); if (title) { await db @@ -1366,7 +1380,7 @@ tabularRouter.post("/:reviewId/chat", requireAuth, async (req, res) => { } } } catch (err) { - console.error("[tabular/chat] error", err); + logger.error({ err }, "[tabular/chat] error"); try { write( `data: ${JSON.stringify({ type: "error", message: String(err) })}\n\n`, @@ -1382,6 +1396,9 @@ tabularRouter.post("/:reviewId/chat", requireAuth, async (req, res) => { function parseCellContent( raw: unknown, + write?: (s: string) => void, + colIndex?: number, + docId?: string, ): { summary: string; flag?: string; reasoning?: string } | null { if (!raw) return null; if (typeof raw === "object" && raw !== null && "summary" in raw) { @@ -1401,30 +1418,31 @@ function parseCellContent( }; } if (typeof raw === "string") { - try { - const p = JSON.parse(raw) as { - summary?: unknown; - value?: unknown; - flag?: unknown; - reasoning?: unknown; - }; - return { - summary: String(p.summary ?? p.value ?? "").trim(), - flag: (["green", "grey", "yellow", "red"] as const).includes( - p.flag as "green", - ) - ? (p.flag as string) - : undefined, - reasoning: typeof p.reasoning === "string" ? p.reasoning : "", - }; - } catch { - return { summary: raw, flag: "grey", reasoning: "" }; + const rawCellJson = raw; + const cellResult = parseLlmJson(rawCellJson, TabularCellSchema); + if (!cellResult.ok) { + logger.warn( + { err: cellResult.error, colIndex, docId }, + "[tabular] per-cell parse failed", + ); + if (write) { + write( + `data: ${JSON.stringify({ type: "tabular_cell_parse_error", col_index: colIndex ?? -1, doc_id: docId ?? "", error: cellResult.error })}\n\n`, + ); + } + // Preserve LLM output verbatim as summary so the user still sees something + return { summary: rawCellJson.slice(0, 2000), flag: "grey", reasoning: "" }; } + return { + summary: String(cellResult.data.summary ?? cellResult.data.value ?? "").trim(), + flag: cellResult.data.flag, + reasoning: typeof cellResult.data.reasoning === "string" ? cellResult.data.reasoning : "", + }; } return null; } -async function queryTabularCell( +async function queryGemini( model: string, filename: string, documentText: string, @@ -1432,6 +1450,9 @@ async function queryTabularCell( format?: string, tags?: string[], apiKeys?: import("../lib/llm").UserApiKeys, + write?: (s: string) => void, + colIndex?: number, + docId?: string, ) { const suffix = formatPromptSuffix(format as never, tags); const fullPrompt = `${columnPrompt}${suffix} If not found, state "Not Found". Leave all reasoning and explanation in the "reasoning" field only.`; @@ -1453,41 +1474,40 @@ The "summary" field must contain only the extracted value with inline citations apiKeys, }); } catch (err) { - console.error("[queryTabularCell] completion failed", err); + logger.error({ err }, "[queryGemini] completion failed"); return null; } - try { - const parsed = JSON.parse( - raw - .replace(/^```(?:json)?\n?/i, "") - .replace(/\n?```$/, "") - .trim(), - ) as { - summary?: unknown; - value?: unknown; - flag?: unknown; - reasoning?: unknown; - }; - return { - summary: - String(parsed.summary ?? parsed.value ?? "").trim() || - "Not addressed", - flag: (["green", "grey", "yellow", "red"] as const).includes( - parsed.flag as "green", - ) - ? (parsed.flag as "green") - : "grey", - reasoning: String(parsed.reasoning ?? ""), - }; - } catch { - return raw.trim() + const rawCellJson = raw + .replace(/^```(?:json)?\n?/i, "") + .replace(/\n?```$/, "") + .trim(); + const cellResult = parseLlmJson(rawCellJson, TabularCellSchema); + if (!cellResult.ok) { + logger.warn( + { err: cellResult.error, colIndex, docId }, + "[tabular] queryGemini per-cell parse failed", + ); + if (write) { + write( + `data: ${JSON.stringify({ type: "tabular_cell_parse_error", col_index: colIndex ?? -1, doc_id: docId ?? "", error: cellResult.error })}\n\n`, + ); + } + // Persist raw text as summary so the user still sees the LLM output verbatim + return rawCellJson ? { - summary: raw.trim().slice(0, 500), + summary: rawCellJson.slice(0, 2000), flag: "grey" as const, reasoning: "", } : null; } + return { + summary: + String(cellResult.data.summary ?? cellResult.data.value ?? "").trim() || + "Not addressed", + flag: cellResult.data.flag ?? "grey", + reasoning: String(cellResult.data.reasoning ?? ""), + }; } async function generateChatTitle( @@ -1579,13 +1599,15 @@ type Column = { tags?: string[]; }; -async function queryTabularAllColumns( +async function queryGeminiAllColumns( model: string, filename: string, documentText: string, columns: Column[], onResult: (columnIndex: number, result: CellResult) => Promise<void>, apiKeys?: import("../lib/llm").UserApiKeys, + write?: (s: string) => void, + docId?: string, ): Promise<void> { const columnsDesc = columns .map((col) => { @@ -1617,28 +1639,28 @@ Rules: const processLine = async (line: string) => { const trimmed = line.trim(); if (!trimmed) return; - try { - const parsed = JSON.parse(trimmed) as { - column_index?: unknown; - summary?: unknown; - flag?: unknown; - reasoning?: unknown; - }; - if (typeof parsed.column_index !== "number") return; - const col = columns.find((c) => c.index === parsed.column_index); - if (!col) return; - await onResult(parsed.column_index, { - summary: String(parsed.summary ?? "").trim() || "Not addressed", - flag: (["green", "grey", "yellow", "red"] as const).includes( - parsed.flag as "green", - ) - ? (parsed.flag as CellResult["flag"]) - : "grey", - reasoning: String(parsed.reasoning ?? ""), - }); - } catch { - // malformed line — skip + const lineResult = parseLlmJson(trimmed, TabularCellLineSchema); + if (!lineResult.ok) { + // col_index unknown at this point (couldn't parse the line) + logger.warn( + { err: lineResult.error, docId }, + "[tabular] queryGeminiAllColumns line parse failed", + ); + if (write) { + write( + `data: ${JSON.stringify({ type: "tabular_cell_parse_error", col_index: -1, doc_id: docId ?? "", error: lineResult.error })}\n\n`, + ); + } + return; } + const { column_index } = lineResult.data; + const col = columns.find((c) => c.index === column_index); + if (!col) return; + await onResult(column_index, { + summary: String(lineResult.data.summary ?? lineResult.data.value ?? "").trim() || "Not addressed", + flag: lineResult.data.flag ?? "grey", + reasoning: String(lineResult.data.reasoning ?? ""), + }); }; try { @@ -1664,7 +1686,7 @@ Rules: }, }); } catch (err) { - console.error("[queryTabularAllColumns] stream failed", err); + logger.error({ err }, "[queryGeminiAllColumns] stream failed"); } if (contentBuffer.trim()) pending.push(processLine(contentBuffer)); diff --git a/backend/src/routes/user.ts b/backend/src/routes/user.ts index 0df2021d6..55d486b37 100644 --- a/backend/src/routes/user.ts +++ b/backend/src/routes/user.ts @@ -1,263 +1,183 @@ import { Router } from "express"; +import { z } from "zod"; import { requireAuth } from "../middleware/auth"; import { createServerSupabase } from "../lib/supabase"; -import { DEFAULT_TABULAR_MODEL, resolveModel } from "../lib/llm"; +import { encryptApiKey } from "../lib/crypto"; +import { logger } from "../lib/logger"; +import { signRestoreToken, verifyRestoreToken } from "../lib/restoreTokens"; import { - type ApiKeyStatus, - getUserApiKeyStatus, - hasEnvApiKey, - normalizeApiKeyProvider, - saveUserApiKey, -} from "../lib/userApiKeys"; + markSoftDelete, + clearSoftDelete, + banUser, + unbanUser, + enqueueDeletionJob, + consumeRestoreToken, + DELETE_GRACE_DAYS, +} from "../lib/accountDeletion"; export const userRouter = Router(); -const MONTHLY_CREDIT_LIMIT = 999999; - -type UserProfileRow = { - display_name: string | null; - organisation: string | null; - message_credits_used: number; - credits_reset_date: string; - tier: string; - tabular_model: string; -}; - -function serializeProfile( - row: UserProfileRow, - apiKeyStatus?: ApiKeyStatus, -) { - const creditsUsed = row.message_credits_used ?? 0; - return { - displayName: row.display_name, - organisation: row.organisation, - messageCreditsUsed: creditsUsed, - creditsResetDate: row.credits_reset_date, - creditsRemaining: Math.max(MONTHLY_CREDIT_LIMIT - creditsUsed, 0), - tier: row.tier || "Free", - tabularModel: resolveModel(row.tabular_model, DEFAULT_TABULAR_MODEL), - ...(apiKeyStatus ? { apiKeyStatus } : {}), - }; -} - -function validateProfilePayload(body: unknown): - | { - ok: true; - update: { - display_name?: string | null; - organisation?: string | null; - tabular_model?: string; - updated_at: string; - }; - } - | { ok: false; detail: string } { - if (!body || typeof body !== "object" || Array.isArray(body)) { - return { ok: false, detail: "Expected a JSON object" }; - } - - const raw = body as Record<string, unknown>; - const allowedFields = new Set([ - "displayName", - "organisation", - "tabularModel", - ]); - const invalidField = Object.keys(raw).find((key) => !allowedFields.has(key)); - if (invalidField) { - return { ok: false, detail: `Unsupported profile field: ${invalidField}` }; - } - - const update: { - display_name?: string | null; - organisation?: string | null; - tabular_model?: string; - updated_at: string; - } = { updated_at: new Date().toISOString() }; - - if ("displayName" in raw) { - if (raw.displayName !== null && typeof raw.displayName !== "string") { - return { ok: false, detail: "displayName must be a string or null" }; - } - update.display_name = raw.displayName?.trim() || null; - } - - if ("organisation" in raw) { - if (raw.organisation !== null && typeof raw.organisation !== "string") { - return { ok: false, detail: "organisation must be a string or null" }; - } - update.organisation = raw.organisation?.trim() || null; - } - - if ("tabularModel" in raw) { - if (typeof raw.tabularModel !== "string") { - return { ok: false, detail: "tabularModel must be a string" }; - } - const resolved = resolveModel(raw.tabularModel, ""); - if (!resolved) { - return { ok: false, detail: "Unsupported tabularModel" }; - } - update.tabular_model = resolved; - } - - return { ok: true, update }; -} +const patchApiKeySchema = z.object({ + provider: z.enum(["claude", "gemini"]), + key: z.string().min(1).nullable(), +}); -async function ensureProfileRow( - db: ReturnType<typeof createServerSupabase>, - userId: string, -) { +// POST /user/profile +userRouter.post("/profile", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const db = createServerSupabase(); const { error } = await db .from("user_profiles") .upsert( { user_id: userId }, { onConflict: "user_id", ignoreDuplicates: true }, ); - return error; -} - -async function loadProfile( - db: ReturnType<typeof createServerSupabase>, - userId: string, - options: { repairMissing?: boolean } = {}, -) { - let { data, error } = await db - .from("user_profiles") - .select( - "display_name, organisation, message_credits_used, credits_reset_date, tier, tabular_model", - ) - .eq("user_id", userId) - .maybeSingle(); - - if (error) return { data: null, error }; - if (!data) { - if (!options.repairMissing) { - return { data: null, error: new Error("Profile not found") }; - } - - const ensureError = await ensureProfileRow(db, userId); - if (ensureError) return { data: null, error: ensureError }; + if (error) return void res.status(500).json({ detail: error.message }); + res.json({ ok: true }); +}); - const created = await db - .from("user_profiles") - .select( - "display_name, organisation, message_credits_used, credits_reset_date, tier, tabular_model", - ) - .eq("user_id", userId) - .single(); - if (created.error) return { data: null, error: created.error }; - data = created.data; +// PATCH /user/api-keys +userRouter.patch("/api-keys", requireAuth, async (req, res) => { + const userId = res.locals.userId as string; + const parsed = patchApiKeySchema.safeParse(req.body); + if (!parsed.success) { + return void res.status(400).json({ + detail: "Invalid request body", + issues: parsed.error.issues, + }); } + const { provider, key } = parsed.data; + const db = createServerSupabase(); - let row = data as UserProfileRow; - if (row.credits_reset_date && new Date() > new Date(row.credits_reset_date)) { - const creditsResetDate = new Date(); - creditsResetDate.setDate(creditsResetDate.getDate() + 30); - const { data: resetData, error: resetError } = await db - .from("user_profiles") - .update({ - message_credits_used: 0, - credits_reset_date: creditsResetDate.toISOString(), - updated_at: new Date().toISOString(), - }) - .eq("user_id", userId) - .select( - "display_name, organisation, message_credits_used, credits_reset_date, tier, tabular_model", - ) - .single(); - - if (resetError) return { data: null, error: resetError }; - row = resetData as UserProfileRow; + const colCT = `${provider}_api_key_ciphertext`; + const colIV = `${provider}_api_key_iv`; + const colTag = `${provider}_api_key_auth_tag`; + + let payload: Record<string, unknown>; + if (key === null) { + payload = { + [colCT]: null, + [colIV]: null, + [colTag]: null, + updated_at: new Date().toISOString(), + }; + } else { + // Supabase JS serialises payloads via JSON.stringify, which renders raw + // Buffer values as `{}` and silently drops every byte. Send PostgreSQL's + // hex bytea text format so PostgREST stores the encrypted bytes exactly. + const enc = encryptApiKey(key); + payload = { + [colCT]: `\\x${enc.ciphertext.toString("hex")}`, + [colIV]: `\\x${enc.iv.toString("hex")}`, + [colTag]: `\\x${enc.authTag.toString("hex")}`, + updated_at: new Date().toISOString(), + }; } - return { data: serializeProfile(row), error: null }; -} - -// POST /user/profile -userRouter.post("/profile", requireAuth, async (_req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - const error = await ensureProfileRow(db, userId); + const { error } = await db + .from("user_profiles") + .update(payload) + .eq("user_id", userId); if (error) return void res.status(500).json({ detail: error.message }); - res.json({ ok: true }); + res.status(204).send(); }); -// GET /user/profile -userRouter.get("/profile", requireAuth, async (_req, res) => { +// GET /user/api-keys/status +userRouter.get("/api-keys/status", requireAuth, async (_req, res) => { const userId = res.locals.userId as string; const db = createServerSupabase(); - const { data, error } = await loadProfile(db, userId, { - repairMissing: true, - }); + const { data, error } = await db + .from("user_profiles") + .select("claude_api_key_ciphertext, gemini_api_key_ciphertext") + .eq("user_id", userId) + .single(); if (error) return void res.status(500).json({ detail: error.message }); - const apiKeyStatus = await getUserApiKeyStatus(userId, db); - res.json({ ...data, apiKeyStatus }); + res.json({ + has_claude: Boolean(data?.claude_api_key_ciphertext), + has_gemini: Boolean(data?.gemini_api_key_ciphertext), + }); }); -// PATCH /user/profile -userRouter.patch("/profile", requireAuth, async (req, res) => { +// DELETE /user/account — soft-delete + restore-token issuance (CLEAN-44) +// Replaces immediate hard-delete; worker (Plan 09) performs hard-delete after 30-day grace. +userRouter.delete("/account", requireAuth, async (req, res) => { const userId = res.locals.userId as string; - const parsed = validateProfilePayload(req.body); - if (!parsed.ok) return void res.status(400).json({ detail: parsed.detail }); - const db = createServerSupabase(); - const ensureError = await ensureProfileRow(db, userId); - if (ensureError) - return void res.status(500).json({ detail: ensureError.message }); - const { error: updateError } = await db - .from("user_profiles") - .update(parsed.update) - .eq("user_id", userId); - if (updateError) - return void res.status(500).json({ detail: updateError.message }); + // 1. Mark soft-delete (idempotent — returns existing deletedAt if already soft-deleted) + const softDelete = await markSoftDelete(userId, db); + if (!softDelete) { + return void res.status(500).json({ detail: "Failed to mark account for deletion" }); + } - const { data, error } = await loadProfile(db, userId); - if (error) return void res.status(500).json({ detail: error.message }); - const apiKeyStatus = await getUserApiKeyStatus(userId, db); - res.json({ ...data, apiKeyStatus }); -}); + const scheduledHardDeleteAt = new Date( + softDelete.deletedAt.getTime() + DELETE_GRACE_DAYS * 86_400_000, + ); -// GET /user/api-keys -userRouter.get("/api-keys", requireAuth, async (_req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - const status = await getUserApiKeyStatus(userId, db); - res.json(status); + // 2. Ban the auth user (idempotent — banning an already-banned user is a no-op for our purposes) + const banned = await banUser(userId, db); + if (!banned) { + return void res.status(500).json({ detail: "Failed to disable auth session" }); + } + + // 3. Enqueue hard-delete job (ON CONFLICT DO NOTHING — re-DELETE doesn't change the schedule) + const enqueued = await enqueueDeletionJob(userId, scheduledHardDeleteAt, db); + if (!enqueued) { + return void res.status(500).json({ detail: "Failed to enqueue deletion job" }); + } + + // 4. Issue a fresh restore token per Open Question 3 — re-DELETE re-issues; + // old tokens still verify until exp, but only one can consume the job (single-use enforcement). + const restoreToken = signRestoreToken(userId, scheduledHardDeleteAt); + + logger.info({ userId, scheduledHardDeleteAt: scheduledHardDeleteAt.toISOString() }, "[user] account soft-deleted"); + + res.json({ + deleted_at: softDelete.deletedAt.toISOString(), + scheduled_hard_delete_at: scheduledHardDeleteAt.toISOString(), + restore_token: restoreToken, + restore_url: `/user/account/restore?token=${restoreToken}`, + }); }); -// PUT /user/api-keys/:provider -userRouter.put("/api-keys/:provider", requireAuth, async (req, res) => { - const userId = res.locals.userId as string; - const provider = normalizeApiKeyProvider(req.params.provider); - if (!provider) - return void res.status(400).json({ detail: "Unsupported provider" }); +// POST /user/account/restore — token-authenticated (NOT requireAuth — user is banned) +// The HMAC token IS the auth. Three-way status-code trichotomy (H6 / RESEARCH.md Open Q5 RESOLVED): +// 401 — token-auth failure (verifyRestoreToken returns null: expired, tampered, malformed, missing) +// 410 — single-use replay (DB row exists, restore_token_used_at already set) +// 404 — no pending job (no account_deletion_jobs row for user) +userRouter.post("/account/restore", async (req, res) => { + const token = String(req.query.token ?? ""); + if (!token) { + return void res.status(401).json({ detail: "Missing token" }); + } - const apiKey = - typeof req.body?.api_key === "string" ? req.body.api_key : null; + const payload = verifyRestoreToken(token); + if (!payload) { + return void res.status(401).json({ detail: "Invalid or expired token" }); + } + + const userId = payload.user_id; const db = createServerSupabase(); - try { - if (hasEnvApiKey(provider)) { - return void res.status(409).json({ - detail: - "This provider is configured by the server environment and cannot be changed from the browser.", - }); + + // 1. Atomically consume the restore token (single-use enforcement — H6 trichotomy) + const consumeResult = await consumeRestoreToken(userId, db); + if (consumeResult.ok === false) { + if (consumeResult.reason === "no_job") { + // 404 Not Found — no row for this user (never soft-deleted, or already cascade-cleared) + return void res.status(404).json({ detail: "No deletion job to restore" }); } - await saveUserApiKey(userId, provider, apiKey, db); - const status = await getUserApiKeyStatus(userId, db); - res.json(status); - } catch (err) { - console.error("[user/api-keys] save failed", { - provider, - error: err instanceof Error ? err.message : String(err), - }); - res.status(500).json({ detail: "Failed to save API key" }); + // 410 Gone — replay of a consumed token (restore_token_used_at already set) + return void res.status(410).json({ detail: "Restore token already used" }); } -}); -// DELETE /user/account -userRouter.delete("/account", requireAuth, async (_req, res) => { - const userId = res.locals.userId as string; - const db = createServerSupabase(); - const { error } = await db.auth.admin.deleteUser(userId); - if (error) return void res.status(500).json({ detail: error.message }); + // 2. Clear soft-delete + unban auth user + const cleared = await clearSoftDelete(userId, db); + const unbanned = await unbanUser(userId, db); + if (!cleared || !unbanned) { + logger.error({ userId, cleared, unbanned }, "[user] restore failed mid-flight"); + return void res.status(500).json({ detail: "Restore failed" }); + } + + logger.info({ userId }, "[user] account restored"); res.status(204).send(); }); diff --git a/backend/src/routes/workflows.ts b/backend/src/routes/workflows.ts index 5f365b3b8..17a6ced4e 100644 --- a/backend/src/routes/workflows.ts +++ b/backend/src/routes/workflows.ts @@ -1,9 +1,47 @@ import { Router } from "express"; +import { z } from "zod"; +import { createClient } from "@supabase/supabase-js"; import { requireAuth } from "../middleware/auth"; import { createServerSupabase } from "../lib/supabase"; +import { parseBody } from "../lib/validate"; +import { BUILTIN_WORKFLOWS } from "../lib/builtinWorkflows"; + +function getAdminClient() { + return createClient( + process.env.SUPABASE_URL ?? "", + process.env.SUPABASE_SECRET_KEY ?? "", + { auth: { autoRefreshToken: false, persistSession: false } }, + ); +} + +const CreateWorkflowSchema = z.object({ + title: z.string().min(1), + type: z.enum(["assistant", "tabular"]), + prompt_md: z.string().optional(), + columns_config: z.unknown().optional(), + practice: z.string().optional().nullable(), +}); + +const PatchWorkflowSchema = z.object({ + title: z.string().min(1).optional(), + prompt_md: z.string().optional(), + columns_config: z.unknown().optional(), + practice: z.string().optional().nullable(), +}); + +const ShareWorkflowSchema = z.object({ + emails: z.array(z.string().email()).min(1), + allow_edit: z.boolean(), +}); export const workflowsRouter = Router(); +// CLEAN-49: single source of truth — return canonical backend BUILTIN_WORKFLOWS +// (mounted before /:id to avoid route shadowing) +workflowsRouter.get("/builtin", requireAuth, (_req, res) => { + res.json({ workflows: BUILTIN_WORKFLOWS }); +}); + type Db = ReturnType<typeof createServerSupabase>; type WorkflowRecord = { @@ -104,7 +142,7 @@ workflowsRouter.get("/", requireAuth, async (req, res) => { : { data: [] }; // Fetch sharer emails via admin client - const admin = createServerSupabase(); + const admin = getAdminClient(); const { data: authData } = await admin.auth.admin.listUsers({ perPage: 1000 }); const authUsers = authData?.users ?? []; @@ -132,19 +170,9 @@ workflowsRouter.get("/", requireAuth, async (req, res) => { // POST /workflows workflowsRouter.post("/", requireAuth, async (req, res) => { const userId = res.locals.userId as string; - const { title, type, prompt_md, columns_config, practice } = req.body as { - title: string; - type: string; - prompt_md?: string; - columns_config?: unknown; - practice?: string | null; - }; - if (!title?.trim()) - return void res.status(400).json({ detail: "title is required" }); - if (!["assistant", "tabular"].includes(type)) - return void res - .status(400) - .json({ detail: "type must be 'assistant' or 'tabular'" }); + const wfBody = parseBody(CreateWorkflowSchema, req, res); + if (!wfBody) return; + const { title, type, prompt_md, columns_config, practice } = wfBody; const db = createServerSupabase(); const { data, error } = await db @@ -168,12 +196,14 @@ async function handleWorkflowUpdate(req: import("express").Request, res: import( const userId = res.locals.userId as string; const userEmail = res.locals.userEmail as string | undefined; const { workflowId } = req.params; + const patchBody = parseBody(PatchWorkflowSchema, req, res); + if (!patchBody) return; const updates: Record<string, unknown> = {}; - if (req.body.title != null) updates.title = req.body.title; - if (req.body.prompt_md != null) updates.prompt_md = req.body.prompt_md; - if (req.body.columns_config != null) - updates.columns_config = req.body.columns_config; - if ("practice" in req.body) updates.practice = req.body.practice ?? null; + if (patchBody.title != null) updates.title = patchBody.title; + if (patchBody.prompt_md != null) updates.prompt_md = patchBody.prompt_md; + if (patchBody.columns_config != null) + updates.columns_config = patchBody.columns_config; + if ("practice" in patchBody) updates.practice = patchBody.practice ?? null; const db = createServerSupabase(); const access = await resolveWorkflowAccess(workflowId, userId, userEmail, db); @@ -212,13 +242,16 @@ workflowsRouter.delete("/:workflowId", requireAuth, async (req, res) => { const userId = res.locals.userId as string; const { workflowId } = req.params; const db = createServerSupabase(); - const { error } = await db + const { data, error } = await db .from("workflows") .delete() .eq("id", workflowId) .eq("user_id", userId) - .eq("is_system", false); + .eq("is_system", false) + .select("id"); if (error) return void res.status(500).json({ detail: error.message }); + if (!data || data.length === 0) + return void res.status(404).json({ detail: "Workflow not found" }); res.status(204).send(); }); @@ -234,12 +267,16 @@ workflowsRouter.get("/hidden", requireAuth, async (req, res) => { res.json((data ?? []).map((r) => r.workflow_id)); }); +const HideWorkflowSchema = z.object({ + workflow_id: z.string().min(1), +}); + // POST /workflows/hidden workflowsRouter.post("/hidden", requireAuth, async (req, res) => { const userId = res.locals.userId as string; - const { workflow_id } = req.body as { workflow_id: string }; - if (!workflow_id?.trim()) - return void res.status(400).json({ detail: "workflow_id is required" }); + const hideBody = parseBody(HideWorkflowSchema, req, res); + if (!hideBody) return; + const { workflow_id } = hideBody; const db = createServerSupabase(); const { error } = await db .from("hidden_workflows") @@ -326,9 +363,9 @@ workflowsRouter.delete("/:workflowId/shares/:shareId", requireAuth, async (req, workflowsRouter.post("/:workflowId/share", requireAuth, async (req, res) => { const userId = res.locals.userId as string; const { workflowId } = req.params; - const { emails, allow_edit } = req.body as { emails: string[]; allow_edit: boolean }; - - if (!emails?.length) return void res.status(400).json({ detail: "emails is required" }); + const shareBody = parseBody(ShareWorkflowSchema, req, res); + if (!shareBody) return; + const { emails, allow_edit } = shareBody; const db = createServerSupabase(); // Verify ownership diff --git a/backend/tsconfig.json b/backend/tsconfig.json index a4b3abf67..00bc0c5b3 100644 --- a/backend/tsconfig.json +++ b/backend/tsconfig.json @@ -10,7 +10,6 @@ "esModuleInterop": true, "skipLibCheck": true, "resolveJsonModule": true, - "types": ["node", "express", "cors", "multer"], "paths": { "@/*": ["./src/*"] } diff --git a/backend/vitest.auth-hardening.config.ts b/backend/vitest.auth-hardening.config.ts new file mode 100644 index 000000000..7e618b1ba --- /dev/null +++ b/backend/vitest.auth-hardening.config.ts @@ -0,0 +1,43 @@ +import { defineConfig } from "vitest/config"; +import path from "path"; + +/** + * Separate vitest config for auth-hardening tests. + * + * Auth-hardening tests that need real Supabase users share the same + * globalSetup as cross-tenant tests (mints two test users, sets TEST_JWT_A etc.). + * Tests that only inspect source files (static-source assertions) skip setup + * gracefully when env vars are absent. + */ +export default defineConfig({ + test: { + environment: "node", + // Stub non-Supabase env vars so env.ts validation passes at test-app import time. + env: { + DOWNLOAD_SIGNING_SECRET: "test-secret-placeholder-32-chars-ok", + FRONTEND_URL: "http://localhost:3000", + R2_ENDPOINT_URL: "http://localhost:9000", + R2_ACCESS_KEY_ID: "test", + R2_SECRET_ACCESS_KEY: "test", + R2_BUCKET_NAME: "test-bucket", + }, + globalSetup: [ + "./tests/cross-tenant/setup.ts", + "./tests/cross-tenant/teardown.ts", + ], + include: ["./tests/auth-hardening/**/*.test.ts"], + exclude: [ + "./tests/auth-hardening/authCache.test.ts", + "./tests/auth-hardening/emptyEmail.test.ts", + "./tests/auth-hardening/randomUuidImport.test.ts", + ], + testTimeout: 30_000, + hookTimeout: 60_000, + reporters: ["verbose"], + }, + resolve: { + alias: { + "@": path.resolve(__dirname, "./src"), + }, + }, +}); diff --git a/backend/vitest.config.ts b/backend/vitest.config.ts new file mode 100644 index 000000000..a804fd48d --- /dev/null +++ b/backend/vitest.config.ts @@ -0,0 +1,38 @@ +import { defineConfig } from "vitest/config"; +import path from "path"; + +export default defineConfig({ + test: { + environment: "node", + // Stub non-Supabase env vars so env.ts validation passes at test-app import time. + // Real Supabase keys must still be supplied via backend/.env for tests to run. + env: { + DOWNLOAD_SIGNING_SECRET: "test-secret-placeholder-32-chars-ok", + FRONTEND_URL: "http://localhost:3000", + R2_ENDPOINT_URL: "http://localhost:9000", + R2_ACCESS_KEY_ID: "test", + R2_SECRET_ACCESS_KEY: "test", + R2_BUCKET_NAME: "test-bucket", + }, + globalSetup: [ + "./tests/cross-tenant/setup.ts", + "./tests/cross-tenant/teardown.ts", + ], + include: [ + "./tests/cross-tenant/**/*.test.ts", + "./tests/auth-hardening/**/*.test.ts", + "./tests/saga/**/*.test.ts", + "./tests/integration/**/*.test.ts", + ], + fileParallelism: false, + maxWorkers: 1, + testTimeout: 30_000, + hookTimeout: 60_000, + reporters: ["verbose"], + }, + resolve: { + alias: { + "@": path.resolve(__dirname, "./src"), + }, + }, +}); diff --git a/backend/vitest.docx.config.ts b/backend/vitest.docx.config.ts new file mode 100644 index 000000000..f1d6f9601 --- /dev/null +++ b/backend/vitest.docx.config.ts @@ -0,0 +1,35 @@ +import { defineConfig } from "vitest/config"; +import path from "path"; + +/** + * Standalone vitest config for Phase 7 docxTrackedChanges round-trip tests. + * + * Pure in-process / no-DB. Tests live under tests/docx-round-trip/ and verify + * that applyTrackedEdits → resolveTrackedChange("accept"/"reject") is a + * semantic no-op across ≥20 DOCX fixture files (CLEAN-31 / CLEAN-36). + */ +export default defineConfig({ + test: { + environment: "node", + env: { + SUPABASE_URL: "http://localhost:54321", + SUPABASE_SECRET_KEY: "test-service-role-key", + DOWNLOAD_SIGNING_SECRET: "test-secret-placeholder-32-chars-ok", + FRONTEND_URL: "http://localhost:3000", + R2_ENDPOINT_URL: "http://localhost:9000", + R2_ACCESS_KEY_ID: "test", + R2_SECRET_ACCESS_KEY: "test", + R2_BUCKET_NAME: "test-bucket", + HUGO_MASTER_KEY: "00".repeat(32), + HUGO_RESTORE_TOKEN_SECRET: "test-restore-secret-placeholder-ok", + }, + include: ["./tests/docx-round-trip/**/*.test.ts"], + testTimeout: 30_000, + reporters: ["verbose"], + }, + resolve: { + alias: { + "@": path.resolve(__dirname, "./src"), + }, + }, +}); diff --git a/backend/vitest.golden-log.config.ts b/backend/vitest.golden-log.config.ts new file mode 100644 index 000000000..b4ddeea8e --- /dev/null +++ b/backend/vitest.golden-log.config.ts @@ -0,0 +1,35 @@ +import { defineConfig } from "vitest/config"; +import path from "path"; + +/** + * Standalone vitest config for Phase 8 golden-log SSE fixture tests. + * + * Pure-mock / no-DB. Tests live under tests/golden-log/ and verify that + * runLLMStream emits a byte-identical SSE event sequence before and + * after the chatTools.ts split (Pitfall 1 mitigation). + */ +export default defineConfig({ + test: { + environment: "node", + env: { + SUPABASE_URL: "http://localhost:54321", + SUPABASE_SECRET_KEY: "test-service-role-key", + DOWNLOAD_SIGNING_SECRET: "test-secret-placeholder-32-chars-ok", + FRONTEND_URL: "http://localhost:3000", + R2_ENDPOINT_URL: "http://localhost:9000", + R2_ACCESS_KEY_ID: "test", + R2_SECRET_ACCESS_KEY: "test", + R2_BUCKET_NAME: "test-bucket", + HUGO_MASTER_KEY: "00".repeat(32), + HUGO_RESTORE_TOKEN_SECRET: "test-restore-secret-placeholder-ok", + }, + include: ["./tests/golden-log/**/*.test.ts"], + testTimeout: 30_000, + reporters: ["verbose"], + }, + resolve: { + alias: { + "@": path.resolve(__dirname, "./src"), + }, + }, +}); diff --git a/backend/vitest.no-db.config.ts b/backend/vitest.no-db.config.ts new file mode 100644 index 000000000..57421fec6 --- /dev/null +++ b/backend/vitest.no-db.config.ts @@ -0,0 +1,65 @@ +import { defineConfig } from "vitest/config"; +import path from "path"; + +/** + * Standalone vitest config for pure-mock / no-DB tests. + * + * No globalSetup — these tests mock Supabase at the function level and do not + * require a live Supabase instance. Covers: + * - auth-hardening: authCache.test.ts and emptyEmail.test.ts (CLEAN-13 / CLEAN-14) + * - unit: crypto, env, restoreTokens, redaction stubs (CLEAN-05 / CLEAN-44) + */ +export default defineConfig({ + test: { + environment: "node", + env: { + SUPABASE_URL: "http://localhost:54321", + SUPABASE_SECRET_KEY: "test-service-role-key", + DOWNLOAD_SIGNING_SECRET: "test-secret-placeholder-32-chars-ok", + FRONTEND_URL: "http://localhost:3000", + R2_ENDPOINT_URL: "http://localhost:9000", + R2_ACCESS_KEY_ID: "test", + R2_SECRET_ACCESS_KEY: "test", + R2_BUCKET_NAME: "test-bucket", + // CLEAN-05: AES-256-GCM master key (64 hex chars = 32 bytes, all-zeros test key) + HUGO_MASTER_KEY: "00".repeat(32), + // CLEAN-44: HMAC secret for restore tokens (min 32 chars) + HUGO_RESTORE_TOKEN_SECRET: "test-restore-secret-placeholder-ok", + }, + include: [ + "./tests/auth-hardening/authCache.test.ts", + "./tests/auth-hardening/emptyEmail.test.ts", + "./tests/auth-hardening/authFailureModes.test.ts", + "./tests/unit/logger.test.ts", + "./tests/unit/geminiDebugGate.test.ts", + "./tests/unit/validate.test.ts", + "./tests/unit/rateLimiter.test.ts", + "./tests/integration/hardening.test.ts", + "./tests/integration/documentsUploadValidation.test.ts", + "./tests/integration/documentVersionConcurrency.test.ts", + "./tests/integration/chatStreamFailures.test.ts", + "./tests/integration/tabularGenerateFailures.test.ts", + "./tests/integration/tabularRegenerateRace.test.ts", + "./tests/unit/**/*.test.ts", + "./tests/integration/generateTitle.test.ts", + "./tests/integration/downloadZip.test.ts", + "./tests/integration/tabularList.test.ts", + "./tests/integration/workflowsBuiltin.test.ts", + "./tests/integration/modelsEndpoint.test.ts", + "./tests/unit/replicateCap.test.ts", + "./tests/integration/apiKeys.test.ts", + "./tests/integration/authDeleted.test.ts", + "./tests/integration/deleteAccount.test.ts", + "./tests/integration/restoreAccount.test.ts", + "./tests/integration/worker.test.ts", + ], + testTimeout: 30_000, + fileParallelism: false, + reporters: ["verbose"], + }, + resolve: { + alias: { + "@": path.resolve(__dirname, "./src"), + }, + }, +}); diff --git a/backend/vitest.saga.config.ts b/backend/vitest.saga.config.ts new file mode 100644 index 000000000..4370cea4e --- /dev/null +++ b/backend/vitest.saga.config.ts @@ -0,0 +1,37 @@ +/** + * Vitest config for saga unit tests only. + * + * Saga tests are pure unit tests with no Supabase dependency — they mock the + * db client and storage functions directly. This config intentionally omits + * the cross-tenant globalSetup so the tests run without a live database. + */ +import { defineConfig } from "vitest/config"; +import path from "path"; + +export default defineConfig({ + test: { + environment: "node", + env: { + DOWNLOAD_SIGNING_SECRET: "test-secret-placeholder-32-chars-ok", + FRONTEND_URL: "http://localhost:3000", + R2_ENDPOINT_URL: "http://localhost:9000", + R2_ACCESS_KEY_ID: "test", + R2_SECRET_ACCESS_KEY: "test", + R2_BUCKET_NAME: "test-bucket", + SUPABASE_URL: "https://test.supabase.co", + SUPABASE_ANON_KEY: "test-anon-key", + SUPABASE_SECRET_KEY: "test-service-role-key", + HUGO_MASTER_KEY: "00".repeat(32), + HUGO_RESTORE_TOKEN_SECRET: "test-restore-secret-placeholder-ok", + }, + include: [ + "./tests/saga/**/*.test.ts", + ], + reporters: ["verbose"], + }, + resolve: { + alias: { + "@": path.resolve(__dirname, "./src"), + }, + }, +});