From 9e969015a5bceefbfb25feb668c3ad3e6a56c6c4 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Fri, 24 Jul 2026 11:48:04 +0000 Subject: [PATCH 1/6] Use JOSE-encrypted credentials locally and Infisical in cloud --- apps/cli/src/index.ts | 8 +- .../cli/src/migrations/encryption-key.test.ts | 47 + apps/cli/src/migrations/index.ts | 22 +- apps/web/scripts/smoke-deployment-mode.ts | 4 + apps/web/src/env.d.ts | 7 + apps/web/src/server.ts | 12 + apps/worker/src/index.ts | 3 + packages/cloud/README.md | 14 + packages/cloud/package.json | 4 +- .../cloud/src/infisical-credentials.test.ts | 84 + packages/cloud/src/infisical-credentials.ts | 89 + packages/config/package.json | 1 + packages/config/src/env-registry.ts | 64 + packages/config/src/env.test.ts | 8 + packages/config/src/env.ts | 14 +- packages/deployment/package.json | 4 +- packages/deployment/src/credentials.ts | 27 + packages/lib/package.json | 2 + .../migrations/0011_provider_credentials.sql | 15 + .../src/db/migrations/meta/0011_snapshot.json | 1823 +++++++++++++++++ .../lib/src/db/migrations/meta/_journal.json | 7 + packages/lib/src/db/schema.ts | 39 +- .../src/providers/registry/anthropic-api.ts | 7 +- .../lib/src/providers/registry/brightdata.ts | 11 +- .../lib/src/providers/registry/dataforseo.ts | 7 +- .../lib/src/providers/registry/mistral-api.ts | 5 +- .../lib/src/providers/registry/olostep.ts | 12 +- .../lib/src/providers/registry/openai-api.ts | 6 +- .../lib/src/providers/registry/openrouter.ts | 7 +- .../lib/src/providers/registry/oxylabs.ts | 5 +- packages/lib/src/secrets/crypto.test.ts | 122 ++ packages/lib/src/secrets/crypto.ts | 75 + packages/lib/src/secrets/index.ts | 18 + packages/lib/src/secrets/store.test.ts | 240 +++ packages/lib/src/secrets/store.ts | 136 ++ pnpm-lock.yaml | 498 +++++ turbo.json | 7 + 37 files changed, 3418 insertions(+), 36 deletions(-) create mode 100644 apps/cli/src/migrations/encryption-key.test.ts create mode 100644 packages/cloud/src/infisical-credentials.test.ts create mode 100644 packages/cloud/src/infisical-credentials.ts create mode 100644 packages/deployment/src/credentials.ts create mode 100644 packages/lib/src/db/migrations/0011_provider_credentials.sql create mode 100644 packages/lib/src/db/migrations/meta/0011_snapshot.json create mode 100644 packages/lib/src/secrets/crypto.test.ts create mode 100644 packages/lib/src/secrets/crypto.ts create mode 100644 packages/lib/src/secrets/index.ts create mode 100644 packages/lib/src/secrets/store.test.ts create mode 100644 packages/lib/src/secrets/store.ts diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index f92fb139..97350871 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -90,8 +90,8 @@ function assertNotCancelled(value: T | symbol): asserts value is T { } } -function generateSecret(bytes = 32): string { - return crypto.randomBytes(bytes).toString("base64url"); +function generateSecret(bytes = 32, encoding: BufferEncoding = "base64url"): string { + return crypto.randomBytes(bytes).toString(encoding); } function link(text: string, url: string): string { @@ -244,6 +244,10 @@ async function runInit(options: InitOptions, version: string): Promise { env.VITE_DEPLOYMENT_MODE = "local"; env.DEPLOYMENT_ID = preservedDeploymentId ?? crypto.randomUUID(); env.BETTER_AUTH_SECRET = generateSecret(); + // Standard base64 (not base64url): the app decodes this with Buffer.from(key, + // "base64") and requires exactly 32 bytes. Enables storing provider + // credentials encrypted in the database via the in-app Providers UI. + env.ELMO_ENCRYPTION_KEY = generateSecret(32, "base64"); env.APP_NAME = DEFAULT_APP_NAME; env.APP_ICON = DEFAULT_APP_ICON; env.VITE_APP_NAME = DEFAULT_APP_NAME; diff --git a/apps/cli/src/migrations/encryption-key.test.ts b/apps/cli/src/migrations/encryption-key.test.ts new file mode 100644 index 00000000..d8ce0606 --- /dev/null +++ b/apps/cli/src/migrations/encryption-key.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { MIGRATIONS } from "./index.js"; +import type { Migration, MigrationContext } from "./types.js"; + +function inMemoryContext(initial: Record = {}): MigrationContext & { + env: () => Record; +} { + let env: Record = { ...initial }; + return { + configDir: "/fake", + log: { info: () => {}, warn: () => {}, step: () => {} }, + readEnv: async () => ({ ...env }), + writeEnv: async (next) => { + env = { ...next }; + }, + env: () => ({ ...env }), + }; +} + +const encryptionKeyMigration = MIGRATIONS.find((m) => m.from === "0.2.17") as Migration; + +describe("ELMO_ENCRYPTION_KEY migration", () => { + it("is registered as the 0.2.17 → 0.2.18 entry", () => { + expect(encryptionKeyMigration).toBeDefined(); + expect(encryptionKeyMigration.to).toBe("0.2.18"); + }); + + it("adds a 32-byte base64 key when absent, leaving other vars untouched", async () => { + const ctx = inMemoryContext({ DATABASE_URL: "postgres://x", OTHER: "keep" }); + await encryptionKeyMigration.run(ctx); + const { ELMO_ENCRYPTION_KEY, ...rest } = ctx.env(); + expect(rest).toEqual({ DATABASE_URL: "postgres://x", OTHER: "keep" }); + expect(Buffer.from(ELMO_ENCRYPTION_KEY, "base64").length).toBe(32); + }); + + it("is a no-op when the key already exists (never clobbers an operator's value)", async () => { + const ctx = inMemoryContext({ ELMO_ENCRYPTION_KEY: "existing", OTHER: "keep" }); + await encryptionKeyMigration.run(ctx); + expect(ctx.env()).toEqual({ ELMO_ENCRYPTION_KEY: "existing", OTHER: "keep" }); + }); + + it("leaves an explicitly-emptied key alone", async () => { + const ctx = inMemoryContext({ ELMO_ENCRYPTION_KEY: "" }); + await encryptionKeyMigration.run(ctx); + expect(ctx.env()).toEqual({ ELMO_ENCRYPTION_KEY: "" }); + }); +}); diff --git a/apps/cli/src/migrations/index.ts b/apps/cli/src/migrations/index.ts index 195d381a..c220cd9d 100644 --- a/apps/cli/src/migrations/index.ts +++ b/apps/cli/src/migrations/index.ts @@ -1,3 +1,4 @@ +import { randomBytes } from "node:crypto"; import type { Migration } from "./types.js"; export * from "./planner.js"; @@ -23,4 +24,23 @@ export * from "./types.js"; // } // }, // } -export const MIGRATIONS: readonly Migration[] = []; +export const MIGRATIONS: readonly Migration[] = [ + { + // Keyed to the last published version without the key (0.2.17) so every + // existing install picks it up on its next upgrade; the planner fires a + // `from` entry when the deployment's version is at or below it. + from: "0.2.17", + to: "0.2.18", + description: "add ELMO_ENCRYPTION_KEY for encrypted in-app provider credentials", + async run(ctx) { + const env = await ctx.readEnv(); + if (env.ELMO_ENCRYPTION_KEY !== undefined) return; + // Standard base64 (not base64url): decoded with Buffer.from(key, + // "base64") to exactly 32 bytes. Absent key = storage UI disabled and + // env-provided credentials keep working, so this is purely additive. + env.ELMO_ENCRYPTION_KEY = randomBytes(32).toString("base64"); + await ctx.writeEnv(env); + ctx.log.info("Added ELMO_ENCRYPTION_KEY — lets you store provider credentials encrypted in the app."); + }, + }, +]; diff --git a/apps/web/scripts/smoke-deployment-mode.ts b/apps/web/scripts/smoke-deployment-mode.ts index 14cb36d7..1bbdac23 100644 --- a/apps/web/scripts/smoke-deployment-mode.ts +++ b/apps/web/scripts/smoke-deployment-mode.ts @@ -83,6 +83,10 @@ const MINIMAL_ENV: Record> = { RESEND_FROM_EMAIL: "Smoke ", GOOGLE_CLIENT_ID: "smoke-google-client-id", GOOGLE_CLIENT_SECRET: "smoke-google-client-secret", + INFISICAL_CLIENT_ID: "smoke-client-id", + INFISICAL_CLIENT_SECRET: "smoke-client-secret", + INFISICAL_PROJECT_ID: "smoke-project-id", + INFISICAL_ENVIRONMENT: "prod", }, }; diff --git a/apps/web/src/env.d.ts b/apps/web/src/env.d.ts index e895f9ee..0e79b354 100644 --- a/apps/web/src/env.d.ts +++ b/apps/web/src/env.d.ts @@ -54,6 +54,13 @@ declare global { readonly OXYLABS_USERNAME?: string; readonly OXYLABS_PASSWORD?: string; readonly JINA_API_KEY?: string; + readonly ELMO_ENCRYPTION_KEY?: string; + readonly INFISICAL_CLIENT_ID?: string; + readonly INFISICAL_CLIENT_SECRET?: string; + readonly INFISICAL_PROJECT_ID?: string; + readonly INFISICAL_ENVIRONMENT?: string; + readonly INFISICAL_SECRET_PATH?: string; + readonly INFISICAL_SITE_URL?: string; readonly DATAFORSEO_LOGIN: string; readonly DATAFORSEO_PASSWORD: string; readonly BETTER_AUTH_SECRET?: string; diff --git a/apps/web/src/server.ts b/apps/web/src/server.ts index db006987..55b04ed9 100644 --- a/apps/web/src/server.ts +++ b/apps/web/src/server.ts @@ -1,6 +1,17 @@ import "../instrument.server.mjs"; import { wrapFetchWithSentry } from "@sentry/tanstackstart-react"; import handler, { createServerEntry } from "@tanstack/react-start/server-entry"; +import { startCredentialRefresh } from "@workspace/deployment/credentials"; + +let credentialsReady: Promise | undefined; + +function ensureCredentialsReady(): Promise { + credentialsReady ??= startCredentialRefresh().catch((error) => { + credentialsReady = undefined; + throw error; + }); + return credentialsReady; +} // HSTS asserts HTTPS-only for the host that served the response. Whitelabel // deployments run on customer-controlled custom domains, where `includeSubDomains` @@ -43,6 +54,7 @@ function addSecurityHeaders(response: Response): Response { export default createServerEntry( wrapFetchWithSentry({ async fetch(request: Request) { + await ensureCredentialsReady(); const response = await handler.fetch(request); return addSecurityHeaders(response); }, diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index ff3ca72b..c91d87b3 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -1,5 +1,6 @@ import * as Sentry from "@sentry/node"; import { getDeployment } from "@workspace/deployment"; +import { startCredentialRefresh } from "@workspace/deployment/credentials"; import { getProvider, parseScrapeTargets, validateScrapeTargets } from "@workspace/lib/providers"; import boss from "./boss"; import { registerHandlers } from "./handlers"; @@ -16,6 +17,8 @@ if (process.env.SENTRY_DSN) { async function main() { console.log("Starting pg-boss worker..."); + await startCredentialRefresh(); + // Fail fast on misconfigured SCRAPE_TARGETS — surfaces unknown providers, // missing API keys, and per-provider target errors before any job runs. validateScrapeTargets(parseScrapeTargets(process.env.SCRAPE_TARGETS), getProvider); diff --git a/packages/cloud/README.md b/packages/cloud/README.md index 18c2f4e5..68ab98bb 100644 --- a/packages/cloud/README.md +++ b/packages/cloud/README.md @@ -12,6 +12,12 @@ The auth and email features in this package need: | `RESEND_FROM_EMAIL` | Sender address, e.g. `Elmo ` | | `GOOGLE_CLIENT_ID` | Google OAuth client ID for social sign-in | | `GOOGLE_CLIENT_SECRET` | Google OAuth client secret | +| `INFISICAL_CLIENT_ID` | Infisical machine identity client ID | +| `INFISICAL_CLIENT_SECRET` | Infisical machine identity client secret | +| `INFISICAL_PROJECT_ID` | Project containing provider credentials | +| `INFISICAL_ENVIRONMENT` | Environment slug containing provider credentials | +| `INFISICAL_SECRET_PATH` | Optional credentials path; defaults to `/` | +| `INFISICAL_SITE_URL` | Optional Infisical site URL; defaults to the US cloud | The canonical list of every cloud-required variable (Stripe, database, etc.) lives in `packages/config/src/env-registry.ts`; env validation fails cloud startup when any of them is missing. @@ -23,6 +29,14 @@ The canonical list of every cloud-required variable (Stripe, database, etc.) liv Email templates are code — `packages/cloud/src/email-templates.ts` — not Resend-hosted templates; there is nothing to configure template-side in Resend. +## Infisical setup + +1. Create a machine identity with Universal Auth and read access to the provider-credential path. +2. Add provider credentials using their canonical environment-variable names, such as `OPENAI_API_KEY`, `OXYLABS_USERNAME`, and `OXYLABS_PASSWORD`. +3. Configure the required project, environment, client ID, and client secret variables above. + +The web server and worker load credentials at startup and refresh them every minute. Failed refreshes retain the last successfully loaded values. Infisical is used only by `DEPLOYMENT_MODE=cloud`; self-hosted deployments use encrypted database credentials or environment variables. + ## Google OAuth setup 1. In the Google Cloud console, create an OAuth client of type **Web application**. diff --git a/packages/cloud/package.json b/packages/cloud/package.json index d13001dc..a0292278 100644 --- a/packages/cloud/package.json +++ b/packages/cloud/package.json @@ -6,12 +6,14 @@ "main": "./src/index.ts", "exports": { ".": "./src/index.ts", - "./auth-hooks": "./src/auth-hooks.ts" + "./auth-hooks": "./src/auth-hooks.ts", + "./infisical-credentials": "./src/infisical-credentials.ts" }, "scripts": { "test": "vitest run" }, "dependencies": { + "@infisical/sdk": "5.0.2", "@workspace/config": "workspace:*", "@workspace/lib": "workspace:*", "better-auth": "^1.6.23", diff --git a/packages/cloud/src/infisical-credentials.test.ts b/packages/cloud/src/infisical-credentials.test.ts new file mode 100644 index 00000000..a6b872eb --- /dev/null +++ b/packages/cloud/src/infisical-credentials.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it, vi } from "vitest"; +import { createInfisicalCredentialLoader } from "./infisical-credentials"; + +const ENV = { + INFISICAL_CLIENT_ID: "client-id", + INFISICAL_CLIENT_SECRET: "client-secret", + INFISICAL_PROJECT_ID: "project-id", + INFISICAL_ENVIRONMENT: "prod", + INFISICAL_SECRET_PATH: "/elmo/providers", + INFISICAL_SITE_URL: "https://eu.infisical.com", +}; + +function fakeClient(secrets: Array<{ secretKey: string; secretValue: string }>) { + const listSecretsWithImports = vi.fn(async () => secrets); + const client = { + auth: () => ({ + universalAuth: { + login: vi.fn(async () => client), + }, + }), + secrets: () => ({ listSecretsWithImports }), + }; + return { client, listSecretsWithImports }; +} + +describe("createInfisicalCredentialLoader", () => { + it("authenticates and returns only canonical provider credentials", async () => { + const { client, listSecretsWithImports } = fakeClient([ + { secretKey: "OPENAI_API_KEY", secretValue: "sk-cloud" }, + { secretKey: "OXYLABS_USERNAME", secretValue: "user" }, + { secretKey: "NOT_AN_ELMO_CREDENTIAL", secretValue: "ignored" }, + ]); + const clientFactory = vi.fn(() => client); + const load = createInfisicalCredentialLoader({ env: ENV, clientFactory }); + + const credentials = await load(); + + expect(clientFactory).toHaveBeenCalledWith("https://eu.infisical.com"); + expect(credentials).toEqual( + new Map([ + ["OPENAI_API_KEY", "sk-cloud"], + ["OXYLABS_USERNAME", "user"], + ]), + ); + expect(listSecretsWithImports).toHaveBeenCalledWith({ + environment: "prod", + projectId: "project-id", + secretPath: "/elmo/providers", + recursive: true, + expandSecretReferences: true, + viewSecretValue: true, + }); + }); + + it("reuses the authenticated client across successful refreshes", async () => { + const { client } = fakeClient([{ secretKey: "OPENAI_API_KEY", secretValue: "sk-cloud" }]); + const clientFactory = vi.fn(() => client); + const load = createInfisicalCredentialLoader({ env: ENV, clientFactory }); + + await load(); + await load(); + + expect(clientFactory).toHaveBeenCalledOnce(); + }); + + it("re-authenticates once after an expired token", async () => { + const first = fakeClient([]); + first.listSecretsWithImports.mockRejectedValueOnce(new Error("unauthorized")); + const second = fakeClient([{ secretKey: "OPENAI_API_KEY", secretValue: "fresh" }]); + const clientFactory = vi.fn().mockReturnValueOnce(first.client).mockReturnValueOnce(second.client); + const load = createInfisicalCredentialLoader({ env: ENV, clientFactory }); + + await expect(load()).resolves.toEqual(new Map([["OPENAI_API_KEY", "fresh"]])); + expect(clientFactory).toHaveBeenCalledTimes(2); + }); + + it("fails fast when cloud authentication configuration is incomplete", () => { + expect(() => + createInfisicalCredentialLoader({ + env: { ...ENV, INFISICAL_CLIENT_SECRET: "" }, + }), + ).toThrow("INFISICAL_CLIENT_SECRET"); + }); +}); diff --git a/packages/cloud/src/infisical-credentials.ts b/packages/cloud/src/infisical-credentials.ts new file mode 100644 index 00000000..84820c86 --- /dev/null +++ b/packages/cloud/src/infisical-credentials.ts @@ -0,0 +1,89 @@ +import { InfisicalSDK } from "@infisical/sdk"; +import { CREDENTIAL_ENV_NAMES } from "@workspace/config/env-registry"; +import type { CredentialSource } from "@workspace/lib/secrets"; + +const CLIENT_ID_ENV = "INFISICAL_CLIENT_ID"; +const CLIENT_SECRET_ENV = "INFISICAL_CLIENT_SECRET"; +const PROJECT_ID_ENV = "INFISICAL_PROJECT_ID"; +const ENVIRONMENT_ENV = "INFISICAL_ENVIRONMENT"; +const SECRET_PATH_ENV = "INFISICAL_SECRET_PATH"; +const SITE_URL_ENV = "INFISICAL_SITE_URL"; + +interface InfisicalClient { + auth(): { + universalAuth: { + login(options: { clientId: string; clientSecret: string }): Promise; + }; + }; + secrets(): { + listSecretsWithImports(options: { + environment: string; + projectId: string; + secretPath: string; + recursive: boolean; + expandSecretReferences: boolean; + viewSecretValue: boolean; + }): Promise>; + }; +} + +export interface InfisicalCredentialLoaderOptions { + env?: Record; + clientFactory?: (siteUrl?: string) => InfisicalClient; +} + +function required(env: Record, name: string): string { + const value = env[name]?.trim(); + if (!value) throw new Error(`${name} is required for cloud provider credentials`); + return value; +} + +/** Cloud-only credential loader. Provider secret names in Infisical match the + * canonical env names (OPENAI_API_KEY, OXYLABS_USERNAME, etc.). */ +export function createInfisicalCredentialLoader(options: InfisicalCredentialLoaderOptions = {}): CredentialSource { + const env = options.env ?? process.env; + const clientId = required(env, CLIENT_ID_ENV); + const clientSecret = required(env, CLIENT_SECRET_ENV); + const projectId = required(env, PROJECT_ID_ENV); + const environment = required(env, ENVIRONMENT_ENV); + const secretPath = env[SECRET_PATH_ENV]?.trim() || "/"; + const siteUrl = env[SITE_URL_ENV]?.trim() || undefined; + const clientFactory = options.clientFactory ?? ((url) => new InfisicalSDK(url ? { siteUrl: url } : undefined)); + + let clientPromise: Promise | null = null; + const authenticate = () => { + clientPromise ??= clientFactory(siteUrl).auth().universalAuth.login({ clientId, clientSecret }); + return clientPromise; + }; + const list = async () => { + const client = await authenticate(); + return client.secrets().listSecretsWithImports({ + environment, + projectId, + secretPath, + recursive: true, + expandSecretReferences: true, + viewSecretValue: true, + }); + }; + + return async () => { + let secrets: Array<{ secretKey: string; secretValue: string }>; + try { + secrets = await list(); + } catch { + // Access tokens expire. Re-authenticate once before surfacing an outage; + // refreshCredentialOverlay keeps the last good values if both attempts fail. + clientPromise = null; + secrets = await list(); + } + + const credentials = new Map(); + for (const secret of secrets) { + if (CREDENTIAL_ENV_NAMES.has(secret.secretKey) && secret.secretValue.trim().length > 0) { + credentials.set(secret.secretKey, secret.secretValue); + } + } + return credentials; + }; +} diff --git a/packages/config/package.json b/packages/config/package.json index 328577f6..87621360 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -10,6 +10,7 @@ "./types": "./src/types.ts", "./constants": "./src/constants.ts", "./env": "./src/env.ts", + "./env-registry": "./src/env-registry.ts", "./scrape-targets": "./src/scrape-targets.ts" }, "devDependencies": { diff --git a/packages/config/src/env-registry.ts b/packages/config/src/env-registry.ts index c6815166..d9829f79 100644 --- a/packages/config/src/env-registry.ts +++ b/packages/config/src/env-registry.ts @@ -202,6 +202,48 @@ export const ENV_REGISTRY: EnvVarSpec[] = [ description: "Optional Jina Reader API key for website-excerpt fetching. When set, requests are authenticated (tracked by key, not IP), which raises the rate limit and avoids the anonymous 'bad network reputation' 401 block.", }, + { + name: "ELMO_ENCRYPTION_KEY", + scope: "server", + requiredBy: "optional", + description: "Base64-encoded 32-byte key enabling encrypted provider credentials in self-hosted deployments.", + }, + { + name: "INFISICAL_CLIENT_ID", + scope: "server", + requiredBy: ["cloud"], + description: "Infisical machine identity client ID used by managed cloud.", + }, + { + name: "INFISICAL_CLIENT_SECRET", + scope: "server", + requiredBy: ["cloud"], + description: "Infisical machine identity client secret used by managed cloud.", + }, + { + name: "INFISICAL_PROJECT_ID", + scope: "server", + requiredBy: ["cloud"], + description: "Infisical project containing managed cloud provider credentials.", + }, + { + name: "INFISICAL_ENVIRONMENT", + scope: "server", + requiredBy: ["cloud"], + description: "Infisical environment slug containing managed cloud provider credentials.", + }, + { + name: "INFISICAL_SECRET_PATH", + scope: "server", + requiredBy: "optional", + description: "Infisical path containing provider credentials. Defaults to the project root.", + }, + { + name: "INFISICAL_SITE_URL", + scope: "server", + requiredBy: "optional", + description: "Infisical API site URL. Defaults to https://app.infisical.com.", + }, { name: "DEPLOYMENT_MODE", scope: "server", @@ -415,3 +457,25 @@ export const ENV_REGISTRY: EnvVarSpec[] = [ "Sender address for transactional email, in the form: Elmo . The domain must be verified in Resend.", }, ]; + +/** + * provider id → the credential env-var names it needs, for every provider that + * declares one (`requiredBy: "dynamic-scrape-targets"`). The single source for + * credential grouping: startup validation (config), the credential overlay + * (lib), and cloud secret loading (Infisical) all read from here. Optional, + * env-only vars such as BRIGHTDATA_SERP_ZONE carry no provider marker and are + * excluded, keeping them out of the stored-credential lifecycle. + */ +export const PROVIDER_CREDENTIAL_KEYS: ReadonlyMap = (() => { + const index = new Map(); + for (const spec of ENV_REGISTRY) { + if (spec.requiredBy !== "dynamic-scrape-targets" || !spec.provider) continue; + const keys = index.get(spec.provider) ?? []; + keys.push(spec.name); + index.set(spec.provider, keys); + } + return index; +})(); + +/** Every canonical provider-credential env-var name, across all providers. */ +export const CREDENTIAL_ENV_NAMES: ReadonlySet = new Set([...PROVIDER_CREDENTIAL_KEYS.values()].flat()); diff --git a/packages/config/src/env.test.ts b/packages/config/src/env.test.ts index d04a637a..160ade48 100644 --- a/packages/config/src/env.test.ts +++ b/packages/config/src/env.test.ts @@ -10,6 +10,10 @@ const CLOUD_ONLY_VARS = [ "GOOGLE_CLIENT_ID", "GOOGLE_CLIENT_SECRET", "RESEND_FROM_EMAIL", + "INFISICAL_CLIENT_ID", + "INFISICAL_CLIENT_SECRET", + "INFISICAL_PROJECT_ID", + "INFISICAL_ENVIRONMENT", ]; // Infra vars every validated mode needs — cloud now among them. const CLOUD_SHARED_VARS = ["DATABASE_URL", "BETTER_AUTH_SECRET", "SCRAPE_TARGETS", "DEPLOYMENT_MODE"]; @@ -52,6 +56,10 @@ describe("cloud env requirements", () => { GOOGLE_CLIENT_ID: "test-google-client-id", GOOGLE_CLIENT_SECRET: "test-google-client-secret", RESEND_FROM_EMAIL: "Elmo ", + INFISICAL_CLIENT_ID: "client-id", + INFISICAL_CLIENT_SECRET: "client-secret", + INFISICAL_PROJECT_ID: "project-id", + INFISICAL_ENVIRONMENT: "prod", }; const { missing } = validateEnvRequirements(cloudReqs, env); const missingIds = new Set(missing.map((entry) => entry.id)); diff --git a/packages/config/src/env.ts b/packages/config/src/env.ts index 58441bd1..e390d716 100644 --- a/packages/config/src/env.ts +++ b/packages/config/src/env.ts @@ -1,4 +1,4 @@ -import { ENV_REGISTRY } from "./env-registry"; +import { ENV_REGISTRY, PROVIDER_CREDENTIAL_KEYS } from "./env-registry"; import { parseScrapeTargets } from "./scrape-targets"; import type { DeploymentMode, EnvRequirement } from "./types"; @@ -68,15 +68,13 @@ function buildProviderKeyRequirements(env: EnvMap = process.env): EnvRequirement const requirements: EnvRequirement[] = []; for (const provider of providers) { - const keys = ENV_REGISTRY.filter( - (spec) => spec.requiredBy === "dynamic-scrape-targets" && spec.provider === provider, - ).map((spec) => spec.name); - if (keys.length === 0) continue; + const keys = PROVIDER_CREDENTIAL_KEYS.get(provider); + if (!keys || keys.length === 0) continue; requirements.push({ id: `PROVIDER_${provider.toUpperCase().replace(/-/g, "_")}`, label: keys.join(" + "), description: `Required by SCRAPE_TARGETS provider "${provider}".`, - isSatisfied: requireAll(keys), + isSatisfied: requireAll([...keys]), }); } @@ -87,7 +85,9 @@ export const ENV_REQUIREMENTS: Record = { local: [...buildStaticRequirements("local"), ...buildProviderKeyRequirements()], demo: [...buildStaticRequirements("demo"), ...buildProviderKeyRequirements()], whitelabel: [...buildStaticRequirements("whitelabel"), ...buildProviderKeyRequirements()], - cloud: [...buildStaticRequirements("cloud"), ...buildProviderKeyRequirements()], + // Cloud provider credentials are loaded from Infisical before providers are + // validated, so their legacy env vars are not startup requirements. + cloud: buildStaticRequirements("cloud"), }; /** diff --git a/packages/deployment/package.json b/packages/deployment/package.json index 758a98f2..74ec7e10 100644 --- a/packages/deployment/package.json +++ b/packages/deployment/package.json @@ -6,11 +6,13 @@ "main": "./src/index.ts", "exports": { ".": "./src/index.ts", - "./client": "./src/client.ts" + "./client": "./src/client.ts", + "./credentials": "./src/credentials.ts" }, "dependencies": { "@workspace/cloud": "workspace:*", "@workspace/config": "workspace:*", + "@workspace/lib": "workspace:*", "@workspace/local": "workspace:*", "@workspace/whitelabel": "workspace:*" }, diff --git a/packages/deployment/src/credentials.ts b/packages/deployment/src/credentials.ts new file mode 100644 index 00000000..81440e88 --- /dev/null +++ b/packages/deployment/src/credentials.ts @@ -0,0 +1,27 @@ +import { getDeploymentModeFromEnv } from "@workspace/config/env"; +import { type CredentialSource, instanceCredentialSource, refreshCredentialOverlay } from "@workspace/lib/secrets"; + +const CREDENTIAL_REFRESH_INTERVAL_MS = 60_000; + +/** Managed cloud loads provider credentials from Infisical; every other mode + * reads the encrypted provider_credentials table (with env fallback). */ +async function getCredentialSource(env: Record): Promise { + if (getDeploymentModeFromEnv(env) !== "cloud") return instanceCredentialSource; + const { createInfisicalCredentialLoader } = await import("@workspace/cloud/infisical-credentials"); + return createInfisicalCredentialLoader({ env }); +} + +export async function startCredentialRefresh( + env: Record = process.env, +): Promise { + const source = await getCredentialSource(env); + const refresh = () => refreshCredentialOverlay(source); + await refresh(); + const timer = setInterval(() => { + refresh().catch((error) => { + console.warn("Provider credential refresh failed — keeping previous credentials:", error); + }); + }, CREDENTIAL_REFRESH_INTERVAL_MS); + timer.unref(); + return timer; +} diff --git a/packages/lib/package.json b/packages/lib/package.json index 92886dfa..7db49dd9 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -24,6 +24,7 @@ "./tag-utils": "./src/tag-utils.ts", "./website-excerpt": "./src/website-excerpt.ts", "./report-metrics": "./src/report-metrics.ts", + "./secrets": "./src/secrets/index.ts", "./providers": "./src/providers/index.ts", "./providers/models": "./src/providers/models.ts", "./providers/types": "./src/providers/types.ts" @@ -40,6 +41,7 @@ "better-auth": "^1.6.23", "dataforseo-client": "^2.0.25", "drizzle-orm": "^0.45.2", + "jose": "6.2.3", "linkedom": "^0.18.13", "olostep": "^1.2.2", "pg": "^8.22.0", diff --git a/packages/lib/src/db/migrations/0011_provider_credentials.sql b/packages/lib/src/db/migrations/0011_provider_credentials.sql new file mode 100644 index 00000000..07733af9 --- /dev/null +++ b/packages/lib/src/db/migrations/0011_provider_credentials.sql @@ -0,0 +1,15 @@ +CREATE TABLE "provider_credentials" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "organization_id" text, + "provider" text NOT NULL, + "encrypted_data" text NOT NULL, + "hint" text, + "last_verified_at" timestamp with time zone, + "last_verify_error" text, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "provider_credentials_provider_org_uidx" UNIQUE NULLS NOT DISTINCT("provider","organization_id") +); +--> statement-breakpoint +ALTER TABLE "provider_credentials" ENABLE ROW LEVEL SECURITY;--> statement-breakpoint +ALTER TABLE "provider_credentials" ADD CONSTRAINT "provider_credentials_organization_id_organization_id_fk" FOREIGN KEY ("organization_id") REFERENCES "public"."organization"("id") ON DELETE cascade ON UPDATE no action; \ No newline at end of file diff --git a/packages/lib/src/db/migrations/meta/0011_snapshot.json b/packages/lib/src/db/migrations/meta/0011_snapshot.json new file mode 100644 index 00000000..9df7051e --- /dev/null +++ b/packages/lib/src/db/migrations/meta/0011_snapshot.json @@ -0,0 +1,1823 @@ +{ + "id": "2402baf0-869d-45b7-86e1-908971084189", + "prevId": "0a10c0de-0010-4010-8010-000000000010", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.brand_opportunities": { + "name": "brand_opportunities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "brand_id": { + "name": "brand_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "report": { + "name": "report", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "brand_opportunities_brand_id_created_at_idx": { + "name": "brand_opportunities_brand_id_created_at_idx", + "columns": [ + { + "expression": "brand_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "brand_opportunities_brand_id_brands_id_fk": { + "name": "brand_opportunities_brand_id_brands_id_fk", + "tableFrom": "brand_opportunities", + "tableTo": "brands", + "columnsFrom": [ + "brand_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": true + }, + "public.brands": { + "name": "brands", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "additional_domains": { + "name": "additional_domains", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "aliases": { + "name": "aliases", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "onboarded": { + "name": "onboarded", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "delay_override_hours": { + "name": "delay_override_hours", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "enabled_models": { + "name": "enabled_models", + "type": "text[]", + "primaryKey": false, + "notNull": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "brands_organization_id_idx": { + "name": "brands_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "brands_organization_id_organization_id_fk": { + "name": "brands_organization_id_organization_id_fk", + "tableFrom": "brands", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": true + }, + "public.citations": { + "name": "citations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "prompt_run_id": { + "name": "prompt_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "prompt_id": { + "name": "prompt_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "brand_id": { + "name": "brand_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "citation_index": { + "name": "citation_index", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "idx_citations_brand_analytics": { + "name": "idx_citations_brand_analytics", + "columns": [ + { + "expression": "brand_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "url", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "title", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "prompt_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "citations_prompt_id_created_at_idx": { + "name": "citations_prompt_id_created_at_idx", + "columns": [ + { + "expression": "prompt_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "citations_domain_idx": { + "name": "citations_domain_idx", + "columns": [ + { + "expression": "domain", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "citations_prompt_run_id_prompt_runs_id_fk": { + "name": "citations_prompt_run_id_prompt_runs_id_fk", + "tableFrom": "citations", + "tableTo": "prompt_runs", + "columnsFrom": [ + "prompt_run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "citations_prompt_id_prompts_id_fk": { + "name": "citations_prompt_id_prompts_id_fk", + "tableFrom": "citations", + "tableTo": "prompts", + "columnsFrom": [ + "prompt_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "citations_brand_id_brands_id_fk": { + "name": "citations_brand_id_brands_id_fk", + "tableFrom": "citations", + "tableTo": "brands", + "columnsFrom": [ + "brand_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": true + }, + "public.competitors": { + "name": "competitors", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "brand_id": { + "name": "brand_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "domains": { + "name": "domains", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "aliases": { + "name": "aliases", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "competitors_brand_id_brands_id_fk": { + "name": "competitors_brand_id_brands_id_fk", + "tableFrom": "competitors", + "tableTo": "brands", + "columnsFrom": [ + "brand_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": true + }, + "public.prompt_runs": { + "name": "prompt_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "prompt_id": { + "name": "prompt_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "brand_id": { + "name": "brand_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "web_search_enabled": { + "name": "web_search_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "raw_output": { + "name": "raw_output", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "web_queries": { + "name": "web_queries", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "brand_mentioned": { + "name": "brand_mentioned", + "type": "boolean", + "primaryKey": false, + "notNull": true + }, + "competitors_mentioned": { + "name": "competitors_mentioned", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "prompt_runs_prompt_id_created_at_idx": { + "name": "prompt_runs_prompt_id_created_at_idx", + "columns": [ + { + "expression": "prompt_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "prompt_runs_created_at_idx": { + "name": "prompt_runs_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "prompt_runs_web_search_created_at_idx": { + "name": "prompt_runs_web_search_created_at_idx", + "columns": [ + { + "expression": "web_search_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "prompt_runs_web_search_model_created_at_idx": { + "name": "prompt_runs_web_search_model_created_at_idx", + "columns": [ + { + "expression": "web_search_enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "prompt_runs_provider_idx": { + "name": "prompt_runs_provider_idx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "prompt_runs_model_created_at_idx": { + "name": "prompt_runs_model_created_at_idx", + "columns": [ + { + "expression": "model", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "prompt_runs_prompt_id_prompts_id_fk": { + "name": "prompt_runs_prompt_id_prompts_id_fk", + "tableFrom": "prompt_runs", + "tableTo": "prompts", + "columnsFrom": [ + "prompt_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "prompt_runs_brand_id_brands_id_fk": { + "name": "prompt_runs_brand_id_brands_id_fk", + "tableFrom": "prompt_runs", + "tableTo": "brands", + "columnsFrom": [ + "brand_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": true + }, + "public.prompts": { + "name": "prompts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "brand_id": { + "name": "brand_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "tags": { + "name": "tags", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "system_tags": { + "name": "system_tags", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "prompts_brand_id_idx": { + "name": "prompts_brand_id_idx", + "columns": [ + { + "expression": "brand_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "prompts_brand_id_enabled_idx": { + "name": "prompts_brand_id_enabled_idx", + "columns": [ + { + "expression": "brand_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "prompts_brand_id_brands_id_fk": { + "name": "prompts_brand_id_brands_id_fk", + "tableFrom": "prompts", + "tableTo": "brands", + "columnsFrom": [ + "brand_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": true + }, + "public.provider_credentials": { + "name": "provider_credentials", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "encrypted_data": { + "name": "encrypted_data", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hint": { + "name": "hint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_verified_at": { + "name": "last_verified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_verify_error": { + "name": "last_verify_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "provider_credentials_organization_id_organization_id_fk": { + "name": "provider_credentials_organization_id_organization_id_fk", + "tableFrom": "provider_credentials", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "provider_credentials_provider_org_uidx": { + "name": "provider_credentials_provider_org_uidx", + "nullsNotDistinct": true, + "columns": [ + "provider", + "organization_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": true + }, + "public.reports": { + "name": "reports", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "brand_name": { + "name": "brand_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "brand_website": { + "name": "brand_website", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "report_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "progress": { + "name": "progress", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "raw_output": { + "name": "raw_output", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "reports_created_at_idx": { + "name": "reports_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": true + }, + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.invitation": { + "name": "invitation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "invitation_organizationId_idx": { + "name": "invitation_organizationId_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "invitation_email_idx": { + "name": "invitation_email_idx", + "columns": [ + { + "expression": "email", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": [ + "inviter_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.member": { + "name": "member", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "member_organizationId_idx": { + "name": "member_organizationId_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "member_userId_idx": { + "name": "member_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organization": { + "name": "organization", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "organization_slug_uidx": { + "name": "organization_slug_uidx", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "nullsNotDistinct": false, + "columns": [ + "slug" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sso_provider": { + "name": "sso_provider", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "issuer": { + "name": "issuer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "oidc_config": { + "name": "oidc_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "saml_config": { + "name": "saml_config", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "domain": { + "name": "domain", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sso_provider_user_id_user_id_fk": { + "name": "sso_provider_user_id_user_id_fk", + "tableFrom": "sso_provider", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "sso_provider_provider_id_unique": { + "name": "sso_provider_provider_id_unique", + "nullsNotDistinct": false, + "columns": [ + "provider_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "has_report_generator_access": { + "name": "has_report_generator_access", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.report_status": { + "name": "report_status", + "schema": "public", + "values": [ + "pending", + "processing", + "completed", + "failed" + ] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/lib/src/db/migrations/meta/_journal.json b/packages/lib/src/db/migrations/meta/_journal.json index 1e435f0f..2b851545 100644 --- a/packages/lib/src/db/migrations/meta/_journal.json +++ b/packages/lib/src/db/migrations/meta/_journal.json @@ -78,6 +78,13 @@ "when": 1784000000000, "tag": "0010_scope_brands_to_orgs", "breakpoints": true + }, + { + "idx": 11, + "version": "7", + "when": 1784845538775, + "tag": "0011_provider_credentials", + "breakpoints": true } ] } diff --git a/packages/lib/src/db/schema.ts b/packages/lib/src/db/schema.ts index 6390bf55..5986bebf 100644 --- a/packages/lib/src/db/schema.ts +++ b/packages/lib/src/db/schema.ts @@ -1,4 +1,16 @@ -import { pgEnum, pgTable, uuid, text, timestamp, boolean, json, index, integer, smallint } from "drizzle-orm/pg-core"; +import { + pgEnum, + pgTable, + uuid, + text, + timestamp, + boolean, + json, + index, + integer, + smallint, + unique, +} from "drizzle-orm/pg-core"; // `organization` is referenced by the brands FK below; the re-export makes it // (and the rest of the auth schema) visible to `import * as schema` consumers. import { organization } from "./schema-auth"; @@ -225,3 +237,28 @@ export const SYSTEM_TAGS = { } as const; export type SystemTag = (typeof SYSTEM_TAGS)[keyof typeof SYSTEM_TAGS]; + +// Self-hosted provider secrets — separate table, strictest access. Managed +// cloud credentials stay in Infisical and never enter this table. +export const providerCredentials = pgTable( + "provider_credentials", + { + id: uuid("id").defaultRandom().primaryKey().notNull(), + organizationId: text("organization_id").references(() => organization.id, { onDelete: "cascade" }), + provider: text("provider").notNull(), + encryptedData: text("encrypted_data").notNull(), + hint: text("hint"), + lastVerifiedAt: timestamp("last_verified_at", { withTimezone: true }), + lastVerifyError: text("last_verify_error"), + createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), + updatedAt: timestamp("updated_at", { withTimezone: true }) + .defaultNow() + .$onUpdate(() => new Date()) + .notNull(), + }, + (table) => ({ + providerOrgUnique: unique("provider_credentials_provider_org_uidx") + .on(table.provider, table.organizationId) + .nullsNotDistinct(), + }), +).enableRLS(); diff --git a/packages/lib/src/providers/registry/anthropic-api.ts b/packages/lib/src/providers/registry/anthropic-api.ts index b8c2f5dd..5630a030 100644 --- a/packages/lib/src/providers/registry/anthropic-api.ts +++ b/packages/lib/src/providers/registry/anthropic-api.ts @@ -16,11 +16,12 @@ import type { StructuredResearchResult, } from "../types"; import type { Citation } from "../../text-extraction"; +import { getCredential } from "../../secrets"; const DEFAULT_RESEARCH_MODEL = "claude-sonnet-4-6"; function getAnthropicLanguageModel(model: string) { - const apiKey = process.env.ANTHROPIC_API_KEY; + const apiKey = getCredential("ANTHROPIC_API_KEY"); return apiKey ? createAnthropic({ apiKey })(model) : anthropic(model); } @@ -29,7 +30,7 @@ function sanitizeForJson(obj: unknown): unknown { } function getClient(): Anthropic { - return new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY! }); + return new Anthropic({ apiKey: getCredential("ANTHROPIC_API_KEY")! }); } async function runAnthropic(prompt: string, model: string, options?: ProviderOptions): Promise { @@ -152,7 +153,7 @@ export const anthropicApi: Provider = { name: "Anthropic API", isConfigured() { - return !!process.env.ANTHROPIC_API_KEY; + return !!getCredential("ANTHROPIC_API_KEY"); }, async run(model: string, prompt: string, options?: ProviderOptions): Promise { diff --git a/packages/lib/src/providers/registry/brightdata.ts b/packages/lib/src/providers/registry/brightdata.ts index b07b6c02..816014b3 100644 --- a/packages/lib/src/providers/registry/brightdata.ts +++ b/packages/lib/src/providers/registry/brightdata.ts @@ -2,6 +2,7 @@ import { bdclient } from "@brightdata/sdk"; import type { Provider, ScrapeResult, ProviderOptions, ModelConfig } from "../types"; import { extractCitationsFromBrightdata, extractTextFromBrightdata, type Citation } from "../../text-extraction"; import { WEB_QUERIES_UNAVAILABLE } from "../../constants"; +import { getCredential } from "../../secrets"; // Google AI Overview isn't a Web Scraper dataset — it's the AI summary block on // a normal Google results page, fetched through BrightData's SERP API instead of @@ -25,7 +26,7 @@ const BD_BASE_URL: Record = { }; function createClient(): bdclient { - return new bdclient({ apiKey: process.env.BRIGHTDATA_API_TOKEN }); + return new bdclient({ apiKey: getCredential("BRIGHTDATA_API_TOKEN") }); } const BRIGHTDATA_REQUEST_URL = "https://api.brightdata.com/request"; @@ -49,7 +50,7 @@ async function runGoogleAiOverview(prompt: string): Promise { const res = await fetch(BRIGHTDATA_REQUEST_URL, { method: "POST", headers: { - Authorization: `Bearer ${process.env.BRIGHTDATA_API_TOKEN}`, + Authorization: `Bearer ${getCredential("BRIGHTDATA_API_TOKEN")}`, "Content-Type": "application/json", }, // `method: "GET"` tells BrightData how to fetch the target URL — without @@ -144,7 +145,7 @@ export const brightdata: Provider = { name: "BrightData", isConfigured() { - return !!process.env.BRIGHTDATA_API_TOKEN; + return !!getCredential("BRIGHTDATA_API_TOKEN"); }, validateTarget(config: ModelConfig) { @@ -189,7 +190,7 @@ export const brightdata: Provider = { { method: "POST", headers: { - Authorization: `Bearer ${process.env.BRIGHTDATA_API_TOKEN}`, + Authorization: `Bearer ${getCredential("BRIGHTDATA_API_TOKEN")}`, "Content-Type": "application/json", }, body: JSON.stringify([ @@ -286,7 +287,7 @@ async function pollUntilReady(snapshotId: string): Promise { async function getSnapshotStatus(snapshotId: string): Promise { try { const res = await fetch(`https://api.brightdata.com/datasets/v3/progress/${snapshotId}`, { - headers: { Authorization: `Bearer ${process.env.BRIGHTDATA_API_TOKEN}` }, + headers: { Authorization: `Bearer ${getCredential("BRIGHTDATA_API_TOKEN")}` }, }); if (!res.ok) return "pending"; const body = (await res.json()) as { status?: string }; diff --git a/packages/lib/src/providers/registry/dataforseo.ts b/packages/lib/src/providers/registry/dataforseo.ts index e0583227..519e3a10 100644 --- a/packages/lib/src/providers/registry/dataforseo.ts +++ b/packages/lib/src/providers/registry/dataforseo.ts @@ -7,6 +7,7 @@ import { extractTextFromGoogle, } from "../../text-extraction"; import type { ModelConfig, Provider, ProviderOptions, ScrapeResult } from "../types"; +import { getCredential } from "../../secrets"; /** * Models served via the SERP Google AI Mode endpoint (SerpApi). These always @@ -48,8 +49,8 @@ function sanitizeForJson(obj: unknown): unknown { } function authFetch(url: string | URL | Request, init?: RequestInit): Promise { - const username = process.env.DATAFORSEO_LOGIN; - const password = process.env.DATAFORSEO_PASSWORD; + const username = getCredential("DATAFORSEO_LOGIN"); + const password = getCredential("DATAFORSEO_PASSWORD"); if (!username || !password) { throw new Error("DataForSEO requires DATAFORSEO_LOGIN and DATAFORSEO_PASSWORD"); } @@ -267,7 +268,7 @@ export const dataforseo: Provider = { name: "DataForSEO", isConfigured() { - return !!process.env.DATAFORSEO_LOGIN && !!process.env.DATAFORSEO_PASSWORD; + return !!getCredential("DATAFORSEO_LOGIN") && !!getCredential("DATAFORSEO_PASSWORD"); }, validateTarget(config: ModelConfig) { diff --git a/packages/lib/src/providers/registry/mistral-api.ts b/packages/lib/src/providers/registry/mistral-api.ts index 19741a59..27991c52 100644 --- a/packages/lib/src/providers/registry/mistral-api.ts +++ b/packages/lib/src/providers/registry/mistral-api.ts @@ -7,6 +7,7 @@ import type { StructuredResearchResult, } from "../types"; import type { Citation } from "../../text-extraction"; +import { getCredential } from "../../secrets"; import { API_PROVIDER_MAX_OUTPUT_TOKENS, warnIfOutputCapped } from "../config"; const MISTRAL_BASE_URL = "https://api.mistral.ai"; @@ -20,7 +21,7 @@ async function mistralPost(path: string, body: object): Promise { const res = await fetch(`${MISTRAL_BASE_URL}${path}`, { method: "POST", headers: { - Authorization: `Bearer ${process.env.MISTRAL_API_KEY}`, + Authorization: `Bearer ${getCredential("MISTRAL_API_KEY")}`, "Content-Type": "application/json", }, body: JSON.stringify(body), @@ -84,7 +85,7 @@ export const mistralApi: Provider = { name: "Mistral API", isConfigured() { - return !!process.env.MISTRAL_API_KEY; + return !!getCredential("MISTRAL_API_KEY"); }, async run(model: string, prompt: string, options?: ProviderOptions): Promise { diff --git a/packages/lib/src/providers/registry/olostep.ts b/packages/lib/src/providers/registry/olostep.ts index 8fd871b3..8b9f4395 100644 --- a/packages/lib/src/providers/registry/olostep.ts +++ b/packages/lib/src/providers/registry/olostep.ts @@ -2,6 +2,7 @@ import Olostep from "olostep"; import type { Provider, ScrapeResult, ProviderOptions, ModelConfig } from "../types"; import type { Citation } from "../../text-extraction"; import { WEB_QUERIES_UNAVAILABLE } from "../../constants"; +import { getCredential } from "../../secrets"; const OLOSTEP_PARSERS: Record string; credits: number }> = { chatgpt: { @@ -37,9 +38,14 @@ const OLOSTEP_PARSERS: Record { diff --git a/packages/lib/src/providers/registry/openrouter.ts b/packages/lib/src/providers/registry/openrouter.ts index f59f1102..8a65d380 100644 --- a/packages/lib/src/providers/registry/openrouter.ts +++ b/packages/lib/src/providers/registry/openrouter.ts @@ -8,6 +8,7 @@ import type { } from "../types"; import type { Citation } from "../../text-extraction"; import { WEB_QUERIES_UNAVAILABLE } from "../../constants"; +import { getCredential } from "../../secrets"; import { API_PROVIDER_MAX_OUTPUT_TOKENS, warnIfOutputCapped } from "../config"; const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"; @@ -20,7 +21,7 @@ const DEFAULT_RESEARCH_MODEL = "openai/gpt-5-mini"; function openrouterHeaders(): Record { return { - Authorization: `Bearer ${process.env.OPENROUTER_API_KEY}`, + Authorization: `Bearer ${getCredential("OPENROUTER_API_KEY")}`, "Content-Type": "application/json", "HTTP-Referer": process.env.APP_URL ?? "https://github.com/elmohq/elmo", "X-Title": "Elmo AEO", @@ -75,7 +76,7 @@ export const openrouter: Provider = { name: "OpenRouter", isConfigured() { - return !!process.env.OPENROUTER_API_KEY; + return !!getCredential("OPENROUTER_API_KEY"); }, async runStructuredResearch({ @@ -150,7 +151,7 @@ export const openrouter: Provider = { const res = await fetch(OPENROUTER_API_URL, { method: "POST", headers: { - Authorization: `Bearer ${process.env.OPENROUTER_API_KEY}`, + Authorization: `Bearer ${getCredential("OPENROUTER_API_KEY")}`, "Content-Type": "application/json", "HTTP-Referer": process.env.APP_URL ?? "https://github.com/elmohq/elmo", "X-Title": "Elmo AEO", diff --git a/packages/lib/src/providers/registry/oxylabs.ts b/packages/lib/src/providers/registry/oxylabs.ts index 57078756..e958f790 100644 --- a/packages/lib/src/providers/registry/oxylabs.ts +++ b/packages/lib/src/providers/registry/oxylabs.ts @@ -1,5 +1,6 @@ import type { Provider, ScrapeResult, ProviderOptions, ModelConfig } from "../types"; import { extractTextFromOxylabs, extractCitationsFromOxylabs, type Citation } from "../../text-extraction"; +import { getCredential } from "../../secrets"; // Oxylabs Web Scraper API sources for AI surfaces. // ChatGPT and Perplexity use `prompt`; the Google surfaces use `query` and @@ -30,7 +31,7 @@ interface OxylabsPayload { } function basicAuthHeader(): string { - const token = btoa(`${process.env.OXYLABS_USERNAME}:${process.env.OXYLABS_PASSWORD}`); + const token = btoa(`${getCredential("OXYLABS_USERNAME")}:${getCredential("OXYLABS_PASSWORD")}`); return `Basic ${token}`; } @@ -158,7 +159,7 @@ export const oxylabs: Provider = { name: "Oxylabs", isConfigured() { - return !!process.env.OXYLABS_USERNAME && !!process.env.OXYLABS_PASSWORD; + return !!getCredential("OXYLABS_USERNAME") && !!getCredential("OXYLABS_PASSWORD"); }, validateTarget(config: ModelConfig) { diff --git a/packages/lib/src/secrets/crypto.test.ts b/packages/lib/src/secrets/crypto.test.ts new file mode 100644 index 00000000..f5a11e82 --- /dev/null +++ b/packages/lib/src/secrets/crypto.test.ts @@ -0,0 +1,122 @@ +import { randomBytes } from "node:crypto"; +import { CompactEncrypt, decodeProtectedHeader } from "jose"; +import { describe, expect, it } from "vitest"; +import { decryptSecret, EncryptionKeyError, encryptSecret, getEncryptionKey, SecretDecryptError } from "./crypto"; + +const KEY = Buffer.alloc(32, 7); +const AAD = "provider-credentials:openai-api"; + +describe("encryptSecret / decryptSecret", () => { + it("round-trips arbitrary plaintext", async () => { + const plaintext = 'sk-abc123-{"nested":true}-🔐'; + const payload = await encryptSecret(plaintext, { key: KEY, aad: AAD }); + await expect(decryptSecret(payload, { key: KEY, aad: AAD })).resolves.toBe(plaintext); + }); + + it("uses the standard direct A256GCM JWE format with authenticated metadata", async () => { + const payload = await encryptSecret("x", { key: KEY, aad: AAD }); + expect(payload.split(".")).toHaveLength(5); + expect(decodeProtectedHeader(payload)).toEqual({ + alg: "dir", + enc: "A256GCM", + typ: "elmo-provider-credentials", + v: 1, + ctx: AAD, + }); + }); + + it("uses a fresh IV every call (no GCM nonce reuse)", async () => { + const ivs = new Set(); + for (let i = 0; i < 200; i++) { + const payload = await encryptSecret("same-plaintext", { key: KEY, aad: AAD }); + ivs.add(payload.split(".")[2]); + } + expect(ivs.size).toBe(200); + }); + + describe("tamper matrix — every failure throws SecretDecryptError, never garbage", () => { + function fresh(): Promise { + return encryptSecret("top-secret-value", { key: KEY, aad: AAD }); + } + + function corruptSegment(payload: string, index: number): string { + const segments = payload.split("."); + const segment = segments[index]; + segments[index] = `${segment[0] === "A" ? "B" : "A"}${segment.slice(1)}`; + return segments.join("."); + } + + it("flipped tag byte", async () => { + await expect(decryptSecret(corruptSegment(await fresh(), 4), { key: KEY, aad: AAD })).rejects.toThrow( + SecretDecryptError, + ); + }); + + it("wrong AAD", async () => { + await expect( + decryptSecret(await fresh(), { key: KEY, aad: "provider-credentials:anthropic-api" }), + ).rejects.toThrow(SecretDecryptError); + }); + + it("wrong key", async () => { + await expect(decryptSecret(await fresh(), { key: Buffer.alloc(32, 8), aad: AAD })).rejects.toThrow( + SecretDecryptError, + ); + }); + + it("truncated ciphertext", async () => { + const segments = (await fresh()).split("."); + segments[3] = segments[3].slice(0, 3); + await expect(decryptSecret(segments.join("."), { key: KEY, aad: AAD })).rejects.toThrow(SecretDecryptError); + }); + + it("unknown version", async () => { + const payload = await new CompactEncrypt(new TextEncoder().encode("top-secret-value")) + .setProtectedHeader({ + alg: "dir", + enc: "A256GCM", + typ: "elmo-provider-credentials", + v: 2, + ctx: AAD, + }) + .encrypt(KEY); + await expect(decryptSecret(payload, { key: KEY, aad: AAD })).rejects.toThrow(SecretDecryptError); + }); + + it("malformed payloads", async () => { + for (const bad of [null, undefined, {}, "nope", 42, { v: 1, keyId: "x", iv: "a", ct: "b" }]) { + await expect(decryptSecret(bad, { key: KEY, aad: AAD })).rejects.toThrow(SecretDecryptError); + } + }); + + it("wrong-length key", async () => { + await expect(decryptSecret(await fresh(), { key: randomBytes(16), aad: AAD })).rejects.toThrow( + SecretDecryptError, + ); + }); + }); +}); + +describe("getEncryptionKey", () => { + it("returns null when unset or blank", () => { + expect(getEncryptionKey({})).toBeNull(); + expect(getEncryptionKey({ ELMO_ENCRYPTION_KEY: "" })).toBeNull(); + expect(getEncryptionKey({ ELMO_ENCRYPTION_KEY: " " })).toBeNull(); + }); + + it("decodes a valid 32-byte base64 key", () => { + const key = randomBytes(32); + const decoded = getEncryptionKey({ ELMO_ENCRYPTION_KEY: key.toString("base64") }); + expect(decoded).not.toBeNull(); + expect(decoded!.equals(key)).toBe(true); + }); + + it("throws EncryptionKeyError for the wrong decoded length", () => { + expect(() => getEncryptionKey({ ELMO_ENCRYPTION_KEY: randomBytes(16).toString("base64") })).toThrow( + EncryptionKeyError, + ); + expect(() => getEncryptionKey({ ELMO_ENCRYPTION_KEY: randomBytes(64).toString("base64") })).toThrow( + EncryptionKeyError, + ); + }); +}); diff --git a/packages/lib/src/secrets/crypto.ts b/packages/lib/src/secrets/crypto.ts new file mode 100644 index 00000000..87cbdce0 --- /dev/null +++ b/packages/lib/src/secrets/crypto.ts @@ -0,0 +1,75 @@ +import { compactDecrypt, CompactEncrypt } from "jose"; + +const KEY_BYTES = 32; // 256-bit +const CURRENT_VERSION = 1; +const JWE_TYPE = "elmo-provider-credentials"; + +export const ENCRYPTION_KEY_ENV = "ELMO_ENCRYPTION_KEY"; + +/** Compact JWE using direct symmetric encryption (`dir`) and AES-256-GCM. */ +export type EncryptedPayload = string; + +/** Thrown for ANY decryption failure — bad tag, wrong AAD, wrong key, + * malformed payload, unknown version. Never leaks plaintext or key material. */ +export class SecretDecryptError extends Error { + constructor(message: string, options?: { cause?: unknown }) { + super(message, options); + this.name = "SecretDecryptError"; + } +} + +/** Thrown when ELMO_ENCRYPTION_KEY is present but unusable (wrong length). */ +export class EncryptionKeyError extends Error { + constructor(message: string) { + super(message); + this.name = "EncryptionKeyError"; + } +} + +export async function encryptSecret( + plaintext: string, + opts: { key: Uint8Array; aad: string }, +): Promise { + return new CompactEncrypt(new TextEncoder().encode(plaintext)) + .setProtectedHeader({ + alg: "dir", + enc: "A256GCM", + typ: JWE_TYPE, + v: CURRENT_VERSION, + ctx: opts.aad, + }) + .encrypt(opts.key); +} + +export async function decryptSecret(payload: unknown, opts: { key: Uint8Array; aad: string }): Promise { + try { + if (typeof payload !== "string") throw new Error("malformed payload"); + const { plaintext, protectedHeader } = await compactDecrypt(payload, opts.key, { + keyManagementAlgorithms: ["dir"], + contentEncryptionAlgorithms: ["A256GCM"], + }); + if (protectedHeader.typ !== JWE_TYPE) throw new Error("unexpected payload type"); + if (protectedHeader.v !== CURRENT_VERSION) { + throw new Error(`unsupported payload version: ${String(protectedHeader.v)}`); + } + if (protectedHeader.ctx !== opts.aad) throw new Error("payload context mismatch"); + return new TextDecoder("utf-8", { fatal: true }).decode(plaintext); + } catch (cause) { + throw new SecretDecryptError("failed to decrypt secret", { cause }); + } +} + +/** Read ELMO_ENCRYPTION_KEY (base64 → exactly 32 bytes). Returns null when + * unset (storage disabled, env credentials unaffected); throws + * EncryptionKeyError when set to the wrong length. */ +export function getEncryptionKey(env: NodeJS.ProcessEnv = process.env): Buffer | null { + const raw = env[ENCRYPTION_KEY_ENV]; + if (typeof raw !== "string" || raw.trim().length === 0) return null; + const key = Buffer.from(raw, "base64"); + if (key.length !== KEY_BYTES) { + throw new EncryptionKeyError( + `${ENCRYPTION_KEY_ENV} must be base64 for exactly ${KEY_BYTES} bytes (decoded to ${key.length})`, + ); + } + return key; +} diff --git a/packages/lib/src/secrets/index.ts b/packages/lib/src/secrets/index.ts new file mode 100644 index 00000000..2899f8b4 --- /dev/null +++ b/packages/lib/src/secrets/index.ts @@ -0,0 +1,18 @@ +export { + decryptSecret, + ENCRYPTION_KEY_ENV, + type EncryptedPayload, + EncryptionKeyError, + encryptSecret, + getEncryptionKey, + SecretDecryptError, +} from "./crypto"; +export { + type CredentialSource, + clearCredentialOverlay, + encryptProviderCredentials, + getCredential, + getCredentialKeysForProvider, + instanceCredentialSource, + refreshCredentialOverlay, +} from "./store"; diff --git a/packages/lib/src/secrets/store.test.ts b/packages/lib/src/secrets/store.test.ts new file mode 100644 index 00000000..e6a1deec --- /dev/null +++ b/packages/lib/src/secrets/store.test.ts @@ -0,0 +1,240 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { EncryptionKeyError, encryptSecret } from "./crypto"; + +// instanceCredentialSource reads rows through drizzle; mock the db module so the +// store is testable with no database. `dbState.rows` is the resolved row set for +// `db.select().from().where()`. +const dbState = vi.hoisted(() => ({ rows: [] as unknown[] })); +vi.mock("../db/db", () => ({ + db: { + select: () => ({ from: () => ({ where: () => Promise.resolve(dbState.rows) }) }), + }, +})); + +import { + clearCredentialOverlay, + encryptProviderCredentials, + getCredential, + getCredentialKeysForProvider, + instanceCredentialSource, + refreshCredentialOverlay, +} from "./store"; + +const KEY = Buffer.alloc(32, 7); +const KEY_B64 = KEY.toString("base64"); + +async function encryptedRow(provider: string, record: Record, key: Buffer = KEY) { + return { + provider, + encryptedData: await encryptSecret(JSON.stringify(record), { key, aad: `provider-credentials:${provider}` }), + }; +} + +beforeEach(() => { + dbState.rows = []; + clearCredentialOverlay(); + vi.spyOn(console, "warn").mockImplementation(() => {}); +}); + +afterEach(() => { + clearCredentialOverlay(); + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +describe("getCredentialKeysForProvider", () => { + it("maps all eight credentialed providers to their env vars", () => { + expect(getCredentialKeysForProvider("olostep")).toEqual(["OLOSTEP_API_KEY"]); + expect(getCredentialKeysForProvider("brightdata")).toEqual(["BRIGHTDATA_API_TOKEN"]); + expect(getCredentialKeysForProvider("oxylabs")).toEqual(["OXYLABS_USERNAME", "OXYLABS_PASSWORD"]); + expect(getCredentialKeysForProvider("openai-api")).toEqual(["OPENAI_API_KEY"]); + expect(getCredentialKeysForProvider("anthropic-api")).toEqual(["ANTHROPIC_API_KEY"]); + expect(getCredentialKeysForProvider("mistral-api")).toEqual(["MISTRAL_API_KEY"]); + expect(getCredentialKeysForProvider("dataforseo")).toEqual(["DATAFORSEO_LOGIN", "DATAFORSEO_PASSWORD"]); + expect(getCredentialKeysForProvider("openrouter")).toEqual(["OPENROUTER_API_KEY"]); + }); + + it("returns [] for credential-less / unknown providers", () => { + expect(getCredentialKeysForProvider("stub")).toEqual([]); + expect(getCredentialKeysForProvider("nope")).toEqual([]); + }); + + it("returns a fresh copy each call (no shared mutable state)", () => { + const keys = getCredentialKeysForProvider("oxylabs"); + keys.push("MUTATED"); + expect(getCredentialKeysForProvider("oxylabs")).toEqual(["OXYLABS_USERNAME", "OXYLABS_PASSWORD"]); + }); +}); + +describe("getCredential", () => { + it("falls back to process.env when the overlay is empty", () => { + vi.stubEnv("OPENAI_API_KEY", "env-value"); + expect(getCredential("OPENAI_API_KEY")).toBe("env-value"); + }); + + it("returns undefined when neither overlay nor env has the var", () => { + vi.stubEnv("SOME_UNSET_CREDENTIAL", undefined); + expect(getCredential("SOME_UNSET_CREDENTIAL")).toBeUndefined(); + }); + + it("a managed value beats process.env, and clearing restores the env fallback", async () => { + vi.stubEnv("ELMO_ENCRYPTION_KEY", KEY_B64); + vi.stubEnv("OPENAI_API_KEY", "env-value"); + dbState.rows = [await encryptedRow("openai-api", { OPENAI_API_KEY: "db-value" })]; + + await refreshCredentialOverlay(instanceCredentialSource); + expect(getCredential("OPENAI_API_KEY")).toBe("db-value"); + + clearCredentialOverlay(); + expect(getCredential("OPENAI_API_KEY")).toBe("env-value"); + }); +}); + +describe("instanceCredentialSource", () => { + beforeEach(() => vi.stubEnv("ELMO_ENCRYPTION_KEY", KEY_B64)); + + it("decrypts rows and applies only the provider's expected keys", async () => { + vi.stubEnv("UNEXPECTED", undefined); + dbState.rows = [ + await encryptedRow("oxylabs", { OXYLABS_USERNAME: "u", OXYLABS_PASSWORD: "p", UNEXPECTED: "drop-me" }), + ]; + + await refreshCredentialOverlay(instanceCredentialSource); + + expect(getCredential("OXYLABS_USERNAME")).toBe("u"); + expect(getCredential("OXYLABS_PASSWORD")).toBe("p"); + expect(getCredential("UNEXPECTED")).toBeUndefined(); + }); + + it("skips encrypted rows when no key is available", async () => { + vi.stubEnv("ELMO_ENCRYPTION_KEY", undefined); + vi.stubEnv("OPENAI_API_KEY", undefined); + dbState.rows = [await encryptedRow("openai-api", { OPENAI_API_KEY: "x" })]; + + await refreshCredentialOverlay(instanceCredentialSource); + + expect(getCredential("OPENAI_API_KEY")).toBeUndefined(); + }); + + it("skips incomplete multi-field credentials instead of mixing them with env values", async () => { + vi.stubEnv("OXYLABS_USERNAME", undefined); + dbState.rows = [await encryptedRow("oxylabs", { OXYLABS_USERNAME: "u" })]; + + await refreshCredentialOverlay(instanceCredentialSource); + + expect(getCredential("OXYLABS_USERNAME")).toBeUndefined(); + }); + + it("does not leak an unknown-provider row's keys onto a real provider", async () => { + vi.stubEnv("OPENAI_API_KEY", undefined); + dbState.rows = [await encryptedRow("mystery", { OPENAI_API_KEY: "x" })]; + + await refreshCredentialOverlay(instanceCredentialSource); + + expect(getCredential("OPENAI_API_KEY")).toBeUndefined(); + }); + + it("a decrypt-failure row omits that provider's keys while healthy providers survive", async () => { + vi.stubEnv("BRIGHTDATA_API_TOKEN", undefined); + dbState.rows = [ + await encryptedRow("openai-api", { OPENAI_API_KEY: "ok" }), + // Encrypted under a different key → GCM auth fails on decrypt. + await encryptedRow("brightdata", { BRIGHTDATA_API_TOKEN: "nope" }, Buffer.alloc(32, 9)), + ]; + + await refreshCredentialOverlay(instanceCredentialSource); + + expect(getCredential("OPENAI_API_KEY")).toBe("ok"); + expect(getCredential("BRIGHTDATA_API_TOKEN")).toBeUndefined(); + }); +}); + +describe("refreshCredentialOverlay", () => { + it("never keeps a stale overlay value once a provider's row stops decrypting", async () => { + vi.stubEnv("ELMO_ENCRYPTION_KEY", KEY_B64); + vi.stubEnv("BRIGHTDATA_API_TOKEN", undefined); // no env fallback for this var + + dbState.rows = [await encryptedRow("brightdata", { BRIGHTDATA_API_TOKEN: "first" })]; + await refreshCredentialOverlay(instanceCredentialSource); + expect(getCredential("BRIGHTDATA_API_TOKEN")).toBe("first"); + + // Same provider, now undecryptable — must not fall back to "first". + dbState.rows = [await encryptedRow("brightdata", { BRIGHTDATA_API_TOKEN: "second" }, Buffer.alloc(32, 3))]; + await refreshCredentialOverlay(instanceCredentialSource); + expect(getCredential("BRIGHTDATA_API_TOKEN")).toBeUndefined(); + }); + + it("degrades to env-only (never throws) when ELMO_ENCRYPTION_KEY is the wrong length", async () => { + vi.stubEnv("ELMO_ENCRYPTION_KEY", Buffer.alloc(16).toString("base64")); + vi.stubEnv("OPENAI_API_KEY", "env-value"); + dbState.rows = [await encryptedRow("openai-api", { OPENAI_API_KEY: "db-value" })]; + + await expect(refreshCredentialOverlay(instanceCredentialSource)).resolves.toBeUndefined(); + expect(getCredential("OPENAI_API_KEY")).toBe("env-value"); + }); + + it("takes managed values from any source and ignores non-provider keys", async () => { + vi.stubEnv("OPENAI_API_KEY", "env-value"); + const source = vi.fn( + async () => + new Map([ + ["OPENAI_API_KEY", "managed-value"], + ["NOT_A_PROVIDER_KEY", "ignored"], + ]), + ); + + await refreshCredentialOverlay(source); + + expect(source).toHaveBeenCalledOnce(); + expect(getCredential("OPENAI_API_KEY")).toBe("managed-value"); + expect(getCredential("NOT_A_PROVIDER_KEY")).toBeUndefined(); + }); + + it("does not mix a partial managed bundle with env values", async () => { + vi.stubEnv("OXYLABS_PASSWORD", "env-password"); + + await refreshCredentialOverlay(async () => new Map([["OXYLABS_USERNAME", "cloud-user"]])); + + expect(getCredential("OXYLABS_USERNAME")).toBeUndefined(); + expect(getCredential("OXYLABS_PASSWORD")).toBe("env-password"); + }); + + it("keeps the current overlay when a refresh fails", async () => { + await refreshCredentialOverlay(async () => new Map([["OPENAI_API_KEY", "current"]])); + + await expect( + refreshCredentialOverlay(async () => { + throw new Error("Infisical unavailable"); + }), + ).rejects.toThrow("Infisical unavailable"); + expect(getCredential("OPENAI_API_KEY")).toBe("current"); + }); +}); + +describe("encryptProviderCredentials", () => { + it("produces a payload that round-trips through the overlay, with a longest-value hint", async () => { + vi.stubEnv("ELMO_ENCRYPTION_KEY", KEY_B64); + const { encryptedData, hint } = await encryptProviderCredentials("oxylabs", { + OXYLABS_USERNAME: "user", + OXYLABS_PASSWORD: "longer-secret", + }); + expect(hint).toBe("cret"); // last 4 of the longest value "longer-secret" + + dbState.rows = [{ provider: "oxylabs", encryptedData }]; + await refreshCredentialOverlay(instanceCredentialSource); + expect(getCredential("OXYLABS_USERNAME")).toBe("user"); + expect(getCredential("OXYLABS_PASSWORD")).toBe("longer-secret"); + }); + + it("rejects incomplete multi-field credentials", async () => { + vi.stubEnv("ELMO_ENCRYPTION_KEY", KEY_B64); + await expect(encryptProviderCredentials("oxylabs", { OXYLABS_USERNAME: "user" })).rejects.toThrow( + /must contain exactly/, + ); + }); + + it("throws EncryptionKeyError when no encryption key is set", async () => { + vi.stubEnv("ELMO_ENCRYPTION_KEY", undefined); + await expect(encryptProviderCredentials("openai-api", { OPENAI_API_KEY: "x" })).rejects.toThrow(EncryptionKeyError); + }); +}); diff --git a/packages/lib/src/secrets/store.ts b/packages/lib/src/secrets/store.ts new file mode 100644 index 00000000..fd5ec365 --- /dev/null +++ b/packages/lib/src/secrets/store.ts @@ -0,0 +1,136 @@ +import { PROVIDER_CREDENTIAL_KEYS } from "@workspace/config/env-registry"; +import { isNull } from "drizzle-orm"; +import { db } from "../db/db"; +import { providerCredentials } from "../db/schema"; +import { + decryptSecret, + ENCRYPTION_KEY_ENV, + type EncryptedPayload, + EncryptionKeyError, + encryptSecret, + getEncryptionKey, +} from "./crypto"; + +// Providers read every credential through getCredential: a managed override if +// one exists, otherwise process.env. The overlay of managed overrides is rebuilt +// on an interval by refreshCredentialOverlay from a CredentialSource — Infisical +// in managed cloud, the encrypted provider_credentials table when self-hosted. +const overlay = new Map(); + +/** A managed credential source: a flat env-name→value map. Only whole provider + * bundles from it reach the overlay (see refreshCredentialOverlay). */ +export type CredentialSource = () => Promise>; + +/** Managed override if present, else process.env. Sync so `isConfigured()` stays sync. */ +export function getCredential(name: string): string | undefined { + return overlay.get(name) ?? process.env[name]; +} + +/** A provider's credential env-var names (fresh array, safe to mutate). */ +export function getCredentialKeysForProvider(providerId: string): string[] { + return [...(PROVIDER_CREDENTIAL_KEYS.get(providerId) ?? [])]; +} + +export function clearCredentialOverlay(): void { + overlay.clear(); +} + +/** AAD binds each ciphertext to one provider, so a payload can't be replayed + * under a different provider even by someone with DB write access. */ +function aadForProvider(provider: string): string { + return `provider-credentials:${provider}`; +} + +/** Rebuild the overlay from a managed source. A provider's keys are taken only as + * a complete, non-empty set, so a managed username never pairs with an env + * password. The overlay is swapped only after the source resolves, so a failed + * load (the source throws) keeps the last good values. */ +export async function refreshCredentialOverlay(source: CredentialSource): Promise { + const managed = await source(); + overlay.clear(); + for (const [provider, keys] of PROVIDER_CREDENTIAL_KEYS) { + const found = keys + .map((name): [string, string | undefined] => [name, managed.get(name)]) + .filter((entry): entry is [string, string] => entry[1] !== undefined && entry[1].trim().length > 0); + if (found.length === keys.length) { + for (const [name, value] of found) overlay.set(name, value); + } else if (found.length > 0) { + console.warn(`[secrets] ignoring partial managed credential for "${provider}"`); + } + } +} + +/** Self-hosted source: decrypt the organization-less provider_credentials rows + * into a flat env-name→value map, scoping each row to the keys its provider + * declares. A missing key, unknown provider, or undecryptable row contributes + * nothing rather than throwing, so one bad row can't take out the others. */ +export const instanceCredentialSource: CredentialSource = async () => { + const managed = new Map(); + + let key: Buffer | null = null; + try { + key = getEncryptionKey(); + } catch (e) { + if (!(e instanceof EncryptionKeyError)) throw e; + console.warn(`[secrets] ${e.message} — encrypted credentials skipped, env credentials unaffected`); + } + if (!key) return managed; + + const rows = await db + .select({ provider: providerCredentials.provider, encryptedData: providerCredentials.encryptedData }) + .from(providerCredentials) + .where(isNull(providerCredentials.organizationId)); + + for (const row of rows) { + const keys = PROVIDER_CREDENTIAL_KEYS.get(row.provider); + if (!keys) continue; + let record: unknown; + try { + record = JSON.parse(await decryptSecret(row.encryptedData, { key, aad: aadForProvider(row.provider) })); + } catch { + console.warn(`[secrets] ignoring undecryptable credential for "${row.provider}"`); + continue; + } + if (typeof record !== "object" || record === null) continue; + for (const name of keys) { + const value = (record as Record)[name]; + if (typeof value === "string") managed.set(name, value); + } + } + + return managed; +}; + +/** Encrypt a provider's full credential set for the write path. `hint` is the + * last 4 chars of the longest value — enough to recognise which secret is stored + * without revealing it. Throws when no encryption key is set. */ +export async function encryptProviderCredentials( + providerId: string, + record: Record, +): Promise<{ encryptedData: EncryptedPayload; hint: string }> { + const keys = getCredentialKeysForProvider(providerId); + if (keys.length === 0) { + throw new Error(`Provider "${providerId}" has no storable credentials`); + } + const isExactBundle = + Object.keys(record).length === keys.length && + keys.every((name) => typeof record[name] === "string" && record[name].trim().length > 0); + if (!isExactBundle) { + throw new Error(`Credentials for "${providerId}" must contain exactly: ${keys.join(", ")}`); + } + + const key = getEncryptionKey(); + if (!key) { + throw new EncryptionKeyError(`${ENCRYPTION_KEY_ENV} is not set — cannot store encrypted credentials`); + } + const encryptedData = await encryptSecret(JSON.stringify(record), { key, aad: aadForProvider(providerId) }); + return { encryptedData, hint: hintFor(record) }; +} + +function hintFor(record: Record): string { + let longest = ""; + for (const value of Object.values(record)) { + if (value.length > longest.length) longest = value; + } + return longest.slice(-4); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0a20f609..fc2f7a4a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -470,6 +470,9 @@ importers: packages/cloud: dependencies: + '@infisical/sdk': + specifier: 5.0.2 + version: 5.0.2 '@workspace/config': specifier: workspace:* version: link:../config @@ -519,6 +522,9 @@ importers: '@workspace/config': specifier: workspace:* version: link:../config + '@workspace/lib': + specifier: workspace:* + version: link:../lib '@workspace/local': specifier: workspace:* version: link:../local @@ -593,6 +599,9 @@ importers: drizzle-orm: specifier: ^0.45.2 version: 0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.0)(@upstash/redis@1.38.0)(kysely@0.28.17)(pg@8.22.0) + jose: + specifier: 6.2.3 + version: 6.2.3 linkedom: specifier: ^0.18.13 version: 0.18.13 @@ -851,6 +860,23 @@ packages: resolution: {integrity: sha512-cTlrKttbrRHEw3W+0/I609A2Matj5JQaRvfLtEIGZvlN0RaPi+3ANsMeqAyCAVlH/lUIW2tmtBlSMni74lcXeg==} engines: {node: '>=12'} + '@aws-crypto/sha256-browser@5.2.0': + resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==} + + '@aws-crypto/sha256-js@5.2.0': + resolution: {integrity: sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==} + engines: {node: '>=16.0.0'} + + '@aws-crypto/supports-web-crypto@5.2.0': + resolution: {integrity: sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==} + + '@aws-crypto/util@5.2.0': + resolution: {integrity: sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==} + + '@aws-sdk/client-cognito-identity@3.993.0': + resolution: {integrity: sha512-7Ne3Yk/bgQPVebAkv7W+RfhiwTRSbfER9BtbhOa2w/+dIr902LrJf6vrZlxiqaJbGj2ALx8M+ZK1YIHVxSwu9A==} + engines: {node: '>=20.0.0'} + '@aws-sdk/client-sts@3.1090.0': resolution: {integrity: sha512-r3XZJpSqnFsM8Ikv1Se4nMlDnlmIxZAV3Zfw6ILQTzeoJ/pZpX4bixBXg960r2t+XAqVtfa21pJ7X2WQ2vlvDQ==} engines: {node: '>=20.0.0'} @@ -899,10 +925,38 @@ packages: resolution: {integrity: sha512-HEfLv6RdiFu8+HO5CUQjjQ/KCWWAHcQCtSPenaF26lYQSjjzKjgvjwAMGBV9/VMrO8V0faIRyoFWhF548UjTOg==} engines: {node: '>=20.0.0'} + '@aws-sdk/credential-providers@3.993.0': + resolution: {integrity: sha512-1M/nukgPSLqe9krzOKHnE8OylUaKAiokAV3xRLdeExVHcRE7WG5uzCTKWTj1imKvPjDqXq/FWhlbbdWIn7xIwA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-host-header@3.972.34': + resolution: {integrity: sha512-NmaFH7Wvrh4SICPO7hRW7M8i+9OvjHxC155HHi/C3JwMUM9CSSXdOjtYhw1Q4cZAoKFnrcu9gIfWPE9seZY5Nw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-logger@3.972.33': + resolution: {integrity: sha512-wnyJLhO66WsWMCo4TPDFQj1Pa3w4IbA0KDsjQIf1ComTyVPs/HxOmqlXMim5ZPXJQsrWvqRgfCe+vYaa/k39SA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-recursion-detection@3.972.35': + resolution: {integrity: sha512-FEdTIfdyyVfMOYfcSEawLZt8ejRBlPoJ51YZOLBdVTBteH5SwLaiDmvAyVwSAhqiI1ooMesdGK0KOwlV+4FDgQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-user-agent@3.972.63': + resolution: {integrity: sha512-ciBdN5V+RjFX2oLXmxqdVbKfADSYIZb4UTFv0eZ2mD9DwARaeSKW9UI9qfX9hJrRjbmYIYtdbMPqFla20x8ZOA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/nested-clients@3.993.0': + resolution: {integrity: sha512-iOq86f2H67924kQUIPOAvlmMaOAvOLoDOIb66I2YqSUpMYB6ufiuJW3RlREgskxv86S5qKzMnfy/X6CqMjK6XQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/nested-clients@3.997.33': resolution: {integrity: sha512-dVZOroI/r3/ENvqNGgjMPul+jjlz9GddfVusgTXlVjfZj5isibOxecLkGQbRPp8XOuX+RAfjXLFgPkD1JS5xrw==} engines: {node: '>=20.0.0'} + '@aws-sdk/region-config-resolver@3.972.37': + resolution: {integrity: sha512-O0mWXYkM1ASgX72KKW3zNG+pEvpmUG5yl6uykgA/p2xkVFi/SjUxwAZBv2ToqKba/TBwDU8tv2wL5Rq5ZTcdMg==} + engines: {node: '>=20.0.0'} + '@aws-sdk/signature-v4-multi-region@3.996.41': resolution: {integrity: sha512-QMUytg+FQMGouc8gHS00KoYih3+N6cqmVI/pQGOIo7Nr7OpQaiXjSYOuL+vsPZ1tymY4LAQ8MYcHJmws5LRxng==} engines: {node: '>=20.0.0'} @@ -915,6 +969,21 @@ packages: resolution: {integrity: sha512-3W6IUtSxFbH6X7Wb7DzGCV5QiFQsd0g8bOfntpmDxQlzBoKWUMBu/JPQR0DwkE+Hpnxd6db1tXbOwdeHddG6cA==} engines: {node: '>=20.0.0'} + '@aws-sdk/util-endpoints@3.993.0': + resolution: {integrity: sha512-j6vioBeRZ4eHX4SWGvGPpwGg/xSOcK7f1GL0VM+rdf3ZFTIsUEhCFmD78B+5r2PgztcECSzEfvHQX01k8dPQPw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/util-locate-window@3.965.8': + resolution: {integrity: sha512-uUbMs1cBZPafD0ohUj6EwNf0fPZ534NvBxHox4hjX+0Rxq5paSYUem7+hi833pYrzrcnBATKIYpR02MDXT5M9g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/util-user-agent-browser@3.972.34': + resolution: {integrity: sha512-QkoA/jvhG6GDeghtIOcsuSQKqMwt75waXVjlf0WIkUhaNE1f/GrZ+/p/01yWI9cJdmO4T4YjyB/ZRdcvU1ToZw==} + + '@aws-sdk/util-user-agent-node@3.973.49': + resolution: {integrity: sha512-gZyJ3fmVq9RONzxk9BYeq1ZRc7d+ZLsytswREgaNTclvBQ7GTm4jlXMEhOTE5g1Ce9rhfWSHmm9xOwldHqtoDQ==} + engines: {node: '>=20.0.0'} + '@aws-sdk/xml-builder@3.972.36': resolution: {integrity: sha512-RdGmS1GLrtaTOLE1ElSluMldNrpk9Emq6uYs8SS8iHlu5xTAmM9rRkM91o48+rIRryBtyO9t+uLYCoMG6jVMVA==} engines: {node: '>=20.0.0'} @@ -1812,6 +1881,10 @@ packages: peerDependencies: react-hook-form: ^7.55.0 + '@infisical/sdk@5.0.2': + resolution: {integrity: sha512-mkgghtmIgBOEOFj1gb2n/AoiroFvN+NqaEEwC8ezEQ87V55yf2yksHhnBMU2MiZ26n6b3sGcpDZHF1/H6JrgyA==} + engines: {node: '>=20'} + '@inquirer/external-editor@1.0.3': resolution: {integrity: sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==} engines: {node: '>=18'} @@ -3859,6 +3932,10 @@ packages: engines: {node: '>= 8.0.0'} hasBin: true + '@smithy/config-resolver@4.6.10': + resolution: {integrity: sha512-OSWakjTQRuMCps5YE8fOxGdDv7gEtanlyn5KtfCBgDE+dvaUPJH7xDQRADRnnBoMh792cGFUsvFLmxobxOpbkg==} + engines: {node: '>=18.0.0'} + '@smithy/core@3.29.5': resolution: {integrity: sha512-i0dk2t5B+CwV/dcJdUHILYkOQF5lof8f44dFCfDWToGCxjT9YQ+CgHqTAvJxzc3+zqQwm2QtVoJ5IqiNar/CnQ==} engines: {node: '>=18.0.0'} @@ -3871,18 +3948,114 @@ packages: resolution: {integrity: sha512-3zpg8yqqyXzoK2TsRDdkqVOj2RDBFfLXwCczOZ5c7TWB4eiaebfSCsbMjDPYB3PJ9ihV62QaeadZ+wLadZtNGA==} engines: {node: '>=18.0.0'} + '@smithy/hash-node@4.4.10': + resolution: {integrity: sha512-y0CfGFpEVyGg6EfrBj0BWkgfaRiXk+dG4t3xYNPMbnkOBBy8Yv8THUDsmkHXe2PyzYIzC6uPlT97kZoX6KF9vg==} + engines: {node: '>=18.0.0'} + + '@smithy/invalid-dependency@4.4.10': + resolution: {integrity: sha512-gF+13AFjZoEjRcUNdkA2Q8tjL2/m3f7ipcZ2w35Hj6Sm9A+gj+yEsuGV70X3UPHar5tTl+385F4Zi9DBaIW3Ag==} + engines: {node: '>=18.0.0'} + + '@smithy/is-array-buffer@2.2.0': + resolution: {integrity: sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==} + engines: {node: '>=14.0.0'} + + '@smithy/middleware-content-length@4.4.10': + resolution: {integrity: sha512-fF04hl+iHSBW1xuXKaKDwp2/OSiClOYHi16+KZxmlTu/DIoy0VyOVaPnJh+ZlKXeYKMz/jR8/eoQ1rDHpft8qA==} + engines: {node: '>=18.0.0'} + + '@smithy/middleware-endpoint@4.6.10': + resolution: {integrity: sha512-AxGe7eMGOwmANI/K6O6a95hTInLocQslDVwCiidV7tat0JDcbpRr0c33VwieO8P3mlh7mqiNhBlyG7VlzZxZIg==} + engines: {node: '>=18.0.0'} + + '@smithy/middleware-retry@4.7.10': + resolution: {integrity: sha512-JY4Oi57f0uVIncDmNnUEGNkyoJZ7bw9rVZPIrgRyEPdDhcCDvhYYomY/FTQBS4JnrJx168c739NLoJzsygGdxA==} + engines: {node: '>=18.0.0'} + + '@smithy/middleware-serde@4.4.10': + resolution: {integrity: sha512-x1LF7xwoZfvnU1G6wYV9Zm788t13CTvx+QPtdBMxNOBnAc1xArpO4ekr0lPEs7nKbvYgCecLwBZa3hjwMHWwtA==} + engines: {node: '>=18.0.0'} + + '@smithy/middleware-stack@4.4.10': + resolution: {integrity: sha512-Uw0gfjz9HgGaaL6+AIMMfs0put11pnuRJltHNU6TU73BfR3Cowms14WM4ovSDxkpJrgNlSQb1XKJVVNUtHhIkQ==} + engines: {node: '>=18.0.0'} + + '@smithy/node-config-provider@4.5.10': + resolution: {integrity: sha512-8J0iQAEQ+WKwUh8NuR7SEr3wdSbXSvTQ2PLMcgWmL8TGqHsy0glEnaZvbuYCQr4RkZjxJ3mfSI2BsbPrxcsRIA==} + engines: {node: '>=18.0.0'} + '@smithy/node-http-handler@4.9.7': resolution: {integrity: sha512-wCU8HCLjAtAVqxxe0j2xff9LcEPw3yjBbg5IdQDIYFnxnPxbxcSLc7rgex7kqm9L/WYOnJEgaWQlfDkZleozMA==} engines: {node: '>=18.0.0'} + '@smithy/property-provider@4.4.10': + resolution: {integrity: sha512-xAxI0w1j9CxzSL3LwRY65/LOuctKBvpnUqEcx5kTNNDqB+8+tw+MjFzCnW2meRswGjSRDYqNvtMie6CxkYmkrA==} + engines: {node: '>=18.0.0'} + + '@smithy/protocol-http@5.5.10': + resolution: {integrity: sha512-0AnsOUvCHQBOXnBJbrX9QGidpvbb3F4aIUFlWPXCfRYxYCcoxj9Uq+b8j75wIZRIwhwiKDWJDZmoHjjQ82GYZA==} + engines: {node: '>=18.0.0'} + '@smithy/signature-v4@5.6.6': resolution: {integrity: sha512-efP6DN3UTFrzIsGO42/xcabv8jU7+9nwEdphFUH7yL0k010ERyAWaO41KFQIDLcFZLZ8xzIQr4wplFxNzslSGQ==} engines: {node: '>=18.0.0'} + '@smithy/smithy-client@4.14.10': + resolution: {integrity: sha512-afUBB2TJBFTlC3axCVDoVStuwkIzMlAAMY/9/pNyE3uspyTxJRczEb/5YiXuE2V1CfX4uGEQPpwZ8mtTzyq5yA==} + engines: {node: '>=18.0.0'} + '@smithy/types@4.16.1': resolution: {integrity: sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==} engines: {node: '>=18.0.0'} + '@smithy/url-parser@4.4.10': + resolution: {integrity: sha512-f1bDXQ9mDozbWok054W8GKwuk7qI33E3S6RvbHtplaBc3BVzxh94zndEqDqTxdxlqKRplsmhuNO0hOhU4AQQ2A==} + engines: {node: '>=18.0.0'} + + '@smithy/util-base64@4.5.10': + resolution: {integrity: sha512-piUTpCteEPeqiIw5nU/8phVUSpjXRU6M5rEF3XUyUJtqP5LUkvVf9zpWCbYqv7kj3SzBW36sKW2c0CiOs2fA/A==} + engines: {node: '>=18.0.0'} + + '@smithy/util-body-length-browser@4.4.10': + resolution: {integrity: sha512-/TTYEuHmnPHDHociNoSsz9EeoeS91oYwuBIHYcCqwGGOW6B+T0Xll03S0Q8UepI40FuSf3mMrkre7npFCHhttg==} + engines: {node: '>=18.0.0'} + + '@smithy/util-body-length-node@4.4.10': + resolution: {integrity: sha512-8HYVvYG32E2wLq4MTlgRlcghZxcj6BvBXkSMqULEJQFLnQeiHptQ2xUYH98KJwaC7CBPrZawQwUfcmEicJs4AQ==} + engines: {node: '>=18.0.0'} + + '@smithy/util-buffer-from@2.2.0': + resolution: {integrity: sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==} + engines: {node: '>=14.0.0'} + + '@smithy/util-defaults-mode-browser@4.5.10': + resolution: {integrity: sha512-bQGtrcVQHfJJQ1bhdZ4I1Q8GwJRw0WqPIcRocvwQ/ukfaCEw22acQhCxEj7UiNYwD7ij5mOXFp8e+FdbVDd6Yw==} + engines: {node: '>=18.0.0'} + + '@smithy/util-defaults-mode-node@4.4.10': + resolution: {integrity: sha512-kwmn00irjhU1hRMYJiPHN3EgXiJbuSh5p6cCHSds3T+8YsMUisWuiVp7zlwe68UsJPFeBKPDGDJ+MM8Kr43Ltw==} + engines: {node: '>=18.0.0'} + + '@smithy/util-endpoints@3.6.10': + resolution: {integrity: sha512-MqqMomCIaSpYm/FDOfhRMt+3IRI+18YSB9ukoIEocXr0MFBUzkKnrsMp6NnBB2zvDfpJtbA/AWGwqOT9lM9sFQ==} + engines: {node: '>=18.0.0'} + + '@smithy/util-middleware@4.4.10': + resolution: {integrity: sha512-jFkCTa/hzSc+rDc3yzHkv6OI/rLjAy88rcgZ0lP8NdkZhgR9N3TiDj6lW1hAYt42bwx8HpwPJCQegj3I2YGRfg==} + engines: {node: '>=18.0.0'} + + '@smithy/util-retry@4.5.10': + resolution: {integrity: sha512-hYu5ieq8myuO29xCQV0IIhicRm1aO0lcNBeMcF1mVVAVQC0oylPGKFliMq5NbxtdOvRxJWCMuir2gwKI91f1lw==} + engines: {node: '>=18.0.0'} + + '@smithy/util-utf8@2.3.0': + resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==} + engines: {node: '>=14.0.0'} + + '@smithy/util-utf8@4.4.10': + resolution: {integrity: sha512-4pWFv2sxrykZqaTixXhkgAsG6+k/VxgAiyZ6M0iLEmsOqcJaR0eidlTCmuKKtMJMIEw8lYwDTvEyR4NyC+V0cw==} + engines: {node: '>=18.0.0'} + '@solid-primitives/event-listener@2.4.5': resolution: {integrity: sha512-nwRV558mIabl4yVAhZKY8cb6G+O1F0M6Z75ttTu5hk+SxdOnKSGj+eetDIu7Oax1P138ZdUU01qnBPR8rnxaEA==} peerDependencies: @@ -8388,6 +8561,11 @@ packages: resolution: {integrity: sha512-YGYEVz3Fm5iy/AybuA0oyNFq7H4CgQNfRp/qfe8nurE1kuCeNm3/vfm9X4Mtl+qLyaKJUh5xrFZwogr41SMjYA==} engines: {node: '>=20'} + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + typescript@7.0.2: resolution: {integrity: sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==} engines: {node: '>=16.20.0'} @@ -8897,6 +9075,9 @@ packages: resolution: {integrity: sha512-dSvYKdvLsAHCDqPOhIwk/q5CvuWtTB3Dgpoe0uVEFjTzIOAmsQpprX25InCvrvJsirEbu1OHyy67n/kAj1Sw/w==} engines: {node: '>=18'} + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + zod@4.1.11: resolution: {integrity: sha512-WPsqwxITS2tzx1bzhIKsEs19ABD5vmCVa4xBo2tq/SrV4RNZtfws1EnCWQXM6yh8bD08a1idvkB5MZSBiZsjwg==} @@ -8978,6 +9159,74 @@ snapshots: escape-html: 1.0.3 xpath: 0.0.32 + '@aws-crypto/sha256-browser@5.2.0': + dependencies: + '@aws-crypto/sha256-js': 5.2.0 + '@aws-crypto/supports-web-crypto': 5.2.0 + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.974.2 + '@aws-sdk/util-locate-window': 3.965.8 + '@smithy/util-utf8': 2.3.0 + tslib: 2.8.1 + + '@aws-crypto/sha256-js@5.2.0': + dependencies: + '@aws-crypto/util': 5.2.0 + '@aws-sdk/types': 3.974.2 + tslib: 2.8.1 + + '@aws-crypto/supports-web-crypto@5.2.0': + dependencies: + tslib: 2.8.1 + + '@aws-crypto/util@5.2.0': + dependencies: + '@aws-sdk/types': 3.974.2 + '@smithy/util-utf8': 2.3.0 + tslib: 2.8.1 + + '@aws-sdk/client-cognito-identity@3.993.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.975.3 + '@aws-sdk/credential-provider-node': 3.972.70 + '@aws-sdk/middleware-host-header': 3.972.34 + '@aws-sdk/middleware-logger': 3.972.33 + '@aws-sdk/middleware-recursion-detection': 3.972.35 + '@aws-sdk/middleware-user-agent': 3.972.63 + '@aws-sdk/region-config-resolver': 3.972.37 + '@aws-sdk/types': 3.974.2 + '@aws-sdk/util-endpoints': 3.993.0 + '@aws-sdk/util-user-agent-browser': 3.972.34 + '@aws-sdk/util-user-agent-node': 3.973.49 + '@smithy/config-resolver': 4.6.10 + '@smithy/core': 3.29.5 + '@smithy/fetch-http-handler': 5.6.7 + '@smithy/hash-node': 4.4.10 + '@smithy/invalid-dependency': 4.4.10 + '@smithy/middleware-content-length': 4.4.10 + '@smithy/middleware-endpoint': 4.6.10 + '@smithy/middleware-retry': 4.7.10 + '@smithy/middleware-serde': 4.4.10 + '@smithy/middleware-stack': 4.4.10 + '@smithy/node-config-provider': 4.5.10 + '@smithy/node-http-handler': 4.9.7 + '@smithy/protocol-http': 5.5.10 + '@smithy/smithy-client': 4.14.10 + '@smithy/types': 4.16.1 + '@smithy/url-parser': 4.4.10 + '@smithy/util-base64': 4.5.10 + '@smithy/util-body-length-browser': 4.4.10 + '@smithy/util-body-length-node': 4.4.10 + '@smithy/util-defaults-mode-browser': 4.5.10 + '@smithy/util-defaults-mode-node': 4.4.10 + '@smithy/util-endpoints': 3.6.10 + '@smithy/util-middleware': 4.4.10 + '@smithy/util-retry': 4.5.10 + '@smithy/util-utf8': 4.4.10 + tslib: 2.8.1 + '@aws-sdk/client-sts@3.1090.0': dependencies: '@aws-sdk/core': 3.975.3 @@ -9112,6 +9361,90 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/credential-providers@3.993.0': + dependencies: + '@aws-sdk/client-cognito-identity': 3.993.0 + '@aws-sdk/core': 3.975.3 + '@aws-sdk/credential-provider-cognito-identity': 3.972.58 + '@aws-sdk/credential-provider-env': 3.972.59 + '@aws-sdk/credential-provider-http': 3.972.61 + '@aws-sdk/credential-provider-ini': 3.973.4 + '@aws-sdk/credential-provider-login': 3.972.66 + '@aws-sdk/credential-provider-node': 3.972.70 + '@aws-sdk/credential-provider-process': 3.972.59 + '@aws-sdk/credential-provider-sso': 3.973.3 + '@aws-sdk/credential-provider-web-identity': 3.972.65 + '@aws-sdk/nested-clients': 3.993.0 + '@aws-sdk/types': 3.974.2 + '@smithy/config-resolver': 4.6.10 + '@smithy/core': 3.29.5 + '@smithy/credential-provider-imds': 4.4.10 + '@smithy/node-config-provider': 4.5.10 + '@smithy/property-provider': 4.4.10 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + + '@aws-sdk/middleware-host-header@3.972.34': + dependencies: + '@aws-sdk/core': 3.975.3 + tslib: 2.8.1 + + '@aws-sdk/middleware-logger@3.972.33': + dependencies: + '@aws-sdk/core': 3.975.3 + tslib: 2.8.1 + + '@aws-sdk/middleware-recursion-detection@3.972.35': + dependencies: + '@aws-sdk/core': 3.975.3 + tslib: 2.8.1 + + '@aws-sdk/middleware-user-agent@3.972.63': + dependencies: + '@aws-sdk/core': 3.975.3 + tslib: 2.8.1 + + '@aws-sdk/nested-clients@3.993.0': + dependencies: + '@aws-crypto/sha256-browser': 5.2.0 + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/core': 3.975.3 + '@aws-sdk/middleware-host-header': 3.972.34 + '@aws-sdk/middleware-logger': 3.972.33 + '@aws-sdk/middleware-recursion-detection': 3.972.35 + '@aws-sdk/middleware-user-agent': 3.972.63 + '@aws-sdk/region-config-resolver': 3.972.37 + '@aws-sdk/types': 3.974.2 + '@aws-sdk/util-endpoints': 3.993.0 + '@aws-sdk/util-user-agent-browser': 3.972.34 + '@aws-sdk/util-user-agent-node': 3.973.49 + '@smithy/config-resolver': 4.6.10 + '@smithy/core': 3.29.5 + '@smithy/fetch-http-handler': 5.6.7 + '@smithy/hash-node': 4.4.10 + '@smithy/invalid-dependency': 4.4.10 + '@smithy/middleware-content-length': 4.4.10 + '@smithy/middleware-endpoint': 4.6.10 + '@smithy/middleware-retry': 4.7.10 + '@smithy/middleware-serde': 4.4.10 + '@smithy/middleware-stack': 4.4.10 + '@smithy/node-config-provider': 4.5.10 + '@smithy/node-http-handler': 4.9.7 + '@smithy/protocol-http': 5.5.10 + '@smithy/smithy-client': 4.14.10 + '@smithy/types': 4.16.1 + '@smithy/url-parser': 4.4.10 + '@smithy/util-base64': 4.5.10 + '@smithy/util-body-length-browser': 4.4.10 + '@smithy/util-body-length-node': 4.4.10 + '@smithy/util-defaults-mode-browser': 4.5.10 + '@smithy/util-defaults-mode-node': 4.4.10 + '@smithy/util-endpoints': 3.6.10 + '@smithy/util-middleware': 4.4.10 + '@smithy/util-retry': 4.5.10 + '@smithy/util-utf8': 4.4.10 + tslib: 2.8.1 + '@aws-sdk/nested-clients@3.997.33': dependencies: '@aws-sdk/core': 3.975.3 @@ -9123,6 +9456,11 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/region-config-resolver@3.972.37': + dependencies: + '@aws-sdk/core': 3.975.3 + tslib: 2.8.1 + '@aws-sdk/signature-v4-multi-region@3.996.41': dependencies: '@aws-sdk/types': 3.974.2 @@ -9144,6 +9482,28 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@aws-sdk/util-endpoints@3.993.0': + dependencies: + '@aws-sdk/types': 3.974.2 + '@smithy/types': 4.16.1 + '@smithy/url-parser': 4.4.10 + '@smithy/util-endpoints': 3.6.10 + tslib: 2.8.1 + + '@aws-sdk/util-locate-window@3.965.8': + dependencies: + tslib: 2.8.1 + + '@aws-sdk/util-user-agent-browser@3.972.34': + dependencies: + '@aws-sdk/core': 3.975.3 + tslib: 2.8.1 + + '@aws-sdk/util-user-agent-node@3.973.49': + dependencies: + '@aws-sdk/core': 3.975.3 + tslib: 2.8.1 + '@aws-sdk/xml-builder@3.972.36': dependencies: '@smithy/types': 4.16.1 @@ -10010,6 +10370,15 @@ snapshots: '@standard-schema/utils': 0.3.0 react-hook-form: 7.82.0(react@19.2.7) + '@infisical/sdk@5.0.2': + dependencies: + '@aws-crypto/sha256-js': 5.2.0 + '@aws-sdk/credential-providers': 3.993.0 + '@smithy/protocol-http': 5.5.10 + '@smithy/signature-v4': 5.6.6 + typescript: 5.9.3 + zod: 3.25.76 + '@inquirer/external-editor@1.0.3(@types/node@26.1.1)': dependencies: chardet: 2.2.0 @@ -11790,6 +12159,11 @@ snapshots: fflate: 0.7.4 string.prototype.codepointat: 0.2.1 + '@smithy/config-resolver@4.6.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + '@smithy/core@3.29.5': dependencies: '@smithy/types': 4.16.1 @@ -11807,22 +12181,142 @@ snapshots: '@smithy/types': 4.16.1 tslib: 2.8.1 + '@smithy/hash-node@4.4.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/invalid-dependency@4.4.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/is-array-buffer@2.2.0': + dependencies: + tslib: 2.8.1 + + '@smithy/middleware-content-length@4.4.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/middleware-endpoint@4.6.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/middleware-retry@4.7.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/middleware-serde@4.4.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/middleware-stack@4.4.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/node-config-provider@4.5.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + '@smithy/node-http-handler@4.9.7': dependencies: '@smithy/core': 3.29.5 '@smithy/types': 4.16.1 tslib: 2.8.1 + '@smithy/property-provider@4.4.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/protocol-http@5.5.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + '@smithy/signature-v4@5.6.6': dependencies: '@smithy/core': 3.29.5 '@smithy/types': 4.16.1 tslib: 2.8.1 + '@smithy/smithy-client@4.14.10': + dependencies: + '@smithy/core': 3.29.5 + '@smithy/types': 4.16.1 + tslib: 2.8.1 + '@smithy/types@4.16.1': dependencies: tslib: 2.8.1 + '@smithy/url-parser@4.4.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/util-base64@4.5.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/util-body-length-browser@4.4.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/util-body-length-node@4.4.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/util-buffer-from@2.2.0': + dependencies: + '@smithy/is-array-buffer': 2.2.0 + tslib: 2.8.1 + + '@smithy/util-defaults-mode-browser@4.5.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/util-defaults-mode-node@4.4.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/util-endpoints@3.6.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/util-middleware@4.4.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/util-retry@4.5.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + + '@smithy/util-utf8@2.3.0': + dependencies: + '@smithy/util-buffer-from': 2.2.0 + tslib: 2.8.1 + + '@smithy/util-utf8@4.4.10': + dependencies: + '@smithy/core': 3.29.5 + tslib: 2.8.1 + '@solid-primitives/event-listener@2.4.5(solid-js@1.9.11)': dependencies: '@solid-primitives/utils': 6.4.0(solid-js@1.9.11) @@ -16800,6 +17294,8 @@ snapshots: dependencies: tagged-tag: 1.0.0 + typescript@5.9.3: {} + typescript@7.0.2: optionalDependencies: '@typescript/typescript-aix-ppc64': 7.0.2 @@ -17204,6 +17700,8 @@ snapshots: normalize-path: 3.0.0 readable-stream: 4.7.0 + zod@3.25.76: {} + zod@4.1.11: {} zod@4.4.3: {} diff --git a/turbo.json b/turbo.json index 0a2c6e98..f7f1195d 100644 --- a/turbo.json +++ b/turbo.json @@ -25,6 +25,13 @@ "OXYLABS_PASSWORD", "OPENROUTER_API_KEY", "JINA_API_KEY", + "ELMO_ENCRYPTION_KEY", + "INFISICAL_CLIENT_ID", + "INFISICAL_CLIENT_SECRET", + "INFISICAL_PROJECT_ID", + "INFISICAL_ENVIRONMENT", + "INFISICAL_SECRET_PATH", + "INFISICAL_SITE_URL", "DEPLOYMENT_MODE", "ADMIN_AUTH0_SUB", "ADMIN_API_KEYS", From 6327f6aaa778282d44a14a35c392a003294fdf1d Mon Sep 17 00:00:00 2001 From: jrhizor Date: Sat, 25 Jul 2026 13:22:48 -0700 Subject: [PATCH 2/6] Address review on encrypted provider credentials MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fail loudly instead of silently unconfiguring providers. An Infisical read that succeeds but returns nothing is an outage, not an empty credential set, so the loader throws and the overlay keeps its last good values. Misread keys, undecryptable rows and vanished credentials log errors with the action to take. Take Infisical off the web request path. The cloud web app runs on Vercel serverless, where a runtime read would add an authentication round trip to every cold start and put Infisical in front of the whole site; Infisical's Vercel secret sync populates the environment at deploy time instead. Only the worker, which is long-lived and spends against these credentials, keeps the SDK loader — so the Infisical vars are no longer web startup requirements. Read one Infisical folder rather than recursing from the project root, so the machine identity needs no broader grant and two folders cannot both define OPENAI_API_KEY with no defined winner. Drop what the payload does not need: typ and v leave the JWE header, since ctx is what stops a ciphertext being re-homed onto another provider. Drop the hint column, which stored the last four characters of the longest value in a bundle — the password, for Oxylabs and DataForSEO — in plaintext beside the ciphertext, and which nothing reads. Delete the unused dataforseo client that captured credentials from the environment at module load. --- .changeset/encrypted-provider-credentials.md | 5 +++ apps/web/scripts/smoke-deployment-mode.ts | 4 -- apps/web/src/server.ts | 13 +------ packages/cloud/README.md | 25 ++++++++++-- .../cloud/src/infisical-credentials.test.ts | 11 +++++- packages/cloud/src/infisical-credentials.ts | 14 ++++++- packages/config/src/env-registry.ts | 16 +++++--- packages/config/src/env.test.ts | 8 ---- packages/config/src/env.ts | 6 ++- packages/deployment/src/credentials.ts | 21 +++++++++- .../docs/developer-guide/configuration.mdx | 9 +++++ packages/lib/package.json | 1 - packages/lib/src/dataforseo.ts | 24 ------------ .../migrations/0011_provider_credentials.sql | 1 - .../src/db/migrations/meta/0011_snapshot.json | 6 --- packages/lib/src/db/schema.ts | 1 - packages/lib/src/secrets/crypto.test.ts | 29 +++++--------- packages/lib/src/secrets/crypto.ts | 23 ++++------- packages/lib/src/secrets/store.test.ts | 33 ++++++++++++++-- packages/lib/src/secrets/store.ts | 39 +++++++++++-------- 20 files changed, 163 insertions(+), 126 deletions(-) create mode 100644 .changeset/encrypted-provider-credentials.md delete mode 100644 packages/lib/src/dataforseo.ts diff --git a/.changeset/encrypted-provider-credentials.md b/.changeset/encrypted-provider-credentials.md new file mode 100644 index 00000000..01275d88 --- /dev/null +++ b/.changeset/encrypted-provider-credentials.md @@ -0,0 +1,5 @@ +--- +"@elmohq/cli": patch +--- + +Added an `ELMO_ENCRYPTION_KEY` to new and upgraded deployments, which lets Elmo store provider credentials encrypted in the database instead of only in `.env`. diff --git a/apps/web/scripts/smoke-deployment-mode.ts b/apps/web/scripts/smoke-deployment-mode.ts index 1bbdac23..14cb36d7 100644 --- a/apps/web/scripts/smoke-deployment-mode.ts +++ b/apps/web/scripts/smoke-deployment-mode.ts @@ -83,10 +83,6 @@ const MINIMAL_ENV: Record> = { RESEND_FROM_EMAIL: "Smoke ", GOOGLE_CLIENT_ID: "smoke-google-client-id", GOOGLE_CLIENT_SECRET: "smoke-google-client-secret", - INFISICAL_CLIENT_ID: "smoke-client-id", - INFISICAL_CLIENT_SECRET: "smoke-client-secret", - INFISICAL_PROJECT_ID: "smoke-project-id", - INFISICAL_ENVIRONMENT: "prod", }, }; diff --git a/apps/web/src/server.ts b/apps/web/src/server.ts index 55b04ed9..17e8b605 100644 --- a/apps/web/src/server.ts +++ b/apps/web/src/server.ts @@ -1,17 +1,9 @@ import "../instrument.server.mjs"; import { wrapFetchWithSentry } from "@sentry/tanstackstart-react"; import handler, { createServerEntry } from "@tanstack/react-start/server-entry"; -import { startCredentialRefresh } from "@workspace/deployment/credentials"; +import { startBackgroundCredentialRefresh } from "@workspace/deployment/credentials"; -let credentialsReady: Promise | undefined; - -function ensureCredentialsReady(): Promise { - credentialsReady ??= startCredentialRefresh().catch((error) => { - credentialsReady = undefined; - throw error; - }); - return credentialsReady; -} +startBackgroundCredentialRefresh(); // HSTS asserts HTTPS-only for the host that served the response. Whitelabel // deployments run on customer-controlled custom domains, where `includeSubDomains` @@ -54,7 +46,6 @@ function addSecurityHeaders(response: Response): Response { export default createServerEntry( wrapFetchWithSentry({ async fetch(request: Request) { - await ensureCredentialsReady(); const response = await handler.fetch(request); return addSecurityHeaders(response); }, diff --git a/packages/cloud/README.md b/packages/cloud/README.md index 68ab98bb..a6795eb3 100644 --- a/packages/cloud/README.md +++ b/packages/cloud/README.md @@ -12,6 +12,11 @@ The auth and email features in this package need: | `RESEND_FROM_EMAIL` | Sender address, e.g. `Elmo ` | | `GOOGLE_CLIENT_ID` | Google OAuth client ID for social sign-in | | `GOOGLE_CLIENT_SECRET` | Google OAuth client secret | + +The worker additionally needs the Infisical variables below. The web app does not — see the Infisical section for why. + +| Variable | Purpose | +| --- | --- | | `INFISICAL_CLIENT_ID` | Infisical machine identity client ID | | `INFISICAL_CLIENT_SECRET` | Infisical machine identity client secret | | `INFISICAL_PROJECT_ID` | Project containing provider credentials | @@ -31,11 +36,23 @@ Email templates are code — `packages/cloud/src/email-templates.ts` — not Res ## Infisical setup -1. Create a machine identity with Universal Auth and read access to the provider-credential path. -2. Add provider credentials using their canonical environment-variable names, such as `OPENAI_API_KEY`, `OXYLABS_USERNAME`, and `OXYLABS_PASSWORD`. -3. Configure the required project, environment, client ID, and client secret variables above. +Provider credentials live in one Infisical folder, named exactly as the canonical environment variables (`OPENAI_API_KEY`, `OXYLABS_USERNAME`, `OXYLABS_PASSWORD`, …). The two runtimes read that folder differently. + +**Worker** — a long-lived VM, so it authenticates once with the SDK and refreshes every minute. Rotating a credential in Infisical reaches it without a deploy. + +1. Create a machine identity with Universal Auth and read access to the provider-credential path — that path only, since the loader is not recursive. +2. Set the Infisical variables above on the worker. + +A refresh that fails, or that returns no provider credentials at all, keeps the last known-good values and logs an error; the worker refuses to start if the very first load fails. + +**Web app** — runs on Vercel serverless. Use Infisical's [Vercel secret sync](https://infisical.com/docs/integrations/secret-syncs/vercel) to push the same folder into the project's environment variables, and the app reads them from `process.env`. + +1. Add a Vercel app connection in Infisical, then a secret sync from the provider-credential folder to the Vercel project. +2. Enable auto-sync, and redeploy after a rotation — Vercel bakes environment variables in at deploy time. + +Reading Infisical at runtime instead would put an authentication round trip on every cold start and Infisical's availability in front of the whole site, to serve credentials the request path barely uses. -The web server and worker load credentials at startup and refresh them every minute. Failed refreshes retain the last successfully loaded values. Infisical is used only by `DEPLOYMENT_MODE=cloud`; self-hosted deployments use encrypted database credentials or environment variables. +Infisical is only used by `DEPLOYMENT_MODE=cloud`. Self-hosted deployments use encrypted database credentials or environment variables. ## Google OAuth setup diff --git a/packages/cloud/src/infisical-credentials.test.ts b/packages/cloud/src/infisical-credentials.test.ts index a6b872eb..a5e7929f 100644 --- a/packages/cloud/src/infisical-credentials.test.ts +++ b/packages/cloud/src/infisical-credentials.test.ts @@ -46,12 +46,21 @@ describe("createInfisicalCredentialLoader", () => { environment: "prod", projectId: "project-id", secretPath: "/elmo/providers", - recursive: true, expandSecretReferences: true, viewSecretValue: true, }); }); + // Cloud has no environment fallback, so an empty read has to fail loudly — + // otherwise a revoked grant or a stale path silently unconfigures every + // provider and the fleet stops scraping with no error anywhere. + it("throws rather than reporting zero credentials", async () => { + const { client } = fakeClient([{ secretKey: "NOT_AN_ELMO_CREDENTIAL", secretValue: "ignored" }]); + const load = createInfisicalCredentialLoader({ env: ENV, clientFactory: () => client }); + + await expect(load()).rejects.toThrow(/no provider credentials/); + }); + it("reuses the authenticated client across successful refreshes", async () => { const { client } = fakeClient([{ secretKey: "OPENAI_API_KEY", secretValue: "sk-cloud" }]); const clientFactory = vi.fn(() => client); diff --git a/packages/cloud/src/infisical-credentials.ts b/packages/cloud/src/infisical-credentials.ts index 84820c86..fe4e9d89 100644 --- a/packages/cloud/src/infisical-credentials.ts +++ b/packages/cloud/src/infisical-credentials.ts @@ -20,7 +20,6 @@ interface InfisicalClient { environment: string; projectId: string; secretPath: string; - recursive: boolean; expandSecretReferences: boolean; viewSecretValue: boolean; }): Promise>; @@ -57,11 +56,13 @@ export function createInfisicalCredentialLoader(options: InfisicalCredentialLoad }; const list = async () => { const client = await authenticate(); + // Non-recursive: the identity reads exactly one folder, so unrelated + // project secrets never reach this process and two folders cannot both + // define OPENAI_API_KEY with no defined winner. return client.secrets().listSecretsWithImports({ environment, projectId, secretPath, - recursive: true, expandSecretReferences: true, viewSecretValue: true, }); @@ -84,6 +85,15 @@ export function createInfisicalCredentialLoader(options: InfisicalCredentialLoad credentials.set(secret.secretKey, secret.secretValue); } } + // Cloud has no environment fallback, so "the call succeeded and returned + // nothing" is an outage — a revoked read grant, or a path/environment slug + // that no longer resolves. Throwing keeps the last good overlay in place + // instead of quietly unconfiguring every provider. + if (credentials.size === 0) { + throw new Error( + `Infisical returned no provider credentials from ${secretPath} in "${environment}" — check the machine identity's read access and ${SECRET_PATH_ENV}`, + ); + } return credentials; }; } diff --git a/packages/config/src/env-registry.ts b/packages/config/src/env-registry.ts index d9829f79..4522399f 100644 --- a/packages/config/src/env-registry.ts +++ b/packages/config/src/env-registry.ts @@ -208,28 +208,32 @@ export const ENV_REGISTRY: EnvVarSpec[] = [ requiredBy: "optional", description: "Base64-encoded 32-byte key enabling encrypted provider credentials in self-hosted deployments.", }, + // Only the managed-cloud worker talks to Infisical. The cloud web app reads + // provider credentials from its environment, which Infisical's Vercel secret + // sync populates at deploy time — so these are not web startup requirements, + // and createInfisicalCredentialLoader fails the worker fast if any is absent. { name: "INFISICAL_CLIENT_ID", scope: "server", - requiredBy: ["cloud"], - description: "Infisical machine identity client ID used by managed cloud.", + requiredBy: "optional", + description: "Infisical machine identity client ID used by the managed cloud worker.", }, { name: "INFISICAL_CLIENT_SECRET", scope: "server", - requiredBy: ["cloud"], - description: "Infisical machine identity client secret used by managed cloud.", + requiredBy: "optional", + description: "Infisical machine identity client secret used by the managed cloud worker.", }, { name: "INFISICAL_PROJECT_ID", scope: "server", - requiredBy: ["cloud"], + requiredBy: "optional", description: "Infisical project containing managed cloud provider credentials.", }, { name: "INFISICAL_ENVIRONMENT", scope: "server", - requiredBy: ["cloud"], + requiredBy: "optional", description: "Infisical environment slug containing managed cloud provider credentials.", }, { diff --git a/packages/config/src/env.test.ts b/packages/config/src/env.test.ts index 160ade48..d04a637a 100644 --- a/packages/config/src/env.test.ts +++ b/packages/config/src/env.test.ts @@ -10,10 +10,6 @@ const CLOUD_ONLY_VARS = [ "GOOGLE_CLIENT_ID", "GOOGLE_CLIENT_SECRET", "RESEND_FROM_EMAIL", - "INFISICAL_CLIENT_ID", - "INFISICAL_CLIENT_SECRET", - "INFISICAL_PROJECT_ID", - "INFISICAL_ENVIRONMENT", ]; // Infra vars every validated mode needs — cloud now among them. const CLOUD_SHARED_VARS = ["DATABASE_URL", "BETTER_AUTH_SECRET", "SCRAPE_TARGETS", "DEPLOYMENT_MODE"]; @@ -56,10 +52,6 @@ describe("cloud env requirements", () => { GOOGLE_CLIENT_ID: "test-google-client-id", GOOGLE_CLIENT_SECRET: "test-google-client-secret", RESEND_FROM_EMAIL: "Elmo ", - INFISICAL_CLIENT_ID: "client-id", - INFISICAL_CLIENT_SECRET: "client-secret", - INFISICAL_PROJECT_ID: "project-id", - INFISICAL_ENVIRONMENT: "prod", }; const { missing } = validateEnvRequirements(cloudReqs, env); const missingIds = new Set(missing.map((entry) => entry.id)); diff --git a/packages/config/src/env.ts b/packages/config/src/env.ts index e390d716..cf024b85 100644 --- a/packages/config/src/env.ts +++ b/packages/config/src/env.ts @@ -85,8 +85,10 @@ export const ENV_REQUIREMENTS: Record = { local: [...buildStaticRequirements("local"), ...buildProviderKeyRequirements()], demo: [...buildStaticRequirements("demo"), ...buildProviderKeyRequirements()], whitelabel: [...buildStaticRequirements("whitelabel"), ...buildProviderKeyRequirements()], - // Cloud provider credentials are loaded from Infisical before providers are - // validated, so their legacy env vars are not startup requirements. + // Managed cloud provisions provider credentials out of band — Infisical's + // Vercel secret sync for the web app, the Infisical SDK for the worker — so a + // missing one is an operator alert, not a reason to replace the whole app with + // the setup screen the self-hosted modes rely on. cloud: buildStaticRequirements("cloud"), }; diff --git a/packages/deployment/src/credentials.ts b/packages/deployment/src/credentials.ts index 81440e88..d6cec24d 100644 --- a/packages/deployment/src/credentials.ts +++ b/packages/deployment/src/credentials.ts @@ -11,6 +11,9 @@ async function getCredentialSource(env: Record): Pro return createInfisicalCredentialLoader({ env }); } +/** Load provider credentials, then keep them fresh on an interval. Rejects if the + * first load fails, so a process that cannot run a job without credentials — + * the worker — dies at boot rather than picking up work it will fail. */ export async function startCredentialRefresh( env: Record = process.env, ): Promise { @@ -19,9 +22,25 @@ export async function startCredentialRefresh( await refresh(); const timer = setInterval(() => { refresh().catch((error) => { - console.warn("Provider credential refresh failed — keeping previous credentials:", error); + console.error("[secrets] credential refresh failed — serving the previous values:", error); }); }, CREDENTIAL_REFRESH_INTERVAL_MS); timer.unref(); return timer; } + +/** The web entry point. Never blocks a request and never rejects: the app has to + * serve sign-in and settings whether or not the credential store is reachable. + * + * Managed cloud is a no-op. That web app runs on Vercel serverless, where + * Infisical's Vercel secret sync writes provider credentials into the + * environment at deploy time — reading them back over the network would add an + * Infisical round trip to every cold start and put Infisical's availability in + * front of the whole site. The worker, which is long-lived and actually spends + * money against these credentials, keeps the live SDK loader. */ +export function startBackgroundCredentialRefresh(env: Record = process.env): void { + if (getDeploymentModeFromEnv(env) === "cloud") return; + void startCredentialRefresh(env).catch((error) => { + console.error("[secrets] initial credential load failed — falling back to environment credentials:", error); + }); +} diff --git a/packages/docs/content/docs/developer-guide/configuration.mdx b/packages/docs/content/docs/developer-guide/configuration.mdx index c1efb71f..f2007fc2 100644 --- a/packages/docs/content/docs/developer-guide/configuration.mdx +++ b/packages/docs/content/docs/developer-guide/configuration.mdx @@ -24,9 +24,18 @@ When you run `elmo init`, the CLI generates a `.env` file in your config directo | `DEPLOYMENT_MODE` | Deployment mode | `local` | | `DATABASE_URL` | PostgreSQL connection string | Generated by CLI | | `BETTER_AUTH_SECRET` | Secret key for session encryption | Auto-generated | +| `ELMO_ENCRYPTION_KEY` | Base64-encoded 32-byte key used to encrypt provider credentials stored in the database | Auto-generated | | `DISABLE_TELEMETRY` | Set to `1` to disable all telemetry. See [Telemetry](/docs/developer-guide/telemetry). | — | | `DEFAULT_DELAY_HOURS` | How often each enabled prompt is re-run against the AI models, in hours. | `24` | + + Back up `ELMO_ENCRYPTION_KEY` alongside your database. Credentials stored in the database are encrypted with it and nothing else — if you lose the key, or restore a database backup onto a deployment that generated a different one, those credentials cannot be recovered and have to be entered again. Elmo logs an error and falls back to the provider keys in your `.env` when a stored credential will not decrypt. + + + + The key protects credentials in the database, not on the host: it lives in the same `.env` as `DATABASE_URL`. That covers a leaked dump, a replica, or anyone with read access to Postgres. It does not defend against someone who can already read your config directory. + + `DEFAULT_DELAY_HOURS` only takes effect for prompts scheduled **after** you change it. Prompts that are already scheduled keep the cadence they were created with, even after a restart. To apply a new cadence to an existing prompt, disable it and then re-enable it under **Settings → Prompts** — re-enabling reschedules it at the current `DEFAULT_DELAY_HOURS`. diff --git a/packages/lib/package.json b/packages/lib/package.json index 7db49dd9..ebd249ca 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -19,7 +19,6 @@ "./constants": "./src/constants.ts", "./overdue": "./src/overdue.ts", "./text-extraction": "./src/text-extraction.ts", - "./dataforseo": "./src/dataforseo.ts", "./onboarding": "./src/onboarding/index.ts", "./tag-utils": "./src/tag-utils.ts", "./website-excerpt": "./src/website-excerpt.ts", diff --git a/packages/lib/src/dataforseo.ts b/packages/lib/src/dataforseo.ts deleted file mode 100644 index 66078c3d..00000000 --- a/packages/lib/src/dataforseo.ts +++ /dev/null @@ -1,24 +0,0 @@ -import * as client from "dataforseo-client"; - -function createAuthenticatedFetch(username: string, password: string) { - return (url: string | URL | Request, init?: RequestInit): Promise => { - const token = btoa(`${username}:${password}`); - const authHeader = { Authorization: `Basic ${token}` }; - - const newInit: RequestInit = { - ...init, - headers: { - ...init?.headers, - ...authHeader, - "Content-Type": "application/json", - }, - }; - - return fetch(url, newInit); - }; -} - -const authFetch = createAuthenticatedFetch(process.env.DATAFORSEO_LOGIN!, process.env.DATAFORSEO_PASSWORD!); -export const dfsLabsApi = new client.DataforseoLabsApi("https://api.dataforseo.com", { fetch: authFetch }); -export const dfsSerpApi = new client.SerpApi("https://api.dataforseo.com", { fetch: authFetch }); -export const dfsBacklinksApi = new client.BacklinksApi("https://api.dataforseo.com", { fetch: authFetch }); diff --git a/packages/lib/src/db/migrations/0011_provider_credentials.sql b/packages/lib/src/db/migrations/0011_provider_credentials.sql index 07733af9..650c5bf8 100644 --- a/packages/lib/src/db/migrations/0011_provider_credentials.sql +++ b/packages/lib/src/db/migrations/0011_provider_credentials.sql @@ -3,7 +3,6 @@ CREATE TABLE "provider_credentials" ( "organization_id" text, "provider" text NOT NULL, "encrypted_data" text NOT NULL, - "hint" text, "last_verified_at" timestamp with time zone, "last_verify_error" text, "created_at" timestamp with time zone DEFAULT now() NOT NULL, diff --git a/packages/lib/src/db/migrations/meta/0011_snapshot.json b/packages/lib/src/db/migrations/meta/0011_snapshot.json index 9df7051e..7756a07e 100644 --- a/packages/lib/src/db/migrations/meta/0011_snapshot.json +++ b/packages/lib/src/db/migrations/meta/0011_snapshot.json @@ -866,12 +866,6 @@ "primaryKey": false, "notNull": true }, - "hint": { - "name": "hint", - "type": "text", - "primaryKey": false, - "notNull": false - }, "last_verified_at": { "name": "last_verified_at", "type": "timestamp with time zone", diff --git a/packages/lib/src/db/schema.ts b/packages/lib/src/db/schema.ts index 5986bebf..96b1e724 100644 --- a/packages/lib/src/db/schema.ts +++ b/packages/lib/src/db/schema.ts @@ -247,7 +247,6 @@ export const providerCredentials = pgTable( organizationId: text("organization_id").references(() => organization.id, { onDelete: "cascade" }), provider: text("provider").notNull(), encryptedData: text("encrypted_data").notNull(), - hint: text("hint"), lastVerifiedAt: timestamp("last_verified_at", { withTimezone: true }), lastVerifyError: text("last_verify_error"), createdAt: timestamp("created_at", { withTimezone: true }).defaultNow().notNull(), diff --git a/packages/lib/src/secrets/crypto.test.ts b/packages/lib/src/secrets/crypto.test.ts index f5a11e82..9043e107 100644 --- a/packages/lib/src/secrets/crypto.test.ts +++ b/packages/lib/src/secrets/crypto.test.ts @@ -1,5 +1,5 @@ import { randomBytes } from "node:crypto"; -import { CompactEncrypt, decodeProtectedHeader } from "jose"; +import { decodeProtectedHeader } from "jose"; import { describe, expect, it } from "vitest"; import { decryptSecret, EncryptionKeyError, encryptSecret, getEncryptionKey, SecretDecryptError } from "./crypto"; @@ -16,13 +16,7 @@ describe("encryptSecret / decryptSecret", () => { it("uses the standard direct A256GCM JWE format with authenticated metadata", async () => { const payload = await encryptSecret("x", { key: KEY, aad: AAD }); expect(payload.split(".")).toHaveLength(5); - expect(decodeProtectedHeader(payload)).toEqual({ - alg: "dir", - enc: "A256GCM", - typ: "elmo-provider-credentials", - v: 1, - ctx: AAD, - }); + expect(decodeProtectedHeader(payload)).toEqual({ alg: "dir", enc: "A256GCM", ctx: AAD }); }); it("uses a fresh IV every call (no GCM nonce reuse)", async () => { @@ -70,17 +64,14 @@ describe("encryptSecret / decryptSecret", () => { await expect(decryptSecret(segments.join("."), { key: KEY, aad: AAD })).rejects.toThrow(SecretDecryptError); }); - it("unknown version", async () => { - const payload = await new CompactEncrypt(new TextEncoder().encode("top-secret-value")) - .setProtectedHeader({ - alg: "dir", - enc: "A256GCM", - typ: "elmo-provider-credentials", - v: 2, - ctx: AAD, - }) - .encrypt(KEY); - await expect(decryptSecret(payload, { key: KEY, aad: AAD })).rejects.toThrow(SecretDecryptError); + // Someone with DB write access moves this row to another provider and + // relabels the header to match. The header is the AEAD's additional data, + // so rewriting it breaks the tag rather than re-homing the credential. + it("ctx rewritten to the provider it was moved to", async () => { + const target = "provider-credentials:anthropic-api"; + const segments = (await fresh()).split("."); + segments[0] = Buffer.from(JSON.stringify({ alg: "dir", enc: "A256GCM", ctx: target })).toString("base64url"); + await expect(decryptSecret(segments.join("."), { key: KEY, aad: target })).rejects.toThrow(SecretDecryptError); }); it("malformed payloads", async () => { diff --git a/packages/lib/src/secrets/crypto.ts b/packages/lib/src/secrets/crypto.ts index 87cbdce0..d1b8e757 100644 --- a/packages/lib/src/secrets/crypto.ts +++ b/packages/lib/src/secrets/crypto.ts @@ -1,16 +1,14 @@ import { compactDecrypt, CompactEncrypt } from "jose"; const KEY_BYTES = 32; // 256-bit -const CURRENT_VERSION = 1; -const JWE_TYPE = "elmo-provider-credentials"; export const ENCRYPTION_KEY_ENV = "ELMO_ENCRYPTION_KEY"; /** Compact JWE using direct symmetric encryption (`dir`) and AES-256-GCM. */ export type EncryptedPayload = string; -/** Thrown for ANY decryption failure — bad tag, wrong AAD, wrong key, - * malformed payload, unknown version. Never leaks plaintext or key material. */ +/** Thrown for ANY decryption failure — bad tag, wrong context, wrong key, + * malformed payload. Never leaks plaintext or key material. */ export class SecretDecryptError extends Error { constructor(message: string, options?: { cause?: unknown }) { super(message, options); @@ -31,13 +29,7 @@ export async function encryptSecret( opts: { key: Uint8Array; aad: string }, ): Promise { return new CompactEncrypt(new TextEncoder().encode(plaintext)) - .setProtectedHeader({ - alg: "dir", - enc: "A256GCM", - typ: JWE_TYPE, - v: CURRENT_VERSION, - ctx: opts.aad, - }) + .setProtectedHeader({ alg: "dir", enc: "A256GCM", ctx: opts.aad }) .encrypt(opts.key); } @@ -48,10 +40,11 @@ export async function decryptSecret(payload: unknown, opts: { key: Uint8Array; a keyManagementAlgorithms: ["dir"], contentEncryptionAlgorithms: ["A256GCM"], }); - if (protectedHeader.typ !== JWE_TYPE) throw new Error("unexpected payload type"); - if (protectedHeader.v !== CURRENT_VERSION) { - throw new Error(`unsupported payload version: ${String(protectedHeader.v)}`); - } + // Both halves are needed. The protected header is the AEAD's additional + // data, so editing `ctx` breaks the tag — but jose authenticates the + // header the payload carries, not the one the caller expected. Comparing + // them is what stops a whole ciphertext being moved to another provider's + // row by someone with DB write access. if (protectedHeader.ctx !== opts.aad) throw new Error("payload context mismatch"); return new TextDecoder("utf-8", { fatal: true }).decode(plaintext); } catch (cause) { diff --git a/packages/lib/src/secrets/store.test.ts b/packages/lib/src/secrets/store.test.ts index e6a1deec..83d6e1fe 100644 --- a/packages/lib/src/secrets/store.test.ts +++ b/packages/lib/src/secrets/store.test.ts @@ -34,6 +34,7 @@ beforeEach(() => { dbState.rows = []; clearCredentialOverlay(); vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); }); afterEach(() => { @@ -209,16 +210,42 @@ describe("refreshCredentialOverlay", () => { ).rejects.toThrow("Infisical unavailable"); expect(getCredential("OPENAI_API_KEY")).toBe("current"); }); + + it("reports a managed credential that its source stopped providing", async () => { + await refreshCredentialOverlay(async () => new Map([["OPENAI_API_KEY", "managed"]])); + await refreshCredentialOverlay(async () => new Map()); + + expect(console.error).toHaveBeenCalledWith(expect.stringContaining("OPENAI_API_KEY is no longer provided")); + }); +}); + +// This PR only adds the ability to store credentials; nobody is moved off +// environment variables yet, so an install with no rows and no key has to behave +// exactly as it did before. +describe("deployments with nothing stored", () => { + it("is a no-op: env credentials win and nothing is logged", async () => { + vi.stubEnv("ELMO_ENCRYPTION_KEY", undefined); + vi.stubEnv("OPENAI_API_KEY", "env-value"); + vi.stubEnv("OXYLABS_USERNAME", "env-user"); + vi.stubEnv("OXYLABS_PASSWORD", "env-password"); + dbState.rows = []; + + await refreshCredentialOverlay(instanceCredentialSource); + + expect(getCredential("OPENAI_API_KEY")).toBe("env-value"); + expect(getCredential("OXYLABS_USERNAME")).toBe("env-user"); + expect(getCredential("OXYLABS_PASSWORD")).toBe("env-password"); + expect(console.error).not.toHaveBeenCalled(); + }); }); describe("encryptProviderCredentials", () => { - it("produces a payload that round-trips through the overlay, with a longest-value hint", async () => { + it("produces a payload that round-trips through the overlay", async () => { vi.stubEnv("ELMO_ENCRYPTION_KEY", KEY_B64); - const { encryptedData, hint } = await encryptProviderCredentials("oxylabs", { + const encryptedData = await encryptProviderCredentials("oxylabs", { OXYLABS_USERNAME: "user", OXYLABS_PASSWORD: "longer-secret", }); - expect(hint).toBe("cret"); // last 4 of the longest value "longer-secret" dbState.rows = [{ provider: "oxylabs", encryptedData }]; await refreshCredentialOverlay(instanceCredentialSource); diff --git a/packages/lib/src/secrets/store.ts b/packages/lib/src/secrets/store.ts index fd5ec365..3658bbaf 100644 --- a/packages/lib/src/secrets/store.ts +++ b/packages/lib/src/secrets/store.ts @@ -47,6 +47,7 @@ function aadForProvider(provider: string): string { * load (the source throws) keeps the last good values. */ export async function refreshCredentialOverlay(source: CredentialSource): Promise { const managed = await source(); + const previous = new Set(overlay.keys()); overlay.clear(); for (const [provider, keys] of PROVIDER_CREDENTIAL_KEYS) { const found = keys @@ -55,7 +56,17 @@ export async function refreshCredentialOverlay(source: CredentialSource): Promis if (found.length === keys.length) { for (const [name, value] of found) overlay.set(name, value); } else if (found.length > 0) { - console.warn(`[secrets] ignoring partial managed credential for "${provider}"`); + const missing = keys.filter((name) => !found.some(([found]) => found === name)); + console.error( + `[secrets] managed credential for "${provider}" is missing ${missing.join(", ")} — dropping the whole bundle so it cannot pair with an unrelated environment value`, + ); + } + } + // A credential that was managed a minute ago and is not any more means the + // source lost it, not that the operator meant to fall back to the environment. + for (const name of previous) { + if (!overlay.has(name)) { + console.error(`[secrets] managed credential ${name} is no longer provided by its source`); } } } @@ -72,7 +83,7 @@ export const instanceCredentialSource: CredentialSource = async () => { key = getEncryptionKey(); } catch (e) { if (!(e instanceof EncryptionKeyError)) throw e; - console.warn(`[secrets] ${e.message} — encrypted credentials skipped, env credentials unaffected`); + console.error(`[secrets] ${e.message} — every stored credential is unreadable until this is corrected`); } if (!key) return managed; @@ -88,7 +99,11 @@ export const instanceCredentialSource: CredentialSource = async () => { try { record = JSON.parse(await decryptSecret(row.encryptedData, { key, aad: aadForProvider(row.provider) })); } catch { - console.warn(`[secrets] ignoring undecryptable credential for "${row.provider}"`); + // There is no key id in the payload, so this cannot distinguish a + // rotated key from a corrupted row — the message has to cover both. + console.error( + `[secrets] stored credential for "${row.provider}" cannot be decrypted with the current ${ENCRYPTION_KEY_ENV} — re-enter it, or restore the key it was saved under`, + ); continue; } if (typeof record !== "object" || record === null) continue; @@ -101,13 +116,12 @@ export const instanceCredentialSource: CredentialSource = async () => { return managed; }; -/** Encrypt a provider's full credential set for the write path. `hint` is the - * last 4 chars of the longest value — enough to recognise which secret is stored - * without revealing it. Throws when no encryption key is set. */ +/** Encrypt a provider's full credential set for the write path. Throws when no + * encryption key is set. */ export async function encryptProviderCredentials( providerId: string, record: Record, -): Promise<{ encryptedData: EncryptedPayload; hint: string }> { +): Promise { const keys = getCredentialKeysForProvider(providerId); if (keys.length === 0) { throw new Error(`Provider "${providerId}" has no storable credentials`); @@ -123,14 +137,5 @@ export async function encryptProviderCredentials( if (!key) { throw new EncryptionKeyError(`${ENCRYPTION_KEY_ENV} is not set — cannot store encrypted credentials`); } - const encryptedData = await encryptSecret(JSON.stringify(record), { key, aad: aadForProvider(providerId) }); - return { encryptedData, hint: hintFor(record) }; -} - -function hintFor(record: Record): string { - let longest = ""; - for (const value of Object.values(record)) { - if (value.length > longest.length) longest = value; - } - return longest.slice(-4); + return encryptSecret(JSON.stringify(record), { key, aad: aadForProvider(providerId) }); } From d9ac9ec4076847eb11a5d2bf4a9d139ffd09f5fa Mon Sep 17 00:00:00 2001 From: jrhizor Date: Sat, 25 Jul 2026 13:30:24 -0700 Subject: [PATCH 3/6] Keep a failed credential load from stopping self-hosted boots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The worker loads credentials before pg-boss starts, so a source that throws took the whole worker down. On an upgrade that source is a fresh query against provider_credentials, which fails if the database is briefly unreachable or not yet migrated — and the compose file only guarantees migration ordering when Postgres runs in Docker. Schedule the retry interval before the first load and let self-hosted modes carry on with their .env credentials when that load fails. Managed cloud still rethrows, since it has no environment fallback and a worker that started anyway would only take jobs it cannot run. This also fixes the web path never retrying after a failed initial load. --- packages/deployment/src/credentials.ts | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/deployment/src/credentials.ts b/packages/deployment/src/credentials.ts index d6cec24d..4506d3c7 100644 --- a/packages/deployment/src/credentials.ts +++ b/packages/deployment/src/credentials.ts @@ -11,21 +11,35 @@ async function getCredentialSource(env: Record): Pro return createInfisicalCredentialLoader({ env }); } -/** Load provider credentials, then keep them fresh on an interval. Rejects if the - * first load fails, so a process that cannot run a job without credentials — - * the worker — dies at boot rather than picking up work it will fail. */ +/** Load provider credentials, then keep them fresh on an interval. The interval + * is scheduled either way, so a source that is briefly unreachable recovers on + * its own rather than leaving the process permanently stale. + * + * Managed cloud rethrows a failed first load: it has no environment fallback, so + * a worker that started anyway would only pick up jobs it cannot run. Every + * other mode logs and continues — self-hosted deployments still have their + * `.env` credentials, and on an upgrade the database may not be migrated yet. */ export async function startCredentialRefresh( env: Record = process.env, ): Promise { const source = await getCredentialSource(env); const refresh = () => refreshCredentialOverlay(source); - await refresh(); const timer = setInterval(() => { refresh().catch((error) => { console.error("[secrets] credential refresh failed — serving the previous values:", error); }); }, CREDENTIAL_REFRESH_INTERVAL_MS); timer.unref(); + + try { + await refresh(); + } catch (error) { + if (getDeploymentModeFromEnv(env) === "cloud") { + clearInterval(timer); + throw error; + } + console.error("[secrets] could not load stored credentials — using environment credentials for now:", error); + } return timer; } From 981aac91a5c4c750632bb97a6f2b6adbceaca468 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Sat, 25 Jul 2026 13:47:18 -0700 Subject: [PATCH 4/6] Require the Infisical credentials across cloud MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only the worker reads them at runtime, but treating them as worker-only let a cloud deployment go out with a worker that had no way to reach Infisical. Make them cloud requirements again so that misconfiguration surfaces at startup. INFISICAL_SECRET_PATH and INFISICAL_SITE_URL stay optional — both have working defaults, and requiring the site URL would break any deployment relying on the US cloud default. --- apps/web/scripts/smoke-deployment-mode.ts | 4 ++++ packages/cloud/README.md | 16 +++++++++------- packages/config/src/env-registry.ts | 16 ++++++++-------- packages/config/src/env.test.ts | 8 ++++++++ 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/apps/web/scripts/smoke-deployment-mode.ts b/apps/web/scripts/smoke-deployment-mode.ts index 14cb36d7..1bbdac23 100644 --- a/apps/web/scripts/smoke-deployment-mode.ts +++ b/apps/web/scripts/smoke-deployment-mode.ts @@ -83,6 +83,10 @@ const MINIMAL_ENV: Record> = { RESEND_FROM_EMAIL: "Smoke ", GOOGLE_CLIENT_ID: "smoke-google-client-id", GOOGLE_CLIENT_SECRET: "smoke-google-client-secret", + INFISICAL_CLIENT_ID: "smoke-client-id", + INFISICAL_CLIENT_SECRET: "smoke-client-secret", + INFISICAL_PROJECT_ID: "smoke-project-id", + INFISICAL_ENVIRONMENT: "prod", }, }; diff --git a/packages/cloud/README.md b/packages/cloud/README.md index a6795eb3..aedfdf69 100644 --- a/packages/cloud/README.md +++ b/packages/cloud/README.md @@ -12,17 +12,19 @@ The auth and email features in this package need: | `RESEND_FROM_EMAIL` | Sender address, e.g. `Elmo ` | | `GOOGLE_CLIENT_ID` | Google OAuth client ID for social sign-in | | `GOOGLE_CLIENT_SECRET` | Google OAuth client secret | - -The worker additionally needs the Infisical variables below. The web app does not — see the Infisical section for why. - -| Variable | Purpose | -| --- | --- | | `INFISICAL_CLIENT_ID` | Infisical machine identity client ID | | `INFISICAL_CLIENT_SECRET` | Infisical machine identity client secret | | `INFISICAL_PROJECT_ID` | Project containing provider credentials | | `INFISICAL_ENVIRONMENT` | Environment slug containing provider credentials | -| `INFISICAL_SECRET_PATH` | Optional credentials path; defaults to `/` | -| `INFISICAL_SITE_URL` | Optional Infisical site URL; defaults to the US cloud | + +Only the worker reads the Infisical variables at runtime — see the Infisical section — but they are required across cloud so a deployment cannot go out with a worker that has no way to reach Infisical. + +These two are optional everywhere: + +| Variable | Purpose | +| --- | --- | +| `INFISICAL_SECRET_PATH` | Credentials path; defaults to `/` | +| `INFISICAL_SITE_URL` | Infisical site URL; defaults to the US cloud | The canonical list of every cloud-required variable (Stripe, database, etc.) lives in `packages/config/src/env-registry.ts`; env validation fails cloud startup when any of them is missing. diff --git a/packages/config/src/env-registry.ts b/packages/config/src/env-registry.ts index 4522399f..c925f62b 100644 --- a/packages/config/src/env-registry.ts +++ b/packages/config/src/env-registry.ts @@ -208,32 +208,32 @@ export const ENV_REGISTRY: EnvVarSpec[] = [ requiredBy: "optional", description: "Base64-encoded 32-byte key enabling encrypted provider credentials in self-hosted deployments.", }, - // Only the managed-cloud worker talks to Infisical. The cloud web app reads - // provider credentials from its environment, which Infisical's Vercel secret - // sync populates at deploy time — so these are not web startup requirements, - // and createInfisicalCredentialLoader fails the worker fast if any is absent. + // Only the worker reads these at runtime — the cloud web app gets provider + // credentials from the environment Infisical's Vercel sync populates. They are + // still required across cloud so a deployment cannot be provisioned with a + // worker that has no way to reach Infisical. { name: "INFISICAL_CLIENT_ID", scope: "server", - requiredBy: "optional", + requiredBy: ["cloud"], description: "Infisical machine identity client ID used by the managed cloud worker.", }, { name: "INFISICAL_CLIENT_SECRET", scope: "server", - requiredBy: "optional", + requiredBy: ["cloud"], description: "Infisical machine identity client secret used by the managed cloud worker.", }, { name: "INFISICAL_PROJECT_ID", scope: "server", - requiredBy: "optional", + requiredBy: ["cloud"], description: "Infisical project containing managed cloud provider credentials.", }, { name: "INFISICAL_ENVIRONMENT", scope: "server", - requiredBy: "optional", + requiredBy: ["cloud"], description: "Infisical environment slug containing managed cloud provider credentials.", }, { diff --git a/packages/config/src/env.test.ts b/packages/config/src/env.test.ts index d04a637a..160ade48 100644 --- a/packages/config/src/env.test.ts +++ b/packages/config/src/env.test.ts @@ -10,6 +10,10 @@ const CLOUD_ONLY_VARS = [ "GOOGLE_CLIENT_ID", "GOOGLE_CLIENT_SECRET", "RESEND_FROM_EMAIL", + "INFISICAL_CLIENT_ID", + "INFISICAL_CLIENT_SECRET", + "INFISICAL_PROJECT_ID", + "INFISICAL_ENVIRONMENT", ]; // Infra vars every validated mode needs — cloud now among them. const CLOUD_SHARED_VARS = ["DATABASE_URL", "BETTER_AUTH_SECRET", "SCRAPE_TARGETS", "DEPLOYMENT_MODE"]; @@ -52,6 +56,10 @@ describe("cloud env requirements", () => { GOOGLE_CLIENT_ID: "test-google-client-id", GOOGLE_CLIENT_SECRET: "test-google-client-secret", RESEND_FROM_EMAIL: "Elmo ", + INFISICAL_CLIENT_ID: "client-id", + INFISICAL_CLIENT_SECRET: "client-secret", + INFISICAL_PROJECT_ID: "project-id", + INFISICAL_ENVIRONMENT: "prod", }; const { missing } = validateEnvRequirements(cloudReqs, env); const missingIds = new Set(missing.map((entry) => entry.id)); From fb528180ee21e2e7727cfa59de9b3bf286d79b45 Mon Sep 17 00:00:00 2001 From: jrhizor Date: Sat, 25 Jul 2026 13:52:21 -0700 Subject: [PATCH 5/6] Keep the Infisical SDK out of the web bundle @infisical/sdk depends on @aws-sdk/credential-providers and @smithy/*, and Nitro traces a dynamic import into the output whether or not the branch can run. The cloud web app never loads credentials over the network, so that was megabytes of AWS SDK shipped into a Vercel serverless function that would never call it. Move the worker's source selection into its own entry point. startCredentialRefresh now takes a source and whether it is required, so the module the web app imports only ever references instanceCredentialSource. Tracing the import graph from apps/web/src/server.ts no longer reaches @infisical/sdk; from apps/worker/src/index.ts it still does. --- apps/worker/src/index.ts | 4 +-- packages/deployment/package.json | 3 +- packages/deployment/src/credentials-worker.ts | 21 +++++++++++++ packages/deployment/src/credentials.ts | 31 +++++++------------ 4 files changed, 37 insertions(+), 22 deletions(-) create mode 100644 packages/deployment/src/credentials-worker.ts diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index c91d87b3..c386a00f 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -1,6 +1,6 @@ import * as Sentry from "@sentry/node"; import { getDeployment } from "@workspace/deployment"; -import { startCredentialRefresh } from "@workspace/deployment/credentials"; +import { startWorkerCredentialRefresh } from "@workspace/deployment/credentials-worker"; import { getProvider, parseScrapeTargets, validateScrapeTargets } from "@workspace/lib/providers"; import boss from "./boss"; import { registerHandlers } from "./handlers"; @@ -17,7 +17,7 @@ if (process.env.SENTRY_DSN) { async function main() { console.log("Starting pg-boss worker..."); - await startCredentialRefresh(); + await startWorkerCredentialRefresh(); // Fail fast on misconfigured SCRAPE_TARGETS — surfaces unknown providers, // missing API keys, and per-provider target errors before any job runs. diff --git a/packages/deployment/package.json b/packages/deployment/package.json index 74ec7e10..423d6636 100644 --- a/packages/deployment/package.json +++ b/packages/deployment/package.json @@ -7,7 +7,8 @@ "exports": { ".": "./src/index.ts", "./client": "./src/client.ts", - "./credentials": "./src/credentials.ts" + "./credentials": "./src/credentials.ts", + "./credentials-worker": "./src/credentials-worker.ts" }, "dependencies": { "@workspace/cloud": "workspace:*", diff --git a/packages/deployment/src/credentials-worker.ts b/packages/deployment/src/credentials-worker.ts new file mode 100644 index 00000000..30917e66 --- /dev/null +++ b/packages/deployment/src/credentials-worker.ts @@ -0,0 +1,21 @@ +import { getDeploymentModeFromEnv } from "@workspace/config/env"; +import { instanceCredentialSource } from "@workspace/lib/secrets"; +import { startCredentialRefresh } from "./credentials"; + +/** The worker entry point: managed cloud reads Infisical, every other mode reads + * the encrypted provider_credentials table (with environment fallback). + * + * This lives apart from `./credentials` because the web app imports that module. + * `@infisical/sdk` pulls in `@aws-sdk/credential-providers` and `@smithy/*`, and + * Nitro traces a dynamic import into the bundle whether or not it can run — so + * keeping the two together would ship megabytes of AWS SDK into a Vercel + * serverless function that never calls it. */ +export async function startWorkerCredentialRefresh( + env: Record = process.env, +): Promise { + if (getDeploymentModeFromEnv(env) !== "cloud") { + return startCredentialRefresh(instanceCredentialSource, { required: false }); + } + const { createInfisicalCredentialLoader } = await import("@workspace/cloud/infisical-credentials"); + return startCredentialRefresh(createInfisicalCredentialLoader({ env }), { required: true }); +} diff --git a/packages/deployment/src/credentials.ts b/packages/deployment/src/credentials.ts index 4506d3c7..3dfcf586 100644 --- a/packages/deployment/src/credentials.ts +++ b/packages/deployment/src/credentials.ts @@ -3,26 +3,19 @@ import { type CredentialSource, instanceCredentialSource, refreshCredentialOverl const CREDENTIAL_REFRESH_INTERVAL_MS = 60_000; -/** Managed cloud loads provider credentials from Infisical; every other mode - * reads the encrypted provider_credentials table (with env fallback). */ -async function getCredentialSource(env: Record): Promise { - if (getDeploymentModeFromEnv(env) !== "cloud") return instanceCredentialSource; - const { createInfisicalCredentialLoader } = await import("@workspace/cloud/infisical-credentials"); - return createInfisicalCredentialLoader({ env }); -} - -/** Load provider credentials, then keep them fresh on an interval. The interval - * is scheduled either way, so a source that is briefly unreachable recovers on - * its own rather than leaving the process permanently stale. +/** Keep the credential overlay fresh from `source`. The interval is scheduled + * before the first load, so a source that is briefly unreachable recovers on its + * own instead of leaving the process permanently stale. * - * Managed cloud rethrows a failed first load: it has no environment fallback, so - * a worker that started anyway would only pick up jobs it cannot run. Every - * other mode logs and continues — self-hosted deployments still have their - * `.env` credentials, and on an upgrade the database may not be migrated yet. */ + * `required` says whether the caller can run without credentials. Only managed + * cloud sets it: there is no environment fallback there, so a worker that + * started anyway would take jobs it cannot run. Self-hosted deployments always + * have their `.env` credentials, and on an upgrade the database may not even be + * migrated yet, so they log and carry on. */ export async function startCredentialRefresh( - env: Record = process.env, + source: CredentialSource, + { required }: { required: boolean }, ): Promise { - const source = await getCredentialSource(env); const refresh = () => refreshCredentialOverlay(source); const timer = setInterval(() => { refresh().catch((error) => { @@ -34,7 +27,7 @@ export async function startCredentialRefresh( try { await refresh(); } catch (error) { - if (getDeploymentModeFromEnv(env) === "cloud") { + if (required) { clearInterval(timer); throw error; } @@ -54,7 +47,7 @@ export async function startCredentialRefresh( * money against these credentials, keeps the live SDK loader. */ export function startBackgroundCredentialRefresh(env: Record = process.env): void { if (getDeploymentModeFromEnv(env) === "cloud") return; - void startCredentialRefresh(env).catch((error) => { + void startCredentialRefresh(instanceCredentialSource, { required: false }).catch((error) => { console.error("[secrets] initial credential load failed — falling back to environment credentials:", error); }); } From a2c3c11f789d9d0f566ca686c8f242f83820e79e Mon Sep 17 00:00:00 2001 From: jrhizor Date: Sat, 25 Jul 2026 13:54:57 -0700 Subject: [PATCH 6/6] Drop the configurable Infisical path and site URL Both only ever held their defaults, and each was another way for a cloud deployment to point the loader somewhere the credentials are not. Read the root of INFISICAL_ENVIRONMENT on Infisical's US cloud, with no way to override either. With the read already non-recursive, that fixes the folder contract in one place: one flat set of canonically named secrets at the environment root, and nothing else in the project is fetched. --- apps/web/src/env.d.ts | 2 -- packages/cloud/README.md | 13 +++------- .../cloud/src/infisical-credentials.test.ts | 6 ++--- packages/cloud/src/infisical-credentials.ts | 25 ++++++++++--------- packages/config/src/env-registry.ts | 12 --------- turbo.json | 2 -- 6 files changed, 19 insertions(+), 41 deletions(-) diff --git a/apps/web/src/env.d.ts b/apps/web/src/env.d.ts index 0e79b354..a2693538 100644 --- a/apps/web/src/env.d.ts +++ b/apps/web/src/env.d.ts @@ -59,8 +59,6 @@ declare global { readonly INFISICAL_CLIENT_SECRET?: string; readonly INFISICAL_PROJECT_ID?: string; readonly INFISICAL_ENVIRONMENT?: string; - readonly INFISICAL_SECRET_PATH?: string; - readonly INFISICAL_SITE_URL?: string; readonly DATAFORSEO_LOGIN: string; readonly DATAFORSEO_PASSWORD: string; readonly BETTER_AUTH_SECRET?: string; diff --git a/packages/cloud/README.md b/packages/cloud/README.md index aedfdf69..d2e481ad 100644 --- a/packages/cloud/README.md +++ b/packages/cloud/README.md @@ -19,12 +19,7 @@ The auth and email features in this package need: Only the worker reads the Infisical variables at runtime — see the Infisical section — but they are required across cloud so a deployment cannot go out with a worker that has no way to reach Infisical. -These two are optional everywhere: - -| Variable | Purpose | -| --- | --- | -| `INFISICAL_SECRET_PATH` | Credentials path; defaults to `/` | -| `INFISICAL_SITE_URL` | Infisical site URL; defaults to the US cloud | +There is nothing else to configure: credentials are read from the root of the given environment, on Infisical's US cloud. The canonical list of every cloud-required variable (Stripe, database, etc.) lives in `packages/config/src/env-registry.ts`; env validation fails cloud startup when any of them is missing. @@ -38,18 +33,18 @@ Email templates are code — `packages/cloud/src/email-templates.ts` — not Res ## Infisical setup -Provider credentials live in one Infisical folder, named exactly as the canonical environment variables (`OPENAI_API_KEY`, `OXYLABS_USERNAME`, `OXYLABS_PASSWORD`, …). The two runtimes read that folder differently. +Provider credentials live at the **root of the `INFISICAL_ENVIRONMENT` environment**, named exactly as the canonical environment variables (`OPENAI_API_KEY`, `OXYLABS_USERNAME`, `OXYLABS_PASSWORD`, …). The path is not configurable, and the read is not recursive, so secrets in subfolders are neither fetched nor used. The two runtimes read that folder differently. **Worker** — a long-lived VM, so it authenticates once with the SDK and refreshes every minute. Rotating a credential in Infisical reaches it without a deploy. -1. Create a machine identity with Universal Auth and read access to the provider-credential path — that path only, since the loader is not recursive. +1. Create a machine identity with Universal Auth and read access to that environment's root. 2. Set the Infisical variables above on the worker. A refresh that fails, or that returns no provider credentials at all, keeps the last known-good values and logs an error; the worker refuses to start if the very first load fails. **Web app** — runs on Vercel serverless. Use Infisical's [Vercel secret sync](https://infisical.com/docs/integrations/secret-syncs/vercel) to push the same folder into the project's environment variables, and the app reads them from `process.env`. -1. Add a Vercel app connection in Infisical, then a secret sync from the provider-credential folder to the Vercel project. +1. Add a Vercel app connection in Infisical, then a secret sync from that same environment root to the Vercel project. 2. Enable auto-sync, and redeploy after a rotation — Vercel bakes environment variables in at deploy time. Reading Infisical at runtime instead would put an authentication round trip on every cold start and Infisical's availability in front of the whole site, to serve credentials the request path barely uses. diff --git a/packages/cloud/src/infisical-credentials.test.ts b/packages/cloud/src/infisical-credentials.test.ts index a5e7929f..4773fbba 100644 --- a/packages/cloud/src/infisical-credentials.test.ts +++ b/packages/cloud/src/infisical-credentials.test.ts @@ -6,8 +6,6 @@ const ENV = { INFISICAL_CLIENT_SECRET: "client-secret", INFISICAL_PROJECT_ID: "project-id", INFISICAL_ENVIRONMENT: "prod", - INFISICAL_SECRET_PATH: "/elmo/providers", - INFISICAL_SITE_URL: "https://eu.infisical.com", }; function fakeClient(secrets: Array<{ secretKey: string; secretValue: string }>) { @@ -35,17 +33,17 @@ describe("createInfisicalCredentialLoader", () => { const credentials = await load(); - expect(clientFactory).toHaveBeenCalledWith("https://eu.infisical.com"); expect(credentials).toEqual( new Map([ ["OPENAI_API_KEY", "sk-cloud"], ["OXYLABS_USERNAME", "user"], ]), ); + // Root of the environment, and not recursive: the whole folder contract. expect(listSecretsWithImports).toHaveBeenCalledWith({ environment: "prod", projectId: "project-id", - secretPath: "/elmo/providers", + secretPath: "/", expandSecretReferences: true, viewSecretValue: true, }); diff --git a/packages/cloud/src/infisical-credentials.ts b/packages/cloud/src/infisical-credentials.ts index fe4e9d89..c58b3e9f 100644 --- a/packages/cloud/src/infisical-credentials.ts +++ b/packages/cloud/src/infisical-credentials.ts @@ -6,8 +6,11 @@ const CLIENT_ID_ENV = "INFISICAL_CLIENT_ID"; const CLIENT_SECRET_ENV = "INFISICAL_CLIENT_SECRET"; const PROJECT_ID_ENV = "INFISICAL_PROJECT_ID"; const ENVIRONMENT_ENV = "INFISICAL_ENVIRONMENT"; -const SECRET_PATH_ENV = "INFISICAL_SECRET_PATH"; -const SITE_URL_ENV = "INFISICAL_SITE_URL"; + +/** Provider credentials live at the root of the configured environment. Combined + * with a non-recursive read, that is the entire folder contract: one flat set of + * canonically named secrets, and nothing else in the project is even fetched. */ +const SECRET_PATH = "/"; interface InfisicalClient { auth(): { @@ -28,7 +31,7 @@ interface InfisicalClient { export interface InfisicalCredentialLoaderOptions { env?: Record; - clientFactory?: (siteUrl?: string) => InfisicalClient; + clientFactory?: () => InfisicalClient; } function required(env: Record, name: string): string { @@ -45,13 +48,11 @@ export function createInfisicalCredentialLoader(options: InfisicalCredentialLoad const clientSecret = required(env, CLIENT_SECRET_ENV); const projectId = required(env, PROJECT_ID_ENV); const environment = required(env, ENVIRONMENT_ENV); - const secretPath = env[SECRET_PATH_ENV]?.trim() || "/"; - const siteUrl = env[SITE_URL_ENV]?.trim() || undefined; - const clientFactory = options.clientFactory ?? ((url) => new InfisicalSDK(url ? { siteUrl: url } : undefined)); + const clientFactory = options.clientFactory ?? (() => new InfisicalSDK()); let clientPromise: Promise | null = null; const authenticate = () => { - clientPromise ??= clientFactory(siteUrl).auth().universalAuth.login({ clientId, clientSecret }); + clientPromise ??= clientFactory().auth().universalAuth.login({ clientId, clientSecret }); return clientPromise; }; const list = async () => { @@ -62,7 +63,7 @@ export function createInfisicalCredentialLoader(options: InfisicalCredentialLoad return client.secrets().listSecretsWithImports({ environment, projectId, - secretPath, + secretPath: SECRET_PATH, expandSecretReferences: true, viewSecretValue: true, }); @@ -86,12 +87,12 @@ export function createInfisicalCredentialLoader(options: InfisicalCredentialLoad } } // Cloud has no environment fallback, so "the call succeeded and returned - // nothing" is an outage — a revoked read grant, or a path/environment slug - // that no longer resolves. Throwing keeps the last good overlay in place - // instead of quietly unconfiguring every provider. + // nothing" is an outage — a revoked read grant, or an environment slug that + // no longer resolves. Throwing keeps the last good overlay in place instead + // of quietly unconfiguring every provider. if (credentials.size === 0) { throw new Error( - `Infisical returned no provider credentials from ${secretPath} in "${environment}" — check the machine identity's read access and ${SECRET_PATH_ENV}`, + `Infisical returned no provider credentials from ${SECRET_PATH} in "${environment}" — check the machine identity's read access and that ${ENVIRONMENT_ENV} is correct`, ); } return credentials; diff --git a/packages/config/src/env-registry.ts b/packages/config/src/env-registry.ts index c925f62b..dcc11f56 100644 --- a/packages/config/src/env-registry.ts +++ b/packages/config/src/env-registry.ts @@ -236,18 +236,6 @@ export const ENV_REGISTRY: EnvVarSpec[] = [ requiredBy: ["cloud"], description: "Infisical environment slug containing managed cloud provider credentials.", }, - { - name: "INFISICAL_SECRET_PATH", - scope: "server", - requiredBy: "optional", - description: "Infisical path containing provider credentials. Defaults to the project root.", - }, - { - name: "INFISICAL_SITE_URL", - scope: "server", - requiredBy: "optional", - description: "Infisical API site URL. Defaults to https://app.infisical.com.", - }, { name: "DEPLOYMENT_MODE", scope: "server", diff --git a/turbo.json b/turbo.json index f7f1195d..2034b33a 100644 --- a/turbo.json +++ b/turbo.json @@ -30,8 +30,6 @@ "INFISICAL_CLIENT_SECRET", "INFISICAL_PROJECT_ID", "INFISICAL_ENVIRONMENT", - "INFISICAL_SECRET_PATH", - "INFISICAL_SITE_URL", "DEPLOYMENT_MODE", "ADMIN_AUTH0_SUB", "ADMIN_API_KEYS",