diff --git a/index.ts b/index.ts index 52f1962..fc54cd1 100644 --- a/index.ts +++ b/index.ts @@ -79,6 +79,11 @@ import { type AdmissionRejectionAuditEntry, } from "./src/admission-control.js"; import { analyzeIntent, applyCategoryBoost } from "./src/intent-analyzer.js"; +import { + resolveEnvVarsSync, + resolveSecretValue, + resolveSecretValues, +} from "./src/secret-resolver.js"; // ============================================================================ // Configuration & Types @@ -245,21 +250,15 @@ function resolveWorkspaceDirFromContext(context: Record | undef } function resolveEnvVars(value: string): string { - return value.replace(/\$\{([^}]+)\}/g, (_, envVar) => { - const envValue = process.env[envVar]; - if (!envValue) { - throw new Error(`Environment variable ${envVar} is not set`); - } - return envValue; - }); + return resolveEnvVarsSync(value); } -function resolveFirstApiKey(apiKey: string | string[]): string { +async function resolveFirstApiKey(apiKey: string | string[]): Promise { const key = Array.isArray(apiKey) ? apiKey[0] : apiKey; if (!key) { throw new Error("embedding.apiKey is empty"); } - return resolveEnvVars(key); + return resolveSecretValue(key); } function resolveOptionalPathWithEnv( @@ -1616,7 +1615,7 @@ const memoryLanceDBProPlugin = { "Enhanced LanceDB-backed long-term memory with hybrid retrieval, multi-scope isolation, and management CLI", kind: "memory" as const, - register(api: OpenClawPluginApi) { + register(api: OpenClawPluginApi): Promise { // Parse and validate configuration const config = parsePluginConfig(api.pluginConfig); @@ -1638,21 +1637,9 @@ const memoryLanceDBProPlugin = { config.embedding.dimensions, ); - // Initialize core components + // store, scopeManager, decayEngine, tierManager, migrator are all sync — + // they do not need resolved secrets and are available immediately. const store = new MemoryStore({ dbPath: resolvedDbPath, vectorDim }); - const embedder = createEmbedder({ - provider: "openai-compatible", - apiKey: config.embedding.apiKey, - model: config.embedding.model || "text-embedding-3-small", - baseURL: config.embedding.baseURL, - dimensions: config.embedding.dimensions, - omitDimensions: config.embedding.omitDimensions, - taskQuery: config.embedding.taskQuery, - taskPassage: config.embedding.taskPassage, - normalized: config.embedding.normalized, - chunking: config.embedding.chunking, - }); - // Initialize decay engine const decayEngine = createDecayEngine({ ...DEFAULT_DECAY_CONFIG, ...(config.decay || {}), @@ -1661,100 +1648,168 @@ const memoryLanceDBProPlugin = { ...DEFAULT_TIER_CONFIG, ...(config.tier || {}), }); - const retriever = createRetriever( - store, - embedder, - { - ...DEFAULT_RETRIEVAL_CONFIG, - ...config.retrieval, - }, - { decayEngine }, - ); const scopeManager = createScopeManager(config.scopes); - // ClawTeam integration: extend accessible scopes via env var - const clawteamScopes = parseClawteamScopes(process.env.CLAWTEAM_MEMORY_SCOPE); - if (clawteamScopes.length > 0) { - applyClawteamScopes(scopeManager, clawteamScopes); - api.logger.info(`memory-lancedb-pro: CLAWTEAM_MEMORY_SCOPE added scopes: ${clawteamScopes.join(", ")}`); - } - - const migrator = createMigrator(store); - - // Initialize smart extraction + // embedder, retriever, and smartExtractor require async secret resolution. + // They are initialized in initPromise and must not be used before it resolves. + // Every hook that uses them awaits initPromise at its top. + // If initPromise rejects, initFailed is set; service.start() logs a warn so + // the user knows why memory features are unavailable. + let embedder!: ReturnType; + let retriever!: ReturnType; let smartExtractor: SmartExtractor | null = null; - if (config.smartExtraction !== false) { - try { - const llmAuth = config.llm?.auth || "api-key"; - const llmApiKey = llmAuth === "oauth" - ? undefined - : config.llm?.apiKey - ? resolveEnvVars(config.llm.apiKey) - : resolveFirstApiKey(config.embedding.apiKey); - const llmBaseURL = llmAuth === "oauth" - ? (config.llm?.baseURL ? resolveEnvVars(config.llm.baseURL) : undefined) - : config.llm?.baseURL - ? resolveEnvVars(config.llm.baseURL) - : config.embedding.baseURL; - const llmModel = config.llm?.model || "openai/gpt-oss-120b"; - const llmOauthPath = llmAuth === "oauth" - ? resolveOptionalPathWithEnv(api, config.llm?.oauthPath, ".memory-lancedb-pro/oauth.json") - : undefined; - const llmOauthProvider = llmAuth === "oauth" - ? config.llm?.oauthProvider - : undefined; - const llmTimeoutMs = resolveLlmTimeoutMs(config); - - const llmClient = createLlmClient({ - auth: llmAuth, - apiKey: llmApiKey, - model: llmModel, - baseURL: llmBaseURL, - oauthProvider: llmOauthProvider, - oauthPath: llmOauthPath, - timeoutMs: llmTimeoutMs, - log: (msg: string) => api.logger.debug(msg), - }); + let llmClientForCli: import("./src/llm-client.js").LlmClient | undefined; + let initFailed = false; + + const initPromise: Promise = (async () => { + const resolvedEmbeddingApiKeys = await resolveSecretValues(config.embedding.apiKey); + const resolvedRerankApiKey = typeof config.retrieval?.rerankApiKey === "string" && config.retrieval.rerankApiKey.trim() + ? await resolveSecretValue(config.retrieval.rerankApiKey) + : undefined; + const resolvedRetrievalConfig = config.retrieval + ? { + ...config.retrieval, + ...(resolvedRerankApiKey ? { rerankApiKey: resolvedRerankApiKey } : {}), + } + : undefined; + embedder = createEmbedder({ + provider: "openai-compatible", + apiKey: resolvedEmbeddingApiKeys.length === 1 ? resolvedEmbeddingApiKeys[0] : resolvedEmbeddingApiKeys, + model: config.embedding.model || "text-embedding-3-small", + baseURL: config.embedding.baseURL, + dimensions: config.embedding.dimensions, + omitDimensions: config.embedding.omitDimensions, + taskQuery: config.embedding.taskQuery, + taskPassage: config.embedding.taskPassage, + normalized: config.embedding.normalized, + chunking: config.embedding.chunking, + }); + retriever = createRetriever( + store, + embedder, + { + ...DEFAULT_RETRIEVAL_CONFIG, + ...resolvedRetrievalConfig, + }, + { decayEngine }, + ); + // Initialize smart extraction inside initPromise + if (config.smartExtraction !== false) { + try { + const llmAuth = config.llm?.auth || "api-key"; + const llmApiKey = llmAuth === "oauth" + ? undefined + : config.llm?.apiKey + ? await resolveSecretValue(config.llm.apiKey) + : await resolveFirstApiKey(config.embedding.apiKey); + const llmBaseURL = llmAuth === "oauth" + ? (config.llm?.baseURL ? resolveEnvVars(config.llm.baseURL) : undefined) + : config.llm?.baseURL + ? resolveEnvVars(config.llm.baseURL) + : config.embedding.baseURL; + const llmModel = config.llm?.model || "openai/gpt-oss-120b"; + const llmOauthPath = llmAuth === "oauth" + ? resolveOptionalPathWithEnv(api, config.llm?.oauthPath, ".memory-lancedb-pro/oauth.json") + : undefined; + const llmOauthProvider = llmAuth === "oauth" + ? config.llm?.oauthProvider + : undefined; + const llmTimeoutMs = resolveLlmTimeoutMs(config); + + const llmClient = createLlmClient({ + auth: llmAuth, + apiKey: llmApiKey, + model: llmModel, + baseURL: llmBaseURL, + oauthProvider: llmOauthProvider, + oauthPath: llmOauthPath, + timeoutMs: llmTimeoutMs, + log: (msg: string) => api.logger.debug(msg), + }); - // Initialize embedding-based noise prototype bank (async, non-blocking) - const noiseBank = new NoisePrototypeBank( - (msg: string) => api.logger.debug(msg), - ); - noiseBank.init(embedder).catch((err) => - api.logger.debug(`memory-lancedb-pro: noise bank init: ${String(err)}`), - ); + // Initialize embedding-based noise prototype bank (async, non-blocking) + const noiseBank = new NoisePrototypeBank( + (msg: string) => api.logger.debug(msg), + ); + noiseBank.init(embedder).catch((err) => + api.logger.debug(`memory-lancedb-pro: noise bank init: ${String(err)}`), + ); - const admissionRejectionAuditWriter = createAdmissionRejectionAuditWriter( - config, - resolvedDbPath, - api, - ); + const admissionRejectionAuditWriter = createAdmissionRejectionAuditWriter( + config, + resolvedDbPath, + api, + ); - smartExtractor = new SmartExtractor(store, embedder, llmClient, { - user: "User", - extractMinMessages: config.extractMinMessages ?? 4, - extractMaxChars: config.extractMaxChars ?? 8000, - defaultScope: config.scopes?.default ?? "global", - workspaceBoundary: config.workspaceBoundary, - admissionControl: config.admissionControl, - onAdmissionRejected: admissionRejectionAuditWriter ?? undefined, - log: (msg: string) => api.logger.info(msg), - debugLog: (msg: string) => api.logger.debug(msg), - noiseBank, - }); + smartExtractor = new SmartExtractor(store, embedder, llmClient, { + user: "User", + extractMinMessages: config.extractMinMessages ?? 4, + extractMaxChars: config.extractMaxChars ?? 8000, + defaultScope: config.scopes?.default ?? "global", + workspaceBoundary: config.workspaceBoundary, + admissionControl: config.admissionControl, + onAdmissionRejected: admissionRejectionAuditWriter ?? undefined, + log: (msg: string) => api.logger.info(msg), + debugLog: (msg: string) => api.logger.debug(msg), + noiseBank, + }); - (isCliMode() ? api.logger.debug : api.logger.info)( - "memory-lancedb-pro: smart extraction enabled (LLM model: " - + llmModel - + ", timeoutMs: " - + llmTimeoutMs - + ", noise bank: ON)", - ); - } catch (err) { - api.logger.warn(`memory-lancedb-pro: smart extraction init failed, falling back to regex: ${String(err)}`); + (isCliMode() ? api.logger.debug : api.logger.info)( + "memory-lancedb-pro: smart extraction enabled (LLM model: " + + llmModel + + ", timeoutMs: " + + llmTimeoutMs + + ", noise bank: ON)", + ); + } catch (err) { + api.logger.warn(`memory-lancedb-pro: smart extraction init failed, falling back to regex: ${String(err)}`); + } + } + // Build the CLI LLM client using already-resolved secrets. + if (config.smartExtraction !== false) { + try { + const llmAuth = config.llm?.auth || "api-key"; + const llmApiKey = llmAuth === "oauth" + ? undefined + : config.llm?.apiKey + ? await resolveSecretValue(config.llm.apiKey) + : await resolveFirstApiKey(config.embedding.apiKey); + const llmBaseURL = llmAuth === "oauth" + ? (config.llm?.baseURL ? resolveEnvVars(config.llm.baseURL) : undefined) + : config.llm?.baseURL + ? resolveEnvVars(config.llm.baseURL) + : config.embedding.baseURL; + const llmOauthPath = llmAuth === "oauth" + ? resolveOptionalPathWithEnv(api, config.llm?.oauthPath, ".memory-lancedb-pro/oauth.json") + : undefined; + const llmOauthProvider = llmAuth === "oauth" ? config.llm?.oauthProvider : undefined; + const llmTimeoutMs = resolveLlmTimeoutMs(config); + llmClientForCli = createLlmClient({ + auth: llmAuth, + apiKey: llmApiKey, + model: config.llm?.model || "openai/gpt-oss-120b", + baseURL: llmBaseURL, + oauthProvider: llmOauthProvider, + oauthPath: llmOauthPath, + timeoutMs: llmTimeoutMs, + log: (msg: string) => api.logger.debug(msg), + }); + } catch { /* llmClientForCli stays undefined */ } } + })().catch((err: unknown) => { + initFailed = true; + api.logger.warn(`memory-lancedb-pro: async secret init failed: ${String(err)}`); + }); + + // ClawTeam integration: extend accessible scopes via env var (sync) + const clawteamScopes = parseClawteamScopes(process.env.CLAWTEAM_MEMORY_SCOPE); + if (clawteamScopes.length > 0) { + applyClawteamScopes(scopeManager, clawteamScopes); + api.logger.info(`memory-lancedb-pro: CLAWTEAM_MEMORY_SCOPE added scopes: ${clawteamScopes.join(", ")}`); } + const migrator = createMigrator(store); + // Extraction rate limiter (Feature 7: Adaptive Extraction Throttling) // NOTE: This rate limiter is global — shared across all agents in multi-agent setups. const extractionRateLimiter = createExtractionRateLimiter({ @@ -2034,18 +2089,24 @@ const memoryLanceDBProPlugin = { // Register Tools // ======================================================================== + // toolContext is mutated by initPromise once secrets are resolved. + const toolContext = { + retriever: retriever as ReturnType, + store, + scopeManager, + embedder: embedder as ReturnType | undefined, + agentId: undefined as string | undefined, + workspaceDir: getDefaultWorkspaceDir(), + mdMirror, + workspaceBoundary: config.workspaceBoundary, + }; + initPromise.then(() => { + toolContext.embedder = embedder; + toolContext.retriever = retriever; + }); registerAllMemoryTools( api, - { - retriever, - store, - scopeManager, - embedder, - agentId: undefined, // Will be determined at runtime from context - workspaceDir: getDefaultWorkspaceDir(), - mdMirror, - workspaceBoundary: config.workspaceBoundary, - }, + toolContext, { enableManagementTools: config.enableManagementTools, enableSelfImprovementTools: config.selfImprovement?.enabled !== false, @@ -2178,46 +2239,22 @@ const memoryLanceDBProPlugin = { // Register CLI Commands // ======================================================================== + // cliContext is mutated by initPromise once secrets are resolved. + const cliContext = { + store, + retriever: retriever as ReturnType, + scopeManager, + migrator, + embedder: embedder as ReturnType | undefined, + llmClient: undefined as import("./src/llm-client.js").LlmClient | undefined, + }; + initPromise.then(() => { + cliContext.retriever = retriever; + cliContext.embedder = embedder; + cliContext.llmClient = llmClientForCli; + }); api.registerCli( - createMemoryCLI({ - store, - retriever, - scopeManager, - migrator, - embedder, - llmClient: smartExtractor ? (() => { - try { - const llmAuth = config.llm?.auth || "api-key"; - const llmApiKey = llmAuth === "oauth" - ? undefined - : config.llm?.apiKey - ? resolveEnvVars(config.llm.apiKey) - : resolveFirstApiKey(config.embedding.apiKey); - const llmBaseURL = llmAuth === "oauth" - ? (config.llm?.baseURL ? resolveEnvVars(config.llm.baseURL) : undefined) - : config.llm?.baseURL - ? resolveEnvVars(config.llm.baseURL) - : config.embedding.baseURL; - const llmOauthPath = llmAuth === "oauth" - ? resolveOptionalPathWithEnv(api, config.llm?.oauthPath, ".memory-lancedb-pro/oauth.json") - : undefined; - const llmOauthProvider = llmAuth === "oauth" - ? config.llm?.oauthProvider - : undefined; - const llmTimeoutMs = resolveLlmTimeoutMs(config); - return createLlmClient({ - auth: llmAuth, - apiKey: llmApiKey, - model: config.llm?.model || "openai/gpt-oss-120b", - baseURL: llmBaseURL, - oauthProvider: llmOauthProvider, - oauthPath: llmOauthPath, - timeoutMs: llmTimeoutMs, - log: (msg: string) => api.logger.debug(msg), - }); - } catch { return undefined; } - })() : undefined, - }), + createMemoryCLI(cliContext), { commands: ["memory-pro"] }, ); @@ -2248,6 +2285,8 @@ const memoryLanceDBProPlugin = { const AUTO_RECALL_TIMEOUT_MS = parsePositiveInt(config.autoRecallTimeoutMs) ?? 5_000; // configurable; default raised from 3s to 5s for remote embedding APIs behind proxies api.on("before_prompt_build", async (event: any, ctx: any) => { + await initPromise; + if (initFailed) return; // Manually increment turn counter for this session const sessionId = ctx?.sessionId || "default"; @@ -2572,6 +2611,8 @@ const memoryLanceDBProPlugin = { // See: https://github.com/CortexReach/memory-lancedb-pro/issues/260 const backgroundRun = (async () => { try { + await initPromise; + if (initFailed) return; // Feature 7: Check extraction rate limit before any work if (extractionRateLimiter.isRateLimited()) { api.logger.debug( @@ -3073,6 +3114,8 @@ const memoryLanceDBProPlugin = { }, { priority: 15 }); api.on("before_prompt_build", async (_event: any, ctx: any) => { + await initPromise; + if (initFailed) return; const sessionKey = typeof ctx.sessionKey === "string" ? ctx.sessionKey : ""; if (isInternalReflectionSessionKey(sessionKey)) return; if (reflectionInjectMode !== "inheritance-only" && reflectionInjectMode !== "inheritance+derived") return; @@ -3100,6 +3143,8 @@ const memoryLanceDBProPlugin = { }, { priority: 12 }); api.on("before_prompt_build", async (_event: any, ctx: any) => { + await initPromise; + if (initFailed) return; const sessionKey = typeof ctx.sessionKey === "string" ? ctx.sessionKey : ""; if (isInternalReflectionSessionKey(sessionKey)) return; const agentId = resolveHookAgentId( @@ -3502,6 +3547,8 @@ const memoryLanceDBProPlugin = { api.on("before_reset", async (event, ctx) => { if (event.reason !== "new") return; + await initPromise; + if (initFailed) return; try { const sessionKey = typeof ctx.sessionKey === "string" ? ctx.sessionKey : ""; @@ -3607,6 +3654,17 @@ const memoryLanceDBProPlugin = { api.registerService({ id: "memory-lancedb-pro", start: async () => { + // Wait for async secret resolution before running startup checks. + await initPromise; + if (initFailed) { + api.logger.warn( + "memory-lancedb-pro: secret resolution failed during startup — " + + "embedding, recall, and capture hooks are disabled. " + + "Check your embedding.apiKey / retrieval.rerankApiKey config.", + ); + return; + } + // IMPORTANT: Do not block gateway startup on external network calls. // If embedding/retrieval tests hang (bad network / slow provider), the gateway // may never bind its HTTP port, causing restart timeouts. @@ -3700,6 +3758,8 @@ const memoryLanceDBProPlugin = { api.logger.info("memory-lancedb-pro: stopped"); }, }); + + return initPromise; }, }; diff --git a/openclaw.plugin.json b/openclaw.plugin.json index a2cfb1f..d599fd3 100644 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -34,7 +34,7 @@ "minItems": 1 } ], - "description": "Single API key or array of keys for round-robin rotation" + "description": "Single API key or array of keys for round-robin rotation. Supports ${ENV_VAR} and bws:// Bitwarden secret references." }, "model": { "type": "string" @@ -307,7 +307,7 @@ }, "rerankApiKey": { "type": "string", - "description": "API key for reranker service (enables cross-encoder reranking)" + "description": "API key for reranker service (enables cross-encoder reranking). Supports ${ENV_VAR} and bws:// Bitwarden secret references." }, "rerankModel": { "type": "string", @@ -676,7 +676,8 @@ "description": "LLM authentication mode. oauth uses the local Codex/ChatGPT login cache instead of llm.apiKey." }, "apiKey": { - "type": "string" + "type": "string", + "description": "LLM API key. Supports ${ENV_VAR} and bws:// Bitwarden secret references." }, "model": { "type": "string", @@ -842,8 +843,8 @@ "embedding.apiKey": { "label": "API Key(s)", "sensitive": true, - "placeholder": "sk-proj-... or [\"key1\", \"key2\"] for rotation", - "help": "Single API key or array of keys for round-robin rotation with automatic failover on rate limits (or use ${OPENAI_API_KEY}; use a dummy value for keyless local endpoints)" + "placeholder": "sk-proj-... or bws:// or [\"key1\", \"key2\"]", + "help": "Single API key or array of keys for round-robin rotation with automatic failover on rate limits. Supports ${OPENAI_API_KEY} and bws:// Bitwarden refs. Use a dummy value for keyless local endpoints." }, "embedding.model": { "label": "Embedding Model", @@ -902,8 +903,8 @@ "llm.apiKey": { "label": "LLM API Key", "sensitive": true, - "placeholder": "sk-... or ${GROQ_API_KEY}", - "help": "API key for LLM used by smart memory extraction (defaults to embedding.apiKey if omitted)" + "placeholder": "sk-... or ${GROQ_API_KEY} or bws://", + "help": "API key for LLM used by smart memory extraction (defaults to embedding.apiKey if omitted). Supports Bitwarden bws:// refs." }, "llm.model": { "label": "LLM Model", @@ -1062,8 +1063,8 @@ "retrieval.rerankApiKey": { "label": "Reranker API Key", "sensitive": true, - "placeholder": "jina_... / sk-... / pcsk_...", - "help": "Reranker API key for cross-encoder reranking", + "placeholder": "jina_... / sk-... / pcsk_... / bws://", + "help": "Reranker API key for cross-encoder reranking. Supports Bitwarden bws:// refs.", "advanced": true }, "retrieval.rerankModel": { diff --git a/package.json b/package.json index cfd47cd..040b8f0 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ ] }, "scripts": { - "test": "node test/embedder-error-hints.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node test/plugin-manifest-regression.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs", + "test": "node test/embedder-error-hints.test.mjs && node test/cjk-recursion-regression.test.mjs && node test/migrate-legacy-schema.test.mjs && node --test test/config-session-strategy-migration.test.mjs && node --test test/scope-access-undefined.test.mjs && node --test test/reflection-bypass-hook.test.mjs && node --test test/smart-extractor-scope-filter.test.mjs && node --test test/store-empty-scope-filter.test.mjs && node --test test/recall-text-cleanup.test.mjs && node test/update-consistency-lancedb.test.mjs && node --test test/strip-envelope-metadata.test.mjs && node test/cli-smoke.mjs && node test/functional-e2e.mjs && node test/retriever-rerank-regression.mjs && node test/smart-memory-lifecycle.mjs && node test/smart-extractor-branches.mjs && node test/plugin-manifest-regression.mjs && node --test test/session-summary-before-reset.test.mjs && node --test test/sync-plugin-version.test.mjs && node test/smart-metadata-v2.mjs && node test/vector-search-cosine.test.mjs && node test/context-support-e2e.mjs && node test/temporal-facts.test.mjs && node test/memory-update-supersede.test.mjs && node test/memory-upgrader-diagnostics.test.mjs && node --test test/llm-api-key-client.test.mjs && node --test test/secret-resolver.test.mjs && node --test test/llm-oauth-client.test.mjs && node --test test/cli-oauth-login.test.mjs && node --test test/workflow-fork-guards.test.mjs && node --test test/clawteam-scope.test.mjs && node --test test/cross-process-lock.test.mjs && node --test test/preference-slots.test.mjs", "test:openclaw-host": "node test/openclaw-host-functional.mjs", "version": "node scripts/sync-plugin-version.mjs openclaw.plugin.json package.json && git add openclaw.plugin.json" }, diff --git a/src/embedder.ts b/src/embedder.ts index 497f68b..0327724 100644 --- a/src/embedder.ts +++ b/src/embedder.ts @@ -421,7 +421,9 @@ export class Embedder { constructor(config: EmbeddingConfig & { chunking?: boolean }) { // Normalize apiKey to array and resolve environment variables const apiKeys = Array.isArray(config.apiKey) ? config.apiKey : [config.apiKey]; - const resolvedKeys = apiKeys.map(k => resolveEnvVars(k)); + // Skip env-var expansion for keys that have already been resolved + // (e.g. pre-resolved Bitwarden secrets that contain no ${} placeholders). + const resolvedKeys = apiKeys.map(k => k.includes("\${") ? resolveEnvVars(k) : k); this._model = config.model; this._baseURL = config.baseURL; diff --git a/src/secret-resolver.ts b/src/secret-resolver.ts new file mode 100644 index 0000000..1f1df49 --- /dev/null +++ b/src/secret-resolver.ts @@ -0,0 +1,214 @@ +import { execFile as execFileCallback, execFileSync } from "node:child_process"; +import { promisify } from "node:util"; + +const execFile = promisify(execFileCallback); + +export type SecretExecFileResult = { + stdout: string; + stderr: string; +}; + +export type SecretExecFile = ( + file: string, + args: string[], + options?: { + env?: NodeJS.ProcessEnv; + timeout?: number; + }, +) => Promise; + +export type SecretResolverOptions = { + env?: NodeJS.ProcessEnv; + execFileImpl?: SecretExecFile; + timeoutMs?: number; +}; + +export type SecretResolverSyncOptions = { + env?: NodeJS.ProcessEnv; + timeoutMs?: number; +}; + +type BitwardenSecretRef = { + id: string; + accessToken?: string; + configFile?: string; + profile?: string; + serverUrl?: string; +}; + +function getEnv(options?: SecretResolverOptions): NodeJS.ProcessEnv { + return options?.env ?? process.env; +} + +export function resolveEnvVarsSync(value: string, env: NodeJS.ProcessEnv = process.env): string { + return value.replace(/\$\{([^}]+)\}/g, (_, envVar) => { + const envValue = env[envVar]; + if (!envValue) { + throw new Error(`Environment variable ${envVar} is not set`); + } + return envValue; + }); +} + +function parseBitwardenSecretRef(value: string, env: NodeJS.ProcessEnv): BitwardenSecretRef | null { + const trimmed = value.trim(); + if (!/^bws:\/\//i.test(trimmed)) return null; + + const parsed = new URL(trimmed); + const rawId = `${parsed.hostname}${parsed.pathname}`.replace(/^\/+/, ""); + const normalizedId = rawId.replace(/^secret\//i, ""); + if (!normalizedId) { + throw new Error(`Invalid Bitwarden secret reference: ${value}`); + } + + const accessTokenRaw = parsed.searchParams.get("accessToken"); + const configFileRaw = parsed.searchParams.get("configFile"); + const profileRaw = parsed.searchParams.get("profile"); + const serverUrlRaw = parsed.searchParams.get("serverUrl"); + + return { + id: normalizedId, + accessToken: accessTokenRaw ? resolveEnvVarsSync(accessTokenRaw, env) : undefined, + configFile: configFileRaw ? resolveEnvVarsSync(configFileRaw, env) : undefined, + profile: profileRaw ? resolveEnvVarsSync(profileRaw, env) : undefined, + serverUrl: serverUrlRaw ? resolveEnvVarsSync(serverUrlRaw, env) : undefined, + }; +} + +async function resolveBitwardenSecret( + ref: BitwardenSecretRef, + options?: SecretResolverOptions, +): Promise { + const execImpl = options?.execFileImpl ?? execFile; + const env = getEnv(options); + const args = ["secret", "get", ref.id, "--output", "json"]; + // Pass access token via env var to avoid exposure in process listings. + const childEnv = ref.accessToken ? { ...env, BWS_ACCESS_TOKEN: ref.accessToken } : env; + if (ref.configFile) args.push("--config-file", ref.configFile); + if (ref.profile) args.push("--profile", ref.profile); + if (ref.serverUrl) args.push("--server-url", ref.serverUrl); + + let stdout = ""; + let stderr = ""; + try { + const result = await execImpl("bws", args, { + env: childEnv, + timeout: options?.timeoutMs ?? 10_000, + }); + stdout = result.stdout; + stderr = result.stderr; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + throw new Error( + `Failed to resolve Bitwarden secret ${ref.id} via bws secret get: ${msg}`, + ); + } + + let parsed: Record; + try { + parsed = JSON.parse(stdout) as Record; + } catch (error) { + throw new Error( + `Bitwarden secret ${ref.id} did not return valid JSON: ${error instanceof Error ? error.message : String(error)}; stderr=${stderr.trim() || "(none)"}`, + ); + } + + const value = + typeof parsed.value === "string" + ? parsed.value + : typeof parsed.note === "string" + ? parsed.note + : null; + if (!value || !value.trim()) { + throw new Error(`Bitwarden secret ${ref.id} has no value`); + } + return value; +} + +function resolveBitwardenSecretSync( + ref: BitwardenSecretRef, + options?: SecretResolverSyncOptions, +): string { + const env = options?.env ?? process.env; + const args = ["secret", "get", ref.id, "--output", "json"]; + // Pass access token via env var to avoid exposure in process listings. + const childEnv = ref.accessToken ? { ...env, BWS_ACCESS_TOKEN: ref.accessToken } : env; + if (ref.configFile) args.push("--config-file", ref.configFile); + if (ref.profile) args.push("--profile", ref.profile); + if (ref.serverUrl) args.push("--server-url", ref.serverUrl); + + let stdout = ""; + try { + stdout = execFileSync("bws", args, { + env: childEnv, + timeout: options?.timeoutMs ?? 10_000, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + throw new Error(`Failed to resolve Bitwarden secret ${ref.id} via bws secret get: ${msg}`); + } + + let parsed: Record; + try { + parsed = JSON.parse(stdout) as Record; + } catch (error) { + throw new Error( + `Bitwarden secret ${ref.id} did not return valid JSON: ${error instanceof Error ? error.message : String(error)}`, + ); + } + + const value = + typeof parsed.value === "string" + ? parsed.value + : typeof parsed.note === "string" + ? parsed.note + : null; + if (!value || !value.trim()) { + throw new Error(`Bitwarden secret ${ref.id} has no value`); + } + return value; +} + +export async function resolveSecretValue( + value: string, + options?: SecretResolverOptions, +): Promise { + const env = getEnv(options); + const envResolved = resolveEnvVarsSync(value, env); + const bitwardenRef = parseBitwardenSecretRef(envResolved, env); + if (!bitwardenRef) { + return envResolved; + } + return resolveBitwardenSecret(bitwardenRef, options); +} + +export function resolveSecretValueSync( + value: string, + options?: SecretResolverSyncOptions, +): string { + const env = options?.env ?? process.env; + const envResolved = resolveEnvVarsSync(value, env); + const bitwardenRef = parseBitwardenSecretRef(envResolved, env); + if (!bitwardenRef) { + return envResolved; + } + return resolveBitwardenSecretSync(bitwardenRef, options); +} + +export async function resolveSecretValues( + value: string | string[], + options?: SecretResolverOptions, +): Promise { + const values = Array.isArray(value) ? value : [value]; + return Promise.all(values.map((entry) => resolveSecretValue(entry, options))); +} + +export function resolveSecretValuesSync( + value: string | string[], + options?: SecretResolverSyncOptions, +): string[] { + const values = Array.isArray(value) ? value : [value]; + return values.map((entry) => resolveSecretValueSync(entry, options)); +} diff --git a/test/plugin-manifest-regression.mjs b/test/plugin-manifest-regression.mjs index 65e9ec2..64f9147 100644 --- a/test/plugin-manifest-regression.mjs +++ b/test/plugin-manifest-regression.mjs @@ -139,6 +139,7 @@ try { { dbPath: path.join(workDir, "db"), autoRecall: false, + selfImprovement: { enabled: false }, embedding: { provider: "openai-compatible", apiKey: "dummy", @@ -149,7 +150,7 @@ try { }, { services }, ); - plugin.register(api); + await plugin.register(api); assert.equal(services.length, 1, "plugin should register its background service"); assert.equal(typeof api.hooks.agent_end, "function", "autoCapture should remain enabled by default"); assert.equal(api.hooks["command:new"], undefined, "sessionMemory should stay disabled by default"); @@ -162,6 +163,7 @@ try { dbPath: path.join(workDir, "db-session-default"), autoCapture: false, autoRecall: false, + selfImprovement: { enabled: false }, sessionMemory: {}, embedding: { provider: "openai-compatible", @@ -171,7 +173,7 @@ try { dimensions: 1536, }, }); - plugin.register(sessionDefaultApi); + await plugin.register(sessionDefaultApi); assert.equal( sessionDefaultApi.hooks["command:new"], undefined, @@ -182,6 +184,7 @@ try { dbPath: path.join(workDir, "db-session-enabled"), autoCapture: false, autoRecall: false, + selfImprovement: { enabled: false }, sessionMemory: { enabled: true }, embedding: { provider: "openai-compatible", @@ -191,7 +194,7 @@ try { dimensions: 1536, }, }); - plugin.register(sessionEnabledApi); + await plugin.register(sessionEnabledApi); assert.equal( typeof sessionEnabledApi.hooks.before_reset, "function", @@ -263,7 +266,7 @@ try { chunking: false, }, }); - plugin.register(chunkingOffApi); + await plugin.register(chunkingOffApi); const chunkingOffTool = chunkingOffApi.toolFactories.memory_store({ agentId: "main", sessionKey: "agent:main:test", @@ -291,7 +294,7 @@ try { chunking: true, }, }); - plugin.register(chunkingOnApi); + await plugin.register(chunkingOnApi); const chunkingOnTool = chunkingOnApi.toolFactories.memory_store({ agentId: "main", sessionKey: "agent:main:test", @@ -318,7 +321,7 @@ try { dimensions: 4, }, }); - plugin.register(withDimensionsApi); + await plugin.register(withDimensionsApi); const withDimensionsTool = withDimensionsApi.toolFactories.memory_store({ agentId: "main", sessionKey: "agent:main:test", @@ -348,7 +351,7 @@ try { omitDimensions: true, }, }); - plugin.register(omitDimensionsApi); + await plugin.register(omitDimensionsApi); const omitDimensionsTool = omitDimensionsApi.toolFactories.memory_store({ agentId: "main", sessionKey: "agent:main:test", diff --git a/test/recall-text-cleanup.test.mjs b/test/recall-text-cleanup.test.mjs index 3788ccd..f181a99 100644 --- a/test/recall-text-cleanup.test.mjs +++ b/test/recall-text-cleanup.test.mjs @@ -415,7 +415,7 @@ describe("recall text cleanup", () => { }, }); - memoryLanceDBProPlugin.register(harness.api); + await memoryLanceDBProPlugin.register(harness.api); const hooks = harness.eventHandlers.get("before_prompt_build") || []; assert.equal(hooks.length, 1, "expected at least one before_prompt_build hook for this config"); @@ -630,7 +630,7 @@ describe("recall text cleanup", () => { }, }); - memoryLanceDBProPlugin.register(harness.api); + await memoryLanceDBProPlugin.register(harness.api); const hooks = harness.eventHandlers.get("before_prompt_build") || []; const [{ handler: autoRecallHook }] = hooks; const output = await autoRecallHook( @@ -688,7 +688,7 @@ describe("recall text cleanup", () => { selfImprovement: { enabled: false, beforeResetNote: false, ensureLearningFiles: false }, }, }); - memoryLanceDBProPlugin.register(harness.api); + await memoryLanceDBProPlugin.register(harness.api); const hooks = harness.eventHandlers.get("before_prompt_build") || []; const [{ handler: autoRecallHook }] = hooks; const output = await autoRecallHook( @@ -808,7 +808,7 @@ describe("recall text cleanup", () => { }, }); - memoryLanceDBProPlugin.register(harness.api); + await memoryLanceDBProPlugin.register(harness.api); const hooks = harness.eventHandlers.get("before_prompt_build") || []; assert.equal(hooks.length, 1); @@ -890,7 +890,7 @@ describe("recall text cleanup", () => { }, }); - memoryLanceDBProPlugin.register(harness.api); + await memoryLanceDBProPlugin.register(harness.api); const hooks = harness.eventHandlers.get("before_prompt_build") || []; assert.equal(hooks.length, 1); diff --git a/test/reflection-bypass-hook.test.mjs b/test/reflection-bypass-hook.test.mjs index e67cfbd..e73805f 100644 --- a/test/reflection-bypass-hook.test.mjs +++ b/test/reflection-bypass-hook.test.mjs @@ -114,7 +114,7 @@ async function invokeReflectionHooks({ workDir, agentId, explicitAgentId = agent pluginConfig, }); - memoryLanceDBProPlugin.register(harness.api); + await memoryLanceDBProPlugin.register(harness.api); const promptHooks = harness.eventHandlers.get("before_prompt_build") || []; diff --git a/test/resolve-env-vars-array.test.mjs b/test/resolve-env-vars-array.test.mjs index 454568b..1d40282 100644 --- a/test/resolve-env-vars-array.test.mjs +++ b/test/resolve-env-vars-array.test.mjs @@ -96,7 +96,7 @@ async function withTestEnv(apiKeyConfig, fn) { registerHook(name, handler) { this.hooks[name] = handler; }, }; - plugin.register(api); + await plugin.register(api); await fn(logs); } finally { await new Promise((r) => embeddingServer.close(r)); diff --git a/test/secret-resolver.test.mjs b/test/secret-resolver.test.mjs new file mode 100644 index 0000000..6e17d5c --- /dev/null +++ b/test/secret-resolver.test.mjs @@ -0,0 +1,104 @@ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { + resolveEnvVarsSync, + resolveSecretValue, + resolveSecretValues, +} = jiti("../src/secret-resolver.ts"); + +describe("secret resolver", () => { + it("resolves environment-variable templates synchronously", () => { + const value = resolveEnvVarsSync("${TEST_SECRET_VALUE}", { + TEST_SECRET_VALUE: "resolved", + }); + assert.equal(value, "resolved"); + }); + + it("passes through plain strings without executing bws", async () => { + let called = false; + const value = await resolveSecretValue("plain-value", { + execFileImpl: async () => { + called = true; + return { stdout: "", stderr: "" }; + }, + }); + assert.equal(value, "plain-value"); + assert.equal(called, false); + }); + + it("resolves bws:// refs via the Bitwarden CLI", async () => { + let captured; + const value = await resolveSecretValue( + "bws://49e0d691-11c4-43cc-a3f9-b40e00f83237?profile=${BWS_PROFILE}", + { + env: { BWS_PROFILE: "ops" }, + execFileImpl: async (file, args) => { + captured = { file, args }; + return { + stdout: JSON.stringify({ + id: "49e0d691-11c4-43cc-a3f9-b40e00f83237", + value: "secret-from-bws", + }), + stderr: "", + }; + }, + }, + ); + + assert.equal(value, "secret-from-bws"); + assert.deepEqual(captured, { + file: "bws", + args: [ + "secret", + "get", + "49e0d691-11c4-43cc-a3f9-b40e00f83237", + "--output", + "json", + "--profile", + "ops", + ], + }); + }); + + it("resolves arrays of secret refs", async () => { + const values = await resolveSecretValues( + ["bws://first-secret", "${SECOND_SECRET}"], + { + env: { SECOND_SECRET: "second-value" }, + execFileImpl: async (_file, args) => ({ + stdout: JSON.stringify({ + id: args[2], + value: `${args[2]}-value`, + }), + stderr: "", + }), + }, + ); + + assert.deepEqual(values, ["first-secret-value", "second-value"]); + }); + + it("throws on bws:// URL with no secret ID (empty hostname and path)", async () => { + await assert.rejects( + () => resolveSecretValue("bws:///", {}), + /Invalid Bitwarden secret reference/, + ); + }); + + it("throws on bws:// URL with only a slash path and no hostname", async () => { + await assert.rejects( + () => resolveSecretValue("bws:///secret/", {}), + /Invalid Bitwarden secret reference/, + ); + }); + + it("throws on bws:// URL where secret ID reduces to empty after prefix strip", async () => { + await assert.rejects( + () => resolveSecretValue("bws://secret/", {}), + /Invalid Bitwarden secret reference/, + ); + }); +}); diff --git a/test/session-summary-before-reset.test.mjs b/test/session-summary-before-reset.test.mjs index d3c7a1a..12afc8d 100644 --- a/test/session-summary-before-reset.test.mjs +++ b/test/session-summary-before-reset.test.mjs @@ -44,6 +44,7 @@ function createApiHarness({ dbPath, embeddingBaseURL }) { dbPath, autoCapture: false, autoRecall: false, + selfImprovement: { enabled: false }, sessionStrategy: "systemSessionMemory", embedding: { provider: "openai-compatible", @@ -101,8 +102,9 @@ describe("systemSessionMemory before_reset", () => { const dbPath = path.join(workDir, "db"); const api = createApiHarness({ dbPath, embeddingBaseURL }); - memoryLanceDBProPlugin.register(api); - + await memoryLanceDBProPlugin.register(api); + // register() returns initPromise; awaiting it ensures secrets are resolved + // and hooks (including before_reset) are fully initialized. assert.equal(typeof api.hooks.before_reset, "function"); assert.equal(api.hooks["command:new"], undefined); diff --git a/test/smart-extractor-branches.mjs b/test/smart-extractor-branches.mjs index 67830f6..e2d98dc 100644 --- a/test/smart-extractor-branches.mjs +++ b/test/smart-extractor-branches.mjs @@ -268,7 +268,7 @@ async function runScenario(mode) { `http://127.0.0.1:${port}`, logs, ); - plugin.register(api); + await plugin.register(api); await seedPreference(dbPath); await runAgentEndHook( @@ -450,7 +450,7 @@ async function runMultiRoundScenario() { `http://127.0.0.1:${port}`, logs, ); - plugin.register(api); + await plugin.register(api); const rounds = [ ["最近我在调整饮品偏好。", "我喜欢乌龙茶。", "这条偏好以后都有效。", "请记住。"], @@ -549,7 +549,7 @@ async function runInjectedRecallScenario() { `http://127.0.0.1:${port}`, logs, ); - plugin.register(api); + await plugin.register(api); await runAgentEndHook( api, @@ -643,7 +643,7 @@ async function runPrependedRecallWithUserTextScenario() { `http://127.0.0.1:${port}`, logs, ); - plugin.register(api); + await plugin.register(api); await runAgentEndHook( api, @@ -735,7 +735,7 @@ async function runInboundMetadataWrappedScenario() { `http://127.0.0.1:${port}`, logs, ); - plugin.register(api); + await plugin.register(api); await runAgentEndHook( api, @@ -791,7 +791,7 @@ async function runSessionDeltaScenario() { "http://127.0.0.1:9", logs, ); - plugin.register(api); + await plugin.register(api); await runAgentEndHook( api, @@ -855,7 +855,7 @@ async function runPendingIngressScenario() { "http://127.0.0.1:9", logs, ); - plugin.register(api); + await plugin.register(api); await api.hooks.message_received( { from: "discord:channel:1", content: "@jige_claw_bot 我的饮品偏好是乌龙茶" }, @@ -911,7 +911,7 @@ async function runRememberCommandContextScenario() { "http://127.0.0.1:9", logs, ); - plugin.register(api); + await plugin.register(api); await api.hooks.message_received( { from: "discord:channel:1", content: "@jige_claw_bot 我的饮品偏好是乌龙茶" }, @@ -1036,7 +1036,7 @@ async function runUserMdExclusiveProfileScenario() { enabled: true, }, }; - plugin.register(api); + await plugin.register(api); await runAgentEndHook( api, @@ -1134,7 +1134,7 @@ async function runBoundarySkipKeepsRegexFallbackScenario() { enabled: true, }, }; - plugin.register(api); + await plugin.register(api); await runAgentEndHook( api, @@ -1236,7 +1236,7 @@ async function runInboundMetadataCleanupScenario() { `http://127.0.0.1:${port}`, logs, ); - plugin.register(api); + await plugin.register(api); await runAgentEndHook( api,