diff --git a/.bumpy/proton-pass-batch-reads.md b/.bumpy/proton-pass-batch-reads.md new file mode 100644 index 000000000..c10d4dd21 --- /dev/null +++ b/.bumpy/proton-pass-batch-reads.md @@ -0,0 +1,5 @@ +--- +"@varlock/proton-pass-plugin": patch +--- + +batch proton pass reads to reduce repeated prompts and add regression tests diff --git a/packages/plugins/proton-pass/src/plugin.ts b/packages/plugins/proton-pass/src/plugin.ts index bdf81c5a0..44827b2f9 100644 --- a/packages/plugins/proton-pass/src/plugin.ts +++ b/packages/plugins/proton-pass/src/plugin.ts @@ -90,6 +90,8 @@ function extractJsonFieldValue( } class ProtonPassPluginInstance { + private static readonly BATCH_READ_TIMEOUT_MS = 50; + private username?: string; private password?: string; private totp?: string; @@ -98,6 +100,12 @@ class ProtonPassPluginInstance { // Cache decrypted values for the current resolution session. private cache = new Map(); + // Batch pending secret reads to reduce repeated interactive prompts. + private readBatch: Record void; + reject: (e: unknown) => void; + }> }> | undefined; + // Login batching / deduping for parallel resolutions. private loginInFlight: Promise | undefined; @@ -220,7 +228,7 @@ class ProtonPassPluginInstance { } } - async getSecret(secretRef: string): Promise { + private async getSecretDirect(secretRef: string): Promise { const cached = this.cache.get(secretRef); if (cached !== undefined) return cached; @@ -262,6 +270,110 @@ class ProtonPassPluginInstance { return plain; } } + + private async executeReadBatch( + batchToExecute: NonNullable, + ): Promise { + const batchSecretRefs = Object.keys(batchToExecute); + debug('executing proton pass batch read', batchSecretRefs); + try { + await this.ensureCliLoggedIn(); + + const envMap: Record = {}; + let i = 1; + for (const secretRef of batchSecretRefs) { + envMap[`VARLOCK_PROTON_PASS_INJECT_${i++}`] = secretRef; + } + + const result = await spawnAsync( + 'pass-cli', + ['run', '--no-masking', '--', 'env', '-0'], + { + env: { + ...(process.env as Record), + ...(this.loginEnv || {}), + ...envMap, + }, + }, + ); + + const unresolvedRefs = new Set(batchSecretRefs); + const lines = result.split('\0'); + for (const line of lines) { + const eqPos = line.indexOf('='); + if (eqPos <= 0) continue; + const key = line.substring(0, eqPos); + const secretRef = envMap[key]; + if (!secretRef) continue; + + const val = line.substring(eqPos + 1); + unresolvedRefs.delete(secretRef); + this.cache.set(secretRef, val); + batchToExecute[secretRef].deferredPromises.forEach((p) => p.resolve(val)); + } + + // Any unresolved refs are retried individually to preserve useful per-secret errors. + if (unresolvedRefs.size) { + debug('batch did not resolve all refs, retrying direct reads', [...unresolvedRefs]); + await Promise.all([...unresolvedRefs].map(async (secretRef) => { + try { + const val = await this.getSecretDirect(secretRef); + batchToExecute[secretRef].deferredPromises.forEach((p) => p.resolve(val)); + } catch (err) { + batchToExecute[secretRef].deferredPromises.forEach((p) => p.reject(err)); + } + })); + } + } catch (err) { + // Retry each ref individually on batch failure so allowMissing + per-ref errors still work. + debug('proton pass batch read failed, retrying per-ref', err instanceof Error ? err.message : String(err)); + await Promise.all(batchSecretRefs.map(async (secretRef) => { + try { + const val = await this.getSecretDirect(secretRef); + batchToExecute[secretRef].deferredPromises.forEach((p) => p.resolve(val)); + } catch (refErr) { + batchToExecute[secretRef].deferredPromises.forEach((p) => p.reject(refErr)); + } + })); + } + } + + async getSecret(secretRef: string): Promise { + const cached = this.cache.get(secretRef); + if (cached !== undefined) return cached; + + let shouldExecuteBatch = false; + if (!this.readBatch) { + this.readBatch = {}; + shouldExecuteBatch = true; + } + this.readBatch[secretRef] ||= { deferredPromises: [] }; + + const deferred = {} as { + promise: Promise; + resolve: (value: string) => void; + reject: (error: unknown) => void; + }; + deferred.promise = new Promise((resolve, reject) => { + deferred.resolve = resolve; + deferred.reject = reject; + }); + this.readBatch[secretRef].deferredPromises.push({ + resolve: deferred.resolve, + reject: deferred.reject, + }); + + if (shouldExecuteBatch) { + setTimeout(async () => { + if (!this.readBatch) return; + const batchToExecute = this.readBatch; + this.readBatch = undefined; + await this.executeReadBatch(batchToExecute); + }, ProtonPassPluginInstance.BATCH_READ_TIMEOUT_MS); + } + + return deferred.promise; + } } const pluginInstances: Record = {}; @@ -487,5 +599,3 @@ plugin.registerResolverFunction({ } }, }); - - diff --git a/packages/plugins/proton-pass/test/fake-pass-cli.cjs b/packages/plugins/proton-pass/test/fake-pass-cli.cjs new file mode 100644 index 000000000..2fe8fd8f0 --- /dev/null +++ b/packages/plugins/proton-pass/test/fake-pass-cli.cjs @@ -0,0 +1,116 @@ +#!/usr/bin/env node +/* eslint-disable @typescript-eslint/no-require-imports */ + +const fs = require('node:fs'); +const path = require('node:path'); + +function loadJson(filePath, fallback) { + try { + return JSON.parse(fs.readFileSync(filePath, 'utf8')); + } catch { + return fallback; + } +} + +function writeJson(filePath, value) { + fs.writeFileSync(filePath, JSON.stringify(value), 'utf8'); +} + +function appendLog(logPath, args) { + if (!logPath) return; + fs.appendFileSync(logPath, `${JSON.stringify(args)}\n`, 'utf8'); +} + +function fail(message, code = 1) { + process.stderr.write(`${message}\n`); + process.exit(code); +} + +const mockDir = process.env.FAKE_PASS_CLI_DIR; +if (!mockDir) fail('fake pass-cli misconfigured: FAKE_PASS_CLI_DIR is required'); + +const configPath = path.join(mockDir, 'config.json'); +const statePath = path.join(mockDir, 'state.json'); +const logPath = path.join(mockDir, 'calls.log'); + +const config = loadJson(configPath, {}); +const state = loadJson(statePath, { loggedIn: false }); +const args = process.argv.slice(2); + +appendLog(logPath, args); + +const ensureLoggedIn = () => { + if (!state.loggedIn) { + fail(config.notLoggedMessage || 'not authenticated'); + } +}; + +const resolveRefsInText = (val) => { + if (typeof val !== 'string') return val; + return val.replace(/pass:\/\/[^\s"'`]+/g, (ref) => { + if (config.refs && Object.prototype.hasOwnProperty.call(config.refs, ref)) { + return String(config.refs[ref]); + } + if (config.missingRefs && config.missingRefs.includes(ref)) { + throw new Error(config.notFoundMessage || `secret not found: ${ref}`); + } + throw new Error(config.notFoundMessage || `secret not found: ${ref}`); + }); +}; + +if (!args.length) fail('missing command'); + +if (args[0] === 'info') { + if (state.loggedIn) process.exit(0); + fail(config.notLoggedMessage || 'not authenticated'); +} + +if (args[0] === 'login') { + const needsPassword = config.requirePassword !== false; + if (needsPassword && !process.env.PROTON_PASS_PASSWORD) { + fail('password missing'); + } + state.loggedIn = true; + writeJson(statePath, state); + process.exit(0); +} + +if (args[0] === 'run') { + ensureLoggedIn(); + if (config.runErrorMessage) fail(config.runErrorMessage); + + const runSkipRefs = new Set(config.runSkipRefs || []); + const outputPairs = []; + for (const [key, rawVal] of Object.entries(process.env)) { + if (!key.startsWith('VARLOCK_PROTON_PASS_INJECT_')) continue; + if (runSkipRefs.has(rawVal)) continue; + try { + const resolved = resolveRefsInText(rawVal); + outputPairs.push(`${key}=${resolved}`); + } catch (err) { + fail((err && err.message) || 'run resolution failed'); + } + } + + process.stdout.write(outputPairs.join('\0')); + if (outputPairs.length) process.stdout.write('\0'); + process.exit(0); +} + +if (args[0] === 'item' && args[1] === 'view') { + ensureLoggedIn(); + const secretRef = args[args.length - 1]; + const parts = String(secretRef).replace(/^pass:\/\//, '').split('/'); + const fieldName = parts[parts.length - 1] || 'value'; + const itemErrors = config.itemViewErrors || {}; + if (itemErrors[secretRef]) fail(itemErrors[secretRef]); + + if (config.refs && Object.prototype.hasOwnProperty.call(config.refs, secretRef)) { + process.stdout.write(JSON.stringify({ [fieldName]: String(config.refs[secretRef]) })); + process.exit(0); + } + + fail(config.notFoundMessage || `secret not found: ${secretRef}`); +} + +fail(`unsupported fake pass-cli command: ${args.join(' ')}`); diff --git a/packages/plugins/proton-pass/test/proton-pass.test.ts b/packages/plugins/proton-pass/test/proton-pass.test.ts new file mode 100644 index 000000000..716e80f13 --- /dev/null +++ b/packages/plugins/proton-pass/test/proton-pass.test.ts @@ -0,0 +1,185 @@ +import path from 'node:path'; +import fs from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { + afterAll, beforeAll, describe, test, +} from 'vitest'; +import { pluginTest } from 'varlock/test-helpers'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const PLUGIN_PATH = path.join(__dirname, '..'); +const FIXTURES_DIR = path.join(__dirname, 'fixtures'); +const FAKE_BIN_DIR = path.join(FIXTURES_DIR, 'bin'); +const FAKE_PASS_CLI_SRC = path.join(__dirname, 'fake-pass-cli.cjs'); +const FAKE_PASS_CLI = path.join(FAKE_BIN_DIR, 'pass-cli'); +const CONFIG_PATH = path.join(FAKE_BIN_DIR, 'config.json'); +const STATE_PATH = path.join(FAKE_BIN_DIR, 'state.json'); + +type FakePassConfig = { + refs?: Record; + missingRefs?: Array; + runSkipRefs?: Array; + notLoggedMessage?: string; + notFoundMessage?: string; + requirePassword?: boolean; + runErrorMessage?: string; + itemViewErrors?: Record; +}; + +function resetFakeCli(config: FakePassConfig) { + fs.mkdirSync(FAKE_BIN_DIR, { recursive: true }); + fs.writeFileSync(CONFIG_PATH, JSON.stringify(config), 'utf8'); + fs.writeFileSync(STATE_PATH, JSON.stringify({ loggedIn: false }), 'utf8'); +} + +async function runProtonPassTest(opts: { + config: FakePassConfig; + schemaItems?: string; + expectValues?: Record; + fullSchema?: string; +}) { + resetFakeCli(opts.config); + + const originalPath = process.env.PATH; + const originalFakeDir = process.env.FAKE_PASS_CLI_DIR; + + process.env.PATH = `${FAKE_BIN_DIR}:${originalPath}`; + process.env.FAKE_PASS_CLI_DIR = FAKE_BIN_DIR; + + try { + const fullSchema = opts.fullSchema ?? ` +# @plugin(${PLUGIN_PATH}) +# @initProtonPass(username=$PROTON_PASS_USERNAME, password=$PROTON_PASS_PASSWORD) +# --- +PROTON_PASS_USERNAME=test@example.com +PROTON_PASS_PASSWORD=test-password +${opts.schemaItems || ''} +`; + await pluginTest({ + schema: fullSchema, + ...(opts.expectValues ? { expectValues: opts.expectValues } : {}), + })(); + } finally { + process.env.PATH = originalPath; + if (originalFakeDir === undefined) { + delete process.env.FAKE_PASS_CLI_DIR; + } else { + process.env.FAKE_PASS_CLI_DIR = originalFakeDir; + } + } +} + +beforeAll(() => { + fs.mkdirSync(FAKE_BIN_DIR, { recursive: true }); + fs.writeFileSync( + FAKE_PASS_CLI, + `#!/usr/bin/env bash\nnode "${FAKE_PASS_CLI_SRC}" "$@"\n`, + 'utf8', + ); + fs.chmodSync(FAKE_PASS_CLI, 0o755); +}); + +afterAll(() => { + fs.rmSync(FIXTURES_DIR, { recursive: true, force: true }); +}); + +describe('proton-pass plugin', () => { + test('batches parallel reads into a single run invocation', async () => { + await runProtonPassTest({ + config: { + refs: { + 'pass://Production/Database/username': 'db-admin', + 'pass://Production/Database/password': 'db-secret', + 'pass://Production/API/key': 'api-key-123', + }, + }, + schemaItems: ` +DB_USER=protonPass(pass://Production/Database/username) +DB_PASS=protonPass(pass://Production/Database/password) +API_KEY=protonPass(pass://Production/API/key) +`, + expectValues: { + DB_USER: 'db-admin', + DB_PASS: 'db-secret', + API_KEY: 'api-key-123', + }, + }); + }); + + test('falls back to per-secret item view when batched run fails', async () => { + await runProtonPassTest({ + config: { + refs: { + 'pass://Production/Database/username': 'db-admin', + 'pass://Production/Database/password': 'db-secret', + }, + runErrorMessage: 'simulated run failure', + }, + schemaItems: ` +DB_USER=protonPass(pass://Production/Database/username) +DB_PASS=protonPass(pass://Production/Database/password) +`, + expectValues: { + DB_USER: 'db-admin', + DB_PASS: 'db-secret', + }, + }); + }); + + test('retries unresolved refs individually when batch omits some refs', async () => { + await runProtonPassTest({ + config: { + refs: { + 'pass://Production/Database/username': 'db-admin', + 'pass://Production/Database/password': 'db-secret', + }, + runSkipRefs: ['pass://Production/Database/password'], + }, + schemaItems: ` +DB_USER=protonPass(pass://Production/Database/username) +DB_PASS=protonPass(pass://Production/Database/password) +`, + expectValues: { + DB_USER: 'db-admin', + DB_PASS: 'db-secret', + }, + }); + }); + + test('missing ref without allowMissing surfaces an error', async () => { + await runProtonPassTest({ + config: { + refs: { + 'pass://Production/Database/username': 'db-admin', + }, + missingRefs: ['pass://Production/Database/missing'], + notFoundMessage: 'secret not found', + }, + schemaItems: ` +DB_USER=protonPass(pass://Production/Database/username) +MISSING=protonPass(pass://Production/Database/missing) +`, + expectValues: { + DB_USER: 'db-admin', + MISSING: Error, + }, + }); + }); + + test('returns auth error when not logged in and password is missing', async () => { + await runProtonPassTest({ + config: { + notLoggedMessage: 'not authenticated', + }, + fullSchema: ` +# @plugin(${PLUGIN_PATH}) +# @initProtonPass(username=test@example.com) +# --- +SECRET=protonPass(pass://Production/Database/password) +`, + expectValues: { + SECRET: Error, + }, + }); + }); +}); diff --git a/packages/plugins/proton-pass/vitest.config.ts b/packages/plugins/proton-pass/vitest.config.ts new file mode 100644 index 000000000..e98196540 --- /dev/null +++ b/packages/plugins/proton-pass/vitest.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + resolve: { + conditions: ['ts-src'], + }, + define: { + __VARLOCK_BUILD_TYPE__: JSON.stringify('test'), + __VARLOCK_SEA_BUILD__: 'false', + }, +}); diff --git a/packages/varlock-website/src/content/docs/plugins/proton-pass.mdx b/packages/varlock-website/src/content/docs/plugins/proton-pass.mdx index 9286fbab6..c62bd7fc3 100644 --- a/packages/varlock-website/src/content/docs/plugins/proton-pass.mdx +++ b/packages/varlock-website/src/content/docs/plugins/proton-pass.mdx @@ -166,5 +166,5 @@ DB_PASSWORD=protonPass(prod, pass://Production/Database/password) This resolver uses: - `pass-cli info` to check authentication state - `pass-cli login --interactive ` when login is required -- `pass-cli item view --output json ` to fetch the requested field - +- `pass-cli run --no-masking -- env -0` to batch multiple `pass://...` refs in one CLI invocation +- `pass-cli item view --output json ` as a fallback for per-secret retries/error handling