Add caching system, random generators, and plugin cache API

.bumpy/cache-system.md

@@ -0,0 +1,5 @@
+---
+"varlock": minor
+---
+
+add caching system with `cache()` resolver, random value generators (`randomInt()`, `randomFloat()`, `randomUuid()`, `randomHex()`, `randomString()`), and plugin cache API

packages/plugins/1password/src/plugin.ts

@@ -1,4 +1,6 @@
-import { type Resolver, plugin } from 'varlock/plugin-lib';
+import {
+  type Resolver, type PluginCacheAccessor, plugin, resolveCacheTtl,
+} from 'varlock/plugin-lib';
 
 import { createDeferredPromise, type DeferredPromise } from '@env-spec/utils/defer';
 import { spawnAsync } from '@env-spec/utils/exec-helpers';
@@ -178,6 +180,15 @@ async function appAuthCliRead(opReference: string, account?: string): Promise<st
 
 plugin.name = '1pass';
 debug('init - version =', plugin.version);
+
+// capture cache accessor while the plugin proxy context is active
+// (the `plugin` proxy is only valid during module initialization, not during resolve())
+let pluginCache: PluginCacheAccessor | undefined;
+try {
+  pluginCache = plugin.cache;
+} catch {
+  // cache not available (e.g., no encryption key)
+}
 plugin.icon = OP_ICON;
 plugin.standardVars = {
   initDecorator: '@initOp',
@@ -268,6 +279,8 @@ class OpPluginInstance {
   private connectToken?: string;
   /** If true, missing items/fields/vaults return undefined instead of throwing */
   allowMissing?: boolean;
+  /** optional cache TTL - when set, resolved values are cached */
+  cacheTtl?: string | number;
 
   /** Per-instance auth mutex for the service-account CLI path (each token is an independent auth context). */
   private cliAuthDeferred?: DeferredPromise<boolean>;
@@ -787,14 +800,15 @@ plugin.registerRootDecorator({
       account,
       connectHost,
       allowMissingResolver: objArgs.allowMissing,
+      cacheTtlResolver: objArgs.cacheTtl,
       tokenResolver: objArgs.token,
       allowAppAuthResolver: objArgs.allowAppAuth,
       connectTokenResolver: objArgs.connectToken,
       useCliWithServiceAccountResolver: objArgs.useCliWithServiceAccount,
     };
   },
   async execute({
-    id, account, connectHost, allowMissingResolver, tokenResolver,
+    id, account, connectHost, allowMissingResolver, cacheTtlResolver, tokenResolver,
     allowAppAuthResolver, connectTokenResolver, useCliWithServiceAccountResolver,
   }) {
     // even if these are empty, we can't throw errors yet
@@ -813,6 +827,10 @@ plugin.registerRootDecorator({
       allowMissing as boolean | undefined,
       !!useCliWithServiceAccount,
     );
+    const cacheTtl = await resolveCacheTtl(cacheTtlResolver);
+    if (cacheTtl !== undefined) {
+      pluginInstances[id].cacheTtl = cacheTtl;
+    }
   },
 });
 
@@ -827,7 +845,6 @@ plugin.registerDataType({
       description: '1Password service accounts',
       url: 'https://developer.1password.com/docs/service-accounts/',
     },
-    'https://example.com',
   ],
   async validate(val) {
     if (!val.startsWith('ops_')) {
@@ -910,6 +927,27 @@ plugin.registerResolverFunction({
     }
     const allowMissing = allowMissingResolver ? !!(await allowMissingResolver.resolve()) : undefined;
     const shouldAllowMissing = allowMissing ?? selectedInstance.allowMissing;
+
+    // check cache if cacheTtl is configured and cache is available
+    if (selectedInstance.cacheTtl !== undefined && pluginCache) {
+      const cacheKey = `op:${instanceId}:${opReference}`;
+      const cached = await pluginCache.get(cacheKey);
+      if (cached !== undefined) {
+        debug('cache hit for %s', cacheKey);
+        return cached;
+      }
+      try {
+        const opValue = await selectedInstance.readItem(opReference);
+        await pluginCache.set(cacheKey, opValue, selectedInstance.cacheTtl);
+        return opValue;
+      } catch (err) {
+        if (shouldAllowMissing && isNotFoundError(err)) {
+          return undefined;
+        }
+        throw err;
+      }
+    }
+
     try {
       const opValue = await selectedInstance.readItem(opReference);
       return opValue;
@@ -978,6 +1016,20 @@ plugin.registerResolverFunction({
     if (typeof environmentId !== 'string') {
       throw new SchemaError('expected environment ID to resolve to a string');
     }
+
+    // check cache if cacheTtl is configured and cache is available
+    if (selectedInstance.cacheTtl !== undefined && pluginCache) {
+      const cacheKey = `opEnv:${instanceId}:${environmentId}`;
+      const cached = await pluginCache.get(cacheKey);
+      if (cached !== undefined) {
+        debug('cache hit for %s', cacheKey);
+        return cached;
+      }
+      const result = await selectedInstance.readEnvironment(environmentId);
+      await pluginCache.set(cacheKey, result, selectedInstance.cacheTtl);
+      return result;
+    }
+
     return await selectedInstance.readEnvironment(environmentId);
   },
 });

packages/plugins/aws-secrets/src/plugin.ts

@@ -1,4 +1,6 @@
-import { type Resolver, plugin } from 'varlock/plugin-lib';
+import {
+  type Resolver, type PluginCacheAccessor, plugin, resolveCacheTtl,
+} from 'varlock/plugin-lib';
 
 import {
   SecretsManagerClient,
@@ -23,6 +25,15 @@ const { debug } = plugin;
 debug('init - version =', plugin.version);
 plugin.icon = AWS_ICON;
 
+// capture cache accessor while the plugin proxy context is active
+// (the `plugin` proxy is only valid during module initialization, not during resolve())
+let pluginCache: PluginCacheAccessor | undefined;
+try {
+  pluginCache = plugin.cache;
+} catch {
+  // cache not available (e.g., no encryption key)
+}
+
 plugin.standardVars = {
   initDecorator: '@initAws',
   params: {
@@ -59,6 +70,8 @@ class AwsPluginInstance {
   private oidcSessionName?: string;
   private oidcToken?: string;
   private cachedOidcCredentials?: OidcCredentials;
+  /** optional cache TTL - when set, resolved values are cached */
+  cacheTtl?: string | number;
 
   constructor(
     readonly id: string,
@@ -500,6 +513,7 @@ plugin.registerRootDecorator({
       oidcRoleArnResolver: objArgs.oidcRoleArn,
       oidcSessionNameResolver: objArgs.oidcSessionName,
       oidcTokenResolver: objArgs.oidcToken,
+      cacheTtlResolver: objArgs.cacheTtl,
     };
   },
   async execute({
@@ -513,6 +527,7 @@ plugin.registerRootDecorator({
     oidcRoleArnResolver,
     oidcSessionNameResolver,
     oidcTokenResolver,
+    cacheTtlResolver,
   }) {
     const region = await regionResolver.resolve();
     const accessKeyId = await accessKeyIdResolver?.resolve();
@@ -534,6 +549,10 @@ plugin.registerRootDecorator({
       oidcSessionName,
       oidcToken,
     );
+    const cacheTtl = await resolveCacheTtl(cacheTtlResolver);
+    if (cacheTtl !== undefined) {
+      pluginInstances[id].cacheTtl = cacheTtl;
+    }
   },
 });
 
@@ -691,6 +710,19 @@ plugin.registerResolverFunction({
     // Apply namePrefix
     const finalSecretId = selectedInstance.applyNamePrefix(secretId);
 
+    // check cache if cacheTtl is configured and cache is available
+    if (selectedInstance.cacheTtl !== undefined && pluginCache) {
+      const cacheKey = `awsSecret:${instanceId}:${finalSecretId}`;
+      const cached = await pluginCache.get(cacheKey);
+      if (cached !== undefined) {
+        debug('cache hit for %s', cacheKey);
+        return cached;
+      }
+      const secretValue = await selectedInstance.getSecret(finalSecretId, jsonKey);
+      await pluginCache.set(cacheKey, secretValue, selectedInstance.cacheTtl);
+      return secretValue;
+    }
+
     const secretValue = await selectedInstance.getSecret(finalSecretId, jsonKey);
     return secretValue;
   },
@@ -810,6 +842,19 @@ plugin.registerResolverFunction({
     // Apply namePrefix
     const finalParameterName = selectedInstance.applyNamePrefix(parameterName);
 
+    // check cache if cacheTtl is configured and cache is available
+    if (selectedInstance.cacheTtl !== undefined && pluginCache) {
+      const cacheKey = `awsParam:${instanceId}:${finalParameterName}`;
+      const cached = await pluginCache.get(cacheKey);
+      if (cached !== undefined) {
+        debug('cache hit for %s', cacheKey);
+        return cached;
+      }
+      const parameterValue = await selectedInstance.getParameter(finalParameterName, jsonKey);
+      await pluginCache.set(cacheKey, parameterValue, selectedInstance.cacheTtl);
+      return parameterValue;
+    }
+
     const parameterValue = await selectedInstance.getParameter(finalParameterName, jsonKey);
     return parameterValue;
   },

packages/plugins/bitwarden/src/plugin.ts

@@ -1,4 +1,6 @@
-import { type Resolver, plugin } from 'varlock/plugin-lib';
+import {
+  type Resolver, type PluginCacheAccessor, plugin, resolveCacheTtl,
+} from 'varlock/plugin-lib';
 import ky from 'ky';
 import { Buffer } from 'node:buffer';
 import { webcrypto } from 'node:crypto';
@@ -13,6 +15,15 @@ const BITWARDEN_ICON = 'simple-icons:bitwarden';
 plugin.name = 'bitwarden';
 const { debug } = plugin;
 debug('init - version =', plugin.version);
+
+// capture cache accessor while the plugin proxy context is active
+// (the `plugin` proxy is only valid during module initialization, not during resolve())
+let pluginCache: PluginCacheAccessor | undefined;
+try {
+  pluginCache = plugin.cache;
+} catch {
+  // cache not available (e.g., no encryption key)
+}
 plugin.icon = BITWARDEN_ICON;
 plugin.standardVars = {
   initDecorator: '@initBitwarden',
@@ -57,6 +68,9 @@ class BitwardenPluginInstance {
   /** In-flight auth promise - prevents parallel resolution from triggering multiple auth requests (rate limit fix) */
   private authInFlight?: Promise<CachedAuth>;
 
+  /** optional cache TTL - when set, resolved values are cached */
+  cacheTtl?: string | number;
+
   constructor(
     readonly id: string,
   ) {}
@@ -336,13 +350,15 @@ plugin.registerRootDecorator({
       apiUrl,
       identityUrl,
       accessTokenResolver: objArgs.accessToken,
+      cacheTtlResolver: objArgs.cacheTtl,
     };
   },
   async execute({
     id,
     apiUrl,
     identityUrl,
     accessTokenResolver,
+    cacheTtlResolver,
   }) {
     // even if the token is empty, we can't throw errors yet
     // in case the instance is never actually used
@@ -353,6 +369,11 @@ plugin.registerRootDecorator({
       apiUrl,
       identityUrl,
     );
+
+    const cacheTtl = await resolveCacheTtl(cacheTtlResolver);
+    if (cacheTtl !== undefined) {
+      pluginInstances[id].cacheTtl = cacheTtl;
+    }
   },
 });
 
@@ -485,7 +506,19 @@ plugin.registerResolverFunction({
       });
     }
 
-    const secretValue = await selectedInstance.getSecret(secretId);
-    return secretValue;
+    // check cache if cacheTtl is configured and cache is available
+    if (selectedInstance.cacheTtl !== undefined && pluginCache) {
+      const cacheKey = `bw:${instanceId}:${secretId}`;
+      const cached = await pluginCache.get(cacheKey);
+      if (cached !== undefined) {
+        debug('cache hit for %s', cacheKey);
+        return cached;
+      }
+      const secretValue = await selectedInstance.getSecret(secretId);
+      await pluginCache.set(cacheKey, secretValue, selectedInstance.cacheTtl);
+      return secretValue;
+    }
+
+    return await selectedInstance.getSecret(secretId);
   },
 });

packages/plugins/google-secret-manager/src/plugin.ts

@@ -1,4 +1,6 @@
-import { type Resolver, plugin } from 'varlock/plugin-lib';
+import {
+  type Resolver, type PluginCacheAccessor, plugin, resolveCacheTtl,
+} from 'varlock/plugin-lib';
 
 import { GoogleAuth } from 'google-auth-library';
 import { getOidcToken } from '@env-spec/utils/oidc-tokens';
@@ -10,6 +12,14 @@ const GSM_ICON = 'devicon:googlecloud';
 plugin.name = 'gsm';
 const { debug } = plugin;
 debug('init - version =', plugin.version);
+// capture cache accessor while the plugin proxy context is active
+// (the `plugin` proxy is only valid during module initialization, not during resolve())
+let pluginCache: PluginCacheAccessor | undefined;
+try {
+  pluginCache = plugin.cache;
+} catch {
+  // cache not available (e.g., no encryption key)
+}
 plugin.icon = GSM_ICON;
 plugin.standardVars = {
   initDecorator: '@initGsm',
@@ -25,6 +35,8 @@ class GsmPluginInstance {
   private workloadIdentityProvider?: string;
   private serviceAccountEmail?: string;
   private oidcToken?: string;
+  /** optional cache TTL - when set, resolved values are cached */
+  cacheTtl?: string | number;
 
   constructor(
     readonly id: string,
@@ -258,6 +270,7 @@ plugin.registerRootDecorator({
 
     return {
       id,
+      cacheTtlResolver: objArgs.cacheTtl,
       projectIdResolver: objArgs.projectId,
       credentialsResolver: objArgs.credentials,
       workloadIdentityProviderResolver: objArgs.workloadIdentityProvider,
@@ -266,7 +279,7 @@ plugin.registerRootDecorator({
     };
   },
   async execute({
-    id, projectIdResolver, credentialsResolver,
+    id, cacheTtlResolver, projectIdResolver, credentialsResolver,
     workloadIdentityProviderResolver, serviceAccountEmailResolver, oidcTokenResolver,
   }) {
     const projectId = await projectIdResolver?.resolve();
@@ -275,6 +288,10 @@ plugin.registerRootDecorator({
     const serviceAccountEmail = await serviceAccountEmailResolver?.resolve();
     const oidcToken = await oidcTokenResolver?.resolve();
     pluginInstances[id].setAuth(projectId, credentials, workloadIdentityProvider, serviceAccountEmail, oidcToken);
+    const cacheTtl = await resolveCacheTtl(cacheTtlResolver);
+    if (cacheTtl !== undefined) {
+      pluginInstances[id].cacheTtl = cacheTtl;
+    }
   },
 });
 
@@ -411,7 +428,19 @@ plugin.registerResolverFunction({
       throw new SchemaError('No secret reference provided');
     }
 
-    const secretValue = await selectedInstance.readSecret(secretRef);
-    return secretValue;
+    // check cache if cacheTtl is configured and cache is available
+    if (selectedInstance.cacheTtl !== undefined && pluginCache) {
+      const cacheKey = `gsm:${instanceId}:${secretRef}`;
+      const cached = await pluginCache.get(cacheKey);
+      if (cached !== undefined) {
+        debug('cache hit for %s', cacheKey);
+        return cached;
+      }
+      const secretValue = await selectedInstance.readSecret(secretRef);
+      await pluginCache.set(cacheKey, secretValue, selectedInstance.cacheTtl);
+      return secretValue;
+    }
+
+    return await selectedInstance.readSecret(secretRef);
   },
 });