diff --git a/src/lifecycle/disposable/ResourceLifecycleRegistry.ts b/src/lifecycle/disposable/ResourceLifecycleRegistry.ts new file mode 100644 index 000000000..ba152b3c6 --- /dev/null +++ b/src/lifecycle/disposable/ResourceLifecycleRegistry.ts @@ -0,0 +1,113 @@ +import { AsyncSeriesWaterfallHook } from 'tapable'; + +export interface Disposable { + dispose(): Promise; + disposed: boolean; +} + +export interface RegisteredResource { + id: string; + /** Higher priority = disposed first (browser contexts before browsers, etc.) */ + priority: number; + resource: Disposable; + registeredAt: number; +} + +export class ResourceLifecycleRegistry { + private readonly resources = new Map(); + private disposing = false; + private disposedAt?: number; + + /** True once disposeAll() has been called (even if some resources failed to dispose) */ + get disposed(): boolean { + return this.disposing || this.disposedAt !== undefined; + } + + /** + * Register a resource for lifecycle-managed disposal. + * If a resource with the same id is already registered, the old one is disposed + * (fire-and-forget) and replaced. + */ + register(id: string, resource: Disposable, priority = 0): void { + if (this.disposed) { + throw new Error(`[lifecycle] Cannot register ${id}: registry is already disposed`); + } + const existing = this.resources.get(id); + if (existing) { + console.warn(`[lifecycle] Duplicate registration of "${id}", disposing old resource first`); + existing.resource.dispose().catch(() => {}); + } + this.resources.set(id, { id, priority, resource, registeredAt: Date.now() }); + console.debug(`[lifecycle] Registered "${id}" (priority=${priority}, total=${this.resources.size})`); + } + + /** Unregister a resource without disposing it (e.g. ownership transferred elsewhere). */ + unregister(id: string): void { + this.resources.delete(id); + console.debug(`[lifecycle] Unregistered "${id}" (remaining=${this.resources.size})`); + } + + get(id: string): Disposable | undefined { + return this.resources.get(id)?.resource; + } + + /** IDs of resources that are registered but not yet disposed — for smoke tests. */ + getOpenHandles(): string[] { + return [...this.resources.values()] + .filter((r) => !r.resource.disposed) + .map((r) => r.id); + } + + /** + * Dispose all registered resources in priority order. + * Each individual dispose() has a 3-second timeout. + */ + async disposeAll(timeoutMs = 10_000): Promise<'ok' | 'timeout'> { + if (this.disposed) return 'ok'; + this.disposing = true; + + const sorted = [...this.resources.values()].sort((a, b) => b.priority - a.priority); + const start = Date.now(); + + await Promise.allSettled( + sorted.map(async (r) => { + if (r.resource.disposed) return; + try { + console.debug(`[lifecycle] Disposing "${r.id}" …`); + await Promise.race([ + r.resource.dispose(), + new Promise<'timeout'>((resolve) => setTimeout(() => resolve('timeout'), 3_000)), + ]); + } catch (err) { + console.error(`[lifecycle] Error disposing "${r.id}":`, err); + } + }), + ); + + this.disposedAt = Date.now(); + const elapsed = Date.now() - start; + console.info(`[lifecycle] disposeAll completed in ${elapsed}ms (handles=${sorted.length})`); + return elapsed > timeoutMs ? 'timeout' : 'ok'; + } + + /** + * Dispose all resources and wait up to `timeoutMs` for completion. + * Logs a warning with the remaining open handles if the timeout is hit. + */ + async disposeAllAndWait(timeoutMs = 30_000): Promise<'ok' | 'timeout'> { + const result = await this.disposeAll(timeoutMs); + if (result === 'timeout') { + const handles = this.getOpenHandles(); + console.error(`[lifecycle] disposeAllAndWait TIMEOUT — open handles: ${handles.join(', ')}`); + } + return result; + } + + /** Number of currently registered resources. */ + get size(): number { + return this.resources.size; + } +} + +/** Singleton registry for agent-scoped resources. */ +export const agentRegistry = new ResourceLifecycleRegistry(); \ No newline at end of file diff --git a/src/lifecycle/disposable/index.ts b/src/lifecycle/disposable/index.ts new file mode 100644 index 000000000..ee3a960e4 --- /dev/null +++ b/src/lifecycle/disposable/index.ts @@ -0,0 +1,2 @@ +export { ResourceLifecycleRegistry, type Disposable, type RegisteredResource } from './ResourceLifecycleRegistry'; +export { agentRegistry } from './ResourceLifecycleRegistry'; \ No newline at end of file diff --git a/src/lifecycle/retention/RetentionPolicy.ts b/src/lifecycle/retention/RetentionPolicy.ts new file mode 100644 index 000000000..b0f380636 --- /dev/null +++ b/src/lifecycle/retention/RetentionPolicy.ts @@ -0,0 +1,172 @@ +/** + * RetentionPolicy — cleanup policy for runtime-generated directories. + * + * Applies TTL and count-based retention to: + * - .pilotdeck-always-on/ (TTL: 7 days, max 50 dirs per project) + * - .agi-output/ (TTL: 30 days) + * - .cci-output/ (TTL: 30 days) + * - TaskOutputStore entries (TTL: 30 days) + */ + +export interface RetentionPolicyOptions { + /** Max age in ms before a directory is considered for deletion. Default: 7 days. */ + alwaysOnTtlMs?: number; + /** Max age in ms before output dirs are cleaned. Default: 30 days. */ + outputTtlMs?: number; + /** Max directories per project for always-on. Default: 50. */ + alwaysOnMaxCount?: number; +} + +interface RetentionEntry { + path: string; + mtimeMs: number; + sizeBytes: number; +} + +const DEFAULT_ALWAYS_ON_TTL_MS = 7 * 24 * 60 * 60 * 1000; // 7 days +const DEFAULT_OUTPUT_TTL_MS = 30 * 24 * 60 * 60 * 1000; // 30 days +const DEFAULT_ALWAYS_ON_MAX_COUNT = 50; + +export class RetentionPolicy { + private readonly alwaysOnTtlMs: number; + private readonly outputTtlMs: number; + private readonly alwaysOnMaxCount: number; + + constructor(options: RetentionPolicyOptions = {}) { + this.alwaysOnTtlMs = options.alwaysOnTtlMs ?? DEFAULT_ALWAYS_ON_TTL_MS; + this.outputTtlMs = options.outputTtlMs ?? DEFAULT_OUTPUT_TTL_MS; + this.alwaysOnMaxCount = options.alwaysOnMaxCount ?? DEFAULT_ALWAYS_ON_MAX_COUNT; + } + + /** + * Scan `rootDir` for retention-target directories and return those that are candidates + * for cleanup (expired TTL or over count limit). + */ + async scanRetentionCandidates(rootDir: string): Promise { + const { promises: fs } = await import('fs'); + const { join } = await import('path'); + const now = Date.now(); + + const candidates: RetentionEntry[] = []; + + // .pilotdeck-always-on/ + try { + const alwaysOnDir = join(rootDir, '.pilotdeck-always-on'); + const stat = await fs.stat(alwaysOnDir); + if (stat.isDirectory()) { + const entries = await fs.readdir(alwaysOnDir); + for (const entry of entries) { + const entryPath = join(alwaysOnDir, entry); + try { + const st = await fs.stat(entryPath); + candidates.push({ + path: entryPath, + mtimeMs: st.mtimeMs, + sizeBytes: 0, // skip size for dirs + }); + } catch { /* skip */ } + } + } + } catch { /* dir doesn't exist */ } + + // .agi-output/, .cci-output/ + for (const subdir of ['.agi-output', '.cci-output']) { + try { + const dir = join(rootDir, subdir); + const stat = await fs.stat(dir); + if (stat.isDirectory()) { + const entries = await fs.readdir(dir); + for (const entry of entries) { + const entryPath = join(dir, entry); + try { + const st = await fs.stat(entryPath); + candidates.push({ + path: entryPath, + mtimeMs: st.mtimeMs, + sizeBytes: 0, + }); + } catch { /* skip */ } + } + } + } catch { /* dir doesn't exist */ } + } + + return candidates; + } + + /** + * Given candidates from scanRetentionCandidates(), return the subset that should + * actually be deleted (TTL expired OR always-on over count limit). + */ + computeDeletionSet( + candidates: RetentionEntry[], + options: { alwaysOnCount?: number } = {}, + ): RetentionEntry[] { + const now = Date.now(); + const toDelete: RetentionEntry[] = []; + + for (const c of candidates) { + const isAlwaysOn = c.path.includes('.pilotdeck-always-on'); + const isOutput = c.path.includes('.agi-output') || c.path.includes('.cci-output'); + const age = now - c.mtimeMs; + + if (isAlwaysOn) { + const overCount = (options.alwaysOnCount ?? this.alwaysOnMaxCount) < candidates.filter( + (x) => x.path.includes('.pilotdeck-always-on'), + ).length; + const expired = age > this.alwaysOnTtlMs; + if (expired || overCount) toDelete.push(c); + } else if (isOutput) { + if (age > this.outputTtlMs) toDelete.push(c); + } + } + + return toDelete; + } + + /** + * Delete a set of entries returned by computeDeletionSet(). + * Removes directories recursively. + */ + async applyDeletions(entries: RetentionEntry[]): Promise<{ deleted: number; freedBytes: number }> { + const { promises: fs } = await import('fs'); + const { join } = await import('path'); + + let deleted = 0; + let freedBytes = 0; + + for (const entry of entries) { + try { + // Estimate freed bytes by reading dir size + let size = 0; + try { + const { stdout } = await import('child_process').exec( + `du -sb "${entry.path}" 2>/dev/null | cut -f1`, + { timeout: 5000 }, + ); + size = parseInt(stdout.trim(), 10) || 0; + } catch { /* ignore */ } + + await fs.rm(entry.path, { recursive: true, force: true }); + deleted++; + freedBytes += size; + console.debug(`[retention] Deleted: ${entry.path} (~${size} bytes)`); + } catch (err) { + console.warn(`[retention] Failed to delete ${entry.path}:`, err); + } + } + + return { deleted, freedBytes }; + } + + /** Convenience: run full scan → compute → delete cycle on `rootDir`. */ + async runRetentionCleanup(rootDir: string): Promise<{ deleted: number; freedBytes: number }> { + const candidates = await this.scanRetentionCandidates(rootDir); + const toDelete = this.computeDeletionSet(candidates); + const result = await this.applyDeletions(toDelete); + console.info( + `[retention] Cleanup complete on ${rootDir}: deleted=${result.deleted}, freed_bytes=${result.freedBytes}`, + ); + return result; + } +} \ No newline at end of file diff --git a/src/lifecycle/retention/index.ts b/src/lifecycle/retention/index.ts new file mode 100644 index 000000000..761d5f5aa --- /dev/null +++ b/src/lifecycle/retention/index.ts @@ -0,0 +1 @@ +export { RetentionPolicy } from './RetentionPolicy'; \ No newline at end of file diff --git a/src/memory/health/CircuitBreakerHealthCheck.ts b/src/memory/health/CircuitBreakerHealthCheck.ts new file mode 100644 index 000000000..1dd729a0c --- /dev/null +++ b/src/memory/health/CircuitBreakerHealthCheck.ts @@ -0,0 +1,167 @@ +/** + * MemoryCircuitBreaker — circuit breaker for APEX-MEM / memory heartbeat services. + * + * After `failureThreshold` failures within `windowMs`, the breaker opens. + * In "open" state, memory operations degrade gracefully instead of hammering a dead service. + * + * Health check performs read-only verification: + * - SQLite integrity_check + * - memories count vs FTS count alignment + * - BM25 max_doc vs actual doc count + */ + +export type MemoryHealthStatus = 'ok' | 'degraded' | 'down'; + +export interface MemoryHealthCheckOptions { + /** Number of consecutive failures before the breaker opens. Default: 3. */ + failureThreshold?: number; + /** Time window in ms to count failures. Default: 30_000 (30 seconds). */ + windowMs?: number; + /** Cooldown period after breaker opens. Default: 300_000 (5 minutes). */ + cooldownMs?: number; +} + +interface CircuitState { + failures: number[]; + firstFailureAt: number | null; + lastFailureAt: number | null; +} + +const DEFAULT_FAILURE_THRESHOLD = 3; +const DEFAULT_WINDOW_MS = 30_000; +const DEFAULT_COOLDOWN_MS = 300_000; + +export class MemoryCircuitBreaker { + private readonly failureThreshold: number; + private readonly windowMs: number; + private readonly cooldownMs: number; + + private state: 'closed' | 'open' | 'half-open' = 'closed'; + private openedAt: number | null = null; + private failures: number[] = []; // timestamps of failures within window + private lastCheckAt: number | null = null; + private lastHealthStatus: MemoryHealthStatus = 'ok'; + + constructor(options: MemoryHealthCheckOptions = {}) { + this.failureThreshold = options.failureThreshold ?? DEFAULT_FAILURE_THRESHOLD; + this.windowMs = options.windowMs ?? DEFAULT_WINDOW_MS; + this.cooldownMs = options.cooldownMs ?? DEFAULT_COOLDOWN_MS; + } + + /** Current circuit state. */ + get circuitState(): 'closed' | 'open' | 'half-open' { + if (this.state === 'open') { + const elapsed = Date.now() - (this.openedAt ?? 0); + if (elapsed >= this.cooldownMs) { + this.state = 'half-open'; + } + } + return this.state; + } + + /** Last recorded health status. */ + get healthStatus(): MemoryHealthStatus { + return this.lastHealthStatus; + } + + /** Record a failed memory operation (network error, 503, etc.). */ + recordFailure(): void { + const now = Date.now(); + this.failures.push(now); + // Prune failures outside the window + const cutoff = now - this.windowMs; + this.failures = this.failures.filter((ts) => ts > cutoff); + + if (this.failures.length >= this.failureThreshold && this.state === 'closed') { + this.state = 'open'; + this.openedAt = now; + console.warn( + `[memory-circuit-breaker] Circuit OPEN. Failures=${this.failures.length} within ${this.windowMs}ms. ` + + `Will retry after ${this.cooldownMs / 1000}s.`, + ); + } + } + + /** Record a successful memory operation — resets the breaker to closed. */ + recordSuccess(): void { + if (this.state !== 'half-open') return; + this.state = 'closed'; + this.failures = []; + this.openedAt = null; + console.info('[memory-circuit-breaker] Circuit CLOSED (half-open probe succeeded)'); + } + + /** + * Returns true if memory operations should be rejected/degraded. + * When open, callers should return cached/degraded results instead of hitting the service. + */ + isOpen(): boolean { + return this.circuitState === 'open'; + } + + /** + * Returns true if the next probe should be allowed through (half-open state). + * Callers use this to decide whether to attempt a health check ping. + */ + isHalfOpen(): boolean { + return this.circuitState === 'half-open'; + } + + /** Last health check timestamp. */ + get lastCheckAt(): number | null { + return this.lastCheckAt; + } + + /** + * Perform a read-only health check. + * Implementors can call this from their health-check endpoint. + * + * Returns 'ok' if all checks pass, 'degraded' if only some pass, 'down' if all fail. + */ + async runHealthCheck(ctx: { + sqliteIntegrity?: () => Promise; + memoriesCount?: () => Promise; + ftsCount?: () => Promise; + bm25MaxDoc?: () => Promise; + }): Promise { + this.lastCheckAt = Date.now(); + const results: boolean[] = []; + + try { + if (ctx.sqliteIntegrity) results.push(await ctx.sqliteIntegrity()); + } catch { results.push(false); } + + try { + if (ctx.memoriesCount && ctx.ftsCount) { + const [mc, fc] = await Promise.all([ctx.memoriesCount(), ctx.ftsCount()]); + results.push(mc === fc); // counts must align + } + } catch { results.push(false); } + + try { + if (ctx.bm25MaxDoc && ctx.memoriesCount) { + const [bm, mc] = await Promise.all([ctx.bm25MaxDoc(), ctx.memoriesCount()]); + results.push(bm === mc); // BM25 index must match memories count + } + } catch { results.push(false); } + + const passCount = results.filter(Boolean).length; + const totalCount = results.length; + + if (passCount === totalCount) { + this.lastHealthStatus = 'ok'; + this.recordSuccess(); + } else if (passCount > 0) { + this.lastHealthStatus = 'degraded'; + // Don't record as success in degraded state + } else { + this.lastHealthStatus = 'down'; + this.recordFailure(); + } + + console.debug( + `[memory-circuit-breaker] Health check: ${passCount}/${totalCount} checks passed, status=${this.lastHealthStatus}`, + ); + return this.lastHealthStatus; + } +} \ No newline at end of file diff --git a/src/memory/health/index.ts b/src/memory/health/index.ts new file mode 100644 index 000000000..cc4a0936c --- /dev/null +++ b/src/memory/health/index.ts @@ -0,0 +1 @@ +export { MemoryCircuitBreaker, type MemoryHealthStatus, type MemoryHealthCheckOptions } from './CircuitBreakerHealthCheck'; \ No newline at end of file diff --git a/src/tool/protocol/circuitBreaker.ts b/src/tool/protocol/circuitBreaker.ts new file mode 100644 index 000000000..e91de4573 --- /dev/null +++ b/src/tool/protocol/circuitBreaker.ts @@ -0,0 +1,115 @@ +/** + * ToolLoopCircuitBreaker — prevents the tool loop from spinning on consecutively + * broken (toolName + validationError) pairs. + * + * After `threshold` consecutive failures of the same (toolName + validationError) + * pattern, the breaker opens and the call is rejected with CircuitOpenError + * until the backoff window expires. + * + * Backoff: 60s → 120s → 240s → 480s (capped), resets on successful call. + */ + +export interface ToolCallCircuitBreakerOptions { + threshold?: number; + initialBackoffMs?: number; + maxBackoffMultiplier?: number; +} + +interface CircuitState { + consecutiveFailures: number; + firstFailureAt: number | null; + currentBackoffMs: number; +} + +const DEFAULT_THRESHOLD = 3; +const DEFAULT_INITIAL_BACKOFF_MS = 60_000; +const DEFAULT_MAX_BACKOFF_MULTIPLIER = 8; + +export class ToolCallCircuitBreaker { + private readonly threshold: number; + private readonly initialBackoffMs: number; + private readonly maxBackoffMultiplier: number; + private readonly circuits = new Map(); + + constructor(options: ToolCallCircuitBreakerOptions = {}) { + this.threshold = options.threshold ?? DEFAULT_THRESHOLD; + this.initialBackoffMs = options.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS; + this.maxBackoffMultiplier = options.maxBackoffMultiplier ?? DEFAULT_MAX_BACKOFF_MULTIPLIER; + } + + private key(toolName: string, validationError: string): string { + return `${toolName}\x00${validationError}`; + } + + private getOrCreate(key: string): CircuitState { + if (!this.circuits.has(key)) { + this.circuits.set(key, { + consecutiveFailures: 0, + firstFailureAt: null, + currentBackoffMs: this.initialBackoffMs, + }); + } + return this.circuits.get(key)!; + } + + private isOpen(state: CircuitState): boolean { + if (state.consecutiveFailures < this.threshold) return false; + if (state.firstFailureAt === null) return false; + return Date.now() - state.firstFailureAt < state.currentBackoffMs; + } + + recordFailure(toolName: string, validationError: string): void { + const state = this.getOrCreate(this.key(toolName, validationError)); + state.consecutiveFailures++; + if (state.firstFailureAt === null) state.firstFailureAt = Date.now(); + if (state.consecutiveFailures === this.threshold) { + state.currentBackoffMs = Math.min( + state.currentBackoffMs * 2, + this.initialBackoffMs * this.maxBackoffMultiplier, + ); + console.warn( + `[circuit-breaker] Tool circuit OPEN for "${toolName}" (error=${JSON.stringify(validationError)}). ` + + `Backoff=${state.currentBackoffMs / 1000}s.`, + ); + } + } + + isOpen(toolName: string, validationError: string): boolean { + return this.isOpen(this.getOrCreate(this.key(toolName, validationError))); + } + + reset(toolName: string, validationError: string): void { + const state = this.circuits.get(this.key(toolName, validationError)); + if (state) { + state.consecutiveFailures = 0; + state.firstFailureAt = null; + state.currentBackoffMs = this.initialBackoffMs; + } + } + + resetAll(): void { + this.circuits.clear(); + } + + get openCircuits(): number { + let count = 0; + for (const s of this.circuits.values()) { + if (this.isOpen(s)) count++; + } + return count; + } +} + +export class CircuitOpenError extends Error { + constructor( + public readonly toolName: string, + public readonly validationError: string, + public readonly retryAfterMs: number, + ) { + super( + `Tool circuit breaker is open for "${toolName}" (validationError=${JSON.stringify(validationError)}). ` + + `Retry allowed after ${retryAfterMs / 1000}s.`, + ); + this.name = 'CircuitOpenError'; + } +} \ No newline at end of file