diff --git a/docs/sbx-integration.md b/docs/sbx-integration.md index 138cf4355..7969ee52f 100644 --- a/docs/sbx-integration.md +++ b/docs/sbx-integration.md @@ -383,13 +383,15 @@ myvm: { That entry makes `runtimeUsesComposeAgent('myvm')` return `false`, which omits its agent from `docker-compose.yml` and skips the Docker network-isolation -override in strict mode. It does **not** select the new manager: the current -main workflow treats every microVM entry as sbx until runtime-specific dispatch is added. +override in strict mode. It does **not** select the new manager: register an +`ExternalRuntimeBackendFactory` in `src/external-runtime-backend-resolver.ts`. -### 2. Implement a manager (mirror `sbx-manager.ts`) +### 2. Implement an external runtime backend -Provide `createSandbox` / `execInSandbox` / `removeSandbox` / `isAvailable` -equivalents for your VMM. Concretely, a KVM backend must: +Implement `ExternalAgentRuntimeBackend` from +`src/external-runtime-backend.ts`, following `SbxRuntimeBackend` in +`src/sbx-runtime-backend.ts`. The backend owns preflight, startup, execution, +diagnostics, and idempotent stop state. Concretely, a KVM backend must: - **Boot a microVM on `/dev/kvm`** with a kernel + rootfs. Confirm KVM is available (`/dev/kvm` present, user in the `kvm` group). On stock @@ -417,16 +419,16 @@ sandbox egress through AWF's host-side Squid: - Reproduce the boundary-crossing addressing that the sbx path uses: Squid at the **bridge gateway IP + published port** (not the internal `172.30.0.x`), and the api-proxy via a host-reachable name (`host.docker.internal`). See the - `SBX_GATEWAY_IP` / `SBX_HOST_DOCKER_INTERNAL` handling in `main-action.ts`. + `SBX_GATEWAY_IP` / `SBX_HOST_DOCKER_INTERNAL` handling in + `src/sbx-runtime-backend.ts`. -### 4. Wire it into `main-action.ts` +### 4. Register the backend -Introduce runtime-specific manager dispatch keyed by `config.containerRuntime`; -do not gate all microVM backends through the current sbx-specific branch. The -selected manager must provide start/run/cleanup wrappers that (a) start -infra-only compose, (b) build the agent environment with its network targets, -(c) create the VM, (d) check api-proxy and Squid across the boundary, and -(e) execute and tear down the agent with that backend's lifecycle commands. +Add the factory to `EXTERNAL_RUNTIME_BACKENDS` in +`src/external-runtime-backend-resolver.ts`. `main-action.ts` resolves exactly one +backend instance, adapts it to `WorkflowDependencies`, and uses that same +instance for cleanup and signal handling. Compose-managed Docker and gVisor +runtimes bypass this adapter. ### 5. Things to get right (lessons from the sbx path) diff --git a/src/cli-workflow.ts b/src/cli-workflow.ts index 977544b85..784da7bb5 100644 --- a/src/cli-workflow.ts +++ b/src/cli-workflow.ts @@ -10,16 +10,11 @@ import { validateEnclavesConfig } from './enclave/preflight'; /** * Dependencies injected into the main workflow. * - * These are implemented by `docker-manager.ts` for the Docker Compose backend. - * A future microVM backend (e.g. Docker sbx) would provide alternative - * implementations that: - * - `writeConfigs` — generate compose for infrastructure only (no agent service) - * - `startContainers` — start Squid + api-proxy via compose, then launch agent - * in a microVM with the sbx proxy chaining through host-side Squid/api-proxy - * - `runAgentCommand` — `sbx run` instead of `docker logs -f` + `docker wait` - * - Cleanup — `sbx rm` + `docker compose down` for infrastructure + * These are implemented by `docker-manager.ts` for Docker Compose agents. + * External agent backends adapt their lifecycle to `startContainers` and + * `runAgentCommand` while continuing to use compose for infrastructure. */ -interface WorkflowDependencies { +export interface WorkflowDependencies { ensureFirewallNetwork: () => Promise<{ squidIp: string; agentIp: string; proxyIp: string; subnet: string }>; setupHostIptables: (squidIp: string, port: number, dnsServers: string[], apiProxyIp?: string, dohProxyIp?: string, hostAccess?: HostAccessConfig, cliProxyConfig?: CliProxyHostConfig) => Promise; writeConfigs: (config: WrapperConfig) => Promise; diff --git a/src/commands/main-action-coverage-gaps.test.ts b/src/commands/main-action-coverage-gaps.test.ts index 049c7a56a..27b644c56 100644 --- a/src/commands/main-action-coverage-gaps.test.ts +++ b/src/commands/main-action-coverage-gaps.test.ts @@ -204,6 +204,28 @@ describe('createMainAction coverage gaps', () => { }); }); + describe('sbx signal handling', () => { + it('stops the selected external backend instead of the compose agent', async () => { + const sbxConfig = { + ...MAIN_ACTION_STUB_CONFIG, + containerRuntime: 'sbx', + keepContainers: false, + } as unknown as import('../types').WrapperConfig; + mockedValidateOptions.validateOptions.mockReturnValue(sbxConfig); + let signalOptions: Parameters[0] | undefined; + mockedSignalHandler.registerSignalHandlers.mockImplementation((options) => { + signalOptions = options; + }); + + const action = createMainAction(getOptionValueSource); + await action(['echo hi'], {}); + await signalOptions!.fastKillAgentContainer(); + + expect(mockedSbxManager.removeSandbox).toHaveBeenCalled(); + expect(mockedDockerManager.fastKillAgentContainer).not.toHaveBeenCalled(); + }); + }); + describe('sbx cleanup: keepContainers=true skips removeSandbox', () => { it('does not call removeSandbox when keepContainers is true', async () => { const sbxConfig = { diff --git a/src/commands/main-action.test.ts b/src/commands/main-action.test.ts index ac5d890e1..84613e183 100644 --- a/src/commands/main-action.test.ts +++ b/src/commands/main-action.test.ts @@ -19,6 +19,13 @@ jest.mock('./signal-handler'); jest.mock('./validate-options'); jest.mock('../sbx-manager'); jest.mock('../enclave/gateway'); +jest.mock('../external-runtime-backend-resolver', () => { + const actual = jest.requireActual('../external-runtime-backend-resolver'); + return { + ...actual, + resolveExternalRuntimeBackend: jest.fn(actual.resolveExternalRuntimeBackend), + }; +}); import { logger } from '../logger'; import * as dockerManager from '../docker-manager'; @@ -33,6 +40,7 @@ import * as signalHandler from './signal-handler'; import * as validateOptions from './validate-options'; import * as sbxManager from '../sbx-manager'; import * as enclaveGateway from '../enclave/gateway'; +import * as externalRuntimeResolver from '../external-runtime-backend-resolver'; import { MAIN_ACTION_STUB_CONFIG, setupMainActionTestHarness } from './main-action.test-utils'; const { @@ -56,6 +64,7 @@ const mockedSignalHandler = signalHandler as jest.Mocked; const mockedValidateOptions = validateOptions as jest.Mocked; const mockedSbxManager = sbxManager as jest.Mocked; const mockedEnclaveGateway = enclaveGateway as jest.Mocked; +const mockedExternalRuntimeResolver = externalRuntimeResolver as jest.Mocked; describe('createMainAction', () => { let processExitSpy: jest.SpyInstance; @@ -387,6 +396,25 @@ describe('createMainAction', () => { expect(mockedHostIptables.cleanupHostIptables).not.toHaveBeenCalled(); expect(processExitSpy).toHaveBeenCalledWith(1); }); + + describe('when external runtime resolution fails', () => { + it('uses fatal-error cleanup and exits with code 1', async () => { + mockedExternalRuntimeResolver.resolveExternalRuntimeBackend.mockImplementationOnce(() => { + throw new Error('backend is not registered'); + }); + + const action = createMainAction(getOptionValueSource); + await expect(action(['echo hi'], {})).rejects.toThrow('process.exit: 1'); + + expect(mockedLogger.error).toHaveBeenCalledWith( + 'Fatal error:', + expect.objectContaining({ message: 'backend is not registered' }), + ); + expect(mockedDockerManager.cleanup).toHaveBeenCalled(); + expect(mockedCliWorkflow.runMainWorkflow).not.toHaveBeenCalled(); + expect(processExitSpy).toHaveBeenCalledWith(1); + }); + }); }); describe('performCleanup with keepContainers=true', () => { diff --git a/src/commands/main-action.ts b/src/commands/main-action.ts index 985ffb54b..29eeee0ba 100644 --- a/src/commands/main-action.ts +++ b/src/commands/main-action.ts @@ -26,15 +26,9 @@ import { validateOptions } from './validate-options'; import { probeSplitFilesystem } from '../dind-probe'; import { assertTopologySupported, connectTopologyContainers } from '../topology'; import { runDindBootstrap } from '../dind-bootstrap'; -import { runtimeUsesComposeAgent } from '../container-runtime'; -import { - assertSbxApiProxyReflect, - createSandbox, - execInSandbox, - removeSandbox, - isSbxAvailable, - SBX_DEFAULT_NAME, -} from '../sbx-manager'; +import { adaptExternalRuntimeBackend } from '../external-runtime-backend'; +import type { ExternalAgentRuntimeBackend } from '../external-runtime-backend'; +import { resolveExternalRuntimeBackend } from '../external-runtime-backend-resolver'; import { prepareEnclaves, teardownEnclaves } from '../enclave/manager'; import { assertEnclaveGatewayReady, @@ -42,16 +36,6 @@ import { shutdownEnclaveGateway, } from '../enclave/gateway'; import type { WrapperConfig } from '../types'; -import { buildAgentEnvironment } from '../services/agent-service'; -import { buildAgentCredentialEnv } from '../services/api-proxy-credential-env'; -import { DEFAULT_DNS_SERVERS } from '../dns-resolver'; -import { AGENT_IP, CLI_PROXY_IP, DOH_PROXY_IP, NETWORK_SUBNET, SQUID_IP } from '../host-iptables-shared'; - -/** Report whether a secret is set (and its length) without exposing the value. */ -function redactSecret(value: string | undefined): string { - if (!value) return '(unset)'; - return `(set, len=${value.length})`; -} const SENSITIVE_CONFIG_KEYS = new Set([ 'openaiApiKey', @@ -112,19 +96,15 @@ function buildCleanupFn( config: WrapperConfig, getContainersStarted: () => boolean, getHostIptablesSetup: () => boolean, + externalRuntimeBackend?: ExternalAgentRuntimeBackend, ) { return async (signal?: string) => { if (signal) { logger.info(`Received ${signal}, cleaning up...`); } - // Clean up sbx sandbox if using microVM runtime - if (!runtimeUsesComposeAgent(config.containerRuntime) && !config.keepContainers) { - try { - await removeSandbox(SBX_DEFAULT_NAME); - } catch { - // Sandbox may not exist yet — that's fine - } + if (externalRuntimeBackend && !config.keepContainers) { + await externalRuntimeBackend.stop(); } // Let the enclave server emit final cleanup telemetry before preserving @@ -299,164 +279,45 @@ export function createMainAction(getOptionValueSource: OptionSourceResolver) { let exitCode = 0; let containersStarted = false; let hostIptablesSetup = false; + let externalRuntimeBackend: ExternalAgentRuntimeBackend | undefined; + try { + externalRuntimeBackend = resolveExternalRuntimeBackend(config, startContainers); + } catch (error) { + logger.error('Fatal error:', error); + await buildCleanupFn( + config, + () => containersStarted, + () => hostIptablesSetup, + )(); + console.error('Process exiting with code: 1'); + process.exit(1); + return; + } const performCleanup = buildCleanupFn( config, () => containersStarted, () => hostIptablesSetup, + externalRuntimeBackend, ); // Register signal handlers for graceful shutdown registerSignalHandlers({ getContainersStarted: () => containersStarted, keepContainers: config.keepContainers, - fastKillAgentContainer, + fastKillAgentContainer: externalRuntimeBackend + ? () => externalRuntimeBackend.stop() + : fastKillAgentContainer, performCleanup, }); try { - // For sbx (microVM) runtime, wrap startContainers and runAgentCommand - // to launch the agent in a sandbox instead of Docker Compose. - const useSbx = !runtimeUsesComposeAgent(config.containerRuntime); - let sbxName: string | undefined; - let sbxEnvironment: Record | undefined; - - const sbxStartContainers = useSbx - ? async ( - workDir: string, - allowedDomains: string[], - proxyLogsDir?: string, - skipPull?: boolean, - onNetworkReady?: () => Promise, - onInfrastructureReady?: () => Promise, - ) => { - // Start infra-only compose (squid, api-proxy — no agent service) - await startContainers( - workDir, - allowedDomains, - proxyLogsDir, - skipPull, - onNetworkReady, - onInfrastructureReady, - ); - - // Verify sbx is available - if (!await isSbxAvailable()) { - throw new Error('Docker sbx CLI not found. Install sbx to use --container-runtime sbx.'); - } - - // For sbx, the microVM can't reach Docker internal IPs (172.30.0.x). - // Published Squid port (3128) is accessible via the sbx gateway IP. - // The api-proxy is on the awf-ext bridge network and reachable from - // inside the sbx via `host.docker.internal` (resolves to the docker0 - // bridge IP, typically 172.17.0.1). - const SBX_GATEWAY_IP = '172.17.0.0'; - const SBX_HOST_DOCKER_INTERNAL = 'host.docker.internal'; - const sbxMounts = [...(config.volumeMounts ?? [])]; - - sbxEnvironment = buildAgentEnvironment({ - config, - networkConfig: { - subnet: NETWORK_SUBNET, - squidIp: SBX_GATEWAY_IP, - agentIp: AGENT_IP, - proxyIp: config.enableApiProxy ? SBX_HOST_DOCKER_INTERNAL : undefined, - dohProxyIp: config.dnsOverHttps ? DOH_PROXY_IP : undefined, - cliProxyIp: config.difcProxyHost ? CLI_PROXY_IP : undefined, - }, - dnsServers: config.dnsServers || DEFAULT_DNS_SERVERS, - }); - - // Merge credential isolation env vars (COPILOT_API_URL, COPILOT_PROVIDER_BASE_URL, etc.) - // In Docker mode these are merged by assembleOptionalServices during compose generation. - // For sbx, we call buildAgentCredentialEnv directly with host.docker.internal - // as the proxy target (the api-proxy is on the awf-ext bridge network). - if (config.enableApiProxy) { - const credentialEnv = buildAgentCredentialEnv({ - config, - networkConfig: { - subnet: NETWORK_SUBNET, - squidIp: SBX_GATEWAY_IP, - agentIp: AGENT_IP, - proxyIp: SBX_HOST_DOCKER_INTERNAL, - }, - }); - Object.assign(sbxEnvironment, credentialEnv); - } - - // Log critical env vars for debugging auth flow (redact secret values) - logger.info(`[sbx-env] COPILOT_API_URL=${sbxEnvironment.COPILOT_API_URL || '(unset)'}`); - logger.info(`[sbx-env] COPILOT_PROVIDER_BASE_URL=${sbxEnvironment.COPILOT_PROVIDER_BASE_URL || '(unset)'}`); - logger.info(`[sbx-env] COPILOT_GITHUB_TOKEN=${redactSecret(sbxEnvironment.COPILOT_GITHUB_TOKEN)}`); - logger.info(`[sbx-env] COPILOT_API_KEY=${redactSecret(sbxEnvironment.COPILOT_API_KEY)}`); - logger.info(`[sbx-env] HTTPS_PROXY=${sbxEnvironment.HTTPS_PROXY || '(unset)'}`); - logger.info(`[sbx-env] COPILOT_PROVIDER_API_KEY=${redactSecret(sbxEnvironment.COPILOT_PROVIDER_API_KEY)}`); - - // Create the sandbox with configured mounts, proxy chaining through Squid - const workspaceDir = process.env.GITHUB_WORKSPACE || process.cwd(); - sbxName = await createSandbox({ - workspaceDir, - squidIp: SQUID_IP, - extraMounts: sbxMounts, - }); - - // gh-aw fetches reflection data from the fixed api-proxy hostname. The - // microVM reaches the sidecar through its published host ports, so install - // that alias and prove the real endpoint before launching the agent. - if (config.enableApiProxy) { - logger.info('[sbx] Verifying api-proxy /reflect access...'); - await assertSbxApiProxyReflect( - sbxName, - sbxEnvironment, - config.containerWorkDir, - ); - } - - // Verify squid proxy is reachable from sandbox - logger.info('[sbx-diag] Verifying squid proxy connectivity...'); - const diagCmd = [ - `echo -n "squid ${SBX_GATEWAY_IP}:3128 → "`, - `curl -sS --max-time 5 --proxy "http://${SBX_GATEWAY_IP}:3128" -o /dev/null -w "%{http_code}" https://api.github.com/ 2>&1`, - 'echo ""', - ].join(' && '); - - const diagResult = await execInSandbox(sbxName, diagCmd, { - timeoutMinutes: 1, - workDir: config.containerWorkDir, - environment: sbxEnvironment, - }); - logger.info(`[sbx-diag] Connectivity check exited with code ${diagResult.exitCode}`); - } - : startContainers; - - const workflowRunAgentCommand = useSbx - ? async (_workDir: string, _allowedDomains: string[], _proxyLogsDir?: string, agentTimeoutMinutes?: number) => { - if (!sbxName) throw new Error('Sandbox not created'); - logger.info(`[sbx] Launching agent command in sandbox "${sbxName}" (timeout: ${agentTimeoutMinutes ?? 'none'} min)`); - logger.debug(`[sbx] Agent command: ${config.agentCommand.substring(0, 200)}...`); - const result = await execInSandbox(sbxName, config.agentCommand, { - timeoutMinutes: agentTimeoutMinutes, - workDir: config.containerWorkDir, - environment: sbxEnvironment, - tty: config.tty, - }); - logger.info(`[sbx] Agent command exited with code ${result.exitCode}`); - - // Dump api-proxy logs for debugging connection issues - if (config.enableApiProxy && result.exitCode !== 0) { - try { - const { execSync } = await import('child_process'); - const proxyLogs = execSync('docker logs --tail 80 awf-api-proxy 2>&1', { encoding: 'utf-8', timeout: 10000 }); - logger.info(`[sbx-diag] api-proxy logs:\n${proxyLogs}`); - const healthStatus = execSync('docker inspect --format={{.State.Health.Status}} awf-api-proxy 2>&1', { encoding: 'utf-8', timeout: 5000 }); - logger.info(`[sbx-diag] api-proxy health status: ${healthStatus.trim()}`); - } catch { /* ignore diagnostic failures */ } - } - - return { exitCode: result.exitCode, blockedDomains: [] as string[] }; - } - : (workDir: string, allowedDomains: string[], proxyLogsDir?: string, agentTimeoutMinutes?: number) => - runAgentCommand(workDir, allowedDomains, proxyLogsDir, agentTimeoutMinutes, config.containerRuntime); + const externalWorkflowDependencies = externalRuntimeBackend + ? adaptExternalRuntimeBackend(externalRuntimeBackend) + : undefined; + const workflowRunAgentCommand = externalWorkflowDependencies?.runAgentCommand + ?? ((workDir: string, allowedDomains: string[], proxyLogsDir?: string, agentTimeoutMinutes?: number) => + runAgentCommand(workDir, allowedDomains, proxyLogsDir, agentTimeoutMinutes, config.containerRuntime)); exitCode = await runMainWorkflow( config, @@ -464,7 +325,7 @@ export function createMainAction(getOptionValueSource: OptionSourceResolver) { ensureFirewallNetwork, setupHostIptables, writeConfigs, - startContainers: sbxStartContainers, + startContainers: externalWorkflowDependencies?.startContainers ?? startContainers, runAgentCommand: workflowRunAgentCommand, collectDiagnosticLogs, assertTopologySupported, diff --git a/src/container-runtime.ts b/src/container-runtime.ts index ff3f70485..f38c6a66b 100644 --- a/src/container-runtime.ts +++ b/src/container-runtime.ts @@ -18,10 +18,9 @@ * non-default OCI runtime but is still orchestrated by `docker compose`. * - `microvm` – agent runs in a hypervisor-isolated microVM (e.g. Docker * sbx). Infrastructure services (Squid, api-proxy) stay in Docker Compose - * on the host; only the agent crosses the hypervisor boundary. The sbx - * proxy chains upstream through AWF's host-side Squid for domain filtering, - * and through the api-proxy for token logging/model routing/credential - * injection. + * on the host; only the agent crosses the hypervisor boundary. A selected + * external runtime backend owns the agent lifecycle and connects it to + * AWF's host-side infrastructure. * * ## Adding a new OCI runtime * @@ -32,11 +31,11 @@ * * ## Adding a microVM backend (e.g. Docker sbx) * - * Add an entry with `executionModel: 'microvm'`. Callers use - * {@link runtimeUsesComposeAgent} to decide whether to include the agent - * service in docker-compose.yml and whether to use `docker logs/wait` or - * the microVM CLI for agent lifecycle management. Infrastructure services - * (Squid, api-proxy) are generated regardless of execution model. + * Add an entry with `executionModel: 'microvm'` and register its backend in + * `external-runtime-backend-resolver.ts`. Callers use + * {@link runtimeUsesComposeAgent} to distinguish Compose-managed agents from + * external backends and to decide whether to include the agent service in + * docker-compose.yml. Infrastructure services are generated in either mode. */ // ─── Registry ──────────────────────────────────────────────────────────────── @@ -105,7 +104,6 @@ const RUNTIME_REGISTRY: Readonly> = { // so skip the iptables-init container and route egress via proxy env vars. usesIptables: false, }, - // Future: Docker sbx microVM backend sbx: { executionModel: 'microvm', dockerRuntime: undefined, diff --git a/src/external-runtime-backend-resolver.ts b/src/external-runtime-backend-resolver.ts new file mode 100644 index 000000000..8de363a6c --- /dev/null +++ b/src/external-runtime-backend-resolver.ts @@ -0,0 +1,47 @@ +import type { WorkflowDependencies } from './cli-workflow'; +import { runtimeUsesComposeAgent } from './container-runtime'; +import type { ExternalAgentRuntimeBackend } from './external-runtime-backend'; +import { createSbxRuntimeBackend } from './sbx-runtime-backend'; +import type { WrapperConfig } from './types'; + +export interface ExternalRuntimeBackendFactoryContext { + config: WrapperConfig; + startInfrastructure: WorkflowDependencies['startContainers']; +} + +export type ExternalRuntimeBackendFactory = ( + context: ExternalRuntimeBackendFactoryContext, +) => ExternalAgentRuntimeBackend; + +export type ExternalRuntimeBackendRegistry = Readonly< + Record +>; + +const EXTERNAL_RUNTIME_BACKENDS: ExternalRuntimeBackendRegistry = { + sbx: ({ config, startInfrastructure }) => + createSbxRuntimeBackend(config, startInfrastructure), +}; + +/** + * Resolves the selected external agent backend. + * + * Compose runtimes intentionally return undefined and continue through the + * existing Docker/gVisor workflow without an additional abstraction layer. + */ +export function resolveExternalRuntimeBackend( + config: WrapperConfig, + startInfrastructure: WorkflowDependencies['startContainers'], + registry: ExternalRuntimeBackendRegistry = EXTERNAL_RUNTIME_BACKENDS, +): ExternalAgentRuntimeBackend | undefined { + if (runtimeUsesComposeAgent(config.containerRuntime)) { + return undefined; + } + + const runtime = config.containerRuntime; + const factory = runtime ? registry[runtime] : undefined; + if (!factory) { + throw new Error(`No external agent runtime backend is registered for "${runtime}"`); + } + + return factory({ config, startInfrastructure }); +} diff --git a/src/external-runtime-backend.test.ts b/src/external-runtime-backend.test.ts new file mode 100644 index 000000000..78e5db9f4 --- /dev/null +++ b/src/external-runtime-backend.test.ts @@ -0,0 +1,98 @@ +import type { WorkflowDependencies } from './cli-workflow'; +import { + adaptExternalRuntimeBackend, + type ExternalAgentRuntimeBackend, +} from './external-runtime-backend'; +import { resolveExternalRuntimeBackend } from './external-runtime-backend-resolver'; +import type { WrapperConfig } from './types'; + +function createBackend(): jest.Mocked { + return { + runtime: 'test-runtime', + preflight: jest.fn().mockResolvedValue(undefined), + start: jest.fn().mockResolvedValue(undefined), + exec: jest.fn().mockResolvedValue({ exitCode: 17 }), + collectDiagnostics: jest.fn().mockResolvedValue(undefined), + stop: jest.fn().mockResolvedValue(undefined), + }; +} + +describe('external runtime backend', () => { + const startInfrastructure = jest.fn() as jest.MockedFunction< + WorkflowDependencies['startContainers'] + >; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('leaves compose runtimes on the existing workflow path', () => { + const factory = jest.fn(); + const backend = resolveExternalRuntimeBackend( + { containerRuntime: 'gvisor' } as WrapperConfig, + startInfrastructure, + { gvisor: factory }, + ); + + expect(backend).toBeUndefined(); + expect(factory).not.toHaveBeenCalled(); + }); + + it('resolves a registered external runtime with compose infrastructure', () => { + const backend = createBackend(); + const factory = jest.fn().mockReturnValue(backend); + const config = { containerRuntime: 'sbx' } as WrapperConfig; + + expect(resolveExternalRuntimeBackend( + config, + startInfrastructure, + { sbx: factory }, + )).toBe(backend); + expect(factory).toHaveBeenCalledWith({ config, startInfrastructure }); + }); + + it('fails explicitly when a microVM runtime has no backend', () => { + expect(() => resolveExternalRuntimeBackend( + { containerRuntime: 'sbx' } as WrapperConfig, + startInfrastructure, + {}, + )).toThrow('No external agent runtime backend is registered for "sbx"'); + }); + + it('adapts start and exec without changing arguments or exit codes', async () => { + const backend = createBackend(); + const adapted = adaptExternalRuntimeBackend(backend); + const onNetworkReady = jest.fn(); + const onInfrastructureReady = jest.fn(); + + await adapted.startContainers( + '/tmp/awf', + ['github.com'], + '/tmp/logs', + true, + onNetworkReady, + onInfrastructureReady, + ); + await expect(adapted.runAgentCommand( + '/tmp/awf', + ['github.com'], + '/tmp/logs', + 9, + )).resolves.toEqual({ exitCode: 17 }); + + expect(backend.start).toHaveBeenCalledWith( + '/tmp/awf', + ['github.com'], + '/tmp/logs', + true, + onNetworkReady, + onInfrastructureReady, + ); + expect(backend.exec).toHaveBeenCalledWith( + '/tmp/awf', + ['github.com'], + '/tmp/logs', + 9, + ); + }); +}); diff --git a/src/external-runtime-backend.ts b/src/external-runtime-backend.ts new file mode 100644 index 000000000..5c75a4495 --- /dev/null +++ b/src/external-runtime-backend.ts @@ -0,0 +1,34 @@ +import type { WorkflowDependencies } from './cli-workflow'; + +/** + * Lifecycle contract for agent runtimes managed outside Docker Compose. + * + * Infrastructure services remain owned by the existing compose implementation; + * the backend owns the external agent's preflight, startup, execution, + * diagnostics, and teardown state. + */ +export interface ExternalAgentRuntimeBackend { + readonly runtime: string; + preflight(): Promise; + start: WorkflowDependencies['startContainers']; + exec: WorkflowDependencies['runAgentCommand']; + collectDiagnostics(): Promise; + stop(): Promise; +} + +export type ExternalRuntimeWorkflowDependencies = Pick< + WorkflowDependencies, + 'startContainers' | 'runAgentCommand' +>; + +/** + * Adapts an external backend to the existing workflow dependency seam. + */ +export function adaptExternalRuntimeBackend( + backend: ExternalAgentRuntimeBackend, +): ExternalRuntimeWorkflowDependencies { + return { + startContainers: backend.start.bind(backend), + runAgentCommand: backend.exec.bind(backend), + }; +} diff --git a/src/sbx-runtime-backend.test.ts b/src/sbx-runtime-backend.test.ts new file mode 100644 index 000000000..636a2960e --- /dev/null +++ b/src/sbx-runtime-backend.test.ts @@ -0,0 +1,175 @@ +import type { WorkflowDependencies } from './cli-workflow'; +import { SBX_DEFAULT_NAME } from './sbx-manager'; +import { + SBX_GATEWAY_IP, + SBX_HOST_DOCKER_INTERNAL, + SbxRuntimeBackend, + type SbxRuntimeBackendDependencies, +} from './sbx-runtime-backend'; +import type { WrapperConfig } from './types'; + +function createConfig(overrides: Partial = {}): WrapperConfig { + return { + allowedDomains: ['github.com'], + agentCommand: 'echo hello', + workDir: '/tmp/awf-test', + containerWorkDir: '/workspace', + keepContainers: false, + enableApiProxy: true, + dnsServers: ['8.8.8.8'], + volumeMounts: ['/tmp/tooling:/tmp/tooling:ro'], + tty: true, + ...overrides, + } as WrapperConfig; +} + +function createDependencies( + overrides: Partial = {}, +): SbxRuntimeBackendDependencies { + return { + startInfrastructure: jest.fn().mockResolvedValue(undefined) as jest.MockedFunction< + WorkflowDependencies['startContainers'] + >, + isAvailable: jest.fn().mockResolvedValue(true), + createSandbox: jest.fn().mockResolvedValue('awf-agent-created'), + execInSandbox: jest.fn().mockResolvedValue({ exitCode: 0 }), + assertApiProxyReflect: jest.fn().mockResolvedValue(undefined), + removeSandbox: jest.fn().mockResolvedValue(undefined), + execHostCommand: jest.fn() + .mockReturnValueOnce('proxy log\n') + .mockReturnValueOnce('healthy\n'), + getWorkspaceDir: jest.fn().mockReturnValue('/github/workspace'), + logger: { + debug: jest.fn(), + info: jest.fn(), + }, + ...overrides, + }; +} + +describe('SbxRuntimeBackend', () => { + it('owns infrastructure startup, preflight, environment, reflection, exec, diagnostics, and stop', async () => { + const dependencies = createDependencies(); + const execInSandbox = dependencies.execInSandbox as jest.MockedFunction< + SbxRuntimeBackendDependencies['execInSandbox'] + >; + execInSandbox + .mockResolvedValueOnce({ exitCode: 0 }) + .mockResolvedValueOnce({ exitCode: 42 }); + const backend = new SbxRuntimeBackend(createConfig(), dependencies); + const onNetworkReady = jest.fn(); + const onInfrastructureReady = jest.fn(); + + await backend.start( + '/tmp/awf-test', + ['github.com'], + '/tmp/proxy-logs', + true, + onNetworkReady, + onInfrastructureReady, + ); + const result = await backend.exec( + '/tmp/awf-test', + ['github.com'], + '/tmp/proxy-logs', + 7, + ); + await backend.stop(); + await backend.stop(); + + expect(dependencies.startInfrastructure).toHaveBeenCalledWith( + '/tmp/awf-test', + ['github.com'], + '/tmp/proxy-logs', + true, + onNetworkReady, + onInfrastructureReady, + ); + expect(dependencies.isAvailable).toHaveBeenCalledTimes(1); + expect(dependencies.createSandbox).toHaveBeenCalledWith({ + workspaceDir: '/github/workspace', + squidIp: expect.any(String), + extraMounts: ['/tmp/tooling:/tmp/tooling:ro'], + }); + expect(dependencies.assertApiProxyReflect).toHaveBeenCalledWith( + 'awf-agent-created', + expect.objectContaining({ + HTTPS_PROXY: `http://${SBX_GATEWAY_IP}:3128`, + AWF_API_PROXY_IP: SBX_HOST_DOCKER_INTERNAL, + }), + '/workspace', + ); + expect(execInSandbox).toHaveBeenLastCalledWith( + 'awf-agent-created', + 'echo hello', + expect.objectContaining({ + timeoutMinutes: 7, + workDir: '/workspace', + tty: true, + }), + ); + expect(result).toEqual({ exitCode: 42 }); + expect(dependencies.execHostCommand).toHaveBeenNthCalledWith( + 1, + 'docker logs --tail 80 awf-api-proxy 2>&1', + { encoding: 'utf-8', timeout: 10_000 }, + ); + expect(dependencies.execHostCommand).toHaveBeenNthCalledWith( + 2, + 'docker inspect --format={{.State.Health.Status}} awf-api-proxy 2>&1', + { encoding: 'utf-8', timeout: 5_000 }, + ); + expect(dependencies.removeSandbox).toHaveBeenCalledTimes(1); + expect(dependencies.removeSandbox).toHaveBeenCalledWith('awf-agent-created'); + }); + + it('preserves startup ordering and fails closed when sbx is unavailable', async () => { + const callOrder: string[] = []; + const dependencies = createDependencies({ + startInfrastructure: jest.fn(async () => { + callOrder.push('infrastructure'); + }), + isAvailable: jest.fn(async () => { + callOrder.push('preflight'); + return false; + }), + }); + const backend = new SbxRuntimeBackend(createConfig(), dependencies); + + await expect(backend.start('/tmp/awf-test', ['github.com'])).rejects.toThrow( + 'Docker sbx CLI not found', + ); + + expect(callOrder).toEqual(['infrastructure', 'preflight']); + expect(dependencies.createSandbox).not.toHaveBeenCalled(); + }); + + it('rejects execution before the sandbox is created', async () => { + const backend = new SbxRuntimeBackend(createConfig(), createDependencies()); + + await expect(backend.exec('/tmp/awf-test', ['github.com'])).rejects.toThrow( + 'Sandbox not created', + ); + }); + + it('stops the deterministic sandbox name after partial startup', async () => { + const dependencies = createDependencies(); + const backend = new SbxRuntimeBackend(createConfig(), dependencies); + + await backend.stop(); + + expect(dependencies.removeSandbox).toHaveBeenCalledWith(SBX_DEFAULT_NAME); + }); + + it('does not collect API proxy diagnostics when the proxy is disabled', async () => { + const dependencies = createDependencies(); + const backend = new SbxRuntimeBackend( + createConfig({ enableApiProxy: false }), + dependencies, + ); + + await backend.collectDiagnostics(); + + expect(dependencies.execHostCommand).not.toHaveBeenCalled(); + }); +}); diff --git a/src/sbx-runtime-backend.ts b/src/sbx-runtime-backend.ts new file mode 100644 index 000000000..71064fd17 --- /dev/null +++ b/src/sbx-runtime-backend.ts @@ -0,0 +1,272 @@ +import { execSync } from 'child_process'; +import type { WorkflowDependencies } from './cli-workflow'; +import type { ExternalAgentRuntimeBackend } from './external-runtime-backend'; +import { logger } from './logger'; +import { + assertSbxApiProxyReflect, + createSandbox, + execInSandbox, + isSbxAvailable, + removeSandbox, + SBX_DEFAULT_NAME, +} from './sbx-manager'; +import { DEFAULT_DNS_SERVERS } from './dns-resolver'; +import { buildAgentEnvironment } from './services/agent-service'; +import { buildAgentCredentialEnv } from './services/api-proxy-credential-env'; +import { + AGENT_IP, + CLI_PROXY_IP, + DOH_PROXY_IP, + NETWORK_SUBNET, + SQUID_IP, +} from './host-iptables-shared'; +import type { WrapperConfig } from './types'; + +export const SBX_GATEWAY_IP = '172.17.0.0'; +export const SBX_HOST_DOCKER_INTERNAL = 'host.docker.internal'; + +interface SbxBackendLogger { + debug(message: string, ...args: unknown[]): void; + info(message: string, ...args: unknown[]): void; +} + +interface HostCommandOptions { + encoding: 'utf-8'; + timeout: number; +} + +export interface SbxRuntimeBackendDependencies { + startInfrastructure: WorkflowDependencies['startContainers']; + isAvailable: typeof isSbxAvailable; + createSandbox: typeof createSandbox; + execInSandbox: typeof execInSandbox; + assertApiProxyReflect: typeof assertSbxApiProxyReflect; + removeSandbox: typeof removeSandbox; + execHostCommand(command: string, options: HostCommandOptions): string; + getWorkspaceDir(): string; + logger: SbxBackendLogger; +} + +function defaultDependencies( + startInfrastructure: WorkflowDependencies['startContainers'], +): SbxRuntimeBackendDependencies { + return { + startInfrastructure, + isAvailable: isSbxAvailable, + createSandbox, + execInSandbox, + assertApiProxyReflect: assertSbxApiProxyReflect, + removeSandbox, + execHostCommand: (command, options) => execSync(command, options), + getWorkspaceDir: () => process.env.GITHUB_WORKSPACE || process.cwd(), + logger, + }; +} + +/** Stateful adapter for the Docker sbx external microVM runtime. */ +export class SbxRuntimeBackend implements ExternalAgentRuntimeBackend { + readonly runtime = 'sbx'; + + private sandboxName = SBX_DEFAULT_NAME; + private environment: Record | undefined; + private sandboxCreated = false; + private stopped = false; + + constructor( + private readonly config: WrapperConfig, + private readonly dependencies: SbxRuntimeBackendDependencies, + ) {} + + async preflight(): Promise { + if (!await this.dependencies.isAvailable()) { + throw new Error('Docker sbx CLI not found. Install sbx to use --container-runtime sbx.'); + } + } + + readonly start: WorkflowDependencies['startContainers'] = async ( + workDir, + allowedDomains, + proxyLogsDir, + skipPull, + onNetworkReady, + onInfrastructureReady, + ) => { + await this.dependencies.startInfrastructure( + workDir, + allowedDomains, + proxyLogsDir, + skipPull, + onNetworkReady, + onInfrastructureReady, + ); + + await this.preflight(); + + this.environment = buildAgentEnvironment({ + config: this.config, + networkConfig: { + subnet: NETWORK_SUBNET, + squidIp: SBX_GATEWAY_IP, + agentIp: AGENT_IP, + proxyIp: this.config.enableApiProxy ? SBX_HOST_DOCKER_INTERNAL : undefined, + dohProxyIp: this.config.dnsOverHttps ? DOH_PROXY_IP : undefined, + cliProxyIp: this.config.difcProxyHost ? CLI_PROXY_IP : undefined, + }, + dnsServers: this.config.dnsServers || DEFAULT_DNS_SERVERS, + }); + + if (this.config.enableApiProxy) { + Object.assign( + this.environment, + buildAgentCredentialEnv({ + config: this.config, + networkConfig: { + subnet: NETWORK_SUBNET, + squidIp: SBX_GATEWAY_IP, + agentIp: AGENT_IP, + proxyIp: SBX_HOST_DOCKER_INTERNAL, + }, + }), + ); + } + + this.logEnvironment(); + + this.sandboxName = await this.dependencies.createSandbox({ + workspaceDir: this.dependencies.getWorkspaceDir(), + squidIp: SQUID_IP, + extraMounts: [...(this.config.volumeMounts ?? [])], + }); + this.sandboxCreated = true; + + if (this.config.enableApiProxy) { + this.dependencies.logger.info('[sbx] Verifying api-proxy /reflect access...'); + await this.dependencies.assertApiProxyReflect( + this.sandboxName, + this.environment, + this.config.containerWorkDir, + ); + } + + this.dependencies.logger.info('[sbx-diag] Verifying squid proxy connectivity...'); + const diagnosticCommand = [ + `echo -n "squid ${SBX_GATEWAY_IP}:3128 -> "`, + `curl -sS --max-time 5 --proxy "http://${SBX_GATEWAY_IP}:3128" -o /dev/null -w "%{http_code}" https://api.github.com/ 2>&1`, + 'echo ""', + ].join(' && '); + const diagnosticResult = await this.dependencies.execInSandbox( + this.sandboxName, + diagnosticCommand, + { + timeoutMinutes: 1, + workDir: this.config.containerWorkDir, + environment: this.environment, + }, + ); + this.dependencies.logger.info( + `[sbx-diag] Connectivity check exited with code ${diagnosticResult.exitCode}`, + ); + }; + + readonly exec: WorkflowDependencies['runAgentCommand'] = async ( + _workDir, + _allowedDomains, + _proxyLogsDir, + agentTimeoutMinutes, + ) => { + if (!this.sandboxCreated) { + throw new Error('Sandbox not created'); + } + + this.dependencies.logger.info( + `[sbx] Launching agent command in sandbox "${this.sandboxName}" (timeout: ${agentTimeoutMinutes ?? 'none'} min)`, + ); + this.dependencies.logger.debug( + `[sbx] Agent command: ${this.config.agentCommand.substring(0, 200)}...`, + ); + const result = await this.dependencies.execInSandbox( + this.sandboxName, + this.config.agentCommand, + { + timeoutMinutes: agentTimeoutMinutes, + workDir: this.config.containerWorkDir, + environment: this.environment, + tty: this.config.tty, + }, + ); + this.dependencies.logger.info(`[sbx] Agent command exited with code ${result.exitCode}`); + + if (this.config.enableApiProxy && result.exitCode !== 0) { + await this.collectDiagnostics(); + } + + return { exitCode: result.exitCode }; + }; + + async collectDiagnostics(): Promise { + if (!this.config.enableApiProxy) return; + + try { + const proxyLogs = this.dependencies.execHostCommand( + 'docker logs --tail 80 awf-api-proxy 2>&1', + { encoding: 'utf-8', timeout: 10_000 }, + ); + this.dependencies.logger.info(`[sbx-diag] api-proxy logs:\n${proxyLogs}`); + const healthStatus = this.dependencies.execHostCommand( + 'docker inspect --format={{.State.Health.Status}} awf-api-proxy 2>&1', + { encoding: 'utf-8', timeout: 5_000 }, + ); + this.dependencies.logger.info( + `[sbx-diag] api-proxy health status: ${healthStatus.trim()}`, + ); + } catch { + // Diagnostics are best effort and must not alter the agent exit code. + } + } + + async stop(): Promise { + if (this.stopped) return; + this.stopped = true; + + try { + await this.dependencies.removeSandbox(this.sandboxName); + } catch { + // The sandbox may not exist yet when startup or signal handling fails. + } + } + + private logEnvironment(): void { + const environment = this.environment!; + this.dependencies.logger.info( + `[sbx-env] COPILOT_API_URL=${environment.COPILOT_API_URL || '(unset)'}`, + ); + this.dependencies.logger.info( + `[sbx-env] COPILOT_PROVIDER_BASE_URL=${environment.COPILOT_PROVIDER_BASE_URL || '(unset)'}`, + ); + this.dependencies.logger.info( + `[sbx-env] COPILOT_GITHUB_TOKEN=${redactSecret(environment.COPILOT_GITHUB_TOKEN)}`, + ); + this.dependencies.logger.info( + `[sbx-env] COPILOT_API_KEY=${redactSecret(environment.COPILOT_API_KEY)}`, + ); + this.dependencies.logger.info( + `[sbx-env] HTTPS_PROXY=${environment.HTTPS_PROXY || '(unset)'}`, + ); + this.dependencies.logger.info( + `[sbx-env] COPILOT_PROVIDER_API_KEY=${redactSecret(environment.COPILOT_PROVIDER_API_KEY)}`, + ); + } +} + +/** Report whether a secret is set (and its length) without exposing the value. */ +function redactSecret(value: string | undefined): string { + if (!value) return '(unset)'; + return `(set, len=${value.length})`; +} + +export function createSbxRuntimeBackend( + config: WrapperConfig, + startInfrastructure: WorkflowDependencies['startContainers'], +): SbxRuntimeBackend { + return new SbxRuntimeBackend(config, defaultDependencies(startInfrastructure)); +}