feat: Persisted Artifact Lifecycle for sandbox

configs/config_openshell.yml

@@ -1,7 +1,5 @@
-# EXPERIMENTAL: AI-Q deep research over a local, single-operator OpenShell sandbox.
-# Run ./scripts/setup_openshell.sh first to provision the gateway and named sandbox.
-# Jobs attach to that shared sandbox; per-job directories prevent filename collisions
-# but are not a security boundary. Do not run mutually untrusted jobs concurrently.
+# EXPERIMENTAL: AI-Q deep research with one policy-bound OpenShell sandbox per job.
+# Run ./scripts/setup_openshell.sh first to provision the gateway, image, and policy.
 
 general:
   use_uvloop: true
@@ -141,13 +139,30 @@ functions:
   deep_research_sandbox:
     _type: deep_research_sandbox
     provider: openshell
-    sandbox_name: ${AIQ_OPENSHELL_SANDBOX_NAME:-aiq-openshell-demo}
+    openshell_image: ${AIQ_OPENSHELL_IMAGE:-aiq-openshell-demo:latest}
     policy: ${AIQ_OPENSHELL_POLICY_FILE:-configs/openshell/generated/aiq-openshell-policy.yaml}
     workdir: /sandbox
-    network: blocked
+    # The OpenShell policy is authoritative. This declaration is an upper bound:
+    # startup fails if the policy grants a hostname not listed here.
+    network: allowlist
+    network_allow:
+      - api.github.com
+      - github.com
+      - integrate.api.nvidia.com
+      - api.tavily.com
+      - google.serper.dev
+      - pypi.org
+      - files.pythonhosted.org
+      - huggingface.co
+      - cdn-lfs.huggingface.co
+      - export.arxiv.org
+      - api.semanticscholar.org
+      - registry.npmjs.org
     timeout: 1200
     idle_timeout: 1800
-    delete_on_exit: false
+    delete_on_exit: true
+    attest: true
+    require_hard_landlock: true
     artifact_capture:
       enabled: true
       max_file_bytes: 50000000

configs/openshell/aiq-research-policy.yaml

@@ -11,7 +11,6 @@ filesystem_policy:
     - /usr
     - /lib
     - /etc
-    - /app
     - /var/log
     - /proc/self
     - /dev/urandom
@@ -21,11 +20,12 @@ filesystem_policy:
     - /tmp
     - /dev/null
 
-# best_effort lets the sandbox start on hosts without Landlock (e.g. Docker Desktop on
-# macOS), but there filesystem confinement is silently dropped. Acceptable only for the
-# local single-operator demo; production must use `hard_requirement` (fail closed).
+# Fail closed when the host cannot enforce Landlock filesystem confinement. Local demo
+# environments that intentionally accept weaker isolation must generate a separate policy
+# with `scripts/setup_openshell.sh --landlock-compatibility best_effort` and configure
+# `require_hard_landlock: false` explicitly.
 landlock:
-  compatibility: best_effort
+  compatibility: hard_requirement
 
 process:
   run_as_user: sandbox

docs/source/architecture/agents/sandbox.md

@@ -6,9 +6,9 @@ SPDX-License-Identifier: Apache-2.0
 # Deep Research Sandbox Notes
 
 Deep research can optionally run DeepAgents `execute` calls through a sandbox provider
-(Modal, OpenShell, or any registered provider). Modal creates a sandbox per job. The
-experimental OpenShell path attaches to a pre-created named sandbox shared by its jobs;
-job-scoped directories prevent filename collisions but are not a security boundary.
+(Modal, OpenShell, or any registered provider). Modal and OpenShell create a fresh physical
+sandbox per job. OpenShell binds the configured policy at creation and attests readiness plus
+the loaded policy revision before exposing the execution backend.
 
 The sandbox is an internal execution detail. There are no sandbox-specific API
 endpoints, and job-level auth remains responsible for submit, stream, status,
@@ -21,12 +21,13 @@ runtime (`.../job/{job_id}/artifacts`), which is also auth-scoped to the job.
 
 ## Current Behavior
 
-- Modal uses one sandbox per deep research job. OpenShell currently attaches jobs to
-  the configured shared sandbox name and is intended for local, single-operator testing.
+- Modal and OpenShell use one physical sandbox per deep research job. OpenShell shared
+  attachment is available only through an explicit debug-only opt-in and is not job-isolated.
 - Synchronous sandbox-enabled runs use an internal per-agent runtime ID.
 - Providers are selected by config (`sandbox.provider` + `providers.<name>`); the
   provider is validated against the registry and gated by its declared capabilities.
-  OpenShell policy is provisioned externally and is not verified when AI-Q attaches.
+  OpenShell policy YAML is parsed strictly against the installed SDK schema, applied in the
+  job's creation spec, checked against the declared network upper bound, and attested before use.
 - Job IDs must satisfy each provider's object-name rules (Modal: 64 chars or fewer,
   alphanumeric plus dash/period/underscore).
 - `timeout` bounds individual execution. Other lifecycle controls are provider-dependent.
@@ -36,13 +37,21 @@ runtime (`.../job/{job_id}/artifacts`), which is also auth-scoped to the job.
 
 ## Operational Notes
 
-- High-concurrency Modal runs create one sandbox per job. OpenShell runs share the named
-  sandbox and must not be used concurrently for mutually untrusted jobs. Optional submit-path
+- High-concurrency Modal and OpenShell runs create one sandbox per job. Optional submit-path
   caps (`AIQ_MAX_SANDBOXES_PER_PRINCIPAL` / `AIQ_MAX_SANDBOXES_GLOBAL`, default-off) bound
-  concurrency/cost but do not provide filesystem isolation.
+  concurrency and cost.
 - Custom client-supplied job IDs must not be reused for a new job.
-- The runtime closes provider sessions on success, failure, cancellation, and timeout.
-  A named OpenShell sandbox persists when `delete_on_exit` is disabled.
+- Manifest checkpoints preserve completed artifacts after successful sandbox commands. The
+  terminal finalizer harvests once before cleanup on success/failure; cancellation harvests
+  only when the provider is idle and otherwise terminates immediately.
+- Job status becomes terminal only after artifact and cleanup events are flushed.
+- Per-job OpenShell mode requires `delete_on_exit: true`. A persistent shared sandbox is
+  possible only through the explicit debug attachment settings.
+- Hosts without a Landlock LSM (local macOS / Docker Desktop) cannot satisfy the production
+  default `landlock.compatibility: hard_requirement`; every sandbox fails prepare there.
+  Generate a local policy with `./scripts/setup_openshell.sh --landlock-compatibility best_effort`
+  and set `require_hard_landlock: false`. See the deep researcher sandbox README ("Local
+  macOS / Docker Desktop") for the two-knob detail.
 
 ## Current Safeguards
 
@@ -54,3 +63,4 @@ The following safeguards are in place:
   with MIME-from-bytes spoof rejection, SVG sanitization, and an inline-render allowlist.
 - Sandbox quota and concurrency controls, and artifact retention via job-expiry cleanup.
 - Structured lifecycle logging for sandbox create, reuse, failure, and cleanup.
+- Structured `sandbox.attestation` and `sandbox.cleanup` events.

frontends/aiq_api/README.md

@@ -72,7 +72,7 @@ Base path: `/v1/jobs/async`
 | `/v1/jobs/async/job/{id}` | GET | Get job status |
 | `/v1/jobs/async/job/{id}/stream` | GET | SSE stream from beginning |
 | `/v1/jobs/async/job/{id}/stream/{last_event_id}` | GET | SSE stream from event ID |
-| `/v1/jobs/async/job/{id}/cancel` | POST | Cancel running job |
+| `/v1/jobs/async/job/{id}/cancel` | POST | Request cancellation; worker publishes `INTERRUPTED` after cleanup |
 | `/v1/jobs/async/job/{id}/state` | GET | Get current UI state |
 | `/v1/jobs/async/job/{id}/report` | GET | Get final report |
 
@@ -149,7 +149,7 @@ Events streamed during job execution:
 | `workflow.start` / `workflow.end` | Workflow lifecycle |
 | `llm.start` / `llm.chunk` / `llm.end` | LLM inference progress |
 | `tool.start` / `tool.end` | Tool invocations |
-| `artifact.update` | Todos, files, citations, output updates |
+| `artifact.update` | Todos, citations, output, and generated-file metadata (`content_url` for durable bytes) |
 | `job.error` | Error occurred |
 
 ## Configuration

frontends/aiq_api/src/aiq_api/jobs/runner.py

@@ -580,7 +580,14 @@ async def run_agent_job(
                     # Signal event stream completion
                     event_stream.on_complete()
 
-                    # Flush any buffered events before updating status
+                    # Finalize before SUCCESS so clients cannot stop streaming before
+                    # artifact and cleanup events are durable.
+                    await asyncio.to_thread(
+                        _teardown_sandbox,
+                        sandbox_runtime,
+                        job_id=job_id,
+                        interrupted=False,
+                    )
                     if hasattr(event_store, "flush"):
                         event_store.flush()
 
@@ -596,17 +603,10 @@ async def run_agent_job(
     except asyncio.CancelledError:
         logger.info("Job %s cancelled", job_id)
         interrupted = True
-        if job_store:
-            try:
-                job = await job_store.get_job(job_id)
-                if job and job.status != JobStatus.INTERRUPTED.value:
-                    await job_store.update_status(job_id, JobStatus.INTERRUPTED, error="cancelled by user")
-            except (ConnectionError, TimeoutError, RuntimeError):
-                pass
-
         if event_store is None:
             event_store = BatchingEventStore(EventStore(db_url, job_id))
 
+        await asyncio.to_thread(_teardown_sandbox, sandbox_runtime, job_id=job_id, interrupted=True)
         event_store.store(
             {
                 "type": "job.cancelled",
@@ -616,14 +616,20 @@ async def run_agent_job(
         if hasattr(event_store, "flush"):
             event_store.flush()
 
-    except Exception as e:
-        logger.exception("Job %s failed: %s", job_id, type(e).__name__)
         if job_store:
-            await job_store.update_status(job_id, JobStatus.FAILURE, error=str(e))
+            try:
+                job = await job_store.get_job(job_id)
+                if job and job.status != JobStatus.INTERRUPTED.value:
+                    await job_store.update_status(job_id, JobStatus.INTERRUPTED, error="cancelled by user")
+            except (ConnectionError, TimeoutError, RuntimeError):
+                pass
 
+    except Exception as e:
+        logger.exception("Job %s failed: %s", job_id, type(e).__name__)
         if event_store is None:
             event_store = BatchingEventStore(EventStore(db_url, job_id))
 
+        await asyncio.to_thread(_teardown_sandbox, sandbox_runtime, job_id=job_id, interrupted=False)
         event_store.store(
             {
                 "type": "job.error",
@@ -635,24 +641,34 @@ async def run_agent_job(
         )
         if hasattr(event_store, "flush"):
             event_store.flush()
+        if job_store:
+            await job_store.update_status(job_id, JobStatus.FAILURE, error=str(e))
 
     finally:
         # Ensure terminal-path events are not left in the batch buffer.
-        if event_store is not None and hasattr(event_store, "flush"):
-            event_store.flush()
+        await _flush_event_store(event_store, job_id=job_id)
         if cancellation_monitor:
             cancellation_monitor.stop()
-        # Release the sandbox off the event loop so the SDK session close never blocks the Dask
-        # worker. The single artifact harvest already ran in agent.run() before this point, so
-        # teardown only closes/terminates; interrupted jobs terminate() to preempt a live execute.
+        # Idempotent fallback for failures before a terminal branch finalized the runtime.
         await asyncio.to_thread(_teardown_sandbox, sandbox_runtime, job_id=job_id, interrupted=interrupted)
+        await _flush_event_store(event_store, job_id=job_id)
         # Clean up job-scoped auth token
         if _auth_token_reset is not None:
             from ._auth_context import job_auth_token
 
             job_auth_token.reset(_auth_token_reset)
 
 
+async def _flush_event_store(event_store: Any | None, *, job_id: str) -> None:
+    """Flush terminal events off-loop without replacing the job result."""
+    if event_store is None or not hasattr(event_store, "flush"):
+        return
+    try:
+        await asyncio.to_thread(event_store.flush)
+    except Exception as exc:  # noqa: BLE001 - terminal observability must not replace the job result
+        logger.warning("Event store flush failed for job %s (%s)", job_id, type(exc).__name__)
+
+
 def _teardown_sandbox(sandbox_runtime: Any | None, *, job_id: str, interrupted: bool) -> None:
     """Release sandbox resources on a terminal path (best-effort, never raises).
 
@@ -662,15 +678,23 @@ def _teardown_sandbox(sandbox_runtime: Any | None, *, job_id: str, interrupted:
     """
     if sandbox_runtime is None:
         return
+    finalize = getattr(sandbox_runtime, "finalize", None)
+    if finalize is not None:
+        try:
+            if not finalize(interrupted=interrupted):
+                logger.warning("Sandbox cleanup reported failure for job %s", job_id)
+        except Exception as exc:  # noqa: BLE001 - cleanup must never replace the job result
+            logger.warning("Sandbox cleanup failed for job %s (%s)", job_id, type(exc).__name__)
+        return
     teardown = getattr(sandbox_runtime, "terminate", None) if interrupted else None
     if teardown is None:
         teardown = getattr(sandbox_runtime, "close", None)
     if teardown is None:
         return
     try:
         teardown()
-    except Exception:  # noqa: BLE001 - cleanup must never raise on the terminal path
-        logger.warning("Sandbox cleanup failed for job %s", job_id, exc_info=True)
+    except Exception as exc:  # noqa: BLE001 - cleanup must never raise on the terminal path
+        logger.warning("Sandbox cleanup failed for job %s (%s)", job_id, type(exc).__name__)
 
 
 def _create_agent_instance(

frontends/aiq_api/src/aiq_api/routes/jobs.py

@@ -758,7 +758,7 @@ async def stream_job_events_from(job_id: str, last_event_id: int) -> StreamingRe
         "/v1/jobs/async/job/{job_id}/cancel",
         tags=["async jobs"],
         summary="Cancel a running job",
-        description="Request cancellation of a running job. The job status will be set to INTERRUPTED.",
+        description="Request cancellation. The worker publishes INTERRUPTED after terminal cleanup completes.",
         responses={
             400: {"description": "Job is not in RUNNING state"},
             404: {"description": "Job not found"},
@@ -772,8 +772,6 @@ async def cancel_job(job_id: str) -> dict:
         if job.status != JobStatus.RUNNING.value:
             raise HTTPException(400, f"Job not running: {job_id} (status: {job.status})")
 
-        await job_store.update_status(job_id, JobStatus.INTERRUPTED, error="cancelled by user")
-
         event_store = EventStore(db_url, job_id)
         event_store.store(
             {
@@ -784,9 +782,14 @@ async def cancel_job(job_id: str) -> dict:
 
         task_cancelled = await _cancel_dask_task(scheduler_address, job_id)
 
-        logger.info("Cancel requested for job %s: status updated, task_cancelled=%s", job_id, task_cancelled)
+        logger.info("Cancel requested for job %s: task_cancelled=%s", job_id, task_cancelled)
 
-        return {"job_id": job_id, "status": JobStatus.INTERRUPTED.value, "task_cancelled": task_cancelled}
+        return {
+            "job_id": job_id,
+            "status": job.status,
+            "cancellation_requested": True,
+            "task_cancelled": task_cancelled,
+        }
 
     @app.get(
         "/v1/jobs/async/job/{job_id}/state",

frontends/ui/src/adapters/api/deep-research-client.spec.ts

@@ -2,7 +2,29 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import { afterEach, describe, expect, test, vi } from 'vitest'
-import { getJobStatus } from './deep-research-client'
+import { createDeepResearchClient, getJobStatus } from './deep-research-client'
+
+class FakeEventSource {
+  static latest: FakeEventSource | null = null
+  onopen: (() => void) | null = null
+  onmessage: ((event: MessageEvent) => void) | null = null
+  onerror: (() => void) | null = null
+  private listeners = new Map<string, (event: MessageEvent) => void>()
+
+  constructor(_url: string) {
+    FakeEventSource.latest = this
+  }
+
+  addEventListener(type: string, listener: EventListener): void {
+    this.listeners.set(type, listener as (event: MessageEvent) => void)
+  }
+
+  close(): void {}
+
+  emit(type: string, data: unknown): void {
+    this.listeners.get(type)?.(new MessageEvent(type, { data: JSON.stringify(data) }))
+  }
+}
 
 describe('deep research REST client', () => {
   afterEach(() => {
@@ -32,4 +54,50 @@ describe('deep research REST client', () => {
       'Failed to get job status: 500 - PROXY_ERROR: fetch failed'
     )
   })
+
+  test('maps generated binary file events to durable artifact metadata', () => {
+    vi.stubGlobal('EventSource', FakeEventSource)
+    const onFileUpdate = vi.fn()
+    const client = createDeepResearchClient({ jobId: 'job-1', callbacks: { onFileUpdate } })
+    client.connect()
+
+    FakeEventSource.latest?.emit('artifact.update', {
+      data: {
+        type: 'file',
+        file_path: 'chart.png',
+        artifact_id: 'art_123',
+        content_url: '/v1/jobs/async/job/job-1/artifacts/art_123/content',
+        mime_type: 'image/png',
+        size_bytes: 2048,
+        sha256: 'a'.repeat(64),
+        inline: true,
+      },
+    })
+
+    expect(onFileUpdate).toHaveBeenCalledWith({
+      filename: 'chart.png',
+      content: undefined,
+      artifactId: 'art_123',
+      contentUrl: '/api/jobs/async/job/job-1/artifacts/art_123/content',
+      mimeType: 'image/png',
+      sizeBytes: 2048,
+      sha256: 'a'.repeat(64),
+      inline: true,
+    })
+  })
+
+  test('preserves legacy text file events', () => {
+    vi.stubGlobal('EventSource', FakeEventSource)
+    const onFileUpdate = vi.fn()
+    const client = createDeepResearchClient({ jobId: 'job-1', callbacks: { onFileUpdate } })
+    client.connect()
+
+    FakeEventSource.latest?.emit('artifact.update', {
+      data: { type: 'file', file_path: 'report.md', content: '# Report' },
+    })
+
+    expect(onFileUpdate).toHaveBeenCalledWith(
+      expect.objectContaining({ filename: 'report.md', content: '# Report' })
+    )
+  })
 })