From 1b8a9521a04639afb55e4078d39f2b98ceff39b7 Mon Sep 17 00:00:00 2001 From: Martin Hausleitner <55828102+Martin-Hausleitner@users.noreply.github.com> Date: Tue, 21 Jul 2026 19:13:29 +0200 Subject: [PATCH 01/32] feat: simplify mobile computer use workspace --- README.md | 2 +- .../mobile/MobileSplitScreen.test.tsx | 80 +++++- .../components/mobile/MobileSplitScreen.tsx | 246 ++++++++++++------ frontend/src/lib/taskHarness.test.ts | 19 ++ frontend/src/lib/taskHarness.ts | 41 ++- frontend/src/styles/globals.css | 42 ++- scripts/mobile_ui_gate.py | 55 ++-- 7 files changed, 387 insertions(+), 98 deletions(-) diff --git a/README.md b/README.md index 67b2eaf7..cabd32b7 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ The [redacted 20-run report](docs/streaming-benchmark-latest.md) remains the war The new pinned r51 Selkies lane is now independently reproducible. In the fresh five-run loopback check, its HTTP shell returned **5/5** successful samples with **2.672 ms median total time** (p95 **3.515 ms**), and `/websockets` returned **5/5** valid upgrades with **2.190 ms median handshake** (p95 **3.722 ms**). This proves transport readiness only: it is not a first-frame, FPS, touch-to-pixel, authenticated product-flow, or mobile-Safari comparison. KasmVNC/noVNC remains the production recommendation because it is still the only candidate integrated and verified through the complete profile, policy, VNC-canvas and mobile-UI path. Full context and limits are in [docs/REMOTE-STREAMING-BENCHMARK.md](docs/REMOTE-STREAMING-BENCHMARK.md). -The current r51 mobile implementation is browser-first: chat starts collapsed, browser tools are centralized, benchmark controls are not shown in the UI, and the task composer accepts only a host bridge explicitly identifying itself as `codex-computer-use`. A missing, generic or mislabeled harness fails closed; task submission is not simulated and no direct vendor API key is required in the UI. The authenticated r51 gate passed **276/276 checks** across five mobile viewports plus the access dashboard and produced **23 screenshots**. A separate Codex Computer Use run logged in as a scoped viewer, reached a real connected VNC canvas, confirmed exactly four persistent actions, exercised the task bridge and found no horizontal overflow at `390 x 844`. The same run then used the admin dashboard to combine `operate` with independent CDP `automate` access for a Paperclip agent. The VCVM/Neko Tailnet check remains transport evidence only: Codex Computer Use completed the protected login and observed `/ws`, but WebRTC ICE failed, so no honest FPS value was recorded. Full details and limits are in [docs/MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md](docs/MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md). +The current r55 mobile implementation is browser-first: chat starts collapsed, the live browser consumes all unused space, benchmark controls are not shown in the UI, and only Full, Tools, Chat and Send remain persistent. Tools use progressive disclosure for View, Sessions and Admin. The bookmark-like Actions row is not a list of URLs; it sends typed Capture, Copy and Paste commands through a host bridge explicitly identifying itself as `codex-computer-use`. Unknown command kinds are dropped at the bridge boundary, while a missing, generic or mislabeled harness fails closed. The fresh live r55 gate passed **276/276 checks** across five mobile viewports and produced **22 screenshots**. The earlier authenticated r51 run additionally covered the access dashboard: Codex Computer Use logged in as a scoped viewer, reached a real connected VNC canvas, confirmed exactly four persistent actions and found no horizontal overflow at `390 x 844`; an administrator then combined `operate` with independent CDP `automate` access for a Paperclip agent. The VCVM/Neko Tailnet check remains transport evidence only: Codex Computer Use completed the protected login and observed `/ws`, but WebRTC ICE failed, so no honest FPS value was recorded. Full details and limits are in [docs/MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md](docs/MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md). ## Development diff --git a/frontend/src/components/mobile/MobileSplitScreen.test.tsx b/frontend/src/components/mobile/MobileSplitScreen.test.tsx index e8e4cb02..239220a4 100644 --- a/frontend/src/components/mobile/MobileSplitScreen.test.tsx +++ b/frontend/src/components/mobile/MobileSplitScreen.test.tsx @@ -66,7 +66,7 @@ function installTaskHarness() { chat: true, streaming: true, clipboard: true, - browser_actions: ["copy", "paste", "fullscreen"], + browser_actions: ["copy", "paste", "screenshot", "fullscreen"], metadata: { mode: "codex-test", provider: codexComputerUseProvider }, }, send, @@ -179,7 +179,7 @@ describe("MobileSplitScreen", () => { expect((screen.getByLabelText("Run task") as HTMLButtonElement).disabled).toBe(true); openBrowserTools(); - expect(screen.getByText("Codex Computer Use Bridge · unavailable")).toBeTruthy(); + expect(screen.getByText("Codex unavailable")).toBeTruthy(); expect(screen.getByText("A verified Codex Computer Use Bridge must be injected by the host before tasks can run.")).toBeTruthy(); fireEvent.submit(input.closest("form") as HTMLFormElement); @@ -300,6 +300,22 @@ describe("MobileSplitScreen", () => { expect(screen.queryByLabelText("Streaming benchmark results")).toBeNull(); }); + it("lets the browser consume unused workspace until chat or tools are opened", () => { + const { container } = runningSplit(); + const workspace = container.querySelector(".mobile-split-root") as HTMLElement; + + expect(workspace.classList.contains("mobile-workspace-collapsed")).toBe(true); + + openBrowserTools(); + expect(workspace.classList.contains("mobile-workspace-collapsed")).toBe(false); + + fireEvent.click(screen.getByLabelText("Close browser tools")); + expect(workspace.classList.contains("mobile-workspace-collapsed")).toBe(true); + + fireEvent.click(screen.getByLabelText("Expand task chat")); + expect(workspace.classList.contains("mobile-workspace-collapsed")).toBe(false); + }); + it("keeps profile and browser actions inside the central tools sheet without a harness picker", () => { runningSplit({ canManageAccess: true }); @@ -312,12 +328,51 @@ describe("MobileSplitScreen", () => { const tools = screen.getByLabelText("Browser tools"); expect(within(tools).getByRole("button", { name: /Stop/i })).toBeTruthy(); + expect(within(tools).queryByLabelText("New profile")).toBeNull(); + fireEvent.click(within(tools).getByLabelText("Toggle browser administration")); expect(within(tools).getByLabelText("New profile")).toBeTruthy(); expect(within(tools).getByLabelText("Edit selected profile")).toBeTruthy(); expect(within(tools).getByLabelText("Browser access controls")).toBeTruthy(); expect(within(tools).queryByLabelText("Select harness runner")).toBeNull(); }); + it("runs compact pinned browser actions only through the verified Codex host", async () => { + const { send } = installTaskHarness(); + runningSplit(); + openBrowserTools(); + + const capture = await screen.findByLabelText("Run Capture with Codex Computer Use"); + await waitFor(() => expect((capture as HTMLButtonElement).disabled).toBe(false)); + fireEvent.click(capture); + + await waitFor(() => + expect(send).toHaveBeenCalledWith( + { + text: "Capture the current browser view.", + commands: [ + { + id: "capture-browser", + label: "Capture", + kind: "screenshot", + scope: "host", + }, + ], + profile_id: runningProfile.id, + metadata: { + runner: "codex-computer-use", + preferred_surface: "codex-computer-use", + browser_visible: true, + source: "pinned-action", + }, + }, + undefined, + ), + ); + expect(await screen.findByText("Harness accepted the task.")).toBeTruthy(); + expect(screen.queryByLabelText("Browser tools")).toBeNull(); + expect(screen.getByLabelText("Chat history")).toBeTruthy(); + }); + it("keeps viewport and grid panels mutually exclusive inside browser tools", async () => { const { props } = renderMobileSplit(); openBrowserTools(); @@ -329,6 +384,7 @@ describe("MobileSplitScreen", () => { fireEvent.click(screen.getByLabelText("Edit browser viewport")); expect(screen.getByLabelText("Viewport controls")).toBeTruthy(); expect(screen.queryByLabelText("Running browser grid")).toBeNull(); + expect(screen.queryByLabelText("Pinned browser actions")).toBeNull(); fireEvent.click(screen.getByText("Tablet")); fireEvent.click(screen.getByText("Apply")); @@ -363,6 +419,7 @@ describe("MobileSplitScreen", () => { openBrowserTools(); expect(livePane.style.getPropertyValue("--mobile-live-pane-basis")).toBe("68%"); + fireEvent.click(screen.getByLabelText("Edit browser viewport")); fireEvent.change(screen.getByLabelText("Browser pane"), { target: { value: "64" } }); fireEvent.change(screen.getByLabelText("Visual zoom"), { target: { value: "135" } }); @@ -377,14 +434,17 @@ describe("MobileSplitScreen", () => { expect(screen.getAllByText("VNC stream")).toHaveLength(1); }); - it("keeps local view controls but hides viewport editing without profile management access", () => { + it("keeps local view controls but hides persistent viewport fields without profile management access", () => { runningSplit({ canManageProfiles: false }); openBrowserTools(); + expect(screen.getByLabelText("Edit browser viewport")).toBeTruthy(); + fireEvent.click(screen.getByLabelText("Edit browser viewport")); expect(screen.getByLabelText("Browser pane")).toBeTruthy(); expect(screen.getByLabelText("Visual zoom")).toBeTruthy(); - expect(screen.queryByLabelText("Edit browser viewport")).toBeNull(); + expect(screen.queryByLabelText("Viewport width")).toBeNull(); + expect(screen.getByText("Viewport changes require profile management access.")).toBeTruthy(); fireEvent.click(screen.getByLabelText("Open fullscreen browser")); expect(screen.getByLabelText("Toggle fullscreen view controls")).toBeTruthy(); @@ -434,6 +494,18 @@ describe("MobileSplitScreen", () => { expect(document.activeElement).toBe(screen.getByLabelText("Open fullscreen browser")); }); + it("keeps a visible fullscreen exit control when no browser is live", () => { + renderMobileSplit(); + + fireEvent.click(screen.getByLabelText("Open fullscreen browser")); + + expect(screen.getByRole("dialog", { name: "Fullscreen browser viewer" })).toBeTruthy(); + expect(screen.getByLabelText("Close fullscreen browser")).toBeTruthy(); + + fireEvent.click(screen.getByLabelText("Close fullscreen browser")); + expect(screen.queryByRole("dialog", { name: "Fullscreen browser viewer" })).toBeNull(); + }); + it("uses Ctrl or Cmd shortcuts for fullscreen chat and browser tools", () => { runningSplit(); diff --git a/frontend/src/components/mobile/MobileSplitScreen.tsx b/frontend/src/components/mobile/MobileSplitScreen.tsx index 1d564444..fc83b4ef 100644 --- a/frontend/src/components/mobile/MobileSplitScreen.tsx +++ b/frontend/src/components/mobile/MobileSplitScreen.tsx @@ -3,7 +3,10 @@ import type { CSSProperties, ChangeEvent, FormEvent, ReactNode } from "react"; import { ArrowLeft, ArrowRight, + Camera, ChevronUp, + ClipboardCopy, + ClipboardPaste, Expand, Grid2X2, Globe2, @@ -23,6 +26,7 @@ import type { Profile } from "../../lib/api"; import { createTaskHarness, taskHarnessReadyEvent, + type TaskHarnessAction, type TaskHarnessCapabilities, } from "../../lib/taskHarness"; import { StatusIndicator } from "../StatusIndicator"; @@ -67,7 +71,7 @@ const presets = [ { label: "Desktop", width: 1440, height: 900 }, ] as const; -const defaultPreviewPanePercent = 42; +const defaultPreviewPanePercent = 68; // The running browser is the primary control surface; chat and settings are // now collapsed into a compact dock by default. const defaultLivePanePercent = 68; @@ -79,6 +83,37 @@ const collapsedLandscapeLivePanePercent = 74; const minimumPhoneFitWidth = 320; const minimumPhoneFitHeight = 480; +type PinnedHarnessAction = Omit & { + kind: "screenshot" | "copy" | "paste"; +}; + +const pinnedHarnessActions = [ + { + id: "capture-browser", + label: "Capture", + kind: "screenshot", + scope: "host", + }, + { + id: "copy-browser-selection", + label: "Copy", + kind: "copy", + scope: "host", + }, + { + id: "paste-into-browser", + label: "Paste", + kind: "paste", + scope: "host", + }, +] satisfies readonly PinnedHarnessAction[]; + +const pinnedHarnessPrompts: Record = { + screenshot: "Capture the current browser view.", + copy: "Copy the current browser selection.", + paste: "Paste the clipboard into the focused browser field.", +}; + function usesCompactLivePane() { return ( typeof window !== "undefined" && @@ -148,6 +183,7 @@ export function MobileSplitScreen({ const [draft, setDraft] = useState(""); const [gridOpen, setGridOpen] = useState(false); const [viewportOpen, setViewportOpen] = useState(false); + const [adminOpen, setAdminOpen] = useState(false); const [viewportSaved, setViewportSaved] = useState(false); const [viewportSaveFailed, setViewportSaveFailed] = useState(false); const [fullscreenOpen, setFullscreenOpen] = useState(false); @@ -216,11 +252,14 @@ export function MobileSplitScreen({ : harnessUnavailable ? "Codex Computer Use Bridge unavailable" : "Ask Codex Computer Use..."; + const compactWorkspace = chatCollapsed && !remoteToolsOpen && !fullscreenOpen; + const toolPanelOpen = viewportOpen || gridOpen || adminOpen; const closeTools = () => { onRemoteToolsOpenChange(false); setViewportOpen(false); setGridOpen(false); + setAdminOpen(false); }; const openTools = () => { @@ -347,6 +386,7 @@ export function MobileSplitScreen({ openTools(); setGridOpen((open) => !open); setViewportOpen(false); + setAdminOpen(false); return; } if (key === "k") { @@ -379,9 +419,7 @@ export function MobileSplitScreen({ setFullscreenViewportOpen(false); }; - const sendMessage = async (event: FormEvent) => { - event.preventDefault(); - const text = draft.trim(); + const runHarnessTask = async (text: string, commands?: readonly TaskHarnessAction[]) => { if (!text || harnessPending || !harnessReady) return; const userMessage: ChatMessage = { id: Date.now(), role: "user", text }; chatAdjustedRef.current = true; @@ -397,11 +435,13 @@ export function MobileSplitScreen({ try { const reply = await (taskHarnessRef.current ?? createTaskHarness(window)).send({ text, + ...(commands ? { commands } : {}), profile_id: selected?.id ?? null, metadata: { runner: "codex-computer-use", preferred_surface: "codex-computer-use", browser_visible: true, + ...(commands ? { source: "pinned-action" } : {}), }, }); setMessages((current) => [ @@ -428,6 +468,19 @@ export function MobileSplitScreen({ } }; + const sendMessage = async (event: FormEvent) => { + event.preventDefault(); + const text = draft.trim(); + if (!text || harnessPending || !harnessReady) return; + setDraft(""); + await runHarnessTask(text); + }; + + const runPinnedHarnessAction = async (action: PinnedHarnessAction) => { + if (!harnessCapabilities?.browser_actions.includes(action.kind)) return; + await runHarnessTask(pinnedHarnessPrompts[action.kind], [action]); + }; + const updateDraft = (event: ChangeEvent) => { setDraft(event.target.value); event.target.style.height = "auto"; @@ -491,15 +544,10 @@ export function MobileSplitScreen({ Save the next browser viewport without leaving the live viewer.

) : null} - {!fullscreen && !isLiveBrowser ? ( + {!fullscreen ? ( <> {renderLiveViewControls()} -
-

Preview controls update this viewer immediately.

- -
+

Pane and zoom update this viewer immediately.

) : null} {canManageProfiles ? ( @@ -786,7 +834,7 @@ export function MobileSplitScreen({ ); return ( -
+
- {isLiveBrowser && fullscreenOpen ? renderFullscreenControls() : null} + {fullscreenOpen ? renderFullscreenControls() : null} {renderBrowserSurface()}
@@ -897,85 +945,135 @@ export function MobileSplitScreen({ {remoteToolsOpen ? (
-
- {canOperate && selected?.status === "running" ? ( - - ) : canOperate ? ( - - ) : null} - {canManageProfiles ? ( - - ) : null} - {canManageProfiles && selected ? ( - - ) : null} - {canManageAccess ? ( - - ) : null} -
+ {!toolPanelOpen && canOperate ? ( +
+ {selected?.status === "running" ? ( + + ) : ( + + )} +
+ ) : null} -
+ {!toolPanelOpen ? ( +
+
+ Actions + {harnessReady ? "Codex ready" : "Codex unavailable"} +
+
+ {pinnedHarnessActions.map((action) => { + const available = Boolean( + harnessReady && harnessCapabilities?.browser_actions.includes(action.kind), + ); + return ( + + ); + })} +
+
+ ) : null} - {isLiveBrowser ? renderLiveViewControls() : null} +
- {canManageProfiles ? ( - - ) : null} + - + {canManageProfiles || canManageAccess ? ( + + ) : null}
- {canManageProfiles && viewportOpen ? renderViewportEditor("inline") : null} + {viewportOpen ? renderViewportEditor("inline") : null} + + {adminOpen ? ( +
+ {canManageProfiles ? ( + + ) : null} + {canManageProfiles && selected ? ( + + ) : null} + {canManageAccess ? ( + + ) : null} +
+ ) : null} {gridOpen ? (
diff --git a/frontend/src/lib/taskHarness.test.ts b/frontend/src/lib/taskHarness.test.ts index 6cf0c690..73146ff0 100644 --- a/frontend/src/lib/taskHarness.test.ts +++ b/frontend/src/lib/taskHarness.test.ts @@ -3,6 +3,7 @@ import { createTaskHarness, codexComputerUseProvider, type InjectedTaskHarness, + type TaskHarnessCapabilities, type TaskHarnessListener, type TaskHarnessMessage, } from "./taskHarness"; @@ -104,6 +105,24 @@ describe("createTaskHarness", () => { expect(unsubscribe).toHaveBeenCalledOnce(); }); + it("drops unknown browser actions at the injected host boundary", async () => { + const capabilities = { + chat: true, + streaming: false, + clipboard: false, + browser_actions: ["screenshot", "delete_everything"], + metadata: { provider: codexComputerUseProvider }, + } as unknown as TaskHarnessCapabilities; + const harness = createTaskHarness(windowWithHarness({ + capabilities, + send: vi.fn(), + })); + + await expect(harness.capabilities()).resolves.toMatchObject({ + browser_actions: ["screenshot"], + }); + }); + it("marks the bridge unavailable when an injected object cannot send", async () => { const harness = createTaskHarness(windowWithHarness({ capabilities: { chat: false } })); diff --git a/frontend/src/lib/taskHarness.ts b/frontend/src/lib/taskHarness.ts index 643c2a78..8103d805 100644 --- a/frontend/src/lib/taskHarness.ts +++ b/frontend/src/lib/taskHarness.ts @@ -8,8 +8,39 @@ export interface TaskHarnessMessage { metadata?: Record; } +export const taskHarnessActionKinds = [ + "navigate", + "click", + "double_click", + "scroll", + "type_text", + "keypress", + "drag", + "move", + "wait", + "copy", + "paste", + "screenshot", + "viewport", + "fullscreen", + "focus_remote", + "focus_chat", +] as const; + +export type TaskHarnessActionKind = (typeof taskHarnessActionKinds)[number]; +export type TaskHarnessActionScope = "ui" | "host"; + +export interface TaskHarnessAction { + id: string; + label: string; + kind: TaskHarnessActionKind; + scope: TaskHarnessActionScope; + args?: Record; +} + export interface TaskHarnessRequest { text: string; + commands?: readonly TaskHarnessAction[]; profile_id?: string | null; conversation_id?: string | null; metadata?: Record; @@ -23,7 +54,7 @@ export interface TaskHarnessCapabilities { chat: boolean; streaming: boolean; clipboard: boolean; - browser_actions: string[]; + browser_actions: TaskHarnessActionKind[]; metadata?: Record; } @@ -68,13 +99,19 @@ const unavailableCapabilities: TaskHarnessCapabilities = { metadata: { mode: "unavailable" }, }; +const taskHarnessActionKindSet = new Set(taskHarnessActionKinds); + +function isTaskHarnessActionKind(value: unknown): value is TaskHarnessActionKind { + return typeof value === "string" && taskHarnessActionKindSet.has(value); +} + function cloneCapabilities(capabilities: TaskHarnessCapabilities): TaskHarnessCapabilities { return { chat: Boolean(capabilities.chat), streaming: Boolean(capabilities.streaming), clipboard: Boolean(capabilities.clipboard), browser_actions: Array.isArray(capabilities.browser_actions) - ? [...capabilities.browser_actions] + ? capabilities.browser_actions.filter(isTaskHarnessActionKind) : [], metadata: capabilities.metadata ? { ...capabilities.metadata } : undefined, }; diff --git a/frontend/src/styles/globals.css b/frontend/src/styles/globals.css index 16b9b8b2..e427aab1 100644 --- a/frontend/src/styles/globals.css +++ b/frontend/src/styles/globals.css @@ -74,7 +74,7 @@ .mobile-live-pane { @apply flex w-full min-w-0 shrink-0 flex-col border-b border-border bg-surface-1; - flex-basis: var(--mobile-live-pane-basis, 42%); + flex-basis: var(--mobile-live-pane-basis, 68%); min-height: 16rem; max-height: 82dvh; } @@ -121,6 +121,16 @@ overscroll-behavior: contain; } + .mobile-workspace-collapsed .mobile-live-pane:not(.mobile-live-pane-fullscreen) { + flex: 1 1 auto; + min-height: 0; + max-height: none; + } + + .mobile-workspace-collapsed .mobile-control-pane { + flex: 0 0 auto; + } + .mobile-browser-frame { @apply flex h-full min-h-48 flex-col overflow-hidden rounded-md border border-border bg-black; touch-action: manipulation; @@ -493,6 +503,26 @@ grid-template-columns: repeat(auto-fit, minmax(5.5rem, 1fr)); } + .mobile-tools-row-primary { + grid-template-columns: minmax(0, 1fr); + } + + .mobile-tool-section { + @apply grid gap-1; + } + + .mobile-tool-section-header { + @apply flex items-center justify-between gap-2 px-0.5 text-[10px] font-semibold uppercase tracking-wide text-gray-500; + } + + .mobile-pinned-actions { + grid-template-columns: repeat(3, minmax(0, 1fr)); + } + + .mobile-admin-tools { + @apply border-t border-border pt-2; + } + .mobile-tool-action { @apply inline-flex min-h-11 min-w-0 items-center justify-center gap-1 rounded-md border border-border bg-surface-1 px-2 text-xs font-medium text-gray-200 transition-colors hover:bg-surface-2 disabled:cursor-not-allowed disabled:text-gray-600 focus:outline-none focus:ring-2 focus:ring-accent/50; } @@ -571,6 +601,16 @@ flex: 1 1 42%; } + .mobile-workspace-collapsed .mobile-live-pane:not(.mobile-live-pane-fullscreen) { + flex: 1 1 auto; + max-width: none; + } + + .mobile-workspace-collapsed .mobile-control-pane { + width: clamp(15rem, 32vw, 18rem); + flex: 0 0 clamp(15rem, 32vw, 18rem); + } + .mobile-toolbar, .mobile-chat-form { padding-top: 0.5rem; diff --git a/scripts/mobile_ui_gate.py b/scripts/mobile_ui_gate.py index affdbe4d..46fca41c 100755 --- a/scripts/mobile_ui_gate.py +++ b/scripts/mobile_ui_gate.py @@ -93,13 +93,14 @@ def mobile_gate_init_script() -> str: chat: true, streaming: true, clipboard: true, - browser_actions: ['copy', 'paste', 'fullscreen'], + browser_actions: ['copy', 'paste', 'screenshot', 'fullscreen'], metadata: {{ mode: 'codex-computer-use-mobile-gate', provider: 'codex-computer-use', }}, }}, send: async (request) => {{ + window.__codexComputerUseLastRequest = request; const text = String(request?.text ?? ''); return {{ id: `mobile-ui-gate-${{Date.now()}}`, @@ -333,6 +334,7 @@ def evaluate(self, expression: str, *, return_by_value: bool = True) -> Any: const commandDock = document.querySelector('.mobile-command-dock'); const composerForm = document.querySelector('.mobile-chat-form'); const composer = document.querySelector('#mobile-task-input'); + const pinnedActions = document.querySelector('[aria-label="Pinned browser actions"]'); const required = [root, live, controls, frame, commandDock, composerForm, composer]; const rect = (node) => node ? node.getBoundingClientRect().toJSON() : null; const visible = (node) => { @@ -365,6 +367,7 @@ def evaluate(self, expression: str, *, return_by_value: bool = True) -> Any: chat: rect(chat), chatHeader: rect(chatHeader), chatCollapsed: !chat && !visible(chatHeader), + compactWorkspace: root?.classList.contains('mobile-workspace-collapsed') ?? false, chatVisibleHeight: Math.round(visibleHeight(chat) * 10) / 10, chatHeaderVisible: fullyVisible(chatHeader), toolsVisible: visible(tools), @@ -378,6 +381,7 @@ def evaluate(self, expression: str, *, return_by_value: bool = True) -> Any: hasBrowserTools: !!document.querySelector('[aria-label="Browser tools"]'), hasAgentRunner: !!document.querySelector('select[aria-label="Select harness runner"]'), hasRunTask: !!document.querySelector('button[aria-label="Run task"]'), + pinnedActionCount: pinnedActions?.querySelectorAll('button')?.length ?? 0, benchmarkNavAbsent: !document.body.innerText.includes('Benchmarks') && !document.querySelector('button[aria-label="Streaming benchmark results"]'), }; @@ -726,14 +730,6 @@ def verify_live_viewport_controls( bool(editor.get("visible")) and editor.get("inputCount") == 2 and bool(editor.get("apply")), editor, ) - click_visible(browser, "button[aria-label='Edit browser viewport']", "live viewport editor toggle") - browser.wait_for( - "!document.querySelector('[aria-label=\"Viewport controls\"]')", - "compact workspace after live profile viewport settings", - 5, - ) - - ensure_browser_tools_open(browser) live_controls_opened = browser.eval( "!!document.querySelector('#mobile-pane-size') && !!document.querySelector('#mobile-browser-zoom')" ) @@ -1341,6 +1337,7 @@ def run_viewport( and bool(structure.get("hasRunTask")) and (structure.get("primaryActionCount") or 0) <= 4 and bool(structure.get("chatCollapsed")) + and bool(structure.get("compactWorkspace")) and bool(structure.get("benchmarkNavAbsent")) and not bool(structure.get("hasAgentRunner")), structure, @@ -1360,8 +1357,10 @@ def run_viewport( tools_structure = browser.eval(STRUCTURE_SCRIPT) add_check( result, - "browser tools omit visible harness picker", - bool(tools_structure.get("toolsVisible")) and not bool(tools_structure.get("hasAgentRunner")), + "browser tools use three pinned Codex actions without a harness picker", + bool(tools_structure.get("toolsVisible")) + and tools_structure.get("pinnedActionCount") == 3 + and not bool(tools_structure.get("hasAgentRunner")), tools_structure, ) browser.eval(r"""(() => { @@ -1554,16 +1553,17 @@ def run_viewport( const input = document.querySelector('#mobile-task-input'); const host = window.cloakBrowserHarness; const hasHost = !!host && typeof host.send === 'function'; - const label = [...document.querySelectorAll('.mobile-chat-form, .mobile-tools-sheet, .mobile-control-pane')] - .map((node) => node.innerText || '') - .join('\n'); - const connected = label.includes('Codex Computer Use Bridge · connected'); - if (!hasHost || !input || input.disabled || !connected || label.includes('unavailable')) return false; + const capture = document.querySelector( + 'button[aria-label="Run Capture with Codex Computer Use"]' + ); + const connected = !!capture && !capture.disabled; + if (!hasHost || !input || input.disabled || !connected) return false; return { hasHost, inputDisabled: input.disabled, placeholder: input.getAttribute('placeholder'), connected, + captureDisabled: capture.disabled, }; })()""", "Codex Computer Use test host harness", @@ -1575,6 +1575,24 @@ def run_viewport( bool(harness_ready), harness_ready, ) + pinned_action = browser.eval(r"""(() => { + const button = document.querySelector('button[aria-label="Run Capture with Codex Computer Use"]'); + if (!button || button.disabled) return false; + button.click(); + return true; + })()""") + pinned_request = browser.wait_for( + "window.__codexComputerUseLastRequest?.commands?.[0]?.kind === 'screenshot' && " + "window.__codexComputerUseLastRequest?.metadata?.source === 'pinned-action'", + "structured Codex pinned action", + 10, + ) + add_check( + result, + "pinned screenshot action uses structured Codex Computer Use command", + bool(pinned_action) and bool(pinned_request), + {"clicked": pinned_action, "requestObserved": bool(pinned_request)}, + ) unique_message = f"Mobile gate {name} {int(time.time() * 1000)}" expected_reply = codex_computer_use_test_reply(unique_message) browser.run("fill", "#mobile-task-input", unique_message) @@ -1831,6 +1849,11 @@ def run_access_dashboard_gate( authenticate_workspace(browser, auth_token) browser.wait_for("!!document.querySelector('.mobile-split-root')", "mobile workspace", 20) ensure_browser_tools_open(browser) + browser.eval(r"""(() => { + const button = document.querySelector('button[aria-label="Toggle browser administration"]'); + if (button?.getAttribute('aria-expanded') !== 'true') button?.click(); + return Boolean(button); + })()""") clicked = browser.eval(r"""(() => { const button = document.querySelector('button[aria-label="Browser access controls"]'); if (!button) return false; From 3dd7e58c29d7b5a4326a9d5a8062f74c47fe8e22 Mon Sep 17 00:00:00 2001 From: Martin Hausleitner <55828102+Martin-Hausleitner@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:05:03 +0200 Subject: [PATCH 02/32] feat: tighten mobile browser workspace ux --- README.md | 8 +- docs/MOBILE-E2E-VALIDATION.md | 6 +- ...STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md | 48 ++++---- docs/MOBILE-UI-UX-HARNESS-AUDIT-2026-07-21.md | 104 ++++++++++++++++++ ...PERCLIP-BROWSER-ACCESS-CONTROL-PROPOSAL.md | 8 +- docs/REMOTE-STREAMING-BENCHMARK.md | 30 ++++- ...ERCLIP-BROWSER-ACCESS-GITHUB-DISCUSSION.md | 2 +- .../mobile/MobileSplitScreen.test.tsx | 7 +- .../components/mobile/MobileSplitScreen.tsx | 81 +++++++------- frontend/src/styles/globals.css | 27 ++--- scripts/mobile_ui_gate.py | 41 +++++++ 11 files changed, 264 insertions(+), 98 deletions(-) create mode 100644 docs/MOBILE-UI-UX-HARNESS-AUDIT-2026-07-21.md diff --git a/README.md b/README.md index cabd32b7..533d6641 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Each CloakBrowser profile generates a completely different device identity. To t - **One-click launch/stop** — each profile runs as an isolated CloakBrowser instance - **Session persistence** — cookies, localStorage, and cache survive browser restarts - **In-browser viewing** — interact with launched browsers via noVNC, directly in the web GUI -- **Mobile task workspace** — browser-first live-VNC split with collapsed chat, central tools, fullscreen shortcuts, editable viewport, visual zoom and an injected Codex Computer Use host bridge +- **Mobile task workspace** — browser-first live-VNC split with collapsed chat, one compact tool sheet, typed Quick actions, fullscreen shortcuts, editable viewport, visual zoom and an injected Codex Computer Use host bridge - **Playwright/Puppeteer API** — connect to any running profile programmatically via CDP, while still watching it live in the browser - **Optional scoped access** — protect the web UI with a bootstrap token, then give people and Paperclip agents only the browser sandboxes they need - **Powered by CloakBrowser** — 32 source-level C++ patches, passes Cloudflare Turnstile, 0.9 reCAPTCHA v3 score @@ -97,9 +97,9 @@ The runner writes JSON plus Markdown reports and separates `measured` candidates The [redacted 20-run report](docs/streaming-benchmark-latest.md) remains the warm r49 product-path baseline: the Manager health endpoint reached a median first byte at **1.438 ms** (p95 **4.005 ms**), and the real KasmVNC/noVNC WebSocket upgrade reached a median **3.457 ms** handshake (p95 **8.931 ms**). That historical report listed Selkies as `not_installed` and Sunshine/Moonlight plus Guacamole as `architecture_only`. -The new pinned r51 Selkies lane is now independently reproducible. In the fresh five-run loopback check, its HTTP shell returned **5/5** successful samples with **2.672 ms median total time** (p95 **3.515 ms**), and `/websockets` returned **5/5** valid upgrades with **2.190 ms median handshake** (p95 **3.722 ms**). This proves transport readiness only: it is not a first-frame, FPS, touch-to-pixel, authenticated product-flow, or mobile-Safari comparison. KasmVNC/noVNC remains the production recommendation because it is still the only candidate integrated and verified through the complete profile, policy, VNC-canvas and mobile-UI path. Full context and limits are in [docs/REMOTE-STREAMING-BENCHMARK.md](docs/REMOTE-STREAMING-BENCHMARK.md). +The fresh r55 comparison provisioned Selkies and Apache Guacamole beside the current product path. Twenty-run shell checks found a **0.756 ms** median Selkies WebSocket upgrade and **1.749 ms** median Guacamole HTTP total time. A separate five-run mobile browser observation measured the first non-black frame at **180 ms median** for the integrated KasmVNC/noVNC product path, **351 ms** for Guacamole and **5,272 ms** for Selkies after four reloads stalled near 5.3 seconds (its first run was 296 ms). These are directional local observations, not interchangeable transport metrics: only KasmVNC/noVNC includes the complete profile, policy, VNC-canvas and mobile-UI chain, and no FPS or WAN touch-to-pixel value is claimed. Full context and limits are in [docs/REMOTE-STREAMING-BENCHMARK.md](docs/REMOTE-STREAMING-BENCHMARK.md). -The current r55 mobile implementation is browser-first: chat starts collapsed, the live browser consumes all unused space, benchmark controls are not shown in the UI, and only Full, Tools, Chat and Send remain persistent. Tools use progressive disclosure for View, Sessions and Admin. The bookmark-like Actions row is not a list of URLs; it sends typed Capture, Copy and Paste commands through a host bridge explicitly identifying itself as `codex-computer-use`. Unknown command kinds are dropped at the bridge boundary, while a missing, generic or mislabeled harness fails closed. The fresh live r55 gate passed **276/276 checks** across five mobile viewports and produced **22 screenshots**. The earlier authenticated r51 run additionally covered the access dashboard: Codex Computer Use logged in as a scoped viewer, reached a real connected VNC canvas, confirmed exactly four persistent actions and found no horizontal overflow at `390 x 844`; an administrator then combined `operate` with independent CDP `automate` access for a Paperclip agent. The VCVM/Neko Tailnet check remains transport evidence only: Codex Computer Use completed the protected login and observed `/ws`, but WebRTC ICE failed, so no honest FPS value was recorded. Full details and limits are in [docs/MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md](docs/MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md). +The current r56 mobile implementation is browser-first: chat starts collapsed, the live browser consumes all unused space, benchmark controls are absent, and only Full, Tools, Chat and Send remain persistent. Tools uses progressive disclosure for Quick actions, View, Sessions, Admin and account controls. Quick actions are not URL bookmarks; Capture, Copy and Paste are typed, host-scoped commands enabled only when a verified `codex-computer-use` bridge reports the matching capability. Unknown command kinds are dropped at the boundary, while a missing, generic or mislabeled harness fails closed. The fresh authenticated r56 gate passed **291/291 checks** across five mobile viewports plus the access dashboard and produced **23 screenshots**. It includes a real connected VNC canvas, one-row composer, 44-pixel touch targets, no horizontal overflow, account controls only inside Tools, fullscreen/viewport/zoom, clipboard/paste and scoped-access checks. The complete UI/UX rationale and ten next input improvements are in [docs/MOBILE-UI-UX-HARNESS-AUDIT-2026-07-21.md](docs/MOBILE-UI-UX-HARNESS-AUDIT-2026-07-21.md); streaming, auth and Tailnet limits remain documented in [docs/MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md](docs/MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md). ## Development @@ -224,7 +224,7 @@ environment: 3. Create a Paperclip agent identity, choose its inherited browser-control tier, independently enable CDP automation when needed, and copy its generated bearer key once into the agent's secret store. 4. Rotate or deactivate a person or agent immediately when access changes. -The access-control implementation and backend test matrix cover profile discovery, VNC, clipboard, launch/stop, CDP HTTP and CDP WebSockets. The r51 dashboard can persist two complementary grants on one sandbox, such as `operate + automate`, and its effective-access disclosure lists the profiles and resulting capabilities before a key is used. The authenticated r51 acceptance run proved that such an agent saw only its `beta` sandbox, reached its CDP endpoint with HTTP 200, passed lifecycle policy to the already-running response, and received HTTP 404 for `alpha`. Denied REST and WebSocket policy decisions are recorded as metadata-only audit events without credentials or browser content. The full suite currently passes **223 backend tests** and **75 frontend tests**. A profile outside a caller's scope still returns the same `404` response as a missing profile. The dashboard is a convenience layer; it is not the security boundary. +The access-control implementation and backend test matrix cover profile discovery, VNC, clipboard, launch/stop, CDP HTTP and CDP WebSockets. The dashboard can persist two complementary grants on one sandbox, such as `operate + automate`, and its effective-access disclosure lists the profiles and resulting capabilities before a key is used. The authenticated r56 acceptance run proved that such an agent saw only its assigned sandbox, reached scoped CDP, passed lifecycle policy and received HTTP 404 for a profile outside its scope; rotating the key invalidated the old key with HTTP 401. Denied REST and WebSocket policy decisions are recorded as metadata-only audit events without credentials or browser content. The full suite currently passes **223 backend tests** and **79 frontend tests**. A profile outside a caller's scope still returns the same `404` response as a missing profile. The dashboard is a convenience layer; it is not the security boundary. | Grant | What it allows | | --- | --- | diff --git a/docs/MOBILE-E2E-VALIDATION.md b/docs/MOBILE-E2E-VALIDATION.md index 0dcea34b..44cc1265 100644 --- a/docs/MOBILE-E2E-VALIDATION.md +++ b/docs/MOBILE-E2E-VALIDATION.md @@ -2,9 +2,9 @@ Stand: 21. Juli 2026 -## Aktueller r51-Nachweis +## Aktueller r56-Nachweis -Der aktuelle authentifizierte Release-Gate umfasst fünf Mobile-Viewports plus Access-Dashboard und bestand **276/276 Checks** mit **23 Screenshots**. Der Composer akzeptiert ausschließlich eine Host-Bridge mit `provider: codex-computer-use`; fehlende oder generische Harnesses bleiben deaktiviert. Ein separater Codex-Computer-Use-Lauf bediente den echten verbundenen VNC-Canvas, den scoped Viewer-Login und die kombinierte `operate + automate`-Vergabe im Access-Dashboard. Details, Grenzen und die aktuellen Performancewerte stehen im [Mobile-/Auth-/Latenz-Audit](MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md). +Der aktuelle authentifizierte Release-Gate umfasst fünf Mobile-Viewports plus Access-Dashboard und bestand **291/291 Checks** mit **23 Screenshots**. Der Composer akzeptiert ausschließlich eine Host-Bridge mit `provider: codex-computer-use`; fehlende oder generische Harnesses bleiben deaktiviert. Der Gate prüft zusätzlich den einzeiligen sichtbaren Composer und dass Kontoaktionen nur hinter Tools liegen. Ein separater Codex-Computer-Use-Lauf bediente den echten verbundenen VNC-Canvas, den scoped Viewer-Login und die kombinierte `operate + automate`-Vergabe im Access-Dashboard. Details, Grenzen und die aktuellen Performancewerte stehen im [Mobile-/Auth-/Latenz-Audit](MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md); die kompakte Informationsarchitektur und universellen Quick actions sind im [Mobile UI/UX and universal browser-action audit](MOBILE-UI-UX-HARNESS-AUDIT-2026-07-21.md) dokumentiert. Die folgenden Abschnitte bewahren ältere, enger abgegrenzte Läufe als Regressionsevidenz. @@ -24,7 +24,7 @@ Die Dauerleisten wurden entfernt: Workspace-Aktionen, Pane-/Zoom-Regler und selt Der Vollbildmodus startet mit drei klaren 44-px-Aktionen (View, Viewport, Exit). Der Viewport-Dialog bietet im Vollbild Phone-fit und Presets, editierbare Breite/Höhe sowie `Apply`, ohne den VNC-Viewer verlassen zu müssen. Der Gate prüft dazu genau einen verbundenen Canvas, RFB-Remote-Eingabe mit CDP-Bestätigung, Ratio und Canvas-Zoom, Grid, den iOS-Paste-Fallback, Vollbild-Viewport-Persistenz, Fokus-Rückgabe, fehlenden horizontalen Overflow und alle sichtbaren Touch-Ziele. Es entstanden 22 lokale PNG-Artefakte; repräsentative iPhone-SE-, Vollbild- und Vollbild-Viewport-Screenshots wurden visuell kontrolliert. -Die nachfolgenden Abschnitte bewahren weitere ältere, enger abgegrenzte Läufe als Vergleichs- und Fehlerhistorie. Sie ersetzen nicht den r51-Nachweis oben. +Die nachfolgenden Abschnitte bewahren weitere ältere, enger abgegrenzte Läufe als Vergleichs- und Fehlerhistorie. Sie ersetzen nicht den r56-Nachweis oben. ## Historischer Kompatibilitäts-Wiederholungslauf diff --git a/docs/MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md b/docs/MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md index 2e466a5a..e3b636b3 100644 --- a/docs/MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md +++ b/docs/MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md @@ -2,35 +2,36 @@ Stand: 21. Juli 2026 -Dieser Audit bewertet den final geprüften r51-Stand des CloakBrowser Managers und ordnet die älteren r49-/r50-Basen ein. Er trennt nachgewiesene Funktion, lokale Messung, Tailnet-Transport und noch offene Produktbehauptung. Browser-Use diente nur als Interaktionsreferenz; fremde Marken-, Cloud- oder UI-Assets wurden nicht übernommen. +Dieser Audit bewertet den final geprüften r56-Stand des CloakBrowser Managers und ordnet die älteren r49-/r50-Basen ein. Er trennt nachgewiesene Funktion, lokale Messung, Tailnet-Transport und noch offene Produktbehauptung. Browser-Use diente nur als Interaktionsreferenz; fremde Marken-, Cloud- oder UI-Assets wurden nicht übernommen. ## Ergebnis in einem Satz -Für das mobile Web-MVP bleibt **KasmVNC 1.3.3 + noVNC 1.4.x** die am besten nachgewiesene Basis: Der authentifizierte r51-Gate bestand fünf mobile Viewpoints plus Access-Dashboard mit **276/276 Checks** und **23 Screenshots**, während die ältere r49-Auth-Suite vier Authentifizierungswege bis zu einem echten verbundenen Canvas belegte. Der warme lokale KasmVNC-WebSocket-Upgrade lag in 20/20 Läufen bei **3,457 ms Median / 8,931 ms p95**. Ein neuer, reproduzierbarer Selkies-Lauf erreichte **2,190 ms Median / 3,722 ms p95** für den isolierten WebSocket-Handshake, aber noch keinen gleichwertigen Produkt-, Frame- oder Eingabe-E2E. Ein zusätzlicher Codex-Computer-Use-Lauf bediente die finale iPhone-Ansicht real und bestätigte Canvas, vier persistente Aktionen, Rechte-Dashboard und den strikt verifizierten Hostvertrag. Das beweist weiterhin keine physische iPhone-, WAN- oder Touch-to-Pixel-Latenz. +Für das mobile Web-MVP bleibt **KasmVNC 1.3.3 + noVNC 1.4.x** die am besten nachgewiesene Basis: Der authentifizierte r56-Gate bestand fünf mobile Viewpoints plus Access-Dashboard mit **291/291 Checks** und **23 Screenshots**, während die ältere r49-Auth-Suite vier Authentifizierungswege bis zu einem echten verbundenen Canvas belegte. Im frischen browserbeobachteten Fünferlauf erreichte der vollständige lokale KasmVNC/noVNC-Produktpfad den ersten nichtschwarzen Frame nach **180 ms Median**, Guacamole nach **351 ms** und Selkies nach **5.272 ms**; der Selkies-Erstlauf lag bei 296 ms, vier Reloads jedoch nahe 5,3 s. Diese Werte sind wegen der unterschiedlichen Integration nur richtungsweisend. Ein zusätzlicher Codex-Computer-Use-Lauf bediente die finale iPhone-Ansicht real und bestätigte Canvas, vier persistente Aktionen, Rechte-Dashboard und den strikt verifizierten Hostvertrag. Das beweist weiterhin keine physische iPhone-, WAN-, FPS- oder Touch-to-Pixel-Latenz. -## r51-Endstand: UI-Architektur und Codex Computer Use +## r56-Endstand: UI-Architektur und Codex Computer Use -Der belegte r51-Stand ist eine Umstrukturierung und vollständige lokale Abnahme der mobilen Oberfläche, nicht ein neuer Performance-Sieg: +Der belegte r56-Stand ist eine Umstrukturierung und vollständige lokale Abnahme der mobilen Oberfläche, nicht ein allgemeiner Performance-Sieg: - **Browser-first:** Der Live-Browser ist die primäre Fläche. Chat startet bei laufendem Browser collapsed, damit VNC nicht von Steuerleisten verdrängt wird. -- **Zentrale Tools:** Browser-Werkzeuge, Viewport, Zoom, Fullscreen und Grid sind in einem zentralen Bedienbereich zusammengeführt. Benchmarks werden nicht in der mobilen UI angezeigt. +- **Zentrale Tools:** Browser-Werkzeuge, Viewport, Zoom, Fullscreen, Sessions und Kontoaktionen sind in einem einzigen Tool-Sheet zusammengeführt. Benchmarks werden nicht in der mobilen UI angezeigt. +- **Quick actions statt Bookmarks:** Capture, Copy und Paste sind typisierte Hostaktionen und keine gespeicherten URLs. Sie werden nur aktiviert, wenn der verifizierte Host die passende Capability meldet. - **Shortcuts:** Chat-Collapse und Fullscreen-Preview sind als schnelle Bedienwege vorgesehen, ohne Touch-only Bedienung zu erzwingen. - **Grid:** Grid bleibt ein kompakter Session-/Profilumschalter und soll nicht still mehrere Streams starten, weil das iPhone-FPS und Akku verfälschen würde. - **Codex-only Hostvertrag:** Der Composer akzeptiert nur eine injizierte Bridge, deren Capabilities explizit `provider: codex-computer-use` melden. Fehlende, generische oder nur umbenannte Harnesses bleiben deaktiviert; auch `send()` kann die Prüfung nicht umgehen. -- **Chat-Anbindung:** Der freigegebene UI- und Testpfad verwendet Codex Computer Use, hält Browser-Credentials außerhalb des Chats und simuliert keinen lokalen Erfolg. Der r51-Stand behauptet noch keine produktive externe Agent-Task-API ohne einen realen Host-Bridge-Prozess. +- **Chat-Anbindung:** Der freigegebene UI- und Testpfad verwendet Codex Computer Use, hält Browser-Credentials außerhalb des Chats und simuliert keinen lokalen Erfolg. Der r56-Stand behauptet noch keine produktive externe Agent-Task-API ohne einen realen Host-Bridge-Prozess. - **Rechte-Dashboard:** Browsersteuerung (`view`/`interact`/`operate`) und CDP-Automation sind getrennte Controls. Dadurch ist `operate + automate` auf derselben Sandbox möglich; eine einklappbare Vorschau zeigt die tatsächlich erreichbaren Profile und effektiven Fähigkeiten. -Der finale r51-Mobile-Gate ist abgeschlossen: fünf Viewports plus Access-Dashboard, **276/276 Checks**, **23 Screenshots**, keine Fehler. Er prüft Touch-Ziele, Fullscreen, Grid, Viewport/Zoom, Chat-Collapse, Clipboard/Paste, eine echte VNC-Verbindung und die authentifizierte Access-Oberfläche. Ein separater Codex-Computer-Use-Lauf bestätigte den finalen Release-Container zusätzlich über echte UI-Interaktionen und vergab einem Paperclip-Agenten kombiniert `operate + automate`. +Der finale r56-Mobile-Gate ist abgeschlossen: fünf Viewports plus Access-Dashboard, **291/291 Checks**, **23 Screenshots**, keine Fehler. Er prüft Touch-Ziele, Fullscreen, Grid, Viewport/Zoom, Chat-Collapse, den einzeiligen Composer, Kontoaktionen nur im Tool-Sheet, Clipboard/Paste, eine echte VNC-Verbindung und die authentifizierte Access-Oberfläche. Ein separater Codex-Computer-Use-Lauf bestätigte den Release-Container zusätzlich über echte UI-Interaktionen und vergab einem Paperclip-Agenten kombiniert `operate + automate`. ## Was jetzt nachgewiesen ist | Bereich | Nachweis | Ergebnis | |---|---|---| -| iPhone 14 Portrait | 390 × 844, echter VNC-Canvas, Split, Grid, Fullscreen, Viewport-Editor und Codex-Computer-Use-Composer | 55/55 | -| iPhone SE Portrait | 375 × 667, kurzer Viewport mit kompaktem Live-Anteil und vollständig erreichbarem Composer | 54/54 | -| iPhone Pro Max Portrait | 430 × 932, inklusive sichtbarem Inline-Viewport-Editor | 56/56 | -| iPhone 14 Landscape | 844 × 390, horizontaler Split und Fullscreen | 53/53 | -| Touch-Tablet | 768 × 1024, coarse pointer, Grid und Fullscreen | 53/53 | +| iPhone 14 Portrait | 390 × 844, echter VNC-Canvas, Split, Grid, Fullscreen, Viewport-Editor und Codex-Computer-Use-Composer | 58/58 | +| iPhone SE Portrait | 375 × 667, kurzer Viewport mit kompaktem Live-Anteil und vollständig erreichbarem Composer | 57/57 | +| iPhone Pro Max Portrait | 430 × 932, inklusive sichtbarem Inline-Viewport-Editor | 59/59 | +| iPhone 14 Landscape | 844 × 390, horizontaler Split und Fullscreen | 56/56 | +| Touch-Tablet | 768 × 1024, coarse pointer, Grid und Fullscreen | 56/56 | | Access-Dashboard | authentifizierter Adminpfad, kompakte Grants und mobile Overflow-Prüfung | 5/5 | | Vision-Artefakte | Empty, Workspace, Grid, Fullscreen, Fullscreen-Viewport und Access-Dashboard | 23 Screenshots | | Legacy-Token | Login, Profilwahl und verbundener Canvas bei 390 × 844 | bestanden | @@ -42,10 +43,9 @@ Der Gate prüfte unter anderem einen echten Canvas, `Connected`, keinen horizont Lokale Belege: -- `artifacts/mobile-ui-gate-r51-codex-access-acceptance-r2/report.json` -- `artifacts/ui-redesign-r51/codex-computer-use-viewer-iphone14.png` -- `artifacts/ui-redesign-r51/codex-computer-use-access-dashboard.png` -- `artifacts/selkies-benchmark/local-r4-independent/streaming-benchmark-report.json` +- `artifacts/mobile-ui-gate-r56-auth-final-r2/report.json` +- `artifacts/selkies-benchmark/r55-local/streaming-benchmark-report.json` +- `artifacts/guacamole-benchmark/r55-local/streaming-benchmark-report.json` - `artifacts/streaming-login-audit-r49/auth-api-summary.json` - `artifacts/streaming-login-audit-r49/auth-ui-summary.json` - `artifacts/streaming-benchmark-r49/streaming-benchmark-report.json` @@ -108,14 +108,14 @@ Zusätzlich wurde ein bereits laufender Neko/Chrome-Stack auf der VCVM über Tai Codex Computer Use führte den geschützten Login erfolgreich aus und sah `/ws`. WebRTC ICE blieb jedoch `checking` und wechselte danach zu `failed`; das Video blieb bei `readyState 0`. Deshalb gibt es aus diesem Lauf **keinen ehrlichen FPS-Wert**. Die wichtigste Performance-Empfehlung ist nicht weiteres UI-Tuning auf Basis erfundener FPS, sondern zuerst direkte Tailscale-Konnektivität und UDP/ICE zu reparieren und danach Frame- sowie Touch-to-Pixel-Messung erneut auszuführen. -### Einordnung der Streaming-Stapel nach dem r51-Nachtest +### Einordnung der Streaming-Stapel nach dem r56-Nachtest -| Stack | r49-Status | Entscheidung | +| Stack | frischer lokaler Stand | Entscheidung | |---|---|---| -| KasmVNC 1.3.3 + noVNC 1.4.x | vollständiger aktueller Produktpfad, 249 Mobile-Checks, vier Loginpfade | beibehalten | -| Selkies | reproduzierbare HTTP-/WebSocket-Bereitschaft, 5/5 + 5/5; kein gleichwertiger Produkt-E2E | weiter messen, noch nicht migrieren | +| KasmVNC 1.3.3 + noVNC 1.4.x | vollständiger aktueller Produktpfad, 291 Mobile-/Access-Checks; erster nichtschwarzer Frame Median 180 ms | beibehalten | +| Selkies | 20/20 HTTP und WebSocket; erster nichtschwarzer Frame Median 5.272 ms nach langsamen Reloads; kein gleichwertiger Produkt-E2E | Reload-/Sessionproblem untersuchen, noch nicht migrieren | | Sunshine/Moonlight | `architecture_only`, keine aktuelle Messung | nicht als Latenzvergleich werten | -| Apache Guacamole | `architecture_only`, keine aktuelle Messung | nicht als Latenzvergleich werten | +| Apache Guacamole 1.6 | 20/20 HTTP; erster nichtschwarzer Frame Median 351 ms; zusätzliche Gateway- und fehlende Policy-/CDP-Integration | kein Web-MVP-Core | Frühere isolierte KasmVNC-1.4- und Selkies-POCs sowie der neue Selkies-Readiness-Lauf sind in `docs/REMOTE-STREAMING-BENCHMARK.md` methodisch getrennt dokumentiert. Einen technologieübergreifenden „Latenz-Sieger“ zu behaupten wäre weiterhin falsch. KasmVNC ist die aktuelle Produktempfehlung, weil nur dieser Pfad vollständig integriert und aktuell end-to-end geprüft ist. @@ -153,9 +153,9 @@ Frühere isolierte KasmVNC-1.4- und Selkies-POCs sowie der neue Selkies-Readines - Der r50-VCVM/Neko-Lauf belegt Tailnet-HTTP und geschützten Login über Codex Computer Use, aber wegen fehlgeschlagenem WebRTC-ICE keine Framerate. - Kein echter Mobilfunk- und kein Touch-to-Pixel-p50/p95-Bericht. - Der Grid-View ist ein schneller Session-/Profilumschalter, kein gleichzeitiges Multi-Canvas-Monitoring. -- Der Chat-Composer akzeptiert in r51 ausschließlich den verifizierten Codex-Computer-Use-Providervertrag, benötigt aber weiterhin einen realen Host-Bridge-Prozess und ist keine eigenständige Vendor-API. -- Selkies ist reproduzierbar provisioniert, aber noch kein gleichwertiges authentifiziertes Produktdeployment; Sunshine/Moonlight und Guacamole bleiben Architekturpfade. +- Der Chat-Composer akzeptiert in r56 ausschließlich den verifizierten Codex-Computer-Use-Providervertrag, benötigt aber weiterhin einen realen Host-Bridge-Prozess und ist keine eigenständige Vendor-API. +- Selkies und Guacamole sind reproduzierbar provisioniert und browserbeobachtet, aber weiterhin keine gleichwertigen authentifizierten Produktdeployments; Sunshine/Moonlight bleibt ein Architekturpfad. ## Freigabeempfehlung -Der r51-Stand ist als **lokales, rollenbasiertes Mobile-Web-MVP** freigabefähig. Für eine externe oder iPhone-spezifische Freigabe fehlen noch Tailscale Serve/HTTPS, ein physisches Safari-Gerät sowie echte Touch-to-Pixel- und Reconnect-Messungen. Die höchste nächste Produktpriorität ist die iOS-IME-Bridge, gefolgt von Keyboard-Zubehörleiste und echter Eingabelatenztelemetrie. +Der r56-Stand ist als **lokales, rollenbasiertes Mobile-Web-MVP** freigabefähig. Für eine externe oder iPhone-spezifische Freigabe fehlen noch Tailscale Serve/HTTPS, ein physisches Safari-Gerät sowie echte Touch-to-Pixel- und Reconnect-Messungen. Die höchste nächste Produktpriorität ist die iOS-IME-Bridge, gefolgt von Keyboard-Zubehörleiste und echter Eingabelatenztelemetrie. Die UI/UX-Details stehen im [Mobile UI/UX and universal browser-action audit](MOBILE-UI-UX-HARNESS-AUDIT-2026-07-21.md). diff --git a/docs/MOBILE-UI-UX-HARNESS-AUDIT-2026-07-21.md b/docs/MOBILE-UI-UX-HARNESS-AUDIT-2026-07-21.md new file mode 100644 index 00000000..a62f5c7b --- /dev/null +++ b/docs/MOBILE-UI-UX-HARNESS-AUDIT-2026-07-21.md @@ -0,0 +1,104 @@ +# Mobile UI/UX and universal browser-action audit + +Status: 21 July 2026 · r56 locally implemented and end-to-end verified + +## Decision + +The mobile workspace uses one dominant live browser, one compact command dock and one composer. Everything else is disclosed on demand. It deliberately does not copy Browser Use branding or proprietary UI assets. The interaction model is informed by Browser Use's documented live preview, streaming messages and follow-up-task flow, Apple's fullscreen and gesture guidance, WCAG 2.2 touch/focus requirements, and OpenAI's screen/mouse/keyboard computer-use loop. + +The default mobile surface contains exactly four persistent actions: **Full**, **Tools**, **Chat** and **Send**. Chat and Tools are mutually exclusive. Chat starts collapsed while a browser is running. Account controls, viewport settings, session switching, profile administration, clipboard tools and typed browser actions live behind Tools. Benchmarks and quality reports stay outside the product UI. + +Primary references: + +- [Browser Use Chat UI tutorial](https://docs.browser-use.com/cloud/tutorials/chat-ui), [live preview](https://docs.browser-use.com/cloud/browser/live-preview) and [follow-up tasks](https://docs.browser-use.com/cloud/agent/follow-up-tasks) +- [OpenAI: Computer-Using Agent](https://openai.com/index/computer-using-agent/) +- [Apple: Going full screen](https://developer.apple.com/design/human-interface-guidelines/going-full-screen) and [Gestures](https://developer.apple.com/design/human-interface-guidelines/gestures) +- [WCAG 2.2](https://www.w3.org/TR/WCAG22/) and [What's New in WCAG 2.2](https://www.w3.org/WAI/standards-guidelines/wcag/new-in-22/) + +## Minimal information architecture + +### 1. Live browser + +- The VNC canvas is the primary surface and consumes all unused height. +- Portrait defaults to a browser-first split; the collapsed workspace allocates up to 82% of the visual height to the live pane. +- Landscape gives the live browser the main horizontal pane and keeps the control pane narrow. +- Fullscreen keeps the same live canvas rather than opening a second stream. +- Visual zoom and pane ratio update immediately. A real profile width/height change is saved for the next browser launch and is labelled accordingly. + +### 2. Command dock + +- **Full** opens the distraction-free live viewer. +- **Tools** opens the only settings/action sheet. +- **Chat** opens or collapses the task history. +- **Send** remains beside the text field in both collapsed and expanded chat states. +- Every visible interactive control is at least 44 by 44 CSS pixels. + +### 3. Progressive disclosure + +Tools contains: + +- Launch/Stop when the signed-in role may operate the profile. +- Three capability-gated Quick actions: Capture, Copy and Paste. +- View, Sessions and role-appropriate Admin disclosures. +- ProfileViewer actions such as CDP, clipboard and the manual iOS paste fallback. +- Signed-in identity and Log out at the bottom of the sheet, never as a permanent footer. + +The Sessions grid is an honest session selector with name, state, platform and resolution. It does not fake live thumbnails or silently start multiple streams. + +## Quick actions are not bookmarks + +Quick actions are reusable, typed browser commands. They do not store destination URLs and they do not become another navigation bar. The current action vocabulary is: + +`navigate`, `click`, `double_click`, `scroll`, `type_text`, `keypress`, `drag`, `move`, `wait`, `copy`, `paste`, `screenshot`, `viewport`, `fullscreen`, `focus_remote`, `focus_chat`. + +Each command carries an ID, label, action kind, `ui` or `host` scope and optional structured arguments. A host reports the exact actions it supports. The UI enables a Quick action only when that capability is present; unknown action kinds are discarded at the bridge boundary. + +This makes the action schema reusable by a harness adapter without binding the UI to a vendor API. For this product build, execution is intentionally stricter: the injected host must identify itself as `codex-computer-use`. Missing, generic or mislabeled bridges fail closed and keep the composer disabled. There is no local fake-success fallback and no browser credential is passed through the chat UI. + +Current trust boundary: the provider identity is an implementation contract, not cryptographic attestation. A stronger deployment should add a signed or session-bound host handshake before broad external exposure. + +## End-to-end evidence + +The authenticated r56 gate ran the production container, selected a real browser profile, connected one live VNC canvas and exercised five device layouts plus the access dashboard. + +| Surface | Result | +|---|---:| +| iPhone 14 portrait, 390 × 844 | 58 checks passed | +| iPhone SE portrait, 375 × 667 | 57 checks passed | +| iPhone Pro Max portrait, 430 × 932 | 59 checks passed | +| iPhone 14 landscape, 844 × 390 | 56 checks passed | +| Touch tablet portrait, 768 × 1024 | 56 checks passed | +| Authenticated access dashboard | 5 checks passed | +| Total | **291/291 checks, 23 screenshots** | + +The gate verifies, among other things: + +- exactly Full, Tools, Chat and Send as the compact primary controls; +- no benchmark navigation or harness picker; +- no horizontal document overflow; +- 44-pixel touch targets and coarse-pointer behavior; +- Tools/Chat mutual exclusion and keyboard shortcuts that do not steal input focus; +- a one-row composer with a visible Send control in collapsed and expanded chat; +- account/logout controls absent from the compact surface and present only inside Tools; +- live zoom, pane ratio, viewport persistence, fullscreen focus/inert behavior and one-canvas preservation; +- real VNC connection, pointer hit-testing, clipboard round-trip and manual iOS paste; +- honest session cards and the authenticated access dashboard. + +Fresh verification also passed 79 frontend tests, 223 backend tests and the production frontend build. The release preview is loopback-only; physical iPhone Safari and private Tailnet HTTPS remain separate release gates. + +## Ten highest-value next input and interaction improvements + +1. **iOS IME bridge.** Use a controlled hidden input for composition events so predictive text, accented characters, emoji and non-Latin keyboards reach the remote browser reliably. +2. **Keyboard accessory strip.** Provide Esc, Tab, Enter, arrows, Backspace and modifier keys in one disclosure that does not cover the canvas. +3. **Direct-touch / trackpad mode.** Make the input model explicit instead of overloading the same gesture with remote click and local scrolling. +4. **Tap-to-control lock.** Require an intentional control mode before remote taps, preventing accidental clicks while the user scrolls the surrounding page. +5. **Paste sheet with preview and queue.** Separate task input from remote-browser paste, show destination state, and report success/failure without recording clipboard contents. +6. **User-defined Quick-action pins.** Let a profile pin typed, capability-checked commands such as Focus chat, Screenshot or Viewport preset; never turn pins into raw script or URL bookmarks. +7. **Risk confirmations.** Require an explicit confirmation for destructive or sensitive host actions, consistent with computer-use safety guidance. +8. **Reconnect input replay protection.** Drop stale pointer/key events across a stream reconnect and visibly restore control state. +9. **Server-backed task sessions.** Persist conversation IDs, task messages, follow-ups and cancellation state so chat history survives refresh without pretending a local component is an agent runtime. +10. **Touch-to-pixel telemetry and physical-device gate.** Measure touch dispatch to observed frame change on a real iPhone over Tailnet HTTPS, alongside reconnect, keyboard and fullscreen tests. + +## Release boundary + +The r56 UI is ready as a compact local mobile-web MVP. It is not yet evidence of a production Codex host bootstrap, physical iPhone Safari compatibility, public-network safety, WAN frame rate or touch-to-pixel latency. Those claims must stay blocked until a real host bridge, private HTTPS and physical-device measurements are present. diff --git a/docs/PAPERCLIP-BROWSER-ACCESS-CONTROL-PROPOSAL.md b/docs/PAPERCLIP-BROWSER-ACCESS-CONTROL-PROPOSAL.md index cd33c237..c24c176d 100644 --- a/docs/PAPERCLIP-BROWSER-ACCESS-CONTROL-PROPOSAL.md +++ b/docs/PAPERCLIP-BROWSER-ACCESS-CONTROL-PROPOSAL.md @@ -1,6 +1,6 @@ # Paperclip-gestützte Browser-Zugriffskontrolle -Stand: 2026-07-21 · Status: r51 lokal implementiert und Ende-zu-Ende geprüft; bereit für Fork-Review +Stand: 2026-07-21 · Status: r56 lokal implementiert und Ende-zu-Ende geprüft; bereit für Fork-Review ## Kurzentscheidung @@ -24,16 +24,16 @@ Die erste, lokale Policy-Schicht ist in diesem Fork umgesetzt. Sie bleibt absich Die Live-Abnahme lief isoliert gegen eine frische lokale Datenbank: zwei Sandboxes (`research`, `finance`), ein `view`-Nutzer und ein `automate`-Agent. Der Nutzer sah nur `research`; direkte `finance`-, Lifecycle- und Admin-Anfragen wurden mit `404` beziehungsweise `403` abgewiesen. Der Agent sah per eigenem Bearer-Key nur `research`; sein alter Key lieferte nach Rotation `401`. -## Frische r51-Browser-E2E-Abnahme (21. Juli 2026) +## Frische r56-Browser-E2E-Abnahme (21. Juli 2026) Die Policy wurde zusätzlich in einem nur auf `127.0.0.1` gebundenen Container mit isolierten Testprofilen und lokalen Wegwerf-Credentials geprüft. Nutzerbereitgestellte Zugangsdaten wurden nicht verwendet oder persistiert. -- Der authentifizierte Mobile-Gate prüfte fünf Viewports plus Access-Dashboard und bestand **276/276 Checks** mit **23 Screenshots**. +- Der authentifizierte Mobile-Gate prüfte fünf Viewports plus Access-Dashboard und bestand **291/291 Checks** mit **23 Screenshots**. Konto/Logout blieb außerhalb des Tool-Sheets verborgen; der Composer blieb in allen Viewports einzeilig und vollständig sichtbar. - Codex Computer Use meldete sich als `view`-Nutzer an, sah ausschließlich das laufende `beta`-Profil, erreichte einen echten verbundenen VNC-Canvas und fand weder Access- noch Launch-/Stop-Aktionen. Die iPhone-14-Ansicht hatte bei 390 px keinen horizontalen Overflow. - Codex Computer Use meldete sich danach als Wegwerf-Admin an und vergab dem Paperclip-Testagenten im echten Dashboard kombiniert `operate + automate`. Die effektive Vorschau zeigte genau das `beta`-Profil mit `Operate + CDP automation`. - Derselbe Agent sah per rotiertem Wegwerf-Key genau ein `beta`-Profil. Ein Lifecycle-Aufruf erreichte die erlaubte „bereits laufend“-Antwort, CDP lieferte HTTP 200 und ein direkter `alpha`-Aufruf HTTP 404. - Abgelehnte REST- und VNC-WebSocket-Entscheidungen werden als `profile.permission.` mit Sandbox-/Profilkennung und `denied` protokolliert, ohne Secrets oder Browserinhalt. -- Die vollständige Backend-Suite lief mit **223 bestanden**; die Frontend-Suite mit **75 bestanden**, gefolgt von einem erfolgreichen Produktions-Build. +- Die vollständige Backend-Suite lief mit **223 bestanden**; die Frontend-Suite mit **79 bestanden**, gefolgt von einem erfolgreichen Produktions-Build. Diese Abnahme belegt die lokale Produktoberfläche und die serverseitige Entscheidung gemeinsam. Sie ersetzt keine externe Production-Abnahme, veröffentlicht keine Test- oder Produktions-Credentials und enthält keine Browserinhalte. diff --git a/docs/REMOTE-STREAMING-BENCHMARK.md b/docs/REMOTE-STREAMING-BENCHMARK.md index 5c505b07..6ac21e8d 100644 --- a/docs/REMOTE-STREAMING-BENCHMARK.md +++ b/docs/REMOTE-STREAMING-BENCHMARK.md @@ -23,9 +23,9 @@ Für den aktuellen CloakBrowser Manager bleibt **KasmVNC 1.3.3 + noVNC 1.4.x bei | KasmVNC 1.3.3 + noVNC 1.4.x | Vollständiger echter Browserstream und Mobile-Vollbild lokal geprüft | E2E-verifiziert | Beibehalten | | KasmVNC 1.4 | Vollständiger isolierter Browser-, Mobile- und RFB/CDP-E2E-Lauf gegen dieselbe App-Konfiguration bestanden; kein Performancegewinn gegenüber 1.3.3 | E2E-verifiziert, lokale Stichprobe | Nicht migrieren | | noVNC 1.7 | Build scheitert ohne Migration an ESM-/Export-Änderungen und entfernter `showDotCursor`-API | Kompatibilitätsprüfung | Nicht direkt aktualisieren | -| Selkies | Echter CloakBrowser auf X11, JPEG- und H.264-Frame im Browser sowie Remote-Eingabe lokal nachgewiesen; keine Produktintegration und kein brauchbares mobiles Raw-UI | Isolierter Browser-E2E-POC | Nicht als Drop-in ersetzen | +| Selkies | Echter Browserstream und Eingabe lokal nachgewiesen; r55 zusätzlich 20 HTTP-/WebSocket- und fünf mobile First-frame-Beobachtungen; keine Produktintegration und instabile Reload-Zeit | Isolierter Browser-E2E-POC | Nicht als Drop-in ersetzen | | Sunshine + Moonlight | Architektur und Clientmodell geprüft; kein passender allgemeiner Web-Embed-Pfad | Architekturprüfung | Kein Web-MVP-Core | -| Apache Guacamole | Browser-/Touch-Unterstützung und Gateway-Architektur geprüft; bestehender KasmVNC-Server hat bewusst keinen Raw-VNC-TCP-Port | Architektur-/Integrationsprüfung | Kein Web-MVP-Core | +| Apache Guacamole | Version 1.6 lokal provisioniert; 20 HTTP-Läufe und fünf mobile First-frame-Beobachtungen bestanden, aber zusätzliche Gateway-, Policy- und CDP-Integration fehlt | Isolierter Gateway-E2E | Kein Web-MVP-Core | | Browser-Use Chat UI | Interaktions- und Informationsarchitektur geprüft | UI-Referenz | Als UX-Referenz, nicht als Streamingstack | ## Upstream-Snapshot @@ -105,6 +105,26 @@ Der unabhängig wiederholte Lauf vom 21. Juli 2026 ergab: Damit ist die lokale Transport-Provisionierung reproduzierbar. Der Lauf misst jedoch weder Profilstart noch ersten nichtschwarzen Frame, Bildrate, Remote-Eingabe, Rechteintegration, Tailnet noch Mobile Safari. Er ist daher kein Beweis, dass Selkies den vollständigen KasmVNC/noVNC-Produktpfad überholt. Die Produktionsentscheidung bleibt unverändert, bis Selkies dieselben authentifizierten Profil-, Policy-, Canvas-, Mobile- und Eingabe-Gates besteht. +### r55 Selkies-/Guacamole-/Kasm-Browserbeobachtung + +Der r55-Nachtest trennte erneut rohe Shell-/Handshake-Zeiten von sichtbarer Browserbereitschaft. Selkies und Guacamole liefen jeweils isoliert auf Loopback; KasmVNC/noVNC lief als vollständiger Manager-Pfad mit Profilwahl und Mobile-UI. Die Shell-Messungen verwendeten 20 Läufe: + +| Kandidat | Erfolg | Median | p95 | Aussagegrenze | +|---|---:|---:|---:|---| +| Selkies HTTP shell | 20/20 | total **0,708 ms** | **1,621 ms** | HTTP, kein Frame | +| Selkies WebSocket | 20/20 | handshake **0,756 ms** | **1,882 ms** | Upgrade, kein Frame | +| Guacamole 1.6 HTTP shell | 20/20 | total **1,749 ms** | **5,443 ms** | Gateway-HTML, kein Frame | + +Anschließend öffnete Codex Computer Use jedes lokale Mobile-UI fünfmal und markierte den ersten nichtschwarzen sichtbaren Frame. Diese Beobachtung ist näher an der wahrgenommenen Bereitschaft, bleibt aber wegen der unterschiedlichen Produktintegration ein Richtungswert: + +| Stack | fünf Beobachtungen | Median | Einordnung | +|---|---|---:|---| +| KasmVNC/noVNC im Manager | 289 / 230 / 180 / 161 / 157 ms | **180 ms** | vollständiger Profil-, Policy-, VNC- und Mobile-UI-Pfad | +| Apache Guacamole 1.6 | 917 / 363 / 351 / 348 / 350 ms | **351 ms** | rohes Gateway-UI; kein Cloak-Policy-/CDP-Lifecycle | +| Selkies Chromium | 296 / 5.249 / 5.272 / 5.296 / 5.323 ms | **5.272 ms** | schneller Erstlauf, danach reproduzierbare Reload-Verzögerung | + +Der Selkies-WebSocket-Handshake ist damit sehr schnell, aber der sichtbare Reload-Pfad in diesem Setup nicht. Guacamole zeigte einen brauchbaren ersten Frame, bringt jedoch eine zusätzliche Gateway-Schicht und keine vorhandene Manager-Integration mit. KasmVNC/noVNC bleibt die Empfehlung, weil es im gemessenen Gesamtpfad zugleich am schnellsten und vollständig integriert war. Es wurde weiterhin kein FPS-, WAN- oder Touch-to-Pixel-Wert erfunden. + ### r50 VCVM/Neko über Tailscale Am 21. Juli 2026 wurde zusätzlich ein bereits laufender Neko/Chrome-Stack auf der VCVM über Tailscale geprüft. Dieser Lauf ist ein Transport- und Login-Beleg, kein fairer Ersatzbenchmark gegen den CloakBrowser-KasmVNC-Pfad. @@ -199,9 +219,9 @@ python3 scripts/streaming_benchmark_runner.py \ --latest-markdown docs/streaming-benchmark-latest.md ``` -Der Runner unterscheidet absichtlich zwischen `measured`, `not_installed` und `architecture_only`. Eine fehlende lokale Installation oder ein reiner Architekturpfad erhält keine erfundenen Zeiten; ein erreichbarer HTTP-/WebSocket-/Command-Kandidat erhält dagegen rohe Messungen und Median-Min-Max-P95-Zusammenfassungen. Der Report ist als Headless-/Offline-Diagnostik gedacht; r51 zeigt Benchmarks nicht in der mobilen UI. Die öffentliche Projektion enthält bewusst keine lokalen Pfade, Endpunkte, Commands, Header oder Prozessausgaben. +Der Runner unterscheidet absichtlich zwischen `measured`, `not_installed` und `architecture_only`. Eine fehlende lokale Installation oder ein reiner Architekturpfad erhält keine erfundenen Zeiten; ein erreichbarer HTTP-/WebSocket-/Command-Kandidat erhält dagegen rohe Messungen und Median-Min-Max-P95-Zusammenfassungen. Der Report ist als Headless-/Offline-Diagnostik gedacht; r56 zeigt Benchmarks nicht in der mobilen UI. Die öffentliche Projektion enthält bewusst keine lokalen Pfade, Endpunkte, Commands, Header oder Prozessausgaben. -Der Browser-/UI-Gate-Runner liegt unter `scripts/mobile_ui_gate.py`. Er prüft fünf Viewports (iPhone 14, iPhone SE, iPhone Pro Max, iPhone 14 Landscape und Touch-Tablet), Touch-Ziele, Overflow, Split-Geometrie, den verifizierten Codex-Computer-Use-Composer, Grid, Fullscreen-Fokus und – mit einer Profil-ID – einen echten VNC-Canvas. Der authentifizierte r51-Lauf bestand **276/276 Checks** und erzeugte **23 Screenshots**, einschließlich Access-Dashboard. Mit Profil-ID öffnet der Runner außerdem den manuellen iOS-Paste-Fallback, prüft dessen Touch-Ziele und bestätigt den kontrollierten Clipboard-Bridge-Rundlauf, ohne Clipboard-Text im Report zu speichern. Optional tippt `--remote-probe-url` eine harmlose, eindeutige URL per Keyboard-Events durch noVNC/RFB und verifiziert die Zielseite danach über den CDP-Proxy. Seine Screenshotprüfung validiert Abmessungen und Mindestdateigröße; die abschließende semantische Sichtprüfung bleibt bewusst ein separater menschlicher oder Vision-Agent-Gate. +Der Browser-/UI-Gate-Runner liegt unter `scripts/mobile_ui_gate.py`. Er prüft fünf Viewports (iPhone 14, iPhone SE, iPhone Pro Max, iPhone 14 Landscape und Touch-Tablet), Touch-Ziele, Overflow, Split-Geometrie, den verifizierten Codex-Computer-Use-Composer, Grid, Fullscreen-Fokus und – mit einer Profil-ID – einen echten VNC-Canvas. Der authentifizierte r56-Lauf bestand **291/291 Checks** und erzeugte **23 Screenshots**, einschließlich Access-Dashboard. Zusätzlich prüft er den einzeiligen sichtbaren Composer und dass Kontoaktionen nur hinter Tools liegen. Mit Profil-ID öffnet der Runner außerdem den manuellen iOS-Paste-Fallback, prüft dessen Touch-Ziele und bestätigt den kontrollierten Clipboard-Bridge-Rundlauf, ohne Clipboard-Text im Report zu speichern. Optional tippt `--remote-probe-url` eine harmlose, eindeutige URL per Keyboard-Events durch noVNC/RFB und verifiziert die Zielseite danach über den CDP-Proxy. Seine Screenshotprüfung validiert Abmessungen und Mindestdateigröße; die abschließende semantische Sichtprüfung bleibt bewusst ein separater menschlicher oder Vision-Agent-Gate. ## Gepinnte Referenzstände @@ -218,7 +238,7 @@ Die folgenden `HEAD`-Stände wurden am 20. Juli 2026 direkt aus den öffentliche ## Offene Nachweise - Selkies: echter Mobile-Safari-/WebKit-Lauf auf einem physischen iPhone, Authentifizierung und Tailscale-HTTPS sowie ein bewusstes mobiles Client-Layout statt des nachgewiesenen Letterbox-POCs. -- Selkies: WebRTC-Modus unter derselben echten Browser- und Eingabeprobe; der reproduzierbare r51-Lauf misst ausschließlich HTTP-/WebSocket-Bereitschaft. +- Selkies: WebRTC-Modus unter derselben echten Browser- und Eingabeprobe; der r55-Lauf ergänzt WebSocket- und First-frame-Beobachtungen, aber noch keinen gleichwertigen WebRTC-/Policy-Produktpfad. - Physisches iPhone: Safari über freigegebenes Tailscale Serve/HTTPS. Der konkrete Aktivierungsversuch wurde vom Tailnet mit `Serve is not enabled on your tailnet` abgelehnt; ein Administrator muss Serve freigeben, bevor eine ehrliche iPhone-URL und der Safari-E2E-Test möglich sind. - Mehrfachmessung der kompletten Interaktionskette: fünf Start-zu-erster-nichtleerer-Frame- und Eingabeantwort-Läufe je Version, nicht nur API-Launches. diff --git a/docs/drafts/PAPERCLIP-BROWSER-ACCESS-GITHUB-DISCUSSION.md b/docs/drafts/PAPERCLIP-BROWSER-ACCESS-GITHUB-DISCUSSION.md index 93c6764a..df1f88c8 100644 --- a/docs/drafts/PAPERCLIP-BROWSER-ACCESS-GITHUB-DISCUSSION.md +++ b/docs/drafts/PAPERCLIP-BROWSER-ACCESS-GITHUB-DISCUSSION.md @@ -29,7 +29,7 @@ VNC and CDP are direct API/WebSocket surfaces. A frontend-only filter would stil - Rotating an agent key invalidated the old key immediately. - A viewer-only noVNC connection displayed live frames. A real click sent through that viewer canvas did not reach a controlled remote button; keyboard, pointer, and clipboard input are filtered server-side after a validated RFB handshake. - The public container check is a metadata-free `/health` endpoint. Runtime/profile counts in `/api/status` require authentication. -- The authenticated mobile acceptance gate passed 276/276 checks across five viewports plus the access dashboard, with 23 screenshots. A separate Codex Computer Use run exercised both the scoped viewer and admin grant-editing flows at 390 px without horizontal overflow. +- The authenticated mobile acceptance gate passed 291/291 checks across five viewports plus the access dashboard, with 23 screenshots. It also verified that account controls remain behind Tools and that the composer stays in one visible row. A separate Codex Computer Use run exercised both the scoped viewer and admin grant-editing flows at 390 px without horizontal overflow. No production deployment, upstream merge, public URL, live credential, or browser content is included in this draft. diff --git a/frontend/src/components/mobile/MobileSplitScreen.test.tsx b/frontend/src/components/mobile/MobileSplitScreen.test.tsx index 239220a4..4b729dcd 100644 --- a/frontend/src/components/mobile/MobileSplitScreen.test.tsx +++ b/frontend/src/components/mobile/MobileSplitScreen.test.tsx @@ -566,7 +566,12 @@ describe("MobileSplitScreen", () => { it("provides a mobile-sized logout action for authenticated sessions", () => { const { props } = renderMobileSplit({ authRequired: true, identityName: "Scoped viewer" }); - const logout = screen.getByRole("button", { name: "Log out" }); + expect(screen.queryByRole("button", { name: "Log out" })).toBeNull(); + openBrowserTools(); + + const tools = screen.getByLabelText("Browser tools"); + expect(within(tools).getByText("Signed in as Scoped viewer")).toBeTruthy(); + const logout = within(tools).getByRole("button", { name: "Log out" }); expect(logout.className).toContain("mobile-logout-button"); fireEvent.click(logout); expect(props.onLogout).toHaveBeenCalledTimes(1); diff --git a/frontend/src/components/mobile/MobileSplitScreen.tsx b/frontend/src/components/mobile/MobileSplitScreen.tsx index fc83b4ef..955db46f 100644 --- a/frontend/src/components/mobile/MobileSplitScreen.tsx +++ b/frontend/src/components/mobile/MobileSplitScreen.tsx @@ -968,33 +968,33 @@ export function MobileSplitScreen({ {!toolPanelOpen ? (
-
- Actions - {harnessReady ? "Codex ready" : "Codex unavailable"} -
-
- {pinnedHarnessActions.map((action) => { - const available = Boolean( - harnessReady && harnessCapabilities?.browser_actions.includes(action.kind), - ); - return ( - - ); - })} -
+
+ Quick actions + {harnessReady ? "Codex ready" : "Codex unavailable"} +
+
+ {pinnedHarnessActions.map((action) => { + const available = Boolean( + harnessReady && harnessCapabilities?.browser_actions.includes(action.kind), + ); + return ( + + ); + })} +
) : null} @@ -1111,6 +1111,20 @@ export function MobileSplitScreen({ ? "A verified Codex Computer Use Bridge must be injected by the host before tasks can run." : "Tasks run only through the verified Codex Computer Use host; browser credentials stay outside the chat UI."}

+ + {authRequired ? ( +
+ {identityName ? Signed in as {identityName} : } + +
+ ) : null}
) : null} @@ -1178,19 +1192,6 @@ export function MobileSplitScreen({
- {authRequired ? ( -
- {identityName ? Signed in as {identityName} : } - -
- ) : null}
diff --git a/frontend/src/styles/globals.css b/frontend/src/styles/globals.css index e427aab1..341389ee 100644 --- a/frontend/src/styles/globals.css +++ b/frontend/src/styles/globals.css @@ -448,12 +448,17 @@ } .mobile-chat-form { - @apply sticky bottom-0 z-10 flex w-full max-w-full shrink-0 flex-col gap-2 border-t border-border bg-surface-1 px-2 py-2; + @apply sticky bottom-0 z-10 grid w-full max-w-full shrink-0 grid-cols-[minmax(0,1fr)_auto] items-start gap-2 border-t border-border bg-surface-1 px-2 py-2; padding-bottom: max(0.5rem, env(safe-area-inset-bottom)); } + .mobile-chat-form textarea { + @apply min-h-11; + height: 2.75rem; + } + .mobile-composer-toolbar { - @apply grid w-full items-center gap-2; + @apply grid w-auto items-center gap-1; grid-template-columns: 2.75rem; } @@ -469,6 +474,10 @@ @apply inline-flex h-11 min-w-11 items-center justify-center rounded-md px-3 text-xs text-gray-500 underline transition-colors hover:bg-surface-2 hover:text-gray-300 focus:outline-none focus:ring-2 focus:ring-accent/50; } + .mobile-account-row { + @apply flex min-h-11 items-center justify-between gap-2 border-t border-border pt-2; + } + .mobile-composer-select { @apply h-11 w-full min-w-0 truncate rounded-md border border-border bg-surface-2 px-2 text-xs text-gray-200 focus:outline-none focus:ring-2 focus:ring-accent/50; } @@ -477,20 +486,6 @@ @apply inline-flex h-11 w-11 shrink-0 items-center justify-center rounded-md bg-accent text-white transition-colors hover:bg-accent-hover disabled:cursor-not-allowed disabled:opacity-45 focus:outline-none focus:ring-2 focus:ring-accent/50; } - .mobile-chat-form-collapsed { - @apply grid grid-cols-[minmax(0,1fr)_auto] items-start gap-2; - } - - .mobile-chat-form-collapsed textarea { - @apply min-h-11; - height: 2.75rem; - } - - .mobile-chat-form-collapsed .mobile-composer-toolbar { - @apply w-auto gap-1; - grid-template-columns: 2.75rem; - } - .mobile-tools-sheet { @apply grid shrink-0 gap-2 border-b border-border bg-surface-0 px-2 py-2; max-height: min(44dvh, 24rem); diff --git a/scripts/mobile_ui_gate.py b/scripts/mobile_ui_gate.py index 46fca41c..7cc0e81e 100755 --- a/scripts/mobile_ui_gate.py +++ b/scripts/mobile_ui_gate.py @@ -334,6 +334,7 @@ def evaluate(self, expression: str, *, return_by_value: bool = True) -> Any: const commandDock = document.querySelector('.mobile-command-dock'); const composerForm = document.querySelector('.mobile-chat-form'); const composer = document.querySelector('#mobile-task-input'); + const send = document.querySelector('button[aria-label="Run task"]'); const pinnedActions = document.querySelector('[aria-label="Pinned browser actions"]'); const required = [root, live, controls, frame, commandDock, composerForm, composer]; const rect = (node) => node ? node.getBoundingClientRect().toJSON() : null; @@ -377,6 +378,18 @@ def evaluate(self, expression: str, *, return_by_value: bool = True) -> Any: composerForm: rect(composerForm), composerFormVisible: fullyVisible(composerForm), composer: rect(composer), + send: rect(send), + sendVisible: visible(send) && fullyVisible(send), + composerSingleRow: (() => { + if (!visible(composer) || !visible(send)) return false; + const inputRect = composer.getBoundingClientRect(); + const sendRect = send.getBoundingClientRect(); + return inputRect.width > 1 && sendRect.width > 1 && + Math.abs(inputRect.top - sendRect.top) <= 2 && Math.abs(inputRect.bottom - sendRect.bottom) <= 2; + })(), + logoutInsideTools: !!tools?.querySelector('button[aria-label="Log out"]'), + logoutOutsideTools: [...document.querySelectorAll('button[aria-label="Log out"]')] + .some((button) => !tools?.contains(button)), hasBrowserToolsToggle: !!document.querySelector('button[aria-label="Open browser tools"], button[aria-label="Close browser tools"]'), hasBrowserTools: !!document.querySelector('[aria-label="Browser tools"]'), hasAgentRunner: !!document.querySelector('select[aria-label="Select harness runner"]'), @@ -1363,6 +1376,25 @@ def run_viewport( and not bool(tools_structure.get("hasAgentRunner")), tools_structure, ) + if auth_token: + add_check( + result, + "account controls stay behind browser tools disclosure", + not bool(structure.get("logoutInsideTools")) + and not bool(structure.get("logoutOutsideTools")) + and bool(tools_structure.get("logoutInsideTools")) + and not bool(tools_structure.get("logoutOutsideTools")), + { + "compact": { + "insideTools": structure.get("logoutInsideTools"), + "outsideTools": structure.get("logoutOutsideTools"), + }, + "toolsOpen": { + "insideTools": tools_structure.get("logoutInsideTools"), + "outsideTools": tools_structure.get("logoutOutsideTools"), + }, + }, + ) browser.eval(r"""(() => { const button = document.querySelector('button[aria-label="Close browser tools"]'); if (button) button.click(); @@ -1395,6 +1427,7 @@ def run_viewport( })()""") chat_visible_again = browser.wait_for("!!document.querySelector('[aria-label=\"Chat history\"]')", "chat after tools", 5) tools_closed_by_chat = browser.eval("!document.querySelector('[aria-label=\"Browser tools\"]')") + expanded_chat_structure = browser.eval(STRUCTURE_SCRIPT) exclusive = { "ready": bool(chat_opened) and bool(tools_opened_after_chat) and bool(chat_reopened), "chatOpen": bool(chat_visible), @@ -1412,6 +1445,14 @@ def run_viewport( and bool(exclusive.get("toolsClosedByChat")), exclusive, ) + add_check( + result, + "expanded chat keeps composer in one compact row", + bool(expanded_chat_structure.get("composerSingleRow")) + and bool(expanded_chat_structure.get("composerFormVisible")) + and bool(expanded_chat_structure.get("sendVisible")), + expanded_chat_structure, + ) dispatch_shortcut = r"""((key, extra = {}) => { window.dispatchEvent(new KeyboardEvent('keydown', {key, bubbles: true, cancelable: true, ctrlKey: true, ...extra})); return true; From a25a9ac71317ce7fae0c7dee4ff409a314971f7d Mon Sep 17 00:00:00 2001 From: Martin Hausleitner <55828102+Martin-Hausleitner@users.noreply.github.com> Date: Tue, 21 Jul 2026 20:51:42 +0200 Subject: [PATCH 03/32] feat: run scoped mobile task workspace on vcvm --- .dockerignore | 36 ++ backend/browser_manager.py | 51 +- backend/database.py | 233 +++++++- backend/main.py | 303 ++++++++++- backend/models.py | 109 ++++ backend/tests/test_access_control.py | 143 +++++ backend/tests/test_database.py | 54 ++ backend/tests/test_task_sessions_api.py | 269 ++++++++++ docker-compose.vcvm.yml | 41 ++ docs/VCVM-DEPLOYMENT.md | 76 +++ frontend/src/App.tsx | 19 +- .../mobile/MobileSplitScreen.test.tsx | 139 ++++- .../components/mobile/MobileSplitScreen.tsx | 221 +++++--- frontend/src/lib/api.test.ts | 174 ++++++ frontend/src/lib/api.ts | 102 ++++ frontend/src/lib/taskHarness.test.ts | 500 ++++++++++++++++-- frontend/src/lib/taskHarness.ts | 333 +++++++++++- frontend/src/styles/globals.css | 23 +- scripts/deploy_vcvm.sh | 238 +++++++++ scripts/test_vcvm_deployment.py | 148 ++++++ 20 files changed, 3013 insertions(+), 199 deletions(-) create mode 100644 .dockerignore create mode 100644 backend/tests/test_task_sessions_api.py create mode 100644 docker-compose.vcvm.yml create mode 100644 docs/VCVM-DEPLOYMENT.md create mode 100755 scripts/deploy_vcvm.sh create mode 100755 scripts/test_vcvm_deployment.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..66ff7ade --- /dev/null +++ b/.dockerignore @@ -0,0 +1,36 @@ +.git +.gitignore +.gitattributes + +.env +.env.* +*.env +*.token +*token* +.env.vcvm + +.venv +backend/.venv +venv +node_modules +frontend/node_modules + +frontend/dist +dist +build +coverage +htmlcov + +.pytest_cache +.ruff_cache +.mypy_cache +.cache +__pycache__ +*.pyc + +backend/.data +artifacts +benchmarks + +.DS_Store +*.log diff --git a/backend/browser_manager.py b/backend/browser_manager.py index 2ec7509b..efdd9c67 100644 --- a/backend/browser_manager.py +++ b/backend/browser_manager.py @@ -192,6 +192,7 @@ def __init__(self): self._launching: set[str] = set() # profile IDs currently being launched self.vnc = VNCManager() self._lock = asyncio.Lock() + self._process_env_lock = asyncio.Lock() self._next_cdp_port = BASE_CDP_PORT self._auto_launch_task: asyncio.Task | None = None @@ -243,26 +244,36 @@ async def launch(self, profile: dict[str, Any]) -> RunningProfile: if proxy: _validate_proxy(proxy) - # Launch CloakBrowser on that display - # DISPLAY is passed via env kwarg to avoid process-wide os.environ mutation - context = await launch_persistent_context_async( - user_data_dir=profile["user_data_dir"], - headless=bool(profile.get("headless", False)), - proxy=proxy, - args=extra_args, - timezone=profile.get("timezone") or None, - locale=profile.get("locale") or None, - humanize=bool(profile.get("humanize", False)), - human_preset=profile.get("human_preset", "default"), - geoip=bool(profile.get("geoip", False)), - color_scheme=profile.get("color_scheme") or None, - user_agent=profile.get("user_agent") or None, - viewport={ - "width": profile.get("screen_width", 1920), - "height": profile.get("screen_height", 1080) - 133, - }, - env={**os.environ, "DISPLAY": f":{display}"}, - ) + # Some CloakBrowser builds do not forward Playwright's env kwarg to + # Chromium, so set DISPLAY around the launch and restore it after. + display_value = f":{display}" + async with self._process_env_lock: + previous_display = os.environ.get("DISPLAY") + os.environ["DISPLAY"] = display_value + try: + context = await launch_persistent_context_async( + user_data_dir=profile["user_data_dir"], + headless=bool(profile.get("headless", False)), + proxy=proxy, + args=extra_args, + timezone=profile.get("timezone") or None, + locale=profile.get("locale") or None, + humanize=bool(profile.get("humanize", False)), + human_preset=profile.get("human_preset", "default"), + geoip=bool(profile.get("geoip", False)), + color_scheme=profile.get("color_scheme") or None, + user_agent=profile.get("user_agent") or None, + viewport={ + "width": profile.get("screen_width", 1920), + "height": profile.get("screen_height", 1080) - 133, + }, + env={**os.environ, "DISPLAY": display_value}, + ) + finally: + if previous_display is None: + os.environ.pop("DISPLAY", None) + else: + os.environ["DISPLAY"] = previous_display await self._fit_window_to_vnc( context, diff --git a/backend/database.py b/backend/database.py index 0cb26d7e..b453d60e 100644 --- a/backend/database.py +++ b/backend/database.py @@ -136,6 +136,52 @@ def init_db(): outcome TEXT NOT NULL, created_at TEXT NOT NULL ); + + CREATE TABLE IF NOT EXISTS task_sessions ( + id TEXT PRIMARY KEY, + profile_id TEXT NOT NULL REFERENCES profiles(id) ON DELETE CASCADE, + sandbox_id TEXT NOT NULL, + title TEXT, + status TEXT NOT NULL DEFAULT 'active' CHECK (status IN ('active', 'archived')), + created_by_kind TEXT NOT NULL, + created_by_id TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + metadata TEXT NOT NULL DEFAULT '{}' + ); + + CREATE INDEX IF NOT EXISTS idx_task_sessions_profile + ON task_sessions(profile_id, created_at DESC); + + CREATE INDEX IF NOT EXISTS idx_task_sessions_sandbox + ON task_sessions(sandbox_id, created_at DESC); + + CREATE TABLE IF NOT EXISTS task_messages ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL REFERENCES task_sessions(id) ON DELETE CASCADE, + role TEXT NOT NULL CHECK (role IN ('user', 'assistant', 'system', 'tool')), + content TEXT NOT NULL, + created_by_kind TEXT NOT NULL, + created_by_id TEXT, + created_at TEXT NOT NULL, + metadata TEXT NOT NULL DEFAULT '{}' + ); + + CREATE INDEX IF NOT EXISTS idx_task_messages_session + ON task_messages(session_id, created_at ASC); + + CREATE TABLE IF NOT EXISTS task_events ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL REFERENCES task_sessions(id) ON DELETE CASCADE, + type TEXT NOT NULL, + payload TEXT NOT NULL DEFAULT '{}', + created_by_kind TEXT NOT NULL, + created_by_id TEXT, + created_at TEXT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_task_events_session + ON task_events(session_id, created_at ASC); """) conn.commit() @@ -277,13 +323,19 @@ def update_profile(profile_id: str, **fields: Any) -> dict[str, Any] | None: if update_cols: update_cols.append("updated_at = ?") - update_vals.append(_now()) + now = _now() + update_vals.append(now) update_vals.append(profile_id) with get_db() as conn: conn.execute( f"UPDATE profiles SET {', '.join(update_cols)} WHERE id = ?", update_vals, ) + if "sandbox_id" in fields: + conn.execute( + "UPDATE task_sessions SET sandbox_id = ?, updated_at = ? WHERE profile_id = ?", + (fields["sandbox_id"], now, profile_id), + ) conn.commit() if tags is not None: @@ -306,6 +358,185 @@ def delete_profile(profile_id: str) -> bool: return cursor.rowcount > 0 +# ── Task session persistence ──────────────────────────────────────────────── + + +def _json_object(value: str | None) -> dict[str, Any]: + if not value: + return {} + try: + decoded = json.loads(value) + except json.JSONDecodeError: + return {} + return decoded if isinstance(decoded, dict) else {} + + +def _task_session_from_row(row: sqlite3.Row) -> dict[str, Any]: + session = dict(row) + session["metadata"] = _json_object(session.get("metadata")) + return session + + +def _task_message_from_row(row: sqlite3.Row) -> dict[str, Any]: + message = dict(row) + message["metadata"] = _json_object(message.get("metadata")) + return message + + +def _task_event_from_row(row: sqlite3.Row) -> dict[str, Any]: + event = dict(row) + event["payload"] = _json_object(event.get("payload")) + return event + + +def create_task_session( + profile_id: str, + sandbox_id: str, + created_by_kind: str, + created_by_id: str | None = None, + title: str | None = None, + metadata: dict[str, Any] | None = None, +) -> dict[str, Any]: + session_id = str(uuid.uuid4()) + now = _now() + with get_db() as conn: + conn.execute( + """INSERT INTO task_sessions + (id, profile_id, sandbox_id, title, status, created_by_kind, created_by_id, + created_at, updated_at, metadata) + VALUES (?, ?, ?, ?, 'active', ?, ?, ?, ?, ?)""", + ( + session_id, + profile_id, + sandbox_id, + title, + created_by_kind, + created_by_id, + now, + now, + json.dumps(metadata or {}, separators=(",", ":")), + ), + ) + conn.commit() + return get_task_session(session_id) # type: ignore[return-value] + + +def get_task_session(session_id: str) -> dict[str, Any] | None: + with get_db() as conn: + row = conn.execute("SELECT * FROM task_sessions WHERE id = ?", (session_id,)).fetchone() + return _task_session_from_row(row) if row else None + + +def list_task_sessions(profile_id: str, limit: int = 100) -> list[dict[str, Any]]: + safe_limit = max(1, min(limit, 200)) + with get_db() as conn: + rows = conn.execute( + """SELECT * FROM task_sessions + WHERE profile_id = ? + ORDER BY created_at DESC + LIMIT ?""", + (profile_id, safe_limit), + ).fetchall() + return [_task_session_from_row(row) for row in rows] + + +def append_task_message( + session_id: str, + role: str, + content: str, + created_by_kind: str, + created_by_id: str | None = None, + metadata: dict[str, Any] | None = None, +) -> dict[str, Any]: + message_id = str(uuid.uuid4()) + now = _now() + with get_db() as conn: + conn.execute( + """INSERT INTO task_messages + (id, session_id, role, content, created_by_kind, created_by_id, created_at, metadata) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", + ( + message_id, + session_id, + role, + content, + created_by_kind, + created_by_id, + now, + json.dumps(metadata or {}, separators=(",", ":")), + ), + ) + conn.execute("UPDATE task_sessions SET updated_at = ? WHERE id = ?", (now, session_id)) + conn.commit() + return get_task_message(message_id) # type: ignore[return-value] + + +def get_task_message(message_id: str) -> dict[str, Any] | None: + with get_db() as conn: + row = conn.execute("SELECT * FROM task_messages WHERE id = ?", (message_id,)).fetchone() + return _task_message_from_row(row) if row else None + + +def list_task_messages(session_id: str, limit: int = 100) -> list[dict[str, Any]]: + safe_limit = max(1, min(limit, 200)) + with get_db() as conn: + rows = conn.execute( + """SELECT * FROM task_messages + WHERE session_id = ? + ORDER BY created_at ASC + LIMIT ?""", + (session_id, safe_limit), + ).fetchall() + return [_task_message_from_row(row) for row in rows] + + +def record_task_event( + session_id: str, + event_type: str, + created_by_kind: str, + created_by_id: str | None = None, + payload: dict[str, Any] | None = None, +) -> dict[str, Any]: + event_id = str(uuid.uuid4()) + now = _now() + with get_db() as conn: + conn.execute( + """INSERT INTO task_events + (id, session_id, type, payload, created_by_kind, created_by_id, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?)""", + ( + event_id, + session_id, + event_type, + json.dumps(payload or {}, separators=(",", ":")), + created_by_kind, + created_by_id, + now, + ), + ) + conn.commit() + return get_task_event(event_id) # type: ignore[return-value] + + +def get_task_event(event_id: str) -> dict[str, Any] | None: + with get_db() as conn: + row = conn.execute("SELECT * FROM task_events WHERE id = ?", (event_id,)).fetchone() + return _task_event_from_row(row) if row else None + + +def list_task_events(session_id: str, limit: int = 100) -> list[dict[str, Any]]: + safe_limit = max(1, min(limit, 200)) + with get_db() as conn: + rows = conn.execute( + """SELECT * FROM task_events + WHERE session_id = ? + ORDER BY created_at ASC + LIMIT ?""", + (session_id, safe_limit), + ).fetchall() + return [_task_event_from_row(row) for row in rows] + + # ── Access control persistence ─────────────────────────────────────────────── diff --git a/backend/main.py b/backend/main.py index 120da818..90c24803 100644 --- a/backend/main.py +++ b/backend/main.py @@ -22,7 +22,7 @@ from urllib.parse import urlparse import httpx -from fastapi import FastAPI, HTTPException, Request, Response, WebSocket, WebSocketDisconnect +from fastapi import FastAPI, HTTPException, Query, Request, Response, WebSocket, WebSocketDisconnect from fastapi.responses import FileResponse, JSONResponse from fastapi.staticfiles import StaticFiles import starlette.requests @@ -50,6 +50,12 @@ ProfileUpdate, StatusResponse, TagResponse, + TaskCommandRequest, + TaskEventResponse, + TaskMessageCreate, + TaskMessageResponse, + TaskSessionCreate, + TaskSessionResponse, ) else: # Support `uvicorn main:app` from the backend directory. import access_control as access @@ -73,6 +79,12 @@ ProfileUpdate, StatusResponse, TagResponse, + TaskCommandRequest, + TaskEventResponse, + TaskMessageCreate, + TaskMessageResponse, + TaskSessionCreate, + TaskSessionResponse, ) logger = logging.getLogger("cloakbrowser.manager") @@ -102,6 +114,19 @@ _LOGIN_FAILURE_TTL_SECONDS = 10 * 60.0 _LOGIN_FAILURE_MAX_KEYS = 1024 _login_failures: dict[tuple[str, str], tuple[int, float, float]] = {} +_TASK_METADATA_MAX_BYTES = 8_192 +_TASK_METADATA_MAX_DEPTH = 4 +_TASK_METADATA_MAX_LIST_ITEMS = 20 +_TASK_SENSITIVE_KEY_PARTS = ( + "authorization", + "cookie", + "credential", + "password", + "secret", + "token", + "api_key", + "apikey", +) _BENCHMARK_REPORT_ENV = "BENCHMARK_REPORT_PATH" _DEFAULT_BENCHMARK_REPORT_PATH = Path("/data/benchmark-report.json") @@ -964,6 +989,107 @@ def _require_profile_permission( return profile, identity +def _can_read_task_sessions(identity: access.AccessIdentity, profile: dict[str, object]) -> bool: + sandbox_id = str(profile.get("sandbox_id") or "default") + return identity.is_admin or access.has_permission(identity, sandbox_id, "view") + + +def _can_write_task_sessions(identity: access.AccessIdentity, profile: dict[str, object]) -> bool: + sandbox_id = str(profile.get("sandbox_id") or "default") + return ( + identity.is_admin + or access.has_permission(identity, sandbox_id, "interact") + or access.has_permission(identity, sandbox_id, "automate") + ) + + +def _require_task_profile( + scope: Scope, profile_id: str, permission: access.Permission = "interact" +) -> tuple[dict[str, object], access.AccessIdentity]: + profile = db.get_profile(profile_id) + if not profile: + raise HTTPException(status_code=404, detail="Profile not found") + identity = _require_identity(scope) + allowed = ( + _can_write_task_sessions(identity, profile) + if permission == "interact" + else _can_read_task_sessions(identity, profile) + ) + if not allowed: + db.record_access_audit_event( + identity.kind, + identity.id, + f"task_session.permission.{permission}", + "denied", + str(profile.get("sandbox_id") or "default"), + profile_id, + ) + raise HTTPException(status_code=404, detail="Profile not found") + return profile, identity + + +def _require_task_session( + scope: Scope, session_id: str, permission: access.Permission = "view" +) -> tuple[dict[str, object], dict[str, object], access.AccessIdentity]: + session = db.get_task_session(session_id) + if not session: + raise HTTPException(status_code=404, detail="Task session not found") + profile = db.get_profile(str(session["profile_id"])) + identity = _require_identity(scope) + allowed = False + if profile: + allowed = ( + _can_write_task_sessions(identity, profile) + if permission == "interact" + else _can_read_task_sessions(identity, profile) + ) + if not profile or not allowed: + if profile: + db.record_access_audit_event( + identity.kind, + identity.id, + f"task_session.permission.{permission}", + "denied", + str(profile.get("sandbox_id") or "default"), + str(profile.get("id") or session["profile_id"]), + ) + raise HTTPException(status_code=404, detail="Task session not found") + return session, profile, identity + + +def _sanitize_task_value(value: object, depth: int = 0) -> object: + if depth >= _TASK_METADATA_MAX_DEPTH: + return "[truncated]" + if value is None or isinstance(value, (bool, int, float, str)): + return value + if isinstance(value, list): + return [ + _sanitize_task_value(item, depth + 1) + for item in value[:_TASK_METADATA_MAX_LIST_ITEMS] + ] + if isinstance(value, dict): + sanitized: dict[str, object] = {} + for raw_key, raw_value in value.items(): + key = str(raw_key)[:120] + key_lower = key.lower() + if any(part in key_lower for part in _TASK_SENSITIVE_KEY_PARTS): + sanitized[key] = "[redacted]" + else: + sanitized[key] = _sanitize_task_value(raw_value, depth + 1) + return sanitized + return str(value)[:500] + + +def _sanitize_task_metadata(metadata: dict[str, object] | None) -> dict[str, object]: + sanitized = _sanitize_task_value(metadata or {}) + if not isinstance(sanitized, dict): + return {} + encoded = json.dumps(sanitized, sort_keys=True, separators=(",", ":")).encode("utf-8") + if len(encoded) <= _TASK_METADATA_MAX_BYTES: + return sanitized + return {"truncated": True} + + async def _require_websocket_profile_permission( websocket: WebSocket, profile_id: str, permission: access.Permission ) -> tuple[dict[str, object], access.AccessIdentity] | None: @@ -1258,6 +1384,181 @@ async def list_access_sandboxes(request: Request): return [{"sandbox_id": key, "profile_count": counts[key]} for key in sorted(counts)] +# ── Task sessions ─────────────────────────────────────────────────────────── + + +@app.post("/api/task-sessions", response_model=TaskSessionResponse, status_code=201) +async def create_task_session(body: TaskSessionCreate, request: Request): + profile, identity = _require_task_profile(request.scope, body.profile_id, "interact") + session = db.create_task_session( + str(profile["id"]), + str(profile.get("sandbox_id") or "default"), + identity.kind, + identity.id, + body.title, + _sanitize_task_metadata(body.metadata), + ) + db.record_task_event( + str(session["id"]), + "task_session.created", + identity.kind, + identity.id, + {"profile_id": str(profile["id"]), "sandbox_id": str(profile.get("sandbox_id") or "default")}, + ) + db.record_access_audit_event( + identity.kind, + identity.id, + "task_session.create", + "allowed", + str(profile.get("sandbox_id") or "default"), + str(profile["id"]), + ) + return TaskSessionResponse(**session) + + +@app.get("/api/task-sessions", response_model=list[TaskSessionResponse]) +async def list_task_sessions( + request: Request, + profile_id: str = Query(..., min_length=1, max_length=120), + limit: int = Query(100, ge=1, le=200), +): + profile, _identity = _require_task_profile(request.scope, profile_id, "view") + return [ + TaskSessionResponse(**session) + for session in db.list_task_sessions(str(profile["id"]), limit=limit) + ] + + +@app.get("/api/task-sessions/{session_id}", response_model=TaskSessionResponse) +async def get_task_session(session_id: str, request: Request): + session, _profile, _identity = _require_task_session(request.scope, session_id, "view") + return TaskSessionResponse(**session) + + +def _append_task_user_message( + scope: Scope, + session_id: str, + text: str, + profile_id: str | None, + commands: list[object], + metadata: dict[str, object], +) -> TaskMessageResponse: + session, profile, identity = _require_task_session(scope, session_id, "interact") + if profile_id and profile_id != str(session["profile_id"]): + requested_profile = db.get_profile(profile_id) + if requested_profile: + db.record_access_audit_event( + identity.kind, + identity.id, + "task_session.profile_mismatch", + "denied", + str(requested_profile.get("sandbox_id") or "default"), + profile_id, + ) + raise HTTPException(status_code=404, detail="Task session not found") + + command_payload = [ + command.model_dump() if hasattr(command, "model_dump") else command + for command in commands + ] + stored_metadata_input = dict(metadata) + if command_payload: + stored_metadata_input["commands"] = command_payload + stored_metadata = _sanitize_task_metadata(stored_metadata_input) + message = db.append_task_message( + str(session["id"]), + "user", + text, + identity.kind, + identity.id, + stored_metadata, + ) + host_command_count = sum( + 1 for command in command_payload + if isinstance(command, dict) and command.get("scope") == "host" + ) + db.record_task_event( + str(session["id"]), + "task_message.appended", + identity.kind, + identity.id, + { + "message_id": str(message["id"]), + "role": "user", + "command_count": len(command_payload), + "host_command_count": host_command_count, + "server_executed": False, + }, + ) + db.record_access_audit_event( + identity.kind, + identity.id, + "task_message.append", + "allowed", + str(profile.get("sandbox_id") or "default"), + str(profile["id"]), + ) + return TaskMessageResponse(**message) + + +@app.post( + "/api/task-sessions/{session_id}/messages", + response_model=TaskMessageResponse, + status_code=201, +) +async def append_task_message(session_id: str, body: TaskMessageCreate, request: Request): + return _append_task_user_message( + request.scope, + session_id, + body.text, + body.profile_id, + body.commands, + body.metadata, + ) + + +@app.post( + "/api/task-sessions/{session_id}/commands", + response_model=TaskMessageResponse, + status_code=201, +) +async def append_task_command(session_id: str, body: TaskCommandRequest, request: Request): + return _append_task_user_message( + request.scope, + session_id, + body.content, + body.profile_id, + body.commands, + body.metadata, + ) + + +@app.get("/api/task-sessions/{session_id}/messages", response_model=list[TaskMessageResponse]) +async def list_task_messages( + session_id: str, + request: Request, + limit: int = Query(100, ge=1, le=200), +): + session, _profile, _identity = _require_task_session(request.scope, session_id, "view") + return [ + TaskMessageResponse(**message) + for message in db.list_task_messages(str(session["id"]), limit=limit) + ] + + +@app.get("/api/task-sessions/{session_id}/events", response_model=list[TaskEventResponse]) +async def list_task_events( + session_id: str, + request: Request, + limit: int = Query(100, ge=1, le=200), +): + session, _profile, _identity = _require_task_session(request.scope, session_id, "view") + return [ + TaskEventResponse(**event) + for event in db.list_task_events(str(session["id"]), limit=limit) + ] + + # ── Profile CRUD ────────────────────────────────────────────────────────────── diff --git a/backend/models.py b/backend/models.py index 6d50393c..bab937ef 100644 --- a/backend/models.py +++ b/backend/models.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from typing import Literal from pydantic import BaseModel, Field, field_validator, model_validator @@ -146,6 +147,114 @@ class ClipboardRequest(BaseModel): text: str = Field(max_length=1_048_576) # 1MB max +class TaskSessionCreate(BaseModel): + profile_id: str = Field(min_length=1, max_length=120) + title: str | None = Field(default=None, max_length=120) + metadata: dict[str, object] = Field(default_factory=dict) + + +class TaskSessionResponse(BaseModel): + id: str + profile_id: str + sandbox_id: str + title: str | None = None + status: Literal["active", "archived"] = "active" + created_by_kind: str + created_by_id: str | None = None + created_at: str + updated_at: str + metadata: dict[str, object] = Field(default_factory=dict) + + +TaskCommandKind = Literal[ + "navigate", + "click", + "double_click", + "scroll", + "type_text", + "keypress", + "drag", + "move", + "wait", + "copy", + "paste", + "screenshot", + "viewport", + "fullscreen", + "focus_remote", + "focus_chat", +] + + +class TaskCommand(BaseModel): + id: str = Field(min_length=1, max_length=120) + label: str = Field(min_length=1, max_length=120) + kind: TaskCommandKind + scope: Literal["ui", "host"] + args: dict[str, str | int | float | bool | None] = Field(default_factory=dict) + + @field_validator("args") + @classmethod + def validate_args(cls, value: dict[str, object]) -> dict[str, object]: + if len(value) > 20: + raise ValueError("Command args may contain at most 20 keys") + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + if len(encoded) > 2_048: + raise ValueError("Command args are too large") + return value + + +class TaskMessageCreate(BaseModel): + text: str = Field(min_length=1, max_length=8_000) + profile_id: str | None = Field(default=None, min_length=1, max_length=120) + commands: list[TaskCommand] = Field(default_factory=list) + metadata: dict[str, object] = Field(default_factory=dict) + + @field_validator("commands") + @classmethod + def validate_commands(cls, value: list[TaskCommand]) -> list[TaskCommand]: + return _validate_task_commands(value) + + +class TaskCommandRequest(BaseModel): + content: str = Field(min_length=1, max_length=8_000) + profile_id: str | None = Field(default=None, min_length=1, max_length=120) + commands: list[TaskCommand] = Field(default_factory=list) + metadata: dict[str, object] = Field(default_factory=dict) + + @field_validator("commands") + @classmethod + def validate_commands(cls, value: list[TaskCommand]) -> list[TaskCommand]: + return _validate_task_commands(value) + + +def _validate_task_commands(value: list[TaskCommand]) -> list[TaskCommand]: + if len(value) > 20: + raise ValueError("A task message may contain at most 20 commands") + return value + + +class TaskMessageResponse(BaseModel): + id: str + session_id: str + role: Literal["user", "assistant", "system", "tool"] + content: str + created_by_kind: str + created_by_id: str | None = None + created_at: str + metadata: dict[str, object] = Field(default_factory=dict) + + +class TaskEventResponse(BaseModel): + id: str + session_id: str + type: str + created_by_kind: str + created_by_id: str | None = None + created_at: str + payload: dict[str, object] = Field(default_factory=dict) + + class LoginRequest(BaseModel): """Bootstrap-token or named-user login request. diff --git a/backend/tests/test_access_control.py b/backend/tests/test_access_control.py index 67ee3921..a389b92e 100644 --- a/backend/tests/test_access_control.py +++ b/backend/tests/test_access_control.py @@ -222,6 +222,149 @@ def test_operator_can_operate_only_its_scoped_profile(client_access: TestClient) assert denied.status_code == 404 +def test_interact_user_can_create_task_session_with_redacted_metadata(client_access: TestClient): + alpha, _beta = create_scoped_profiles() + client_access.post( + "/api/access/users", + headers=bootstrap_headers(), + json={ + "username": "interact-task", + "password": "interact-task-password-123", + "grants": [{"sandbox_id": "alpha", "permission": "interact"}], + }, + ) + client_access.cookies.clear() + assert client_access.post( + "/api/auth/login", + json={"username": "interact-task", "password": "interact-task-password-123"}, + ).status_code == 200 + + created = client_access.post( + "/api/task-sessions", + json={ + "profile_id": alpha["id"], + "title": "Scoped harness task", + "metadata": { + "source": "vcvm-e2e", + "authorization": "Bearer should-not-persist", + "nested": {"password": "should-not-persist"}, + }, + }, + ) + + assert created.status_code == 201 + assert created.json()["metadata"] == { + "source": "vcvm-e2e", + "authorization": "[redacted]", + "nested": {"password": "[redacted]"}, + } + + +def test_interact_user_cannot_create_task_session_outside_granted_sandbox( + client_access: TestClient, +): + _alpha, beta = create_scoped_profiles() + client_access.post( + "/api/access/users", + headers=bootstrap_headers(), + json={ + "username": "interact-task-denied", + "password": "interact-task-denied-password-123", + "grants": [{"sandbox_id": "alpha", "permission": "interact"}], + }, + ) + client_access.cookies.clear() + assert client_access.post( + "/api/auth/login", + json={ + "username": "interact-task-denied", + "password": "interact-task-denied-password-123", + }, + ).status_code == 200 + + denied = client_access.post( + "/api/task-sessions", + json={"profile_id": beta["id"], "title": "Out of scope"}, + ) + + assert denied.status_code == 404 + assert denied.json()["detail"] == "Profile not found" + + +def test_interact_user_cannot_operate_or_automate_profile(client_access: TestClient): + alpha, _beta = create_scoped_profiles() + client_access.post( + "/api/access/users", + headers=bootstrap_headers(), + json={ + "username": "interact-no-lifecycle", + "password": "interact-no-lifecycle-password-123", + "grants": [{"sandbox_id": "alpha", "permission": "interact"}], + }, + ) + client_access.cookies.clear() + assert client_access.post( + "/api/auth/login", + json={ + "username": "interact-no-lifecycle", + "password": "interact-no-lifecycle-password-123", + }, + ).status_code == 200 + + launch_denied = client_access.post(f"/api/profiles/{alpha['id']}/launch") + cdp_denied = client_access.get(f"/api/profiles/{alpha['id']}/cdp") + + assert launch_denied.status_code == 404 + assert cdp_denied.status_code == 404 + + +def test_paperclip_agent_command_metadata_is_redacted(client_access: TestClient): + alpha, _beta = create_scoped_profiles() + created = client_access.post( + "/api/access/agents", + headers=bootstrap_headers(), + json={ + "display_name": "Paperclip task agent", + "paperclip_agent_id": "paperclip-agent-task", + "grants": [ + {"sandbox_id": "alpha", "permission": "interact"}, + {"sandbox_id": "alpha", "permission": "automate"}, + ], + }, + ) + agent_headers = {"Authorization": f"Bearer {created.json()['api_key']}"} + identity = client_access.get("/api/access/me", headers=agent_headers) + assert identity.status_code == 200, identity.text + assert identity.json()["kind"] == "agent" + visible = client_access.get("/api/profiles", headers=agent_headers) + assert visible.status_code == 200, visible.text + assert [profile["id"] for profile in visible.json()] == [alpha["id"]] + session = client_access.post( + "/api/task-sessions", + headers=agent_headers, + json={"profile_id": alpha["id"], "title": "Agent task"}, + ) + assert session.status_code == 201, session.text + + command = client_access.post( + f"/api/task-sessions/{session.json()['id']}/commands", + headers=agent_headers, + json={ + "content": "type into browser", + "metadata": { + "harness": "paperclip", + "cookie": "should-not-persist", + "api_key": "should-not-persist", + }, + }, + ) + + assert command.status_code == 201 + assert command.json()["metadata"]["harness"] == "paperclip" + assert command.json()["metadata"]["cookie"] == "[redacted]" + assert command.json()["metadata"]["api_key"] == "[redacted]" + + def test_admin_can_update_a_user_grants_from_the_access_dashboard_payload(client_access: TestClient): """Pydantic serializes nested grants before main.py receives the update. diff --git a/backend/tests/test_database.py b/backend/tests/test_database.py index 3f00cc13..8353e963 100644 --- a/backend/tests/test_database.py +++ b/backend/tests/test_database.py @@ -200,6 +200,60 @@ def test_list_profiles_includes_tags(tmp_db: Path): assert len(profiles[0]["tags"]) == 1 +# ── task sessions ──────────────────────────────────────────────────────────── + + +def test_task_session_message_and_event_roundtrip(tmp_db: Path): + profile = db.create_profile("Task Browser", sandbox_id="tasks") + session = db.create_task_session( + profile["id"], + profile["sandbox_id"], + "user", + "user-1", + "Research", + {"source": "test"}, + ) + + message = db.append_task_message( + session["id"], + "user", + "Open the dashboard", + "user", + "user-1", + {"intent": "navigate"}, + ) + event = db.record_task_event( + session["id"], + "task_command.appended", + "user", + "user-1", + {"message_id": message["id"]}, + ) + + assert db.get_task_session(session["id"])["metadata"] == {"source": "test"} + assert db.list_task_sessions(profile["id"])[0]["id"] == session["id"] + assert db.list_task_messages(session["id"]) == [message] + assert db.list_task_events(session["id"]) == [event] + + +def test_task_sessions_are_deleted_with_profile(tmp_db: Path): + profile = db.create_profile("Task Browser") + session = db.create_task_session( + profile["id"], + profile["sandbox_id"], + "bootstrap", + None, + ) + db.append_task_message(session["id"], "user", "hello", "bootstrap") + db.record_task_event(session["id"], "task_session.created", "bootstrap") + + assert db.delete_profile(profile["id"]) is True + + assert db.get_task_session(session["id"]) is None + assert db.list_task_messages(session["id"]) == [] + assert db.list_task_events(session["id"]) == [] + + # ── update_profile ─────────────────────────────────────────────────────────── diff --git a/backend/tests/test_task_sessions_api.py b/backend/tests/test_task_sessions_api.py new file mode 100644 index 00000000..9a40865c --- /dev/null +++ b/backend/tests/test_task_sessions_api.py @@ -0,0 +1,269 @@ +"""API tests for sandbox-scoped browser task sessions.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest +from starlette.testclient import TestClient + +from backend import database as db + + +@pytest.fixture() +def client_access(tmp_db, monkeypatch): + from backend import main + + monkeypatch.setattr(main, "AUTH_TOKEN", "bootstrap-test-secret") + monkeypatch.setattr(main, "ACCESS_CONTROL_ENABLED", True) + main._login_failures.clear() + monkeypatch.setattr(main.browser_mgr, "cleanup_stale", AsyncMock()) + monkeypatch.setattr(main.browser_mgr, "cleanup_all", AsyncMock()) + monkeypatch.setattr(main.browser_mgr.vnc, "cleanup_stale", AsyncMock()) + with TestClient(main.app) as client: + yield client + + +def bootstrap_headers() -> dict[str, str]: + return {"Authorization": "Bearer bootstrap-test-secret"} + + +def create_user( + client: TestClient, + username: str, + sandbox_id: str, + permission: str, +) -> str: + password = f"{username}-password-123" + response = client.post( + "/api/access/users", + headers=bootstrap_headers(), + json={ + "username": username, + "password": password, + "grants": [{"sandbox_id": sandbox_id, "permission": permission}], + }, + ) + assert response.status_code == 201 + return password + + +def login(client: TestClient, username: str, password: str) -> None: + client.cookies.clear() + response = client.post( + "/api/auth/login", + json={"username": username, "password": password}, + ) + assert response.status_code == 200 + + +def test_task_session_messages_persist_commands_and_never_fake_assistant( + client_access: TestClient, +): + profile = db.create_profile("Alpha browser", sandbox_id="alpha") + password = create_user(client_access, "alpha-operator", "alpha", "interact") + login(client_access, "alpha-operator", password) + + created = client_access.post( + "/api/task-sessions", + json={ + "profile_id": profile["id"], + "metadata": {"source": "mobile", "api_token": "secret-token"}, + }, + ) + assert created.status_code == 201 + session = created.json() + assert session["profile_id"] == profile["id"] + assert session["sandbox_id"] == "alpha" + assert session["metadata"]["api_token"] == "[redacted]" + + posted = client_access.post( + f"/api/task-sessions/{session['id']}/messages", + json={ + "text": "Copy the visible result", + "profile_id": profile["id"], + "commands": [ + { + "id": "copy-visible", + "label": "Copy visible result", + "kind": "copy", + "scope": "host", + "args": {"format": "text"}, + } + ], + "metadata": {"password": "do-not-store", "client": "mobile"}, + }, + ) + assert posted.status_code == 201 + message = posted.json() + assert message["role"] == "user" + assert message["content"] == "Copy the visible result" + assert message["metadata"]["password"] == "[redacted]" + assert message["metadata"]["commands"][0]["kind"] == "copy" + assert message["metadata"]["commands"][0]["scope"] == "host" + + history = client_access.get(f"/api/task-sessions/{session['id']}/messages") + assert history.status_code == 200 + assert history.json() == [message] + assert all(item["role"] != "assistant" for item in history.json()) + + events = client_access.get(f"/api/task-sessions/{session['id']}/events") + assert events.status_code == 200 + appended = [event for event in events.json() if event["type"] == "task_message.appended"][0] + assert appended["payload"]["command_count"] == 1 + assert appended["payload"]["host_command_count"] == 1 + assert appended["payload"]["server_executed"] is False + + +def test_view_grant_can_read_history_but_cannot_create_or_send(client_access: TestClient): + profile = db.create_profile("Alpha browser", sandbox_id="alpha") + session = db.create_task_session(profile["id"], "alpha", "bootstrap") + db.append_task_message(session["id"], "user", "stored command", "bootstrap") + password = create_user(client_access, "alpha-viewer", "alpha", "view") + login(client_access, "alpha-viewer", password) + + listed = client_access.get(f"/api/task-sessions?profile_id={profile['id']}") + assert listed.status_code == 200 + assert [item["id"] for item in listed.json()] == [session["id"]] + + messages = client_access.get(f"/api/task-sessions/{session['id']}/messages") + assert messages.status_code == 200 + assert messages.json()[0]["content"] == "stored command" + + denied_create = client_access.post( + "/api/task-sessions", + json={"profile_id": profile["id"]}, + ) + assert denied_create.status_code == 404 + assert denied_create.json()["detail"] == "Profile not found" + + denied_send = client_access.post( + f"/api/task-sessions/{session['id']}/messages", + json={"text": "should not append"}, + ) + assert denied_send.status_code == 404 + assert denied_send.json()["detail"] == "Task session not found" + + +def test_cross_sandbox_task_sessions_are_indistinguishable_404( + client_access: TestClient, +): + alpha = db.create_profile("Alpha browser", sandbox_id="alpha") + beta = db.create_profile("Beta browser", sandbox_id="beta") + beta_session = db.create_task_session(beta["id"], "beta", "bootstrap") + password = create_user(client_access, "alpha-user", "alpha", "interact") + login(client_access, "alpha-user", password) + + assert client_access.get(f"/api/task-sessions?profile_id={alpha['id']}").status_code == 200 + denied_profile = client_access.get(f"/api/task-sessions?profile_id={beta['id']}") + assert denied_profile.status_code == 404 + assert denied_profile.json()["detail"] == "Profile not found" + + denied_session = client_access.get(f"/api/task-sessions/{beta_session['id']}/messages") + assert denied_session.status_code == 404 + assert denied_session.json()["detail"] == "Task session not found" + + with db.get_db() as conn: + denied_events = { + (row["action"], row["sandbox_id"], row["profile_id"], row["outcome"]) + for row in conn.execute( + """SELECT action, sandbox_id, profile_id, outcome + FROM access_audit_events WHERE outcome = 'denied'""" + ).fetchall() + } + assert ("task_session.permission.view", "beta", beta["id"], "denied") in denied_events + + +def test_task_message_rejects_missing_profile_mismatch_and_unknown_commands( + client_access: TestClient, +): + alpha = db.create_profile("Alpha browser", sandbox_id="alpha") + beta = db.create_profile("Beta browser", sandbox_id="beta") + password = create_user(client_access, "alpha-operator", "alpha", "interact") + login(client_access, "alpha-operator", password) + + missing = client_access.post("/api/task-sessions", json={"profile_id": "missing"}) + assert missing.status_code == 404 + + session = client_access.post( + "/api/task-sessions", + json={"profile_id": alpha["id"]}, + ).json() + + mismatch = client_access.post( + f"/api/task-sessions/{session['id']}/messages", + json={"text": "wrong browser", "profile_id": beta["id"]}, + ) + assert mismatch.status_code == 404 + assert mismatch.json()["detail"] == "Task session not found" + + invalid = client_access.post( + f"/api/task-sessions/{session['id']}/messages", + json={ + "text": "bad command", + "commands": [ + { + "id": "bad", + "label": "Bad", + "kind": "shell", + "scope": "host", + } + ], + }, + ) + assert invalid.status_code == 422 + + +def test_legacy_commands_route_rejects_more_than_twenty_commands(client_access: TestClient): + alpha = db.create_profile("Alpha browser", sandbox_id="alpha") + password = create_user(client_access, "alpha-operator", "alpha", "interact") + login(client_access, "alpha-operator", password) + session = client_access.post( + "/api/task-sessions", + json={"profile_id": alpha["id"]}, + ).json() + + too_many = client_access.post( + f"/api/task-sessions/{session['id']}/commands", + json={ + "content": "run pinned actions", + "commands": [ + { + "id": f"cmd-{index}", + "label": f"Command {index}", + "kind": "screenshot", + "scope": "ui", + } + for index in range(21) + ], + }, + ) + + assert too_many.status_code == 422 + + +def test_task_session_history_follows_profile_sandbox_move(client_access: TestClient): + profile = db.create_profile("Movable browser", sandbox_id="alpha") + session = db.create_task_session(profile["id"], "alpha", "bootstrap") + db.append_task_message(session["id"], "user", "move-safe history", "bootstrap") + + updated = client_access.put( + f"/api/profiles/{profile['id']}", + headers=bootstrap_headers(), + json={"sandbox_id": "beta"}, + ) + assert updated.status_code == 200 + assert updated.json()["sandbox_id"] == "beta" + assert db.get_task_session(session["id"])["sandbox_id"] == "beta" + + old_password = create_user(client_access, "old-alpha-viewer", "alpha", "view") + login(client_access, "old-alpha-viewer", old_password) + old_view = client_access.get(f"/api/task-sessions/{session['id']}/messages") + assert old_view.status_code == 404 + assert old_view.json()["detail"] == "Task session not found" + + new_password = create_user(client_access, "new-beta-viewer", "beta", "view") + login(client_access, "new-beta-viewer", new_password) + new_view = client_access.get(f"/api/task-sessions/{session['id']}/messages") + assert new_view.status_code == 200 + assert new_view.json()[0]["content"] == "move-safe history" diff --git a/docker-compose.vcvm.yml b/docker-compose.vcvm.yml new file mode 100644 index 00000000..5538528d --- /dev/null +++ b/docker-compose.vcvm.yml @@ -0,0 +1,41 @@ +name: cloakbrowser-manager-vcvm + +services: + manager: + container_name: cloakbrowser-manager-vcvm + image: cloakbrowser-manager:vcvm + platform: linux/amd64 + build: + context: . + dockerfile: Dockerfile + restart: unless-stopped + network_mode: bridge + ports: + - "127.0.0.1:${MANAGER_PORT:-18115}:8080" + environment: + AUTH_TOKEN: ${AUTH_TOKEN:?AUTH_TOKEN is required for VCVM deployments} + ACCESS_CONTROL_ENABLED: "1" + volumes: + - cloakbrowser-manager-vcvm-data:/data + shm_size: "${VCVM_SHM_SIZE:-2gb}" + cpus: "${VCVM_CPUS:-16.0}" + mem_limit: "${VCVM_MEMORY_LIMIT:-32g}" + pids_limit: 4096 + ulimits: + nofile: + soft: 65535 + hard: 65535 + healthcheck: + test: + - CMD + - python + - -c + - "import urllib.request; urllib.request.urlopen('http://localhost:8080/health', timeout=5)" + interval: 30s + timeout: 5s + retries: 5 + start_period: 60s + +volumes: + cloakbrowser-manager-vcvm-data: + name: cloakbrowser-manager-vcvm-data diff --git a/docs/VCVM-DEPLOYMENT.md b/docs/VCVM-DEPLOYMENT.md new file mode 100644 index 00000000..15a8898b --- /dev/null +++ b/docs/VCVM-DEPLOYMENT.md @@ -0,0 +1,76 @@ +# VCVM Deployment + +This deployment path runs the whole CloakBrowser Manager product on the VCVM: +FastAPI, the built React UI, SQLite data, KasmVNC and launched browser profiles +stay inside one Docker service with one persistent Docker volume. + +## Safety model + +- Host: `vcvm` +- Remote path: `/home/coder/cloakbrowser-manager` +- Docker project and container: `cloakbrowser-manager-vcvm` +- Data volume: `cloakbrowser-manager-vcvm-data` +- Manager bind: `127.0.0.1:${MANAGER_PORT:-18115}` on the VCVM only +- Required auth: `AUTH_TOKEN` +- Required policy layer: `ACCESS_CONTROL_ENABLED=1` +- Optional private iPhone access: Tailscale Serve HTTPS after the app proves + `auth_required=true` and `access_control_enabled=true` + +The compose file does not publish any raw VNC port. Browser viewing remains +behind the authenticated Manager proxy. + +## Deploy + +Create a long bootstrap token in a local secret file with mode `600`. The token +is sent to the VCVM over SSH and written to +`/home/coder/cloakbrowser-manager/.env.vcvm` with mode `600`. + +```bash +mkdir -p ~/.config/cloakbrowser +openssl rand -base64 48 > ~/.config/cloakbrowser/vcvm-auth-token +chmod 600 ~/.config/cloakbrowser/vcvm-auth-token +./scripts/deploy_vcvm.sh --auth-token-file ~/.config/cloakbrowser/vcvm-auth-token +``` + +The script syncs the current checkout to the VCVM, builds the Docker image on +the VCVM, starts the stack and checks: + +1. `/health` answers locally on the VCVM. +2. `/api/auth/status` reports required auth. +3. `/api/auth/status` reports access control enabled. + +## Private Tailscale HTTPS + +If Tailscale Serve is enabled for the tailnet and a private HTTPS port is free: + +```bash +./scripts/deploy_vcvm.sh --auth-token-file ~/.config/cloakbrowser/vcvm-auth-token --serve-private +``` + +Use a different private HTTPS port if `443` is already configured: + +```bash +TAILSCALE_HTTPS_PORT=8443 ./scripts/deploy_vcvm.sh --auth-token-file ~/.config/cloakbrowser/vcvm-auth-token --serve-private +``` + +The script refuses to replace an existing Serve entry on the selected HTTPS +port. It also refuses to publish unless the protected Manager is already running +with scoped access control. + +## Validation + +Run the local deployment-surface checks before changing the VCVM: + +```bash +python3 scripts/test_vcvm_deployment.py +``` + +Run a remote smoke after deploy: + +```bash +ssh vcvm 'curl -fsS http://127.0.0.1:18115/health' +ssh vcvm 'curl -fsS http://127.0.0.1:18115/api/auth/status' +``` + +The second command must report `auth_required: true` and +`access_control_enabled: true`; do not publish a URL if it does not. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 445a14a7..dd441813 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -122,6 +122,21 @@ function AppContent({ authRequired, accessControlEnabled, identity, onLogout }: const canOperateSelected = Boolean(selected && canAccess(identity, selected, "operate")); const canInteractSelected = Boolean(selected && canAccess(identity, selected, "interact")); + useEffect(() => { + if (!isMobile || loading || profiles.length === 0) return; + + const selectedStillExists = selectedId + ? profiles.some((profile) => profile.id === selectedId) + : false; + if (selectedStillExists) return; + + const nextProfile = profiles.find((profile) => profile.status === "running") ?? profiles[0]; + if (!nextProfile) return; + + setSelectedId(nextProfile.id); + setView("view"); + }, [isMobile, loading, profiles, selectedId]); + const handleSelect = useCallback((id: string) => { setSelectedId(id); const profile = profiles.find((p) => p.id === id); @@ -168,8 +183,8 @@ function AppContent({ authRequired, accessControlEnabled, identity, onLogout }: }, [identity, profiles, selectedId, stop]); const handleVncDisconnect = useCallback(() => { - setView(canManageProfiles ? "edit" : "empty"); - }, [canManageProfiles]); + setView(isMobile ? "view" : canManageProfiles ? "edit" : "empty"); + }, [canManageProfiles, isMobile]); const handleViewportApply = useCallback(async (width: number, height: number) => { if (!selectedId || !canManageProfiles) return false; diff --git a/frontend/src/components/mobile/MobileSplitScreen.test.tsx b/frontend/src/components/mobile/MobileSplitScreen.test.tsx index 4b729dcd..ecaefc2a 100644 --- a/frontend/src/components/mobile/MobileSplitScreen.test.tsx +++ b/frontend/src/components/mobile/MobileSplitScreen.test.tsx @@ -1,7 +1,7 @@ import { fireEvent, render, screen, waitFor, within } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useState } from "react"; -import type { Profile } from "../../lib/api"; +import { api, type Profile } from "../../lib/api"; import { codexComputerUseProvider, taskHarnessReadyEvent } from "../../lib/taskHarness"; import { MobileSplitScreen } from "./MobileSplitScreen"; @@ -142,6 +142,9 @@ function runningSplit(overrides: Partial[0] beforeEach(() => { vi.restoreAllMocks(); + vi.spyOn(api, "listTaskSessions").mockResolvedValue([]); + vi.spyOn(api, "listTaskSessionMessages").mockResolvedValue([]); + vi.spyOn(api, "listTaskSessionEvents").mockResolvedValue([]); installTaskHarness(); }); @@ -154,7 +157,7 @@ describe("MobileSplitScreen", () => { it("renders the default Codex Computer Use composer with browser tools and chat collapsed", async () => { renderMobileSplit(); - expect(screen.getByText("Codex Computer Use")).toBeTruthy(); + expect(await screen.findByText("Codex Computer Use")).toBeTruthy(); expect(await screen.findByPlaceholderText("Ask Codex Computer Use...")).toBeTruthy(); expect(screen.getByLabelText("Open browser tools")).toBeTruthy(); expect(screen.queryByLabelText("Browser tools")).toBeNull(); @@ -163,7 +166,7 @@ describe("MobileSplitScreen", () => { await waitFor(() => expect(window.cloakBrowserHarness?.send).toBeTruthy()); }); - it("disables the composer when the injected Codex Computer Use Bridge is invalid", async () => { + it("labels the server fallback as save-only and never fabricates an assistant reply", async () => { window.cloakBrowserHarness = { capabilities: { chat: true, @@ -172,28 +175,109 @@ describe("MobileSplitScreen", () => { browser_actions: ["paste"], }, }; - renderMobileSplit(); + vi.spyOn(api, "createTaskSession").mockResolvedValue({ + id: "server-session-1", + profile_id: stoppedProfile.id, + sandbox_id: stoppedProfile.sandbox_id, + title: null, + status: "active", + created_by_kind: "user", + created_by_id: "user-1", + created_at: "2026-07-21T10:00:00.000Z", + updated_at: "2026-07-21T10:00:00.000Z", + metadata: {}, + }); + const append = vi.spyOn(api, "appendTaskMessage").mockResolvedValue({ + id: "server-message-1", + session_id: "server-session-1", + role: "user", + content: "Save this task", + created_by_kind: "user", + created_by_id: "user-1", + created_at: "2026-07-21T10:00:01.000Z", + metadata: {}, + }); + const { container } = renderMobileSplit(); - const input = await screen.findByPlaceholderText("Codex Computer Use Bridge unavailable"); - expect((input as HTMLTextAreaElement).disabled).toBe(true); - expect((screen.getByLabelText("Run task") as HTMLButtonElement).disabled).toBe(true); + const input = await screen.findByPlaceholderText("Save task to server history..."); + expect((input as HTMLTextAreaElement).disabled).toBe(false); openBrowserTools(); - expect(screen.getByText("Codex unavailable")).toBeTruthy(); - expect(screen.getByText("A verified Codex Computer Use Bridge must be injected by the host before tasks can run.")).toBeTruthy(); + expect(screen.getByText("Save only")).toBeTruthy(); + expect(screen.getByText("Tasks are saved to scoped server history only. Nothing executes until a verified Codex host attaches.")).toBeTruthy(); + fireEvent.click(screen.getByLabelText("Close browser tools")); + + fireEvent.change(input, { target: { value: "Save this task" } }); + fireEvent.click(screen.getByLabelText("Run task")); + + await waitFor(() => expect(append).toHaveBeenCalledTimes(1)); + expect(await screen.findByText("Saved to server history · not executed.")).toBeTruthy(); + expect(screen.getAllByText("Save this task")).toHaveLength(1); + expect(container.querySelector(".mobile-message-assistant")).toBeNull(); + }); + + it("loads the latest scoped server conversation and continues it after reload", async () => { + delete window.cloakBrowserHarness; + vi.mocked(api.listTaskSessions).mockResolvedValue([ + { + id: "server-session-existing", + profile_id: stoppedProfile.id, + sandbox_id: stoppedProfile.sandbox_id, + title: "Checkout follow-up", + status: "active", + created_by_kind: "user", + created_by_id: "user-1", + created_at: "2026-07-21T09:00:00.000Z", + updated_at: "2026-07-21T09:05:00.000Z", + metadata: {}, + }, + ]); + vi.mocked(api.listTaskSessionMessages).mockResolvedValue([ + { + id: "history-message-1", + session_id: "server-session-existing", + role: "user", + content: "Open checkout", + created_by_kind: "user", + created_by_id: "user-1", + created_at: "2026-07-21T09:01:00.000Z", + metadata: {}, + }, + ]); + const create = vi.spyOn(api, "createTaskSession"); + const append = vi.spyOn(api, "appendTaskMessage").mockResolvedValue({ + id: "history-message-2", + session_id: "server-session-existing", + role: "user", + content: "Continue checkout", + created_by_kind: "user", + created_by_id: "user-1", + created_at: "2026-07-21T09:06:00.000Z", + metadata: {}, + }); + + renderMobileSplit(); + fireEvent.click(screen.getByLabelText("Expand task chat")); + expect(await screen.findByText("Open checkout")).toBeTruthy(); - fireEvent.submit(input.closest("form") as HTMLFormElement); + const input = await screen.findByPlaceholderText("Save task to server history..."); + fireEvent.change(input, { target: { value: "Continue checkout" } }); + fireEvent.click(screen.getByLabelText("Run task")); - expect(screen.queryByText(/Queued locally/)).toBeNull(); - expect(screen.queryByText(/could not queue/)).toBeNull(); + await waitFor(() => expect(append).toHaveBeenCalledWith( + "server-session-existing", + expect.objectContaining({ text: "Continue checkout" }), + { signal: undefined }, + )); + expect(create).not.toHaveBeenCalled(); }); it("enables the composer when a valid host bridge is injected after mount", async () => { delete window.cloakBrowserHarness; renderMobileSplit(); - const unavailableInput = await screen.findByPlaceholderText("Codex Computer Use Bridge unavailable"); - expect((unavailableInput as HTMLTextAreaElement).disabled).toBe(true); + const serverInput = await screen.findByPlaceholderText("Save task to server history..."); + expect((serverInput as HTMLTextAreaElement).disabled).toBe(false); const { send } = installTaskHarness(); window.dispatchEvent(new Event(taskHarnessReadyEvent)); @@ -213,7 +297,7 @@ describe("MobileSplitScreen", () => { profile_id: stoppedProfile.id, metadata: { runner: "codex-computer-use", - preferred_surface: "codex-computer-use", + execution: "host", browser_visible: true, }, }, @@ -239,7 +323,7 @@ describe("MobileSplitScreen", () => { profile_id: stoppedProfile.id, metadata: { runner: "codex-computer-use", - preferred_surface: "codex-computer-use", + execution: "host", browser_visible: true, }, }, @@ -360,7 +444,7 @@ describe("MobileSplitScreen", () => { profile_id: runningProfile.id, metadata: { runner: "codex-computer-use", - preferred_surface: "codex-computer-use", + execution: "host", browser_visible: true, source: "pinned-action", }, @@ -463,6 +547,17 @@ describe("MobileSplitScreen", () => { expect(screen.getAllByText(/412 x 892/).length).toBeGreaterThan(0); }); + it("keeps the complete device viewport when phone-fit is applied to a live browser", async () => { + const { props } = runningSplit(); + vi.stubGlobal("visualViewport", { width: 390, height: 844 }); + + openBrowserTools(); + fireEvent.click(screen.getByLabelText("Edit browser viewport")); + fireEvent.click(screen.getByText("Phone fit")); + + await waitFor(() => expect(props.onViewportApply).toHaveBeenCalledWith(390, 844)); + }); + it("keeps editable viewport settings and zoom available in fullscreen while background controls are inert", async () => { const { props } = runningSplit(); @@ -494,15 +589,11 @@ describe("MobileSplitScreen", () => { expect(document.activeElement).toBe(screen.getByLabelText("Open fullscreen browser")); }); - it("keeps a visible fullscreen exit control when no browser is live", () => { + it("does not offer fullscreen when no browser is live", () => { renderMobileSplit(); - fireEvent.click(screen.getByLabelText("Open fullscreen browser")); - - expect(screen.getByRole("dialog", { name: "Fullscreen browser viewer" })).toBeTruthy(); - expect(screen.getByLabelText("Close fullscreen browser")).toBeTruthy(); - - fireEvent.click(screen.getByLabelText("Close fullscreen browser")); + expect(screen.queryByLabelText("Open fullscreen browser")).toBeNull(); + fireEvent.keyDown(window, { key: "b", ctrlKey: true }); expect(screen.queryByRole("dialog", { name: "Fullscreen browser viewer" })).toBeNull(); }); diff --git a/frontend/src/components/mobile/MobileSplitScreen.tsx b/frontend/src/components/mobile/MobileSplitScreen.tsx index 955db46f..5200e91f 100644 --- a/frontend/src/components/mobile/MobileSplitScreen.tsx +++ b/frontend/src/components/mobile/MobileSplitScreen.tsx @@ -24,10 +24,13 @@ import { } from "lucide-react"; import type { Profile } from "../../lib/api"; import { + cloakServerProvider, + codexComputerUseProvider, createTaskHarness, taskHarnessReadyEvent, type TaskHarnessAction, type TaskHarnessCapabilities, + type TaskHarnessMessage, } from "../../lib/taskHarness"; import { StatusIndicator } from "../StatusIndicator"; @@ -60,7 +63,7 @@ interface MobileSplitScreenProps { } interface ChatMessage { - id: number; + id: string; role: "task" | "assistant" | "user" | "tool"; text: string; } @@ -127,14 +130,6 @@ function defaultPanePercent(isLiveBrowser: boolean, compactLivePane: boolean) { return compactLivePane ? compactLivePanePercent : defaultLivePanePercent; } -const initialMessages: ChatMessage[] = [ - { - id: 1, - role: "assistant", - text: "Codex Computer Use Bridge status appears here while browser control stays visible.", - }, -]; - function isInteractiveShortcutTarget(target: EventTarget | null) { if (!(target instanceof HTMLElement)) return false; if ( @@ -148,6 +143,18 @@ function isInteractiveShortcutTarget(target: EventTarget | null) { return Boolean(target.closest("canvas, .mobile-browser-frame, .profile-viewer")); } +function toChatMessage(message: TaskHarnessMessage): ChatMessage { + return { + id: message.id, + role: message.role === "assistant" + ? "assistant" + : message.role === "user" + ? "user" + : "tool", + text: message.content, + }; +} + export function MobileSplitScreen({ profiles, selected, @@ -179,7 +186,10 @@ export function MobileSplitScreen({ width: selected?.screen_width ?? presets[0].width, height: selected?.screen_height ?? presets[0].height, }); - const [messages, setMessages] = useState(initialMessages); + const [messages, setMessages] = useState([]); + const [conversationId, setConversationId] = useState(null); + const [historyPending, setHistoryPending] = useState(false); + const [harnessNotice, setHarnessNotice] = useState(null); const [draft, setDraft] = useState(""); const [gridOpen, setGridOpen] = useState(false); const [viewportOpen, setViewportOpen] = useState(false); @@ -240,18 +250,33 @@ export function MobileSplitScreen({ : browserConnectionStatus === "failed" ? "mobile-connection-failed" : "mobile-connection-pending"; - const harnessReady = harnessCapabilities?.chat === true; + const harnessProvider = harnessCapabilities?.metadata?.provider; + const codexHostReady = Boolean( + harnessCapabilities?.chat && harnessProvider === codexComputerUseProvider, + ); + const serverHistoryReady = Boolean( + harnessCapabilities?.chat && harnessProvider === cloakServerProvider, + ); + const harnessReady = codexHostReady || serverHistoryReady; + const composerReady = Boolean(harnessReady && canInteract && selected); const harnessLabel = harnessCapabilities === null - ? "Codex Computer Use Bridge · checking" - : harnessCapabilities.chat === false - ? "Codex Computer Use Bridge · unavailable" - : "Codex Computer Use Bridge · connected"; - const harnessUnavailable = harnessCapabilities?.chat === false; + ? "Task connection · checking" + : codexHostReady + ? "Codex Computer Use · connected" + : serverHistoryReady + ? "Server history · save only" + : "Task connection · unavailable"; const harnessPlaceholder = harnessCapabilities === null - ? "Checking Codex Computer Use Bridge..." - : harnessUnavailable - ? "Codex Computer Use Bridge unavailable" - : "Ask Codex Computer Use..."; + ? "Checking task connection..." + : !selected + ? "Select a browser profile" + : !canInteract + ? "View-only access" + : codexHostReady + ? "Ask Codex Computer Use..." + : serverHistoryReady + ? "Save task to server history..." + : "Task connection unavailable"; const compactWorkspace = chatCollapsed && !remoteToolsOpen && !fullscreenOpen; const toolPanelOpen = viewportOpen || gridOpen || adminOpen; @@ -275,6 +300,7 @@ export function MobileSplitScreen({ const currentRequestId = ++requestId; const harness = createTaskHarness(window); taskHarnessRef.current = harness; + setHarnessCapabilities(null); harness.capabilities() .then((capabilities) => { if (!cancelled && currentRequestId === requestId) { @@ -303,6 +329,49 @@ export function MobileSplitScreen({ }; }, []); + useEffect(() => { + const profileId = selected?.id; + const harness = taskHarnessRef.current; + const controller = new AbortController(); + let cancelled = false; + + setMessages([]); + setConversationId(null); + setHarnessNotice(null); + setHarnessError(null); + + if (!profileId || !harness || !harnessCapabilities?.chat) { + setHistoryPending(false); + return () => controller.abort(); + } + + setHistoryPending(true); + void harness.listConversations(profileId, { signal: controller.signal }) + .then(async (conversations) => { + if (cancelled) return; + const latest = conversations.find((conversation) => conversation.status === "active") + ?? conversations[0]; + if (!latest) return; + const history = await harness.listMessages(latest.id, { signal: controller.signal }); + if (cancelled) return; + setConversationId(latest.id); + setMessages(history.map(toChatMessage)); + }) + .catch((err) => { + if (cancelled || (err instanceof DOMException && err.name === "AbortError")) return; + console.warn("[task-harness] history failed:", err); + setHarnessNotice("History is temporarily unavailable."); + }) + .finally(() => { + if (!cancelled) setHistoryPending(false); + }); + + return () => { + cancelled = true; + controller.abort(); + }; + }, [harnessCapabilities, selected?.id]); + useEffect(() => { if (viewportProfileIdRef.current !== selected?.id) { setViewportSaved(false); @@ -328,7 +397,7 @@ export function MobileSplitScreen({ }, [paneAdjusted, preferredPanePercent, selected?.id, selected?.status]); useEffect(() => { - if (chatAdjustedRef.current) return; + chatAdjustedRef.current = false; setChatCollapsed(true); }, [selected?.id]); @@ -404,6 +473,7 @@ export function MobileSplitScreen({ }); const openFullscreen = () => { + if (!isLiveBrowser) return; restoreFullscreenFocusRef.current = true; setGridOpen(false); setViewportOpen(false); @@ -420,8 +490,9 @@ export function MobileSplitScreen({ }; const runHarnessTask = async (text: string, commands?: readonly TaskHarnessAction[]) => { - if (!text || harnessPending || !harnessReady) return; - const userMessage: ChatMessage = { id: Date.now(), role: "user", text }; + if (!text || harnessPending || !composerReady) return; + const localMessageId = `local-${Date.now()}`; + const userMessage: ChatMessage = { id: localMessageId, role: "user", text }; chatAdjustedRef.current = true; closeTools(); setChatCollapsed(false); @@ -432,37 +503,35 @@ export function MobileSplitScreen({ setDraft(""); setHarnessPending(true); setHarnessError(null); + setHarnessNotice(null); try { const reply = await (taskHarnessRef.current ?? createTaskHarness(window)).send({ text, ...(commands ? { commands } : {}), profile_id: selected?.id ?? null, + ...(conversationId ? { conversation_id: conversationId } : {}), metadata: { - runner: "codex-computer-use", - preferred_surface: "codex-computer-use", + runner: codexHostReady ? codexComputerUseProvider : cloakServerProvider, + execution: codexHostReady ? "host" : "persist-only", browser_visible: true, ...(commands ? { source: "pinned-action" } : {}), }, }); - setMessages((current) => [ - ...current, - { - id: Date.now() + 1, - role: reply.role === "tool" ? "tool" : "assistant", - text: reply.content, - }, - ]); + if (reply.role === "user") { + setMessages((current) => current.map((message) => ( + message.id === localMessageId ? toChatMessage(reply) : message + ))); + setHarnessNotice( + serverHistoryReady + ? "Saved to server history · not executed." + : "Task recorded · no execution result received.", + ); + } else { + setMessages((current) => [...current, toChatMessage(reply)]); + } } catch (err) { const message = err instanceof Error ? err.message : "Task harness request failed"; setHarnessError(message); - setMessages((current) => [ - ...current, - { - id: Date.now() + 1, - role: "assistant", - text: `Codex Computer Use could not queue that task: ${message}`, - }, - ]); } finally { setHarnessPending(false); } @@ -471,13 +540,13 @@ export function MobileSplitScreen({ const sendMessage = async (event: FormEvent) => { event.preventDefault(); const text = draft.trim(); - if (!text || harnessPending || !harnessReady) return; + if (!text || harnessPending || !composerReady) return; setDraft(""); await runHarnessTask(text); }; const runPinnedHarnessAction = async (action: PinnedHarnessAction) => { - if (!harnessCapabilities?.browser_actions.includes(action.kind)) return; + if (!codexHostReady || !harnessCapabilities?.browser_actions.includes(action.kind)) return; await runHarnessTask(pinnedHarnessPrompts[action.kind], [action]); }; @@ -505,7 +574,7 @@ export function MobileSplitScreen({ height: Math.round( Math.max( minimumPhoneFitHeight, - (visualViewport?.height ?? window.innerHeight ?? presets[0].height) - (isLiveBrowser ? 96 : 0), + visualViewport?.height ?? window.innerHeight ?? presets[0].height, ), ), }; @@ -896,17 +965,19 @@ export function MobileSplitScreen({ ) : null}
- + {isLiveBrowser ? ( + + ) : null}
+ {!historyPending && messages.length === 0 ? ( +
+ No saved tasks for this browser yet. +
+ ) : null} {messages.map((message) => (
) : null} + {harnessNotice ? ( +
+ {harnessNotice} +
+ ) : null}
) : null}