From 1b8a9521a04639afb55e4078d39f2b98ceff39b7 Mon Sep 17 00:00:00 2001
From: Martin Hausleitner
<55828102+Martin-Hausleitner@users.noreply.github.com>
Date: Tue, 21 Jul 2026 19:13:29 +0200
Subject: [PATCH 01/32] feat: simplify mobile computer use workspace
---
README.md | 2 +-
.../mobile/MobileSplitScreen.test.tsx | 80 +++++-
.../components/mobile/MobileSplitScreen.tsx | 246 ++++++++++++------
frontend/src/lib/taskHarness.test.ts | 19 ++
frontend/src/lib/taskHarness.ts | 41 ++-
frontend/src/styles/globals.css | 42 ++-
scripts/mobile_ui_gate.py | 55 ++--
7 files changed, 387 insertions(+), 98 deletions(-)
diff --git a/README.md b/README.md
index 67b2eaf7..cabd32b7 100644
--- a/README.md
+++ b/README.md
@@ -99,7 +99,7 @@ The [redacted 20-run report](docs/streaming-benchmark-latest.md) remains the war
The new pinned r51 Selkies lane is now independently reproducible. In the fresh five-run loopback check, its HTTP shell returned **5/5** successful samples with **2.672 ms median total time** (p95 **3.515 ms**), and `/websockets` returned **5/5** valid upgrades with **2.190 ms median handshake** (p95 **3.722 ms**). This proves transport readiness only: it is not a first-frame, FPS, touch-to-pixel, authenticated product-flow, or mobile-Safari comparison. KasmVNC/noVNC remains the production recommendation because it is still the only candidate integrated and verified through the complete profile, policy, VNC-canvas and mobile-UI path. Full context and limits are in [docs/REMOTE-STREAMING-BENCHMARK.md](docs/REMOTE-STREAMING-BENCHMARK.md).
-The current r51 mobile implementation is browser-first: chat starts collapsed, browser tools are centralized, benchmark controls are not shown in the UI, and the task composer accepts only a host bridge explicitly identifying itself as `codex-computer-use`. A missing, generic or mislabeled harness fails closed; task submission is not simulated and no direct vendor API key is required in the UI. The authenticated r51 gate passed **276/276 checks** across five mobile viewports plus the access dashboard and produced **23 screenshots**. A separate Codex Computer Use run logged in as a scoped viewer, reached a real connected VNC canvas, confirmed exactly four persistent actions, exercised the task bridge and found no horizontal overflow at `390 x 844`. The same run then used the admin dashboard to combine `operate` with independent CDP `automate` access for a Paperclip agent. The VCVM/Neko Tailnet check remains transport evidence only: Codex Computer Use completed the protected login and observed `/ws`, but WebRTC ICE failed, so no honest FPS value was recorded. Full details and limits are in [docs/MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md](docs/MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md).
+The current r55 mobile implementation is browser-first: chat starts collapsed, the live browser consumes all unused space, benchmark controls are not shown in the UI, and only Full, Tools, Chat and Send remain persistent. Tools use progressive disclosure for View, Sessions and Admin. The bookmark-like Actions row is not a list of URLs; it sends typed Capture, Copy and Paste commands through a host bridge explicitly identifying itself as `codex-computer-use`. Unknown command kinds are dropped at the bridge boundary, while a missing, generic or mislabeled harness fails closed. The fresh live r55 gate passed **276/276 checks** across five mobile viewports and produced **22 screenshots**. The earlier authenticated r51 run additionally covered the access dashboard: Codex Computer Use logged in as a scoped viewer, reached a real connected VNC canvas, confirmed exactly four persistent actions and found no horizontal overflow at `390 x 844`; an administrator then combined `operate` with independent CDP `automate` access for a Paperclip agent. The VCVM/Neko Tailnet check remains transport evidence only: Codex Computer Use completed the protected login and observed `/ws`, but WebRTC ICE failed, so no honest FPS value was recorded. Full details and limits are in [docs/MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md](docs/MOBILE-STREAMING-AUTH-LATENCY-AUDIT-2026-07-21.md).
## Development
diff --git a/frontend/src/components/mobile/MobileSplitScreen.test.tsx b/frontend/src/components/mobile/MobileSplitScreen.test.tsx
index e8e4cb02..239220a4 100644
--- a/frontend/src/components/mobile/MobileSplitScreen.test.tsx
+++ b/frontend/src/components/mobile/MobileSplitScreen.test.tsx
@@ -66,7 +66,7 @@ function installTaskHarness() {
chat: true,
streaming: true,
clipboard: true,
- browser_actions: ["copy", "paste", "fullscreen"],
+ browser_actions: ["copy", "paste", "screenshot", "fullscreen"],
metadata: { mode: "codex-test", provider: codexComputerUseProvider },
},
send,
@@ -179,7 +179,7 @@ describe("MobileSplitScreen", () => {
expect((screen.getByLabelText("Run task") as HTMLButtonElement).disabled).toBe(true);
openBrowserTools();
- expect(screen.getByText("Codex Computer Use Bridge · unavailable")).toBeTruthy();
+ expect(screen.getByText("Codex unavailable")).toBeTruthy();
expect(screen.getByText("A verified Codex Computer Use Bridge must be injected by the host before tasks can run.")).toBeTruthy();
fireEvent.submit(input.closest("form") as HTMLFormElement);
@@ -300,6 +300,22 @@ describe("MobileSplitScreen", () => {
expect(screen.queryByLabelText("Streaming benchmark results")).toBeNull();
});
+ it("lets the browser consume unused workspace until chat or tools are opened", () => {
+ const { container } = runningSplit();
+ const workspace = container.querySelector(".mobile-split-root") as HTMLElement;
+
+ expect(workspace.classList.contains("mobile-workspace-collapsed")).toBe(true);
+
+ openBrowserTools();
+ expect(workspace.classList.contains("mobile-workspace-collapsed")).toBe(false);
+
+ fireEvent.click(screen.getByLabelText("Close browser tools"));
+ expect(workspace.classList.contains("mobile-workspace-collapsed")).toBe(true);
+
+ fireEvent.click(screen.getByLabelText("Expand task chat"));
+ expect(workspace.classList.contains("mobile-workspace-collapsed")).toBe(false);
+ });
+
it("keeps profile and browser actions inside the central tools sheet without a harness picker", () => {
runningSplit({ canManageAccess: true });
@@ -312,12 +328,51 @@ describe("MobileSplitScreen", () => {
const tools = screen.getByLabelText("Browser tools");
expect(within(tools).getByRole("button", { name: /Stop/i })).toBeTruthy();
+ expect(within(tools).queryByLabelText("New profile")).toBeNull();
+ fireEvent.click(within(tools).getByLabelText("Toggle browser administration"));
expect(within(tools).getByLabelText("New profile")).toBeTruthy();
expect(within(tools).getByLabelText("Edit selected profile")).toBeTruthy();
expect(within(tools).getByLabelText("Browser access controls")).toBeTruthy();
expect(within(tools).queryByLabelText("Select harness runner")).toBeNull();
});
+ it("runs compact pinned browser actions only through the verified Codex host", async () => {
+ const { send } = installTaskHarness();
+ runningSplit();
+ openBrowserTools();
+
+ const capture = await screen.findByLabelText("Run Capture with Codex Computer Use");
+ await waitFor(() => expect((capture as HTMLButtonElement).disabled).toBe(false));
+ fireEvent.click(capture);
+
+ await waitFor(() =>
+ expect(send).toHaveBeenCalledWith(
+ {
+ text: "Capture the current browser view.",
+ commands: [
+ {
+ id: "capture-browser",
+ label: "Capture",
+ kind: "screenshot",
+ scope: "host",
+ },
+ ],
+ profile_id: runningProfile.id,
+ metadata: {
+ runner: "codex-computer-use",
+ preferred_surface: "codex-computer-use",
+ browser_visible: true,
+ source: "pinned-action",
+ },
+ },
+ undefined,
+ ),
+ );
+ expect(await screen.findByText("Harness accepted the task.")).toBeTruthy();
+ expect(screen.queryByLabelText("Browser tools")).toBeNull();
+ expect(screen.getByLabelText("Chat history")).toBeTruthy();
+ });
+
it("keeps viewport and grid panels mutually exclusive inside browser tools", async () => {
const { props } = renderMobileSplit();
openBrowserTools();
@@ -329,6 +384,7 @@ describe("MobileSplitScreen", () => {
fireEvent.click(screen.getByLabelText("Edit browser viewport"));
expect(screen.getByLabelText("Viewport controls")).toBeTruthy();
expect(screen.queryByLabelText("Running browser grid")).toBeNull();
+ expect(screen.queryByLabelText("Pinned browser actions")).toBeNull();
fireEvent.click(screen.getByText("Tablet"));
fireEvent.click(screen.getByText("Apply"));
@@ -363,6 +419,7 @@ describe("MobileSplitScreen", () => {
openBrowserTools();
expect(livePane.style.getPropertyValue("--mobile-live-pane-basis")).toBe("68%");
+ fireEvent.click(screen.getByLabelText("Edit browser viewport"));
fireEvent.change(screen.getByLabelText("Browser pane"), { target: { value: "64" } });
fireEvent.change(screen.getByLabelText("Visual zoom"), { target: { value: "135" } });
@@ -377,14 +434,17 @@ describe("MobileSplitScreen", () => {
expect(screen.getAllByText("VNC stream")).toHaveLength(1);
});
- it("keeps local view controls but hides viewport editing without profile management access", () => {
+ it("keeps local view controls but hides persistent viewport fields without profile management access", () => {
runningSplit({ canManageProfiles: false });
openBrowserTools();
+ expect(screen.getByLabelText("Edit browser viewport")).toBeTruthy();
+ fireEvent.click(screen.getByLabelText("Edit browser viewport"));
expect(screen.getByLabelText("Browser pane")).toBeTruthy();
expect(screen.getByLabelText("Visual zoom")).toBeTruthy();
- expect(screen.queryByLabelText("Edit browser viewport")).toBeNull();
+ expect(screen.queryByLabelText("Viewport width")).toBeNull();
+ expect(screen.getByText("Viewport changes require profile management access.")).toBeTruthy();
fireEvent.click(screen.getByLabelText("Open fullscreen browser"));
expect(screen.getByLabelText("Toggle fullscreen view controls")).toBeTruthy();
@@ -434,6 +494,18 @@ describe("MobileSplitScreen", () => {
expect(document.activeElement).toBe(screen.getByLabelText("Open fullscreen browser"));
});
+ it("keeps a visible fullscreen exit control when no browser is live", () => {
+ renderMobileSplit();
+
+ fireEvent.click(screen.getByLabelText("Open fullscreen browser"));
+
+ expect(screen.getByRole("dialog", { name: "Fullscreen browser viewer" })).toBeTruthy();
+ expect(screen.getByLabelText("Close fullscreen browser")).toBeTruthy();
+
+ fireEvent.click(screen.getByLabelText("Close fullscreen browser"));
+ expect(screen.queryByRole("dialog", { name: "Fullscreen browser viewer" })).toBeNull();
+ });
+
it("uses Ctrl or Cmd shortcuts for fullscreen chat and browser tools", () => {
runningSplit();
diff --git a/frontend/src/components/mobile/MobileSplitScreen.tsx b/frontend/src/components/mobile/MobileSplitScreen.tsx
index 1d564444..fc83b4ef 100644
--- a/frontend/src/components/mobile/MobileSplitScreen.tsx
+++ b/frontend/src/components/mobile/MobileSplitScreen.tsx
@@ -3,7 +3,10 @@ import type { CSSProperties, ChangeEvent, FormEvent, ReactNode } from "react";
import {
ArrowLeft,
ArrowRight,
+ Camera,
ChevronUp,
+ ClipboardCopy,
+ ClipboardPaste,
Expand,
Grid2X2,
Globe2,
@@ -23,6 +26,7 @@ import type { Profile } from "../../lib/api";
import {
createTaskHarness,
taskHarnessReadyEvent,
+ type TaskHarnessAction,
type TaskHarnessCapabilities,
} from "../../lib/taskHarness";
import { StatusIndicator } from "../StatusIndicator";
@@ -67,7 +71,7 @@ const presets = [
{ label: "Desktop", width: 1440, height: 900 },
] as const;
-const defaultPreviewPanePercent = 42;
+const defaultPreviewPanePercent = 68;
// The running browser is the primary control surface; chat and settings are
// now collapsed into a compact dock by default.
const defaultLivePanePercent = 68;
@@ -79,6 +83,37 @@ const collapsedLandscapeLivePanePercent = 74;
const minimumPhoneFitWidth = 320;
const minimumPhoneFitHeight = 480;
+type PinnedHarnessAction = Omit
Preview controls update this viewer immediately.
- -Pane and zoom update this viewer immediately.
> ) : null} {canManageProfiles ? ( @@ -786,7 +834,7 @@ export function MobileSplitScreen({ ); return ( -