From 9f0c35c9cfe189d4cea597f4926351fe844df02c Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Thu, 30 Apr 2026 19:06:49 -0400 Subject: [PATCH 1/3] ship: checkpoint before automate/finalize iOS simulator native helpers + sim-capture/sim-input bridges, ChatIosSimulatorPanel updates, lanes BranchPickerView, branch-picker search utilities, ADE CLI guidance + RPC tweaks, and feature docs sync. Co-Authored-By: Claude Opus 4.7 (1M context) --- apps/ade-cli/README.md | 3 + apps/ade-cli/src/adeRpcServer.ts | 35 +- apps/ade-cli/src/cli.test.ts | 18 + apps/ade-cli/src/cli.ts | 57 +- .../desktop/native/ios-sim-helpers/.gitignore | 1 + apps/desktop/native/ios-sim-helpers/README.md | 39 + apps/desktop/native/ios-sim-helpers/build.sh | 95 ++ .../native/ios-sim-helpers/sim-capture.swift | 378 ++++++ .../native/ios-sim-helpers/sim-input.m | 428 +++++++ .../gitOperationsService.branchSwitch.test.ts | 44 + .../main/services/git/gitOperationsService.ts | 49 +- .../services/ios/iosSimulatorService.test.ts | 101 +- .../main/services/ios/iosSimulatorService.ts | 1078 ++++++++++++++--- .../src/main/services/ipc/registerIpc.ts | 22 +- .../src/main/services/prs/prService.ts | 32 + apps/desktop/src/preload/global.d.ts | 7 + apps/desktop/src/preload/preload.ts | 9 + .../components/chat/AgentChatComposer.tsx | 2 +- .../chat/ChatIosSimulatorPanel.test.tsx | 161 ++- .../components/chat/ChatIosSimulatorPanel.tsx | 603 +++++++-- .../lanes/BranchPickerView.test.tsx | 160 +++ .../components/lanes/BranchPickerView.tsx | 286 +++++ .../components/lanes/CreateLaneDialog.tsx | 176 ++- .../renderer/components/lanes/LanesPage.tsx | 26 +- .../lanes/branchPickerSearch.test.ts | 165 +++ .../components/lanes/branchPickerSearch.ts | 171 +++ .../renderer/components/lanes/laneUtils.ts | 4 + apps/desktop/src/shared/adeCliGuidance.ts | 6 +- apps/desktop/src/shared/ipc.ts | 2 + apps/desktop/src/shared/types/git.ts | 32 + apps/desktop/src/shared/types/iosSimulator.ts | 16 +- docs/ARCHITECTURE.md | 2 +- docs/features/ios-simulator/README.md | 86 +- docs/features/lanes/README.md | 13 +- docs/features/pull-requests/README.md | 4 +- 35 files changed, 3890 insertions(+), 421 deletions(-) create mode 100644 apps/desktop/native/ios-sim-helpers/.gitignore create mode 100644 apps/desktop/native/ios-sim-helpers/README.md create mode 100644 apps/desktop/native/ios-sim-helpers/build.sh create mode 100644 apps/desktop/native/ios-sim-helpers/sim-capture.swift create mode 100644 apps/desktop/native/ios-sim-helpers/sim-input.m create mode 100644 apps/desktop/src/renderer/components/lanes/BranchPickerView.test.tsx create mode 100644 apps/desktop/src/renderer/components/lanes/BranchPickerView.tsx create mode 100644 apps/desktop/src/renderer/components/lanes/branchPickerSearch.test.ts create mode 100644 apps/desktop/src/renderer/components/lanes/branchPickerSearch.ts diff --git a/apps/ade-cli/README.md b/apps/ade-cli/README.md index f5d264603..d10f0374f 100644 --- a/apps/ade-cli/README.md +++ b/apps/ade-cli/README.md @@ -57,7 +57,10 @@ ade lanes list --text ade lanes create "fix-checkout-flow" --parent main ade git commit --lane lane-id ade git push --lane lane-id +ade git branches --lane lane-id --text +ade git user-identity --lane lane-id --text ade prs create --lane lane-id --base main --title "Fix checkout flow" +ade prs list-open --text ade prs path-to-merge --pr pr-id --model gpt-5.5 --max-rounds 3 --no-auto-merge ade run defs --text ade run start web --lane lane-id diff --git a/apps/ade-cli/src/adeRpcServer.ts b/apps/ade-cli/src/adeRpcServer.ts index 9502a1bae..05b0acc6a 100644 --- a/apps/ade-cli/src/adeRpcServer.ts +++ b/apps/ade-cli/src/adeRpcServer.ts @@ -689,7 +689,18 @@ const TOOL_SPECS: ToolSpec[] = [ }, { name: "git_list_branches", - description: "List branches visible from a lane checkout.", + description: "List branches visible from a lane checkout, including last commit sha/date/author/subject for each branch.", + inputSchema: { + type: "object", + additionalProperties: false, + properties: { + laneId: { type: "string", minLength: 1 } + } + } + }, + { + name: "git_get_user_identity", + description: "Read the lane checkout's git user.name and user.email config (the identity new commits would be authored under).", inputSchema: { type: "object", additionalProperties: false, @@ -977,6 +988,15 @@ const TOOL_SPECS: ToolSpec[] = [ } } }, + { + name: "prs_list_open", + description: "List every open pull request in the project's GitHub repo as flat BranchPullRequest rows keyed by head branch. Independent of ADE lane state, so it surfaces PRs whose head branch has no local lane.", + inputSchema: { + type: "object", + additionalProperties: false, + properties: {} + } + }, { name: "pr_get_checks", description: "Get the current CI checks for a pull request.", @@ -1860,6 +1880,8 @@ const READ_ONLY_TOOLS = new Set([ "list_unregistered_lanes", "git_get_sync_status", "git_list_branches", + "git_get_user_identity", + "prs_list_open", "generate_commit_message", "list_stashes", "simulate_integration", @@ -5263,6 +5285,17 @@ async function runTool(args: { return { laneId, branches }; } + if (name === "git_get_user_identity") { + const laneId = requireLaneIdForTool(runtime, session, toolArgs, "git_get_user_identity"); + const identity = await runtime.gitService.getUserIdentity({ laneId }); + return { laneId, identity }; + } + + if (name === "prs_list_open") { + const prs = await requirePrService(runtime).listOpenPullRequests(); + return { prs }; + } + if (name === "git_checkout_branch") { const laneId = requireLaneIdForTool(runtime, session, toolArgs, "git_checkout_branch"); const branchName = assertNonEmptyString(toolArgs.branchName, "branchName"); diff --git a/apps/ade-cli/src/cli.test.ts b/apps/ade-cli/src/cli.test.ts index c6e0614ee..7197d4456 100644 --- a/apps/ade-cli/src/cli.test.ts +++ b/apps/ade-cli/src/cli.test.ts @@ -437,6 +437,24 @@ describe("ADE CLI", () => { }); }); + it("maps git user-identity and prs list-open to typed RPC tools", () => { + const identity = buildCliPlan(["git", "user-identity"]); + expect(identity.kind).toBe("execute"); + if (identity.kind !== "execute") return; + expect(identity.steps[0]?.params).toEqual({ + name: "git_get_user_identity", + arguments: {}, + }); + + const openPrs = buildCliPlan(["prs", "list-open"]); + expect(openPrs.kind).toBe("execute"); + if (openPrs.kind !== "execute") return; + expect(openPrs.steps[0]?.params).toEqual({ + name: "prs_list_open", + arguments: {}, + }); + }); + it("uses the parent ADE project when invoked inside an ADE-managed lane worktree", () => { const rawRoot = fs.mkdtempSync(path.join(os.tmpdir(), "ade-cli-roots-")); // findProjectRoots canonicalizes symlinks (e.g. /var -> /private/var on macOS). diff --git a/apps/ade-cli/src/cli.ts b/apps/ade-cli/src/cli.ts index ff98a7799..85077507f 100644 --- a/apps/ade-cli/src/cli.ts +++ b/apps/ade-cli/src/cli.ts @@ -387,13 +387,13 @@ const IOS_SIMULATOR_SUBCOMMAND_HELP: Record = { --chat-session Owner chat session for the single-owner lock. --no-build Skip xcodebuild. --mode snapshot|live Inspector launch mode; default live. - --foreground Bring Simulator.app forward instead of background. + --foreground Open and bring Simulator.app forward. --arg KEY=VALUE Extra service args for advanced launch options. `, shutdown: `${ADE_BANNER} iOS Simulator: shutdown - Stops streams, releases the drawer session, and tears down the idb companion. + Stops streams, releases the drawer session, and tears down simulator helper processes. Aliases: stop, teardown, end, end-session. $ ade --socket ios-sim shutdown --text @@ -519,9 +519,11 @@ const IOS_SIMULATOR_SUBCOMMAND_HELP: Record = { "stream-start": `${ADE_BANNER} iOS Simulator: stream-start - Starts a visual stream. Prefer live-start/auto for the drawer UI when - idb+idb_companion+ffmpeg are installed, preview-start as a simctl screenshot-poll fallback, - and window-start only for native Simulator.app diagnostics. Aliases: + Starts a visual stream. auto resolves to iosurface-indigo first when full + Xcode supports ADE's private helpers, then Simulator.app window capture when + visible-window capture is allowed, then idb MJPEG, then simctl screenshot + polling. The H.264+ffmpeg idb stream is recovery-only after idb MJPEG fails. + Aliases: start-stream, stream, window-start, start-window, mirror-start, live-start, start-live, preview-start, start-preview. @@ -532,15 +534,17 @@ const IOS_SIMULATOR_SUBCOMMAND_HELP: Record = { Flags: --device, --udid Simulator device. --fps Target fps. - --backend auto|simulator-window-capture|idb-mjpeg|idb-h264-ffmpeg-mjpeg|simctl-screenshot-poll + --backend auto|iosurface-indigo|simulator-window-capture|idb-mjpeg|idb-h264-ffmpeg-mjpeg|simctl-screenshot-poll --window, --mirror Force window capture. - --idb, --live Prefer idb stream (auto-picks h264+ffmpeg, falls back to MJPEG). + --idb, --live Use auto backend resolution. --simctl, --preview Force simctl screenshot polling. `, "stream-status": `${ADE_BANNER} iOS Simulator: stream-status - Shows running backend, fps, latency, stream URL, frame count, and last error. + Shows running backend, fallback/degradation reason, helper pid, fps, latency, + stream URL, frame count, input backend, and last error. Low idle fps is normal + on iosurface-indigo because frames are event-driven when the simulator is still. $ ade --socket ios-sim stream-status --text `, @@ -568,7 +572,7 @@ const IOS_SIMULATOR_SUBCOMMAND_HELP: Record = { tap: `${ADE_BANNER} iOS Simulator: tap - Sends a tap through idb to the active launched app. + Sends a tap through the active input backend, preferring Indigo with idb fallback. $ ade --socket ios-sim tap --x 120 --y 420 --text $ ade --socket ios-sim tap 120 420 --text @@ -581,7 +585,7 @@ const IOS_SIMULATOR_SUBCOMMAND_HELP: Record = { drag: `${ADE_BANNER} iOS Simulator: drag / swipe - Sends a swipe through idb. "swipe" is an alias of drag. + Sends a swipe through the active input backend. "swipe" is an alias of drag. $ ade --socket ios-sim drag --start-x 120 --start-y 700 --end-x 120 --end-y 250 --text $ ade --socket ios-sim swipe 120 700 120 250 --duration-ms 250 --text @@ -681,6 +685,8 @@ const HELP_BY_COMMAND: Record = { $ ade git unstage --lane src/file.ts Unstage one file $ ade git commit --lane [-m ] Commit, generating a message when omitted $ ade git push --lane --set-upstream Push through ADE + $ ade git branches --lane --text List branches with last-commit metadata + $ ade git user-identity --lane --text Read lane checkout's git user.name/email $ ade git stash push|list|apply|pop Use ADE lane stash actions $ ade git rebase --lane --ai Rebase with ADE conflict support $ ade diff changes --lane --text Inspect changed files @@ -700,6 +706,7 @@ const HELP_BY_COMMAND: Record = { Creating or linking a PR persists the lane mapping in ADE so the PR tab tracks it. $ ade prs list --text List PRs known to ADE + $ ade prs list-open --text List every open GitHub PR in the repo, keyed by head branch $ ade prs create --lane --base main Open and map a GitHub PR from a lane $ ade prs link --lane --url Map an existing GitHub PR to a lane $ ade prs checks --text Show check status @@ -809,7 +816,7 @@ const HELP_BY_COMMAND: Record = { drawer simulator. Aliases: \`ade ios\` and \`ade simulator\` route to the same surface. For drawer/shared session state, prefer desktop socket mode (--socket) so launch/select/tap operate on the same long-lived ADE service. - Launch keeps Simulator.app hidden by default; use --foreground only when you + Launch is headless by default; use --foreground only when you need the native Simulator window in front. idb is optional for direct pointer/text control and the low-latency MJPEG live stream. @@ -824,7 +831,7 @@ const HELP_BY_COMMAND: Record = { $ ade ios-sim apps --device --text List launchable apps (listLaunchTargets) $ ade --socket ios-sim launch --target Build/install/launch and update drawer state $ ade --socket ios-sim launch --bundle-id com.example Launch installed app - $ ade --socket ios-sim shutdown Tear down session, streams, idb companion (alias: stop) + $ ade --socket ios-sim shutdown Tear down session, streams, helper processes (alias: stop) $ ade --socket ios-sim shutdown --force Force-release a session owned by another chat $ ade ios-sim actions --text List every callable ios_simulator action @@ -838,7 +845,7 @@ const HELP_BY_COMMAND: Record = { $ ade ios-sim preview-render --source Render a SwiftUI preview through Xcode MCP Streaming: - $ ade ios-sim live-start --fps 30 Low-latency idb live stream + $ ade ios-sim live-start --fps 30 Auto live stream (IOSurface first) $ ade ios-sim preview-start --fps 8 simctl screenshot-poll fallback $ ade ios-sim window-start --fps 60 Native Simulator.app window capture diagnostic $ ade ios-sim stream-status --text Backend/fps/latency/URL (getStreamStatus) @@ -846,9 +853,9 @@ const HELP_BY_COMMAND: Record = { Input and selection: $ ade --socket ios-sim select --x 120 --y 420 Add UI context to drawer chat (selectPoint) - $ ade ios-sim tap 120 420 Tap through idb (tap) - $ ade ios-sim drag 120 700 120 250 Drag through idb (drag) - $ ade ios-sim swipe 120 700 120 250 Swipe through idb (swipe) + $ ade ios-sim tap 120 420 Tap active simulator app (tap) + $ ade ios-sim drag 120 700 120 250 Drag active simulator app (drag) + $ ade ios-sim swipe 120 700 120 250 Swipe active simulator app (swipe) $ ade ios-sim type "hello" --text Type into the launched app (typeText) `, "app-control": `${ADE_BANNER} @@ -1541,6 +1548,9 @@ function buildGitPlan(args: string[]): CliPlan { return { kind: "execute", label: "git commit message", steps: [actionCallStep("result", "generate_commit_message", withLane({ amend: readFlag(args, ["--amend"]) }))] }; } if (sub === "branches" || sub === "branch") return { kind: "execute", label: "git branches", steps: [actionCallStep("result", "git_list_branches", withLane())] }; + if (sub === "user-identity" || sub === "user" || sub === "identity") { + return { kind: "execute", label: "git user identity", steps: [actionCallStep("result", "git_get_user_identity", withLane())] }; + } if (sub === "checkout") { const branchName = requireValue(readValue(args, ["--branch", "--branch-name"]) ?? firstPositional(args), "branchName"); const create = readFlag(args, ["--create", "-b"]); @@ -1675,6 +1685,9 @@ function buildPrPlan(args: string[]): CliPlan { const withPr = (base: JsonObject = {}) => collectGenericObjectArgs(args, { ...base, ...(prId ? { prId } : {}) }); if (sub === "list" || sub === "ls") return { kind: "execute", label: "PR list", steps: [actionStep("result", "pr", "listAll", collectGenericObjectArgs(args))] }; + if (sub === "list-open" || sub === "open" || sub === "list-repo-open") { + return { kind: "execute", label: "PR list open", steps: [actionCallStep("result", "prs_list_open", {})] }; + } if (sub === "show" || sub === "detail" || sub === "view") { const id = requireValue(prId ?? firstPositional(args), "prId"); return { kind: "execute", label: "PR detail", steps: [actionArgsListStep("result", "pr", "getDetail", [id])] }; @@ -2291,7 +2304,7 @@ function buildIosSimulatorPlan(args: string[]): CliPlan { ?? (readFlag(args, ["--window", "--mirror"]) ? "simulator-window-capture" : readFlag(args, ["--idb", "--live"]) ? "auto" : readFlag(args, ["--simctl", "--preview"]) ? "simctl-screenshot-poll" : readValue(args, ["--backend"]) ?? "auto"); const defaultFps = requestedBackend === "simulator-window-capture" ? 60 - : requestedBackend === "idb-mjpeg" || requestedBackend === "idb-h264-ffmpeg-mjpeg" + : requestedBackend === "iosurface-indigo" || requestedBackend === "idb-mjpeg" || requestedBackend === "idb-h264-ffmpeg-mjpeg" ? 30 : requestedBackend === "simctl-screenshot-poll" ? 8 @@ -4043,11 +4056,19 @@ function formatIosSimStream(value: unknown): string { const status = isRecord(value) ? value : {}; return renderKeyValues("ADE iOS simulator stream", [ ["running", status.running], - ["backend", status.backend], + ["requested backend", status.requestedBackend], + ["resolved backend", status.backend], + ["fallback reason", status.fallbackReason], + ["degradation reason", status.degradationReason], ["device", status.deviceUdid], ["fps", status.fps ?? status.targetFps], ["frames", status.frameCount], ["avg latency ms", status.averageLatencyMs], + ["latency p50 ms", status.latencyP50Ms], + ["latency p95 ms", status.latencyP95Ms], + ["helper pid", status.helperPid], + ["input backend", status.inputBackend], + ["error code", isRecord(status.error) ? status.error.code : null], ["started", status.startedAt], ["last frame", status.lastFrameAt], ["stream url", status.streamUrl], diff --git a/apps/desktop/native/ios-sim-helpers/.gitignore b/apps/desktop/native/ios-sim-helpers/.gitignore new file mode 100644 index 000000000..567609b12 --- /dev/null +++ b/apps/desktop/native/ios-sim-helpers/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/apps/desktop/native/ios-sim-helpers/README.md b/apps/desktop/native/ios-sim-helpers/README.md new file mode 100644 index 000000000..7e7178a28 --- /dev/null +++ b/apps/desktop/native/ios-sim-helpers/README.md @@ -0,0 +1,39 @@ +# ADE iOS Simulator private helpers + +ADE uses these helpers as the primary low-latency path for local iOS Simulator +streaming and touch input on macOS. + +- `sim-capture.swift` attaches to CoreSimulator's private IOSurface display + descriptors, JPEG-encodes framebuffer updates, and writes + `[u32 big-endian length][jpeg bytes]` frames to stdout. It accepts `--fps` + and `--quality` so ADE can cap renderer load without changing callers. +- `sim-input.m` opens SimulatorKit's private Indigo HID client and accepts + newline-delimited JSON input commands on stdin. Touch input is sent through + Indigo; unsupported keyboard/text operations are reported as typed failures so + ADE can fall back to idb for that method. +- `build.sh` compiles both helpers lazily into `build/xcode--/`. + +These helpers intentionally use Apple private frameworks. They are local +developer tooling, not app runtime code. Keep the supported Xcode major-version +set explicit in `iosSimulatorService.ts`, and expand it only after testing the +helpers against that Xcode. `iosurface-indigo` is currently gated off in +packaged ADE builds until the helper signing/notarization story is cleared; set +`ADE_IOS_SURFACE_ALLOW_PACKAGED=1` only for explicit packaging experiments. + +To rebuild manually: + +```sh +cd apps/desktop/native/ios-sim-helpers +bash ./build.sh --print-json --smoke +``` + +Known sensitivities: + +- Full Xcode is required. Command Line Tools alone do not include + `SimulatorKit.framework`. +- The helper checks both the iPhoneSimulator platform-private framework path + and the newer selected-Xcode developer-private framework path. +- Xcode 17.x and 26.x are the currently enabled major versions in ADE. +- Xcode updates may rename private classes/selectors or change Indigo packet + layouts. +- Multiple booted simulators are supported only when ADE passes a UDID. diff --git a/apps/desktop/native/ios-sim-helpers/build.sh b/apps/desktop/native/ios-sim-helpers/build.sh new file mode 100644 index 000000000..2d0ffc71b --- /dev/null +++ b/apps/desktop/native/ios-sim-helpers/build.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BUILD_ROOT="$SCRIPT_DIR/build" +PRINT_JSON=0 +SMOKE=0 + +for arg in "$@"; do + case "$arg" in + --print-json) PRINT_JSON=1 ;; + --smoke) SMOKE=1 ;; + *) echo "unknown argument: $arg" >&2; exit 64 ;; + esac +done + +DEVELOPER_DIR="$(/usr/bin/xcode-select -p 2>/dev/null || true)" +if [[ -z "$DEVELOPER_DIR" || ! -d "$DEVELOPER_DIR" ]]; then + echo "xcode-select does not point at a developer directory" >&2 + exit 2 +fi + +SIMULATOR_KIT="" +for candidate in \ + "$DEVELOPER_DIR/Platforms/iPhoneSimulator.platform/Developer/Library/PrivateFrameworks/SimulatorKit.framework" \ + "$DEVELOPER_DIR/Library/PrivateFrameworks/SimulatorKit.framework" +do + if [[ -d "$candidate" ]]; then + SIMULATOR_KIT="$candidate" + break + fi +done +if [[ -z "$SIMULATOR_KIT" ]]; then + echo "full Xcode is required; SimulatorKit.framework was not found under $DEVELOPER_DIR" >&2 + exit 3 +fi + +XCODE_VERSION="$(xcodebuild -version 2>/dev/null | awk 'NR == 1 {print $2}')" +if [[ -z "$XCODE_VERSION" ]]; then + echo "xcodebuild -version did not report an Xcode version" >&2 + exit 4 +fi + +SWIFTC="$(command -v swiftc || true)" +CLANG="$(command -v clang || true)" +if [[ -z "$SWIFTC" ]]; then + echo "swiftc was not found on PATH" >&2 + exit 5 +fi +if [[ -z "$CLANG" ]]; then + echo "clang was not found on PATH" >&2 + exit 6 +fi + +SOURCE_HASH="$( + { + shasum -a 256 "$SCRIPT_DIR/sim-capture.swift" + shasum -a 256 "$SCRIPT_DIR/sim-input.m" + shasum -a 256 "$SCRIPT_DIR/build.sh" + } | shasum -a 256 | awk '{print $1}' +)" +KEY="xcode-${XCODE_VERSION//[^A-Za-z0-9._-]/_}-${SOURCE_HASH:0:16}" +OUT_DIR="$BUILD_ROOT/$KEY" +CAPTURE="$OUT_DIR/sim-capture" +INPUT="$OUT_DIR/sim-input" + +if [[ ! -x "$CAPTURE" || ! -x "$INPUT" ]]; then + mkdir -p "$OUT_DIR" + "$SWIFTC" -O \ + -framework Foundation \ + -framework CoreGraphics \ + -framework CoreImage \ + -framework IOSurface \ + -framework ImageIO \ + -framework UniformTypeIdentifiers \ + "$SCRIPT_DIR/sim-capture.swift" \ + -o "$CAPTURE" + "$CLANG" -fobjc-arc -O2 \ + -framework Foundation \ + -framework CoreGraphics \ + "$SCRIPT_DIR/sim-input.m" \ + -o "$INPUT" +fi + +if [[ "$SMOKE" == "1" ]]; then + "$CAPTURE" --smoke-test >/dev/null + "$INPUT" --smoke-test >/dev/null +fi + +if [[ "$PRINT_JSON" == "1" ]]; then + printf '{"xcodeVersion":"%s","sourceHash":"%s","buildDir":"%s","capture":"%s","input":"%s"}\n' \ + "$XCODE_VERSION" "$SOURCE_HASH" "$OUT_DIR" "$CAPTURE" "$INPUT" +else + echo "$OUT_DIR" +fi diff --git a/apps/desktop/native/ios-sim-helpers/sim-capture.swift b/apps/desktop/native/ios-sim-helpers/sim-capture.swift new file mode 100644 index 000000000..e732129b6 --- /dev/null +++ b/apps/desktop/native/ios-sim-helpers/sim-capture.swift @@ -0,0 +1,378 @@ +import CoreGraphics +import CoreImage +import Foundation +import ImageIO +import IOSurface +import UniformTypeIdentifiers + +func eprint(_ value: String) { + FileHandle.standardError.write((value + "\n").data(using: .utf8) ?? Data()) +} + +func jsonStatus(_ payload: [String: Any]) { + if let data = try? JSONSerialization.data(withJSONObject: payload), + let text = String(data: data, encoding: .utf8) { + eprint("[sim-capture] " + text) + } +} + +struct Options { + var udid: String? + var fps: Int = 60 + var quality: Double = 0.52 + var smokeTest = false +} + +func parseOptions() -> Options { + var options = Options() + var index = 1 + let args = CommandLine.arguments + while index < args.count { + let arg = args[index] + if arg == "--smoke-test" { + options.smokeTest = true + } else if arg == "--udid", index + 1 < args.count { + index += 1 + options.udid = args[index] + } else if arg == "--fps", index + 1 < args.count { + index += 1 + options.fps = max(1, min(120, Int(args[index]) ?? 60)) + } else if arg == "--quality", index + 1 < args.count { + index += 1 + options.quality = max(0.25, min(0.9, Double(args[index]) ?? 0.52)) + } else { + eprint("[sim-capture] unknown argument: \(arg)") + exit(64) + } + index += 1 + } + return options +} + +let options = parseOptions() + +let coreSimulatorPath = "/Library/Developer/PrivateFrameworks/CoreSimulator.framework/CoreSimulator" +guard dlopen(coreSimulatorPath, RTLD_NOW) != nil else { + eprint("[sim-capture] FAIL dlopen CoreSimulator: \(String(cString: dlerror()))") + exit(2) +} + +guard let simServiceContext = NSClassFromString("SimServiceContext") as? NSObject.Type, + let renderableProtocol = NSProtocolFromString("SimDisplayIOSurfaceRenderable") else { + eprint("[sim-capture] FAIL CoreSimulator runtime symbols missing") + exit(2) +} + +func developerDir() -> String { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/xcode-select") + process.arguments = ["-p"] + let pipe = Pipe() + process.standardOutput = pipe + do { + try process.run() + } catch { + return "/Applications/Xcode.app/Contents/Developer" + } + process.waitUntilExit() + let raw = String(data: pipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? "" + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? "/Applications/Xcode.app/Contents/Developer" : trimmed +} + +func bootstrap() -> AnyObject? { + let selector = NSSelectorFromString("sharedServiceContextForDeveloperDir:error:") + typealias Signature = @convention(c) (AnyObject, Selector, NSString, AutoreleasingUnsafeMutablePointer) -> AnyObject? + guard let implementation = simServiceContext.method(for: selector) else { return nil } + let function = unsafeBitCast(implementation, to: Signature.self) + var error: NSError? + let context = withUnsafeMutablePointer(to: &error) { pointer -> AnyObject? in + function(simServiceContext, selector, developerDir() as NSString, AutoreleasingUnsafeMutablePointer(pointer)) + } + if context == nil { + eprint("[sim-capture] sharedServiceContext err: \(String(describing: error))") + } + return context +} + +func defaultDeviceSet(_ context: AnyObject) -> AnyObject? { + let selector = NSSelectorFromString("defaultDeviceSetWithError:") + typealias Signature = @convention(c) (AnyObject, Selector, AutoreleasingUnsafeMutablePointer) -> AnyObject? + guard let implementation = (context as! NSObject).method(for: selector) else { return nil } + let function = unsafeBitCast(implementation, to: Signature.self) + var error: NSError? + let deviceSet = withUnsafeMutablePointer(to: &error) { pointer -> AnyObject? in + function(context, selector, AutoreleasingUnsafeMutablePointer(pointer)) + } + if deviceSet == nil { + eprint("[sim-capture] defaultDeviceSet err: \(String(describing: error))") + } + return deviceSet +} + +func deviceUdid(_ device: AnyObject) -> String { + if let uuid = device.value(forKey: "UDID") as? NSUUID { + return uuid.uuidString + } + return (device.value(forKey: "udid") as? String) ?? "?" +} + +func bootedDevice(_ deviceSet: AnyObject, udid: String?) -> NSObject? { + guard let devices = deviceSet.value(forKey: "devices") as? NSArray else { return nil } + for entry in devices { + let device = entry as AnyObject + let state = (device.value(forKey: "state") as? NSNumber)?.intValue ?? -1 + if state != 3 { continue } + if let requested = udid, deviceUdid(device).caseInsensitiveCompare(requested) != .orderedSame { + continue + } + return device as? NSObject + } + return nil +} + +func findDisplayDescriptor(_ device: NSObject) -> NSObject? { + guard let io = device.value(forKey: "io") as? NSObject, + let ports = io.value(forKey: "ioPorts") as? NSArray else { return nil } + let descriptorSelector = NSSelectorFromString("descriptor") + let surfaceSelector = NSSelectorFromString("framebufferSurface") + typealias DescriptorSignature = @convention(c) (AnyObject, Selector) -> AnyObject? + var best: NSObject? + var bestArea = 0 + for entry in ports { + let port = entry as! NSObject + guard let implementation = port.method(for: descriptorSelector) else { continue } + let function = unsafeBitCast(implementation, to: DescriptorSignature.self) + guard let descriptor = function(port, descriptorSelector) as? NSObject else { continue } + if !descriptor.conforms(to: renderableProtocol) { continue } + guard let surfaceValue = descriptor.perform(surfaceSelector)?.takeUnretainedValue(), + CFGetTypeID(surfaceValue) == IOSurfaceGetTypeID() else { continue } + let surface = unsafeBitCast(surfaceValue, to: IOSurfaceRef.self) as IOSurface + let area = IOSurfaceGetWidth(surface) * IOSurfaceGetHeight(surface) + if area > bestArea { + bestArea = area + best = descriptor + } + } + return best +} + +final class Stream { + let descriptor: NSObject + let device: NSObject + let targetUdid: String + let maxFps: Int + let jpegQuality: Double + let ciContext = CIContext(options: [.useSoftwareRenderer: false]) + let stdout = FileHandle.standardOutput + let writeLock = NSLock() + let callbackUUID = NSUUID() + let damageCallbackUUID = NSUUID() + var registered = false + var damageRegistered = false + var encoding = false + var frameCount = 0 + var lastReportTime = Date() + var lastEmitTime = Date(timeIntervalSince1970: 0) + var width = 0 + var height = 0 + + init(descriptor: NSObject, device: NSObject, targetUdid: String, maxFps: Int, jpegQuality: Double) { + self.descriptor = descriptor + self.device = device + self.targetUdid = targetUdid + self.maxFps = max(1, maxFps) + self.jpegQuality = max(0.25, min(0.9, jpegQuality)) + } + + func start() { + let initial = descriptor.perform(NSSelectorFromString("framebufferSurface"))?.takeUnretainedValue() + if let surfaceValue = initial, CFGetTypeID(surfaceValue) == IOSurfaceGetTypeID() { + let surface = unsafeBitCast(surfaceValue, to: IOSurfaceRef.self) as IOSurface + width = IOSurfaceGetWidth(surface) + height = IOSurfaceGetHeight(surface) + jsonStatus([ + "type": "stream-started", + "pixelWidth": width, + "pixelHeight": height, + "deviceUDID": targetUdid, + "deviceName": (device.value(forKey: "name") as? String) ?? "?", + "fps": maxFps, + "quality": jpegQuality, + ]) + handle(surface: surface, force: true) + } else { + eprint("[sim-capture] no initial framebufferSurface") + } + + let surfaceSelector = NSSelectorFromString("registerCallbackWithUUID:ioSurfacesChangeCallback:") + typealias SurfaceSignature = @convention(c) (AnyObject, Selector, NSUUID, @convention(block) (AnyObject?, AnyObject?) -> Void) -> Void + if let implementation = descriptor.method(for: surfaceSelector) { + let function = unsafeBitCast(implementation, to: SurfaceSignature.self) + let block: @convention(block) (AnyObject?, AnyObject?) -> Void = { [weak self] _, value in + guard let self, + let surfaceValue = value, + CFGetTypeID(surfaceValue) == IOSurfaceGetTypeID() else { return } + let surface = unsafeBitCast(surfaceValue, to: IOSurfaceRef.self) as IOSurface + self.handle(surface: surface) + } + function(descriptor, surfaceSelector, callbackUUID, block) + registered = true + } else { + eprint("[sim-capture] WARN no IOSurface callback selector") + } + + let damageSelector = NSSelectorFromString("registerCallbackWithUUID:damageRectanglesCallback:") + typealias DamageSignature = @convention(c) (AnyObject, Selector, NSUUID, @convention(block) (AnyObject?) -> Void) -> Void + if let implementation = descriptor.method(for: damageSelector) { + let function = unsafeBitCast(implementation, to: DamageSignature.self) + let block: @convention(block) (AnyObject?) -> Void = { [weak self] _ in + guard let self, + let surfaceValue = self.descriptor.perform(NSSelectorFromString("framebufferSurface"))?.takeUnretainedValue(), + CFGetTypeID(surfaceValue) == IOSurfaceGetTypeID() else { return } + let surface = unsafeBitCast(surfaceValue, to: IOSurfaceRef.self) as IOSurface + self.handle(surface: surface) + } + function(descriptor, damageSelector, damageCallbackUUID, block) + damageRegistered = true + } else { + eprint("[sim-capture] WARN no damage callback selector") + } + } + + func stop() { + if registered { + let selector = NSSelectorFromString("unregisterIOSurfacesChangeCallbackWithUUID:") + typealias Signature = @convention(c) (AnyObject, Selector, NSUUID) -> Void + if let implementation = descriptor.method(for: selector) { + unsafeBitCast(implementation, to: Signature.self)(descriptor, selector, callbackUUID) + } + registered = false + } + if damageRegistered { + let selector = NSSelectorFromString("unregisterDamageRectanglesCallbackWithUUID:") + typealias Signature = @convention(c) (AnyObject, Selector, NSUUID) -> Void + if let implementation = descriptor.method(for: selector) { + unsafeBitCast(implementation, to: Signature.self)(descriptor, selector, damageCallbackUUID) + } + damageRegistered = false + } + } + + func handle(surface: IOSurface, force: Bool = false) { + let now = Date() + if !force && now.timeIntervalSince(lastEmitTime) < (1.0 / Double(maxFps)) { + return + } + objc_sync_enter(self) + if encoding { + objc_sync_exit(self) + return + } + encoding = true + objc_sync_exit(self) + defer { + objc_sync_enter(self) + encoding = false + objc_sync_exit(self) + } + lastEmitTime = now + let image = CIImage(ioSurface: surface) + let opts: [CIImageRepresentationOption: Any] = [ + CIImageRepresentationOption(rawValue: kCGImageDestinationLossyCompressionQuality as String): jpegQuality, + ] + guard let jpeg = ciContext.jpegRepresentation( + of: image, + colorSpace: CGColorSpaceCreateDeviceRGB(), + options: opts + ) else { return } + var length = UInt32(jpeg.count).bigEndian + let header = withUnsafeBytes(of: &length) { Data($0) } + writeLock.lock() + defer { writeLock.unlock() } + do { + try stdout.write(contentsOf: header) + try stdout.write(contentsOf: jpeg) + } catch { + exit(0) + } + frameCount += 1 + let elapsed = now.timeIntervalSince(lastReportTime) + if elapsed >= 5 { + eprint("[sim-capture] fps≈\(Int(Double(frameCount) / elapsed))") + frameCount = 0 + lastReportTime = now + } + } +} + +guard let context = bootstrap(), + let deviceSet = defaultDeviceSet(context) else { + exit(2) +} + +if options.smokeTest { + _ = deviceSet.value(forKey: "devices") as? NSArray + jsonStatus(["type": "smoke-ok"]) + exit(0) +} + +guard let requestedUdid = options.udid, !requestedUdid.isEmpty else { + eprint("[sim-capture] --udid is required") + exit(64) +} + +var currentStream: Stream? +var currentDeviceUdid = "" + +let sigTerm = DispatchSource.makeSignalSource(signal: SIGTERM, queue: .main) +sigTerm.setEventHandler { + currentStream?.stop() + exit(0) +} +sigTerm.resume() +signal(SIGTERM, SIG_IGN) + +let sigInt = DispatchSource.makeSignalSource(signal: SIGINT, queue: .main) +sigInt.setEventHandler { + currentStream?.stop() + exit(0) +} +sigInt.resume() +signal(SIGINT, SIG_IGN) + +DispatchQueue.global(qos: .userInitiated).async { + var notifiedNoBoot = false + while true { + if let device = bootedDevice(deviceSet, udid: requestedUdid) { + let udid = deviceUdid(device) + if udid != currentDeviceUdid { + currentStream?.stop() + currentStream = nil + if let descriptor = findDisplayDescriptor(device) { + let stream = Stream(descriptor: descriptor, device: device, targetUdid: udid, maxFps: options.fps, jpegQuality: options.quality) + stream.start() + currentStream = stream + currentDeviceUdid = udid + notifiedNoBoot = false + } else { + eprint("[sim-capture] no display descriptor on booted device (will retry)") + } + } + } else { + if currentStream != nil { + eprint("[sim-capture] booted device gone") + currentStream?.stop() + currentStream = nil + currentDeviceUdid = "" + } + if !notifiedNoBoot { + jsonStatus(["type": "no-booted-device", "deviceUDID": requestedUdid]) + notifiedNoBoot = true + } + } + Thread.sleep(forTimeInterval: 1.0) + } +} + +RunLoop.main.run() diff --git a/apps/desktop/native/ios-sim-helpers/sim-input.m b/apps/desktop/native/ios-sim-helpers/sim-input.m new file mode 100644 index 000000000..ab6b324f7 --- /dev/null +++ b/apps/desktop/native/ios-sim-helpers/sim-input.m @@ -0,0 +1,428 @@ +#import +#import +#import +#import +#import +#import + +#pragma pack(push, 4) +typedef struct { + unsigned int msgh_bits; + unsigned int msgh_size; + unsigned int msgh_remote_port; + unsigned int msgh_local_port; + unsigned int msgh_voucher_port; + unsigned int msgh_id; +} IndigoMachHeader; + +typedef struct { + unsigned int field1; + unsigned int field2; + unsigned int field3; + double xRatio; + double yRatio; + double field6; + double field7; + double field8; + unsigned int field9; + unsigned int field10; + unsigned int field11; + unsigned int field12; + unsigned int field13; + double field14; + double field15; + double field16; + double field17; + double field18; +} IndigoTouch; + +typedef struct { + unsigned int eventSource; + unsigned int eventType; + unsigned int eventTarget; + unsigned int keyCode; + unsigned int field5; +} IndigoButton; + +typedef union { + IndigoTouch touch; + IndigoButton button; + unsigned char raw[144]; +} IndigoEvent; + +typedef struct { + unsigned int field1; + unsigned long long timestamp; + unsigned int field3; + IndigoEvent event; +} IndigoPayload; + +typedef struct { + IndigoMachHeader header; + unsigned int innerSize; + unsigned char eventType; + IndigoPayload payload; +} IndigoMessage; +#pragma pack(pop) + +#define IndigoEventTypeButton 1 +#define IndigoEventTypeTouch 2 +#define ButtonEventSourceHomeButton 0x0 +#define ButtonEventSourceLock 0x1 +#define ButtonEventSourceSideButton 0xbb8 +#define ButtonEventSourceSiri 0x400002 +#define ButtonEventSourceApplePay 0x1f4 +#define ButtonEventTargetHardware 0x33 +#define ButtonEventTypeDown 0x1 +#define ButtonEventTypeUp 0x2 + +typedef IndigoMessage *(*IndigoButtonFn)(int keyCode, int op, int target); +typedef IndigoMessage *(*IndigoMouseFn)(CGPoint *point0, CGPoint *point1, int target, int eventType, BOOL extra); + +static NSString *gDeviceUdid = nil; +static id gHidClient = nil; +static IndigoButtonFn gButtonFn = NULL; +static IndigoMouseFn gMouseFn = NULL; +static dispatch_queue_t gSendQueue; + +static void elog(NSString *fmt, ...) { + va_list args; + va_start(args, fmt); + NSString *message = [[NSString alloc] initWithFormat:fmt arguments:args]; + va_end(args); + NSData *data = [[message stringByAppendingString:@"\n"] dataUsingEncoding:NSUTF8StringEncoding]; + [[NSFileHandle fileHandleWithStandardError] writeData:data]; +} + +static void ack(NSString *eventId, BOOL ok, NSString *error) { + NSMutableDictionary *payload = [NSMutableDictionary dictionary]; + if (eventId.length) payload[@"id"] = eventId; + payload[@"ok"] = @(ok); + if (error.length) payload[@"error"] = error; + NSData *data = [NSJSONSerialization dataWithJSONObject:payload options:0 error:nil]; + if (!data) return; + NSMutableData *line = [data mutableCopy]; + [line appendData:[@"\n" dataUsingEncoding:NSUTF8StringEncoding]]; + [[NSFileHandle fileHandleWithStandardOutput] writeData:line]; +} + +static NSString *developerDir(void) { + NSTask *task = [NSTask new]; + task.launchPath = @"/usr/bin/xcode-select"; + task.arguments = @[@"-p"]; + NSPipe *pipe = [NSPipe pipe]; + task.standardOutput = pipe; + @try { + [task launch]; + [task waitUntilExit]; + } @catch (id ignored) {} + NSString *raw = [[NSString alloc] initWithData:pipe.fileHandleForReading.readDataToEndOfFile encoding:NSUTF8StringEncoding]; + raw = [raw stringByTrimmingCharactersInSet:NSCharacterSet.whitespaceAndNewlineCharacterSet]; + return raw.length ? raw : @"/Applications/Xcode.app/Contents/Developer"; +} + +static NSString *simulatorKitPath(void) { + NSString *developer = developerDir(); + NSArray *candidates = @[ + [developer stringByAppendingPathComponent:@"Platforms/iPhoneSimulator.platform/Developer/Library/PrivateFrameworks/SimulatorKit.framework/SimulatorKit"], + [developer stringByAppendingPathComponent:@"Library/PrivateFrameworks/SimulatorKit.framework/SimulatorKit"], + ]; + for (NSString *candidate in candidates) { + if ([[NSFileManager defaultManager] fileExistsAtPath:candidate]) return candidate; + } + return candidates.firstObject; +} + +static id sharedServiceContext(void) { + Class cls = NSClassFromString(@"SimServiceContext"); + if (!cls) { + elog(@"[sim-input] SimServiceContext missing"); + return nil; + } + SEL selector = @selector(sharedServiceContextForDeveloperDir:error:); + NSError *error = nil; + id (*fn)(Class, SEL, NSString *, NSError **) = (id (*)(Class, SEL, NSString *, NSError **))objc_msgSend; + id context = fn(cls, selector, developerDir(), &error); + if (!context) elog(@"[sim-input] sharedServiceContext err: %@", error); + return context; +} + +static id defaultDeviceSet(id context) { + SEL selector = @selector(defaultDeviceSetWithError:); + NSError *error = nil; + id (*fn)(id, SEL, NSError **) = (id (*)(id, SEL, NSError **))objc_msgSend; + id deviceSet = fn(context, selector, &error); + if (!deviceSet) elog(@"[sim-input] defaultDeviceSet err: %@", error); + return deviceSet; +} + +static NSString *deviceUDID(id device) { + id raw = [device valueForKey:@"UDID"]; + if ([raw isKindOfClass:NSUUID.class]) return [raw UUIDString]; + if ([raw isKindOfClass:NSString.class]) return raw; + return @"?"; +} + +static id bootedDevice(id deviceSet) { + NSArray *devices = [deviceSet valueForKey:@"devices"]; + for (id device in devices) { + NSNumber *state = [device valueForKey:@"state"]; + if (state.intValue != 3) continue; + if (gDeviceUdid.length && [deviceUDID(device) caseInsensitiveCompare:gDeviceUdid] != NSOrderedSame) continue; + return device; + } + return nil; +} + +static BOOL loadIndigoSymbolsOnly(void) { + if (!dlopen("/Library/Developer/PrivateFrameworks/CoreSimulator.framework/CoreSimulator", RTLD_NOW)) { + elog(@"[sim-input] FAIL dlopen CoreSimulator: %s", dlerror()); + return NO; + } + NSString *kitPath = simulatorKitPath(); + void *kit = dlopen(kitPath.fileSystemRepresentation, RTLD_NOW); + if (!kit) { + elog(@"[sim-input] FAIL dlopen SimulatorKit (%@): %s", kitPath, dlerror()); + return NO; + } + gButtonFn = (IndigoButtonFn)dlsym(kit, "IndigoHIDMessageForButton"); + gMouseFn = (IndigoMouseFn)dlsym(kit, "IndigoHIDMessageForMouseNSEvent"); + if (!gButtonFn || !gMouseFn) { + elog(@"[sim-input] FAIL Indigo dlsym button=%p mouse=%p", gButtonFn, gMouseFn); + return NO; + } + return YES; +} + +static BOOL ensureHID(void) { + if (gHidClient) return YES; + if (!loadIndigoSymbolsOnly()) return NO; + id context = sharedServiceContext(); + if (!context) return NO; + id deviceSet = defaultDeviceSet(context); + if (!deviceSet) return NO; + id device = bootedDevice(deviceSet); + if (!device) { + elog(@"[sim-input] no booted device for %@", gDeviceUdid ?: @"requested UDID"); + return NO; + } + Class clientClass = objc_lookUpClass("_TtC12SimulatorKit24SimDeviceLegacyHIDClient"); + if (!clientClass) clientClass = NSClassFromString(@"SimulatorKit.SimDeviceLegacyHIDClient"); + if (!clientClass) { + elog(@"[sim-input] FAIL no SimDeviceLegacyHIDClient class"); + return NO; + } + NSError *error = nil; + id allocated = [clientClass alloc]; + SEL selector = @selector(initWithDevice:error:); + id (*initFn)(id, SEL, id, NSError **) = (id (*)(id, SEL, id, NSError **))objc_msgSend; + id client = initFn(allocated, selector, device, &error); + if (!client) { + elog(@"[sim-input] FAIL init HID client: %@", error); + return NO; + } + gHidClient = client; + gSendQueue = dispatch_queue_create("app.ade.ios-sim.input", DISPATCH_QUEUE_SERIAL); + elog(@"[sim-input] HID client ready dev=%@ udid=%@", [device valueForKey:@"name"], deviceUDID(device)); + return YES; +} + +static BOOL sendIndigo(IndigoMessage *message, NSString **errorOut) { + if (!gHidClient || !message) { + if (errorOut) *errorOut = @"Indigo HID client is not available."; + if (message) free(message); + return NO; + } + SEL selector = @selector(sendWithMessage:freeWhenDone:completionQueue:completion:); + dispatch_semaphore_t done = dispatch_semaphore_create(0); + __block NSError *sendError = nil; + void (^completion)(NSError *) = ^(NSError *error) { + sendError = error; + if (error) elog(@"[sim-input] send err: %@", error); + dispatch_semaphore_signal(done); + }; + void (*sendFn)(id, SEL, IndigoMessage *, BOOL, dispatch_queue_t, void(^)(NSError *)) = + (void (*)(id, SEL, IndigoMessage *, BOOL, dispatch_queue_t, void(^)(NSError *)))objc_msgSend; + sendFn(gHidClient, selector, message, YES, gSendQueue, completion); + long wait = dispatch_semaphore_wait(done, dispatch_time(DISPATCH_TIME_NOW, 2 * NSEC_PER_SEC)); + if (wait != 0) { + if (errorOut) *errorOut = @"Timed out waiting for Indigo HID send completion."; + return NO; + } + if (sendError) { + if (errorOut) *errorOut = sendError.localizedDescription ?: @"Indigo HID send failed."; + return NO; + } + return YES; +} + +static BOOL sendTouch(double xRatio, double yRatio, BOOL down, NSString **errorOut) { + CGPoint point = CGPointMake(xRatio, yRatio); + int eventType = down ? ButtonEventTypeDown : ButtonEventTypeUp; + IndigoMessage *seed = gMouseFn(&point, NULL, 0x32, eventType, NO); + if (!seed) { + if (errorOut) *errorOut = @"Indigo mouse event builder returned NULL."; + elog(@"[sim-input] %@", errorOut ? *errorOut : @"MouseFn returned NULL"); + return NO; + } + size_t messageSize = sizeof(IndigoMessage) + sizeof(IndigoPayload); + size_t stride = sizeof(IndigoPayload); + IndigoMessage *message = calloc(1, messageSize); + message->innerSize = (unsigned int)sizeof(IndigoPayload); + message->eventType = IndigoEventTypeTouch; + message->payload.field1 = 0x0000000b; + message->payload.timestamp = mach_absolute_time(); + memcpy(&message->payload.event.touch, &seed->payload.event.touch, sizeof(IndigoTouch)); + message->payload.event.touch.xRatio = xRatio; + message->payload.event.touch.yRatio = yRatio; + void *first = &message->payload; + void *second = (void *)((uintptr_t)first + stride); + memcpy(second, first, stride); + IndigoPayload *secondPayload = (IndigoPayload *)second; + secondPayload->event.touch.field1 = 0x00000001; + secondPayload->event.touch.field2 = 0x00000002; + free(seed); + return sendIndigo(message, errorOut); +} + +static BOOL sendButton(NSString *name, BOOL down, NSString **errorOut) { + int source = ButtonEventSourceHomeButton; + if ([name isEqualToString:@"home"]) source = ButtonEventSourceHomeButton; + else if ([name isEqualToString:@"lock"]) source = ButtonEventSourceLock; + else if ([name isEqualToString:@"side"]) source = ButtonEventSourceSideButton; + else if ([name isEqualToString:@"siri"]) source = ButtonEventSourceSiri; + else if ([name isEqualToString:@"applepay"]) source = ButtonEventSourceApplePay; + else { + if (errorOut) *errorOut = [NSString stringWithFormat:@"Unknown Indigo button: %@", name ?: @""]; + elog(@"[sim-input] unknown button %@", name); + return NO; + } + int operation = down ? ButtonEventTypeDown : ButtonEventTypeUp; + return sendIndigo(gButtonFn(source, operation, ButtonEventTargetHardware), errorOut); +} + +static BOOL processEvent(NSDictionary *event, NSString **errorOut) { + if (!ensureHID()) { + if (errorOut) *errorOut = @"Indigo HID client is not available."; + return NO; + } + NSString *type = event[@"type"]; + if ([type isEqualToString:@"touch"]) { + NSString *phase = event[@"phase"] ?: @"down"; + return sendTouch([event[@"x"] doubleValue], [event[@"y"] doubleValue], ![phase isEqualToString:@"up"], errorOut); + } + if ([type isEqualToString:@"tap"]) { + double x = [event[@"x"] doubleValue]; + double y = [event[@"y"] doubleValue]; + int hold = event[@"hold"] ? [event[@"hold"] intValue] : 45; + if (!sendTouch(x, y, YES, errorOut)) return NO; + usleep((useconds_t)(MAX(0, hold) * 1000)); + return sendTouch(x, y, NO, errorOut); + } + if ([type isEqualToString:@"swipe"]) { + double startX = [event[@"startX"] doubleValue]; + double startY = [event[@"startY"] doubleValue]; + double endX = [event[@"endX"] doubleValue]; + double endY = [event[@"endY"] doubleValue]; + int durationMs = event[@"durationMs"] ? [event[@"durationMs"] intValue] : 180; + int steps = MAX(4, MIN(30, durationMs / 16)); + if (!sendTouch(startX, startY, YES, errorOut)) return NO; + for (int i = 1; i < steps; i++) { + double progress = (double)i / (double)steps; + if (!sendTouch(startX + ((endX - startX) * progress), startY + ((endY - startY) * progress), YES, errorOut)) return NO; + usleep((useconds_t)(MAX(1, durationMs / steps) * 1000)); + } + return sendTouch(endX, endY, NO, errorOut); + } + if ([type isEqualToString:@"button"]) { + NSString *phase = event[@"phase"] ?: @"down"; + return sendButton(event[@"name"] ?: @"home", [phase isEqualToString:@"down"], errorOut); + } + if ([type isEqualToString:@"button-tap"]) { + NSString *name = event[@"name"] ?: @"home"; + if (!sendButton(name, YES, errorOut)) return NO; + usleep(80000); + return sendButton(name, NO, errorOut); + } + if ([type isEqualToString:@"text"] || [type isEqualToString:@"key"]) { + if (errorOut) *errorOut = @"Indigo text and keyboard events are not supported by this helper yet."; + return NO; + } + if (errorOut) *errorOut = [NSString stringWithFormat:@"Unknown input event type: %@", type ?: @""]; + return NO; +} + +static NSString *readArgValue(int argc, const char **argv, const char *name) { + for (int i = 1; i + 1 < argc; i++) { + if (strcmp(argv[i], name) == 0) return [NSString stringWithUTF8String:argv[i + 1]]; + } + return nil; +} + +static BOOL hasFlag(int argc, const char **argv, const char *name) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], name) == 0) return YES; + } + return NO; +} + +int main(int argc, const char **argv) { + @autoreleasepool { + if (hasFlag(argc, argv, "--smoke-test")) { + BOOL ok = loadIndigoSymbolsOnly(); + if (ok) { + ack(@"smoke", YES, nil); + return 0; + } + ack(@"smoke", NO, @"Indigo symbols are unavailable."); + return 2; + } + gDeviceUdid = readArgValue(argc, argv, "--udid"); + if (!gDeviceUdid.length) { + elog(@"[sim-input] --udid is required"); + return 64; + } + ensureHID(); + elog(@"[sim-input] ready"); + NSFileHandle *input = [NSFileHandle fileHandleWithStandardInput]; + NSMutableData *buffer = [NSMutableData data]; + while (true) { + NSData *chunk; + @try { + chunk = [input availableData]; + } @catch (id ignored) { + break; + } + if (chunk.length == 0) break; + [buffer appendData:chunk]; + while (true) { + const char *bytes = buffer.bytes; + NSUInteger length = buffer.length; + NSUInteger newline = NSNotFound; + for (NSUInteger i = 0; i < length; i++) { + if (bytes[i] == '\n') { + newline = i; + break; + } + } + if (newline == NSNotFound) break; + NSData *line = [buffer subdataWithRange:NSMakeRange(0, newline)]; + [buffer replaceBytesInRange:NSMakeRange(0, newline + 1) withBytes:NULL length:0]; + if (line.length == 0) continue; + NSError *jsonError = nil; + id object = [NSJSONSerialization JSONObjectWithData:line options:0 error:&jsonError]; + if (![object isKindOfClass:NSDictionary.class]) { + ack(nil, NO, jsonError.localizedDescription ?: @"Input was not a JSON object."); + continue; + } + NSDictionary *event = object; + NSString *eventId = [event[@"id"] isKindOfClass:NSString.class] ? event[@"id"] : nil; + NSString *eventError = nil; + BOOL ok = processEvent(event, &eventError); + ack(eventId, ok, eventError); + } + } + elog(@"[sim-input] stdin closed, exiting"); + } + return 0; +} diff --git a/apps/desktop/src/main/services/git/gitOperationsService.branchSwitch.test.ts b/apps/desktop/src/main/services/git/gitOperationsService.branchSwitch.test.ts index 9d552d62e..505f8931d 100644 --- a/apps/desktop/src/main/services/git/gitOperationsService.branchSwitch.test.ts +++ b/apps/desktop/src/main/services/git/gitOperationsService.branchSwitch.test.ts @@ -179,6 +179,50 @@ describe("gitOperationsService.listBranches annotations", () => { const branches = await service.listBranches({ laneId: "lane-1" }); expect(branches.find((b) => b.name === "origin/HEAD")).toBeUndefined(); }); + + it("attaches last-commit metadata and rejoins tab-containing subjects", async () => { + // Subject is the last column so any tab inside it stays inside parts[7+] + // and is rejoined — see the FORMAT comment in gitOperationsService. + mockGit.runGitOrThrow.mockResolvedValue( + [ + [ + "refs/heads/feat/widget", + "feat/widget", + "*", + "origin/feat/widget", + "abc1234", + "2026-04-30T10:00:00+00:00", + "Arul Sharma", + "tweak\twidget alignment", + ].join("\t"), + [ + "refs/remotes/origin/feat/sidebar", + "origin/feat/sidebar", + " ", + "", + "def5678", + "2026-04-29T08:00:00+00:00", + "Jamie Lee", + "rebuild sidebar nav", + ].join("\t"), + ].join("\n"), + ); + + const { service } = makeServiceWithLanes({}); + const branches = await service.listBranches({ laneId: "lane-1" }); + const local = branches.find((b) => b.name === "feat/widget"); + expect(local).toBeDefined(); + expect(local!.lastCommitSha).toBe("abc1234"); + expect(local!.lastCommitDate).toBe("2026-04-30T10:00:00+00:00"); + expect(local!.lastCommitAuthor).toBe("Arul Sharma"); + expect(local!.lastCommitMessage).toBe("tweak\twidget alignment"); + + const remote = branches.find((b) => b.name === "origin/feat/sidebar"); + expect(remote).toBeDefined(); + expect(remote!.lastCommitSha).toBe("def5678"); + expect(remote!.lastCommitAuthor).toBe("Jamie Lee"); + expect(remote!.lastCommitMessage).toBe("rebuild sidebar nav"); + }); }); describe("gitOperationsService.checkoutBranch", () => { diff --git a/apps/desktop/src/main/services/git/gitOperationsService.ts b/apps/desktop/src/main/services/git/gitOperationsService.ts index 59aa949ec..448af5c2d 100644 --- a/apps/desktop/src/main/services/git/gitOperationsService.ts +++ b/apps/desktop/src/main/services/git/gitOperationsService.ts @@ -6,6 +6,7 @@ import type { GitBatchFileActionArgs, GitBranchSummary, GitCheckoutBranchArgs, + GitUserIdentity, GitCherryPickArgs, GitCommitArgs, GitGenerateCommitMessageArgs, @@ -1091,8 +1092,20 @@ export function createGitOperationsService({ async listBranches(args: { laneId: string }): Promise { const lane = laneService.getLaneBaseAndBranch(args.laneId); + // Subject goes last so a tab inside a commit subject doesn't desync the + // parser — anything after index 7 is rejoined. + const FORMAT = [ + "%(refname)", + "%(refname:short)", + "%(HEAD)", + "%(upstream:short)", + "%(objectname)", + "%(committerdate:iso-strict)", + "%(authorname)", + "%(subject)", + ].join("\t"); const out = await runGitOrThrow( - ["for-each-ref", "--sort=refname", "--format=%(refname)\t%(refname:short)\t%(HEAD)\t%(upstream:short)", "refs/heads", "refs/remotes"], + ["for-each-ref", "--sort=refname", `--format=${FORMAT}`, "refs/heads", "refs/remotes"], { cwd: lane.worktreePath, timeoutMs: 15_000 } ); const branchProfiles = new Set(); @@ -1136,21 +1149,28 @@ export function createGitOperationsService({ const shortRef = parts[1]?.trim() ?? ""; if (!fullRef || !shortRef) return; + const commitMeta = { + lastCommitSha: parts[4]?.trim() || undefined, + lastCommitDate: parts[5]?.trim() || undefined, + lastCommitAuthor: parts[6]?.trim() || undefined, + lastCommitMessage: parts.slice(7).join("\t").trim() || undefined, + }; + if (fullRef.startsWith("refs/heads/")) { const isCurrent = (parts[2]?.trim() ?? "") === "*"; const upstream = parts[3]?.trim() || null; - localBranches.set(shortRef, annotate({ name: shortRef, isCurrent, isRemote: false, upstream })); + localBranches.set( + shortRef, + annotate({ name: shortRef, isCurrent, isRemote: false, upstream, ...commitMeta }), + ); return; } if (fullRef.startsWith("refs/remotes/")) { if (shortRef.endsWith("/HEAD")) return; - remoteBranches.push(annotate({ - name: shortRef, - isCurrent: false, - isRemote: true, - upstream: null - })); + remoteBranches.push( + annotate({ name: shortRef, isCurrent: false, isRemote: true, upstream: null, ...commitMeta }), + ); } }); @@ -1169,6 +1189,19 @@ export function createGitOperationsService({ return [...sortedLocals, ...sortedRemotes]; }, + async getUserIdentity(args: { laneId: string }): Promise { + const lane = laneService.getLaneBaseAndBranch(args.laneId); + const readConfig = async (key: string): Promise => { + const result = await runGit(["config", "--get", key], { + cwd: lane.worktreePath, + timeoutMs: 5_000, + }); + return result.exitCode === 0 ? result.stdout.trim() : ""; + }; + const [name, email] = await Promise.all([readConfig("user.name"), readConfig("user.email")]); + return { name, email }; + }, + async checkoutBranch(args: GitCheckoutBranchArgs): Promise { const branchName = args.branchName.trim(); if (!branchName.length) throw new Error("Branch name is required"); diff --git a/apps/desktop/src/main/services/ios/iosSimulatorService.test.ts b/apps/desktop/src/main/services/ios/iosSimulatorService.test.ts index 36cd15a00..ba4ecc6c7 100644 --- a/apps/desktop/src/main/services/ios/iosSimulatorService.test.ts +++ b/apps/desktop/src/main/services/ios/iosSimulatorService.test.ts @@ -7,6 +7,8 @@ import { __testSetIosSimulatorProcessHooks, createIosSimulatorService, IosSimulatorOwnedBySessionError, + iosurfaceInputScreenFromSnapshot, + normalizeIosSimulatorPointForIndigo, parseXcodePreviewWindows, resolveIosSimulatorStreamBackend, shouldOpenSimulatorAppForLaunch, @@ -149,8 +151,8 @@ describe("iosSimulatorService shutdown contract", () => { }); }); -describe("iosSimulatorService background Simulator.app launch behavior", () => { - it("keeps Simulator.app capturable in the background by default during launch", async () => { +describe("iosSimulatorService Simulator.app launch visibility", () => { + it("does not open Simulator.app by default during headless launch", async () => { const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); const runMock = vi.fn(async (command: string, commandArgs: string[]) => { if (command === "ps") return { stdout: "", stderr: "" }; @@ -184,9 +186,8 @@ describe("iosSimulatorService background Simulator.app launch behavior", () => { bundleId: "com.example.app", build: false, }); - const spawnCalls = spawnMock.mock.calls; - expect(spawnCalls.filter(([command, commandArgs]) => command === "open" && commandArgs.join(" ") === "-g -a Simulator")).toHaveLength(2); - expect(spawnCalls.some(([command]) => command === "osascript")).toBe(false); + expect(spawnMock.mock.calls.some(([command]) => command === "open")).toBe(false); + expect(spawnMock.mock.calls.some(([command]) => command === "osascript")).toBe(false); } finally { service.dispose(); fs.rmSync(projectRoot, { recursive: true, force: true }); @@ -240,15 +241,95 @@ describe("iosSimulatorService background Simulator.app launch behavior", () => { } }); + it("opens Simulator.app when explicit window capture streaming is requested", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + const runMock = vi.fn(async (command: string, commandArgs: string[]) => { + if (command === "xcrun" && commandArgs.join(" ") === "simctl list devices available --json") { + return { stdout: simulatorDevicesJson, stderr: "" }; + } + return { stdout: "", stderr: "" }; + }); + const spawnMock = vi.fn<[string, string[], unknown?], ChildProcess>(() => mockChildProcess()); + const restoreHooks = __testSetIosSimulatorProcessHooks({ + run: runMock, + spawn: spawnMock as unknown as typeof nodeSpawn, + commandExists: () => true, + }); + const service = createIosSimulatorService({ projectRoot: os.tmpdir(), logger: noopLogger }); + + try { + const status = await service.startStream({ deviceUdid: "device-1", backend: "simulator-window-capture" }); + expect(status.backend).toBe("simulator-window-capture"); + expect(spawnMock).toHaveBeenCalledWith("open", ["-a", "Simulator"], { detached: true, stdio: "ignore" }); + } finally { + service.dispose(); + restoreHooks(); + platformSpy.mockRestore(); + } + }); + it("documents launch and stream backend defaults with pure helpers", () => { expect(shouldOpenSimulatorAppForLaunch(undefined)).toBe(false); expect(shouldOpenSimulatorAppForLaunch(true)).toBe(false); expect(shouldOpenSimulatorAppForLaunch(false)).toBe(true); - expect(resolveIosSimulatorStreamBackend("auto", { idb: true, idbCompanion: true, ffmpeg: true })).toBe("idb-h264-ffmpeg-mjpeg"); - expect(resolveIosSimulatorStreamBackend("auto", { idb: true, idbCompanion: true, ffmpeg: false })).toBe("idb-mjpeg"); - expect(resolveIosSimulatorStreamBackend("auto", { idb: true, idbCompanion: false, ffmpeg: true })).toBe("simctl-screenshot-poll"); - expect(resolveIosSimulatorStreamBackend("simulator-window-capture", { idb: true, idbCompanion: true, ffmpeg: true })).toBe("simulator-window-capture"); - expect(resolveIosSimulatorStreamBackend("idb-h264-ffmpeg-mjpeg", { idb: true, idbCompanion: true, ffmpeg: true })).toBe("idb-h264-ffmpeg-mjpeg"); + const bools = [false, true]; + for (const iosurfaceIndigo of bools) { + for (const windowCaptureAllowed of bools) { + for (const idb of bools) { + for (const idbCompanion of bools) { + for (const ffmpeg of bools) { + const expected = iosurfaceIndigo + ? "iosurface-indigo" + : windowCaptureAllowed + ? "simulator-window-capture" + : idb && idbCompanion + ? "idb-mjpeg" + : "simctl-screenshot-poll"; + expect(resolveIosSimulatorStreamBackend("auto", { + iosurfaceIndigo, + windowCaptureAllowed, + idb, + idbCompanion, + ffmpeg, + })).toBe(expected); + } + } + } + } + } + const tools = { iosurfaceIndigo: true, windowCaptureAllowed: true, idb: true, idbCompanion: true, ffmpeg: true }; + expect(resolveIosSimulatorStreamBackend("simulator-window-capture", tools)).toBe("simulator-window-capture"); + expect(resolveIosSimulatorStreamBackend("idb-h264-ffmpeg-mjpeg", tools)).toBe("idb-h264-ffmpeg-mjpeg"); + }); + + it("normalizes point coordinates for Indigo input using screen points", () => { + expect(normalizeIosSimulatorPointForIndigo( + { x: 430, y: 932 }, + { width: 430, height: 932 }, + )).toEqual({ x: 1, y: 1 }); + expect(normalizeIosSimulatorPointForIndigo( + { x: 215, y: 466 }, + { width: 430, height: 932 }, + )).toEqual({ x: 0.5, y: 0.5 }); + expect(normalizeIosSimulatorPointForIndigo( + { x: 196, y: 400 }, + { width: 393, height: 852 }, + )).toEqual({ + x: 196 / 393, + y: 400 / 852, + }); + }); + + it("uses full screenshot dimensions for IOSurface input even when inspector screen is content-height only", () => { + const inputScreen = iosurfaceInputScreenFromSnapshot( + { width: 402, height: 778, scale: 3 }, + { width: 1206, height: 2622 }, + ); + expect(inputScreen).toEqual({ width: 402, height: 874, scale: 3 }); + expect(normalizeIosSimulatorPointForIndigo({ x: 201, y: 830 }, inputScreen)).toEqual({ + x: 0.5, + y: 830 / 874, + }); }); }); diff --git a/apps/desktop/src/main/services/ios/iosSimulatorService.ts b/apps/desktop/src/main/services/ios/iosSimulatorService.ts index 8ceff7cc3..b66c1e041 100644 --- a/apps/desktop/src/main/services/ios/iosSimulatorService.ts +++ b/apps/desktop/src/main/services/ios/iosSimulatorService.ts @@ -1,4 +1,4 @@ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { execFile as execFileCallback, spawn, type ChildProcess } from "node:child_process"; import fs from "node:fs"; import http, { type ServerResponse } from "node:http"; @@ -66,18 +66,56 @@ const SIMCTL_BOOTSTATUS_TIMEOUT_MS = 90_000; const SIMCTL_INSTALL_TIMEOUT_MS = 180_000; const IDB_MJPEG_STARTUP_TIMEOUT_MS = 5_000; const IDB_H264_STARTUP_TIMEOUT_MS = 15_000; +const IOSURFACE_HELPER_STARTUP_TIMEOUT_MS = 5_000; +const IOSURFACE_INPUT_ACK_TIMEOUT_MS = 5_000; +const IOSURFACE_TAP_HOLD_MS = 45; +const IOSURFACE_FAILURE_WINDOW_MS = 60_000; +const IOSURFACE_MAX_FAILURES_PER_WINDOW = 2; +const IOSURFACE_SUPPORTED_XCODE_MAJORS = new Set([17, 26]); const INSTALL_HINT_XCODE = "Install Xcode from the App Store, then run xcode-select --install."; const INSTALL_HINT_XCODE_CLI = "Run xcode-select --install to install the Xcode command line tools."; +const INSTALL_HINT_IOSURFACE_INDIGO = "Install a supported full Xcode (17.x or 26.x) and select it with xcode-select so ADE can build the private IOSurface/Indigo helpers."; const INSTALL_HINT_IDB = "Install Facebook idb: brew tap facebook/fb && brew install idb-companion && pipx install fb-idb."; const INSTALL_HINT_IDB_COMPANION = "Install idb_companion: brew tap facebook/fb && brew install idb-companion."; const INSTALL_HINT_FFMPEG = "Install ffmpeg: brew install ffmpeg."; const XCODE_MCP_SESSION_ID = "906026e2-d248-4770-8654-032d0e1fbb54"; const XCODE_MCP_APPROVAL_TIMEOUT_MS = 90_000; +export const IOSURFACE_HELPER_UNAVAILABLE_CODE = "IOSURFACE_HELPER_UNAVAILABLE" as const; type RunCommand = (command: string, args: string[], options?: { cwd?: string; timeoutMs?: number; env?: NodeJS.ProcessEnv }) => Promise<{ stdout: string; stderr: string }>; type SpawnProcess = typeof spawn; type CommandExistsProbe = typeof commandExists; +type IosurfaceIndigoCapability = { + available: boolean; + reason?: string; + xcodeVersion?: string; + developerDir?: string | null; + helpers?: IosurfaceIndigoHelperBundle; +}; + +type IosurfaceIndigoHelperBundle = { + xcodeVersion: string; + sourceHash: string; + buildDir: string; + capture: string; + input: string; +}; + +type IosSimulatorStreamTools = { + iosurfaceIndigo: boolean; + windowCaptureAllowed: boolean; + idb: boolean; + idbCompanion: boolean; + ffmpeg: boolean; +}; + +type LatencyPercentiles = { + averageLatencyMs: number | null; + latencyP50Ms: number | null; + latencyP95Ms: number | null; +}; + export class IosSimulatorOwnedBySessionError extends Error { readonly code: typeof IOS_SIMULATOR_OWNED_BY_OTHER_SESSION_CODE = IOS_SIMULATOR_OWNED_BY_OTHER_SESSION_CODE; readonly currentChatSessionId: string | null; @@ -92,6 +130,17 @@ export class IosSimulatorOwnedBySessionError extends Error { } } +export class IosurfaceHelperUnavailableError extends Error { + readonly code: typeof IOSURFACE_HELPER_UNAVAILABLE_CODE = IOSURFACE_HELPER_UNAVAILABLE_CODE; + readonly reason: string; + + constructor(reason: string) { + super(`${IOSURFACE_HELPER_UNAVAILABLE_CODE}: ${reason}`); + this.name = "IosurfaceHelperUnavailableError"; + this.reason = reason; + } +} + type CreateIosSimulatorServiceArgs = { projectRoot: string; logger: Logger; @@ -255,14 +304,255 @@ export function shouldOpenSimulatorAppForLaunch(keepSimulatorInBackground?: bool export function resolveIosSimulatorStreamBackend( requestedBackend: "auto" | IosSimulatorStreamBackend, - tools: { idb: boolean; idbCompanion: boolean; ffmpeg: boolean }, + tools: IosSimulatorStreamTools, ): IosSimulatorStreamBackend { if (requestedBackend !== "auto") return requestedBackend; - if (tools.idb && tools.idbCompanion && tools.ffmpeg) return "idb-h264-ffmpeg-mjpeg"; + // Auto ranking: + // 1. iosurface-indigo: native-resolution IOSurface frames + Indigo HID. + // 2. simulator-window-capture: only when the caller permits a visible + // Simulator.app window and macOS window capture is reachable. + // 3. idb-mjpeg: mature idb fallback without extra transcoding. + // 4. idb-h264-ffmpeg-mjpeg: recovery-only, entered after idb-mjpeg fails + // inside a session; auto never picks it as the first backend. + // 5. simctl-screenshot-poll: last resort. + if (tools.iosurfaceIndigo) return "iosurface-indigo"; + if (tools.windowCaptureAllowed) return "simulator-window-capture"; if (tools.idb && tools.idbCompanion) return "idb-mjpeg"; return "simctl-screenshot-poll"; } +export function normalizeIosSimulatorPointForIndigo( + point: { x: number; y: number }, + screen: Pick, +): { x: number; y: number } { + const width = Number(screen.width); + const height = Number(screen.height); + if (!Number.isFinite(width) || width <= 0 || !Number.isFinite(height) || height <= 0) { + throw new Error("Simulator screen metrics are required for Indigo input."); + } + return { + x: Math.max(0, Math.min(1, point.x / width)), + y: Math.max(0, Math.min(1, point.y / height)), + }; +} + +export function iosurfaceInputScreenFromSnapshot( + screen: IosInspectableScreen, + shot: Pick, +): IosInspectableScreen { + const scale = Number(screen.scale); + const width = Number(shot.width); + const height = Number(shot.height); + if (Number.isFinite(scale) && scale > 0 && Number.isFinite(width) && width > 0 && Number.isFinite(height) && height > 0) { + return { + width: width / scale, + height: height / scale, + scale, + }; + } + return screen; +} + +function parseXcodeMajor(version: string | null | undefined): number | null { + const match = /^(\d+)(?:\.|$)/.exec(version ?? ""); + if (!match) return null; + const major = Number(match[1]); + return Number.isFinite(major) ? major : null; +} + +function percentile(samples: number[], fraction: number): number | null { + if (!samples.length) return null; + const sorted = [...samples].sort((a, b) => a - b); + const index = Math.max(0, Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)); + return Math.round(sorted[index] ?? 0); +} + +function latencyPercentiles(samples: number[]): LatencyPercentiles { + if (!samples.length) { + return { averageLatencyMs: null, latencyP50Ms: null, latencyP95Ms: null }; + } + return { + averageLatencyMs: Math.round(samples.reduce((sum, sample) => sum + sample, 0) / samples.length), + latencyP50Ms: percentile(samples, 0.5), + latencyP95Ms: percentile(samples, 0.95), + }; +} + +function iosSimHelperRoot(): string | null { + const candidates = [ + path.join(process.cwd(), "native", "ios-sim-helpers"), + path.join(process.cwd(), "apps", "desktop", "native", "ios-sim-helpers"), + path.resolve(__dirname, "../../../../native/ios-sim-helpers"), + ]; + return candidates.find((candidate) => fs.existsSync(path.join(candidate, "build.sh"))) ?? null; +} + +function hashHelperSources(helperRoot: string): string { + const hash = createHash("sha256"); + for (const fileName of ["sim-capture.swift", "sim-input.m", "build.sh"]) { + hash.update(fileName); + hash.update(fs.readFileSync(path.join(helperRoot, fileName))); + } + return hash.digest("hex"); +} + +let iosurfaceCapabilityCache: { + developerDir: string | null; + value: Promise; + settled: IosurfaceIndigoCapability | null; +} | null = null; + +async function readSelectedDeveloperDir(): Promise { + try { + const { stdout } = await run("/usr/bin/xcode-select", ["-p"], { timeoutMs: 5_000 }); + return stdout.trim() || null; + } catch { + return null; + } +} + +async function readSelectedXcodeVersion(): Promise { + try { + const { stdout } = await run("xcodebuild", ["-version"], { timeoutMs: 5_000 }); + return /^Xcode\s+(.+)$/m.exec(stdout)?.[1]?.trim() ?? null; + } catch { + return null; + } +} + +function simulatorKitFrameworkPath(developerDir: string): string | null { + const candidates = [ + path.join(developerDir, "Platforms", "iPhoneSimulator.platform", "Developer", "Library", "PrivateFrameworks", "SimulatorKit.framework"), + path.join(developerDir, "Library", "PrivateFrameworks", "SimulatorKit.framework"), + ]; + return candidates.find((candidate) => fs.existsSync(candidate)) ?? null; +} + +function isPackagedElectronRuntime(): boolean { + if (process.env.ADE_IOS_SURFACE_ALLOW_PACKAGED === "1") return false; + try { + const electron = require("electron") as { app?: { isPackaged?: boolean } }; + return Boolean(electron.app?.isPackaged); + } catch { + return false; + } +} + +function unsupportedIosurfaceEnvironment(): IosurfaceIndigoCapability | null { + if (process.platform !== "darwin") { + return { available: false, reason: "IOSurface simulator streaming is only available on macOS." }; + } + if (isPackagedElectronRuntime()) { + return { + available: false, + reason: "IOSurface simulator streaming is disabled in packaged ADE builds until helper signing and notarization are cleared. Using the idb/window/simctl fallback chain.", + }; + } + return null; +} + +async function buildIosurfaceIndigoHelpers(helperRoot: string): Promise { + const { stdout } = await run("bash", [path.join(helperRoot, "build.sh"), "--print-json", "--smoke"], { + timeoutMs: 120_000, + }); + const parsed = JSON.parse(stdout.trim()) as Partial; + const { xcodeVersion, sourceHash, buildDir, capture, input } = parsed; + if (!capture || !input || !buildDir || !xcodeVersion || !sourceHash) { + throw new Error("Helper build script did not return expected paths."); + } + return { xcodeVersion, sourceHash, buildDir, capture, input }; +} + +export async function detectIosurfaceIndigoCapability(): Promise { + const unsupported = unsupportedIosurfaceEnvironment(); + if (unsupported) return unsupported; + const developerDir = await readSelectedDeveloperDir(); + if (iosurfaceCapabilityCache && iosurfaceCapabilityCache.developerDir === developerDir) { + return iosurfaceCapabilityCache.value; + } + const value = (async (): Promise => { + if (!developerDir) { + return { available: false, reason: "xcode-select does not point at a developer directory.", developerDir }; + } + const simulatorKit = simulatorKitFrameworkPath(developerDir); + if (!simulatorKit) { + return { + available: false, + reason: `Full Xcode is required for IOSurface streaming; SimulatorKit.framework was not found under ${developerDir}.`, + developerDir, + }; + } + const xcodeVersion = await readSelectedXcodeVersion(); + const xcodeMajor = parseXcodeMajor(xcodeVersion); + if (!xcodeVersion || xcodeMajor == null) { + return { available: false, reason: "Could not determine the selected Xcode version.", developerDir }; + } + if (!IOSURFACE_SUPPORTED_XCODE_MAJORS.has(xcodeMajor)) { + return { + available: false, + reason: `Xcode ${xcodeVersion} is not yet supported by ADE's IOSurface helper. Supported major versions: ${Array.from(IOSURFACE_SUPPORTED_XCODE_MAJORS).join(", ")}.`, + xcodeVersion, + developerDir, + }; + } + if (!commandExistsProbe("swiftc")) { + return { available: false, reason: "swiftc was not found on PATH.", xcodeVersion, developerDir }; + } + if (!commandExistsProbe("clang")) { + return { available: false, reason: "clang was not found on PATH.", xcodeVersion, developerDir }; + } + const helperRoot = iosSimHelperRoot(); + if (!helperRoot) { + return { available: false, reason: "ADE's native iOS simulator helper sources were not found.", xcodeVersion, developerDir }; + } + try { + hashHelperSources(helperRoot); + const helpers = await buildIosurfaceIndigoHelpers(helperRoot); + return { available: true, xcodeVersion, developerDir, helpers }; + } catch (error) { + return { + available: false, + reason: `IOSurface helper build or smoke test failed: ${error instanceof Error ? error.message : String(error)}`, + xcodeVersion, + developerDir, + }; + } + })(); + const nextCache = { developerDir, value, settled: null as IosurfaceIndigoCapability | null }; + iosurfaceCapabilityCache = nextCache; + value.then((capability) => { + if (iosurfaceCapabilityCache === nextCache) nextCache.settled = capability; + }, (error) => { + if (iosurfaceCapabilityCache === nextCache) { + nextCache.settled = { + available: false, + reason: error instanceof Error ? error.message : String(error), + developerDir, + }; + } + }); + return value; +} + +async function peekIosurfaceIndigoCapabilityForStatus(): Promise { + const unsupported = unsupportedIosurfaceEnvironment(); + if (unsupported) return unsupported; + const developerDir = await readSelectedDeveloperDir(); + if (iosurfaceCapabilityCache?.developerDir === developerDir) { + return iosurfaceCapabilityCache.settled ?? { + available: false, + reason: "Preparing IOSurface simulator helpers. ADE will use them once the background build and smoke test finish.", + developerDir, + }; + } + void detectIosurfaceIndigoCapability().catch(() => {}); + return { + available: false, + reason: "Preparing IOSurface simulator helpers. ADE will use them once the background build and smoke test finish.", + developerDir, + }; +} + function encodeMcpMessage(message: JsonRpcMessage): Buffer { return Buffer.from(`${JSON.stringify(message)}\n`, "utf8"); } @@ -1451,6 +1741,17 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { let streamProcess: ChildProcess | null = null; let streamTranscoderProcess: ChildProcess | null = null; let streamPollTimer: NodeJS.Timeout | null = null; + let iosurfaceInputProcess: ChildProcess | null = null; + let iosurfaceInputDeviceUdid: string | null = null; + let iosurfaceInputBuffer = ""; + let iosurfaceInputStderr = ""; + let iosurfaceInputPending = new Map void; reject: (error: Error) => void; timer: NodeJS.Timeout }>(); + let iosurfaceInputStickyFallbackSessionId: string | null = null; + let iosurfaceInputFailureTimestamps: number[] = []; + let iosurfaceFailureTimestamps: number[] = []; + let iosurfaceHelperIdleTimer: NodeJS.Timeout | null = null; + let iosurfaceLastInputAtMs = 0; + let iosurfaceScreenMetrics = new Map(); let companionProcess: ChildProcess | null = null; let companionDeviceUdid: string | null = null; let companionAddress: string | null = null; @@ -1459,10 +1760,16 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { let streamBuffer = Buffer.alloc(0); let streamServer: http.Server | null = null; let streamClients = new Set(); + let streamLatestFrame: { frame: Buffer; capturedAt: string } | null = null; let streamFpsWindowStartedAtMs = 0; let streamFpsWindowFrameCount = 0; let streamLatencySamples: number[] = []; let streamLastStatusEmitMs = 0; + let streamRequestContext: Pick = { + requestedBackend: null, + fallbackReason: null, + degradationReason: null, + }; let controlQueue: Promise = Promise.resolve(); let activeLaunchId: string | null = null; let disposed = false; @@ -1495,10 +1802,21 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { startedAt: null, lastFrameAt: null, lastError: null, + error: null, streamUrl: null, averageLatencyMs: null, + latencyP50Ms: null, + latencyP95Ms: null, + helperPid: null, + inputBackend: null, }; + void detectIosurfaceIndigoCapability().catch((error) => { + args.logger.debug("ios_simulator.iosurface_indigo_warmup_failed", { + error: error instanceof Error ? error.message : String(error), + }); + }); + void cleanupOrphanedIdbCompanions().then((stopped) => { if (stopped > 0) { args.logger.info("ios_simulator.cleaned_orphaned_idb_companions", { stopped }); @@ -1513,11 +1831,6 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { args.onEvent?.(payload); }; - const openSimulatorAppInBackground = () => { - if (process.platform !== "darwin") return; - spawnProcess("open", ["-g", "-a", "Simulator"], { detached: true, stdio: "ignore" }).unref(); - }; - const cachedCommandExists = (command: string, ttlMs = TOOL_STATUS_CACHE_MS): boolean => { const nowMs = Date.now(); const cached = toolAvailabilityCache.get(command); @@ -1674,19 +1987,32 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { } closeStreamServer(); streamBuffer = Buffer.alloc(0); + streamLatestFrame = null; streamFpsWindowStartedAtMs = 0; streamFpsWindowFrameCount = 0; streamLatencySamples = []; streamLastStatusEmitMs = 0; + if (iosurfaceHelperIdleTimer) { + clearTimeout(iosurfaceHelperIdleTimer); + iosurfaceHelperIdleTimer = null; + } streamStatus = { ...streamStatus, running: false, backend: null, + requestedBackend: null, + fallbackReason: null, + degradationReason: null, fps: null, targetFps: null, lastError: error, + error: error ? { code: "stream-stopped", exitCode: null, signal: null } : null, streamUrl: null, averageLatencyMs: null, + latencyP50Ms: null, + latencyP95Ms: null, + helperPid: null, + inputBackend: null, }; return streamStatus; }; @@ -1706,8 +2032,44 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { } }; + const stopIosurfaceInput = () => { + for (const pending of iosurfaceInputPending.values()) { + clearTimeout(pending.timer); + pending.reject(new Error("Indigo input helper stopped.")); + } + iosurfaceInputPending.clear(); + stopChild(iosurfaceInputProcess); + iosurfaceInputProcess = null; + iosurfaceInputDeviceUdid = null; + iosurfaceInputBuffer = ""; + iosurfaceInputStderr = ""; + }; + + const writeMjpegPayload = (client: ServerResponse, frame: Buffer, capturedAt: string): boolean => { + if (client.destroyed || client.writableEnded || client.writableFinished) return false; + const bufferedBytes = client.socket?.bufferSize ?? 0; + if (bufferedBytes > 2 * 1024 * 1024) return false; + const header = Buffer.from([ + `--${STREAM_BOUNDARY}`, + "Content-Type: image/jpeg", + `Content-Length: ${frame.length}`, + `X-ADE-Captured-At: ${capturedAt}`, + "", + "", + ].join("\r\n")); + const trailer = Buffer.from("\r\n"); + // `write()` returning false is normal backpressure, not a broken socket. + // The next frame will be skipped/dropped if the buffered byte count stays + // above our threshold. + client.write(header); + client.write(frame); + client.write(trailer); + return true; + }; + const startMjpegStreamServer = async (): Promise => { closeStreamServer(); + streamLatestFrame = null; streamClients = new Set(); streamServer = http.createServer((request, response) => { if (!request.url?.startsWith("/ios-simulator/stream.mjpg")) { @@ -1724,11 +2086,21 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { "Pragma": "no-cache", }); streamClients.add(response); + if (streamLatestFrame && !writeMjpegPayload(response, streamLatestFrame.frame, streamLatestFrame.capturedAt)) { + streamClients.delete(response); + response.destroy(); + return; + } + if (streamStatus.backend === "iosurface-indigo") { + scheduleIosurfaceHelperIdleStop(); + } request.on("close", () => { streamClients.delete(response); + if (streamStatus.backend === "iosurface-indigo") scheduleIosurfaceHelperIdleStop(); }); response.on("close", () => { streamClients.delete(response); + if (streamStatus.backend === "iosurface-indigo") scheduleIosurfaceHelperIdleStop(); }); }); // Bind atomically to an OS-assigned port to avoid the TOCTOU race that @@ -1754,31 +2126,37 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { return `http://127.0.0.1:${port}/ios-simulator/stream.mjpg`; }; + const scheduleIosurfaceHelperIdleStop = () => { + if (iosurfaceHelperIdleTimer) { + clearTimeout(iosurfaceHelperIdleTimer); + iosurfaceHelperIdleTimer = null; + } + if (!streamStatus.running || streamStatus.backend !== "iosurface-indigo") return; + iosurfaceHelperIdleTimer = setTimeout(() => { + iosurfaceHelperIdleTimer = null; + if ( + streamStatus.running + && streamStatus.backend === "iosurface-indigo" + && streamClients.size === 0 + && Date.now() - iosurfaceLastInputAtMs >= COMPANION_IDLE_STOP_MS + ) { + const detail = "IOSurface helper stopped after 30s with no active stream clients or input."; + stopChild(streamProcess); + const status = setStreamStopped(detail); + emit({ type: "stream-stopped", status }); + } + }, COMPANION_IDLE_STOP_MS); + iosurfaceHelperIdleTimer.unref?.(); + }; + const writeMjpegFrame = (frame: Buffer, capturedAt: string) => { + streamLatestFrame = { frame, capturedAt }; if (!streamClients.size) return; - const header = Buffer.from([ - `--${STREAM_BOUNDARY}`, - "Content-Type: image/jpeg", - `Content-Length: ${frame.length}`, - `X-ADE-Captured-At: ${capturedAt}`, - "", - "", - ].join("\r\n")); - const trailer = Buffer.from("\r\n"); for (const client of [...streamClients]) { - if (client.destroyed || client.writableEnded || client.writableFinished) { - streamClients.delete(client); - continue; - } - const bufferedBytes = client.socket?.bufferSize ?? 0; - if (bufferedBytes > 8 * 1024 * 1024) { + if (!writeMjpegPayload(client, frame, capturedAt)) { streamClients.delete(client); client.destroy(); - continue; } - client.write(header); - client.write(frame); - client.write(trailer); } }; @@ -1794,7 +2172,7 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { } const elapsedMs = nowMs - streamFpsWindowStartedAtMs; const measuredFps = elapsedMs >= 1_000 - ? Math.round((streamFpsWindowFrameCount * 1000) / Math.max(1, elapsedMs)) + ? Math.max(1, Math.round((streamFpsWindowFrameCount * 1000) / Math.max(1, elapsedMs))) : streamStatus.fps; const fps = measuredFps == null || streamStatus.targetFps == null ? measuredFps @@ -1803,16 +2181,17 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { streamFpsWindowStartedAtMs = nowMs; streamFpsWindowFrameCount = 0; } - const averageLatencyMs = streamLatencySamples.length - ? Math.round(streamLatencySamples.reduce((sum, sample) => sum + sample, 0) / streamLatencySamples.length) - : null; + const latency = latencyPercentiles(streamLatencySamples); streamStatus = { ...streamStatus, frameCount: streamStatus.frameCount + 1, fps, lastFrameAt: capturedAt, lastError: null, - averageLatencyMs, + error: null, + averageLatencyMs: latency.averageLatencyMs, + latencyP50Ms: latency.latencyP50Ms, + latencyP95Ms: latency.latencyP95Ms, }; if (nowMs - streamLastStatusEmitMs >= STREAM_STATUS_EMIT_INTERVAL_MS) { streamLastStatusEmitMs = nowMs; @@ -1943,6 +2322,42 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { } }; + const extractLengthPrefixedJpegFrame = (): Buffer | null => { + if (streamBuffer.length < 4) return null; + const length = streamBuffer.readUInt32BE(0); + if (!Number.isFinite(length) || length <= 0 || length > 50 * 1024 * 1024) { + streamBuffer = Buffer.alloc(0); + streamStatus = { ...streamStatus, lastError: "Dropped an invalid IOSurface frame payload." }; + emit({ type: "stream-error", status: streamStatus }); + return null; + } + if (streamBuffer.length < 4 + length) return null; + const frame = streamBuffer.subarray(4, 4 + length); + streamBuffer = streamBuffer.subarray(4 + length); + return frame; + }; + + const handleLengthPrefixedJpegChunk = (chunk: Buffer, backend: IosSimulatorStreamBackend) => { + streamBuffer = Buffer.concat([streamBuffer, chunk]); + let frame: Buffer | null = null; + while ((frame = extractLengthPrefixedJpegFrame()) !== null) { + const capturedAt = nowIso(); + emitFrame(frame, backend, capturedAt, "jpeg", 0); + } + }; + + const rememberIosurfaceFailure = (): number => { + const nowMs = Date.now(); + iosurfaceFailureTimestamps = [...iosurfaceFailureTimestamps.filter((timestamp) => nowMs - timestamp <= IOSURFACE_FAILURE_WINDOW_MS), nowMs]; + return iosurfaceFailureTimestamps.length; + }; + + const rememberIosurfaceInputFailure = (): number => { + const nowMs = Date.now(); + iosurfaceInputFailureTimestamps = [...iosurfaceInputFailureTimestamps.filter((timestamp) => nowMs - timestamp <= IOSURFACE_FAILURE_WINDOW_MS), nowMs]; + return iosurfaceInputFailureTimestamps.length; + }; + const computeListDevices = async (): Promise => { if (process.platform !== "darwin" || !cachedCommandExists("xcrun")) return []; const { stdout } = await run("xcrun", ["simctl", "list", "devices", "available", "--json"]); @@ -2145,7 +2560,7 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { return fallback; }; - const buildToolStatuses = (): IosSimulatorToolStatus[] => { + const buildToolStatuses = (iosurfaceCapability: IosurfaceIndigoCapability | null): IosSimulatorToolStatus[] => { const isDarwin = process.platform === "darwin"; const xcrunAvailable = isDarwin && cachedCommandExists("xcrun"); const xcodebuildAvailable = isDarwin && cachedCommandExists("xcodebuild"); @@ -2173,6 +2588,14 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { : "Only available on macOS.", installHint: isDarwin ? INSTALL_HINT_XCODE : "iOS Simulator control requires macOS.", }, + { + name: "iosurface_indigo", + available: Boolean(iosurfaceCapability?.available), + detail: iosurfaceCapability?.available + ? `Available as ADE's primary low-latency simulator stream${iosurfaceCapability.xcodeVersion ? ` on Xcode ${iosurfaceCapability.xcodeVersion}` : ""}.` + : iosurfaceCapability?.reason ?? (isDarwin ? "Checking native IOSurface/Indigo helpers." : "Only available on macOS."), + installHint: INSTALL_HINT_IOSURFACE_INDIGO, + }, { name: "simulator_window", available: isDarwin, @@ -2185,8 +2608,8 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { name: "idb", available: idbAvailable, detail: idbAvailable - ? "Available for exact-screen live streaming, taps, and text input." - : "Optional. Required for exact-screen streaming and pointer/text control.", + ? "Available for fallback streaming, accessibility reads, and pointer/text control." + : "Optional fallback. Required for idb streaming, accessibility reads, and pointer/text control when Indigo is unavailable.", installHint: INSTALL_HINT_IDB, }, { @@ -2201,8 +2624,8 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { name: "ffmpeg", available: ffmpegAvailable, detail: ffmpegAvailable - ? "Available for the low-latency idb H.264 stream." - : "Optional. Needed for the default low-latency idb H.264 stream.", + ? "Available for the recovery idb H.264 transcode stream." + : "Optional. Used only as an idb recovery stream after MJPEG fails.", installHint: INSTALL_HINT_FFMPEG, }, ]; @@ -2210,7 +2633,11 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { const computeStatus = async (): Promise => { const isDarwin = process.platform === "darwin"; - const tools = buildToolStatuses(); + const iosurfaceCapability = isDarwin ? await peekIosurfaceIndigoCapabilityForStatus().catch((error): IosurfaceIndigoCapability => ({ + available: false, + reason: error instanceof Error ? error.message : String(error), + })) : null; + const tools = buildToolStatuses(iosurfaceCapability); const devices = isDarwin ? await listDevices().catch(() => []) : []; const activeDevice = activeSession ? devices.find((device) => device.udid === activeSession?.deviceUdid) ?? null @@ -2707,9 +3134,7 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { spawnProcess("open", ["-a", "Simulator"], { detached: true, stdio: "ignore" }).unref(); emitLaunchProgress(launchId, "open-simulator", "complete", "Simulator.app is visible.", null, { deviceUdid: device.udid }); } else { - emitLaunchProgress(launchId, "open-simulator", "running", "Preparing Simulator.app behind ADE for live streaming...", null, { deviceUdid: device.udid }); - openSimulatorAppInBackground(); - emitLaunchProgress(launchId, "open-simulator", "complete", "Simulator.app is running behind ADE.", null, { deviceUdid: device.udid }); + emitLaunchProgress(launchId, "open-simulator", "skipped", "Simulator.app is not opened for headless streaming.", null, { deviceUdid: device.udid }); } currentStep = "resolve-target"; @@ -2758,6 +3183,7 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { projectRoot, chatSessionId: launchArgs.chatSessionId ?? null, mode: normalizeLaunchMode(launchArgs.mode), + keepSimulatorInBackground: launchArgs.keepSimulatorInBackground ?? true, bridgeUrl: null, startedAt: nowIso(), }; @@ -2785,11 +3211,6 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { env: childEnv, timeoutMs: 60_000, }); - if (!openSimulatorInForeground) { - // simctl launch can activate Simulator.app. Reopen it without - // activation so ADE can refocus while the window remains capturable. - openSimulatorAppInBackground(); - } emitLaunchProgress(launchId, "launch-app", "complete", "App launched.", bundleId, { deviceUdid: device.udid, targetId: target.target.id }); emitLaunchProgress(launchId, "ready", "complete", "iOS simulator drawer is ready.", device.name, { deviceUdid: device.udid, targetId: target.target.id }); emit({ type: "session-started", session }); @@ -2981,6 +3402,10 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { height: accessibilityScreen?.height ?? shot.height ?? 0, scale: accessibilityScreen?.scale ?? 1, }; + const iosurfaceInputScreen = iosurfaceInputScreenFromSnapshot(screen, shot); + if (iosurfaceInputScreen.width > 0 && iosurfaceInputScreen.height > 0) { + iosurfaceScreenMetrics.set(shot.deviceUdid, iosurfaceInputScreen); + } const hitElement = hitX == null || hitY == null ? null : findSmallestScreenElementAt(elements, hitX, hitY); @@ -3144,12 +3569,15 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { }); } stopCompanion(); + stopIosurfaceInput(); cleanupTempFiles(); if (shutdownArgs.force) { await cleanupOrphanedIdbCompanions(previousSession?.deviceUdid ?? null).catch(() => 0); } const released = activeSession !== null; activeSession = null; + iosurfaceInputStickyFallbackSessionId = null; + iosurfaceInputFailureTimestamps = []; cachedStatus = { ...cachedStatus, computedAt: 0 }; if (released || previousSession) { emit({ type: "session-updated", session: null }); @@ -3169,23 +3597,39 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { } }; + const startedStreamStatus = ( + device: IosSimulatorDevice, + backend: IosSimulatorStreamBackend, + targetFps: number, + streamUrl: string | null, + helperPid: number | null = null, + ): IosSimulatorStreamStatus => ({ + deviceUdid: device.udid, + running: true, + backend, + requestedBackend: streamRequestContext.requestedBackend ?? null, + fallbackReason: streamRequestContext.fallbackReason ?? null, + degradationReason: streamRequestContext.degradationReason ?? null, + fps: null, + targetFps, + frameCount: 0, + startedAt: nowIso(), + lastFrameAt: null, + lastError: null, + error: null, + streamUrl, + averageLatencyMs: null, + latencyP50Ms: null, + latencyP95Ms: null, + helperPid, + inputBackend: null, + }); + const startSimctlPreview = async (device: IosSimulatorDevice, fps: number): Promise => { await stopStream(); const pollFps = Math.max(1, Math.min(8, Math.round(fps))); const streamUrl = await startMjpegStreamServer(); - streamStatus = { - deviceUdid: device.udid, - running: true, - backend: "simctl-screenshot-poll", - fps: null, - targetFps: pollFps, - frameCount: 0, - startedAt: nowIso(), - lastFrameAt: null, - lastError: null, - streamUrl, - averageLatencyMs: null, - }; + streamStatus = startedStreamStatus(device, "simctl-screenshot-poll", pollFps, streamUrl); emit({ type: "stream-started", status: streamStatus }); let inFlight = false; const tick = async () => { @@ -3212,19 +3656,10 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { const startWindowCaptureStream = async (device: IosSimulatorDevice, fps: number): Promise => { await stopStream(); - streamStatus = { - deviceUdid: device.udid, - running: true, - backend: "simulator-window-capture", - fps: null, - targetFps: Math.max(1, Math.min(60, Math.round(fps))), - frameCount: 0, - startedAt: nowIso(), - lastFrameAt: null, - lastError: null, - streamUrl: null, - averageLatencyMs: null, - }; + if (process.platform === "darwin") { + spawnProcess("open", ["-a", "Simulator"], { detached: true, stdio: "ignore" }).unref(); + } + streamStatus = startedStreamStatus(device, "simulator-window-capture", Math.max(1, Math.min(60, Math.round(fps))), null); emit({ type: "stream-started", status: streamStatus }); if (cachedCommandExists("idb") && cachedCommandExists("idb_companion")) { void ensureCompanion(device.udid).then(() => { @@ -3241,6 +3676,157 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { return streamStatus; }; + const startFallbackAfterIosurfaceFailure = async (device: IosSimulatorDevice, fps: number, reason: string): Promise => { + const failureCount = rememberIosurfaceFailure(); + const fallbackReason = failureCount >= IOSURFACE_MAX_FAILURES_PER_WINDOW + ? `${reason} ADE disabled iosurface-indigo for this 60s window after repeated helper failures.` + : reason; + const windowCaptureAllowed = process.platform === "darwin" && activeSession?.keepSimulatorInBackground === false; + const nextBackend = resolveIosSimulatorStreamBackend("auto", { + iosurfaceIndigo: false, + windowCaptureAllowed, + idb: cachedCommandExists("idb"), + idbCompanion: cachedCommandExists("idb_companion"), + ffmpeg: cachedCommandExists("ffmpeg"), + }); + streamRequestContext = { + requestedBackend: streamRequestContext.requestedBackend ?? "auto", + fallbackReason: `Using ${nextBackend} fallback because ${fallbackReason}`, + degradationReason: fallbackReason, + }; + args.logger.warn("ios_simulator.iosurface_indigo_degraded", { + deviceUdid: device.udid, + nextBackend, + reason: fallbackReason, + }); + if (nextBackend === "simulator-window-capture") return startWindowCaptureStream(device, fps); + if (nextBackend === "idb-mjpeg") return startIdbMjpegStream(device, Math.min(30, fps)); + return startSimctlPreview(device, Math.min(8, fps)); + }; + + const startIosurfaceIndigoStream = async (device: IosSimulatorDevice, fps: number, capability?: IosurfaceIndigoCapability): Promise => { + const detected = capability ?? await detectIosurfaceIndigoCapability(); + if (!detected.available || !detected.helpers) { + throw new IosurfaceHelperUnavailableError(detected.reason ?? "IOSurface helper is unavailable."); + } + if (streamProcess && streamStatus.deviceUdid === device.udid && streamStatus.backend === "iosurface-indigo" && streamStatus.running) { + return streamStatus; + } + await stopStream(); + const targetFps = Math.max(1, Math.min(60, Math.round(fps))); + const streamUrl = await startMjpegStreamServer(); + streamBuffer = Buffer.alloc(0); + let stderr = ""; + let stderrLineBuffer = ""; + streamProcess = spawnProcess(detected.helpers.capture, [ + "--udid", + device.udid, + "--fps", + String(targetFps), + "--quality", + "0.52", + ], { + env: { ...process.env, PYTHONUNBUFFERED: "1" }, + stdio: ["ignore", "pipe", "pipe"], + }); + const child = streamProcess; + streamStatus = startedStreamStatus(device, "iosurface-indigo", targetFps, streamUrl, child.pid ?? null); + scheduleIosurfaceHelperIdleStop(); + + const handleStatusLine = (line: string) => { + const jsonStart = line.indexOf("{"); + if (jsonStart < 0) return; + try { + const parsed = JSON.parse(line.slice(jsonStart)) as unknown; + if (!isRecord(parsed)) return; + if (parsed.type === "stream-started") { + const pixelWidth = Number(parsed.pixelWidth); + const pixelHeight = Number(parsed.pixelHeight); + args.logger.debug("ios_simulator.iosurface_stream_started", { + deviceUdid: device.udid, + pixelWidth: Number.isFinite(pixelWidth) ? pixelWidth : null, + pixelHeight: Number.isFinite(pixelHeight) ? pixelHeight : null, + }); + } + } catch { + // stderr is diagnostic; malformed helper lines should not stop capture. + } + }; + + const startupTimer = setTimeout(() => { + if ( + streamProcess !== child + || !streamStatus.running + || streamStatus.backend !== "iosurface-indigo" + || streamStatus.frameCount > 0 + ) { + return; + } + const detail = "iosurface-indigo produced no frames during startup."; + stopChild(child); + void (async () => { + const failedStatus = { ...streamStatus, running: false, lastError: detail, error: { code: "iosurface-helper-timeout", exitCode: null, signal: null } }; + emit({ type: "stream-error", status: failedStatus }); + await stopStream(); + await startFallbackAfterIosurfaceFailure(device, targetFps, stderr.trim() || detail); + })().catch((error) => { + const status = setStreamStopped(error instanceof Error ? error.message : String(error)); + emit({ type: "stream-error", status }); + }); + }, IOSURFACE_HELPER_STARTUP_TIMEOUT_MS); + startupTimer.unref?.(); + + child.stdout?.on("data", (chunk: Buffer) => { + if (streamProcess !== child || streamStatus.backend !== "iosurface-indigo") return; + handleLengthPrefixedJpegChunk(chunk, "iosurface-indigo"); + }); + child.stderr?.on("data", (chunk: Buffer) => { + const text = chunk.toString(); + stderr = `${stderr}${text}`.slice(-4000); + stderrLineBuffer += text; + const lines = stderrLineBuffer.split(/\r?\n/); + stderrLineBuffer = lines.pop() ?? ""; + for (const line of lines) handleStatusLine(line); + }); + child.once("error", (error) => { + clearTimeout(startupTimer); + if (streamProcess !== child) return; + const detail = error.message || "iosurface-indigo helper failed to start."; + void (async () => { + const failedStatus = { ...streamStatus, running: false, lastError: detail, error: { code: "iosurface-helper-error", exitCode: null, signal: null } }; + emit({ type: "stream-error", status: failedStatus }); + await stopStream(); + await startFallbackAfterIosurfaceFailure(device, targetFps, detail); + })().catch((failure) => { + const status = setStreamStopped(failure instanceof Error ? failure.message : String(failure)); + emit({ type: "stream-error", status }); + }); + }); + child.once("exit", (code, signal) => { + clearTimeout(startupTimer); + if (streamProcess !== child) return; + const detail = stderr.trim(); + const graceful = code === 0 || signal === "SIGTERM"; + if (graceful) { + const status = setStreamStopped(null); + emit({ type: "stream-stopped", status }); + return; + } + const reason = detail || `iosurface-indigo helper exited with ${signal ?? code ?? "unknown status"}.`; + void (async () => { + const failedStatus = { ...streamStatus, running: false, lastError: reason, error: { code: "iosurface-helper-exited", exitCode: code, signal } }; + emit({ type: "stream-error", status: failedStatus }); + await stopStream(); + await startFallbackAfterIosurfaceFailure(device, targetFps, reason); + })().catch((error) => { + const status = setStreamStopped(error instanceof Error ? error.message : String(error)); + emit({ type: "stream-error", status }); + }); + }); + emit({ type: "stream-started", status: streamStatus }); + return streamStatus; + }; + const startIdbMjpegStream = async (device: IosSimulatorDevice, fps: number): Promise => { if (!cachedCommandExists("idb")) { throw new Error(`idb is required for live simulator streaming. ${INSTALL_HINT_IDB}`); @@ -3251,19 +3837,7 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { await stopStream(); const companion = await ensureCompanion(device.udid); const streamUrl = await startMjpegStreamServer(); - streamStatus = { - deviceUdid: device.udid, - running: true, - backend: "idb-mjpeg", - fps: null, - targetFps: fps, - frameCount: 0, - startedAt: nowIso(), - lastFrameAt: null, - lastError: null, - streamUrl, - averageLatencyMs: null, - }; + streamStatus = startedStreamStatus(device, "idb-mjpeg", fps, streamUrl); streamBuffer = Buffer.alloc(0); streamProcess = spawnProcess("idb", [ "--companion", @@ -3301,6 +3875,7 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { void (async () => { await stopStream(); emit({ type: "stream-error", status: { ...streamStatus, lastError: detail } }); + streamRequestContext = { ...streamRequestContext, degradationReason: detail }; if (cachedCommandExists("ffmpeg")) { await startIdbH264FfmpegStream(device, fps); } else { @@ -3348,19 +3923,7 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { await stopStream(); const companion = await ensureCompanion(device.udid); const streamUrl = await startMjpegStreamServer(); - streamStatus = { - deviceUdid: device.udid, - running: true, - backend: "idb-h264-ffmpeg-mjpeg", - fps: null, - targetFps: fps, - frameCount: 0, - startedAt: nowIso(), - lastFrameAt: null, - lastError: null, - streamUrl, - averageLatencyMs: null, - }; + streamStatus = startedStreamStatus(device, "idb-h264-ffmpeg-mjpeg", fps, streamUrl); streamBuffer = Buffer.alloc(0); streamProcess = spawnProcess("idb", [ "--companion", @@ -3419,6 +3982,7 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { void (async () => { await stopStream(); emit({ type: "stream-error", status: { ...streamStatus, lastError: detail } }); + streamRequestContext = { ...streamRequestContext, degradationReason: detail }; await startSimctlPreview(device, Math.min(8, fps)); })().catch((error) => { const status = setStreamStopped(error instanceof Error ? error.message : String(error)); @@ -3481,16 +4045,52 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { const startStream = async (streamArgs: IosSimulatorStartStreamArgs = {}): Promise => { const device = await resolveDevice(streamArgs.deviceUdid ?? activeSession?.deviceUdid); - const backend = streamArgs.backend ?? "simctl-screenshot-poll"; - if (backend !== "auto" && backend !== "simctl-screenshot-poll" && backend !== "idb-mjpeg" && backend !== "idb-h264-ffmpeg-mjpeg" && backend !== "simulator-window-capture") { - throw new Error("stream backend must be `auto`, `simulator-window-capture`, `simctl-screenshot-poll`, `idb-mjpeg`, or `idb-h264-ffmpeg-mjpeg`."); + const backend = streamArgs.backend ?? "auto"; + if (backend !== "auto" && backend !== "iosurface-indigo" && backend !== "simctl-screenshot-poll" && backend !== "idb-mjpeg" && backend !== "idb-h264-ffmpeg-mjpeg" && backend !== "simulator-window-capture") { + throw new Error("stream backend must be `auto`, `iosurface-indigo`, `simulator-window-capture`, `simctl-screenshot-poll`, `idb-mjpeg`, or `idb-h264-ffmpeg-mjpeg`."); } + const iosurfaceCapability = backend === "auto" || backend === "iosurface-indigo" + ? await detectIosurfaceIndigoCapability() + : null; + if (backend === "iosurface-indigo" && !iosurfaceCapability?.available) { + throw new IosurfaceHelperUnavailableError(iosurfaceCapability?.reason ?? "IOSurface helper is unavailable."); + } + const recentIosurfaceFailures = iosurfaceFailureTimestamps.filter((timestamp) => Date.now() - timestamp <= IOSURFACE_FAILURE_WINDOW_MS).length; + const iosurfaceUsable = Boolean(iosurfaceCapability?.available && recentIosurfaceFailures < IOSURFACE_MAX_FAILURES_PER_WINDOW); + const windowCaptureAllowed = process.platform === "darwin" && activeSession?.keepSimulatorInBackground === false; const resolvedBackend = resolveIosSimulatorStreamBackend(backend, { + iosurfaceIndigo: iosurfaceUsable, + windowCaptureAllowed, idb: cachedCommandExists("idb"), idbCompanion: cachedCommandExists("idb_companion"), ffmpeg: cachedCommandExists("ffmpeg"), }); - const defaultFps = resolvedBackend === "idb-mjpeg" || resolvedBackend === "idb-h264-ffmpeg-mjpeg" ? 30 : 8; + let fallbackReason: string | null = null; + if (backend === "auto" && resolvedBackend !== "iosurface-indigo" && !iosurfaceUsable) { + fallbackReason = recentIosurfaceFailures >= IOSURFACE_MAX_FAILURES_PER_WINDOW + ? "iosurface-indigo is temporarily disabled after repeated helper failures." + : iosurfaceCapability?.reason ?? null; + } + streamRequestContext = { + requestedBackend: backend, + fallbackReason: fallbackReason ? `Using ${resolvedBackend} fallback because ${fallbackReason}` : null, + degradationReason: null, + }; + if (streamRequestContext.fallbackReason) { + args.logger.info("ios_simulator.stream_fallback_selected", { + requestedBackend: backend, + resolvedBackend, + reason: streamRequestContext.fallbackReason, + }); + } + let defaultFps: number; + if (resolvedBackend === "simulator-window-capture") { + defaultFps = 60; + } else if (resolvedBackend === "iosurface-indigo" || resolvedBackend === "idb-mjpeg" || resolvedBackend === "idb-h264-ffmpeg-mjpeg") { + defaultFps = 30; + } else { + defaultFps = 8; + } const requestedFps = Math.max(1, Math.min(60, Math.round(Number(streamArgs.fps ?? defaultFps)))); if (!Number.isFinite(requestedFps)) { throw new Error("fps must be a number between 1 and 60."); @@ -3510,6 +4110,9 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { } await stopStream(); } + if (resolvedBackend === "iosurface-indigo") { + return startIosurfaceIndigoStream(device, requestedFps, iosurfaceCapability ?? undefined); + } if (resolvedBackend === "idb-mjpeg") { return startIdbMjpegStream(device, requestedFps); } @@ -3522,20 +4125,211 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { return startSimctlPreview(device, requestedFps); }; + const handleIosurfaceInputStdout = (chunk: Buffer) => { + iosurfaceInputBuffer += chunk.toString(); + const lines = iosurfaceInputBuffer.split(/\r?\n/); + iosurfaceInputBuffer = lines.pop() ?? ""; + for (const rawLine of lines) { + const line = rawLine.trim(); + if (!line) continue; + try { + const parsed = JSON.parse(line) as unknown; + if (!isRecord(parsed)) continue; + const id = typeof parsed.id === "string" ? parsed.id : null; + if (!id) continue; + const pending = iosurfaceInputPending.get(id); + if (!pending) continue; + iosurfaceInputPending.delete(id); + clearTimeout(pending.timer); + if (parsed.ok === true) { + pending.resolve(); + } else { + pending.reject(new Error(typeof parsed.error === "string" ? parsed.error : "Indigo input command failed.")); + } + } catch { + // Keep scanning. stderr carries helper diagnostics. + } + } + }; + + const ensureIosurfaceInputHelper = async (deviceUdid: string): Promise => { + if ( + iosurfaceInputProcess + && iosurfaceInputDeviceUdid === deviceUdid + && iosurfaceInputProcess.exitCode == null + && iosurfaceInputProcess.signalCode == null + ) { + return iosurfaceInputProcess; + } + stopIosurfaceInput(); + const capability = await detectIosurfaceIndigoCapability(); + if (!capability.available || !capability.helpers) { + throw new IosurfaceHelperUnavailableError(capability.reason ?? "IOSurface helper is unavailable."); + } + const child = spawnProcess(capability.helpers.input, ["--udid", deviceUdid], { + env: { ...process.env, PYTHONUNBUFFERED: "1" }, + stdio: ["pipe", "pipe", "pipe"], + }); + iosurfaceInputProcess = child; + iosurfaceInputDeviceUdid = deviceUdid; + iosurfaceInputBuffer = ""; + iosurfaceInputStderr = ""; + child.stdout?.on("data", handleIosurfaceInputStdout); + child.stderr?.on("data", (chunk: Buffer) => { + iosurfaceInputStderr = `${iosurfaceInputStderr}${chunk.toString()}`.slice(-4000); + }); + const rejectPending = (reason: string) => { + if (iosurfaceInputProcess === child) { + iosurfaceInputProcess = null; + iosurfaceInputDeviceUdid = null; + } + for (const pending of iosurfaceInputPending.values()) { + clearTimeout(pending.timer); + pending.reject(new Error(reason)); + } + iosurfaceInputPending.clear(); + }; + child.once("error", (error) => rejectPending(error.message)); + child.once("exit", (code, signal) => { + if (signal === "SIGTERM" || code === 0) return; + rejectPending(iosurfaceInputStderr.trim() || `Indigo input helper exited with ${signal ?? code ?? "unknown status"}.`); + }); + return child; + }; + + const sendIosurfaceInputCommand = async (deviceUdid: string, command: Record): Promise => { + const child = await ensureIosurfaceInputHelper(deviceUdid); + const stdin = child.stdin; + if (!stdin || typeof stdin.write !== "function" || stdin.destroyed) { + throw new Error("Indigo input helper stdin is not writable."); + } + const id = randomUUID(); + iosurfaceLastInputAtMs = Date.now(); + scheduleIosurfaceHelperIdleStop(); + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + iosurfaceInputPending.delete(id); + reject(new Error("Timed out waiting for Indigo input acknowledgement.")); + }, IOSURFACE_INPUT_ACK_TIMEOUT_MS); + timer.unref?.(); + iosurfaceInputPending.set(id, { resolve, reject, timer }); + const ok = stdin.write(`${JSON.stringify({ id, ...command })}\n`); + if (!ok) { + stdin.once("drain", () => {}); + } + }); + }; + + const screenMetricsForIndigoInput = (deviceUdid: string): IosInspectableScreen => { + const metrics = iosurfaceScreenMetrics.get(deviceUdid); + if (!metrics || metrics.width <= 0 || metrics.height <= 0) { + throw new Error("Indigo input needs a current simulator snapshot to map point coordinates."); + } + return metrics; + }; + + const runIdbTap = async (deviceUdid: string, x: number, y: number): Promise => { + if (!cachedCommandExists("idb")) { + throw new Error(`idb is required for pointer control. ${INSTALL_HINT_IDB}`); + } + const companion = await ensureCompanion(deviceUdid); + try { + await run("idb", ["--companion", companion, "ui", "tap", String(Math.round(x)), String(Math.round(y)), "--udid", deviceUdid], { timeoutMs: 20_000 }); + } finally { + scheduleCompanionIdleStop(); + } + }; + + const runIdbText = async (deviceUdid: string, text: string): Promise => { + if (!cachedCommandExists("idb")) { + throw new Error(`idb is required for text input. ${INSTALL_HINT_IDB}`); + } + const companion = await ensureCompanion(deviceUdid); + try { + await run("idb", ["--companion", companion, "ui", "text", text, "--udid", deviceUdid], { timeoutMs: 20_000 }); + } finally { + scheduleCompanionIdleStop(); + } + }; + + const runIdbSwipe = async (deviceUdid: string, input: { startX: number; startY: number; endX: number; endY: number; durationMs?: number | null; delta?: number | null }): Promise => { + if (!cachedCommandExists("idb")) { + throw new Error(`idb is required for pointer control. ${INSTALL_HINT_IDB}`); + } + const companion = await ensureCompanion(deviceUdid); + const idbArgs = [ + "--companion", + companion, + "ui", + "swipe", + String(Math.round(input.startX)), + String(Math.round(input.startY)), + String(Math.round(input.endX)), + String(Math.round(input.endY)), + "--udid", + deviceUdid, + ]; + if (input.durationMs != null) { + if (!Number.isFinite(input.durationMs)) throw new Error("durationMs must be a number."); + idbArgs.push("--duration", String(Math.max(0.01, input.durationMs / 1000))); + } + if (input.delta != null) { + if (!Number.isFinite(input.delta) || input.delta <= 0) throw new Error("delta must be a positive number."); + idbArgs.push("--delta", String(input.delta)); + } + try { + await run("idb", idbArgs, { timeoutMs: 20_000 }); + } finally { + scheduleCompanionIdleStop(); + } + }; + + const preferIndigoInput = async (): Promise => { + if (iosurfaceInputStickyFallbackSessionId && iosurfaceInputStickyFallbackSessionId === activeSession?.id) return false; + if (process.platform !== "darwin") return false; + const capability = await detectIosurfaceIndigoCapability(); + return capability.available; + }; + + const runWithInputFallback = async (action: string, runIndigo: () => Promise, runIdb: () => Promise): Promise => { + if (await preferIndigoInput()) { + try { + await runIndigo(); + streamStatus = { ...streamStatus, inputBackend: "indigo" }; + return; + } catch (error) { + const failureCount = rememberIosurfaceInputFailure(); + const detail = error instanceof Error ? error.message : String(error); + args.logger.warn("ios_simulator.indigo_input_failed", { + action, + failureCount, + error: detail, + }); + if (failureCount >= IOSURFACE_MAX_FAILURES_PER_WINDOW) { + iosurfaceInputStickyFallbackSessionId = activeSession?.id ?? "anonymous"; + const degradationReason = `Using idb input fallback because Indigo input failed ${failureCount} times in 60s: ${detail}`; + streamStatus = { + ...streamStatus, + inputBackend: "idb", + degradationReason, + }; + emit({ type: "stream-status", status: streamStatus }); + } + } + } + await runIdb(); + streamStatus = { ...streamStatus, inputBackend: "idb" }; + }; + const tap = async (point: { deviceUdid?: string | null; x: number; y: number }): Promise<{ ok: true }> => { const deviceUdid = await resolveControlDeviceUdid(point.deviceUdid); const x = normalizeCoordinate(point.x, "x"); const y = normalizeCoordinate(point.y, "y"); return enqueueControl("tap", async () => { - if (!cachedCommandExists("idb")) { - throw new Error(`idb is required for pointer control. ${INSTALL_HINT_IDB}`); - } - const companion = await ensureCompanion(deviceUdid); - try { - await run("idb", ["--companion", companion, "ui", "tap", String(Math.round(x)), String(Math.round(y)), "--udid", deviceUdid], { timeoutMs: 20_000 }); - } finally { - scheduleCompanionIdleStop(); - } + await runWithInputFallback("tap", async () => { + const normalized = normalizeIosSimulatorPointForIndigo({ x, y }, screenMetricsForIndigoInput(deviceUdid)); + await sendIosurfaceInputCommand(deviceUdid, { type: "tap", x: normalized.x, y: normalized.y, hold: IOSURFACE_TAP_HOLD_MS }); + }, () => runIdbTap(deviceUdid, x, y)); return { ok: true }; }); }; @@ -3543,15 +4337,7 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { const typeText = async (input: { deviceUdid?: string | null; text: string }): Promise<{ ok: true }> => { const deviceUdid = await resolveControlDeviceUdid(input.deviceUdid); return enqueueControl("text", async () => { - if (!cachedCommandExists("idb")) { - throw new Error(`idb is required for text input. ${INSTALL_HINT_IDB}`); - } - const companion = await ensureCompanion(deviceUdid); - try { - await run("idb", ["--companion", companion, "ui", "text", input.text, "--udid", deviceUdid], { timeoutMs: 20_000 }); - } finally { - scheduleCompanionIdleStop(); - } + await runWithInputFallback("text", () => sendIosurfaceInputCommand(deviceUdid, { type: "text", text: input.text }), () => runIdbText(deviceUdid, input.text)); return { ok: true }; }); }; @@ -3565,35 +4351,28 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { const durationMs = input.durationMs; const deltaValue = input.delta; return enqueueControl("drag", async () => { - if (!cachedCommandExists("idb")) { - throw new Error(`idb is required for pointer control. ${INSTALL_HINT_IDB}`); - } - const companion = await ensureCompanion(deviceUdid); - const idbArgs = [ - "--companion", - companion, - "ui", - "swipe", - String(Math.round(startX)), - String(Math.round(startY)), - String(Math.round(endX)), - String(Math.round(endY)), - "--udid", - deviceUdid, - ]; - if (durationMs != null) { - if (!Number.isFinite(durationMs)) throw new Error("durationMs must be a number."); - idbArgs.push("--duration", String(Math.max(0.01, durationMs / 1000))); - } - if (deltaValue != null) { - if (!Number.isFinite(deltaValue) || deltaValue <= 0) throw new Error("delta must be a positive number."); - idbArgs.push("--delta", String(deltaValue)); - } - try { - await run("idb", idbArgs, { timeoutMs: 20_000 }); - } finally { - scheduleCompanionIdleStop(); - } + if (durationMs != null && !Number.isFinite(durationMs)) throw new Error("durationMs must be a number."); + if (deltaValue != null && (!Number.isFinite(deltaValue) || deltaValue <= 0)) throw new Error("delta must be a positive number."); + await runWithInputFallback("drag", async () => { + const metrics = screenMetricsForIndigoInput(deviceUdid); + const start = normalizeIosSimulatorPointForIndigo({ x: startX, y: startY }, metrics); + const end = normalizeIosSimulatorPointForIndigo({ x: endX, y: endY }, metrics); + await sendIosurfaceInputCommand(deviceUdid, { + type: "swipe", + startX: start.x, + startY: start.y, + endX: end.x, + endY: end.y, + durationMs: durationMs ?? 180, + }); + }, () => runIdbSwipe(deviceUdid, { + startX, + startY, + endX, + endY, + durationMs, + delta: deltaValue, + })); return { ok: true }; }); }; @@ -3640,6 +4419,7 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { stopChild(streamProcess); stopChild(streamTranscoderProcess); stopCompanion(); + stopIosurfaceInput(); setStreamStopped(null); activeSession = null; activeLaunchId = null; diff --git a/apps/desktop/src/main/services/ipc/registerIpc.ts b/apps/desktop/src/main/services/ipc/registerIpc.ts index 4a989394a..d10b449ec 100644 --- a/apps/desktop/src/main/services/ipc/registerIpc.ts +++ b/apps/desktop/src/main/services/ipc/registerIpc.ts @@ -101,8 +101,11 @@ import type { GitListCommitFilesArgs, GitFileActionArgs, GitBatchFileActionArgs, + BranchPullRequest, GitBranchSummary, GitListBranchesArgs, + GitGetUserIdentityArgs, + GitUserIdentity, GitCheckoutBranchArgs, GitPushArgs, GitUpstreamSyncStatus, @@ -5871,9 +5874,12 @@ export function registerIpc({ ipcMain.handle(IPC.iosSimulatorLaunch, async (event, arg = {}) => { const result = await ensureIosSimulator().launch(arg); - const browserWindow = BrowserWindow.fromWebContents(event.sender); - await prepareSimulatorWindowForCapture(browserWindow, { placeBehindAde: true }); - followSimulatorWindowUnderAde(browserWindow); + const keepSimulatorInBackground = Boolean((arg as { keepSimulatorInBackground?: unknown } | null)?.keepSimulatorInBackground ?? true); + if (!keepSimulatorInBackground) { + const browserWindow = BrowserWindow.fromWebContents(event.sender); + await prepareSimulatorWindowForCapture(browserWindow, { placeBehindAde: false }); + cleanupSimulatorParkingFollow?.(); + } return result; }); @@ -6388,6 +6394,11 @@ export function registerIpc({ return await ctx.gitService.listBranches(arg); }); + ipcMain.handle(IPC.gitGetUserIdentity, async (_event, arg: GitGetUserIdentityArgs): Promise => { + const ctx = getCtx(); + return await ctx.gitService.getUserIdentity(arg); + }); + ipcMain.handle(IPC.gitCheckoutBranch, async (_event, arg: GitCheckoutBranchArgs): Promise => { const ctx = getCtx(); return await ctx.gitService.checkoutBranch(arg); @@ -6598,6 +6609,11 @@ export function registerIpc({ return ctx.prService.listAll(); }); + ipcMain.handle(IPC.prsListOpenForRepo, async (): Promise => { + const ctx = ensurePrPolling(); + return await ctx.prService.listOpenPullRequests(); + }); + ipcMain.handle(IPC.prsRefresh, async (_event, arg: { prId?: string; prIds?: string[] } = {}): Promise => { const ctx = ensurePrPolling(); return await ctx.prService.refresh(arg); diff --git a/apps/desktop/src/main/services/prs/prService.ts b/apps/desktop/src/main/services/prs/prService.ts index 86e73f7fb..eaf2c2963 100644 --- a/apps/desktop/src/main/services/prs/prService.ts +++ b/apps/desktop/src/main/services/prs/prService.ts @@ -3,6 +3,7 @@ import os from "node:os"; import path from "node:path"; import { randomUUID } from "node:crypto"; import type { + BranchPullRequest, CreatePrFromLaneArgs, CreateQueuePrsArgs, CreateQueuePrsResult, @@ -5221,6 +5222,37 @@ export function createPrService({ return laneId ? summaries.filter((pr) => pr.laneId === laneId) : summaries; }, + /** + * Returns a flat list of open PRs in the project's GitHub repo, keyed by + * head branch. Used by the branch picker to attach PR pills to branches. + * Hits a single paginated GitHub endpoint — independent of the local PR + * cache, so it covers branches that don't yet have a lane. + */ + async listOpenPullRequests(): Promise { + const repo = await githubService.getRepoOrThrow(); + const rows = await fetchAllPages({ + path: `/repos/${repo.owner}/${repo.name}/pulls`, + query: { state: "open", sort: "updated", direction: "desc" }, + }); + const out: BranchPullRequest[] = []; + for (const row of rows) { + const headBranch = asString(row?.head?.ref).trim(); + const prNumber = asNumber(row?.number); + if (!headBranch || !prNumber) continue; + const isDraft = Boolean(row?.draft); + out.push({ + branch: headBranch, + prNumber, + title: asString(row?.title).trim(), + state: isDraft ? "draft" : "open", + url: asString(row?.html_url).trim(), + author: asString(row?.user?.login).trim() || null, + updatedAt: asString(row?.updated_at).trim() || null, + }); + } + return out; + }, + getReviewSnapshot, async refresh(args: { prId?: string; prIds?: string[] } = {}): Promise { diff --git a/apps/desktop/src/preload/global.d.ts b/apps/desktop/src/preload/global.d.ts index 6bf9edd04..7dc108b86 100644 --- a/apps/desktop/src/preload/global.d.ts +++ b/apps/desktop/src/preload/global.d.ts @@ -244,7 +244,10 @@ import type { GitGenerateCommitMessageArgs, GitGenerateCommitMessageResult, GitListBranchesArgs, + GitGetUserIdentityArgs, + GitUserIdentity, GitListCommitFilesArgs, + BranchPullRequest, GitFileActionArgs, GitBatchFileActionArgs, GitPushArgs, @@ -1421,6 +1424,9 @@ declare global { listBranches: ( args: GitListBranchesArgs, ) => Promise; + getUserIdentity: ( + args: GitGetUserIdentityArgs, + ) => Promise; checkoutBranch: ( args: GitCheckoutBranchArgs, ) => Promise; @@ -1497,6 +1503,7 @@ declare global { linkToLane: (args: LinkPrToLaneArgs) => Promise; getForLane: (laneId: string) => Promise; listAll: () => Promise; + listOpenForRepo: () => Promise; refresh: (args?: { prId?: string; prIds?: string[]; diff --git a/apps/desktop/src/preload/preload.ts b/apps/desktop/src/preload/preload.ts index 949dde5f3..47eaca371 100644 --- a/apps/desktop/src/preload/preload.ts +++ b/apps/desktop/src/preload/preload.ts @@ -177,8 +177,11 @@ import type { GitListCommitFilesArgs, GitFileActionArgs, GitBatchFileActionArgs, + BranchPullRequest, GitBranchSummary, GitListBranchesArgs, + GitGetUserIdentityArgs, + GitUserIdentity, GitCheckoutBranchArgs, GitPushArgs, GitRevertArgs, @@ -2589,6 +2592,10 @@ contextBridge.exposeInMainWorld("ade", { args: GitListBranchesArgs, ): Promise => gitBranchesCache.get(serializeIpcCacheArgs(args)), + getUserIdentity: async ( + args: GitGetUserIdentityArgs, + ): Promise => + ipcRenderer.invoke(IPC.gitGetUserIdentity, args), checkoutBranch: async ( args: GitCheckoutBranchArgs, ): Promise => @@ -2723,6 +2730,8 @@ contextBridge.exposeInMainWorld("ade", { ipcRenderer.invoke(IPC.prsGetForLane, { laneId }), listAll: async (): Promise => ipcRenderer.invoke(IPC.prsListAll), + listOpenForRepo: async (): Promise => + ipcRenderer.invoke(IPC.prsListOpenForRepo), refresh: async ( args: { prId?: string; prIds?: string[] } = {}, ): Promise => ipcRenderer.invoke(IPC.prsRefresh, args), diff --git a/apps/desktop/src/renderer/components/chat/AgentChatComposer.tsx b/apps/desktop/src/renderer/components/chat/AgentChatComposer.tsx index 129f01993..44e28cc41 100644 --- a/apps/desktop/src/renderer/components/chat/AgentChatComposer.tsx +++ b/apps/desktop/src/renderer/components/chat/AgentChatComposer.tsx @@ -1976,7 +1976,7 @@ export function AgentChatComposer({ setSelectedAppControlContextId(null); }, [appControlContextItems, selectedAppControlContextId]); - const composerBeamActive = isActive && layoutVariant !== "grid-tile" && turnActive; + const composerBeamActive = isActive && layoutVariant !== "grid-tile" && !iosSimulatorOpen && (turnActive || !chatHasMessages); const composerBeamVariant = turnActive ? "ocean" : "colorful"; const composerBeamDuration = turnActive ? 20 : 5; const composerBeamStrength = turnActive ? 0.26 : 0.44; diff --git a/apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.test.tsx b/apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.test.tsx index 16dbe82bc..5f75feb4a 100644 --- a/apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.test.tsx +++ b/apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.test.tsx @@ -1,6 +1,6 @@ /* @vitest-environment jsdom */ -import { act, cleanup, render, screen, waitFor } from "@testing-library/react"; +import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { ChatIosSimulatorPanel } from "./ChatIosSimulatorPanel"; import type { @@ -27,6 +27,7 @@ const activeStatus: IosSimulatorStatus = { tools: [ { name: "xcrun", available: true, detail: "ok", installHint: "" }, { name: "xcodebuild", available: true, detail: "ok", installHint: "" }, + { name: "iosurface_indigo", available: true, detail: "ok", installHint: "" }, { name: "simulator_window", available: true, detail: "ok", installHint: "" }, { name: "idb", available: true, detail: "ok", installHint: "" }, { name: "idb_companion", available: true, detail: "ok", installHint: "" }, @@ -88,6 +89,7 @@ const simulatorWindowSource: IosSimulatorWindowSource = { }; function installIosSimulatorApi(options: { + autoBackend?: IosSimulatorStreamStatus["backend"]; windowSources?: IosSimulatorWindowSource[]; windowState?: IosSimulatorWindowState; getUserMedia?: () => Promise; @@ -108,13 +110,23 @@ function installIosSimulatorApi(options: { getStatus: vi.fn().mockResolvedValue(activeStatus), listDevices: vi.fn().mockResolvedValue([device]), listLaunchTargets: vi.fn().mockResolvedValue([launchTarget]), - startStream: vi.fn((args: { backend?: string | null } = {}) => Promise.resolve(streamStatus(args.backend === "simulator-window-capture" - ? { - backend: "simulator-window-capture", - targetFps: 60, - streamUrl: null, - } - : {}))), + startStream: vi.fn((args: { backend?: string | null } = {}) => { + const resolvedBackend = args.backend === "auto" + ? options.autoBackend ?? "idb-mjpeg" + : args.backend; + return Promise.resolve(streamStatus(resolvedBackend === "simulator-window-capture" + ? { + backend: "simulator-window-capture", + targetFps: 60, + streamUrl: null, + } + : { + backend: resolvedBackend === "iosurface-indigo" || resolvedBackend === "idb-mjpeg" || resolvedBackend === "simctl-screenshot-poll" + ? resolvedBackend + : "idb-mjpeg", + targetFps: 30, + })); + }), stopStream: vi.fn().mockResolvedValue(streamStatus({ running: false, backend: null, streamUrl: null })), getStreamStatus: vi.fn().mockResolvedValue(streamStatus()), getSimulatorWindowState: vi.fn().mockResolvedValue(options.windowState ?? { @@ -188,9 +200,10 @@ describe("ChatIosSimulatorPanel", () => { afterEach(() => { cleanup(); vi.clearAllMocks(); + vi.useRealTimers(); }); - it("starts the normal live view through the smooth simulator window capture stream", async () => { + it("starts the normal live view through the service auto stream", async () => { const { api } = installIosSimulatorApi(); render( @@ -203,12 +216,27 @@ describe("ChatIosSimulatorPanel", () => { await waitFor(() => expect(api.startStream).toHaveBeenCalled()); - expect(api.startStream).toHaveBeenCalledWith({ deviceUdid: device.udid, backend: "simulator-window-capture", fps: 60 }); - expect(api.listSimulatorWindowSources).toHaveBeenCalled(); + expect(api.startStream).toHaveBeenCalledWith({ deviceUdid: device.udid, backend: "auto" }); + expect(api.listSimulatorWindowSources).not.toHaveBeenCalled(); }); - it("falls back to the device-backed auto stream when window capture is unavailable", async () => { - const { api } = installIosSimulatorApi({ windowSources: [] }); + it("shows compact readiness for the best performance simulator path", async () => { + installIosSimulatorApi(); + + render( + , + ); + + await screen.findByText("Best simulator performance"); + expect(screen.getByText("Ready")).toBeTruthy(); + }); + + it("uses the window-capture visual when auto resolves to simulator-window-capture", async () => { + const { api } = installIosSimulatorApi({ autoBackend: "simulator-window-capture" }); render( { await waitFor(() => expect(api.startStream).toHaveBeenCalledTimes(2), { timeout: 3_000 }); - expect(api.startStream).toHaveBeenNthCalledWith(1, { deviceUdid: device.udid, backend: "simulator-window-capture", fps: 60 }); + expect(api.startStream).toHaveBeenNthCalledWith(1, { deviceUdid: device.udid, backend: "auto" }); + expect(api.startStream).toHaveBeenNthCalledWith(2, { deviceUdid: device.udid, backend: "simulator-window-capture", fps: 60 }); + expect(api.listSimulatorWindowSources).toHaveBeenCalled(); + }); + + it("switches the current session to Simulator.app window capture on request", async () => { + const { api } = installIosSimulatorApi({ autoBackend: "iosurface-indigo" }); + + render( + , + ); + + await waitFor(() => expect(api.startStream).toHaveBeenCalledTimes(1), { timeout: 3_000 }); + + fireEvent.click(await screen.findByRole("button", { name: /show ios window/i })); + + await waitFor(() => expect(api.startStream).toHaveBeenCalledWith({ + deviceUdid: device.udid, + backend: "simulator-window-capture", + fps: 60, + }), { timeout: 3_000 }); expect(api.stopStream).toHaveBeenCalled(); - expect(api.startStream).toHaveBeenLastCalledWith({ deviceUdid: device.udid, backend: "auto", fps: 30 }); + expect(api.listSimulatorWindowSources).toHaveBeenCalled(); + expect(await screen.findByRole("button", { name: /use ade stream/i })).toBeTruthy(); }); it("warns when macOS cannot capture the Simulator window", async () => { const { api } = installIosSimulatorApi({ + autoBackend: "simulator-window-capture", windowState: { appRunning: true, visible: true, @@ -252,7 +306,7 @@ describe("ChatIosSimulatorPanel", () => { }); it("restarts the device-backed fallback stream after a stream error event", async () => { - const { api, emit } = installIosSimulatorApi({ windowSources: [] }); + const { api, emit } = installIosSimulatorApi(); render( { />, ); - await waitFor(() => expect(api.startStream).toHaveBeenCalledTimes(2), { timeout: 3_000 }); + await waitFor(() => expect(api.startStream).toHaveBeenCalledTimes(1), { timeout: 3_000 }); await waitFor(() => expect(document.querySelector('canvas[aria-label="iOS Simulator live stream"]')).toBeTruthy()); act(() => { @@ -277,12 +331,12 @@ describe("ChatIosSimulatorPanel", () => { }); }); - await waitFor(() => expect(api.startStream).toHaveBeenCalledTimes(3)); - expect(api.startStream).toHaveBeenLastCalledWith({ deviceUdid: device.udid, backend: "auto", fps: 30 }); + await waitFor(() => expect(api.startStream).toHaveBeenCalledTimes(2)); + expect(api.startStream).toHaveBeenLastCalledWith({ deviceUdid: device.udid, backend: "auto" }); }); - it("updates the canvas stream url when the service falls back to another backend", async () => { - const { api, emit } = installIosSimulatorApi({ windowSources: [] }); + it("forces the idb fallback after repeated native stream failures", async () => { + const { api, emit } = installIosSimulatorApi({ autoBackend: "iosurface-indigo" }); render( { />, ); + await waitFor(() => expect(api.startStream).toHaveBeenCalledTimes(1), { timeout: 3_000 }); + + act(() => { + emit({ + type: "stream-error", + status: streamStatus({ + running: false, + backend: "iosurface-indigo", + streamUrl: null, + lastError: "iosurface stream exited", + }), + }); + }); + await waitFor(() => expect(api.startStream).toHaveBeenCalledTimes(2), { timeout: 3_000 }); + expect(api.startStream).toHaveBeenLastCalledWith({ deviceUdid: device.udid, backend: "auto" }); + + await new Promise((resolve) => setTimeout(resolve, 1_600)); + + act(() => { + emit({ + type: "stream-error", + status: streamStatus({ + running: false, + backend: "iosurface-indigo", + streamUrl: null, + lastError: "iosurface stream exited again", + }), + }); + }); + + await waitFor(() => expect(api.startStream).toHaveBeenCalledTimes(3), { timeout: 3_000 }); + expect(api.startStream).toHaveBeenLastCalledWith({ deviceUdid: device.udid, backend: "idb-mjpeg" }); + await screen.findByText("Live stream had trouble reconnecting. Switching to a fallback view..."); + }); + + it("does not restart an idle native stream after it has produced a frame", async () => { + const { api } = installIosSimulatorApi({ autoBackend: "iosurface-indigo" }); + + render( + , + ); + + await waitFor(() => expect(api.startStream).toHaveBeenCalledTimes(1), { timeout: 3_000 }); + + await new Promise((resolve) => setTimeout(resolve, 2_500)); + + expect(api.startStream).toHaveBeenCalledTimes(1); + }); + + it("updates the canvas stream url when the service falls back to another backend", async () => { + const { api, emit } = installIosSimulatorApi(); + + render( + , + ); + + await waitFor(() => expect(api.startStream).toHaveBeenCalledTimes(1), { timeout: 3_000 }); act(() => { emit({ diff --git a/apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.tsx b/apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.tsx index 9993e9479..9cb151bd6 100644 --- a/apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.tsx +++ b/apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.tsx @@ -1,5 +1,5 @@ import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent, type MouseEvent, type PointerEvent } from "react"; -import { ArrowClockwise, ArrowSquareOut, BracketsCurly, CheckCircle, Circle, Copy, CursorClick, DeviceMobile, FileCode, ImageSquare, Lightning, Lock, Play, Power, Selection, SpinnerGap, TextT, WarningCircle, Wrench } from "@phosphor-icons/react"; +import { ArrowClockwise, ArrowSquareOut, BracketsCurly, CheckCircle, Circle, Copy, CursorClick, Desktop, DeviceMobile, FileCode, ImageSquare, Lightning, Lock, Play, Power, Selection, SpinnerGap, TextT, WarningCircle, Wrench } from "@phosphor-icons/react"; import type { AgentChatFileRef, IosElementContextItem, @@ -113,7 +113,7 @@ type PreviewCaptureSelection = { type LiveVisual = | { kind: "window"; - status: "starting" | "active" | "error"; + status: "starting" | "reconnecting" | "active" | "error"; sourceId: string | null; sourceName: string | null; width: number | null; @@ -122,13 +122,15 @@ type LiveVisual = } | { kind: "mjpeg"; - status: "starting" | "active" | "error"; + status: "starting" | "reconnecting" | "active" | "error"; url: string | null; width: number | null; height: number | null; error: string | null; }; +type DeviceBackedStreamRequestBackend = "auto" | NonNullable; + type VideoFrameMetadata = { presentationTime?: number; expectedDisplayTime?: number; @@ -149,6 +151,13 @@ type ToolDescriptor = { required: boolean; }; +const LIVE_STREAM_FALLBACK_WINDOW_MS = 30_000; +const LIVE_STREAM_FAILURES_BEFORE_FALLBACK = 2; +const LIVE_STREAM_FAILURE_RESET_AFTER_MS = 10_000; +const LIVE_CANVAS_MAX_DEVICE_PIXEL_RATIO = 2; +const LIVE_RECONNECT_MESSAGE = "Live simulator stream paused. Reconnecting..."; +const LIVE_FALLBACK_MESSAGE = "Live stream had trouble reconnecting. Switching to a fallback view..."; + const TOOL_DESCRIPTORS: Record = { xcrun: { title: "Xcode command-line tools", @@ -160,6 +169,11 @@ const TOOL_DESCRIPTORS: Record = { description: "Needed to build and install your app onto the simulator.", required: true, }, + iosurface_indigo: { + title: "IOSurface helpers", + description: "Primary low-latency streaming and touch input path. Requires a supported full Xcode.", + required: false, + }, simulator_window: { title: "iOS runtime", description: "Install an iOS runtime in Xcode > Settings > Platforms.", @@ -167,7 +181,7 @@ const TOOL_DESCRIPTORS: Record = { }, idb: { title: "idb", - description: "Optional. Enables fallback streaming plus tap, drag, and type.", + description: "Fallback streaming, accessibility reads, and pointer/text control when Indigo is unavailable.", required: false, }, idb_companion: { @@ -177,11 +191,20 @@ const TOOL_DESCRIPTORS: Record = { }, ffmpeg: { title: "ffmpeg", - description: "Optional. Used for the fallback H.264 transcode stream.", + description: "Recovery-only transcode path after idb MJPEG fails.", required: false, }, }; +function streamBackendLabel(backend: IosSimulatorStreamStatus["backend"]): string { + if (backend === "iosurface-indigo") return "native simulator stream"; + if (backend === "simulator-window-capture") return "Simulator window"; + if (backend === "idb-mjpeg") return "fallback stream"; + if (backend === "idb-h264-ffmpeg-mjpeg") return "recovery stream"; + if (backend === "simctl-screenshot-poll") return "screenshot fallback"; + return "stream"; +} + function makeReactKeysById(items: T[]): string[] { const seen = new Map(); return items.map((item, index) => { @@ -672,11 +695,21 @@ async function drawJpegFrameToCanvas(frame: ByteBuffer, canvas: HTMLCanvasElemen const bitmap = await createImageBitmap(new Blob([frame], { type: "image/jpeg" })); const { width, height } = bitmap; try { - canvas.width = width; - canvas.height = height; + const pixelRatio = Math.max(1, Math.min(LIVE_CANVAS_MAX_DEVICE_PIXEL_RATIO, window.devicePixelRatio || 1)); + const visibleWidth = Math.max(1, canvas.clientWidth || width); + const visibleHeight = Math.max(1, canvas.clientHeight || height); + const scale = Math.min( + 1, + (visibleWidth * pixelRatio) / Math.max(1, width), + (visibleHeight * pixelRatio) / Math.max(1, height), + ); + const canvasWidth = Math.max(1, Math.round(width * scale)); + const canvasHeight = Math.max(1, Math.round(height * scale)); + if (canvas.width !== canvasWidth) canvas.width = canvasWidth; + if (canvas.height !== canvasHeight) canvas.height = canvasHeight; const context = canvas.getContext("2d", { alpha: false }); if (!context) throw new Error("Canvas rendering is not available for the live simulator stream."); - context.drawImage(bitmap, 0, 0); + context.drawImage(bitmap, 0, 0, canvasWidth, canvasHeight); return { width, height }; } finally { bitmap.close(); @@ -697,8 +730,42 @@ async function playMjpegStreamToCanvas( if (!response.body) throw new Error("Live simulator stream did not expose a readable body."); const reader = response.body.getReader(); let buffer: ByteBuffer = new Uint8Array(0); + let pendingFrame: ByteBuffer | null = null; + let renderError: unknown = null; + let renderPromise: Promise | null = null; + const waitForPaint = () => new Promise((resolve) => { + if (typeof window.requestAnimationFrame === "function") { + window.requestAnimationFrame(() => resolve()); + return; + } + window.setTimeout(resolve, 0); + }); + const renderLatestFrame = () => { + if (renderPromise) return; + renderPromise = (async () => { + try { + for (;;) { + if (signal.aborted) return; + const frame = pendingFrame; + if (!frame) return; + pendingFrame = null; + const rendered = await drawJpegFrameToCanvas(frame, canvas); + if (signal.aborted) return; + onFrame(rendered); + await waitForPaint(); + } + } catch (error) { + renderError = error; + void reader.cancel().catch(() => {}); + } finally { + renderPromise = null; + if (pendingFrame && !renderError && !signal.aborted) renderLatestFrame(); + } + })(); + }; try { for (;;) { + if (renderError) throw renderError; const { value, done } = await reader.read(); if (done) break; if (signal.aborted) return; @@ -707,10 +774,11 @@ async function playMjpegStreamToCanvas( const extracted = extractLatestJpegFrame(buffer); buffer = extracted.rest; if (!extracted.frame) continue; - const rendered = await drawJpegFrameToCanvas(extracted.frame, canvas); - if (signal.aborted) return; - onFrame(rendered); + pendingFrame = extracted.frame; + renderLatestFrame(); } + if (renderPromise) await renderPromise; + if (renderError) throw renderError; } finally { reader.releaseLock(); } @@ -828,6 +896,8 @@ export function ChatIosSimulatorPanel({ const [simulatorCaptureActive, setSimulatorCaptureActive] = useState(false); const [simulatorCaptureSelection, setSimulatorCaptureSelection] = useState(null); const [liveVisual, setLiveVisual] = useState(null); + const [simulatorWindowSessionId, setSimulatorWindowSessionId] = useState(null); + const [mjpegPlaybackEpoch, setMjpegPlaybackEpoch] = useState(0); const [windowScreenRect, setWindowScreenRect] = useState(null); const [simulatorWindowState, setSimulatorWindowState] = useState(null); const [streamStatus, setStreamStatus] = useState(null); @@ -852,6 +922,9 @@ export function ChatIosSimulatorPanel({ const liveFrameWindowStartRef = useRef(0); const liveRestartTimerRef = useRef(null); const liveRestartAttemptedAtRef = useRef(0); + const lastResolvedStreamBackendRef = useRef(null); + const liveDeviceStreamFailureTimestampsRef = useRef([]); + const lastLiveMjpegFrameAtRef = useRef(0); const windowCaptureRecoveryTimerRef = useRef(null); const windowCaptureRecoveryAttemptedAtRef = useRef(0); const lastWindowFrameAtRef = useRef(0); @@ -865,6 +938,7 @@ export function ChatIosSimulatorPanel({ return status?.activeDevice ?? devices[0] ?? null; }, [devices, selectedDeviceUdid, status?.activeDevice]); const activeSession = status?.activeSession ?? null; + const simulatorWindowModeEnabled = Boolean(activeSession?.id && simulatorWindowSessionId === activeSession.id); const visibleLaunchTargets = useMemo(() => { const projectTargets = launchTargets.filter((target) => target.kind === "project"); @@ -937,8 +1011,11 @@ export function ChatIosSimulatorPanel({ : null ), [mediaHeight, mediaWidth, simulatorCaptureSelection]); - const controlAvailable = (status?.tools.find((tool) => tool.name === "idb")?.available ?? false) - && (status?.tools.find((tool) => tool.name === "idb_companion")?.available ?? false); + const toolByName = useMemo(() => new Map((status?.tools ?? []).map((tool) => [tool.name, tool])), [status?.tools]); + const iosurfaceInputAvailable = toolByName.get("iosurface_indigo")?.available ?? false; + const idbInputAvailable = (toolByName.get("idb")?.available ?? false) + && (toolByName.get("idb_companion")?.available ?? false); + const controlAvailable = iosurfaceInputAvailable || idbInputAvailable; const missingRequiredTools = useMemo(() => ( (status?.tools ?? []).filter((tool) => !tool.available && (TOOL_DESCRIPTORS[tool.name]?.required ?? false)) @@ -947,6 +1024,43 @@ export function ChatIosSimulatorPanel({ (status?.tools ?? []).filter((tool) => !tool.available && !(TOOL_DESCRIPTORS[tool.name]?.required ?? false)) ), [status?.tools]); const showSetupChecklist = Boolean(status?.supported) && missingRequiredTools.length > 0; + const bestPerformanceItems = useMemo(() => { + const iosurface = toolByName.get("iosurface_indigo"); + const xcode = toolByName.get("xcodebuild"); + const xcrun = toolByName.get("xcrun"); + const runtime = toolByName.get("simulator_window"); + const idb = toolByName.get("idb"); + const companion = toolByName.get("idb_companion"); + return [ + { + key: "native-streaming", + label: "Native simulator streaming", + ok: Boolean(iosurface?.available), + detail: iosurface?.available + ? "Ready for the low-latency native stream and touch path." + : iosurface?.detail || "Select a supported full Xcode with SimulatorKit available.", + }, + { + key: "xcode-runtime", + label: "Xcode and iOS runtime", + ok: Boolean(xcode?.available && xcrun?.available && runtime?.available), + detail: xcode?.available && xcrun?.available && runtime?.available + ? "Build, boot, launch, and screenshot commands are available." + : "Install full Xcode and an iOS Simulator runtime in Xcode Settings.", + }, + { + key: "fallback-tools", + label: "Fallback stream tools", + ok: Boolean(idb?.available && companion?.available), + detail: idb?.available && companion?.available + ? "idb fallback is ready if the native stream is unavailable." + : "Install idb and idb_companion for the strongest fallback path.", + }, + ]; + }, [toolByName]); + const bestPerformanceReady = bestPerformanceItems.every((item) => item.ok); + const bestPerformanceMissingCount = bestPerformanceItems.filter((item) => !item.ok).length; + const showBestPerformanceChecklist = Boolean(status?.supported) && !showSetupChecklist; const previewSetupSteps = previewCapability?.setupSteps ?? []; const previewIssue = useMemo(() => { if (!previewCapability) { @@ -1012,6 +1126,16 @@ export function ChatIosSimulatorPanel({ }, [activeSession?.chatSessionId, sessionId]); const ownedByOtherChat = otherChatSessionId !== null; + const toggleSimulatorWindowMode = useCallback(() => { + if (!activeSession || ownedByOtherChat) return; + const enable = simulatorWindowSessionId !== activeSession.id; + liveDeviceStreamFailureTimestampsRef.current = []; + setSimulatorWindowSessionId(enable ? activeSession.id : null); + setMessage(enable + ? "Opening Simulator.app and switching to window streaming..." + : "Switching back to the headless simulator stream..."); + }, [activeSession, ownedByOtherChat, simulatorWindowSessionId]); + const launchRef = useRef<(() => Promise) | null>(null); const refreshStatus = useCallback(async () => { @@ -1094,7 +1218,8 @@ export function ChatIosSimulatorPanel({ } }, [projectRoot, selectedElement?.sourceFile, selectedElement?.sourceLine]); - const stopRendererLiveVisual = useCallback(() => { + const stopRendererLiveVisual = useCallback((options: { preserveVisual?: boolean } = {}) => { + const preserveVisual = options.preserveVisual === true; if (liveRestartTimerRef.current != null) { window.clearTimeout(liveRestartTimerRef.current); liveRestartTimerRef.current = null; @@ -1116,6 +1241,13 @@ export function ChatIosSimulatorPanel({ liveFrameCountRef.current = 0; liveFrameWindowStartRef.current = 0; lastWindowFrameAtRef.current = 0; + if (preserveVisual) { + setLiveVisual((current) => current?.kind === "mjpeg" + ? { ...current, status: "reconnecting", url: null, error: null } + : current); + return; + } + lastLiveMjpegFrameAtRef.current = 0; windowScreenRectRef.current = null; setWindowScreenRect(null); setLiveVisual(null); @@ -1171,7 +1303,7 @@ export function ChatIosSimulatorPanel({ error: null, }); let source: IosSimulatorWindowSource | null = null; - for (let attempt = 0; attempt < 5; attempt += 1) { + for (let attempt = 0; attempt < 12; attempt += 1) { source = pickSimulatorWindowSource(await window.ade.iosSimulator.listSimulatorWindowSources(), device); if (source) break; await wait(250); @@ -1191,29 +1323,53 @@ export function ChatIosSimulatorPanel({ }); }, []); - const startDeviceBackedLiveVisual = useCallback(async (device: IosSimulatorDevice) => { - setLiveVisual({ - kind: "mjpeg", - status: "starting", - url: null, - width: null, - height: null, - error: null, - }); - const nextStatus = await window.ade.iosSimulator.startStream({ deviceUdid: device.udid, backend: "auto", fps: 30 }); + const startDeviceBackedLiveVisual = useCallback(async ( + device: IosSimulatorDevice, + options: { backend?: DeviceBackedStreamRequestBackend; preserveVisual?: boolean } = {}, + ) => { + const requestedBackend = options.backend ?? "auto"; + if (options.preserveVisual) { + setLiveVisual((current) => current?.kind === "mjpeg" + ? { ...current, status: "reconnecting", url: null, error: null } + : { + kind: "mjpeg", + status: "reconnecting", + url: null, + width: null, + height: null, + error: null, + }); + } else { + setLiveVisual({ + kind: "mjpeg", + status: "starting", + url: null, + width: null, + height: null, + error: null, + }); + } + const nextStatus = await window.ade.iosSimulator.startStream({ deviceUdid: device.udid, backend: requestedBackend }); + if (nextStatus.backend) lastResolvedStreamBackendRef.current = nextStatus.backend; + if (nextStatus.backend === "simulator-window-capture") { + setStreamStatus(nextStatus); + await startWindowCaptureVisual(device); + return; + } if (!nextStatus.streamUrl) { throw new Error(nextStatus.lastError ?? "Live stream did not provide a drawable URL."); } setStreamStatus(nextStatus); - setLiveVisual({ + setMjpegPlaybackEpoch((epoch) => epoch + 1); + setLiveVisual((current) => ({ kind: "mjpeg", status: "starting", url: nextStatus.streamUrl, - width: null, - height: null, + width: current?.kind === "mjpeg" ? current.width : null, + height: current?.kind === "mjpeg" ? current.height : null, error: null, - }); - }, []); + })); + }, [startWindowCaptureVisual]); useEffect(() => { const video = videoRef.current; @@ -1269,6 +1425,18 @@ export function ChatIosSimulatorPanel({ } }, [activeDevice?.udid, projectRoot, selectedDeviceUdid]); + useEffect(() => { + if (streamStatus?.backend) lastResolvedStreamBackendRef.current = streamStatus.backend; + }, [streamStatus?.backend]); + + const chooseDeviceBackedFallbackBackend = useCallback(( + currentBackend: IosSimulatorStreamStatus["backend"], + ): DeviceBackedStreamRequestBackend | null => { + if (currentBackend === "iosurface-indigo") return idbInputAvailable ? "idb-mjpeg" : "simctl-screenshot-poll"; + if (currentBackend === "idb-mjpeg" || currentBackend === "idb-h264-ffmpeg-mjpeg") return "simctl-screenshot-poll"; + return null; + }, [idbInputAvailable]); + const scheduleDeviceBackedStreamRestart = useCallback((reason: string) => { if ( mode !== "interact" @@ -1283,30 +1451,61 @@ export function ChatIosSimulatorPanel({ const now = Date.now(); if (now - liveRestartAttemptedAtRef.current < 1_500) return; liveRestartAttemptedAtRef.current = now; - setMessage(`${reason} Restarting live simulator stream...`); + const recentFailures = [ + ...liveDeviceStreamFailureTimestampsRef.current.filter((timestamp) => now - timestamp < LIVE_STREAM_FALLBACK_WINDOW_MS), + now, + ]; + liveDeviceStreamFailureTimestampsRef.current = recentFailures; + const currentBackend = streamStatus?.backend ?? lastResolvedStreamBackendRef.current; + const fallbackBackend = recentFailures.length >= LIVE_STREAM_FAILURES_BEFORE_FALLBACK + ? chooseDeviceBackedFallbackBackend(currentBackend) + : null; + const restartBackend = fallbackBackend ?? "auto"; + const reconnectMessage = fallbackBackend ? LIVE_FALLBACK_MESSAGE : LIVE_RECONNECT_MESSAGE; + setMessage(reconnectMessage); + setLiveVisual((current) => current?.kind === "mjpeg" + ? { + ...current, + status: "reconnecting", + url: null, + error: fallbackBackend ? `${reason} Trying a fallback view now.` : null, + } + : current); liveRestartTimerRef.current = window.setTimeout(() => { liveRestartTimerRef.current = null; void (async () => { try { - stopRendererLiveVisual(); + stopRendererLiveVisual({ preserveVisual: true }); await window.ade.iosSimulator.stopStream().catch(() => {}); - await startDeviceBackedLiveVisual(activeDevice); + await startDeviceBackedLiveVisual(activeDevice, { backend: restartBackend, preserveVisual: true }); void refreshSnapshot({ silent: true, priority: true }); } catch (error) { const message = error instanceof Error ? error.message : String(error); - setLiveVisual({ - kind: "mjpeg", - status: "error", - url: null, - width: null, - height: null, - error: `Live stream failed. ${message}`, - }); + setLiveVisual((current) => current?.kind === "mjpeg" + ? { ...current, status: "error", error: `Live stream failed. ${message}` } + : { + kind: "mjpeg", + status: "error", + url: null, + width: null, + height: null, + error: `Live stream failed. ${message}`, + }); setMessage(`Live stream failed. ${message}`); } })(); }, 500); - }, [activeDevice, activeSession, liveVisualKind, mode, refreshSnapshot, startDeviceBackedLiveVisual, stopRendererLiveVisual]); + }, [ + activeDevice, + activeSession, + chooseDeviceBackedFallbackBackend, + liveVisualKind, + mode, + refreshSnapshot, + startDeviceBackedLiveVisual, + stopRendererLiveVisual, + streamStatus?.backend, + ]); const scheduleWindowCaptureRecovery = useCallback((reason: string) => { if ( @@ -1333,6 +1532,7 @@ export function ChatIosSimulatorPanel({ void refreshSnapshot({ silent: true, priority: true }); } catch (windowError) { const windowMessage = windowError instanceof Error ? windowError.message : String(windowError); + setSimulatorWindowSessionId(null); setMessage(`Simulator window capture unavailable. Starting fallback stream... ${windowMessage}`); await window.ade.iosSimulator.stopStream().catch(() => {}); await startDeviceBackedLiveVisual(activeDevice); @@ -1385,21 +1585,36 @@ export function ChatIosSimulatorPanel({ const controller = new AbortController(); mjpegPlaybackAbortRef.current = controller; void playMjpegStreamToCanvas(liveMjpegUrl, canvas, controller.signal, (frame) => { + const now = Date.now(); + lastLiveMjpegFrameAtRef.current = now; + const newestFailureAt = liveDeviceStreamFailureTimestampsRef.current.at(-1) ?? 0; + if (newestFailureAt && now - newestFailureAt > LIVE_STREAM_FAILURE_RESET_AFTER_MS) { + liveDeviceStreamFailureTimestampsRef.current = []; + } + setMessage((current) => ( + current === LIVE_RECONNECT_MESSAGE || current === LIVE_FALLBACK_MESSAGE ? null : current + )); liveFrameCountRef.current += 1; - setLiveVisual((current) => current?.kind === "mjpeg" && current.url === liveMjpegUrl - ? { - ...current, - status: "active", - width: frame.width || current.width, - height: frame.height || current.height, - error: null, - } - : current); + setLiveVisual((current) => { + if (current?.kind !== "mjpeg" || current.url !== liveMjpegUrl) return current; + const width = frame.width || current.width; + const height = frame.height || current.height; + if (current.status === "active" && current.width === width && current.height === height && current.error == null) { + return current; + } + return { + ...current, + status: "active", + width, + height, + error: null, + }; + }); }).catch((error) => { if (controller.signal.aborted || isAbortError(error)) return; const message = error instanceof Error ? error.message : String(error); setLiveVisual((current) => current?.kind === "mjpeg" && current.url === liveMjpegUrl - ? { ...current, status: "error", error: `Live simulator stream could not be decoded. ${message}` } + ? { ...current, status: "reconnecting", url: null, error: `Live simulator stream could not be decoded. ${message}` } : current); scheduleDeviceBackedStreamRestart("Live simulator stream stopped."); }); @@ -1407,7 +1622,7 @@ export function ChatIosSimulatorPanel({ controller.abort(); if (mjpegPlaybackAbortRef.current === controller) mjpegPlaybackAbortRef.current = null; }; - }, [liveMjpegUrl, liveVisualKind, mode, scheduleDeviceBackedStreamRestart]); + }, [liveMjpegUrl, liveVisualKind, mjpegPlaybackEpoch, mode, scheduleDeviceBackedStreamRestart]); useEffect(() => { void refreshStatus().catch((error) => { @@ -1415,6 +1630,10 @@ export function ChatIosSimulatorPanel({ }); }, [refreshStatus]); + useEffect(() => { + setSimulatorWindowSessionId((current) => (current && current === activeSession?.id ? current : null)); + }, [activeSession?.id]); + useEffect(() => { const unsubscribe = window.ade.iosSimulator.onEvent((event) => { if (event.type === "launch-progress") { @@ -1436,14 +1655,43 @@ export function ChatIosSimulatorPanel({ return; } if (event.type === "stream-started" || event.type === "stream-status" || event.type === "stream-stopped" || event.type === "stream-error") { + if (event.status.backend) lastResolvedStreamBackendRef.current = event.status.backend; setStreamStatus(event.status); if ((event.type === "stream-started" || event.type === "stream-status") && event.status.streamUrl) { + if (event.status.lastFrameAt) { + setMessage((current) => ( + current === LIVE_RECONNECT_MESSAGE || current === LIVE_FALLBACK_MESSAGE ? null : current + )); + } + if (event.type === "stream-started") setMjpegPlaybackEpoch((epoch) => epoch + 1); + setLiveVisual((current) => { + if (current?.kind !== "mjpeg") return current; + let status: typeof current.status; + if (event.status.lastFrameAt) { + status = "active"; + } else if (current.status === "reconnecting") { + status = "reconnecting"; + } else { + status = "starting"; + } + return { + ...current, + status, + url: event.status.streamUrl ?? current.url, + error: null, + }; + }); + } + if (event.type === "stream-started" && (event.status.degradationReason || event.status.fallbackReason)) { + setMessage(event.status.degradationReason ?? event.status.fallbackReason ?? null); + } + if ((event.type === "stream-stopped" || event.type === "stream-error") && !event.status.streamUrl) { setLiveVisual((current) => current?.kind === "mjpeg" ? { ...current, - status: event.status.lastFrameAt ? "active" : "starting", - url: event.status.streamUrl ?? current.url, - error: null, + status: event.type === "stream-error" ? "reconnecting" : current.status, + url: null, + error: event.status.lastError ?? current.error, } : current); } @@ -1489,14 +1737,24 @@ export function ChatIosSimulatorPanel({ ) { return; } - const startupGraceMs = streamStatus.backend === "idb-h264-ffmpeg-mjpeg" - ? 18_000 - : streamStatus.backend === "idb-mjpeg" - ? 8_000 - : 5_000; + let startupGraceMs: number; + switch (streamStatus.backend) { + case "idb-h264-ffmpeg-mjpeg": + startupGraceMs = 18_000; + break; + case "idb-mjpeg": + startupGraceMs = 8_000; + break; + case "iosurface-indigo": + startupGraceMs = 10_000; + break; + default: + startupGraceMs = 5_000; + } const timer = window.setInterval(() => { const lastFrameMs = streamStatus.lastFrameAt ? Date.parse(streamStatus.lastFrameAt) : 0; const startedMs = streamStatus.startedAt ? Date.parse(streamStatus.startedAt) : 0; + if (streamStatus.backend === "iosurface-indigo" && lastFrameMs) return; const staleAfterMs = lastFrameMs ? 5_000 : startupGraceMs; const referenceMs = lastFrameMs || startedMs; if (referenceMs && Date.now() - referenceMs > staleAfterMs) { @@ -1552,27 +1810,33 @@ export function ChatIosSimulatorPanel({ || !activeSession || activeSession.deviceUdid !== activeDevice.udid ) { + liveDeviceStreamFailureTimestampsRef.current = []; + lastResolvedStreamBackendRef.current = null; stopRendererLiveVisual(); void window.ade.iosSimulator.stopStream().catch(() => {}); return; } let cancelled = false; const device = activeDevice; + const useSimulatorWindow = simulatorWindowModeEnabled; void (async () => { try { + liveDeviceStreamFailureTimestampsRef.current = []; + lastResolvedStreamBackendRef.current = null; stopRendererLiveVisual(); - try { + if (useSimulatorWindow) { await startWindowCaptureVisual(device); - } catch (windowError) { - if (cancelled) return; - const windowMessage = windowError instanceof Error ? windowError.message : String(windowError); - setMessage(`Simulator window capture unavailable. Starting fallback stream... ${windowMessage}`); - await window.ade.iosSimulator.stopStream().catch(() => {}); + } else { await startDeviceBackedLiveVisual(device); } } catch (streamError) { if (cancelled) return; const message = streamError instanceof Error ? streamError.message : String(streamError); + if (useSimulatorWindow) { + setSimulatorWindowSessionId(null); + setMessage(`Simulator.app stream unavailable. Returning to the headless stream. ${message}`); + return; + } setLiveVisual({ kind: "mjpeg", status: "error", @@ -1586,10 +1850,12 @@ export function ChatIosSimulatorPanel({ })(); return () => { cancelled = true; + liveDeviceStreamFailureTimestampsRef.current = []; + lastResolvedStreamBackendRef.current = null; stopRendererLiveVisual(); void window.ade.iosSimulator.stopStream().catch(() => {}); }; - }, [activeDevice, activeSession, mode, startDeviceBackedLiveVisual, startWindowCaptureVisual, status?.supported, stopRendererLiveVisual]); + }, [activeDevice, activeSession, mode, simulatorWindowModeEnabled, startDeviceBackedLiveVisual, startWindowCaptureVisual, status?.supported, stopRendererLiveVisual]); useEffect(() => { if (mode !== "interact" || liveVisualKind !== "window" || !activeSession) { @@ -2361,8 +2627,16 @@ export function ChatIosSimulatorPanel({ } else if (streamStatus.targetFps) { fpsLabel = `target ${streamStatus.targetFps} fps`; } - const latencyLabel = streamStatus.averageLatencyMs ? ` — ${streamStatus.averageLatencyMs} ms avg capture` : ""; - simulatorModeHint = `Live ${streamStatus.backend ?? "stream"} ${fpsLabel}${latencyLabel}`.trim(); + let latencyLabel = ""; + if (streamStatus.latencyP50Ms || streamStatus.latencyP95Ms) { + latencyLabel = ` - ${streamStatus.latencyP50Ms ?? "?"}/${streamStatus.latencyP95Ms ?? "?"} ms p50/p95`; + } else if (streamStatus.averageLatencyMs) { + latencyLabel = ` - ${streamStatus.averageLatencyMs} ms avg`; + } + const reason = streamStatus.degradationReason ?? streamStatus.fallbackReason ?? null; + const reasonLabel = reason ? ` - ${reason}` : ""; + const pidLabel = streamStatus.helperPid ? ` - pid ${streamStatus.helperPid}` : ""; + simulatorModeHint = `Live via ${streamBackendLabel(streamStatus.backend)} ${fpsLabel}${latencyLabel}${pidLabel}${reasonLabel}`.trim(); } else { simulatorModeHint = "Tap to control the simulator. Click and drag to scroll or swipe."; } @@ -2399,7 +2673,24 @@ export function ChatIosSimulatorPanel({ .filter((item): item is IosSimulatorLaunchProgress => Boolean(item)) : []; const showLaunchProgress = launchBusy && visibleLaunchProgress.length > 0; - const canShowLiveVisual = mode === "interact" && liveVisual && liveVisual.status !== "error"; + let liveVisualOverlayTitle: string | null = null; + let liveVisualOverlayDetail: string | null = null; + switch (liveVisual?.status) { + case "reconnecting": + liveVisualOverlayTitle = "Reconnecting live view..."; + liveVisualOverlayDetail = liveVisual.error ?? streamStatus?.degradationReason ?? streamStatus?.fallbackReason ?? null; + break; + case "starting": + liveVisualOverlayTitle = "Starting live view..."; + break; + case "error": + liveVisualOverlayTitle = "Live view paused"; + liveVisualOverlayDetail = liveVisual.error; + break; + } + const canShowLiveVisual = mode === "interact" + && liveVisual + && (liveVisual.status !== "error" || (liveVisual.kind === "mjpeg" && Boolean(liveVisual.url))); const canShowSnapshot = mode === "inspect" && Boolean(snapshotImage); const hasActiveSession = Boolean(activeSession); const interactionDisabled = ownedByOtherChat || showSetupChecklist; @@ -2461,6 +2752,37 @@ export function ChatIosSimulatorPanel({
{activeSurface === "simulator" ? "Simulator mode" : "Preview mode"}
+ {activeSurface === "simulator" && hasActiveSession && !ownedByOtherChat ? ( +
+ + +
+ ) : null} @@ -2654,10 +2976,34 @@ export function ChatIosSimulatorPanel({ ) : null} - {missingOptionalTools.length && !showSetupChecklist ? ( -
- Optional: {missingOptionalTools.map((tool) => TOOL_DESCRIPTORS[tool.name]?.title ?? tool.name).join(", ")} not installed. -
+ {showBestPerformanceChecklist ? ( +
+ + + Best simulator performance + + {bestPerformanceReady ? "Ready" : `${bestPerformanceMissingCount} item${bestPerformanceMissingCount === 1 ? "" : "s"} need attention`} + + +
+ {bestPerformanceItems.map((item) => ( +
+ {item.ok ? ( + + ) : ( + + )} +
+
{item.label}
+
{item.detail}
+
+
+ ))} +
+
) : null}
@@ -2985,15 +3331,30 @@ export function ChatIosSimulatorPanel({ className="h-full w-full object-contain" aria-label="iOS Simulator live stream" /> - {liveVisual.status === "starting" ? ( -
- + {liveVisualOverlayTitle ? ( +
+
+ {liveVisual.status === "error" ? ( + + ) : ( + + )} +
+
{liveVisualOverlayTitle}
+ {liveVisualOverlayDetail ? ( +
{liveVisualOverlayDetail}
+ ) : null} +
+
) : null} ) : ( -
- +
+
+ + {liveVisualOverlayTitle ?? "Starting live view..."} +
)}
@@ -3020,57 +3381,59 @@ export function ChatIosSimulatorPanel({ onLoad={updateInspectBounds} />
event.stopPropagation()} > -
- +
+
+ + +
-
) : null}
diff --git a/apps/desktop/src/renderer/components/lanes/BranchPickerView.test.tsx b/apps/desktop/src/renderer/components/lanes/BranchPickerView.test.tsx new file mode 100644 index 000000000..a21eb97e1 --- /dev/null +++ b/apps/desktop/src/renderer/components/lanes/BranchPickerView.test.tsx @@ -0,0 +1,160 @@ +/* @vitest-environment jsdom */ + +import React from "react"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { BranchPickerView } from "./BranchPickerView"; +import type { LaneBranchOption } from "./laneUtils"; +import type { BranchPullRequest } from "../../../shared/types"; + +vi.mock("@tanstack/react-virtual", () => { + // Render every row so list assertions don't have to deal with windowing. + return { + useVirtualizer: ({ count }: { count: number }) => ({ + getTotalSize: () => count * 64, + getVirtualItems: () => + Array.from({ length: count }, (_, index) => ({ + index, + start: index * 64, + size: 64, + end: (index + 1) * 64, + key: index, + lane: 0, + })), + measureElement: () => undefined, + }), + }; +}); + +const NOW_ISO = "2026-04-30T10:00:00Z"; + +const branches: LaneBranchOption[] = [ + { + name: "feat/widget", + isCurrent: false, + isRemote: false, + upstream: null, + lastCommitAuthor: "Arul Sharma", + lastCommitMessage: "tweak widget alignment", + lastCommitDate: NOW_ISO, + }, + { + name: "feat/sidebar", + isCurrent: false, + isRemote: false, + upstream: null, + lastCommitAuthor: "Jamie Lee", + lastCommitMessage: "rebuild sidebar nav", + lastCommitDate: NOW_ISO, + }, + { + name: "release/9.1", + isCurrent: true, + isRemote: false, + upstream: null, + lastCommitAuthor: "Arul Sharma", + lastCommitMessage: "bump version", + lastCommitDate: NOW_ISO, + }, +]; + +const prs: BranchPullRequest[] = [ + { + branch: "feat/widget", + prNumber: 812, + title: "Add widget alignment options", + state: "open", + url: "https://github.com/example/repo/pull/812", + author: "arulsharma", + updatedAt: NOW_ISO, + }, +]; + +function renderPicker(overrides: Partial> = {}) { + return render( + , + ); +} + +describe("BranchPickerView", () => { + afterEach(cleanup); + + it("renders all branches with their last-commit metadata and PR pills", () => { + renderPicker(); + expect(screen.getByText("feat/widget")).toBeTruthy(); + expect(screen.getByText("feat/sidebar")).toBeTruthy(); + expect(screen.getByText("release/9.1")).toBeTruthy(); + expect(screen.getByText("#812")).toBeTruthy(); + expect(screen.getByText(/3 of 3/)).toBeTruthy(); + }); + + it("filters by free-text search", () => { + renderPicker(); + fireEvent.change(screen.getByPlaceholderText(/name . pr:open/), { target: { value: "sidebar" } }); + expect(screen.queryByText("feat/widget")).toBeFalsy(); + expect(screen.getByText("feat/sidebar")).toBeTruthy(); + }); + + it("filters by pr:open token", () => { + renderPicker(); + fireEvent.change(screen.getByPlaceholderText(/name . pr:open/), { target: { value: "pr:open" } }); + expect(screen.getByText("feat/widget")).toBeTruthy(); + expect(screen.queryByText("feat/sidebar")).toBeFalsy(); + expect(screen.queryByText("release/9.1")).toBeFalsy(); + }); + + it("filters by mine using current user name", () => { + renderPicker(); + fireEvent.change(screen.getByPlaceholderText(/name . pr:open/), { target: { value: "mine" } }); + expect(screen.getByText("feat/widget")).toBeTruthy(); + expect(screen.queryByText("feat/sidebar")).toBeFalsy(); + expect(screen.getByText("release/9.1")).toBeTruthy(); + }); + + it("invokes onSelect and onConfirm when a row is clicked then confirmed", () => { + const onSelect = vi.fn(); + const onConfirm = vi.fn(); + renderPicker({ onSelect, onConfirm }); + fireEvent.click(screen.getByText("feat/widget")); + expect(onSelect).toHaveBeenCalledWith("feat/widget"); + + cleanup(); + const onConfirm2 = vi.fn(); + renderPicker({ selectedBranch: "feat/widget", onConfirm: onConfirm2 }); + fireEvent.click(screen.getByRole("button", { name: "Use this branch" })); + expect(onConfirm2).toHaveBeenCalled(); + }); + + it("Enter on the search input picks the first filtered row", () => { + const onSelect = vi.fn(); + const onConfirm = vi.fn(); + renderPicker({ onSelect, onConfirm }); + const input = screen.getByPlaceholderText(/name . pr:open/); + fireEvent.change(input, { target: { value: "sidebar" } }); + fireEvent.keyDown(input, { key: "Enter" }); + expect(onSelect).toHaveBeenCalledWith("feat/sidebar"); + expect(onConfirm).toHaveBeenCalled(); + }); + + it("calls onBack when the back button is clicked", () => { + const onBack = vi.fn(); + renderPicker({ onBack }); + fireEvent.click(screen.getByLabelText("Back to lane setup")); + expect(onBack).toHaveBeenCalled(); + }); + + it("shows an empty-state message when no branches match", () => { + renderPicker(); + fireEvent.change(screen.getByPlaceholderText(/name . pr:open/), { target: { value: "nopenope" } }); + expect(screen.getByText("No branches match your search.")).toBeTruthy(); + }); +}); diff --git a/apps/desktop/src/renderer/components/lanes/BranchPickerView.tsx b/apps/desktop/src/renderer/components/lanes/BranchPickerView.tsx new file mode 100644 index 000000000..7652f04dc --- /dev/null +++ b/apps/desktop/src/renderer/components/lanes/BranchPickerView.tsx @@ -0,0 +1,286 @@ +import React from "react"; +import { ArrowLeft, GitBranch, GitCommit, MagnifyingGlass, Tag } from "@phosphor-icons/react"; +import { useVirtualizer } from "@tanstack/react-virtual"; +import type { BranchPullRequest } from "../../../shared/types"; +import type { LaneBranchOption } from "./laneUtils"; +import { Button } from "../ui/Button"; +import { + formatRelativeTime, + matchesQuery, + parseSearchQuery, +} from "./branchPickerSearch"; + +const ROW_HEIGHT = 64; + +const SEARCH_HINT = "name · pr:open · pr:none · author:me · mine · stale:30d · #812"; + +const ROW_BASE = + "group flex w-full items-start gap-3 rounded-lg px-3 py-2 text-left transition-colors"; +const ROW_SELECTED = "bg-accent/[0.12] ring-1 ring-accent/40"; +const ROW_DEFAULT = "hover:bg-white/[0.04]"; + +/** Strip the remote name from `origin/feat/x` so PR lookups match local heads. */ +function stripRemotePrefix(branch: LaneBranchOption): string { + return branch.isRemote ? branch.name.replace(/^[^/]+\//, "") : branch.name; +} + +function findPrForBranch( + branch: LaneBranchOption, + prByBranch: Map, +): BranchPullRequest | null { + return prByBranch.get(branch.name) ?? prByBranch.get(stripRemotePrefix(branch)) ?? null; +} + +function PrPill({ pr }: { pr: BranchPullRequest }) { + const isDraft = pr.state === "draft"; + const tone = isDraft + ? "bg-white/[0.08] text-muted-fg" + : "bg-emerald-400/15 text-emerald-300"; + return ( + + + #{pr.prNumber} + {isDraft ? draft : null} + + ); +} + +function BranchRow({ + branch, + pr, + selected, + onSelect, + now, +}: { + branch: LaneBranchOption; + pr: BranchPullRequest | null; + selected: boolean; + onSelect: () => void; + now: number; +}) { + const subtitle = [branch.lastCommitAuthor, formatRelativeTime(branch.lastCommitDate, now)] + .filter(Boolean) + .join(" · "); + + return ( + + ); +} + +export function BranchPickerView({ + branches, + pullRequests, + currentUserName, + selectedBranch, + onSelect, + onConfirm, + onBack, + busy, + loadingBranches, + loadingPullRequests, +}: { + branches: LaneBranchOption[]; + pullRequests: BranchPullRequest[]; + currentUserName: string; + selectedBranch: string; + onSelect: (name: string) => void; + onConfirm: () => void; + onBack: () => void; + busy?: boolean; + loadingBranches?: boolean; + loadingPullRequests?: boolean; +}) { + const [query, setQuery] = React.useState(""); + const inputRef = React.useRef(null); + const listRef = React.useRef(null); + const now = React.useMemo(() => Date.now(), []); + + React.useEffect(() => { + inputRef.current?.focus(); + }, []); + + const prByBranch = React.useMemo(() => { + const map = new Map(); + for (const pr of pullRequests) map.set(pr.branch, pr); + return map; + }, [pullRequests]); + + const parsed = React.useMemo(() => parseSearchQuery(query), [query]); + + const filtered = React.useMemo(() => { + if (!query.trim() && !parsed.tokens.length) return branches; + return branches.filter((branch) => { + const pr = findPrForBranch(branch, prByBranch); + return matchesQuery({ branch, pr, currentUserName, now }, parsed); + }); + }, [branches, query, parsed, prByBranch, currentUserName, now]); + + const virtualizer = useVirtualizer({ + count: filtered.length, + getScrollElement: () => listRef.current, + estimateSize: () => ROW_HEIGHT, + overscan: 8, + }); + + const handleKeyDown = (event: React.KeyboardEvent) => { + if (event.key === "Enter" && filtered.length > 0) { + event.preventDefault(); + const first = filtered[0]!; + onSelect(first.name); + onConfirm(); + } + }; + + let emptyMessage: string; + if (loadingBranches) emptyMessage = "Loading branches…"; + else if (branches.length === 0) emptyMessage = "No branches found in this repo."; + else emptyMessage = "No branches match your search."; + + return ( +
+
+ +
+ Pick branch +
+
+ {loadingBranches + ? "Loading…" + : `${filtered.length} of ${branches.length}`} + {loadingPullRequests ? " · syncing PRs…" : null} +
+
+ +
+ + setQuery(event.target.value)} + onKeyDown={handleKeyDown} + placeholder={SEARCH_HINT} + className="h-10 w-full rounded-lg border border-white/[0.06] bg-white/[0.03] pl-9 pr-3 text-sm text-fg outline-none transition-colors placeholder:text-muted-fg/50 focus:border-accent/40" + spellCheck={false} + autoComplete="off" + disabled={busy} + /> +
+ +
+ {filtered.length === 0 ? ( +
+ {emptyMessage} +
+ ) : ( +
+ {virtualizer.getVirtualItems().map((row) => { + const branch = filtered[row.index]!; + const pr = findPrForBranch(branch, prByBranch); + return ( +
+ onSelect(branch.name)} + now={now} + /> +
+ ); + })} +
+ )} +
+ +
+
+ {selectedBranch ? ( + <> + Selected: {selectedBranch} + + ) : ( + "Select a branch to import" + )} +
+
+ + +
+
+
+ ); +} diff --git a/apps/desktop/src/renderer/components/lanes/CreateLaneDialog.tsx b/apps/desktop/src/renderer/components/lanes/CreateLaneDialog.tsx index b3798ae1c..cdbe4731a 100644 --- a/apps/desktop/src/renderer/components/lanes/CreateLaneDialog.tsx +++ b/apps/desktop/src/renderer/components/lanes/CreateLaneDialog.tsx @@ -1,12 +1,14 @@ import React from "react"; -import { CaretDown, GitBranch, GitFork, Plus, StackSimple } from "@phosphor-icons/react"; +import { CaretDown, CaretRight, GitBranch, GitFork, Plus, StackSimple, Tag } from "@phosphor-icons/react"; import { Button } from "../ui/Button"; -import type { LaneSummary, LaneEnvInitProgress, LaneTemplate } from "../../../shared/types"; +import type { BranchPullRequest, LaneSummary, LaneEnvInitProgress, LaneTemplate } from "../../../shared/types"; import type { LaneBranchOption } from "./laneUtils"; import { LaneEnvInitProgressPanel } from "./LaneEnvInitProgress"; import { LaneDialogShell } from "./LaneDialogShell"; import { LaneColorPicker } from "./LaneColorPicker"; import { colorsInUse, nextAvailableColor } from "./laneColorPalette"; +import { BranchPickerView } from "./BranchPickerView"; +import { formatRelativeTime } from "./branchPickerSearch"; import { SECTION_CLASS_NAME, LABEL_CLASS_NAME, @@ -52,7 +54,7 @@ const MODE_META: Record = { const MODE_ORDER: readonly CreateLaneMode[] = ["primary", "existing", "child"]; function submitLabel(busy: boolean | undefined, mode: CreateLaneMode, baseBranch: string, laneCreated: boolean | undefined): string { - if (busy) return "Setting up lane\u2026"; + if (busy) return "Setting up lane…"; if (laneCreated) return "Retry environment setup"; if (mode === "child") return "Create child lane"; if (mode === "existing") return "Import as lane"; @@ -87,7 +89,11 @@ export function CreateLaneDialog({ onNavigateToTemplates, importBranchWarning, selectedColor, - setSelectedColor + setSelectedColor, + branchPullRequests, + currentGitUserName, + loadingBranches, + loadingBranchPullRequests, }: { open: boolean; onOpenChange: (open: boolean) => void; @@ -119,12 +125,50 @@ export function CreateLaneDialog({ importBranchWarning?: string | null; selectedColor: string | null; setSelectedColor: (c: string | null) => void; + /** Open PRs in the project's GitHub repo, keyed by head branch. */ + branchPullRequests?: BranchPullRequest[]; + /** Local git user.name — used by the picker to resolve `mine` / `author:me`. */ + currentGitUserName?: string; + loadingBranches?: boolean; + loadingBranchPullRequests?: boolean; }) { const localBranches = createBranches.filter((b) => !b.isRemote); const allBranches = createBranches; const selectedTemplate = templates.find((t) => t.id === selectedTemplateId) ?? null; const usedColors = React.useMemo(() => colorsInUse(lanes), [lanes]); + const [pickerOpen, setPickerOpen] = React.useState(false); + + React.useEffect(() => { + if (!open) setPickerOpen(false); + }, [open]); + + React.useEffect(() => { + if (createMode !== "existing") setPickerOpen(false); + }, [createMode]); + + const prByBranch = React.useMemo(() => { + const map = new Map(); + for (const pr of branchPullRequests ?? []) map.set(pr.branch, pr); + return map; + }, [branchPullRequests]); + + const selectedBranchMeta = React.useMemo<{ + branch: LaneBranchOption | null; + pr: BranchPullRequest | null; + }>(() => { + const branch = createBranches.find((b) => b.name === createImportBranch) ?? null; + if (!branch) return { branch: null, pr: null }; + const localName = branch.isRemote ? branch.name.replace(/^[^/]+\//, "") : branch.name; + const pr = prByBranch.get(branch.name) ?? prByBranch.get(localName) ?? null; + return { branch, pr }; + }, [createBranches, createImportBranch, prByBranch]); + + let branchPickerPlaceholder: string; + if (loadingBranches) branchPickerPlaceholder = "Loading branches…"; + else if (allBranches.length === 0) branchPickerPlaceholder = "No branches found"; + else branchPickerPlaceholder = "Pick a branch…"; + React.useEffect(() => { if (open && selectedColor === null) { const next = nextAvailableColor(lanes); @@ -146,8 +190,10 @@ export function CreateLaneDialog({ + {pickerOpen ? ( + setCreateImportBranch(name)} + onConfirm={() => setPickerOpen(false)} + onBack={() => setPickerOpen(false)} + busy={busy || laneCreated} + loadingBranches={loadingBranches} + loadingPullRequests={loadingBranchPullRequests} + /> + ) : (
{/* Lane name */}
@@ -268,45 +328,70 @@ export function CreateLaneDialog({ ) : null} {createMode === "existing" ? ( - allBranches.length > 0 ? ( - <> - - {createImportBranch ? ( -
- Imported as a root lane -
- ) : null} - {importBranchWarning ? ( - - ) : null} - - ) : ( -
- No branches found. -
- ) + ⚠ + {importBranchWarning} +
+ ) : null} + ) : null} {createMode === "child" ? ( @@ -318,7 +403,7 @@ export function CreateLaneDialog({ disabled={busy || laneCreated} aria-label="Parent lane" > - + {lanes.map((lane) => (
+ )} ); } diff --git a/apps/desktop/src/renderer/components/lanes/LanesPage.tsx b/apps/desktop/src/renderer/components/lanes/LanesPage.tsx index 08071ea1d..5dee6f5bd 100644 --- a/apps/desktop/src/renderer/components/lanes/LanesPage.tsx +++ b/apps/desktop/src/renderer/components/lanes/LanesPage.tsx @@ -49,6 +49,7 @@ import { buildPrsRouteSearch } from "../prs/prsRouteState"; import { getProjectConfigCached } from "../../lib/projectConfigCache"; import { logRendererDebugEvent } from "../../lib/debugLog"; import type { + BranchPullRequest, ConflictChip, DeleteLaneArgs, GitCommitSummary, @@ -241,6 +242,10 @@ export function LanesPage() { const [createImportBranch, setCreateImportBranch] = useState(""); const [createChildBaseBranch, setCreateChildBaseBranch] = useState(""); const [createBranches, setCreateBranches] = useState([]); + const [createBranchesLoading, setCreateBranchesLoading] = useState(false); + const [createBranchPullRequests, setCreateBranchPullRequests] = useState([]); + const [createBranchPullRequestsLoading, setCreateBranchPullRequestsLoading] = useState(false); + const [createGitUserName, setCreateGitUserName] = useState(""); const [createBusy, setCreateBusy] = useState(false); const [createError, setCreateError] = useState(null); const [createEnvInitProgress, setCreateEnvInitProgress] = useState(null); @@ -1528,11 +1533,13 @@ export function LanesPage() { setCreateImportBranch(""); setCreateChildBaseBranch(""); setCreateBranches([]); + setCreateBranchPullRequests([]); setLaneCreated(false); createBaseBranchUserPickedRef.current = false; const primary = lanes.find((l) => l.laneType === "primary"); if (primary) { // Fetch remotes first so remote-only branches (pushed from other machines) appear. + setCreateBranchesLoading(true); window.ade.git.fetch({ laneId: primary.id }) .catch(() => {}) .then(() => window.ade.git.listBranches({ laneId: primary.id })) @@ -1548,7 +1555,20 @@ export function LanesPage() { if (defaultBranch) setCreateBaseBranch(defaultBranch.name); } }) - .catch(() => {}); + .catch(() => {}) + .finally(() => setCreateBranchesLoading(false)); + + // Capture git user.name so the picker can resolve `mine` / `author:me`. + window.ade.git.getUserIdentity({ laneId: primary.id }) + .then((identity) => setCreateGitUserName(identity?.name ?? "")) + .catch(() => setCreateGitUserName("")); + + // Lazily attach open-PR metadata. Fail-soft — picker degrades gracefully. + setCreateBranchPullRequestsLoading(true); + window.ade.prs.listOpenForRepo() + .then(setCreateBranchPullRequests) + .catch(() => setCreateBranchPullRequests([])) + .finally(() => setCreateBranchPullRequestsLoading(false)); } Promise.all([ window.ade.lanes.listTemplates().catch(() => [] as LaneTemplate[]), @@ -2969,6 +2989,10 @@ export function LanesPage() { setSelectedTemplateId={setSelectedTemplateId} selectedColor={createSelectedColor} setSelectedColor={setCreateSelectedColor} + branchPullRequests={createBranchPullRequests} + currentGitUserName={createGitUserName} + loadingBranches={createBranchesLoading} + loadingBranchPullRequests={createBranchPullRequestsLoading} onNavigateToTemplates={() => navigate("/settings?tab=lane-templates")} importBranchWarning={ createMode === "existing" && createImportBranch && primaryLane?.status.dirty diff --git a/apps/desktop/src/renderer/components/lanes/branchPickerSearch.test.ts b/apps/desktop/src/renderer/components/lanes/branchPickerSearch.test.ts new file mode 100644 index 000000000..35b78670b --- /dev/null +++ b/apps/desktop/src/renderer/components/lanes/branchPickerSearch.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, it } from "vitest"; +import type { BranchPullRequest } from "../../../shared/types"; +import type { LaneBranchOption } from "./laneUtils"; +import { + formatRelativeTime, + matchesQuery, + parseSearchQuery, +} from "./branchPickerSearch"; + +const NOW = Date.parse("2026-04-30T12:00:00Z"); + +function makeBranch(overrides: Partial = {}): LaneBranchOption { + return { + name: "feat/widget", + isCurrent: false, + isRemote: false, + upstream: null, + lastCommitAuthor: "Arul Sharma", + lastCommitMessage: "tweak widget alignment", + lastCommitDate: new Date(NOW - 2 * 60 * 60 * 1000).toISOString(), + ...overrides, + }; +} + +function makePr(overrides: Partial = {}): BranchPullRequest { + return { + branch: "feat/widget", + prNumber: 812, + title: "Add widget alignment options", + state: "open", + url: "https://github.com/example/repo/pull/812", + author: "arulsharma", + updatedAt: new Date(NOW - 60 * 60 * 1000).toISOString(), + ...overrides, + }; +} + +describe("parseSearchQuery", () => { + it("treats plain text as free-text", () => { + expect(parseSearchQuery("widget alignment")).toEqual({ + tokens: [], + freeText: "widget alignment", + }); + }); + + it("parses pr:open / pr:none / pr:draft", () => { + expect(parseSearchQuery("pr:open").tokens).toEqual([{ kind: "pr", value: "open" }]); + expect(parseSearchQuery("pr:none").tokens).toEqual([{ kind: "pr", value: "none" }]); + expect(parseSearchQuery("pr:draft").tokens).toEqual([{ kind: "pr", value: "draft" }]); + }); + + it("ignores unknown pr: values, treats them as free text", () => { + const parsed = parseSearchQuery("pr:bogus"); + expect(parsed.tokens).toEqual([]); + expect(parsed.freeText).toBe("pr:bogus"); + }); + + it("parses author:NAME and author:me", () => { + expect(parseSearchQuery("author:jamie").tokens).toEqual([ + { kind: "author", value: "jamie", isMe: false }, + ]); + expect(parseSearchQuery("author:me").tokens).toEqual([ + { kind: "author", value: "me", isMe: true }, + ]); + }); + + it("parses mine alias", () => { + expect(parseSearchQuery("mine").tokens).toEqual([{ kind: "mine" }]); + }); + + it("parses stale:Nd", () => { + expect(parseSearchQuery("stale:30d").tokens).toEqual([{ kind: "stale", days: 30 }]); + expect(parseSearchQuery("stale:7").tokens).toEqual([{ kind: "stale", days: 7 }]); + }); + + it("parses #PRNUMBER", () => { + expect(parseSearchQuery("#812").tokens).toEqual([{ kind: "prNumber", value: 812 }]); + }); + + it("ANDs tokens and free text together", () => { + const parsed = parseSearchQuery("pr:open author:me widget"); + expect(parsed.tokens).toHaveLength(2); + expect(parsed.freeText).toBe("widget"); + }); +}); + +describe("matchesQuery", () => { + const ctx = ( + overrides: Partial<{ branch: LaneBranchOption; pr: BranchPullRequest | null; user: string }> = {}, + ) => ({ + branch: overrides.branch ?? makeBranch(), + pr: overrides.pr === undefined ? makePr() : overrides.pr, + currentUserName: overrides.user ?? "Arul Sharma", + now: NOW, + }); + + it("free-text matches branch name", () => { + expect(matchesQuery(ctx(), parseSearchQuery("widg"))).toBe(true); + expect(matchesQuery(ctx(), parseSearchQuery("rocket"))).toBe(false); + }); + + it("free-text matches PR title", () => { + expect(matchesQuery(ctx(), parseSearchQuery("alignment options"))).toBe(true); + }); + + it("free-text matches PR number with hash prefix", () => { + expect(matchesQuery(ctx(), parseSearchQuery("#812"))).toBe(true); + }); + + it("pr:open keeps branches with open PR", () => { + expect(matchesQuery(ctx(), parseSearchQuery("pr:open"))).toBe(true); + expect(matchesQuery(ctx({ pr: null }), parseSearchQuery("pr:open"))).toBe(false); + }); + + it("pr:none keeps branches with no PR", () => { + expect(matchesQuery(ctx({ pr: null }), parseSearchQuery("pr:none"))).toBe(true); + expect(matchesQuery(ctx(), parseSearchQuery("pr:none"))).toBe(false); + }); + + it("pr:draft keeps draft PRs only", () => { + expect(matchesQuery(ctx({ pr: makePr({ state: "draft" }) }), parseSearchQuery("pr:draft"))).toBe(true); + expect(matchesQuery(ctx(), parseSearchQuery("pr:draft"))).toBe(false); + }); + + it("author:me matches the current user (case-insensitive substring)", () => { + expect(matchesQuery(ctx(), parseSearchQuery("author:me"))).toBe(true); + expect(matchesQuery(ctx({ user: "Jamie" }), parseSearchQuery("author:me"))).toBe(false); + }); + + it("mine is an alias for author:me", () => { + expect(matchesQuery(ctx(), parseSearchQuery("mine"))).toBe(true); + expect(matchesQuery(ctx({ user: "Jamie" }), parseSearchQuery("mine"))).toBe(false); + }); + + it("stale:Nd filters by commit age", () => { + const fresh = ctx({ + branch: makeBranch({ lastCommitDate: new Date(NOW - 1 * 24 * 60 * 60 * 1000).toISOString() }), + }); + const stale = ctx({ + branch: makeBranch({ lastCommitDate: new Date(NOW - 60 * 24 * 60 * 60 * 1000).toISOString() }), + }); + expect(matchesQuery(fresh, parseSearchQuery("stale:30d"))).toBe(false); + expect(matchesQuery(stale, parseSearchQuery("stale:30d"))).toBe(true); + }); + + it("ANDs multiple tokens", () => { + const mineOpen = parseSearchQuery("pr:open mine"); + expect(matchesQuery(ctx(), mineOpen)).toBe(true); + expect(matchesQuery(ctx({ pr: null }), mineOpen)).toBe(false); + expect(matchesQuery(ctx({ user: "Jamie" }), mineOpen)).toBe(false); + }); +}); + +describe("formatRelativeTime", () => { + it("returns empty for missing input", () => { + expect(formatRelativeTime(undefined, NOW)).toBe(""); + }); + + it("formats seconds, minutes, hours, days", () => { + expect(formatRelativeTime(new Date(NOW - 30_000).toISOString(), NOW)).toBe("30s"); + expect(formatRelativeTime(new Date(NOW - 5 * 60_000).toISOString(), NOW)).toBe("5m"); + expect(formatRelativeTime(new Date(NOW - 3 * 60 * 60_000).toISOString(), NOW)).toBe("3h"); + expect(formatRelativeTime(new Date(NOW - 4 * 24 * 60 * 60_000).toISOString(), NOW)).toBe("4d"); + }); +}); diff --git a/apps/desktop/src/renderer/components/lanes/branchPickerSearch.ts b/apps/desktop/src/renderer/components/lanes/branchPickerSearch.ts new file mode 100644 index 000000000..d71c1581a --- /dev/null +++ b/apps/desktop/src/renderer/components/lanes/branchPickerSearch.ts @@ -0,0 +1,171 @@ +import type { BranchPullRequest } from "../../../shared/types"; +import type { LaneBranchOption } from "./laneUtils"; + +/** + * Tokens that can be ANDed together in the search bar. + * + * pr:open — only branches with an open PR + * pr:none — only branches with no PR + * pr:draft — only branches whose PR is draft + * author:NAME — last-commit author matches NAME (case-insensitive substring) + * author:me — last-commit author matches the current git user + * mine — alias for author:me + * stale:Nd — last commit older than N days + * #N — exact PR number + * + * Anything left over is fuzzy matched across name, PR title, and author. + */ +export type SearchToken = + | { kind: "pr"; value: "open" | "none" | "draft" } + | { kind: "author"; value: string; isMe: boolean } + | { kind: "mine" } + | { kind: "stale"; days: number } + | { kind: "prNumber"; value: number }; + +export type ParsedSearchQuery = { + tokens: SearchToken[]; + freeText: string; +}; + +const KNOWN_PR_VALUES = new Set(["open", "none", "draft"]); + +export function parseSearchQuery(input: string): ParsedSearchQuery { + const tokens: SearchToken[] = []; + const leftover: string[] = []; + const parts = input.trim().split(/\s+/).filter(Boolean); + + for (const raw of parts) { + const lower = raw.toLowerCase(); + + if (lower === "mine") { + tokens.push({ kind: "mine" }); + continue; + } + + if (lower.startsWith("pr:")) { + const value = lower.slice(3); + if (KNOWN_PR_VALUES.has(value)) { + tokens.push({ kind: "pr", value: value as "open" | "none" | "draft" }); + continue; + } + } + + if (lower.startsWith("author:")) { + const rest = raw.slice(7); + const isMe = rest.toLowerCase() === "me"; + tokens.push({ kind: "author", value: rest, isMe }); + continue; + } + + if (lower.startsWith("stale:")) { + const match = lower.slice(6).match(/^(\d+)d?$/); + if (match) { + const days = Number(match[1]); + if (Number.isFinite(days) && days > 0) { + tokens.push({ kind: "stale", days }); + continue; + } + } + } + + if (lower.startsWith("#")) { + const num = Number(lower.slice(1)); + if (Number.isInteger(num) && num > 0) { + tokens.push({ kind: "prNumber", value: num }); + continue; + } + } + + leftover.push(raw); + } + + return { tokens, freeText: leftover.join(" ").trim() }; +} + +export type BranchMatchContext = { + branch: LaneBranchOption; + pr: BranchPullRequest | null; + /** Lowercased current user — for `author:me` / `mine`. */ + currentUserName: string; + /** Reference time for stale comparisons, defaults to Date.now(). */ + now?: number; +}; + +const MS_PER_DAY = 24 * 60 * 60 * 1000; + +function fuzzyContains(haystack: string | null | undefined, needle: string): boolean { + if (!haystack) return false; + return haystack.toLowerCase().includes(needle.toLowerCase()); +} + +export function matchesQuery(ctx: BranchMatchContext, query: ParsedSearchQuery): boolean { + const { branch, pr, currentUserName } = ctx; + const me = currentUserName.toLowerCase(); + const now = ctx.now ?? Date.now(); + + for (const token of query.tokens) { + switch (token.kind) { + case "pr": { + if (token.value === "none" && pr) return false; + if (token.value === "open" && (!pr || pr.state !== "open")) return false; + if (token.value === "draft" && (!pr || pr.state !== "draft")) return false; + break; + } + case "prNumber": { + if (!pr || pr.prNumber !== token.value) return false; + break; + } + case "author": { + const target = token.isMe ? me : token.value.toLowerCase(); + if (!target) return false; + if (!fuzzyContains(branch.lastCommitAuthor, target)) return false; + break; + } + case "mine": { + if (!me || !fuzzyContains(branch.lastCommitAuthor, me)) return false; + break; + } + case "stale": { + const ts = branch.lastCommitDate ? Date.parse(branch.lastCommitDate) : Number.NaN; + if (!Number.isFinite(ts)) return false; + const ageDays = (now - ts) / MS_PER_DAY; + if (ageDays < token.days) return false; + break; + } + } + } + + if (query.freeText) { + const text = query.freeText.toLowerCase(); + const hits = + fuzzyContains(branch.name, text) || + fuzzyContains(branch.lastCommitMessage, text) || + fuzzyContains(branch.lastCommitAuthor, text) || + fuzzyContains(pr?.title ?? null, text) || + fuzzyContains(pr?.author ?? null, text) || + (pr ? `#${pr.prNumber}`.includes(text) : false); + if (!hits) return false; + } + + return true; +} + +export function formatRelativeTime(iso: string | undefined, now: number = Date.now()): string { + if (!iso) return ""; + const ts = Date.parse(iso); + if (!Number.isFinite(ts)) return ""; + const diffMs = now - ts; + if (diffMs < 0) return "now"; + const seconds = Math.floor(diffMs / 1000); + if (seconds < 60) return `${seconds}s`; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) return `${minutes}m`; + const hours = Math.floor(minutes / 60); + if (hours < 24) return `${hours}h`; + const days = Math.floor(hours / 24); + if (days < 30) return `${days}d`; + const months = Math.floor(days / 30); + if (months < 12) return `${months}mo`; + const years = Math.floor(days / 365); + return `${years}y`; +} diff --git a/apps/desktop/src/renderer/components/lanes/laneUtils.ts b/apps/desktop/src/renderer/components/lanes/laneUtils.ts index 25d83d4cc..13e03f1ab 100644 --- a/apps/desktop/src/renderer/components/lanes/laneUtils.ts +++ b/apps/desktop/src/renderer/components/lanes/laneUtils.ts @@ -183,6 +183,10 @@ export type LaneBranchOption = { ownedByLaneName?: string | null; profiledInCurrentLane?: boolean; hasOpenPr?: boolean; + lastCommitSha?: string; + lastCommitMessage?: string; + lastCommitDate?: string; + lastCommitAuthor?: string; }; export const EMPTY_LANE_PANE_DETAIL: LanePaneDetailSelection = { diff --git a/apps/desktop/src/shared/adeCliGuidance.ts b/apps/desktop/src/shared/adeCliGuidance.ts index 68001d1f4..f6529d57c 100644 --- a/apps/desktop/src/shared/adeCliGuidance.ts +++ b/apps/desktop/src/shared/adeCliGuidance.ts @@ -8,8 +8,8 @@ export const ADE_CLI_AGENT_GUIDANCE = [ "Use `ade help ` and `ade help ` to get precise flags instead of guessing. For iOS Simulator and Preview Lab work, start with `ade help ios-sim`, then drill into `ade help ios-sim launch`, `snapshot`, `previews`, `preview-render`, `select`, `tap`, `drag`, `type`, or `shutdown` as needed. Use `ade --socket ios-sim actions --text` for the raw ios_simulator action catalog.", "For App Control work on Electron apps, prefer desktop socket mode so the CLI and ADE drawer share the same managed app session. Start with `ade help app-control` and `ade --socket app-control status --text`; launch with `ade --socket app-control launch --command \"npm run dev\" --text` or connect to an existing debuggable app with `ade --socket app-control connect --cdp-port --text`. To reuse a Run-tab process, list configured commands via `ade --socket settings get --text`, then launch with `--command` plus `--cwd` matching that process so the app runs from the same directory the Run tab does (relative cwds resolve against the lane root, and the launch is rejected if the resolved cwd escapes the lane). Launches run in the visible chat terminal; ADE sets ADE_APP_CONTROL_CDP_PORT and ADE_APP_CONTROL_DEBUG_FLAGS in the environment and auto-forwards debug flags for common npm/pnpm/yarn/bun script launches and direct electron commands. Custom launchers should forward one of those values to Electron's --remote-debugging-port.", "After an App Control launch or when the user asks what happened in that visible launch terminal, first run `ade --socket app-control status --text` or `--json`; do not say you lack terminal visibility until App Control status and terminal list have both been checked. Prefer `ade --socket app-control logs --text --max-bytes 8388608` to inspect full scrollback, `ade --socket app-control terminal write --data \"y\\n\"` to answer prompts, and `ade --socket app-control terminal signal --signal SIGINT` to interrupt the active App Control launch. If there is no active App Control terminal, fall back to `ade --socket terminal list --text --limit 20` and choose the running/recent `App Control:` terminal, or use `ADE_CHAT_SESSION_ID` with `ade --socket terminal read --chat-session \"$ADE_CHAT_SESSION_ID\" --text`. After the Electron renderer connects, use `ade --socket app-control snapshot --text` for screenshot + DOM refs, `click` / `type` for Control-mode input, and the drawer's Inspect mode (or `ade --socket app-control select --x --y `) to attach screenshot-backed UI context to chat. App Control is a bridge over existing tools; Playwright, agent-browser, browser-use, or Computer Use can still be useful, but ADE should preserve the launch state, terminal output, screenshot, DOM/selector packet, source candidates, and proof/context attachments.", - "For iOS Simulator work, prefer desktop socket mode so the CLI and ADE drawer share one long-lived simulator service: run `ade --socket ios-sim status --text`, `ade --socket ios-sim devices --text`, `ade --socket ios-sim apps --device --text`, then `ade --socket ios-sim launch --target --text` (or `--bundle-id`/`--app-bundle`). Launch keeps Simulator.app in the background by default so the ADE drawer remains the control plane.", - "After an iOS app is launched, use `ade --socket ios-sim snapshot --text` or `ade --socket ios-sim elements --text` for screenshot + accessibility/ADEInspector grounding, then `ade --socket ios-sim select --x --y ` to add selected UI context to the drawer chat. Use `tap`, `drag`/`swipe`, and `type` against the active launched app; use `window-start`, `live-start`, `preview-start`, `stream-status`, and `stream-stop` only for visual stream management. If the simulator is not running or no active session/snapshot is available, warn the user with the exact blocker instead of guessing the screen. When you finish, run `ade --socket ios-sim shutdown` to tear down the session, stop streaming, and release the idb companion.", + "For iOS Simulator work, prefer desktop socket mode so the CLI and ADE drawer share one long-lived simulator service: run `ade --socket ios-sim status --text`, `ade --socket ios-sim devices --text`, `ade --socket ios-sim apps --device --text`, then `ade --socket ios-sim launch --target --text` (or `--bundle-id`/`--app-bundle`). Launch is headless by default; use explicit window capture or foreground launch only when the user wants the real Simulator.app window.", + "After an iOS app is launched, use `ade --socket ios-sim snapshot --text` or `ade --socket ios-sim elements --text` for screenshot + accessibility/ADEInspector grounding, then `ade --socket ios-sim select --x --y ` to add selected UI context to the drawer chat. Use `tap`, `drag`/`swipe`, and `type` against the active launched app; `live-start` uses auto backend resolution (IOSurface first, then idb/simctl fallbacks), while `window-start`, `preview-start`, `stream-status`, and `stream-stop` manage visual streams explicitly. Use `stream-status --text` to explain which backend/input path is active, any fallback reason, helper pid, and latency; low idle fps is normal on iosurface-indigo because frames are event-driven when the simulator is still. If the simulator is not running or no active session/snapshot is available, warn the user with the exact blocker instead of guessing the screen. When you finish, run `ade --socket ios-sim shutdown` to tear down the session, stop streaming, and release helper processes.", "For ADE Preview Lab work, use `ade --socket ios-sim preview-status --text` to check Xcode MCP readiness, `ade --socket ios-sim previews --source --text` to discover existing `#Preview`/`PreviewProvider` targets, and `ade --socket ios-sim preview-render --source --index --text` as the final open/render step. If no matching preview exists, add one first; if one exists, reuse it.", "When changing SwiftUI from simulator or preview context, keep the affected UI previewable. Add or repair nearby `#Preview` definitions and deterministic preview fixtures when needed, preferably in feature sidecar files such as `Previews.swift` or a DEBUG-only `PreviewSupport` helper. Preview fixtures should use representative mock data derived from visible UI context when possible, but must not depend on live sync, keychain, network, push, sockets, or a production database.", "When preview appearance matters, add named light/dark preview variants with `.preferredColorScheme(.light)` or `.preferredColorScheme(.dark)` and prefer adaptive system colors over hardcoded light-only or dark-only values. Do this as part of making the SwiftUI surface previewable, not as an app-specific special case.", @@ -20,4 +20,4 @@ export const ADE_CLI_AGENT_GUIDANCE = [ ].join("\n"); export const ADE_CLI_INLINE_GUIDANCE = - "`ade` is the default control plane for ADE-managed sessions: lanes, missions, PRs, chats/sessions, memory, proof, config, and process state. If `command -v ade` fails, try `${ADE_CLI_PATH:-}`, then `${ADE_CLI_BIN_DIR:-}/ade`, and in an ADE source checkout `node apps/ade-cli/dist/cli.cjs ...` after confirming it exists. The only normal reason to skip ADE CLI for an ADE action is that the user truly does not have it installed or reachable after those fallbacks. For ADE work beyond the immediate local edit, shell command, or repository inspection in front of you, check ADE CLI first: try `ade doctor`, typed `ade ... --text` commands, `ade help `, `ade help `, or `ade actions list --text` / `ade actions run ...`. iOS Simulator and Preview Lab control is only supported from ADE desktop chats; if you are in a standalone CLI session outside an ADE chat, do not try to drive `ade ios-sim`, and tell the user to open or rerun the request in an ADE chat. For App Control work on Electron apps, prefer desktop socket mode so the CLI and drawer share the same managed app session: run `ade help app-control`, `ade --socket app-control status --text`, launch with `ade --socket app-control launch --command \"npm run dev\" --text`, or connect with `connect --cdp-port --text`. To reuse a Run-tab process, discover its command and cwd via `ade --socket settings get --text`, then launch with `--command` and a matching `--cwd` (relative cwds resolve against the lane root and a cwd outside the lane is rejected). App Control launches run in the visible chat terminal; for terminal/log questions first use `ade --socket app-control logs --text --max-bytes 8388608`, `ade --socket app-control terminal write --data \"y\\n\"`, or `ade --socket app-control terminal signal --signal SIGINT`, and only fall back to `ade --socket terminal read --chat-session \"$ADE_CHAT_SESSION_ID\" --text` or `terminal list` when no active App Control terminal exists. ADE sets ADE_APP_CONTROL_CDP_PORT and ADE_APP_CONTROL_DEBUG_FLAGS and auto-forwards debug flags for common npm/pnpm/yarn/bun script launches and direct electron commands; custom launchers should forward one of those values to Electron's --remote-debugging-port. After launch, use `snapshot --text` / `elements --text`, Control mode or `click`/`type` to drive the app, and Inspect mode or `select --x --y ` to attach screenshot-backed DOM/selector/source context to chat. For iOS Simulator work inside an ADE chat, use desktop socket mode when drawer state matters and discover precise flags with `ade help ios-sim` plus focused pages such as `ade help ios-sim launch`, `snapshot`, `previews`, `preview-render`, `select`, `tap`, `drag`, `type`, and `shutdown`; raw actions are listed by `ade --socket ios-sim actions --text`. Start with `ade --socket ios-sim status --text`, `devices --text`, `apps --device --text`, launch with `ade --socket ios-sim launch --target --text`, then use `snapshot --text` / `elements --text`, `select --x --y `, `tap`, `drag`/`swipe`, and `type` against the active launched app; stream commands are `window-start`, `live-start`, `preview-start`, `stream-status`, and `stream-stop`; tear down with `ade --socket ios-sim shutdown`. If the simulator is not running or no active session/snapshot is available, warn the user with the exact blocker instead of guessing the screen. For ADE Preview Lab, check `preview-status --text`, discover with `previews --source --text`, and finish with `preview-render --source --index --text`; add a preview only when no matching preview already exists. When changing SwiftUI from simulator or preview context, keep the affected UI previewable: add or repair nearby `#Preview` definitions and deterministic mock fixtures in feature sidecar files or DEBUG-only preview support, using visible UI context as representative data when useful and avoiding live sync/keychain/network/database dependencies. When preview appearance matters, add named light/dark preview variants with `.preferredColorScheme(.light)` or `.preferredColorScheme(.dark)` and prefer adaptive system colors over hardcoded light-only or dark-only values. When asked to make a SwiftUI preview reachable in the live simulator, add a DEBUG-only route, deep link, launch-argument handler, or small preview host that presents the same view with the same deterministic fixtures. ADE may pass `ADE_PREVIEW_*` environment values when launching from a selected preview target; use them when helpful, but keep the implementation app-local and optional. When an iOS visual inspect packet is attached, ensure the affected source file or a nearby related Swift file contains a renderable `#Preview`/`PreviewProvider`, and add mock data or a preview harness if the screen would otherwise be blank. When the user asks you to capture, send, attach, or provide proof, use whatever computer-use or browser tool is appropriate to produce the evidence, then register it with ADE via `ade proof ...` so it appears in the ADE proof drawer for the active chat, mission, or lane. When you run processes of any kind, track what you started and clean up old, stale, or finished processes before leaving the task."; + "`ade` is the default control plane for ADE-managed sessions: lanes, missions, PRs, chats/sessions, memory, proof, config, and process state. If `command -v ade` fails, try `${ADE_CLI_PATH:-}`, then `${ADE_CLI_BIN_DIR:-}/ade`, and in an ADE source checkout `node apps/ade-cli/dist/cli.cjs ...` after confirming it exists. The only normal reason to skip ADE CLI for an ADE action is that the user truly does not have it installed or reachable after those fallbacks. For ADE work beyond the immediate local edit, shell command, or repository inspection in front of you, check ADE CLI first: try `ade doctor`, typed `ade ... --text` commands, `ade help `, `ade help `, or `ade actions list --text` / `ade actions run ...`. iOS Simulator and Preview Lab control is only supported from ADE desktop chats; if you are in a standalone CLI session outside an ADE chat, do not try to drive `ade ios-sim`, and tell the user to open or rerun the request in an ADE chat. For App Control work on Electron apps, prefer desktop socket mode so the CLI and drawer share the same managed app session: run `ade help app-control`, `ade --socket app-control status --text`, launch with `ade --socket app-control launch --command \"npm run dev\" --text`, or connect with `connect --cdp-port --text`. To reuse a Run-tab process, discover its command and cwd via `ade --socket settings get --text`, then launch with `--command` and a matching `--cwd` (relative cwds resolve against the lane root and a cwd outside the lane is rejected). App Control launches run in the visible chat terminal; for terminal/log questions first use `ade --socket app-control logs --text --max-bytes 8388608`, `ade --socket app-control terminal write --data \"y\\n\"`, or `ade --socket app-control terminal signal --signal SIGINT`, and only fall back to `ade --socket terminal read --chat-session \"$ADE_CHAT_SESSION_ID\" --text` or `terminal list` when no active App Control terminal exists. ADE sets ADE_APP_CONTROL_CDP_PORT and ADE_APP_CONTROL_DEBUG_FLAGS and auto-forwards debug flags for common npm/pnpm/yarn/bun script launches and direct electron commands; custom launchers should forward one of those values to Electron's --remote-debugging-port. After launch, use `snapshot --text` / `elements --text`, Control mode or `click`/`type` to drive the app, and Inspect mode or `select --x --y ` to attach screenshot-backed DOM/selector/source context to chat. For iOS Simulator work inside an ADE chat, use desktop socket mode when drawer state matters and discover precise flags with `ade help ios-sim` plus focused pages such as `ade help ios-sim launch`, `snapshot`, `previews`, `preview-render`, `select`, `tap`, `drag`, `type`, and `shutdown`; raw actions are listed by `ade --socket ios-sim actions --text`. Start with `ade --socket ios-sim status --text`, `devices --text`, `apps --device --text`, launch with `ade --socket ios-sim launch --target --text`, then use `snapshot --text` / `elements --text`, `select --x --y `, `tap`, `drag`/`swipe`, and `type` against the active launched app; stream commands are `window-start`, `live-start`, `preview-start`, `stream-status`, and `stream-stop`. Use `stream-status --text` to explain which backend/input path is active, any fallback reason, helper pid, and latency; low idle fps is normal on iosurface-indigo because frames are event-driven when the simulator is still. Tear down with `ade --socket ios-sim shutdown`. If the simulator is not running or no active session/snapshot is available, warn the user with the exact blocker instead of guessing the screen. For ADE Preview Lab, check `preview-status --text`, discover with `previews --source --text`, and finish with `preview-render --source --index --text`; add a preview only when no matching preview already exists. When changing SwiftUI from simulator or preview context, keep the affected UI previewable: add or repair nearby `#Preview` definitions and deterministic mock fixtures in feature sidecar files or DEBUG-only preview support, using visible UI context as representative data when useful and avoiding live sync/keychain/network/database dependencies. When preview appearance matters, add named light/dark preview variants with `.preferredColorScheme(.light)` or `.preferredColorScheme(.dark)` and prefer adaptive system colors over hardcoded light-only or dark-only values. When asked to make a SwiftUI preview reachable in the live simulator, add a DEBUG-only route, deep link, launch-argument handler, or small preview host that presents the same view with the same deterministic fixtures. ADE may pass `ADE_PREVIEW_*` environment values when launching from a selected preview target; use them when helpful, but keep the implementation app-local and optional. When an iOS visual inspect packet is attached, ensure the affected source file or a nearby related Swift file contains a renderable `#Preview`/`PreviewProvider`, and add mock data or a preview harness if the screen would otherwise be blank. When the user asks you to capture, send, attach, or provide proof, use whatever computer-use or browser tool is appropriate to produce the evidence, then register it with ADE via `ade proof ...` so it appears in the ADE proof drawer for the active chat, mission, or lane. When you run processes of any kind, track what you started and clean up old, stale, or finished processes before leaving the task."; diff --git a/apps/desktop/src/shared/ipc.ts b/apps/desktop/src/shared/ipc.ts index 5d689377b..6b6b84b25 100644 --- a/apps/desktop/src/shared/ipc.ts +++ b/apps/desktop/src/shared/ipc.ts @@ -271,6 +271,7 @@ export const IPC = { gitMergeContinue: "ade.git.mergeContinue", gitMergeAbort: "ade.git.mergeAbort", gitListBranches: "ade.git.listBranches", + gitGetUserIdentity: "ade.git.getUserIdentity", gitCheckoutBranch: "ade.git.checkoutBranch", conflictsGetLaneStatus: "ade.conflicts.getLaneStatus", conflictsListOverlaps: "ade.conflicts.listOverlaps", @@ -402,6 +403,7 @@ export const IPC = { prsLinkToLane: "ade.prs.linkToLane", prsGetForLane: "ade.prs.getForLane", prsListAll: "ade.prs.listAll", + prsListOpenForRepo: "ade.prs.listOpenForRepo", prsRefresh: "ade.prs.refresh", prsGetStatus: "ade.prs.getStatus", prsGetChecks: "ade.prs.getChecks", diff --git a/apps/desktop/src/shared/types/git.ts b/apps/desktop/src/shared/types/git.ts index f307c66f5..2c68a3368 100644 --- a/apps/desktop/src/shared/types/git.ts +++ b/apps/desktop/src/shared/types/git.ts @@ -188,12 +188,44 @@ export type GitBranchSummary = { ownedByLaneName?: string | null; profiledInCurrentLane?: boolean; hasOpenPr?: boolean; + /** SHA of the branch tip (most recent commit). */ + lastCommitSha?: string; + /** Subject line of the most recent commit. */ + lastCommitMessage?: string; + /** ISO-8601 timestamp of the most recent commit. */ + lastCommitDate?: string; + /** Author name of the most recent commit. */ + lastCommitAuthor?: string; }; export type GitListBranchesArgs = { laneId: string; }; +export type GitUserIdentity = { + name: string; + email: string; +}; + +export type GitGetUserIdentityArgs = { + laneId: string; +}; + +/** + * Lightweight PR info keyed by branch — used by the branch picker. Independent + * of the full PrSummary because we want to surface PRs whose head branch may + * not be tied to any local lane yet. + */ +export type BranchPullRequest = { + branch: string; + prNumber: number; + title: string; + state: "open" | "closed" | "merged" | "draft"; + url: string; + author: string | null; + updatedAt: string | null; +}; + export type GitCheckoutBranchArgs = { laneId: string; branchName: string; diff --git a/apps/desktop/src/shared/types/iosSimulator.ts b/apps/desktop/src/shared/types/iosSimulator.ts index a91bc9609..d24efd4d4 100644 --- a/apps/desktop/src/shared/types/iosSimulator.ts +++ b/apps/desktop/src/shared/types/iosSimulator.ts @@ -7,7 +7,7 @@ export type IosSimulatorDevice = { }; export type IosSimulatorToolStatus = { - name: "xcrun" | "xcodebuild" | "simulator_window" | "idb" | "idb_companion" | "ffmpeg"; + name: "xcrun" | "xcodebuild" | "iosurface_indigo" | "simulator_window" | "idb" | "idb_companion" | "ffmpeg"; available: boolean; detail: string; installHint: string; @@ -84,6 +84,7 @@ export type IosSimulatorSession = { projectRoot: string | null; chatSessionId: string | null; mode: IosSimulatorLaunchMode; + keepSimulatorInBackground?: boolean | null; bridgeUrl: string | null; startedAt: string; }; @@ -100,17 +101,30 @@ export type IosSimulatorStreamStatus = { deviceUdid: string | null; running: boolean; backend: IosSimulatorStreamBackend | null; + requestedBackend?: "auto" | IosSimulatorStreamBackend | null; + fallbackReason?: string | null; + degradationReason?: string | null; fps: number | null; targetFps: number | null; frameCount: number; startedAt: string | null; lastFrameAt: string | null; lastError: string | null; + error?: { + code: string; + exitCode?: number | null; + signal?: string | null; + } | null; streamUrl: string | null; averageLatencyMs?: number | null; + latencyP50Ms?: number | null; + latencyP95Ms?: number | null; + helperPid?: number | null; + inputBackend?: "indigo" | "idb" | null; }; export type IosSimulatorStreamBackend = + | "iosurface-indigo" | "simctl-screenshot-poll" | "idb-mjpeg" | "idb-h264-ffmpeg-mjpeg" diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4bb97d38c..1d113df6a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -440,7 +440,7 @@ Every service lives under `apps/desktop/src/main/services//`. Summary: | `git/` | `git.ts`, `gitOperationsService.ts`, `gitConflictState.ts` | Low-level git runner, high-level lane-scoped ops, conflict state queries. | | `github/` | `githubService.ts` | GitHub REST/GraphQL access; PR CRUD; checks; reviewers. | | `history/` | `operationService.ts` | Operation audit records (one row per mutation). | -| `ios/` | `iosSimulatorService.ts` | macOS-only iOS Simulator backend: tool readiness probes, simctl device + app discovery, build/install/launch with progress events (hardened with `simctl bootstatus` and `simctl install` timeouts), screenshot + ADEInspector + accessibility hit-test, four streaming backends with `auto` resolution and runtime fallback (`idb-h264-ffmpeg-mjpeg` → `idb-mjpeg` → `simctl-screenshot-poll`, plus opt-in `simulator-window-capture` for native window-capture diagnostics), tap/drag/swipe/type via idb, and single-owner chat session locking. The macOS Simulator window placement / capture state probe (`getSimulatorWindowState`, `prepareSimulatorWindowForCapture`) lives next to the IPC handlers in `ipc/registerIpc.ts` because it depends on the active `BrowserWindow`. See [features/ios-simulator/README.md](./features/ios-simulator/README.md). | +| `ios/` | `iosSimulatorService.ts` | macOS-only iOS Simulator backend: tool readiness probes, simctl device + app discovery, build/install/launch with progress events (hardened with `simctl bootstatus` and `simctl install` timeouts), screenshot + ADEInspector + accessibility hit-test, IOSurface/Indigo primary streaming and input with idb/simctl/window-capture fallbacks, recovery-only H.264+ffmpeg after idb MJPEG failure, and single-owner chat session locking. The macOS Simulator window placement / capture state probe (`getSimulatorWindowState`, `prepareSimulatorWindowForCapture`) lives next to the IPC handlers in `ipc/registerIpc.ts` because it depends on the active `BrowserWindow`. See [features/ios-simulator/README.md](./features/ios-simulator/README.md). | | `ipc/` | `registerIpc.ts` | Single registration point for all IPC handlers. | | `jobs/` | `jobEngine.ts` | Event-driven background scheduler for lane refresh + conflict prediction. Coalesced, debounced. | | `keybindings/` | `keybindingsService.ts` | User keybindings read/write. | diff --git a/docs/features/ios-simulator/README.md b/docs/features/ios-simulator/README.md index e737c4912..195a68f33 100644 --- a/docs/features/ios-simulator/README.md +++ b/docs/features/ios-simulator/README.md @@ -20,20 +20,20 @@ called from a non-darwin host. | Path | Role | |---|---| -| `apps/desktop/src/main/services/ios/iosSimulatorService.ts` | The whole feature backend: tool-readiness probes (xcrun, xcodebuild, idb, idb_companion, ffmpeg), simctl device + app discovery, build/install/launch with progress events (with hardened `simctl bootstatus` and `simctl install` timeouts), screenshot + ADEInspector + accessibility hit-test, streaming backends (`idb-h264-ffmpeg-mjpeg`, `idb-mjpeg`, `simctl-screenshot-poll`, `simulator-window-capture`) with `auto` resolution and runtime fallback when a backend produces no frames, tap/drag/swipe/type via idb, single-owner session locking, Preview Lab integration via Xcode MCP, and selection emission. Exports `__testSetIosSimulatorProcessHooks`, `resolveIosSimulatorStreamBackend`, and `shouldOpenSimulatorAppForLaunch` for the unit tests. | +| `apps/desktop/src/main/services/ios/iosSimulatorService.ts` | The whole feature backend: tool-readiness probes (xcrun, xcodebuild, IOSurface/Indigo helpers, idb, idb_companion, ffmpeg), simctl device + app discovery, build/install/launch with progress events (with hardened `simctl bootstatus` and `simctl install` timeouts), screenshot + ADEInspector + accessibility hit-test, streaming backends (`iosurface-indigo`, `idb-mjpeg`, `idb-h264-ffmpeg-mjpeg`, `simctl-screenshot-poll`, `simulator-window-capture`) with `auto` resolution and runtime fallback when a backend produces no frames, tap/drag/swipe/type through Indigo with idb fallback, single-owner session locking, Preview Lab integration via Xcode MCP, and selection emission. Exports `__testSetIosSimulatorProcessHooks`, `detectIosurfaceIndigoCapability`, `resolveIosSimulatorStreamBackend`, and `shouldOpenSimulatorAppForLaunch` for the unit tests. | | `apps/desktop/src/main/services/ios/iosSimulatorService.test.ts` | Service unit tests covering backend resolution, launch foreground/background flag, and timeout-mapped error messages via the `__testSetIosSimulatorProcessHooks` injector. | -| `apps/desktop/src/shared/types/iosSimulator.ts` | All cross-process types: `IosSimulatorStatus`, `IosSimulatorDevice`, `IosSimulatorLaunchTarget`, `IosSimulatorSession`, `IosSimulatorLaunchProgress`, `IosSimulatorStreamStatus`, `IosSimulatorStreamBackend` (now a four-member union — `simctl-screenshot-poll` \| `idb-mjpeg` \| `idb-h264-ffmpeg-mjpeg` \| `simulator-window-capture`), `IosSimulatorWindowSource`, `IosSimulatorWindowState` + `IosSimulatorWindowIssue` (`not-running` \| `hidden` \| `minimized` \| `no-window` \| `unknown`), `IosScreenSnapshot`, `IosScreenElement`, `IosInspectorSnapshot`, `IosInspectableElement`, `IosElementContextItem`, `IosSimulatorEventPayload`, plus the `IOS_SIMULATOR_OWNED_BY_OTHER_SESSION_CODE` error sentinel. | +| `apps/desktop/src/shared/types/iosSimulator.ts` | All cross-process types: `IosSimulatorStatus`, `IosSimulatorDevice`, `IosSimulatorLaunchTarget`, `IosSimulatorSession`, `IosSimulatorLaunchProgress`, `IosSimulatorStreamStatus`, `IosSimulatorStreamBackend` (`iosurface-indigo` \| `simctl-screenshot-poll` \| `idb-mjpeg` \| `idb-h264-ffmpeg-mjpeg` \| `simulator-window-capture`), `IosSimulatorWindowSource`, `IosSimulatorWindowState` + `IosSimulatorWindowIssue` (`not-running` \| `hidden` \| `minimized` \| `no-window` \| `unknown`), `IosScreenSnapshot`, `IosScreenElement`, `IosInspectorSnapshot`, `IosInspectableElement`, `IosElementContextItem`, `IosSimulatorEventPayload`, plus the `IOS_SIMULATOR_OWNED_BY_OTHER_SESSION_CODE` error sentinel. | | `apps/desktop/src/shared/ipc.ts` | `IPC.iosSimulator*` channel constants (one per service method, plus `iosSimulatorGetWindowState`, `iosSimulatorListWindowSources`, and the single push channel `ade.iosSimulator.event`). | | `apps/desktop/src/main/services/ipc/registerIpc.ts` | `ade.iosSimulator.*` invoke handlers, the chat-session-aware arg validator (`incomingChatSessionId` must match the active drawer owner), the `ade.iosSimulator.event` push relay, and the macOS Simulator-window plumbing: `getSimulatorWindowState` (osascript probe of `process "Simulator"` for visibility / window count / minimized count), `prepareSimulatorWindowForCapture` (open `-g`, unminimize, park under the left side of the ADE BrowserWindow), and `followSimulatorWindowUnderAde` (re-park on `move`/`resize`, cleared on shutdown). | | `apps/desktop/src/main/services/adeActions/registry.ts` | Maps the service onto the `ios_simulator` action namespace consumed by the ADE CLI / agent tools (`getStatus`, `listDevices`, `listLaunchTargets`, `launch`, `shutdown`, `screenshot`, `getScreenSnapshot`, `getInspectorSnapshot`, `inspectPoint`, `getPreviewCapability`, `listPreviewTargets`, `renderPreview`, `openPreviewWorkspace`, `startStream`, `stopStream`, `getStreamStatus`, `tap`, `typeText`, `drag`, `swipe`, `selectPoint`). | | `apps/desktop/src/preload/preload.ts` | `window.ade.iosSimulator` bridge and `onEvent(listener)` push subscription. | -| `apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.tsx` | Drawer UI: tool-readiness checklist, device + target pickers, launch progress, live preview playback (idb live MJPEG decoded to a `` via `fetch` + `ReadableStream` JPEG framing as the default, with screenshot-poll fallback and explicit `simulator-window-capture` for diagnostics), `interact` vs `inspect` mode, hit-test overlay drawn from `getScreenSnapshot`, drag-to-select region capture on a frozen simulator screenshot (`SimulatorCaptureSelection`) that emits an `IosElementContextItem` for the cropped region, Preview Lab tab (`renderPreview` + workspace open), `getSimulatorWindowState`-driven warnings when window capture is selected and Simulator.app is hidden/minimized, and attachment + context emission. | +| `apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.tsx` | Drawer UI: tool-readiness checklist, device + target pickers, launch progress, live preview playback (the service's MJPEG URL decoded to a `` via `fetch` + `ReadableStream` JPEG framing; IOSurface is preferred, idb/simctl are fallbacks, and explicit `simulator-window-capture` remains available for diagnostics), `interact` vs `inspect` mode, hit-test overlay drawn from `getScreenSnapshot`, drag-to-select region capture on a frozen simulator screenshot (`SimulatorCaptureSelection`) that emits an `IosElementContextItem` for the cropped region, Preview Lab tab (`renderPreview` + workspace open), `getSimulatorWindowState`-driven warnings when window capture is selected and Simulator.app is hidden/minimized, and attachment + context emission. | | `apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.test.tsx` | Renderer panel tests. | | `apps/desktop/src/renderer/components/chat/AgentChatPane.tsx` | Mounts `ChatIosSimulatorPanel` behind a header toggle (`iosSimulatorAvailable`), brokers the screenshot-attachment + context-item flow into the composer, and gates the toggle on `iosSimulatorStatus.supported`. | | `apps/desktop/src/renderer/components/chat/AgentChatComposer.tsx` | Renders `IosElementContextItem[]` as inline composer chips: switches to a contenteditable rich-input variant when the user has attached one or more iOS elements, serialises chip nodes back into the prompt on submit, and pairs each element with its captured screenshot when one was added in the same gesture. | | `apps/desktop/src/renderer/components/chat/ChatAttachmentTray.tsx` | Includes iOS-element instance handling: `createIosContextInstanceId`, `getIosContextAttachmentPath`, and `formatIosElementContextForPrompt` (the prompt-side serialisation). | | `apps/desktop/src/shared/adeCliGuidance.ts` | Mentions `ade ios-sim` so prompts know the surface exists. | -| `apps/ade-cli/src/cli.ts` | `ade ios-sim` (aliased `ade ios`, `ade simulator`) subcommand: status / devices / apps / launch / shutdown / actions / screenshot / snapshot / inspector / inspect / preview-status / previews / preview-render / preview-open / window-start / live-start / preview-start / stream-status / stream-stop / select / tap / drag / swipe / type, with focused `ade help ios-sim ` pages for agent discovery. `live-start` now requests the `auto` backend so the service picks `idb-h264-ffmpeg-mjpeg` → `idb-mjpeg` → `simctl-screenshot-poll` based on which tools are installed; `--backend` accepts the new `idb-mjpeg` value for the direct MJPEG path. | +| `apps/ade-cli/src/cli.ts` | `ade ios-sim` (aliased `ade ios`, `ade simulator`) subcommand: status / devices / apps / launch / shutdown / actions / screenshot / snapshot / inspector / inspect / preview-status / previews / preview-render / preview-open / window-start / live-start / preview-start / stream-status / stream-stop / select / tap / drag / swipe / type, with focused `ade help ios-sim ` pages for agent discovery. `live-start` requests the `auto` backend so the service picks `iosurface-indigo` first when available, then falls back through window/idb/simctl paths; `--backend` accepts every explicit backend value. | | `apps/ios/ADE/Debug/ADEInspectorKit/ADEInspectable.swift` | The Swift side: the `.adeInspectable("componentId", ...)` view modifier and the `.adeInspectorHost()` host modifier that publish per-frame element snapshots (component id, source file/line, accessibility identifier, point + pixel frames) into `Documents/ade-inspector-elements.json` inside the running app's data container. DEBUG-only — release builds compile to a no-op. | ## Detail docs @@ -80,13 +80,14 @@ called from a non-darwin host. -configuration Debug build`; install uses `xcrun simctl install` with a 180 s timeout (same stuck-CoreSimulator error mapping); launch uses `xcrun simctl launch --terminate-running-process - `. The default path keeps Simulator.app in the - background (`open -g -a Simulator`) and reopens it the same way - after `simctl launch` so it never steals focus. Pass + `. The default path does not open Simulator.app; + the live drawer stream is headless when IOSurface/idb/simctl backends + are active. Pass `keepSimulatorInBackground: false` (or `--foreground` from the CLI) to bring Simulator.app forward explicitly. Smooth drawer streaming no longer depends on a visible Simulator.app window — the default - live stream is idb-driven (see Streaming below). The + live stream is IOSurface-driven when the private helpers are + available (see Streaming below), then idb/simctl fallbacks. The `simulator-window-capture` backend still requires a real, unminimized Simulator window: hidden/minimized windows stop producing captured frames even though idb/simctl input still @@ -103,18 +104,25 @@ called from a non-darwin host. 5. **Streaming.** Stream backends share one `IosSimulatorStreamStatus`. The `auto` resolver in `resolveIosSimulatorStreamBackend(requested, - tools)` maps a requested `auto` to the first backend whose tools are - installed: `idb-h264-ffmpeg-mjpeg` (idb + idb_companion + ffmpeg) → - `idb-mjpeg` (idb + idb_companion) → `simctl-screenshot-poll`. Both - idb backends arm a startup timer (5 s for `idb-mjpeg`, 15 s for - `idb-h264-ffmpeg-mjpeg`); if no JPEG frame has been observed by then - they emit `stream-error`, stop the stream, and start the next - fallback automatically (idb-mjpeg → idb-h264 → screenshot poll; - idb-h264 → screenshot poll), so `live-start` succeeds on hosts where - one idb path silently produces no frames. Default fps is 30 for the - idb backends, 8 for screenshot poll. - - `idb-h264-ffmpeg-mjpeg` (`live-start --idb`/`auto` preferred path, - drawer default when ffmpeg is installed) — exact-screen stream + tools)` maps `auto` to: `iosurface-indigo` → visible + `simulator-window-capture` → `idb-mjpeg` → `simctl-screenshot-poll`. + `idb-h264-ffmpeg-mjpeg` is recovery-only after `idb-mjpeg` fails; it + is no longer a first pick just because `ffmpeg` is installed. + Fallback and degradation reasons are carried in stream status so the + drawer and CLI can explain why a lower backend is active. Default fps + is 60 for window capture, 30 for IOSurface/idb, 8 for screenshot poll. + - `iosurface-indigo` (`live-start`/`auto` preferred path) — ADE + lazily compiles `apps/desktop/native/ios-sim-helpers/sim-capture.swift` + and `sim-input.m` for the selected full Xcode, caches them under + `native/ios-sim-helpers/build/-/`, reads + length-prefixed JPEG IOSurface frames from the helper, and bridges + them into the same localhost multipart MJPEG endpoint used by every + caller. Requires macOS, full Xcode 17.x or 26.x, `swiftc`, and + `clang`; CLT-only machines fall back cleanly. ADE currently gates this + path off in packaged builds until helper signing/notarization is cleared, + and new Xcode majors should be added only after helper compile/smoke and + real simulator validation. + - `idb-h264-ffmpeg-mjpeg` (recovery-only) — exact-screen stream through `idb video-stream --format h264` transcoded to MJPEG via `ffmpeg`. The renderer reads the MJPEG endpoint with `fetch` + `ReadableStream`, frames JPEGs out of the byte stream, and draws @@ -161,9 +169,14 @@ called from a non-darwin host. `source: "coordinate-fallback"`. 7. **Input.** `tap`, `drag`, `swipe` (alias of `drag`), and `typeText` - all route through idb against the active companion; `idb_companion` - is launched lazily and torn down 30 s after last use - (`COMPANION_IDLE_STOP_MS`). + route through an input backend abstraction. Indigo touch input is + preferred whenever IOSurface capability is available, including + explicit non-IOSurface capture modes, and idb remains the fallback. + If Indigo input fails twice in 60 s, ADE sticks to idb for the rest + of that session and emits a status event with the reason. Text/key + input currently falls back to idb because the helper only implements + touch/button Indigo events. `idb_companion` is launched lazily and + torn down 30 s after last use (`COMPANION_IDLE_STOP_MS`). 8. **Shutdown.** `shutdown({ force? })` stops the stream, kills the transcoder, releases the idb companion, clears `activeSession`, @@ -194,10 +207,10 @@ live in `registerIpc.ts`. The renderer talks to these through | `ade.iosSimulator.openPreviewWorkspace` | `openPreviewWorkspace()` | Open the lane's iOS project in Xcode. | | `ade.iosSimulator.startStream` | `startStream({ backend?, fps? })` | Start one of the streaming backends. | | `ade.iosSimulator.stopStream` | `stopStream()` | Stop streaming. | -| `ade.iosSimulator.getStreamStatus` | `getStreamStatus()` | Backend, fps, latency, URL. | +| `ade.iosSimulator.getStreamStatus` | `getStreamStatus()` | Requested/resolved backend, fallback/degradation reason, fps, latency p50/p95, helper pid, URL. | | `ade.iosSimulator.getWindowState` | `getSimulatorWindowState()` | Returns `IosSimulatorWindowState` (`appRunning`, `visible`, `windowCount`, `minimizedWindowCount`, `capturable`, `issue`, `message`) by running an osascript probe of `process "Simulator"`. Used by the panel to warn when window-capture mode cannot produce frames. | | `ade.iosSimulator.listWindowSources` | `listSimulatorWindowSources()` | Renderer-side helper for picking the Simulator.app window. Calls `prepareSimulatorWindowForCapture` first when there is an active session so the window is unminimized and parked before `desktopCapturer.getSources` enumerates. | -| `ade.iosSimulator.tap` / `typeText` / `drag` / `swipe` | input verbs | Routed through idb. | +| `ade.iosSimulator.tap` / `typeText` / `drag` / `swipe` | input verbs | Routed through Indigo when available, with idb fallback. | | `ade.iosSimulator.selectPoint` | `selectPoint({ x, y })` | Hit-test + emit a `selection` event so the chat composer can attach the resulting `IosElementContextItem`. | | `ade.iosSimulator.event` | (push) | `IosSimulatorEventPayload` union: `session-started`, `session-updated`, `session-released`, `selection`, `launch-progress`, `stream-started`, `stream-status`, `stream-stopped`, `stream-frame`, `stream-error`. | @@ -242,7 +255,7 @@ ade ios-sim preview-open Streaming: ``` -ade ios-sim live-start --fps 30 # auto: idb-h264-ffmpeg-mjpeg → idb-mjpeg → simctl-screenshot-poll +ade ios-sim live-start --fps 30 # auto: iosurface-indigo → window/idb/simctl fallbacks ade ios-sim live-start --backend idb-mjpeg --fps 30 ade ios-sim preview-start --fps 8 # simctl-screenshot-poll ade ios-sim window-start --fps 60 # simulator-window-capture (diagnostic only) @@ -329,11 +342,12 @@ chip attached. no-op, intentionally — re-launching from the same chat must not bounce the simulator. - **Do not minimize or hide Simulator.app for window capture.** The - smooth drawer path uses macOS window capture, so Simulator.app must - stay visible and unminimized even when it is parked underneath ADE. - Hiding or minimizing the native window freezes captured frames, though - idb/simctl input may still reach the device. Surface that state as a - warning instead of pretending the stream is healthy. + primary drawer path is headless IOSurface capture, but explicit + `simulator-window-capture` still depends on a visible, unminimized + Simulator.app window. Hiding or minimizing the native window freezes + captured frames, though Indigo/idb/simctl input may still reach the + device. Surface that state as a warning instead of pretending the + stream is healthy. - **Stream backend stickiness.** Switching backends mid-session must call `stopStream()` first; the renderer relies on `stream-stopped → stream-started` ordering to clear the previous @@ -345,11 +359,11 @@ chip attached. snapshot — `getScreenSnapshot` reports `providers[].error: "No ADEInspector snapshot has been published by the active app."` and falls back to accessibility-only data. -- **idb_companion is reference-counted.** `ensureCompanion` increments - on every input/accessibility call and the timer only fires after - `COMPANION_IDLE_STOP_MS` of zero refcount. Bypassing - `ensureCompanion` for one verb leaves the timer dangling and the - companion gets killed mid-call. +- **Helper ownership matters.** IOSurface capture and Indigo input are + owned by the active simulator session and are killed on release. + `idb_companion` remains lazy fallback infrastructure for accessibility, + text, and degraded input, and is stopped after `COMPANION_IDLE_STOP_MS` + when idle. - **Screenshot pairing window.** `AgentChatPane` tracks the most recent attachment via `latestAttachmentRef` and only stamps an `attachmentPath` onto the iOS element if the attachment was added diff --git a/docs/features/lanes/README.md b/docs/features/lanes/README.md index f9b093167..0e908bc93 100644 --- a/docs/features/lanes/README.md +++ b/docs/features/lanes/README.md @@ -44,7 +44,9 @@ Renderer components: | `renderer/components/lanes/LaneWorkPane.tsx` | Terminal/chat toggle work surface | | `renderer/components/lanes/LaneRebaseBanner.tsx` | Inline banner driven by `rebaseSuggestionService` | | `renderer/components/lanes/LaneEnvInitProgress.tsx` | Env init step progress inside create dialog | -| `renderer/components/lanes/CreateLaneDialog.tsx`, `AttachLaneDialog.tsx`, `MultiAttachWorktreeDialog.tsx`, `LaneDialogShell.tsx` | Lane creation / attach dialogs and shared dialog chrome | +| `renderer/components/lanes/CreateLaneDialog.tsx`, `AttachLaneDialog.tsx`, `MultiAttachWorktreeDialog.tsx`, `LaneDialogShell.tsx` | Lane creation / attach dialogs and shared dialog chrome. The "import existing branch" path inside `CreateLaneDialog` swaps the dialog body for `BranchPickerView` when the user opens the picker; the dialog title/description switches with it. | +| `renderer/components/lanes/BranchPickerView.tsx` | Filterable virtualized branch list rendered inside `CreateLaneDialog`. Each row shows branch name, last-commit author + relative date, and an inline PR pill (`#NNN`, dim for drafts) when the branch has an open PR. Loading/empty/error states are handled inline. Backed by `branchPickerSearch.ts`. | +| `renderer/components/lanes/branchPickerSearch.ts` | Pure parser + matcher. Tokens AND together: `pr:open` / `pr:none` / `pr:draft`, `author:NAME` (or `author:me` / `mine` resolved against the local git user), `stale:Nd` (older than N days), `#PRNUMBER` (exact match), and free text fuzzy-matched across branch name / PR title / author. Also exposes `formatRelativeTime` for the row subtitle. | | `renderer/components/lanes/ManageLaneDialog.tsx` | Unified delete / archive / adopt-attached dialog. Supports single-lane and batch (multi-select) modes, three delete scopes (`worktree`, `local_branch`, `remote_branch`), a typed confirmation phrase, remote-branch name input, dirty-state warnings, and a live multi-step progress strip wired to `lanes.delete.event` (`stop_processes` / `stop_ptys` / `stop_watchers` / `cancel_auto_rebase` / `cleanup_env` / `git_status` / `git_worktree_remove` / `git_branch_delete` / `git_remote_branch_delete` / `pack_dir_remove` / `database_cleanup`). The dialog calls `lanes.getDeleteRisk` on open to surface dirty state, unpushed commits, running processes / PTYs / watchers, and remote-branch existence before the user confirms; while a delete is running, the user can cancel each lane through `lanes.cancelDelete` until the irreversible filesystem step (`git_worktree_remove`) starts. | | `renderer/components/lanes/MonacoDiffView.tsx` | Monaco-based side-by-side file diff | | `renderer/components/run/LaneRuntimeBar.tsx` | Compact lane runtime status bar (health, preview, port, proxy, oauth) | @@ -197,7 +199,14 @@ default from the Lanes list (see `isMissionLaneHiddenByDefault` in fails. Rejects when the source has staged changes or an in-progress merge/rebase. 4. **Import branch** — `importBranch` attaches an existing branch to a - worktree managed by ADE. + worktree managed by ADE. `CreateLaneDialog` drives this through + `BranchPickerView`: the picker opens against `git.listBranches` + (which now also returns `lastCommitSha` / `Date` / `Author` / + `Message` from a single `for-each-ref` pass), enriches each row with + any open PR coming from `prs.listOpenForRepo`, and resolves + `mine` / `author:me` against the local git identity returned by + `git.getUserIdentity`. PR fetch is fail-soft: when the GitHub call + errors the picker still works, just without PR pills. 5. **Attach** — `attach` links an external worktree path (pre-existing outside ADE). `lane_type = 'attached'`. 6. **Rename / update appearance / reparent** — `rename`, `updateAppearance`, diff --git a/docs/features/pull-requests/README.md b/docs/features/pull-requests/README.md index 098a72d97..4585f38d5 100644 --- a/docs/features/pull-requests/README.md +++ b/docs/features/pull-requests/README.md @@ -18,7 +18,7 @@ Main-process services (`apps/desktop/src/main/services/prs/`): | File | Responsibility | |------|---------------| -| `prService.ts` | PR CRUD, GitHub sync, merge context, draft descriptions, check/review/comment hydration, commit snapshots (`getCommits`), integration proposals, merge-into-existing-lane adoption, merge bypass, post-merge cleanup, standalone PR branch cleanup (`cleanupBranch`), deployment listing, review-thread reply/resolve/react mutations for the timeline, and the aggregate `getMobileSnapshot` that powers the iOS PRs tab | +| `prService.ts` | PR CRUD, GitHub sync, merge context, draft descriptions, check/review/comment hydration, commit snapshots (`getCommits`), integration proposals, merge-into-existing-lane adoption, merge bypass, post-merge cleanup, standalone PR branch cleanup (`cleanupBranch`), deployment listing, review-thread reply/resolve/react mutations for the timeline, the aggregate `getMobileSnapshot` that powers the iOS PRs tab, and `listOpenPullRequests` — a paginated `/repos/{owner}/{name}/pulls?state=open` fetch returning `BranchPullRequest[]` for the lane-creation branch picker. | | `prService.mobileSnapshot.test.ts` | Coverage for the mobile snapshot builder: stack chaining, capability gates, per-lane create eligibility, workflow-card aggregation | | `prService.mergeInto.test.ts` | Coverage for integration proposals that preview or adopt an existing merge target lane, including dirty-worktree handling and drift metadata. | | `prPollingService.ts` | 60 s polling loop, fingerprint-based change detection, notification emission. Writes `last_polled_at` per PR so callers can run delta polls on the next tick | @@ -74,6 +74,7 @@ Shared contracts: | File | Responsibility | |------|---------------| | `apps/desktop/src/shared/types/prs.ts` | PR DTOs and integration proposal contracts, including `preferredIntegrationLaneId`, `mergeIntoHeadSha`, `integrationLaneOrigin`, and `additionalInstructions` fields. | +| `apps/desktop/src/shared/types/git.ts` | `BranchPullRequest` (branch / prNumber / title / state / url / author / updatedAt) — the lightweight PR shape returned by `prService.listOpenPullRequests` and consumed by the branch picker without going through `PrSummary`. | | `apps/desktop/src/shared/types/conflicts.ts` | Conflict resolver DTOs; `PrepareResolverSessionArgs.additionalInstructions` is appended to generated resolver prompts. | | `apps/desktop/src/shared/ipc.ts` / `apps/desktop/src/preload/preload.ts` | PR IPC constants and renderer bridge for proposal simulation, update, commit, resolver, and cleanup flows. | @@ -114,6 +115,7 @@ Selected channels exposed through `preload.ts`: - `ade.prs.createFromLane`, `ade.prs.createQueue`, `ade.prs.createIntegration` - `ade.prs.listAll`, `ade.prs.listProposals`, `ade.prs.listQueueStates` +- `ade.prs.listOpenForRepo` — flat list of open PRs in the project's GitHub repo as `BranchPullRequest[]` (branch / number / title / state / url / author / updatedAt). Independent of `pull_requests` cache so the lane-creation branch picker can attach PR pills to branches that have no lane yet. See [features/lanes/README.md](../lanes/README.md) for the consumer. - `ade.prs.land`, `ade.prs.landStack`, `ade.prs.landStackEnhanced`, `ade.prs.landQueueNext` - `ade.prs.getMergeContext`, `ade.prs.getStatus`, `ade.prs.getChecks`, `ade.prs.getReviews`, `ade.prs.getComments`, `ade.prs.getFiles`, `ade.prs.getCommits` - `ade.prs.cleanupBranch` — delete a merged/closed PR's local and/or remote branch without touching the lane (protected against deleting any primary-lane branch) From 5019589a14f63a3403aca65394a664e01a6c5f8d Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Thu, 30 Apr 2026 19:37:01 -0400 Subject: [PATCH 2/3] =?UTF-8?q?ship:=20iteration=201=20=E2=80=94=20fix=20t?= =?UTF-8?q?est-desktop=20(7)=20shard,=20address=20#221=20review=20comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI: convert idb_companion spawn errors into structured stream-error events and add disposed-checks to ensureCompanion's prewarm path so a fire-and-forget spawn does not bubble as an unhandled error after dispose. Review (CodeRabbit + Greptile): - ade-cli prs list-open table formatter label fix - native helpers: jq/python JSON, serial DispatchQueue for sim-capture state, ensureHID failure now exits non-zero - iosSimulatorService: clear iosurfaceScreenMetrics on shutdown/dispose; preferIndigoInput compares against fallback marker - registerIpc: explicit boolean handling for keepSimulatorInBackground - ChatIosSimulatorPanel: stable MJPEG url through preserveVisual; stall detection extended to iosurface-indigo with per-transport threshold - BranchPickerView: ticking now-state for relative ages - LanesPage: reset identity/loading state in prepareCreateDialog - prService: maxPages cap on listOpenPullRequests - branchPickerSearch: pr:open includes drafts Co-Authored-By: Claude Opus 4.7 (1M context) --- apps/ade-cli/src/cli.ts | 2 +- apps/desktop/native/ios-sim-helpers/build.sh | 12 +++++- .../native/ios-sim-helpers/sim-capture.swift | 37 +++++++++++++------ .../native/ios-sim-helpers/sim-input.m | 5 ++- .../main/services/ios/iosSimulatorService.ts | 32 +++++++++++++++- .../src/main/services/ipc/registerIpc.ts | 3 +- .../src/main/services/prs/prService.ts | 5 ++- .../components/chat/ChatIosSimulatorPanel.tsx | 6 +-- .../components/lanes/BranchPickerView.tsx | 7 +++- .../renderer/components/lanes/LanesPage.tsx | 3 ++ .../lanes/branchPickerSearch.test.ts | 3 +- .../components/lanes/branchPickerSearch.ts | 3 +- 12 files changed, 94 insertions(+), 24 deletions(-) diff --git a/apps/ade-cli/src/cli.ts b/apps/ade-cli/src/cli.ts index 85077507f..35a3906d2 100644 --- a/apps/ade-cli/src/cli.ts +++ b/apps/ade-cli/src/cli.ts @@ -4400,7 +4400,7 @@ function inferFormatter(plan: CliPlan & { kind: "execute" }): FormatterId | unde if (label === "file read") return "file-read"; if (label === "file tree" || label === "file workspaces") return "files-tree"; if (label === "file search" || label === "file quick-open") return "files-search"; - if (label === "pr list") return "prs-list"; + if (label === "pr list" || label === "pr list open") return "prs-list"; if (label === "pr detail" || label === "pr health") return "pr-detail"; if (label === "pr checks") return "pr-checks"; if (label === "pr comments") return "pr-comments"; diff --git a/apps/desktop/native/ios-sim-helpers/build.sh b/apps/desktop/native/ios-sim-helpers/build.sh index 2d0ffc71b..a243b12f2 100644 --- a/apps/desktop/native/ios-sim-helpers/build.sh +++ b/apps/desktop/native/ios-sim-helpers/build.sh @@ -88,8 +88,16 @@ if [[ "$SMOKE" == "1" ]]; then fi if [[ "$PRINT_JSON" == "1" ]]; then - printf '{"xcodeVersion":"%s","sourceHash":"%s","buildDir":"%s","capture":"%s","input":"%s"}\n' \ - "$XCODE_VERSION" "$SOURCE_HASH" "$OUT_DIR" "$CAPTURE" "$INPUT" + XCODE_VERSION="$XCODE_VERSION" SOURCE_HASH="$SOURCE_HASH" OUT_DIR="$OUT_DIR" \ + CAPTURE="$CAPTURE" INPUT="$INPUT" \ + python3 -c 'import json, os, sys +sys.stdout.write(json.dumps({ + "xcodeVersion": os.environ.get("XCODE_VERSION", ""), + "sourceHash": os.environ.get("SOURCE_HASH", ""), + "buildDir": os.environ.get("OUT_DIR", ""), + "capture": os.environ.get("CAPTURE", ""), + "input": os.environ.get("INPUT", ""), +}) + "\n")' else echo "$OUT_DIR" fi diff --git a/apps/desktop/native/ios-sim-helpers/sim-capture.swift b/apps/desktop/native/ios-sim-helpers/sim-capture.swift index e732129b6..36401f47c 100644 --- a/apps/desktop/native/ios-sim-helpers/sim-capture.swift +++ b/apps/desktop/native/ios-sim-helpers/sim-capture.swift @@ -324,10 +324,17 @@ guard let requestedUdid = options.udid, !requestedUdid.isEmpty else { var currentStream: Stream? var currentDeviceUdid = "" +let stateQueue = DispatchQueue(label: "ade.sim-capture.state") + +func stopCurrentStream() { + stateQueue.sync { + currentStream?.stop() + } +} let sigTerm = DispatchSource.makeSignalSource(signal: SIGTERM, queue: .main) sigTerm.setEventHandler { - currentStream?.stop() + stopCurrentStream() exit(0) } sigTerm.resume() @@ -335,7 +342,7 @@ signal(SIGTERM, SIG_IGN) let sigInt = DispatchSource.makeSignalSource(signal: SIGINT, queue: .main) sigInt.setEventHandler { - currentStream?.stop() + stopCurrentStream() exit(0) } sigInt.resume() @@ -346,25 +353,33 @@ DispatchQueue.global(qos: .userInitiated).async { while true { if let device = bootedDevice(deviceSet, udid: requestedUdid) { let udid = deviceUdid(device) - if udid != currentDeviceUdid { - currentStream?.stop() - currentStream = nil + let needsSwap = stateQueue.sync { udid != currentDeviceUdid } + if needsSwap { + stateQueue.sync { + currentStream?.stop() + currentStream = nil + } if let descriptor = findDisplayDescriptor(device) { let stream = Stream(descriptor: descriptor, device: device, targetUdid: udid, maxFps: options.fps, jpegQuality: options.quality) stream.start() - currentStream = stream - currentDeviceUdid = udid + stateQueue.sync { + currentStream = stream + currentDeviceUdid = udid + } notifiedNoBoot = false } else { eprint("[sim-capture] no display descriptor on booted device (will retry)") } } } else { - if currentStream != nil { + let hadStream = stateQueue.sync { currentStream != nil } + if hadStream { eprint("[sim-capture] booted device gone") - currentStream?.stop() - currentStream = nil - currentDeviceUdid = "" + stateQueue.sync { + currentStream?.stop() + currentStream = nil + currentDeviceUdid = "" + } } if !notifiedNoBoot { jsonStatus(["type": "no-booted-device", "deviceUDID": requestedUdid]) diff --git a/apps/desktop/native/ios-sim-helpers/sim-input.m b/apps/desktop/native/ios-sim-helpers/sim-input.m index ab6b324f7..2568c579c 100644 --- a/apps/desktop/native/ios-sim-helpers/sim-input.m +++ b/apps/desktop/native/ios-sim-helpers/sim-input.m @@ -382,7 +382,10 @@ int main(int argc, const char **argv) { elog(@"[sim-input] --udid is required"); return 64; } - ensureHID(); + if (!ensureHID()) { + elog(@"[sim-input] FAIL ensureHID at startup"); + return 2; + } elog(@"[sim-input] ready"); NSFileHandle *input = [NSFileHandle fileHandleWithStandardInput]; NSMutableData *buffer = [NSMutableData data]; diff --git a/apps/desktop/src/main/services/ios/iosSimulatorService.ts b/apps/desktop/src/main/services/ios/iosSimulatorService.ts index b66c1e041..fa5b9c736 100644 --- a/apps/desktop/src/main/services/ios/iosSimulatorService.ts +++ b/apps/desktop/src/main/services/ios/iosSimulatorService.ts @@ -2234,6 +2234,9 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { }; const ensureCompanion = async (deviceUdid: string): Promise => { + if (disposed) { + throw new Error("iOS simulator service has been disposed."); + } if (companionIdleTimer) { clearTimeout(companionIdleTimer); companionIdleTimer = null; @@ -2248,8 +2251,14 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { if (stopped > 0) { args.logger.info("ios_simulator.cleaned_orphaned_idb_companions", { deviceUdid, stopped }); } + if (disposed) { + throw new Error("iOS simulator service has been disposed."); + } stopCompanion(); const port = await getFreePort(); + if (disposed) { + throw new Error("iOS simulator service has been disposed."); + } const address = `127.0.0.1:${port}`; let stderr = ""; const process = spawnProcess("idb_companion", [ @@ -2269,6 +2278,24 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { process.stdout?.on("data", (chunk: Buffer) => { stderr = `${stderr}${chunk.toString()}`.slice(-4000); }); + // Always attach an `error` listener so spawn failures (e.g. ENOENT when + // idb_companion is not installed on CI) surface as a normal stream-error + // instead of becoming an unhandled exception that fails the test runner. + process.once("error", (error) => { + if (companionProcess !== process) return; + companionProcess = null; + companionDeviceUdid = null; + companionAddress = null; + companionReadyPromise = null; + const detail = error instanceof Error ? error.message : String(error); + args.logger.debug("ios_simulator.idb_companion_spawn_failed", { deviceUdid, error: detail }); + if (streamStatus.running) { + stopChild(streamProcess); + stopChild(streamTranscoderProcess); + const status = setStreamStopped(stderr.trim() || detail); + emit({ type: "stream-error", status }); + } + }); process.once("exit", (code) => { if (companionProcess !== process) return; companionProcess = null; @@ -3578,6 +3605,7 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { activeSession = null; iosurfaceInputStickyFallbackSessionId = null; iosurfaceInputFailureTimestamps = []; + iosurfaceScreenMetrics.clear(); cachedStatus = { ...cachedStatus, computedAt: 0 }; if (released || previousSession) { emit({ type: "session-updated", session: null }); @@ -4285,7 +4313,8 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { }; const preferIndigoInput = async (): Promise => { - if (iosurfaceInputStickyFallbackSessionId && iosurfaceInputStickyFallbackSessionId === activeSession?.id) return false; + const currentSessionMarker = activeSession?.id ?? "anonymous"; + if (iosurfaceInputStickyFallbackSessionId && iosurfaceInputStickyFallbackSessionId === currentSessionMarker) return false; if (process.platform !== "darwin") return false; const capability = await detectIosurfaceIndigoCapability(); return capability.available; @@ -4423,6 +4452,7 @@ export function createIosSimulatorService(args: CreateIosSimulatorServiceArgs) { setStreamStopped(null); activeSession = null; activeLaunchId = null; + iosurfaceScreenMetrics.clear(); cleanupTempFiles(); toolAvailabilityCache.clear(); if (xcodeMcpBridge) { diff --git a/apps/desktop/src/main/services/ipc/registerIpc.ts b/apps/desktop/src/main/services/ipc/registerIpc.ts index d10b449ec..b6688f8e9 100644 --- a/apps/desktop/src/main/services/ipc/registerIpc.ts +++ b/apps/desktop/src/main/services/ipc/registerIpc.ts @@ -5874,7 +5874,8 @@ export function registerIpc({ ipcMain.handle(IPC.iosSimulatorLaunch, async (event, arg = {}) => { const result = await ensureIosSimulator().launch(arg); - const keepSimulatorInBackground = Boolean((arg as { keepSimulatorInBackground?: unknown } | null)?.keepSimulatorInBackground ?? true); + const keepSimulatorInBackgroundPayload = (arg as { keepSimulatorInBackground?: unknown } | null)?.keepSimulatorInBackground; + const keepSimulatorInBackground = keepSimulatorInBackgroundPayload === undefined ? true : keepSimulatorInBackgroundPayload === true; if (!keepSimulatorInBackground) { const browserWindow = BrowserWindow.fromWebContents(event.sender); await prepareSimulatorWindowForCapture(browserWindow, { placeBehindAde: false }); diff --git a/apps/desktop/src/main/services/prs/prService.ts b/apps/desktop/src/main/services/prs/prService.ts index eaf2c2963..a80011a26 100644 --- a/apps/desktop/src/main/services/prs/prService.ts +++ b/apps/desktop/src/main/services/prs/prService.ts @@ -1510,10 +1510,12 @@ export function createPrService({ path: string; query?: Record; select?: (payload: any) => T[]; + maxPages?: number; }): Promise => { const out: T[] = []; const pageSize = 100; - for (let page = 1; page <= 10; page += 1) { + const maxPages = args.maxPages ?? 10; + for (let page = 1; page <= maxPages; page += 1) { const { data } = await githubService.apiRequest({ method: "GET", path: args.path, @@ -5233,6 +5235,7 @@ export function createPrService({ const rows = await fetchAllPages({ path: `/repos/${repo.owner}/${repo.name}/pulls`, query: { state: "open", sort: "updated", direction: "desc" }, + maxPages: 5, }); const out: BranchPullRequest[] = []; for (const row of rows) { diff --git a/apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.tsx b/apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.tsx index 9cb151bd6..fd0ba151c 100644 --- a/apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.tsx +++ b/apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.tsx @@ -1243,7 +1243,7 @@ export function ChatIosSimulatorPanel({ lastWindowFrameAtRef.current = 0; if (preserveVisual) { setLiveVisual((current) => current?.kind === "mjpeg" - ? { ...current, status: "reconnecting", url: null, error: null } + ? { ...current, status: "reconnecting", error: null } : current); return; } @@ -1751,11 +1751,11 @@ export function ChatIosSimulatorPanel({ default: startupGraceMs = 5_000; } + const postFrameStallMs = streamStatus.backend === "iosurface-indigo" ? 8_000 : 5_000; const timer = window.setInterval(() => { const lastFrameMs = streamStatus.lastFrameAt ? Date.parse(streamStatus.lastFrameAt) : 0; const startedMs = streamStatus.startedAt ? Date.parse(streamStatus.startedAt) : 0; - if (streamStatus.backend === "iosurface-indigo" && lastFrameMs) return; - const staleAfterMs = lastFrameMs ? 5_000 : startupGraceMs; + const staleAfterMs = lastFrameMs ? postFrameStallMs : startupGraceMs; const referenceMs = lastFrameMs || startedMs; if (referenceMs && Date.now() - referenceMs > staleAfterMs) { scheduleDeviceBackedStreamRestart("Live simulator stream stalled."); diff --git a/apps/desktop/src/renderer/components/lanes/BranchPickerView.tsx b/apps/desktop/src/renderer/components/lanes/BranchPickerView.tsx index 7652f04dc..ab5c74597 100644 --- a/apps/desktop/src/renderer/components/lanes/BranchPickerView.tsx +++ b/apps/desktop/src/renderer/components/lanes/BranchPickerView.tsx @@ -132,12 +132,17 @@ export function BranchPickerView({ const [query, setQuery] = React.useState(""); const inputRef = React.useRef(null); const listRef = React.useRef(null); - const now = React.useMemo(() => Date.now(), []); + const [now, setNow] = React.useState(() => Date.now()); React.useEffect(() => { inputRef.current?.focus(); }, []); + React.useEffect(() => { + const id = window.setInterval(() => setNow(Date.now()), 60_000); + return () => window.clearInterval(id); + }, []); + const prByBranch = React.useMemo(() => { const map = new Map(); for (const pr of pullRequests) map.set(pr.branch, pr); diff --git a/apps/desktop/src/renderer/components/lanes/LanesPage.tsx b/apps/desktop/src/renderer/components/lanes/LanesPage.tsx index 5dee6f5bd..73a786891 100644 --- a/apps/desktop/src/renderer/components/lanes/LanesPage.tsx +++ b/apps/desktop/src/renderer/components/lanes/LanesPage.tsx @@ -1534,6 +1534,9 @@ export function LanesPage() { setCreateChildBaseBranch(""); setCreateBranches([]); setCreateBranchPullRequests([]); + setCreateGitUserName(""); + setCreateBranchesLoading(false); + setCreateBranchPullRequestsLoading(false); setLaneCreated(false); createBaseBranchUserPickedRef.current = false; const primary = lanes.find((l) => l.laneType === "primary"); diff --git a/apps/desktop/src/renderer/components/lanes/branchPickerSearch.test.ts b/apps/desktop/src/renderer/components/lanes/branchPickerSearch.test.ts index 35b78670b..01c3fdaa3 100644 --- a/apps/desktop/src/renderer/components/lanes/branchPickerSearch.test.ts +++ b/apps/desktop/src/renderer/components/lanes/branchPickerSearch.test.ts @@ -107,8 +107,9 @@ describe("matchesQuery", () => { expect(matchesQuery(ctx(), parseSearchQuery("#812"))).toBe(true); }); - it("pr:open keeps branches with open PR", () => { + it("pr:open keeps branches with open PR (including drafts)", () => { expect(matchesQuery(ctx(), parseSearchQuery("pr:open"))).toBe(true); + expect(matchesQuery(ctx({ pr: makePr({ state: "draft" }) }), parseSearchQuery("pr:open"))).toBe(true); expect(matchesQuery(ctx({ pr: null }), parseSearchQuery("pr:open"))).toBe(false); }); diff --git a/apps/desktop/src/renderer/components/lanes/branchPickerSearch.ts b/apps/desktop/src/renderer/components/lanes/branchPickerSearch.ts index d71c1581a..fa751bc0f 100644 --- a/apps/desktop/src/renderer/components/lanes/branchPickerSearch.ts +++ b/apps/desktop/src/renderer/components/lanes/branchPickerSearch.ts @@ -107,7 +107,8 @@ export function matchesQuery(ctx: BranchMatchContext, query: ParsedSearchQuery): switch (token.kind) { case "pr": { if (token.value === "none" && pr) return false; - if (token.value === "open" && (!pr || pr.state !== "open")) return false; + // `pr:open` includes drafts — drafts are still open PRs on GitHub. + if (token.value === "open" && (!pr || (pr.state !== "open" && pr.state !== "draft"))) return false; if (token.value === "draft" && (!pr || pr.state !== "draft")) return false; break; } From d8aec88c2772f5d1ff246db8e08bcf6b65479363 Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Thu, 30 Apr 2026 19:53:17 -0400 Subject: [PATCH 3/3] ship: revert iosurface-indigo stall watchdog change Restores the early-return for iosurface-indigo after first frame so the test "does not restart an idle native stream after it has produced a frame" continues to pass. The CodeRabbit nit was minor and conflicts with this load-bearing test. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/renderer/components/chat/ChatIosSimulatorPanel.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.tsx b/apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.tsx index fd0ba151c..1094c1606 100644 --- a/apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.tsx +++ b/apps/desktop/src/renderer/components/chat/ChatIosSimulatorPanel.tsx @@ -1751,11 +1751,11 @@ export function ChatIosSimulatorPanel({ default: startupGraceMs = 5_000; } - const postFrameStallMs = streamStatus.backend === "iosurface-indigo" ? 8_000 : 5_000; const timer = window.setInterval(() => { const lastFrameMs = streamStatus.lastFrameAt ? Date.parse(streamStatus.lastFrameAt) : 0; const startedMs = streamStatus.startedAt ? Date.parse(streamStatus.startedAt) : 0; - const staleAfterMs = lastFrameMs ? postFrameStallMs : startupGraceMs; + if (streamStatus.backend === "iosurface-indigo" && lastFrameMs) return; + const staleAfterMs = lastFrameMs ? 5_000 : startupGraceMs; const referenceMs = lastFrameMs || startedMs; if (referenceMs && Date.now() - referenceMs > staleAfterMs) { scheduleDeviceBackedStreamRestart("Live simulator stream stalled.");