diff --git a/apps/desktop/scripts/build-release.sh b/apps/desktop/scripts/build-release.sh index 97ee180..546a922 100755 --- a/apps/desktop/scripts/build-release.sh +++ b/apps/desktop/scripts/build-release.sh @@ -51,6 +51,25 @@ fetch_marksyncr() { [ -n "$MKS_SRC" ] || echo " ! MarkSyncr fetch skipped (non-fatal)" } +# The Node automation runtime for `tron snapshot|click|fill` (PRD M3.2). The +# @tronbrowser/browser-core source has no runtime deps, so its compiled dist tree +# is self-contained; ship it with a {"type":"module"} marker and the shell +# dispatcher runs it via node. Best-effort like the extension fetches — a build +# host without node/pnpm simply omits it (the CLI then reports "run tron upgrade"). +stage_automation() { # dest dir + local s="$1" + command -v node >/dev/null 2>&1 && command -v pnpm >/dev/null 2>&1 || { + echo " ! automation runtime skipped (needs node + pnpm)"; return; } + if ( cd "$REPO_ROOT" && pnpm --filter @tronbrowser/browser-core build >/dev/null 2>&1 ); then + rm -rf "$s/automate" + cp -R "$REPO_ROOT/packages/browser-core/dist" "$s/automate" + printf '{\n "type": "module"\n}\n' > "$s/automate/package.json" + echo " + bundled automation runtime (tron snapshot/click/fill)" + else + echo " ! automation runtime skipped (browser-core build failed)" + fi +} + stage() { # dest dir local s="$1" mkdir -p "$s/extensions" @@ -62,6 +81,7 @@ stage() { # dest dir # Managed-session engine for `tron browser …` / `tron open` (PRD M3.1). Sits # next to the shim; the `tron` dispatcher resolves it relative to $CURRENT. install -m 0755 "$DESKTOP/launcher/tron-session" "$s/tron-session" + stage_automation "$s" # -L dereferences the branding symlinks (icons/logo.svg -> repo-root logo.svg) # so the package contains real files, not dangling links. cp -RL "$DESKTOP/extensions/ai-sidebar" "$s/extensions/ai-sidebar" diff --git a/apps/web/public/install.sh b/apps/web/public/install.sh index ed5e6a7..b28ea20 100755 --- a/apps/web/public/install.sh +++ b/apps/web/public/install.sh @@ -84,6 +84,9 @@ Usage: tron browser status Show managed-session status (--json for machine output) tron browser tabs List tabs in the managed session (--json) tron browser close Close the managed session + tron snapshot Structured, ref-tagged page snapshot (--json) + tron click Click a snapshot ref, e.g. @e3 + tron fill Fill an input by ref, e.g. tron fill @e4 "hi@x.com" tron upgrade Update to the latest release tron remove Uninstall TronBrowser (keeps your profile data) tron version Print the installed version @@ -131,6 +134,20 @@ session_bin() { echo "$_ld/tron-session" } +# Node automation runtime entry for `tron snapshot|click|fill|type` (M3.2). +automate_entry() { + _ld="$(dirname "$(readlink -f "$CURRENT" 2>/dev/null || echo "$CURRENT")")" + echo "$_ld/automate/automate-bin.js" +} + +# Route a CDP automation subcommand to the Node runtime, or explain what's missing. +run_automation() { + ENTRY="$(automate_entry)" + command -v node >/dev/null 2>&1 || { echo "tron $1 needs Node.js (>=22) on PATH." >&2; exit 1; } + [ -f "$ENTRY" ] || { echo "This TronBrowser build lacks the automation runtime. Run: tron upgrade" >&2; exit 1; } + exec node "$ENTRY" "$@" +} + case "${1:-}" in open) shift @@ -151,6 +168,9 @@ case "${1:-}" in SESSION="$(session_bin)" [ -x "$SESSION" ] || { echo "This TronBrowser build has no managed-session support (missing tron-session). Run: tron upgrade" >&2; exit 1; } exec "$SESSION" browser "$@" ;; + snapshot|click|fill|type) + # CDP automation on the managed session's current page (PRD M3.2). + run_automation "$@" ;; restart) # Force-quit any running TronBrowser, then launch fresh. Chromium forwards a # new launch to an already-running instance (which keeps the OLD extension diff --git a/docs/snapshots-and-refs.md b/docs/snapshots-and-refs.md new file mode 100644 index 0000000..446adb3 --- /dev/null +++ b/docs/snapshots-and-refs.md @@ -0,0 +1,58 @@ +# Snapshots and refs (M3.2) + +Once a managed session is running (`tron browser launch`, see +[managed-sessions.md](./managed-sessions.md)), the `tron` CLI can read the +current page as a compact, ref-tagged structure and act on it by ref. + +```sh +tron snapshot # compact text snapshot of the current page +tron snapshot --json # machine-readable snapshot +tron snapshot --include-hidden +tron click @e3 # click a ref from the last snapshot +tron fill @e4 "hi@example.com" # fill an input/textarea by ref +``` + +Text output: + +```txt +Page: Contact Us +URL: https://example.com/contact + +@e1 heading "Contact Us" +@e2 textbox "Name" +@e3 textbox "Email" +@e4 link "Privacy" -> https://example.com/privacy +@e5 button "Submit" +``` + +## Refs + +A snapshot assigns `@e1`, `@e2`, … to visible interactive elements (and +headings) in document order and tags each element in the page with a +`data-tron-ref` attribute. Because the ref lives in the DOM, a later +`tron click @e3` — a separate process — resolves it with a plain attribute +selector. If the element is gone (navigation, re-render), the action returns a +recoverable **STALE_REF** error (exit code 5) telling you to re-`snapshot`, +rather than acting on the wrong node. Prefer refs over CSS selectors for agents. + +Password values are never echoed in snapshots; `--json` includes `role`, +`name`, `value`, `href`, visibility, and interactivity per element. + +## How it works + +- `snapshot`/`click`/`fill` are Node subcommands the shell `tron` dispatcher + delegates to. They attach to the session's current page via the descriptor's + `webSocketDebuggerUrl` and drive it over the Chrome DevTools Protocol + (`Runtime.evaluate`). +- The CDP client uses Node's global `WebSocket` (Node >= 22) — no dependency. + The runtime is `@tronbrowser/browser-core`'s compiled tree, shipped in the + launcher payload; the dispatcher runs it with `node`. +- Everything stays on `127.0.0.1` — no page content leaves the machine. + +## Scope / limitations + +- Requires Node.js (>= 22) on PATH, plus a running managed session. +- The snapshot targets the session's current tab (`tron browser use ` to + switch). Shadow DOM and cross-origin iframes are out of scope for M3.2. +- Contracts and CDP/DOM logic are unit-tested in + `packages/browser-core/src/automation` and `src/automate-*.test.ts`. diff --git a/packages/browser-core/package.json b/packages/browser-core/package.json index 0d9210f..2ae5036 100644 --- a/packages/browser-core/package.json +++ b/packages/browser-core/package.json @@ -19,6 +19,7 @@ "lint": "eslint src" }, "devDependencies": { + "happy-dom": "^20.10.6", "typescript": "^5.6.3", "vitest": "^2.1.4" } diff --git a/packages/browser-core/src/automate-bin.ts b/packages/browser-core/src/automate-bin.ts new file mode 100644 index 0000000..69d0f77 --- /dev/null +++ b/packages/browser-core/src/automate-bin.ts @@ -0,0 +1,14 @@ +/** + * Executable wrapper around the automation CLI. Built into a self-contained + * `automate.js` (see apps/desktop/scripts/build-release.sh) that the shell + * `tron` dispatcher runs via `node`. + */ +import { run } from './automate-cli.js'; + +run(process.argv.slice(2)).then( + (code) => process.exit(code), + (err: unknown) => { + process.stderr.write(`tron: ${err instanceof Error ? err.message : String(err)}\n`); + process.exit(1); + }, +); diff --git a/packages/browser-core/src/automate-cli.test.ts b/packages/browser-core/src/automate-cli.test.ts new file mode 100644 index 0000000..72cf28d --- /dev/null +++ b/packages/browser-core/src/automate-cli.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it, vi } from 'vitest'; +import { EXIT, run, type CliDeps } from './automate-cli.js'; +import type { CdpConnection } from './automation/cdp-client.js'; +import type { AgentSnapshot } from './automation/snapshot-script.js'; +import type { SessionDescriptor } from './automation/types.js'; + +const descriptor: SessionDescriptor = { + version: 1, + pid: 1, + host: '127.0.0.1', + port: 9222, + profileDir: '/x', + profileName: 'agent', + headless: false, + ephemeral: false, + createdAt: '2026-07-04T00:00:00.000Z', + activeTabId: 'p1', +}; + +const snap: AgentSnapshot = { + url: 'https://example.com', + title: 'Example', + timestamp: '2026-07-04T00:00:00.000Z', + elements: [ + { ref: '@e1', role: 'link', name: 'More', tag: 'a', interactive: true, visible: true, href: 'https://x' }, + ], +}; + +/** A CdpConnection whose Runtime.evaluate yields `evalValue`. */ +function conn(evalValue: unknown): CdpConnection { + return { + send: (async (method: string) => + method === 'Runtime.evaluate' ? { result: { value: evalValue } } : {}) as CdpConnection['send'], + on: vi.fn(), + close: vi.fn(), + }; +} + +function harness(overrides: Partial = {}) { + const out: string[] = []; + const err: string[] = []; + const deps: Partial = { + env: {}, + loadDescriptor: async () => descriptor, + fetchTargets: async () => [ + { id: 'p1', type: 'page', url: 'https://example.com', webSocketDebuggerUrl: 'ws://x/p1' }, + ], + connect: async () => conn(snap), + out: (t) => out.push(t), + err: (t) => err.push(t), + ...overrides, + }; + return { deps, out, err }; +} + +describe('automate-cli run', () => { + it('prints a text snapshot', async () => { + const { deps, out } = harness(); + const code = await run(['snapshot'], deps); + expect(code).toBe(EXIT.ok); + expect(out.join('\n')).toContain('@e1 link "More"'); + }); + + it('prints JSON with --json', async () => { + const { deps, out } = harness(); + await run(['snapshot', '--json'], deps); + expect(JSON.parse(out.join('\n')).title).toBe('Example'); + }); + + it('clicks a ref', async () => { + const { deps, out } = harness({ connect: async () => conn({ ok: true, ref: '@e1' }) }); + const code = await run(['click', '@e1'], deps); + expect(code).toBe(EXIT.ok); + expect(out.join('\n')).toContain('clicked @e1'); + }); + + it('fills a ref', async () => { + const { deps, out } = harness({ connect: async () => conn({ ok: true, ref: '@e2' }) }); + const code = await run(['fill', '@e2', 'hello'], deps); + expect(code).toBe(EXIT.ok); + expect(out.join('\n')).toContain('filled @e2'); + }); + + it('exits staleRef when a ref no longer resolves', async () => { + const { deps, err } = harness({ + connect: async () => conn({ ok: false, error: 'STALE_REF', ref: '@e9' }), + }); + const code = await run(['click', '@e9'], deps); + expect(code).toBe(EXIT.staleRef); + expect(err.join('\n')).toMatch(/stale/i); + }); + + it('exits noSession when there is no descriptor', async () => { + const { deps, err } = harness({ + loadDescriptor: async () => { + throw new Error('ENOENT'); + }, + }); + const code = await run(['snapshot'], deps); + expect(code).toBe(EXIT.noSession); + expect(err.join('\n')).toContain('tron browser launch'); + }); + + it('exits usage when click is missing a ref', async () => { + const { deps } = harness(); + expect(await run(['click'], deps)).toBe(EXIT.usage); + }); +}); diff --git a/packages/browser-core/src/automate-cli.ts b/packages/browser-core/src/automate-cli.ts new file mode 100644 index 0000000..f49299f --- /dev/null +++ b/packages/browser-core/src/automate-cli.ts @@ -0,0 +1,143 @@ +/** + * `tron-automate` — Node entrypoint for the CDP-driven automation subcommands + * the shell `tron` dispatcher delegates to (PRD M3.2): + * + * tron snapshot [--json] [--include-hidden] + * tron click + * tron fill + * + * It attaches to the M3.1-managed session via its descriptor + the page target's + * webSocketDebuggerUrl. Dependencies (descriptor read, target fetch, CDP connect) + * are injectable so the command layer is testable without a real browser. + */ +import { readFile } from 'node:fs/promises'; +import { CdpClient, type CdpConnection } from './automation/cdp-client.js'; +import { cdpListUrl } from './automation/cdp.js'; +import { + descriptorPath, + parseDescriptor, + resolveDataDir, +} from './automation/descriptor.js'; +import { + captureSnapshot, + clickRef, + enableRuntime, + fillRef, + formatSnapshotText, + StaleRefError, +} from './automation/page.js'; +import { resolvePageWsUrl } from './automation/page-target.js'; +import type { SessionDescriptor, CdpTarget } from './automation/types.js'; + +/** Process exit codes shared with the shell dispatcher. */ +export const EXIT = { + ok: 0, + usage: 2, + noSession: 4, + staleRef: 5, + failed: 1, +} as const; + +export interface CliDeps { + env: NodeJS.ProcessEnv; + loadDescriptor(path: string): Promise; + fetchTargets(listUrl: string): Promise; + connect(wsUrl: string): Promise; + out(text: string): void; + err(text: string): void; +} + +const defaultDeps: CliDeps = { + env: process.env, + async loadDescriptor(path) { + return parseDescriptor(await readFile(path, 'utf8')); + }, + async fetchTargets(listUrl) { + const res = await fetch(listUrl); + if (!res.ok) throw new Error(`DevTools /json/list returned ${res.status}`); + return (await res.json()) as CdpTarget[]; + }, + connect: (wsUrl) => CdpClient.connect(wsUrl), + out: (t) => process.stdout.write(t + '\n'), + err: (t) => process.stderr.write(t + '\n'), +}; + +/** Attach to the current page of the managed session, or throw a coded error. */ +async function attach(deps: CliDeps): Promise { + const dataDir = resolveDataDir(deps.env); + let descriptor: SessionDescriptor; + try { + descriptor = await deps.loadDescriptor(descriptorPath(dataDir)); + } catch { + const e = new Error('No managed session. Run: tron browser launch') as Error & { exit?: number }; + e.exit = EXIT.noSession; + throw e; + } + const targets = await deps.fetchTargets( + cdpListUrl({ host: descriptor.host, port: descriptor.port }), + ); + const wsUrl = resolvePageWsUrl(targets, descriptor.activeTabId); + const conn = await deps.connect(wsUrl); + await enableRuntime(conn); + return conn; +} + +export async function run(argv: string[], overrides: Partial = {}): Promise { + const deps: CliDeps = { ...defaultDeps, ...overrides }; + const [command, ...rest] = argv; + + if (command === undefined || command === 'help' || command === '--help') { + deps.out('usage: tron snapshot [--json] [--include-hidden] | click | fill '); + return EXIT.ok; + } + + let conn: CdpConnection | undefined; + try { + switch (command) { + case 'snapshot': { + const json = rest.includes('--json'); + const includeHidden = rest.includes('--include-hidden'); + conn = await attach(deps); + const snap = await captureSnapshot(conn, includeHidden ? { includeHidden } : {}); + deps.out(json ? JSON.stringify(snap, null, 2) : formatSnapshotText(snap)); + return EXIT.ok; + } + case 'click': { + const ref = rest[0]; + if (!ref) { + deps.err('usage: tron click '); + return EXIT.usage; + } + conn = await attach(deps); + const res = await clickRef(conn, ref); + deps.out(`clicked ${res.ref}`); + return EXIT.ok; + } + case 'fill': { + const ref = rest[0]; + const value = rest[1]; + if (!ref || value === undefined) { + deps.err('usage: tron fill '); + return EXIT.usage; + } + conn = await attach(deps); + const res = await fillRef(conn, ref, value); + deps.out(`filled ${res.ref}`); + return EXIT.ok; + } + default: + deps.err(`unknown automation command: ${command}`); + return EXIT.usage; + } + } catch (err) { + if (err instanceof StaleRefError) { + deps.err(err.message); + return EXIT.staleRef; + } + const coded = err as Error & { exit?: number }; + deps.err(`tron: ${coded.message}`); + return typeof coded.exit === 'number' ? coded.exit : EXIT.failed; + } finally { + conn?.close(); + } +} diff --git a/packages/browser-core/src/automate-e2e.test.ts b/packages/browser-core/src/automate-e2e.test.ts new file mode 100644 index 0000000..bae561b --- /dev/null +++ b/packages/browser-core/src/automate-e2e.test.ts @@ -0,0 +1,152 @@ +// End-to-end: drive the automation CLI with its REAL default deps (global fetch +// + real CdpClient over a real WebSocket) against a mock DevTools server. This +// covers the glue the unit tests exercise only in isolation: descriptor -> +// /json/list -> page WS -> Runtime.evaluate -> formatted output. The in-page +// scripts themselves are verified against a real DOM in snapshot-script.test.ts, +// so here the mock returns canned evaluate results. +import { createHash } from 'node:crypto'; +import { createServer, type Server } from 'node:http'; +import type { Socket } from 'node:net'; +import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { EXIT, run } from './automate-cli.js'; +import { serializeDescriptor } from './automation/descriptor.js'; +import type { SessionDescriptor } from './automation/types.js'; +import type { AgentSnapshot } from './automation/snapshot-script.js'; + +const GUID = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'; + +function decodeFrame(buf: Buffer): string | null { + if ((buf[0] & 0x0f) === 0x8) return null; + const masked = (buf[1] & 0x80) !== 0; + let len = buf[1] & 0x7f; + let off = 2; + if (len === 126) { len = buf.readUInt16BE(2); off = 4; } + let mask: Buffer | null = null; + if (masked) { mask = buf.subarray(off, off + 4); off += 4; } + const p = buf.subarray(off, off + len); + const out = Buffer.alloc(len); + for (let i = 0; i < len; i += 1) out[i] = mask ? p[i] ^ mask[i % 4] : p[i]; + return out.toString('utf8'); +} +function encodeFrame(str: string): Buffer { + const p = Buffer.from(str, 'utf8'); + if (p.length < 126) return Buffer.concat([Buffer.from([0x81, p.length]), p]); + const h = Buffer.alloc(4); + h[0] = 0x81; h[1] = 126; h.writeUInt16BE(p.length, 2); + return Buffer.concat([h, p]); +} + +interface Mock { + port: number; + close: () => Promise; +} + +/** Mock DevTools server: HTTP /json/list + a page WS that answers Runtime.*. */ +async function startMock(evaluate: (expression: string) => unknown): Promise { + const sockets = new Set(); + const server: Server = createServer((req, res) => { + if (req.url === '/json/list') { + const port = (server.address() as { port: number }).port; + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify([ + { id: 'p1', type: 'page', url: 'https://example.com', webSocketDebuggerUrl: `ws://127.0.0.1:${port}/devtools/page/p1` }, + ]), + ); + return; + } + res.writeHead(404).end(); + }); + server.on('upgrade', (req, socket) => { + sockets.add(socket as Socket); + socket.on('close', () => sockets.delete(socket as Socket)); + const accept = createHash('sha1').update((req.headers['sec-websocket-key'] ?? '') + GUID).digest('base64'); + socket.write( + 'HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n' + + `Sec-WebSocket-Accept: ${accept}\r\n\r\n`, + ); + socket.on('data', (buf: Buffer) => { + const text = decodeFrame(buf); + if (text === null) { socket.destroy(); return; } + const msg = JSON.parse(text) as { id: number; method: string; params?: { expression?: string } }; + // Real CDP nests twice: {result: {result: , exceptionDetails?}}. + const result = + msg.method === 'Runtime.evaluate' + ? { result: { result: { value: evaluate(msg.params?.expression ?? '') } } } + : {}; + socket.write(encodeFrame(JSON.stringify({ id: msg.id, ...result }))); + }); + socket.on('error', () => {}); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return { + port: (server.address() as { port: number }).port, + close: () => + new Promise((resolve) => { + for (const s of sockets) s.destroy(); + server.close(() => resolve()); + }), + }; +} + +const snap: AgentSnapshot = { + url: 'https://example.com/contact', + title: 'Contact Us', + timestamp: '2026-07-04T00:00:00.000Z', + elements: [ + { ref: '@e1', role: 'textbox', name: 'Email', tag: 'input', interactive: true, visible: true }, + ], +}; + +let mock: Mock; +let dataDir: string; + +function writeDescriptor(port: number): void { + const d: SessionDescriptor = { + version: 1, pid: process.pid, host: '127.0.0.1', port, + profileDir: '/x', profileName: 'agent', headless: false, ephemeral: false, + createdAt: '2026-07-04T00:00:00.000Z', activeTabId: 'p1', + }; + mkdirSync(join(dataDir, 'automation'), { recursive: true }); + writeFileSync(join(dataDir, 'automation', 'session.json'), serializeDescriptor(d)); +} + +beforeEach(() => { + dataDir = mkdtempSync(join(tmpdir(), 'automate-e2e-')); +}); +afterEach(async () => { + await mock.close(); + rmSync(dataDir, { recursive: true, force: true }); +}); + +describe('automation CLI end-to-end over HTTP + WebSocket', () => { + it('snapshots the current page through the real transport', async () => { + mock = await startMock(() => snap); + writeDescriptor(mock.port); + const out: string[] = []; + const code = await run(['snapshot'], { env: { TRONBROWSER_DATA: dataDir }, out: (t) => out.push(t) }); + expect(code).toBe(EXIT.ok); + expect(out.join('\n')).toContain('@e1 textbox "Email"'); + }); + + it('clicks a ref end-to-end', async () => { + mock = await startMock(() => ({ ok: true, ref: '@e1' })); + writeDescriptor(mock.port); + const out: string[] = []; + const code = await run(['click', '@e1'], { env: { TRONBROWSER_DATA: dataDir }, out: (t) => out.push(t) }); + expect(code).toBe(EXIT.ok); + expect(out.join('\n')).toContain('clicked @e1'); + }); + + it('returns staleRef end-to-end when the ref is gone', async () => { + mock = await startMock(() => ({ ok: false, error: 'STALE_REF', ref: '@e9' })); + writeDescriptor(mock.port); + const err: string[] = []; + const code = await run(['click', '@e9'], { env: { TRONBROWSER_DATA: dataDir }, err: (t) => err.push(t) }); + expect(code).toBe(EXIT.staleRef); + expect(err.join('\n')).toMatch(/stale/i); + }); +}); diff --git a/packages/browser-core/src/automation/action-script.ts b/packages/browser-core/src/automation/action-script.ts new file mode 100644 index 0000000..868137e --- /dev/null +++ b/packages/browser-core/src/automation/action-script.ts @@ -0,0 +1,66 @@ +/** + * In-page scripts for ref-based actions (PRD M3.2): click, fill, type. + * + * Each resolves the ref via its `data-tron-ref` attribute (set by the last + * snapshot). A missing element returns `{ok:false, error:'STALE_REF'}` so the + * caller can raise a recoverable error telling the agent to re-snapshot, rather + * than acting on the wrong node. + */ + +/** A `data-tron-ref` value: strip a leading `@`, require the `e` form. */ +export function normalizeRef(ref: string): string { + const trimmed = ref.trim(); + const bare = trimmed.startsWith('@') ? trimmed.slice(1) : trimmed; + if (!/^e[0-9]+$/.test(bare)) { + throw new Error(`Not a snapshot ref: "${ref}" (expected @e1, @e2, …)`); + } + return bare; +} + +/** Shared prelude: resolve `ref` to an element or bail with STALE_REF. */ +function resolvePrelude(ref: string): string { + const bare = normalizeRef(ref); + return `const el = document.querySelector('[data-tron-ref=' + ${JSON.stringify( + JSON.stringify(bare), + )} + ']'); + if (!el) return { ok: false, error: 'STALE_REF', ref: ${JSON.stringify('@' + bare)} };`; +} + +/** Click the element referenced by `ref`. */ +export function clickExpression(ref: string): string { + return `(() => { + ${resolvePrelude(ref)} + el.scrollIntoView({ block: 'center', inline: 'center' }); + el.click(); + return { ok: true, ref: ${JSON.stringify('@' + normalizeRef(ref))} }; +})()`; +} + +/** Fill an input/textarea/contenteditable referenced by `ref` with `value`. */ +export function fillExpression(ref: string, value: string): string { + return `(() => { + ${resolvePrelude(ref)} + const value = ${JSON.stringify(value)}; + el.scrollIntoView({ block: 'center', inline: 'center' }); + if (el.isContentEditable) { + el.focus(); + el.textContent = value; + el.dispatchEvent(new Event('input', { bubbles: true })); + return { ok: true, ref: ${JSON.stringify('@' + normalizeRef(ref))} }; + } + const proto = el instanceof HTMLTextAreaElement ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype; + const desc = Object.getOwnPropertyDescriptor(proto, 'value'); + el.focus(); + if (desc && desc.set) { desc.set.call(el, value); } else { el.value = value; } + el.dispatchEvent(new Event('input', { bubbles: true })); + el.dispatchEvent(new Event('change', { bubbles: true })); + return { ok: true, ref: ${JSON.stringify('@' + normalizeRef(ref))} }; +})()`; +} + +/** Result shape returned by the action scripts (via Runtime.evaluate). */ +export interface ActionResult { + ok: boolean; + ref: string; + error?: string; +} diff --git a/packages/browser-core/src/automation/cdp-client.test.ts b/packages/browser-core/src/automation/cdp-client.test.ts new file mode 100644 index 0000000..ef92f8c --- /dev/null +++ b/packages/browser-core/src/automation/cdp-client.test.ts @@ -0,0 +1,144 @@ +import { createHash } from 'node:crypto'; +import { createServer, type Server } from 'node:http'; +import type { Socket } from 'node:net'; +import { afterEach, describe, expect, it } from 'vitest'; +import { CdpClient, CdpError } from './cdp-client.js'; + +// A tiny WebSocket server (handshake + single-frame text codec) so the CDP +// client is exercised over a real socket without pulling in a `ws` dependency. +const GUID = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'; + +function decodeFrame(buf: Buffer): string | null { + const opcode = buf[0] & 0x0f; + if (opcode === 0x8) return null; // close + const masked = (buf[1] & 0x80) !== 0; + let len = buf[1] & 0x7f; + let offset = 2; + if (len === 126) { + len = buf.readUInt16BE(2); + offset = 4; + } + let mask: Buffer | null = null; + if (masked) { + mask = buf.subarray(offset, offset + 4); + offset += 4; + } + const payload = buf.subarray(offset, offset + len); + const out = Buffer.alloc(len); + for (let i = 0; i < len; i += 1) out[i] = mask ? payload[i] ^ mask[i % 4] : payload[i]; + return out.toString('utf8'); +} + +function encodeFrame(str: string): Buffer { + const payload = Buffer.from(str, 'utf8'); + const len = payload.length; + if (len < 126) return Buffer.concat([Buffer.from([0x81, len]), payload]); + const head = Buffer.alloc(4); + head[0] = 0x81; + head[1] = 126; + head.writeUInt16BE(len, 2); + return Buffer.concat([head, payload]); +} + +type Handler = (msg: { id?: number; method?: string; params?: unknown }, socket: Socket) => void; + +interface Mock { + url: string; + close: () => Promise; +} + +async function startMock(handler: Handler): Promise { + const server: Server = createServer(); + const sockets = new Set(); + server.on('upgrade', (req, socket) => { + sockets.add(socket as Socket); + socket.on('close', () => sockets.delete(socket as Socket)); + const key = req.headers['sec-websocket-key'] ?? ''; + const accept = createHash('sha1').update(key + GUID).digest('base64'); + socket.write( + 'HTTP/1.1 101 Switching Protocols\r\n' + + 'Upgrade: websocket\r\nConnection: Upgrade\r\n' + + `Sec-WebSocket-Accept: ${accept}\r\n\r\n`, + ); + socket.on('data', (buf: Buffer) => { + const text = decodeFrame(buf); + if (text === null) { + socket.destroy(); // client close frame → drop the socket + return; + } + handler(JSON.parse(text), socket as Socket); + }); + socket.on('error', () => {}); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const addr = server.address(); + const port = typeof addr === 'object' && addr ? addr.port : 0; + return { + url: `ws://127.0.0.1:${port}/devtools/page/mock`, + close: () => + new Promise((resolve) => { + for (const s of sockets) s.destroy(); + sockets.clear(); + server.close(() => resolve()); + }), + }; +} + +function reply(socket: Socket, obj: unknown): void { + socket.write(encodeFrame(JSON.stringify(obj))); +} + +let mock: Mock; +let client: CdpClient | undefined; + +afterEach(async () => { + client?.close(); + client = undefined; + await mock.close(); +}); + +describe('CdpClient over a WebSocket', () => { + it('matches command responses by id', async () => { + mock = await startMock((msg, socket) => { + reply(socket, { id: msg.id, result: { echoed: msg.method } }); + }); + client = await CdpClient.connect(mock.url); + const res = await client.send<{ echoed: string }>('Runtime.evaluate', { expression: '1' }); + expect(res.echoed).toBe('Runtime.evaluate'); + }); + + it('rejects with CdpError on an error result', async () => { + mock = await startMock((msg, socket) => { + reply(socket, { id: msg.id, error: { code: -32000, message: 'no such target' } }); + }); + client = await CdpClient.connect(mock.url); + const err = await client.send('Bad.method').catch((e) => e); + expect(err).toBeInstanceOf(CdpError); + expect(err.code).toBe(-32000); + expect(err.message).toContain('no such target'); + }); + + it('dispatches protocol events to on() handlers', async () => { + mock = await startMock((msg, socket) => { + // Reply, then emit an unsolicited event. + reply(socket, { id: msg.id, result: {} }); + reply(socket, { method: 'Page.loadEventFired', params: { timestamp: 42 } }); + }); + client = await CdpClient.connect(mock.url); + const event = new Promise((resolve) => client!.on('Page.loadEventFired', resolve)); + await client.send('Page.enable'); + await expect(event).resolves.toEqual({ timestamp: 42 }); + }); + + it('rejects pending commands when the connection closes', async () => { + mock = await startMock(() => { + /* never respond */ + }); + client = await CdpClient.connect(mock.url); + const pending = client.send('Runtime.evaluate').catch((e) => e); + client.close(); + const err = await pending; + expect(err).toBeInstanceOf(Error); + expect(err.message).toMatch(/closed/); + }); +}); diff --git a/packages/browser-core/src/automation/cdp-client.ts b/packages/browser-core/src/automation/cdp-client.ts new file mode 100644 index 0000000..643bb2f --- /dev/null +++ b/packages/browser-core/src/automation/cdp-client.ts @@ -0,0 +1,160 @@ +/** + * Minimal Chrome DevTools Protocol client over a WebSocket (PRD M3.2). + * + * Uses the Node global `WebSocket` (Node >= 22), so it needs no dependency. This + * is the programmatic control channel M3.1's session descriptor points at via + * `webSocketDebuggerUrl`; snapshots and ref actions drive a page target through + * it. Commands are JSON-RPC ({id, method, params} -> {id, result|error}); + * unmatched messages are protocol events dispatched to `on` handlers. + */ + +/** The subset of the CDP transport the snapshot/action layer depends on. */ +export interface CdpConnection { + send(method: string, params?: Record): Promise; + on(method: string, handler: (params: unknown) => void): void; + close(): void; +} + +/** A CDP command returned an error result. */ +export class CdpError extends Error { + readonly code: number; + constructor(method: string, code: number, message: string) { + super(`CDP ${method} failed (${code}): ${message}`); + this.name = 'CdpError'; + this.code = code; + } +} + +interface Pending { + resolve: (value: unknown) => void; + reject: (reason: Error) => void; + method: string; +} + +export interface CdpConnectOptions { + timeoutMs?: number; +} + +export class CdpClient implements CdpConnection { + #ws: WebSocket; + #nextId = 1; + #pending = new Map(); + #handlers = new Map void>>(); + #closed = false; + + private constructor(ws: WebSocket) { + this.#ws = ws; + ws.onmessage = (ev: MessageEvent) => this.#onMessage(ev); + ws.onclose = () => this.#onClose(); + } + + /** Open a CDP connection to a DevTools WebSocket URL. */ + static connect(url: string, options: CdpConnectOptions = {}): Promise { + const timeoutMs = options.timeoutMs ?? 10_000; + return new Promise((resolve, reject) => { + let ws: WebSocket; + try { + ws = new WebSocket(url); + } catch (err) { + reject(err instanceof Error ? err : new Error(String(err))); + return; + } + const timer = setTimeout(() => { + try { + ws.close(); + } catch { + // already closing + } + reject(new Error(`CDP connect timed out after ${timeoutMs}ms`)); + }, timeoutMs); + ws.onopen = () => { + clearTimeout(timer); + resolve(new CdpClient(ws)); + }; + ws.onerror = () => { + clearTimeout(timer); + reject(new Error(`CDP connect failed for ${url}`)); + }; + }); + } + + send(method: string, params: Record = {}): Promise { + if (this.#closed) return Promise.reject(new Error('CDP connection is closed')); + const id = this.#nextId++; + const payload = JSON.stringify({ id, method, params }); + return new Promise((resolve, reject) => { + this.#pending.set(id, { + resolve: resolve as (value: unknown) => void, + reject, + method, + }); + try { + this.#ws.send(payload); + } catch (err) { + this.#pending.delete(id); + reject(err instanceof Error ? err : new Error(String(err))); + } + }); + } + + on(method: string, handler: (params: unknown) => void): void { + let set = this.#handlers.get(method); + if (!set) { + set = new Set(); + this.#handlers.set(method, set); + } + set.add(handler); + } + + close(): void { + if (this.#closed) return; + this.#closed = true; + try { + this.#ws.close(); + } catch { + // ignore + } + this.#onClose(); + } + + #onMessage(ev: MessageEvent): void { + const raw = typeof ev.data === 'string' ? ev.data : String(ev.data); + let msg: { + id?: number; + result?: unknown; + error?: { code?: number; message?: string }; + method?: string; + params?: unknown; + }; + try { + msg = JSON.parse(raw); + } catch { + return; // ignore malformed frames + } + if (typeof msg.id === 'number') { + const pending = this.#pending.get(msg.id); + if (!pending) return; + this.#pending.delete(msg.id); + if (msg.error) { + pending.reject( + new CdpError(pending.method, msg.error.code ?? -1, msg.error.message ?? 'unknown'), + ); + } else { + pending.resolve(msg.result); + } + return; + } + if (typeof msg.method === 'string') { + const set = this.#handlers.get(msg.method); + if (set) for (const h of set) h(msg.params); + } + } + + #onClose(): void { + this.#closed = true; + if (this.#pending.size === 0) return; + const err = new Error('CDP connection closed'); + for (const p of this.#pending.values()) p.reject(err); + this.#pending.clear(); + } +} diff --git a/packages/browser-core/src/automation/index.ts b/packages/browser-core/src/automation/index.ts index e67b2d1..bcbb99c 100644 --- a/packages/browser-core/src/automation/index.ts +++ b/packages/browser-core/src/automation/index.ts @@ -8,3 +8,10 @@ export * from './types.js'; export * from './cdp.js'; export * from './descriptor.js'; + +// Snapshots and ref actions (PRD M3.2). +export * from './cdp-client.js'; +export * from './snapshot-script.js'; +export * from './action-script.js'; +export * from './page-target.js'; +export * from './page.js'; diff --git a/packages/browser-core/src/automation/page-target.test.ts b/packages/browser-core/src/automation/page-target.test.ts new file mode 100644 index 0000000..a9b2086 --- /dev/null +++ b/packages/browser-core/src/automation/page-target.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest'; +import { resolvePageWsUrl, selectPageTarget } from './page-target.js'; +import type { CdpTarget } from './types.js'; + +const targets: CdpTarget[] = [ + { id: 'sw', type: 'service_worker', url: 'x' }, + { id: 'p1', type: 'page', url: 'https://a', webSocketDebuggerUrl: 'ws://h/p1' }, + { id: 'p2', type: 'page', url: 'https://b', webSocketDebuggerUrl: 'ws://h/p2' }, +]; + +describe('selectPageTarget', () => { + it('prefers the active tab', () => { + expect(selectPageTarget(targets, 'p2')?.id).toBe('p2'); + }); + it('falls back to the first page when active is absent/closed', () => { + expect(selectPageTarget(targets, 'gone')?.id).toBe('p1'); + expect(selectPageTarget(targets)?.id).toBe('p1'); + }); + it('returns undefined when there are no pages', () => { + expect(selectPageTarget([{ id: 'sw', type: 'service_worker' }])).toBeUndefined(); + }); +}); + +describe('resolvePageWsUrl', () => { + it('returns the chosen page ws url', () => { + expect(resolvePageWsUrl(targets, 'p2')).toBe('ws://h/p2'); + }); + it('throws when there is no page target', () => { + expect(() => resolvePageWsUrl([])).toThrow(/No page target/); + }); + it('throws when the page has no ws url', () => { + expect(() => resolvePageWsUrl([{ id: 'p', type: 'page' }])).toThrow(/no webSocketDebuggerUrl/); + }); +}); diff --git a/packages/browser-core/src/automation/page-target.ts b/packages/browser-core/src/automation/page-target.ts new file mode 100644 index 0000000..ea5561f --- /dev/null +++ b/packages/browser-core/src/automation/page-target.ts @@ -0,0 +1,34 @@ +/** + * Resolve which page target's DevTools WebSocket to drive (PRD M3.2). + * + * A managed session can have several page targets; snapshot/click/fill act on + * the "current" one — the descriptor's active tab when present, else the first + * page — matching how `tron browser tabs` marks the current tab. + */ +import type { CdpTarget } from './types.js'; + +/** Page target chosen to act on: the active tab if present, else the first page. */ +export function selectPageTarget( + targets: readonly CdpTarget[], + activeTabId?: string, +): CdpTarget | undefined { + const pages = targets.filter((t) => t.type === 'page'); + if (activeTabId !== undefined) { + const active = pages.find((t) => t.id === activeTabId); + if (active) return active; + } + return pages[0]; +} + +/** The page WebSocket URL to attach to, or throw a clear error if none. */ +export function resolvePageWsUrl( + targets: readonly CdpTarget[], + activeTabId?: string, +): string { + const target = selectPageTarget(targets, activeTabId); + if (!target) throw new Error('No page target in the managed session'); + if (!target.webSocketDebuggerUrl) { + throw new Error(`Page target ${target.id} has no webSocketDebuggerUrl`); + } + return target.webSocketDebuggerUrl; +} diff --git a/packages/browser-core/src/automation/page.test.ts b/packages/browser-core/src/automation/page.test.ts new file mode 100644 index 0000000..a1f5a01 --- /dev/null +++ b/packages/browser-core/src/automation/page.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { CdpConnection } from './cdp-client.js'; +import { + captureSnapshot, + clickRef, + fillRef, + formatSnapshotText, + StaleRefError, +} from './page.js'; +import type { AgentSnapshot } from './snapshot-script.js'; + +/** A CdpConnection whose Runtime.evaluate returns a canned by-value result. */ +function fakeConn(evalValue: unknown, opts: { exception?: string } = {}): CdpConnection { + const send = vi.fn(async (method: string) => { + if (method === 'Runtime.evaluate') { + return opts.exception + ? { exceptionDetails: { text: opts.exception } } + : { result: { value: evalValue } }; + } + return {}; + }); + return { send: send as unknown as CdpConnection['send'], on: vi.fn(), close: vi.fn() }; +} + +const snap: AgentSnapshot = { + url: 'https://example.com/contact', + title: 'Contact Us', + timestamp: '2026-07-04T00:00:00.000Z', + elements: [ + { ref: '@e1', role: 'heading', name: 'Contact Us', tag: 'h1', interactive: false, visible: true }, + { ref: '@e2', role: 'textbox', name: 'Email', tag: 'input', interactive: true, visible: true, value: 'a@b.com' }, + { ref: '@e3', role: 'link', name: 'More', tag: 'a', interactive: true, visible: true, href: 'https://x/y' }, + ], +}; + +describe('captureSnapshot', () => { + it('returns the page-provided snapshot value', async () => { + const result = await captureSnapshot(fakeConn(snap)); + expect(result.title).toBe('Contact Us'); + expect(result.elements).toHaveLength(3); + }); + + it('throws when the page evaluation raises', async () => { + await expect(captureSnapshot(fakeConn(null, { exception: 'boom' }))).rejects.toThrow( + /Page evaluation failed: boom/, + ); + }); +}); + +describe('ref actions', () => { + it('clickRef returns the action result', async () => { + const res = await clickRef(fakeConn({ ok: true, ref: '@e3' }), '@e3'); + expect(res.ok).toBe(true); + }); + + it('clickRef throws StaleRefError when the ref is gone', async () => { + await expect( + clickRef(fakeConn({ ok: false, error: 'STALE_REF', ref: '@e9' }), '@e9'), + ).rejects.toBeInstanceOf(StaleRefError); + }); + + it('fillRef throws StaleRefError when the ref is gone', async () => { + const err = await fillRef( + fakeConn({ ok: false, error: 'STALE_REF', ref: '@e9' }), + '@e9', + 'x', + ).catch((e) => e); + expect(err).toBeInstanceOf(StaleRefError); + expect(err.recoverable).toBe(true); + expect(err.code).toBe('STALE_REF'); + }); + + it('rejects a malformed ref before touching the page', async () => { + await expect(clickRef(fakeConn({}), 'not-a-ref')).rejects.toThrow(/snapshot ref/); + }); +}); + +describe('formatSnapshotText', () => { + it('renders compact ref lines with value and href hints', () => { + const text = formatSnapshotText(snap); + expect(text).toContain('Page: Contact Us'); + expect(text).toContain('URL: https://example.com/contact'); + expect(text).toContain('@e1 heading "Contact Us"'); + expect(text).toContain('@e2 textbox "Email" = "a@b.com"'); + expect(text).toContain('@e3 link "More" -> https://x/y'); + }); + + it('notes when there are no interactive elements', () => { + const text = formatSnapshotText({ ...snap, elements: [] }); + expect(text).toContain('(no interactive elements)'); + }); +}); diff --git a/packages/browser-core/src/automation/page.ts b/packages/browser-core/src/automation/page.ts new file mode 100644 index 0000000..340f972 --- /dev/null +++ b/packages/browser-core/src/automation/page.ts @@ -0,0 +1,106 @@ +/** + * Page-level automation over a CDP connection (PRD M3.2): evaluate the snapshot + * and ref-action scripts, parse their results, and surface a recoverable + * STALE_REF error when a ref no longer resolves. + */ +import type { CdpConnection } from './cdp-client.js'; +import { + clickExpression, + fillExpression, + normalizeRef, + type ActionResult, +} from './action-script.js'; +import { + snapshotExpression, + type AgentSnapshot, + type SnapshotElement, + type SnapshotOptions, +} from './snapshot-script.js'; + +/** A ref no longer resolves in the page; the caller should re-snapshot. */ +export class StaleRefError extends Error { + readonly ref: string; + readonly code = 'STALE_REF' as const; + readonly recoverable = true; + constructor(ref: string) { + super( + `Ref ${ref} not found on the page — it may be stale. Run \`tron snapshot\` and use a current ref.`, + ); + this.name = 'StaleRefError'; + this.ref = ref; + } +} + +interface EvalResult { + result?: { value?: unknown }; + exceptionDetails?: { exception?: { description?: string }; text?: string }; +} + +/** Evaluate an expression in the page and return its by-value result. */ +async function evaluate(conn: CdpConnection, expression: string): Promise { + const res = await conn.send('Runtime.evaluate', { + expression, + returnByValue: true, + awaitPromise: true, + }); + if (res.exceptionDetails) { + const detail = + res.exceptionDetails.exception?.description ?? + res.exceptionDetails.text ?? + 'evaluation failed'; + throw new Error(`Page evaluation failed: ${detail}`); + } + return res.result?.value as T; +} + +/** Enable the CDP Runtime domain (idempotent) before evaluating. */ +export async function enableRuntime(conn: CdpConnection): Promise { + await conn.send('Runtime.enable'); +} + +/** Capture a structured, ref-tagged snapshot of the current page. */ +export async function captureSnapshot( + conn: CdpConnection, + options: SnapshotOptions = {}, +): Promise { + return evaluate(conn, snapshotExpression(options)); +} + +/** Click the element referenced by `ref` (throws StaleRefError if gone). */ +export async function clickRef(conn: CdpConnection, ref: string): Promise { + const result = await evaluate(conn, clickExpression(ref)); + if (!result.ok && result.error === 'STALE_REF') throw new StaleRefError(`@${normalizeRef(ref)}`); + return result; +} + +/** Fill the element referenced by `ref` with `value` (throws StaleRefError if gone). */ +export async function fillRef( + conn: CdpConnection, + ref: string, + value: string, +): Promise { + const result = await evaluate(conn, fillExpression(ref, value)); + if (!result.ok && result.error === 'STALE_REF') throw new StaleRefError(`@${normalizeRef(ref)}`); + return result; +} + +/** Render a snapshot as compact text (the default `tron snapshot` output). */ +export function formatSnapshotText(snapshot: AgentSnapshot): string { + const lines: string[] = [ + `Page: ${snapshot.title || '(untitled)'}`, + `URL: ${snapshot.url}`, + '', + ]; + for (const el of snapshot.elements) { + lines.push(formatElementLine(el)); + } + if (snapshot.elements.length === 0) lines.push('(no interactive elements)'); + return lines.join('\n'); +} + +function formatElementLine(el: SnapshotElement): string { + let line = `${el.ref} ${el.role} ${JSON.stringify(el.name)}`; + if (el.value !== undefined && el.value !== '') line += ` = ${JSON.stringify(el.value)}`; + if (el.href) line += ` -> ${el.href}`; + return line; +} diff --git a/packages/browser-core/src/automation/snapshot-script.test.ts b/packages/browser-core/src/automation/snapshot-script.test.ts new file mode 100644 index 0000000..dbfde35 --- /dev/null +++ b/packages/browser-core/src/automation/snapshot-script.test.ts @@ -0,0 +1,119 @@ +// @vitest-environment happy-dom +import { beforeEach, describe, expect, it } from 'vitest'; +import { snapshotExpression, type AgentSnapshot } from './snapshot-script.js'; +import { clickExpression, fillExpression, type ActionResult } from './action-script.js'; + +// happy-dom does no layout, so getBoundingClientRect() is all zeros. The snapshot +// script uses a non-zero box as a visibility signal; give visible elements one so +// the display/hidden/visibility filters (which happy-dom does honor) are what's +// under test. +function run(expr: string): T { + return new Function('return ' + expr)() as T; +} + +beforeEach(() => { + document.head.innerHTML = ''; + document.body.innerHTML = ''; + Element.prototype.getBoundingClientRect = function () { + return { width: 120, height: 20, top: 0, left: 0, right: 120, bottom: 20, x: 0, y: 0, toJSON() {} }; + } as typeof Element.prototype.getBoundingClientRect; +}); + +describe('snapshotExpression', () => { + it('tags interactive + heading elements with refs in document order', () => { + document.body.innerHTML = ` +

Contact Us

+
+ + + + More information + + +
`; + const snap = run(snapshotExpression()); + + expect(snap.title).toBe(document.title); + const byRole = Object.fromEntries(snap.elements.map((e) => [e.name, e])); + expect(snap.elements.map((e) => e.ref)).toEqual(['@e1', '@e2', '@e3', '@e4', '@e5', '@e6']); + expect(byRole['Contact Us'].role).toBe('heading'); + expect(byRole['Name'].role).toBe('textbox'); + expect(byRole['Email'].value).toBe('a@b.com'); + expect(byRole['Message'].role).toBe('textbox'); + expect(byRole['More information'].role).toBe('link'); + expect(byRole['More information'].href).toContain('example.com/more'); + expect(byRole['Submit'].role).toBe('button'); + // The hidden input has no layout role here and type=hidden is excluded. + expect(snap.elements.some((e) => e.name === 'csrf')).toBe(false); + }); + + it('writes data-tron-ref attributes so later actions can resolve refs', () => { + document.body.innerHTML = ``; + run(snapshotExpression()); + expect(document.querySelector('[data-tron-ref="e1"]')?.textContent).toBe('Go'); + }); + + it('redacts password values', () => { + document.body.innerHTML = ``; + const snap = run(snapshotExpression()); + expect(snap.elements[0].value).not.toContain('hunter2'); + }); + + it('excludes display:none and [hidden] elements by default', () => { + document.body.innerHTML = ` + + + `; + const snap = run(snapshotExpression()); + expect(snap.elements.map((e) => e.name)).toEqual(['Yes']); + }); + + it('includes hidden elements when asked', () => { + document.body.innerHTML = ``; + const snap = run(snapshotExpression({ includeHidden: true })); + expect(snap.elements.map((e) => e.name)).toEqual(['Nope']); + expect(snap.elements[0].visible).toBe(false); + }); + + it('reports the focused ref', () => { + document.body.innerHTML = ``; + (document.getElementById('b') as HTMLInputElement).focus(); + const snap = run(snapshotExpression()); + expect(snap.focusedRef).toBe('@e2'); + }); +}); + +describe('action expressions', () => { + it('clicks the referenced element', () => { + document.body.innerHTML = ``; + run(snapshotExpression()); + let clicked = false; + document.querySelector('button')!.addEventListener('click', () => { + clicked = true; + }); + const res = run(clickExpression('@e1')); + expect(res.ok).toBe(true); + expect(clicked).toBe(true); + }); + + it('fills an input and dispatches input/change', () => { + document.body.innerHTML = ``; + run(snapshotExpression()); + const input = document.getElementById('x') as HTMLInputElement; + const events: string[] = []; + input.addEventListener('input', () => events.push('input')); + input.addEventListener('change', () => events.push('change')); + const res = run(fillExpression('@e1', 'hello@example.com')); + expect(res.ok).toBe(true); + expect(input.value).toBe('hello@example.com'); + expect(events).toEqual(['input', 'change']); + }); + + it('returns STALE_REF when the ref no longer resolves', () => { + document.body.innerHTML = ``; + // No snapshot taken, so no data-tron-ref exists. + const res = run(clickExpression('@e9')); + expect(res.ok).toBe(false); + expect(res.error).toBe('STALE_REF'); + }); +}); diff --git a/packages/browser-core/src/automation/snapshot-script.ts b/packages/browser-core/src/automation/snapshot-script.ts new file mode 100644 index 0000000..17a174e --- /dev/null +++ b/packages/browser-core/src/automation/snapshot-script.ts @@ -0,0 +1,189 @@ +/** + * The in-page snapshot script (PRD M3.2) and its result types. + * + * `SNAPSHOT_JS` is evaluated in the page via CDP `Runtime.evaluate`. It tags each + * surfaced element with a `data-tron-ref` attribute and returns a compact, + * LLM-friendly list. Encoding the ref in the DOM (rather than a server-side node + * map) is what lets a later `tron click @e3` — a separate process — resolve the + * ref with a plain attribute selector, and makes a vanished element a clean + * STALE_REF instead of a dangling handle. + */ + +/** DOM attribute that carries a snapshot ref (e.g. `e3` for `@e3`). */ +export const TRON_REF_ATTR = 'data-tron-ref'; + +export interface SnapshotElement { + ref: string; // "@e3" + role: string; + name: string; + tag: string; + interactive: boolean; + visible: boolean; + value?: string; + href?: string; +} + +export interface AgentSnapshot { + url: string; + title: string; + timestamp: string; + elements: SnapshotElement[]; + focusedRef?: string; +} + +export interface SnapshotOptions { + includeHidden?: boolean; +} + +/** + * Build the in-page snapshot expression. Returns an IIFE string suitable for + * `Runtime.evaluate` with `returnByValue: true`. + */ +export function snapshotExpression(options: SnapshotOptions = {}): string { + const includeHidden = options.includeHidden === true; + return `(() => { + const ATTR = ${JSON.stringify(TRON_REF_ATTR)}; + const includeHidden = ${includeHidden ? 'true' : 'false'}; + const INTERACTIVE = 'a[href], button, input:not([type=hidden]), select, textarea, ' + + '[role=button], [role=link], [role=checkbox], [role=radio], [role=tab], ' + + '[role=menuitem], [role=switch], [role=textbox], [contenteditable=""], ' + + '[contenteditable=true], summary, [tabindex]:not([tabindex="-1"])'; + const HEADING = 'h1, h2, h3, h4, h5, h6, [role=heading]'; + + for (const el of document.querySelectorAll('[' + ATTR + ']')) el.removeAttribute(ATTR); + + const isVisible = (el) => { + if (el.hasAttribute('hidden')) return false; + const st = getComputedStyle(el); + if (st.display === 'none' || st.visibility === 'hidden' || st.visibility === 'collapse') return false; + if (parseFloat(st.opacity || '1') === 0) return false; + const r = el.getBoundingClientRect(); + return r.width > 0 && r.height > 0; + }; + + const roleOf = (el) => { + const explicit = el.getAttribute('role'); + if (explicit) return explicit; + const tag = el.tagName.toLowerCase(); + if (tag === 'a') return el.hasAttribute('href') ? 'link' : 'generic'; + if (tag === 'button' || tag === 'summary') return 'button'; + if (tag === 'select') return 'combobox'; + if (tag === 'textarea') return 'textbox'; + if (/^h[1-6]$/.test(tag)) return 'heading'; + if (tag === 'input') { + const t = (el.getAttribute('type') || 'text').toLowerCase(); + if (t === 'checkbox') return 'checkbox'; + if (t === 'radio') return 'radio'; + if (t === 'button' || t === 'submit' || t === 'reset') return 'button'; + if (t === 'range') return 'slider'; + return 'textbox'; + } + return 'generic'; + }; + + const escapeId = (id) => (typeof CSS !== 'undefined' && CSS.escape ? CSS.escape(id) : id.replace(/["\\\\]/g, '\\\\$&')); + const labelFor = (el) => { + try { + if (el.id) { + const lab = document.querySelector('label[for="' + escapeId(el.id) + '"]'); + if (lab && lab.textContent) return lab.textContent.trim(); + } + } catch (_) { /* bad id selector — fall through */ } + const wrap = el.closest('label'); + if (wrap && wrap.textContent) return wrap.textContent.trim(); + return ''; + }; + + const nameOf = (el) => { + const aria = el.getAttribute('aria-label'); + if (aria) return aria.trim(); + const labelledby = el.getAttribute('aria-labelledby'); + if (labelledby) { + const parts = labelledby.split(/\\s+/).map((id) => { + const n = document.getElementById(id); + return n && n.textContent ? n.textContent.trim() : ''; + }).filter(Boolean); + if (parts.length) return parts.join(' '); + } + const lab = labelFor(el); + if (lab) return lab; + const tag = el.tagName.toLowerCase(); + if (tag === 'input' || tag === 'textarea') { + const ph = el.getAttribute('placeholder'); + if (ph) return ph.trim(); + const nm = el.getAttribute('name'); + if (nm) return nm.trim(); + } + if (tag === 'img') { + const alt = el.getAttribute('alt'); + if (alt) return alt.trim(); + } + const text = (el.textContent || '').replace(/\\s+/g, ' ').trim(); + if (text) return text.slice(0, 120); + const title = el.getAttribute('title'); + return title ? title.trim() : ''; + }; + + const valueOf = (el) => { + const tag = el.tagName.toLowerCase(); + if (tag === 'input') { + const t = (el.getAttribute('type') || 'text').toLowerCase(); + if (t === 'password') return '\\u2022\\u2022\\u2022'; // never echo secrets + if (t === 'checkbox' || t === 'radio') return el.checked ? 'checked' : 'unchecked'; + return el.value || ''; + } + if (tag === 'textarea' || tag === 'select') return el.value || ''; + return undefined; + }; + + const seen = new Set(); + const nodes = []; + const collect = (sel, interactive) => { + for (const el of document.querySelectorAll(sel)) { + if (seen.has(el)) continue; + seen.add(el); + const visible = isVisible(el); + if (!visible && !includeHidden) continue; + nodes.push({ el, interactive, visible }); + } + }; + collect(INTERACTIVE, true); + collect(HEADING, false); + + // Document order keeps refs stable and readable. + nodes.sort((a, b) => { + const p = a.el.compareDocumentPosition(b.el); + if (p & Node.DOCUMENT_POSITION_FOLLOWING) return -1; + if (p & Node.DOCUMENT_POSITION_PRECEDING) return 1; + return 0; + }); + + const active = document.activeElement; + let focusedRef; + const elements = nodes.map((n, i) => { + const ref = 'e' + (i + 1); + n.el.setAttribute(ATTR, ref); + if (n.el === active) focusedRef = '@' + ref; + const out = { + ref: '@' + ref, + role: roleOf(n.el), + name: nameOf(n.el), + tag: n.el.tagName.toLowerCase(), + interactive: n.interactive, + visible: n.visible, + }; + const v = valueOf(n.el); + if (v !== undefined) out.value = v; + if (n.el.tagName.toLowerCase() === 'a' && n.el.href) out.href = n.el.href; + return out; + }); + + return { + url: location.href, + title: document.title, + timestamp: new Date().toISOString(), + elements, + focusedRef, + }; +})()`; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e8c90e3..160a2b6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -31,7 +31,7 @@ importers: version: 8.62.0(eslint@9.39.4)(typescript@5.9.3) vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) apps/desktop: devDependencies: @@ -40,7 +40,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) apps/docs: devDependencies: @@ -49,7 +49,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) apps/extensions: devDependencies: @@ -58,7 +58,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) apps/mobile: dependencies: @@ -92,7 +92,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) apps/web: devDependencies: @@ -101,7 +101,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) packages/agent-runtime: devDependencies: @@ -110,7 +110,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) packages/ai-core: devDependencies: @@ -119,7 +119,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) packages/auth: devDependencies: @@ -128,16 +128,19 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) packages/browser-core: devDependencies: + happy-dom: + specifier: ^20.10.6 + version: 20.10.6 typescript: specifier: ^5.6.3 version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) packages/model-providers: devDependencies: @@ -146,7 +149,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) packages/payments: devDependencies: @@ -155,7 +158,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) packages/plugins: devDependencies: @@ -164,7 +167,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) packages/sdk: devDependencies: @@ -173,7 +176,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) packages/shared: devDependencies: @@ -182,7 +185,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) packages/storage: devDependencies: @@ -191,7 +194,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) packages/sync: devDependencies: @@ -200,7 +203,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) packages/ui: devDependencies: @@ -209,7 +212,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) packages/workflow-engine: devDependencies: @@ -218,7 +221,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) services/api: dependencies: @@ -252,7 +255,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) services/scheduler: devDependencies: @@ -261,7 +264,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) services/sync-server: devDependencies: @@ -270,7 +273,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) services/worker: devDependencies: @@ -279,7 +282,7 @@ importers: version: 5.9.3 vitest: specifier: ^2.1.4 - version: 2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0) + version: 2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0) packages: @@ -1379,6 +1382,9 @@ packages: '@types/react@19.2.17': resolution: {integrity: sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw==} + '@types/whatwg-mimetype@3.0.2': + resolution: {integrity: sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA==} + '@types/ws@8.18.1': resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} @@ -1653,6 +1659,10 @@ packages: buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + buffer-image-size@0.6.4: + resolution: {integrity: sha512-nEh+kZOPY1w+gcCMobZ6ETUp9WfibndnosbpwB1iJk/8Gt5ZF2bhS6+B6bPYz424KtwsR6Rflc3tCz1/ghX2dQ==} + engines: {node: '>=4.0'} + bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -1855,6 +1865,10 @@ packages: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} + entities@7.0.1: + resolution: {integrity: sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==} + engines: {node: '>=0.12'} + error-stack-parser@2.1.4: resolution: {integrity: sha512-Sk5V6wVazPhq5MhpO+AUxJn5x7XSXGl1R93Vn7i+zS15KDVxQijejNCrz8340/2bgLBjR9GtEG8ZVKONDjcqGQ==} @@ -2163,6 +2177,10 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + happy-dom@20.10.6: + resolution: {integrity: sha512-6QD0ilzDDt93tX44y8tbmZdAcdTRYDhUP+Asgi6pC8Pp5IA3cvaZGyoVN/EGtlq9ziT65iPuBBn3ASLr6hCgVw==} + engines: {node: '>=20.0.0'} + has-flag@3.0.0: resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} engines: {node: '>=4'} @@ -3299,6 +3317,10 @@ packages: whatwg-fetch@3.6.20: resolution: {integrity: sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==} + whatwg-mimetype@3.0.0: + resolution: {integrity: sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==} + engines: {node: '>=12'} + whatwg-url-minimum@0.1.2: resolution: {integrity: sha512-XPEm0XFQWNVG292lII1PrRRJl3sItrs7CettZ4ncYxuDVpLyy+NwlGyut2hXI0JswcJUxeCH+CyOJK0ZzAXD6A==} @@ -4694,6 +4716,8 @@ snapshots: dependencies: csstype: 3.2.3 + '@types/whatwg-mimetype@3.0.2': {} + '@types/ws@8.18.1': dependencies: '@types/node': 24.13.2 @@ -5043,6 +5067,10 @@ snapshots: buffer-from@1.1.2: {} + buffer-image-size@0.6.4: + dependencies: + '@types/node': 24.13.2 + bytes@3.1.2: {} cac@6.7.14: {} @@ -5237,6 +5265,8 @@ snapshots: encodeurl@2.0.0: {} + entities@7.0.1: {} + error-stack-parser@2.1.4: dependencies: stackframe: 1.3.4 @@ -5579,6 +5609,19 @@ snapshots: graceful-fs@4.2.11: {} + happy-dom@20.10.6: + dependencies: + '@types/node': 24.13.2 + '@types/whatwg-mimetype': 3.0.2 + '@types/ws': 8.18.1 + buffer-image-size: 0.6.4 + entities: 7.0.1 + whatwg-mimetype: 3.0.0 + ws: 8.21.0 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + has-flag@3.0.0: {} has-flag@4.0.0: {} @@ -6696,7 +6739,7 @@ snapshots: lightningcss: 1.32.0 terser: 5.48.0 - vitest@2.1.9(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0): + vitest@2.1.9(@types/node@24.13.2)(happy-dom@20.10.6)(lightningcss@1.32.0)(terser@5.48.0): dependencies: '@vitest/expect': 2.1.9 '@vitest/mocker': 2.1.9(vite@5.4.21(@types/node@24.13.2)(lightningcss@1.32.0)(terser@5.48.0)) @@ -6720,6 +6763,7 @@ snapshots: why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 24.13.2 + happy-dom: 20.10.6 transitivePeerDependencies: - less - lightningcss @@ -6745,6 +6789,8 @@ snapshots: whatwg-fetch@3.6.20: {} + whatwg-mimetype@3.0.0: {} + whatwg-url-minimum@0.1.2: {} which@2.0.2: