Skip to content

Commit 3e030fb

Browse files
ralyodioclaude
andcommitted
feat(m3.5): AI analyze — deterministic form fill + safety + bounded execute
Adds `tron analyze` and wires the SDK page.analyze/step/runTask (PRD §11). Dry-run by default; --execute runs a bounded, safety-gated fill loop. tron analyze "Fill this contact form" --data ./lead.json tron analyze "..." --data ./lead.json --execute --no-submit tron analyze form --json packages/agent-runtime/src/analyze — deterministic, no LLM: - data/matching: flatten --data to dot-paths; match label/name/placeholder/ARIA to data with synonyms + confidence (values referenced by path, never echoed). - policy: risk classification — credential/payment/PII fields never auto-fill; submit needs --allow-submit; payment/irreversible submits blocked even then; CAPTCHA stops the loop. - form-script: in-page reader (refs from the snapshot's data-tron-ref, required, type, submit). forms/planner: build a form map + safe ordered plan; report missing required data and ambiguous mappings instead of guessing. - analyze: orchestration — dry-run returns the plan; execute fills low-risk fields then stops before the gated submit (bounded by --max-steps). - analyze-cli/-bin: `tron analyze` with --data (file/inline/stdin), --execute, --no-submit, --allow-submit, --policy, --json. Open-ended navigation goals return AI_PROVIDER_NOT_CONFIGURED (deterministic form path is complete; the BYOK/local planner is a follow-up). Wiring: install.sh routes `tron analyze` via a new generic bin launcher (tron-node.mjs) that resolves agent-runtime's @tronbrowser/* imports; build- release ships agent-runtime dist as analyze/; the SDK depends on agent-runtime and implements page.analyze/step/runTask; tron-run resolver maps agent-runtime. Tests (+38): matching/policy/data units, the in-page form reader (happy-dom), the analyze orchestration (dry-run map, missing data, CAPTCHA, high-risk field never filled, execute+no-submit, allow-submit, high-risk submit refused), and the CLI with fakes. Verified `tron analyze` end-to-end (dry-run + execute) via a WS CDP stub. Full workspace suite green in CI order. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 9059aed commit 3e030fb

28 files changed

Lines changed: 1393 additions & 19 deletions
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// Generic launcher for shipped Node bins that import @tronbrowser/* packages by
2+
// bare specifier (e.g. analyze-bin -> @tronbrowser/browser-core). Registers the
3+
// resolver hook that maps those specifiers to the sibling dist trees in the
4+
// launcher payload, then runs the target entry.
5+
//
6+
// node tron-node.mjs <entry.js> [args...]
7+
import { register } from 'node:module';
8+
import { dirname, join } from 'node:path';
9+
import { fileURLToPath, pathToFileURL } from 'node:url';
10+
11+
const here = dirname(fileURLToPath(import.meta.url));
12+
register(pathToFileURL(join(here, 'tron-run-hooks.mjs')), import.meta.url, {
13+
data: {
14+
'@tronbrowser/browser-core': pathToFileURL(join(here, 'automate', 'index.js')).href,
15+
'@tronbrowser/agent-runtime': pathToFileURL(join(here, 'analyze', 'index.js')).href,
16+
'@tronbrowser/sdk': pathToFileURL(join(here, 'sdk', 'index.js')).href,
17+
},
18+
});
19+
20+
const entry = process.argv[2];
21+
if (!entry) {
22+
process.stderr.write('tron-node: missing entry\n');
23+
process.exit(2);
24+
}
25+
process.argv = [process.argv[0], entry, ...process.argv.slice(3)];
26+
await import(pathToFileURL(entry).href);

apps/desktop/launcher/tron-run.mjs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { fileURLToPath, pathToFileURL } from 'node:url';
1010
const here = dirname(fileURLToPath(import.meta.url));
1111
const sdkEntry = join(here, 'sdk', 'index.js');
1212
const coreEntry = join(here, 'automate', 'index.js');
13+
const agentEntry = join(here, 'analyze', 'index.js');
1314

1415
const argv = process.argv.slice(2);
1516
let script;
@@ -43,6 +44,7 @@ register(pathToFileURL(join(here, 'tron-run-hooks.mjs')), import.meta.url, {
4344
data: {
4445
'@tronbrowser/sdk': pathToFileURL(sdkEntry).href,
4546
'@tronbrowser/browser-core': pathToFileURL(coreEntry).href,
47+
'@tronbrowser/agent-runtime': pathToFileURL(agentEntry).href,
4648
},
4749
});
4850

apps/desktop/scripts/build-release.sh

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -61,15 +61,17 @@ stage_automation() { # dest dir
6161
local s="$1"
6262
command -v node >/dev/null 2>&1 && command -v pnpm >/dev/null 2>&1 || {
6363
echo " ! automation runtime skipped (needs node + pnpm)"; return; }
64-
if ( cd "$REPO_ROOT" && pnpm --filter @tronbrowser/browser-core --filter @tronbrowser/sdk build >/dev/null 2>&1 ); then
65-
rm -rf "$s/automate" "$s/sdk"
64+
if ( cd "$REPO_ROOT" && pnpm --filter @tronbrowser/browser-core --filter @tronbrowser/agent-runtime --filter @tronbrowser/sdk build >/dev/null 2>&1 ); then
65+
rm -rf "$s/automate" "$s/analyze" "$s/sdk"
6666
cp -R "$REPO_ROOT/packages/browser-core/dist" "$s/automate"
6767
printf '{\n "type": "module"\n}\n' > "$s/automate/package.json"
68+
cp -R "$REPO_ROOT/packages/agent-runtime/dist" "$s/analyze"
69+
printf '{\n "type": "module"\n}\n' > "$s/analyze/package.json"
6870
cp -R "$REPO_ROOT/packages/sdk/dist" "$s/sdk"
6971
printf '{\n "type": "module"\n}\n' > "$s/sdk/package.json"
70-
echo " + bundled automation runtime + SDK (tron snapshot/extract/run)"
72+
echo " + bundled automation + analyze + SDK (tron snapshot/extract/analyze/run)"
7173
else
72-
echo " ! automation runtime skipped (browser-core/sdk build failed)"
74+
echo " ! automation runtime skipped (browser-core/agent-runtime/sdk build failed)"
7375
fi
7476
}
7577

@@ -84,9 +86,11 @@ stage() { # dest dir
8486
# Managed-session engine for `tron browser …` / `tron open` (PRD M3.1). Sits
8587
# next to the shim; the `tron` dispatcher resolves it relative to $CURRENT.
8688
install -m 0755 "$DESKTOP/launcher/tron-session" "$s/tron-session"
87-
# `tron run` launcher + ESM resolver hook (PRD M3.4).
89+
# `tron run` launcher + ESM resolver hook (PRD M3.4) and the generic bin
90+
# launcher used by `tron analyze` (PRD M3.5).
8891
install -m 0644 "$DESKTOP/launcher/tron-run.mjs" "$s/tron-run.mjs"
8992
install -m 0644 "$DESKTOP/launcher/tron-run-hooks.mjs" "$s/tron-run-hooks.mjs"
93+
install -m 0644 "$DESKTOP/launcher/tron-node.mjs" "$s/tron-node.mjs"
9094
stage_automation "$s"
9195
# -L dereferences the branding symlinks (icons/logo.svg -> repo-root logo.svg)
9296
# so the package contains real files, not dangling links.

apps/web/public/install.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ Usage:
9191
tron screenshot <p> Save a PNG of the current page (--full-page)
9292
tron headless <url> One-shot: --snapshot | --screenshot <p> | --extract <mode>
9393
tron run <script> Run a JS/TS script using @tronbrowser/sdk (--headless/--trace)
94+
tron analyze [goal] Analyze/fill a form or page (--data, --execute, --json)
9495
tron upgrade Update to the latest release
9596
tron remove Uninstall TronBrowser (keeps your profile data)
9697
tron version Print the installed version
@@ -176,6 +177,14 @@ case "${1:-}" in
176177
snapshot|click|fill|type|extract|screenshot|pdf)
177178
# CDP automation on the managed session's current page (PRD M3.2/M3.3).
178179
run_automation "$@" ;;
180+
analyze)
181+
# AI-assisted unknown-interface analysis / form fill (PRD M3.5). Runs via
182+
# tron-node.mjs so agent-runtime's @tronbrowser/* imports resolve.
183+
_ld="$(dirname "$(readlink -f "$CURRENT" 2>/dev/null || echo "$CURRENT")")"
184+
ENTRY="$_ld/analyze/analyze-bin.js"
185+
command -v node >/dev/null 2>&1 || { echo "tron analyze needs Node.js (>=22) on PATH." >&2; exit 1; }
186+
[ -f "$ENTRY" ] || { echo "This TronBrowser build lacks the analyze runtime. Run: tron upgrade" >&2; exit 1; }
187+
exec node "$_ld/tron-node.mjs" "$ENTRY" "$@" ;;
179188
headless)
180189
# One-shot: launch a headless ephemeral session, navigate, act, tear down.
181190
run_automation "$@" ;;

docs/analyze.md

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# AI analyze (M3.5)
2+
3+
`tron analyze` inspects the current page of a managed session and answers "what
4+
is this asking for, and what should I do next?" — mapping forms to your data,
5+
proposing a safe plan, and (with `--execute`) filling low-risk fields under a
6+
strict safety policy. **Dry-run by default.**
7+
8+
```sh
9+
tron analyze # describe the page's forms
10+
tron analyze form --json # machine-readable form map
11+
tron analyze "Fill this contact form" --data ./lead.json
12+
tron analyze "Fill this contact form but do not submit" --data ./lead.json --execute --no-submit
13+
tron analyze "Fill and submit" --data ./lead.json --execute --allow-submit
14+
```
15+
16+
`--data` accepts a file, inline JSON, or `-` (stdin):
17+
18+
```json
19+
{ "lead": { "name": "Jane Doe", "email": "jane@example.com", "message": "Please send pricing." } }
20+
```
21+
22+
## What it does
23+
24+
- **Maps fields to data** deterministically: labels/placeholders/ARIA/`name` are
25+
matched to your data paths (synonyms like *e-mail → email*, *organization →
26+
company*) with a confidence score. Plans reference `lead.email`, never the
27+
value — so values stay out of logs and traces.
28+
- **Reports missing required data** and **ambiguous** mappings instead of
29+
guessing.
30+
- **Safety policy** (`--policy safe|auto|ask`, default `safe`):
31+
- Credential/payment/PII fields (password, card, CVV, SSN, API key…) are
32+
**never auto-filled**.
33+
- A final submit needs `--allow-submit`; `--no-submit` fills but never submits.
34+
- Payment/irreversible submits (*Pay*, *Delete account*, *Transfer*…) are
35+
**blocked even with `--allow-submit`**.
36+
- A CAPTCHA/challenge stops the loop.
37+
- **`--execute`** runs a bounded (`--max-steps`, default 8), validated loop:
38+
fill low-risk fields, then stop before the gated submit. Stale refs, missing
39+
data, ambiguity, challenges, and max-steps all end the loop cleanly.
40+
41+
## Output
42+
43+
Text by default; `--json` for a machine-readable `AnalyzeResult` (status,
44+
`detectedForms`, `plan`, `nextAction`, `missingData`, `ambiguous`, `reason`).
45+
Statuses: `planned`, `acted`, `complete`, `needs_confirmation`, `blocked`,
46+
`ambiguous`, `failed`.
47+
48+
## From the SDK
49+
50+
```ts
51+
const result = await page.analyze('Fill contact form', { data: lead }); // dry-run
52+
await page.analyze('Fill contact form', { data: lead, execute: true, noSubmit: true });
53+
```
54+
55+
`page.step()` runs one bounded step; `page.runTask()` runs the loop to
56+
completion.
57+
58+
## Scope
59+
60+
- The **form-fill path is deterministic** (no LLM) and fully covered by tests.
61+
- **Open-ended navigation** goals (e.g. "click through onboarding until the
62+
dashboard") need a configured AI provider; without one, analyze returns
63+
`AI_PROVIDER_NOT_CONFIGURED` rather than guessing. Wiring the BYOK/local
64+
provider planner is a follow-up.
65+
- Requires Node ≥22 and a running managed session (`tron browser launch`).
66+
- Logic lives in `packages/agent-runtime/src/analyze`.

packages/agent-runtime/package.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,11 @@
1818
"test": "vitest run --passWithNoTests",
1919
"lint": "eslint src"
2020
},
21+
"dependencies": {
22+
"@tronbrowser/browser-core": "workspace:*"
23+
},
2124
"devDependencies": {
25+
"happy-dom": "^20.10.6",
2226
"typescript": "^5.6.3",
2327
"vitest": "^2.1.4"
2428
}
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
/**
2+
* Executable wrapper for `tron analyze`. Built into the launcher payload as
3+
* `analyze/analyze-bin.js`; the shell dispatcher runs it via node.
4+
*/
5+
import { run } from './analyze-cli.js';
6+
7+
run(process.argv.slice(2)).then(
8+
(code) => process.exit(code),
9+
(err: unknown) => {
10+
process.stderr.write(`tron: ${err instanceof Error ? err.message : String(err)}\n`);
11+
process.exit(1);
12+
},
13+
);
Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
import { describe, expect, it, vi } from 'vitest';
2+
import { EXIT, run, type AnalyzeCliDeps } from './analyze-cli.js';
3+
import type { AnalyzeBrowser } from './analyze/analyze.js';
4+
import type { RawFormsResult } from './analyze/form-script.js';
5+
6+
const contactForm: RawFormsResult = {
7+
challenge: false,
8+
forms: [{ name: 'contact', submitRef: '@e4', submitLabel: 'Send', fields: [
9+
{ ref: '@e1', label: 'Name', name: 'name', type: 'text', role: 'input', required: true },
10+
{ ref: '@e2', label: 'Email', name: 'email', type: 'email', role: 'input', required: true },
11+
] }],
12+
};
13+
const lead = { lead: { name: 'Jane', email: 'jane@example.com' } };
14+
15+
function browser(raw: RawFormsResult, hooks: Partial<AnalyzeBrowser> = {}): AnalyzeBrowser {
16+
return {
17+
snapshot: async () => ({ url: 'https://x/contact', title: 'Contact', timestamp: 't', elements: [] }),
18+
readForms: async () => raw,
19+
fill: hooks.fill ?? (async () => {}),
20+
click: hooks.click ?? (async () => {}),
21+
};
22+
}
23+
24+
function harness(overrides: Partial<AnalyzeCliDeps> = {}, raw = contactForm) {
25+
const out: string[] = [];
26+
const err: string[] = [];
27+
const close = vi.fn();
28+
const deps: Partial<AnalyzeCliDeps> = {
29+
env: {},
30+
attach: async () => ({ browser: browser(raw), close }),
31+
readData: async () => lead,
32+
out: (t) => out.push(t),
33+
err: (t) => err.push(t),
34+
...overrides,
35+
};
36+
return { deps, out, err, close };
37+
}
38+
39+
describe('analyze CLI', () => {
40+
it('prints a JSON plan and closes the session', async () => {
41+
const { deps, out, close } = harness();
42+
const code = await run(['Fill contact form', '--data', './lead.json', '--json'], deps);
43+
expect(code).toBe(EXIT.ok);
44+
const result = JSON.parse(out.join('\n'));
45+
expect(result.status).toBe('planned');
46+
expect(result.detectedForms[0].fields[1].valueFrom).toBe('lead.email');
47+
expect(close).toHaveBeenCalled();
48+
});
49+
50+
it('treats a bare mode keyword as mode, not a goal', async () => {
51+
const { deps, out } = harness();
52+
await run(['form', '--json'], deps);
53+
expect(JSON.parse(out.join('\n')).goal).toBeUndefined();
54+
});
55+
56+
it('exits notOk (6) when required data is missing', async () => {
57+
const { deps } = harness({ readData: async () => ({ lead: { name: 'Jane' } }) });
58+
expect(await run(['Fill', '--data', 'x', '--json'], deps)).toBe(EXIT.notOk);
59+
});
60+
61+
it('exits noSession (4) when no managed session', async () => {
62+
const { deps, err } = harness({
63+
attach: async () => {
64+
const e = new Error('No managed session. Run: tron browser launch') as Error & { exit?: number };
65+
e.exit = EXIT.noSession;
66+
throw e;
67+
},
68+
});
69+
const code = await run(['form'], deps);
70+
expect(code).toBe(EXIT.noSession);
71+
expect(err.join('\n')).toContain('tron browser launch');
72+
});
73+
74+
it('executes fills but not submit with --execute --no-submit', async () => {
75+
const fill = vi.fn(async () => {});
76+
const click = vi.fn(async () => {});
77+
const { deps } = harness({ attach: async () => ({ browser: browser(contactForm, { fill, click }), close: vi.fn() }) });
78+
const code = await run(['Fill', '--data', 'x', '--execute', '--no-submit', '--json'], deps);
79+
expect(code).toBe(EXIT.ok);
80+
expect(fill).toHaveBeenCalledTimes(2);
81+
expect(click).not.toHaveBeenCalled();
82+
});
83+
84+
it('rejects an invalid --policy', async () => {
85+
const { deps } = harness();
86+
expect(await run(['form', '--policy', 'wild'], deps)).toBe(EXIT.usage);
87+
});
88+
});

0 commit comments

Comments
 (0)