diff --git a/cli/src/commands.rs b/cli/src/commands.rs index d0e4969e..35b9f11f 100644 --- a/cli/src/commands.rs +++ b/cli/src/commands.rs @@ -1935,7 +1935,7 @@ fn parse_set(rest: &[&str], id: &str) -> Result { } fn parse_network(rest: &[&str], id: &str) -> Result { - const VALID: &[&str] = &["route", "unroute", "requests"]; + const VALID: &[&str] = &["route", "unroute", "requests", "response", "dump"]; match rest.first().copied() { Some("route") => { @@ -1957,12 +1957,69 @@ fn parse_network(rest: &[&str], id: &str) -> Result { } Some("requests") => { let clear = rest.contains(&"--clear"); + let redact = rest.contains(&"--redact"); let filter_idx = rest.iter().position(|&s| s == "--filter"); let filter = filter_idx.and_then(|i| rest.get(i + 1).copied()); + let host_idx = rest.iter().position(|&s| s == "--host"); + let host = host_idx.and_then(|i| rest.get(i + 1).copied()); + let type_idx = rest.iter().position(|&s| s == "--type"); + let rtype = type_idx.and_then(|i| rest.get(i + 1).copied()); let mut cmd = json!({ "id": id, "action": "requests", "clear": clear }); if let Some(f) = filter { cmd["filter"] = json!(f); } + if let Some(h) = host { + cmd["host"] = json!(h); + } + if let Some(t) = rtype { + cmd["type"] = json!(t); + } + if redact { + cmd["redact"] = json!(true); + } + Ok(cmd) + } + Some("response") => { + let url = rest.get(1).ok_or_else(|| ParseError::MissingArguments { + context: "network response".to_string(), + usage: "network response [--timeout ]", + })?; + let timeout_idx = rest.iter().position(|&s| s == "--timeout"); + let timeout = timeout_idx.and_then(|i| rest.get(i + 1).and_then(|s| s.parse::().ok())); + let mut cmd = json!({ "id": id, "action": "responsebody", "url": url }); + if let Some(t) = timeout { + cmd["timeout"] = json!(t); + } + Ok(cmd) + } + Some("dump") => { + let out_idx = rest.iter().position(|&s| s == "--out"); + let out_path = out_idx + .and_then(|i| rest.get(i + 1).copied()) + .ok_or_else(|| ParseError::MissingArguments { + context: "network dump".to_string(), + usage: "network dump --out [--filter ] [--host ] [--type ] [--redact]", + })?; + let redact = rest.contains(&"--redact"); + let filter_idx = rest.iter().position(|&s| s == "--filter"); + let filter = filter_idx.and_then(|i| rest.get(i + 1).copied()); + let host_idx = rest.iter().position(|&s| s == "--host"); + let host = host_idx.and_then(|i| rest.get(i + 1).copied()); + let type_idx = rest.iter().position(|&s| s == "--type"); + let rtype = type_idx.and_then(|i| rest.get(i + 1).copied()); + let mut cmd = json!({ "id": id, "action": "networkdump", "outputPath": out_path }); + if let Some(f) = filter { + cmd["filter"] = json!(f); + } + if let Some(h) = host { + cmd["host"] = json!(h); + } + if let Some(t) = rtype { + cmd["type"] = json!(t); + } + if redact { + cmd["redact"] = json!(true); + } Ok(cmd) } Some(sub) => Err(ParseError::UnknownSubcommand { @@ -1971,7 +2028,7 @@ fn parse_network(rest: &[&str], id: &str) -> Result { }), None => Err(ParseError::MissingArguments { context: "network".to_string(), - usage: "network [args...]", + usage: "network [args...]", }), } } diff --git a/src/action-policy.ts b/src/action-policy.ts index b9847d2e..aa1a9d1f 100644 --- a/src/action-policy.ts +++ b/src/action-policy.ts @@ -75,11 +75,12 @@ const ACTION_CATEGORIES: Record = { isvisible: 'get', isenabled: 'get', ischecked: 'get', - responsebody: 'get', route: 'network', unroute: 'network', requests: 'network', + responsebody: 'network', + networkdump: 'network', state_save: 'state', state_load: 'state', diff --git a/src/actions.ts b/src/actions.ts index 5b5b06f6..ce334325 100644 --- a/src/actions.ts +++ b/src/actions.ts @@ -126,6 +126,7 @@ import type { MultiSelectCommand, WaitForDownloadCommand, ResponseBodyCommand, + NetworkDumpCommand, ScreencastStartCommand, ScreencastStopCommand, InputMouseCommand, @@ -569,6 +570,8 @@ async function dispatchAction(command: Command, browser: BrowserManager): Promis return await handleWaitForDownload(command, browser); case 'responsebody': return await handleResponseBody(command, browser); + case 'networkdump': + return await handleNetworkDump(command, browser); case 'screencast_start': return await handleScreencastStart(command, browser); case 'screencast_stop': @@ -1416,7 +1419,12 @@ async function handleRequests( // Start tracking if not already browser.startRequestTracking(); - const requests = browser.getRequests(command.filter); + const requests = browser.getRequests({ + filter: command.filter, + host: command.host, + type: command.type, + redact: command.redact, + }); return successResponse(command.id, { requests }); } @@ -2478,10 +2486,31 @@ async function handleResponseBody( return successResponse(command.id, { url: response.url(), status: response.status(), + headers: response.headers(), body: parsed, }); } +async function handleNetworkDump( + command: NetworkDumpCommand, + browser: BrowserManager +): Promise { + browser.startRequestTracking(); + const requests = browser.getRequests({ + filter: command.filter, + host: command.host, + type: command.type, + redact: command.redact, + }); + const outputDir = path.dirname(command.outputPath); + mkdirSync(outputDir, { recursive: true }); + fs.writeFileSync(command.outputPath, JSON.stringify({ requests }, null, 2)); + return successResponse(command.id, { + path: command.outputPath, + count: requests.length, + }); +} + // Screencast and input injection handlers async function handleScreencastStart( diff --git a/src/browser.ts b/src/browser.ts index c32aea37..63c6365f 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -72,11 +72,14 @@ export interface ScreencastOptions { } interface TrackedRequest { + id: number; url: string; method: string; headers: Record; timestamp: number; resourceType: string; + statusCode?: number; + responseHeaders?: Record; } interface ConsoleMessage { @@ -109,6 +112,9 @@ export class BrowserManager { private activeFrame: Frame | null = null; private dialogHandler: ((dialog: Dialog) => Promise) | null = null; private trackedRequests: TrackedRequest[] = []; + private trackedRequestsById: Map = new Map(); + private requestIdCounter: number = 0; + private isTrackingRequests: boolean = false; private routes: Map Promise> = new Map(); private consoleMessages: ConsoleMessage[] = []; private pageErrors: PageError[] = []; @@ -451,26 +457,88 @@ export class BrowserManager { * Start tracking requests */ startRequestTracking(): void { - const page = this.getPage(); + if (this.isTrackingRequests) return; + this.isTrackingRequests = true; + // Attach to current page + this.attachRequestListeners(this.getPage()); + } + + private attachRequestListeners(page: Page): void { + const requestMap = new WeakMap(); page.on('request', (request: Request) => { - this.trackedRequests.push({ + const id = ++this.requestIdCounter; + requestMap.set(request, id); + const tracked: TrackedRequest = { + id, url: request.url(), method: request.method(), headers: request.headers(), timestamp: Date.now(), resourceType: request.resourceType(), - }); + }; + this.trackedRequests.push(tracked); + this.trackedRequestsById.set(id, tracked); + }); + page.on('response', (response) => { + const reqId = requestMap.get(response.request()); + if (reqId !== undefined) { + const tracked = this.trackedRequestsById.get(reqId); + if (tracked) { + tracked.statusCode = response.status(); + tracked.responseHeaders = response.headers(); + } + } }); } /** * Get tracked requests */ - getRequests(filter?: string): TrackedRequest[] { - if (filter) { - return this.trackedRequests.filter((r) => r.url.includes(filter)); + getRequests(options?: { + filter?: string; + host?: string; + type?: string; + redact?: boolean; + }): TrackedRequest[] { + let results = this.trackedRequests; + + if (options?.filter) { + const f = options.filter; + results = results.filter((r) => r.url.includes(f)); + } + if (options?.host) { + const host = options.host; + results = results.filter((r) => { + try { + return new URL(r.url).hostname.includes(host); + } catch { + return false; + } + }); } - return this.trackedRequests; + if (options?.type) { + const types = options.type.split(',').map((t) => t.trim().toLowerCase()); + results = results.filter((r) => types.includes(r.resourceType)); + } + if (options?.redact) { + const sensitiveKeys = ['authorization', 'cookie', 'set-cookie', 'x-api-key', 'x-auth-token']; + results = results.map((r) => ({ + ...r, + headers: Object.fromEntries( + Object.entries(r.headers).map(([k, v]) => + sensitiveKeys.includes(k.toLowerCase()) ? [k, '[REDACTED]'] : [k, v] + ) + ), + responseHeaders: r.responseHeaders + ? Object.fromEntries( + Object.entries(r.responseHeaders).map(([k, v]) => + sensitiveKeys.includes(k.toLowerCase()) ? [k, '[REDACTED]'] : [k, v] + ) + ) + : undefined, + })); + } + return results; } /** @@ -478,6 +546,7 @@ export class BrowserManager { */ clearRequests(): void { this.trackedRequests = []; + this.trackedRequestsById.clear(); } /** @@ -1747,6 +1816,10 @@ export class BrowserManager { } } }); + + if (this.isTrackingRequests) { + this.attachRequestListeners(page); + } } /** diff --git a/src/protocol.ts b/src/protocol.ts index e93da1d3..2d3fa455 100644 --- a/src/protocol.ts +++ b/src/protocol.ts @@ -243,6 +243,9 @@ const unrouteSchema = baseCommandSchema.extend({ const requestsSchema = baseCommandSchema.extend({ action: z.literal('requests'), filter: z.string().optional(), + host: z.string().optional(), + type: z.string().optional(), + redact: z.boolean().optional(), clear: z.boolean().optional(), }); @@ -688,6 +691,15 @@ const responseBodySchema = baseCommandSchema.extend({ timeout: z.number().positive().optional(), }); +const networkDumpSchema = baseCommandSchema.extend({ + action: z.literal('networkdump'), + outputPath: z.string().min(1), + filter: z.string().optional(), + host: z.string().optional(), + type: z.string().optional(), + redact: z.boolean().optional(), +}); + // Screencast schemas for streaming browser viewport const screencastStartSchema = baseCommandSchema.extend({ action: z.literal('screencast_start'), @@ -1050,6 +1062,7 @@ const commandSchema = z.discriminatedUnion('action', [ multiSelectSchema, waitForDownloadSchema, responseBodySchema, + networkDumpSchema, screencastStartSchema, screencastStopSchema, inputMouseSchema, diff --git a/src/types.ts b/src/types.ts index df18acf1..97031b8e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -231,7 +231,10 @@ export interface UnrouteCommand extends BaseCommand { // Request inspection export interface RequestsCommand extends BaseCommand { action: 'requests'; - filter?: string; // URL pattern to filter + filter?: string; + host?: string; + type?: string; + redact?: boolean; clear?: boolean; } @@ -491,6 +494,15 @@ export interface ResponseBodyCommand extends BaseCommand { timeout?: number; } +export interface NetworkDumpCommand extends BaseCommand { + action: 'networkdump'; + outputPath: string; + filter?: string; + host?: string; + type?: string; + redact?: boolean; +} + // Screencast commands for streaming browser viewport export interface ScreencastStartCommand extends BaseCommand { action: 'screencast_start'; @@ -1016,6 +1028,7 @@ export type Command = | MultiSelectCommand | WaitForDownloadCommand | ResponseBodyCommand + | NetworkDumpCommand | ScreencastStartCommand | ScreencastStopCommand | InputMouseCommand