Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions src/browser.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,20 +122,32 @@ describe('BrowserBridge state', () => {

it('rejects connect() while already connecting', async () => {
const bridge = new BrowserBridge();
(bridge as any)._state = 'connecting';
(bridge as unknown as { _state: string })._state = 'connecting';

await expect(bridge.connect()).rejects.toThrow('Already connecting');
});

it('rejects connect() while closing', async () => {
const bridge = new BrowserBridge();
(bridge as any)._state = 'closing';
(bridge as unknown as { _state: string })._state = 'closing';

await expect(bridge.connect()).rejects.toThrow('Session is closing');
});

it('fails fast when daemon is running but extension is disconnected', async () => {
vi.spyOn(daemonClient, 'getDaemonHealth').mockResolvedValue({ state: 'no-extension', status: { extensionConnected: false } as any });
vi.spyOn(daemonClient, 'getDaemonHealth').mockResolvedValue({
state: 'no-extension',
status: {
ok: true,
pid: 1,
uptime: 0,
extensionConnected: false,
pending: 0,
lastCliRequestTime: 0,
memoryMB: 0,
port: 0,
},
});

const bridge = new BrowserBridge();

Expand Down
6 changes: 3 additions & 3 deletions src/browser/cdp.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@ const { MockWebSocket } = vi.hoisted(() => {
class MockWebSocket {
static OPEN = 1;
readyState = 1;
private handlers = new Map<string, Array<(...args: any[]) => void>>();
private handlers = new Map<string, Array<(...args: unknown[]) => void>>();

constructor(_url: string) {
queueMicrotask(() => this.emit('open'));
}

on(event: string, handler: (...args: any[]) => void): void {
on(event: string, handler: (...args: unknown[]) => void): void {
const handlers = this.handlers.get(event) ?? [];
handlers.push(handler);
this.handlers.set(event, handlers);
Expand All @@ -22,7 +22,7 @@ const { MockWebSocket } = vi.hoisted(() => {
this.readyState = 3;
}

private emit(event: string, ...args: any[]): void {
private emit(event: string, ...args: unknown[]): void {
for (const handler of this.handlers.get(event) ?? []) {
handler(...args);
}
Expand Down
19 changes: 10 additions & 9 deletions src/browser/dom-helpers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { autoScrollJs, waitForCaptureJs, waitForSelectorJs } from './dom-helpers

describe('autoScrollJs', () => {
it('returns early without error when document.body is null', async () => {
const g = globalThis as any;
const g = globalThis as unknown as Record<string, unknown>;
const origDoc = g.document;
g.document = { body: null, documentElement: {} };
g.window = g;
Expand All @@ -26,19 +26,20 @@ describe('waitForCaptureJs', () => {
});

it('resolves "captured" when __opencli_xhr is populated before deadline', async () => {
const g = globalThis as any;
g.__opencli_xhr = [];
const g = globalThis as unknown as Record<string, unknown>;
const captured: unknown[] = [];
g.__opencli_xhr = captured;
g.window = g; // stub window for Node eval
const code = waitForCaptureJs(1000);
const promise = eval(code) as Promise<string>;
g.__opencli_xhr.push({ data: 'test' });
captured.push({ data: 'test' });
await expect(promise).resolves.toBe('captured');
delete g.__opencli_xhr;
delete g.window;
});

it('rejects when __opencli_xhr stays empty past deadline', async () => {
const g = globalThis as any;
const g = globalThis as unknown as Record<string, unknown>;
g.__opencli_xhr = [];
g.window = g;
const code = waitForCaptureJs(50); // 50ms timeout
Expand All @@ -49,7 +50,7 @@ describe('waitForCaptureJs', () => {
});

it('resolves immediately when __opencli_xhr already has data', async () => {
const g = globalThis as any;
const g = globalThis as unknown as Record<string, unknown>;
g.__opencli_xhr = [{ data: 'already here' }];
g.window = g;
const code = waitForCaptureJs(1000);
Expand All @@ -69,7 +70,7 @@ describe('waitForSelectorJs', () => {
});

it('resolves "found" immediately when selector already present', async () => {
const g = globalThis as any;
const g = globalThis as unknown as Record<string, unknown>;
const fakeEl = { tagName: 'DIV' };
g.document = { querySelector: (_: string) => fakeEl };
const code = waitForSelectorJs('[data-testid="primaryColumn"]', 1000);
Expand All @@ -78,7 +79,7 @@ describe('waitForSelectorJs', () => {
});

it('resolves "found" when selector appears after DOM mutation', async () => {
const g = globalThis as any;
const g = globalThis as unknown as Record<string, unknown>;
let mutationCallback!: () => void;
g.MutationObserver = class {
constructor(cb: () => void) { mutationCallback = cb; }
Expand All @@ -99,7 +100,7 @@ describe('waitForSelectorJs', () => {
});

it('rejects when selector never appears within timeout', async () => {
const g = globalThis as any;
const g = globalThis as unknown as Record<string, unknown>;
g.MutationObserver = class {
constructor(_cb: () => void) {}
observe() {}
Expand Down
13 changes: 11 additions & 2 deletions src/cascade.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,14 @@ import { Strategy } from './registry.js';
import type { IPage } from './types.js';
import { getErrorMessage } from './errors.js';

/** Shape returned by the in-page fetch probe JS (see buildFetchProbeJs). */
interface FetchProbeResponse {
ok?: boolean;
status?: number;
hasData?: boolean;
preview?: string;
}

/** Strategy cascade order (simplest → most complex) */
const CASCADE_ORDER: Strategy[] = [
Strategy.PUBLIC,
Expand Down Expand Up @@ -103,12 +111,13 @@ export async function probeEndpoint(
try {
const opts = PROBE_OPTIONS[strategy];
if (opts) {
const resp = await page.evaluate(buildFetchProbeJs(url, opts));
const resp = (await page.evaluate(buildFetchProbeJs(url, opts))) as FetchProbeResponse | undefined;
result.statusCode = resp?.status;
result.success = resp?.ok && resp?.hasData;
result.success = !!(resp?.ok && resp?.hasData);
result.hasData = resp?.hasData;
result.responsePreview = resp?.preview;
} else {
// INTERCEPT / UI require site-specific implementation.
result.error = `Strategy ${strategy} requires site-specific implementation`;
}
} catch (err) {
Expand Down
10 changes: 6 additions & 4 deletions src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,7 @@ export function createProgram(BUILTIN_CLIS: string, USER_CLIS: string): Command
const page = await getBrowserPage();
await fn(page, ...args);
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
const msg = getErrorMessage(err);
if (msg.includes('Extension not connected') || msg.includes('Daemon')) {
console.error(`Browser not connected. Run 'opencli doctor' to diagnose.`);
} else if (msg.includes('attach failed') || msg.includes('chrome-extension://')) {
Expand Down Expand Up @@ -693,10 +693,12 @@ cli({
console.log(` Executing: opencli ${site} ${command}${limitFlag}\n`);
console.log(output);
console.log(`\n ✓ Adapter works!`);
} catch (err: any) {
} catch (err) {
console.log(` Executing: opencli ${site} ${command}${limitFlag}\n`);
if (err.stdout) console.log(err.stdout);
if (err.stderr) console.error(err.stderr.slice(0, 500));
// execFileSync attaches captured stdout/stderr on its thrown Error.
const execErr = err as { stdout?: string | Buffer; stderr?: string | Buffer };
if (execErr.stdout) console.log(String(execErr.stdout));
if (execErr.stderr) console.error(String(execErr.stderr).slice(0, 500));
console.log(`\n ✗ Adapter failed. Fix the code and try again.`);
process.exitCode = EXIT_CODES.GENERIC_ERROR;
}
Expand Down
2 changes: 1 addition & 1 deletion src/download/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,7 @@ export async function saveDocument(
content: string,
destPath: string,
format: 'json' | 'markdown' | 'html' | 'text' = 'markdown',
metadata?: Record<string, any>,
metadata?: Record<string, unknown>,
): Promise<{ success: boolean; size: number; error?: string }> {
try {
const dir = path.dirname(destPath);
Expand Down
6 changes: 3 additions & 3 deletions src/engine.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,13 @@ cli({
});
`);

delete (globalThis as any).__opencli_helper_loaded__;
delete (globalThis as { __opencli_helper_loaded__?: unknown }).__opencli_helper_loaded__;
await discoverClis(tempRoot);

expect((globalThis as any).__opencli_helper_loaded__).toBeUndefined();
expect((globalThis as { __opencli_helper_loaded__?: unknown }).__opencli_helper_loaded__).toBeUndefined();
expect(getRegistry().get('temp-site/hello')).toBeDefined();
} finally {
delete (globalThis as any).__opencli_helper_loaded__;
delete (globalThis as { __opencli_helper_loaded__?: unknown }).__opencli_helper_loaded__;
await fs.promises.rm(tempRoot, { recursive: true, force: true });
}
});
Expand Down
12 changes: 6 additions & 6 deletions src/output.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,38 +21,38 @@ describe('output TTY detection', () => {
Object.defineProperty(process.stdout, 'isTTY', { value: false, writable: true });
// commanderAdapter always passes fmt:'table' as default — this must still trigger downgrade
render([{ name: 'alice', score: 10 }], { fmt: 'table', columns: ['name', 'score'] });
const out = logSpy.mock.calls.map((c: any[]) => c[0]).join('\n');
const out = logSpy.mock.calls.map((c: unknown[]) => c[0]).join('\n');
expect(out).toContain('name: alice');
expect(out).toContain('score: 10');
});

it('outputs table in TTY when format is default table', () => {
Object.defineProperty(process.stdout, 'isTTY', { value: true, writable: true });
render([{ name: 'alice', score: 10 }], { fmt: 'table', columns: ['name', 'score'] });
const out = logSpy.mock.calls.map((c: any[]) => c[0]).join('\n');
const out = logSpy.mock.calls.map((c: unknown[]) => c[0]).join('\n');
expect(out).toContain('alice');
});

it('respects explicit -f json even in non-TTY', () => {
Object.defineProperty(process.stdout, 'isTTY', { value: false, writable: true });
render([{ name: 'alice' }], { fmt: 'json' });
const out = logSpy.mock.calls.map((c: any[]) => c[0]).join('\n');
const out = logSpy.mock.calls.map((c: unknown[]) => c[0]).join('\n');
expect(JSON.parse(out)).toEqual([{ name: 'alice' }]);
});

it('OUTPUT env var overrides default table in non-TTY', () => {
Object.defineProperty(process.stdout, 'isTTY', { value: false, writable: true });
process.env.OUTPUT = 'json';
render([{ name: 'alice' }], { fmt: 'table' });
const out = logSpy.mock.calls.map((c: any[]) => c[0]).join('\n');
const out = logSpy.mock.calls.map((c: unknown[]) => c[0]).join('\n');
expect(JSON.parse(out)).toEqual([{ name: 'alice' }]);
});

it('explicit -f flag takes precedence over OUTPUT env var', () => {
Object.defineProperty(process.stdout, 'isTTY', { value: false, writable: true });
process.env.OUTPUT = 'json';
render([{ name: 'alice' }], { fmt: 'csv', fmtExplicit: true });
const out = logSpy.mock.calls.map((c: any[]) => c[0]).join('\n');
const out = logSpy.mock.calls.map((c: unknown[]) => c[0]).join('\n');
expect(out).toContain('name');
expect(out).toContain('alice');
expect(out).not.toContain('"name"'); // not JSON
Expand All @@ -61,7 +61,7 @@ describe('output TTY detection', () => {
it('explicit -f table overrides non-TTY auto-downgrade', () => {
Object.defineProperty(process.stdout, 'isTTY', { value: false, writable: true });
render([{ name: 'alice' }], { fmt: 'table', fmtExplicit: true, columns: ['name'] });
const out = logSpy.mock.calls.map((c: any[]) => c[0]).join('\n');
const out = logSpy.mock.calls.map((c: unknown[]) => c[0]).join('\n');
// Should be table output, not YAML
expect(out).not.toContain('name: alice');
expect(out).toContain('alice');
Expand Down
2 changes: 1 addition & 1 deletion src/pipeline/executor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ describe('executePipeline', () => {
});

it('skips null/invalid steps', async () => {
const result = await executePipeline(null, [null, undefined, 42] as any);
const result = await executePipeline(null, [null, undefined, 42]);
expect(result).toBeNull();
});

Expand Down
62 changes: 43 additions & 19 deletions src/pipeline/steps/download.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,32 +101,55 @@ function dedupeCookies(
* type: auto
* ```
*/
interface DownloadParams {
url?: string;
dir?: string;
filename?: string;
concurrency?: number;
skip_existing?: boolean;
timeout?: number;
use_ytdlp?: boolean;
ytdlp_args?: unknown;
type?: string;
progress?: boolean;
content?: string;
metadata?: Record<string, unknown>;
}

export async function stepDownload(
page: IPage | null,
params: any,
data: any,
args: Record<string, any>,
): Promise<any> {
params: unknown,
data: unknown,
args: Record<string, unknown>,
): Promise<unknown> {
// Parse parameters with defaults
const urlTemplate = typeof params === 'string' ? params : (params?.url ?? '');
const dirTemplate = params?.dir ?? './downloads';
const filenameTemplate = params?.filename ?? '';
const concurrency = typeof params?.concurrency === 'number' ? params.concurrency : 3;
const skipExisting = params?.skip_existing !== false;
const timeout = typeof params?.timeout === 'number' ? params.timeout * 1000 : 30000;
const useYtdlp = params?.use_ytdlp ?? false;
const ytdlpArgs = Array.isArray(params?.ytdlp_args) ? params.ytdlp_args : [];
const contentType = params?.type ?? 'auto';
const showProgress = params?.progress !== false;
const contentTemplate = params?.content;
const metadataTemplate = params?.metadata;
const p: DownloadParams =
typeof params === 'object' && params !== null ? (params as DownloadParams) : {};
const urlTemplate = typeof params === 'string' ? params : (p.url ?? '');
const dirTemplate = p.dir ?? './downloads';
const filenameTemplate = p.filename ?? '';
const concurrency = typeof p.concurrency === 'number' ? p.concurrency : 3;
const skipExisting = p.skip_existing !== false;
const timeout = typeof p.timeout === 'number' ? p.timeout * 1000 : 30000;
const useYtdlp = p.use_ytdlp ?? false;
const ytdlpArgs: string[] = Array.isArray(p.ytdlp_args)
? p.ytdlp_args.map((v) => String(v))
: [];
const contentType = p.type ?? 'auto';
const showProgress = p.progress !== false;
const contentTemplate = p.content;
const metadataTemplate = p.metadata;

// Resolve output directory
const dir = String(render(dirTemplate, { args, data }));
fs.mkdirSync(dir, { recursive: true });

// Normalize data to array
const items: any[] = Array.isArray(data) ? data : data ? [data] : [];
// Normalize data to array. Items are row records (string-keyed) produced by
// upstream steps; we treat them as Record<string, unknown> and narrow per-use.
const items: Array<Record<string, unknown>> =
Array.isArray(data) ? (data as Array<Record<string, unknown>>)
: data ? [data as Record<string, unknown>]
: [];
if (items.length === 0) {
return [];
}
Expand Down Expand Up @@ -171,7 +194,8 @@ export async function stepDownload(
}

// Process downloads with concurrency
const results = await mapConcurrent(items, concurrency, async (item, index): Promise<any> => {
type DownloadedItem = Record<string, unknown> & { _download: DownloadResult };
const results = await mapConcurrent(items, concurrency, async (item, index): Promise<DownloadedItem> => {
const startTime = Date.now();

// Render URL
Expand Down
22 changes: 17 additions & 5 deletions src/pipeline/steps/intercept.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,20 @@
import type { IPage } from '../../types.js';
import { render, normalizeEvaluateSource } from '../template.js';

export async function stepIntercept(page: IPage | null, params: any, data: any, args: Record<string, any>): Promise<any> {
const cfg = typeof params === 'object' ? params : {};
interface InterceptParams {
trigger?: string;
capture?: string;
timeout?: number;
select?: string;
}

export async function stepIntercept(
page: IPage | null,
params: unknown,
data: unknown,
args: Record<string, unknown>,
): Promise<unknown> {
const cfg: InterceptParams = typeof params === 'object' && params !== null ? (params as InterceptParams) : {};
const trigger = cfg.trigger ?? '';
const capturePattern = cfg.capture ?? '';
const timeout = cfg.timeout ?? 8;
Expand Down Expand Up @@ -38,14 +50,14 @@ export async function stepIntercept(page: IPage | null, params: any, data: any,
const matchingResponses = await page!.getInterceptedRequests();

// Step 5: Select from response if specified
let result = matchingResponses.length === 1 ? matchingResponses[0] :
let result: unknown = matchingResponses.length === 1 ? matchingResponses[0] :
matchingResponses.length > 1 ? matchingResponses : data;

if (selectPath && result) {
let current = result;
let current: unknown = result;
for (const part of String(selectPath).split('.')) {
if (current && typeof current === 'object' && !Array.isArray(current)) {
current = current[part];
current = (current as Record<string, unknown>)[part];
} else break;
}
result = current ?? result;
Expand Down
Loading
Loading