Merge pull request #89 from ulixee/humanaint

chore(core)!: rename emulators/humanoids

.eslintrc.js

@@ -48,7 +48,7 @@ module.exports = {
   },
   overrides: [
     {
-      files: 'emulator-plugins/shared/test/*.ts',
+      files: 'emulate-browsers/base/test/*.ts',
       rules: {
         'no-console': 'off',
       },
@@ -83,7 +83,13 @@ module.exports = {
       },
     },
     {
-      files: ['**/install*', '**/*Install*', '**/prepare-*.js', '**/scripts/*.ts'],
+      files: [
+        '**/install*',
+        '**/*Install*',
+        '**/prepare-*.js',
+        '**/scripts/*.ts',
+        '**/data-scripts/*.ts',
+      ],
       rules: {
         'no-console': 'off',
       },
@@ -92,7 +98,7 @@ module.exports = {
   ignorePatterns: [
     '**/node_modules',
     'node_modules',
-    "**/test/assets/**",
+    '**/test/assets/**',
     'build',
     'build-dist',
     '**/build/**',
@@ -116,7 +122,15 @@ module.exports = {
     // 'import/no-default-export': 'error',
     'import/no-extraneous-dependencies': [
       'error',
-      { devDependencies: ['**/test/**', '**/examples/**', '**/scripts/**', '**/*.test.ts'] },
+      {
+        devDependencies: [
+          '**/**/test/**',
+          '**/examples/**',
+          '**/scripts/**',
+          '**/data-scripts/**',
+          '**/*.test.ts',
+        ],
+      },
     ],
     'no-use-before-define': 'off', // use typescript one
     'no-prototype-builtins': 'off',

.github/workflows/lint-and-test.yml

@@ -118,7 +118,6 @@ jobs:
         working-directory: ./build
         env:
           SA_SHOW_REPLAY: false
-          CACHE_DIR: .sessions
           NODE_ENV: test
 
       - name: Coverage

README.md

@@ -5,7 +5,7 @@ SecretAgent is a web browser that's built for scraping.
 - [x] **Built for scraping** - it's the first modern headless browsers designed specifically for scraping instead of just automated testing.
 - [x] **Designed for web developers** - We've recreated a fully compliant DOM directly in NodeJS allowing you bypass the headaches of previous scraper tools.
 - [x] **Powered by Chromium** - The powerful Chromium engine sits under the hood, allowing for lightning fast rendering.
-- [x] **Emulates any modern browser** - Emulator plugins make it easy to disguise your script as practically any browser.
+- [x] **Emulates any modern browser** - BrowserEmulators make it easy to disguise your script as practically any browser.
 - [x] **Avoids detection along the entire stack** - Don't be blocked because of TLS fingerprints in your networking stack.
 
 Check out our [website for more details](https://secretagent.dev).

client/index.ts

@@ -1,43 +1,34 @@
 // setup must go first
-import './lib/SetupAwaitedHandler';
-
-import { ILocationTrigger, LocationStatus } from '@secret-agent/core-interfaces/Location';
-import IConfigureOptions from '@secret-agent/core-interfaces/IConfigureOptions';
-import { RenderingOption } from '@secret-agent/core-interfaces/ITabOptions';
-import os from 'os';
-import initializeConstantsAndProperties from 'awaited-dom/base/initializeConstantsAndProperties';
-import { IRequestInit } from 'awaited-dom/base/interfaces/official';
-import { ISuperElement } from 'awaited-dom/base/interfaces/super';
-import IDomStorage from '@secret-agent/core-interfaces/IDomStorage';
-import ISessionOptions from '@secret-agent/core-interfaces/ISessionOptions';
-import IUserProfile from '@secret-agent/core-interfaces/IUserProfile';
-import IWaitForResourceOptions from '@secret-agent/core-interfaces/IWaitForResourceOptions';
-import IWaitForElementOptions from '@secret-agent/core-interfaces/IWaitForElementOptions';
-import StateMachine from 'awaited-dom/base/StateMachine';
-import Request from 'awaited-dom/impl/official-klasses/Request';
-import { bindFunctions } from '@secret-agent/commons/utils';
-import ICreateSessionOptions from '@secret-agent/core-interfaces/ICreateSessionOptions';
-import ICreateSecretAgentOptions from './interfaces/ICreateSecretAgentOptions';
-import CoreClient from './lib/CoreClient';
-import ISecretAgentClass, { SecretAgentStatics, ISecretAgentConfigureOptions } from "./interfaces/ISecretAgentClass";
-import ISecretAgent, { ISecretAgentEvents } from './interfaces/ISecretAgent';
-import CoreTab from './lib/CoreTab';
-import Tab, { createTab, getCoreTab } from './lib/Tab';
-import IInteractions, {
-  Command,
-  IMousePosition,
-  ITypeInteraction,
-} from './interfaces/IInteractions';
-import Interactor from './lib/Interactor';
-import IWaitForResourceFilter from './interfaces/IWaitForResourceFilter';
-import AwaitedEventTarget from './lib/AwaitedEventTarget';
-import ScriptInstance from './lib/ScriptInstance';
+import "./lib/SetupAwaitedHandler";
+
+import { ILocationTrigger, LocationStatus } from "@secret-agent/core-interfaces/Location";
+import { RenderingOption } from "@secret-agent/core-interfaces/ITabOptions";
+import initializeConstantsAndProperties from "awaited-dom/base/initializeConstantsAndProperties";
+import { IRequestInit } from "awaited-dom/base/interfaces/official";
+import { ISuperElement } from "awaited-dom/base/interfaces/super";
+import IDomStorage from "@secret-agent/core-interfaces/IDomStorage";
+import ISessionOptions from "@secret-agent/core-interfaces/ISessionOptions";
+import IUserProfile from "@secret-agent/core-interfaces/IUserProfile";
+import IWaitForResourceOptions from "@secret-agent/core-interfaces/IWaitForResourceOptions";
+import IWaitForElementOptions from "@secret-agent/core-interfaces/IWaitForElementOptions";
+import StateMachine from "awaited-dom/base/StateMachine";
+import Request from "awaited-dom/impl/official-klasses/Request";
+import { bindFunctions } from "@secret-agent/commons/utils";
+import ICreateSessionOptions from "@secret-agent/core-interfaces/ICreateSessionOptions";
+import ICreateSecretAgentOptions from "./interfaces/ICreateSecretAgentOptions";
+import CoreClient from "./lib/CoreClient";
+import ISecretAgentClass, { ISecretAgentConfigureOptions, SecretAgentStatics } from "./interfaces/ISecretAgentClass";
+import ISecretAgent, { ISecretAgentEvents } from "./interfaces/ISecretAgent";
+import CoreTab from "./lib/CoreTab";
+import Tab, { createTab, getCoreTab } from "./lib/Tab";
+import IInteractions, { Command, IMousePosition, ITypeInteraction } from "./interfaces/IInteractions";
+import Interactor from "./lib/Interactor";
+import IWaitForResourceFilter from "./interfaces/IWaitForResourceFilter";
+import AwaitedEventTarget from "./lib/AwaitedEventTarget";
+import ScriptInstance from "./lib/ScriptInstance";
 import Signals = NodeJS.Signals;
 
 const DefaultOptions = {
-  maxConcurrentSessionCount: 10,
-  localProxyPortStart: 10e3,
-  sessionsDir: os.tmpdir(),
   defaultRenderingOptions: [RenderingOption.All],
   defaultUserProfile: {},
 };
@@ -283,12 +274,12 @@ export function SecretAgentClientGenerator(
 
     public static async configure(options: Partial<ISecretAgentConfigureOptions>): Promise<void> {
       this.options = { ...DefaultOptions, ...this.options, ...options };
-      await coreClient.configure(options as IConfigureOptions);
+      await coreClient.configure(options);
     }
 
     public static async prewarm(options: Partial<ISecretAgentConfigureOptions> = {}) {
       this.options = { ...DefaultOptions, ...this.options, ...options };
-      await coreClient.prewarm(options as IConfigureOptions);
+      await coreClient.prewarm(options);
     }
 
     public static async recordUnhandledError(error: Error) {

client/interfaces/ISecretAgentClass.ts

@@ -1,13 +1,14 @@
-import IConfigureOptions from "@secret-agent/core-interfaces/IConfigureOptions";
+import ICoreConfigureOptions from "@secret-agent/core-interfaces/ICoreConfigureOptions";
 import { IRenderingOption } from "@secret-agent/core-interfaces/ITabOptions";
 import IUserProfile from "@secret-agent/core-interfaces/IUserProfile";
-import ICreateSecretAgentOptions from './ICreateSecretAgentOptions';
+import ICreateSecretAgentOptions from "./ICreateSecretAgentOptions";
 import ISecretAgent from "./ISecretAgent";
 
-export interface ISecretAgentConfigureOptions extends IConfigureOptions {
+export interface ISecretAgentConfigureOptions extends Pick<ICoreConfigureOptions, 'browserEmulatorIds'> {
   defaultRenderingOptions: IRenderingOption[];
   defaultUserProfile: IUserProfile;
 }
+
 export default interface ISecretAgentClass {
   // private options: ISecretAgentConfigureOptions;
   configure(options: Partial<ISecretAgentConfigureOptions>): Promise<void>;

client/lib/CoreClient.ts

@@ -1,8 +1,8 @@
-import ICreateSessionOptions from '@secret-agent/core-interfaces/ICreateSessionOptions';
-import IConfigureOptions from '@secret-agent/core-interfaces/IConfigureOptions';
-import ISessionMeta from '@secret-agent/core-interfaces/ISessionMeta';
-import CoreCommandQueue from './CoreCommandQueue';
-import CoreTab from './CoreTab';
+import ICreateSessionOptions from "@secret-agent/core-interfaces/ICreateSessionOptions";
+import ISessionMeta from "@secret-agent/core-interfaces/ISessionMeta";
+import CoreCommandQueue from "./CoreCommandQueue";
+import CoreTab from "./CoreTab";
+import { ISecretAgentConfigureOptions } from "../interfaces/ISecretAgentClass";
 
 export default class CoreClient {
   public readonly commandQueue: CoreCommandQueue;
@@ -17,7 +17,7 @@ export default class CoreClient {
     this.commandQueue = new CoreCommandQueue(null, this);
   }
 
-  public async configure(options: IConfigureOptions): Promise<void> {
+  public async configure(options: Partial<ISecretAgentConfigureOptions>): Promise<void> {
     await this.commandQueue.run('configure', options);
   }
 
@@ -57,7 +57,7 @@ export default class CoreClient {
     await this.commandQueue.run('logUnhandledError', error);
   }
 
-  public async prewarm(options?: IConfigureOptions): Promise<void> {
+  public async prewarm(options?: Partial<ISecretAgentConfigureOptions>): Promise<void> {
     await this.commandQueue.run('prewarm', options);
   }
 

commons/Logger.ts

@@ -1,4 +1,6 @@
 // eslint-disable-next-line max-classes-per-file
+import ILog, { ILogData } from "@secret-agent/core-interfaces/ILog";
+
 let logId = 0;
 class Log implements ILog {
   public readonly level: string = process.env.DEBUG ? 'stats' : 'warn';
@@ -153,24 +155,6 @@ interface ILogBuilder {
   log: ILog;
 }
 
-export interface ILog extends IBoundLog<ILogData> {
-  level: string;
-  flush();
-}
-
-export interface IBoundLog<Base = any> {
-  stats<T extends Base>(action: string, data?: T): number;
-  info<T extends Base>(action: string, data?: T): number;
-  warn<T extends Base>(action: string, data?: T): number;
-  error<T extends Base>(action: string, data?: T): number;
-  createChild(module, boundData?: any): IBoundLog;
-}
-
-interface ILogData {
-  sessionId: string;
-  parentLogId?: number;
-}
-
 function extractPathFromModule(module: NodeModule) {
   const fullPath = typeof module === 'string' ? module : module.filename || module.id || '';
   return fullPath.replace(/^(.*)\/secret-agent\/(.*)$/, '$2');

commons/Queue.ts

@@ -1,4 +1,5 @@
-import { createPromise, IResolvablePromise } from './utils';
+import IResolvablePromise from '@secret-agent/core-interfaces/IResolvablePromise';
+import { createPromise } from './utils';
 
 type Callback<T> = (value?: any) => Promise<T>;
 

commons/eventUtils.ts

@@ -1,14 +1,10 @@
 import { EventEmitter } from 'events';
+import ITypedEventEmitter from '@secret-agent/core-interfaces/ITypedEventEmitter';
+import IRegisteredEventListener from "@secret-agent/core-interfaces/IRegisteredEventListener";
+import { IBoundLog} from "@secret-agent/core-interfaces/ILog";
 import { createPromise } from './utils';
-import ITypedEventEmitter from './interfaces/ITypedEventEmitter';
 import IPendingWaitEvent, { CanceledPromiseError } from './interfaces/IPendingWaitEvent';
-import { IBoundLog } from './Logger';
 
-export interface IRegisteredEventListener {
-  emitter: EventEmitter | ITypedEventEmitter<any>;
-  eventName: string | symbol;
-  handler: (...args: any[]) => void;
-}
 
 export function addEventListener(
   emitter: EventEmitter,

commons/interfaces/IPendingWaitEvent.ts

@@ -1,4 +1,4 @@
-import { IResolvablePromise } from '../utils';
+import IResolvablePromise from '@secret-agent/core-interfaces/IResolvablePromise';
 
 export class CanceledPromiseError extends Error {}
 

commons/utils.ts

@@ -1,3 +1,5 @@
+import IResolvablePromise from '@secret-agent/core-interfaces/IResolvablePromise';
+
 export function assert(value: unknown, message?: string, reject?) {
   if (value) return;
   const error = new Error(message);
@@ -8,6 +10,11 @@ export function assert(value: unknown, message?: string, reject?) {
   }
 }
 
+export function pickRandom<T>(array: T[]) {
+  if (!array.length) throw new Error('Empty array provided to "pickRandom"');
+  return array[Math.floor(Math.random() * array.length)];
+}
+
 export function bindFunctions(self: any) {
   let object = self;
   do {
@@ -56,11 +63,3 @@ export function createPromise<T = any>(
   (response as any).catch = response.promise.catch.bind(response.promise);
   return response;
 }
-
-export interface IResolvablePromise<T = any> {
-  isResolved: boolean;
-  promise?: Promise<T>;
-  resolve?: (value?: T | PromiseLike<T>) => void;
-  reject?: (reason?: any) => void;
-  timeout?: NodeJS.Timeout;
-}

core-interfaces/IBrowserEmulator.ts

@@ -0,0 +1,14 @@
+import INetworkInterceptorDelegate from './INetworkInterceptorDelegate';
+import IUserProfile from './IUserProfile';
+import IUserAgent from './IUserAgent';
+import INewDocumentInjectedScript from './INewDocumentInjectedScript';
+
+export default interface IBrowserEmulator {
+  readonly userAgent: IUserAgent;
+  readonly canPolyfill: boolean;
+  readonly networkInterceptorDelegate: INetworkInterceptorDelegate;
+  locale: string;
+  userProfile: IUserProfile;
+
+  newDocumentInjectedScripts(): Promise<INewDocumentInjectedScript[]>;
+}

core-interfaces/IBrowserEmulatorClass.ts

@@ -0,0 +1,13 @@
+import IUserAgent from './IUserAgent';
+import IBrowserEmulator from './IBrowserEmulator';
+
+export default interface IBrowserEmulatorClass {
+  id: string;
+  roundRobinPercent: number;
+  engine: { browser: string; executablePath: string; revision: string };
+  new (userAgent?: IUserAgent): IBrowserEmulator;
+}
+
+// decorator for browser emulator classes. hacky way to check the class implements statics we need
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+export function BrowserEmulatorClassDecorator(constructor: IBrowserEmulatorClass) {}

core-interfaces/IBrowserEngine.ts

@@ -0,0 +1,5 @@
+export default interface IBrowserEngine {
+  browser: string;
+  revision: string;
+  executablePath: string;
+}

core-interfaces/IConfigureOptions.ts → core-interfaces/ICoreConfigureOptions.ts

@@ -1,7 +1,7 @@
-export default interface IConfigureOptions {
+export default interface ICoreConfigureOptions {
   maxConcurrentSessionsCount?: number;
   localProxyPortStart?: number;
   replayServerPort?: number;
   sessionsDir?: string;
-  activeEmulatorIds?: string[];
+  browserEmulatorIds?: string[];
 }

core-interfaces/ICreateSessionOptions.ts

@@ -1,11 +1,11 @@
-import IUserProfile from './IUserProfile';
-import ISessionOptions from './ISessionOptions';
-import IScriptInstanceMeta from './IScriptInstanceMeta';
-import IViewport from './IViewport';
+import IUserProfile from "./IUserProfile";
+import ISessionOptions from "./ISessionOptions";
+import IScriptInstanceMeta from "./IScriptInstanceMeta";
+import IViewport from "./IViewport";
 
 export default interface ICreateSessionOptions extends ISessionOptions {
   sessionName?: string;
-  emulatorId?: string;
+  browserEmulatorId?: string;
   userProfile?: IUserProfile;
   scriptInstanceMeta?: IScriptInstanceMeta;
   viewport?: IViewport;