Merge pull request #90 from ulixee/emulators

Browser Emulators Clarification

.eslintrc.js

@@ -124,7 +124,7 @@ module.exports = {
       'error',
       {
         devDependencies: [
-          '**/test/**',
+          '**/**/test/**',
           '**/examples/**',
           '**/scripts/**',
           '**/data-scripts/**',

commons/Logger.ts

@@ -1,4 +1,6 @@
 // eslint-disable-next-line max-classes-per-file
+import ILog, { ILogData } from "@secret-agent/core-interfaces/ILog";
+
 let logId = 0;
 class Log implements ILog {
   public readonly level: string = process.env.DEBUG ? 'stats' : 'warn';
@@ -153,24 +155,6 @@ interface ILogBuilder {
   log: ILog;
 }
 
-export interface ILog extends IBoundLog<ILogData> {
-  level: string;
-  flush();
-}
-
-export interface IBoundLog<Base = any> {
-  stats<T extends Base>(action: string, data?: T): number;
-  info<T extends Base>(action: string, data?: T): number;
-  warn<T extends Base>(action: string, data?: T): number;
-  error<T extends Base>(action: string, data?: T): number;
-  createChild(module, boundData?: any): IBoundLog;
-}
-
-interface ILogData {
-  sessionId: string;
-  parentLogId?: number;
-}
-
 function extractPathFromModule(module: NodeModule) {
   const fullPath = typeof module === 'string' ? module : module.filename || module.id || '';
   return fullPath.replace(/^(.*)\/secret-agent\/(.*)$/, '$2');

commons/Queue.ts

@@ -1,4 +1,5 @@
-import { createPromise, IResolvablePromise } from './utils';
+import IResolvablePromise from '@secret-agent/core-interfaces/IResolvablePromise';
+import { createPromise } from './utils';
 
 type Callback<T> = (value?: any) => Promise<T>;
 

commons/eventUtils.ts

@@ -1,14 +1,10 @@
 import { EventEmitter } from 'events';
+import ITypedEventEmitter from '@secret-agent/core-interfaces/ITypedEventEmitter';
+import IRegisteredEventListener from "@secret-agent/core-interfaces/IRegisteredEventListener";
+import { IBoundLog} from "@secret-agent/core-interfaces/ILog";
 import { createPromise } from './utils';
-import ITypedEventEmitter from './interfaces/ITypedEventEmitter';
 import IPendingWaitEvent, { CanceledPromiseError } from './interfaces/IPendingWaitEvent';
-import { IBoundLog } from './Logger';
 
-export interface IRegisteredEventListener {
-  emitter: EventEmitter | ITypedEventEmitter<any>;
-  eventName: string | symbol;
-  handler: (...args: any[]) => void;
-}
 
 export function addEventListener(
   emitter: EventEmitter,

commons/interfaces/IPendingWaitEvent.ts

@@ -1,4 +1,4 @@
-import { IResolvablePromise } from '../utils';
+import IResolvablePromise from '@secret-agent/core-interfaces/IResolvablePromise';
 
 export class CanceledPromiseError extends Error {}
 

commons/utils.ts

@@ -1,3 +1,5 @@
+import IResolvablePromise from '@secret-agent/core-interfaces/IResolvablePromise';
+
 export function assert(value: unknown, message?: string, reject?) {
   if (value) return;
   const error = new Error(message);
@@ -61,11 +63,3 @@ export function createPromise<T = any>(
   (response as any).catch = response.promise.catch.bind(response.promise);
   return response;
 }
-
-export interface IResolvablePromise<T = any> {
-  isResolved: boolean;
-  promise?: Promise<T>;
-  resolve?: (value?: T | PromiseLike<T>) => void;
-  reject?: (reason?: any) => void;
-  timeout?: NodeJS.Timeout;
-}

core-interfaces/IBrowserEmulator.ts

@@ -1,16 +1,14 @@
-import IHttpRequestModifierDelegate from '@secret-agent/commons/interfaces/IHttpRequestModifierDelegate';
+import INetworkInterceptorDelegate from './INetworkInterceptorDelegate';
 import IUserProfile from './IUserProfile';
 import IUserAgent from './IUserAgent';
-import IPageOverride from './IPageOverride';
+import INewDocumentInjectedScript from './INewDocumentInjectedScript';
 
 export default interface IBrowserEmulator {
   readonly userAgent: IUserAgent;
-  canPolyfill: boolean;
-  engineExecutablePath: string;
-  engine: { browser: string; revision: string };
-  delegate: IHttpRequestModifierDelegate;
+  readonly canPolyfill: boolean;
+  readonly networkInterceptorDelegate: INetworkInterceptorDelegate;
   locale: string;
   userProfile: IUserProfile;
 
-  generatePageOverrides(): Promise<IPageOverride[]>;
+  newDocumentInjectedScripts(): Promise<INewDocumentInjectedScript[]>;
 }

core-interfaces/IBrowserEmulatorClass.ts

@@ -3,8 +3,8 @@ import IBrowserEmulator from './IBrowserEmulator';
 
 export default interface IBrowserEmulatorClass {
   id: string;
-  statcounterBrowser: string;
-  engine: { browser: string; revision: string };
+  roundRobinPercent: number;
+  engine: { browser: string; executablePath: string; revision: string };
   new (userAgent?: IUserAgent): IBrowserEmulator;
 }
 

core-interfaces/IBrowserEngine.ts

@@ -0,0 +1,5 @@
+export default interface IBrowserEngine {
+  browser: string;
+  revision: string;
+  executablePath: string;
+}

commons/interfaces/IHttpResourceLoadDetails.ts → core-interfaces/IHttpResourceLoadDetails.ts

@@ -1,7 +1,7 @@
-import ResourceType from '@secret-agent/core-interfaces/ResourceType';
-import { URL } from 'url';
-import IResourceHeaders from '@secret-agent/core-interfaces/IResourceHeaders';
-import OriginType from './OriginType';
+import { URL } from "url";
+import ResourceType from "./ResourceType";
+import IResourceHeaders from "./IResourceHeaders";
+import OriginType from "./OriginType";
 
 export default interface IHttpResourceLoadDetails {
   isSSL: boolean;

core-interfaces/ILog.ts

@@ -0,0 +1,17 @@
+export default interface ILog extends IBoundLog<ILogData> {
+  level: string;
+  flush();
+}
+
+export interface IBoundLog<Base = any> {
+  stats<T extends Base>(action: string, data?: T): number;
+  info<T extends Base>(action: string, data?: T): number;
+  warn<T extends Base>(action: string, data?: T): number;
+  error<T extends Base>(action: string, data?: T): number;
+  createChild(module, boundData?: any): IBoundLog;
+}
+
+export interface ILogData {
+  sessionId: string;
+  parentLogId?: number;
+}

core-interfaces/INavigation.ts

@@ -1,4 +1,4 @@
-import { IResolvablePromise } from '@secret-agent/commons/utils';
+import IResolvablePromise from './IResolvablePromise';
 import { IPipelineStatus } from './Location';
 
 export default interface INavigation {

core-interfaces/INetworkInterceptorDelegate.ts

@@ -0,0 +1,40 @@
+import { ConnectionOptions } from 'tls';
+import ResourceType from './ResourceType';
+import IResourceHeaders from './IResourceHeaders';
+import OriginType from './OriginType';
+import IHttpResourceLoadDetails from './IHttpResourceLoadDetails';
+
+export default interface INetworkInterceptorDelegate {
+  tcp?: { windowSize: number; ttl: number };
+  tls?: {
+    emulatorProfileId: string;
+  };
+  dns?: {
+    dnsOverTlsConnection: ConnectionOptions;
+  };
+  connections?: {
+    socketsPerOrigin: number;
+  };
+  http: {
+    requestHeaders?: (request: IResourceToModify) => { [key: string]: string };
+    cookieHeader?: (resource: IHttpResourceLoadDetails) => Promise<string>;
+    onSetCookie?: (
+      cookie: string,
+      resource: IHttpResourceLoadDetails,
+      statusCode: number,
+    ) => Promise<void>;
+    onOriginHasFirstPartyInteraction?: (documentUrl: string) => void;
+  };
+}
+
+export interface IResourceToModify {
+  isServerHttp2: boolean;
+  isClientHttp2: boolean;
+  sessionId: string;
+  resourceType: ResourceType;
+  isSSL: boolean;
+  method: string;
+  originType: OriginType;
+  lowerHeaders: IResourceHeaders;
+  headers: IResourceHeaders;
+}

core-interfaces/IPageOverride.ts → core-interfaces/INewDocumentInjectedScript.ts

@@ -1,4 +1,4 @@
-export default interface IPageOverride {
+export default interface INewDocumentInjectedScript {
   script: string;
   callback?: (json: any) => void;
   callbackWindowName?: string;

core-interfaces/IRegisteredEventListener.ts

@@ -0,0 +1,8 @@
+import { EventEmitter } from "events";
+import ITypedEventEmitter from "./ITypedEventEmitter";
+
+export default interface IRegisteredEventListener {
+  emitter: EventEmitter | ITypedEventEmitter<any>;
+  eventName: string | symbol;
+  handler: (...args: any[]) => void;
+}

core-interfaces/IResolvablePromise.ts

@@ -0,0 +1,7 @@
+export default interface IResolvablePromise<T = any> {
+  isResolved: boolean;
+  promise?: Promise<T>;
+  resolve?: (value?: T | PromiseLike<T>) => void;
+  reject?: (reason?: any) => void;
+  timeout?: NodeJS.Timeout;
+}

core-interfaces/package.json

@@ -4,6 +4,7 @@
   "description": "Core interfaces used by SecretAgent",
   "dependencies": {
     "@secret-agent/commons": "1.1.0-alpha.1",
+    "@secret-agent/puppet-interfaces": "1.1.0-alpha.1",
     "awaited-dom": "^1.1.7"
   },
   "devDependencies": {

core/lib/BrowserEmulators.ts

@@ -1,6 +1,5 @@
 import IBrowserEmulatorClass from '@secret-agent/core-interfaces/IBrowserEmulatorClass';
 import { pickRandom } from '@secret-agent/commons/utils';
-import BrowserDistribution from '@secret-agent/emulate-browsers-base/lib/BrowserDistribution';
 
 export default class BrowserEmulators {
   public static defaultEmulatorId: string;
@@ -12,13 +11,13 @@ export default class BrowserEmulators {
     if (!this.defaultEmulatorId) this.defaultEmulatorId = BrowserEmulatorClass.id;
     this.emulatorsById[BrowserEmulatorClass.id] = BrowserEmulatorClass;
 
-    const usagePct = BrowserDistribution.getBrowserConsumerUsage(BrowserEmulatorClass);
+    const usagePct = BrowserEmulatorClass.roundRobinPercent ?? 1;
     for (let i = 0; i < usagePct; i += 1) {
       this.emulatorPublicUsageDistribution.push(BrowserEmulatorClass.id);
     }
   }
 
-  public static create(browserEmulatorId: string) {
+  public static getClass(browserEmulatorId: string) {
     let BrowserEmulator = this.emulatorsById[browserEmulatorId];
     if (!BrowserEmulator) {
       const fromShortId = `@secret-agent/emulate-${browserEmulatorId}`;
@@ -35,7 +34,7 @@ export default class BrowserEmulators {
     if (!BrowserEmulator) {
       throw new Error(`BrowserEmulator could not be found: ${browserEmulatorId}`);
     }
-    return new BrowserEmulator();
+    return BrowserEmulator;
   }
 
   public static getId(emulatorId?: string) {

core/lib/DomEnv.ts

@@ -1,14 +1,15 @@
 import * as fs from 'fs';
 import { IJsPath } from 'awaited-dom/base/AwaitedPath';
 import { IRequestInit } from 'awaited-dom/base/interfaces/official';
-import Log, { IBoundLog } from '@secret-agent/commons/Logger';
+import Log  from '@secret-agent/commons/Logger';
 import Typeson from 'typeson';
 import TypesonRegistry from 'typeson-registry/dist/presets/builtin';
 import IElementRect from '@secret-agent/injected-scripts/interfaces/IElementRect';
 import IExecJsPathResult from '@secret-agent/injected-scripts/interfaces/IExecJsPathResult';
 import IAttachedState from '@secret-agent/injected-scripts/interfaces/IAttachedStateCopy';
-import { IPuppetPage } from '@secret-agent/puppet/interfaces/IPuppetPage';
+import { IPuppetPage } from '@secret-agent/puppet-interfaces/IPuppetPage';
 import injectedSourceUrl from '@secret-agent/core-interfaces/injectedSourceUrl';
+import { IBoundLog } from '@secret-agent/core-interfaces/ILog';
 import DomEnvError from './DomEnvError';
 import { Serializable } from '../interfaces/ISerializable';
 

core/lib/DomRecorder.ts

@@ -1,8 +1,9 @@
-import fs from 'fs';
-import Log from '@secret-agent/commons/Logger';
-import { PageRecorderResultSet } from '@secret-agent/injected-scripts/scripts/pageEventsRecorder';
-import { IRegisteredEventListener, removeEventListeners } from '@secret-agent/commons/eventUtils';
-import { IPuppetPage } from '@secret-agent/puppet/interfaces/IPuppetPage';
+import fs from "fs";
+import Log from "@secret-agent/commons/Logger";
+import { PageRecorderResultSet } from "@secret-agent/injected-scripts/scripts/pageEventsRecorder";
+import { removeEventListeners } from "@secret-agent/commons/eventUtils";
+import { IPuppetPage } from "@secret-agent/puppet-interfaces/IPuppetPage";
+import IRegisteredEventListener from "@secret-agent/core-interfaces/IRegisteredEventListener";
 
 const domObserver = fs.readFileSync(
   require.resolve('@secret-agent/injected-scripts/scripts/pageEventsRecorder.js'),