[scout] use project deps as global hooks for parallel tests (elastic#211409)

## Summary

Currently we are using `globalSetup` [script in configuration
file](https://playwright.dev/docs/test-global-setup-teardown#option-2-configure-globalsetup-and-globalteardown)
to ingest Elasticsearch data before running the tests in parallel
against the same ES/Kibana instances.

This approach doesn't work well when you need to adjust `globalSetup`
logic based on specific condition, e.g. configuration file defining
where servers are hosted, its credentials, etc.

Not only global hook, but `ScoutConfig` fixture expects an argument to
define where servers configuration is defined:


https://github.com/elastic/kibana/blob/cd502acea12979979497f62897be663044ade3aa/packages/kbn-scout/src/playwright/fixtures/worker/core_fixtures.ts#L65-L75

`testInfo` is how Playwright exposes currently running configuration in
a form of `project` interface:
[projects](https://playwright.dev/docs/test-projects) can be used to
group tests, e.g. for specific envs or browsers.

Unfortunately `testInfo` is not exposed in global scripts, because in
Playwright project design `globalSetup` scripts are run before multiple
projects and projects can have its own `setup` hooks via
[dependencies](https://playwright.dev/docs/test-global-setup-teardown#option-1-project-dependencies):

```
    {
      name: 'setup',
      testMatch: /global.setup\.ts/,
    },
    {
      name: 'local',
      use: { ...devices['Desktop Chrome'], configName: 'local' },
      dependencies: 'setup',
    },
``` 

We already use project API to get `serversConfigDir` path, where we plan
to store local and cloud server configurations. This PR proposes to
define projects as `local` and `cloud` (maybe even separate `cloud-mki`,
`cloud-ech`) as a way to provide playwright information about servers
configuration.

Advantages:
1. we can re-use existing fixtures as-is, without adding custom exported
helper functions for ES data ingestion
2. project dependency is displayed as `setup` in Playwright report
3. way better and simpler design for consumers:
```
import { globalSetupHook } from '@kbn/scout';

globalSetupHook('Ingest data to Elasticsearch', async ({ esArchiver, log }) => {
  // add archives to load, if needed
  const archives = [
    testData.ES_ARCHIVES.LOGSTASH,
  ];

  log.debug('[setup] loading test data (only if indexes do not exist)...');
  for (const archive of archives) {
    await esArchiver.loadIfNeeded(archive);
  }
});
```
4. it is supported by VSCode Playwright plugin
<img width="1271" alt="Screenshot 2025-02-17 at 11 26 12"
src="https://github.com/user-attachments/assets/ba7eeb38-d39d-4785-9c11-18647599ec4a"
/>

I find it extremely useful because you don't need to change env var when
you want to switch b/w local or cloud run, all the configurations are
loaded automatically and you just tick the checkbox!

Disadvantages:
1. it is important to run `playwright test` with `--project` flag to use
the proper configuration
2. we have to define how `projects` are used for local and cloud
configuration, and make sure it meets requirements of multiple teams. We
can expose a way to pass custom project definitions in
`createPlaywrightConfig` function, but it might complicate the support
effort when every Team has too many custom projects.
3. `project` term is something we can't change and might be confusing
4. Since it is a Playwright feature, we might not have consistency with
API tests runner under Scout

For reviewers: 

Playing with it locally might give a better understanding about the
pros/cons, especially with IDE playwright plugin installed.

Running servers with tests:
```
node scripts/scout.js run-tests --serverless=oblt --testTarget=local --config x-pack/platform/plugins/private/discover_enhanced/ui_tests/playwright.config.ts

node scripts/scout.js run-tests --serverless=oblt --config x-pack/platform/plugins/private/discover_enhanced/ui_tests/playwright.config.ts
```

Running test only requires passing `project` argument:

```
npx playwright test --project=local --config x-pack/platform/plugins/private/discover_enhanced/ui_tests/playwright.config.ts

npx playwright test --project=local --config x-pack/platform/plugins/private/discover_enhanced/ui_tests/parallel.playwright.config.ts
```

---------

Co-authored-by: kibanamachine <[email protected]>
Co-authored-by: jennypavlova <[email protected]>

Author: 3 people

.eslintrc.js

@@ -1471,7 +1471,8 @@ module.exports = {
         'playwright/no-slowed-test': 'error',
         'playwright/no-standalone-expect': 'error',
         'playwright/no-unsafe-references': 'error',
-        'playwright/no-wait-for-selector': 'warn',
+        'playwright/no-useless-await': 'error',
+        'playwright/no-wait-for-selector': 'error',
         'playwright/max-nested-describe': ['error', { max: 1 }],
         'playwright/missing-playwright-await': 'error',
         'playwright/prefer-comparison-matcher': 'error',

src/platform/packages/private/kbn-scout-reporting/src/helpers/cli_processing.test.ts

@@ -12,10 +12,19 @@ import { getRunTarget, stripRunCommand } from './cli_processing';
 describe('cli_processing', () => {
   describe('stripRunCommand', () => {
     it(`should return the correct run command when started with 'npx'`, () => {
-      const argv = ['npx', 'playwright', 'test', '--config', 'path/to/config', '--grep=@svlSearch'];
+      const argv = [
+        'npx',
+        'playwright',
+        'test',
+        '--config',
+        'path/to/config',
+        '--project',
+        'local',
+        '--grep=@svlSearch',
+      ];
 
       expect(stripRunCommand(argv)).toBe(
-        'npx playwright test --config path/to/config --grep=@svlSearch'
+        'npx playwright test --config path/to/config --project local --grep=@svlSearch'
       );
     });
 
@@ -26,11 +35,13 @@ describe('cli_processing', () => {
         'test',
         '--config',
         'path/to/config',
+        '--project',
+        'local',
         '--grep=@svlSearch',
       ];
 
       expect(stripRunCommand(argv)).toBe(
-        'npx playwright test --config path/to/config --grep=@svlSearch'
+        'npx playwright test --config path/to/config --project local --grep=@svlSearch'
       );
     });
 

src/platform/packages/shared/kbn-scout/README.md

@@ -186,9 +186,11 @@ If the servers are already running, you can execute tests independently using ei
 - Command Line: Use the following command to run tests:
 
 ```bash
-npx playwright test --config <plugin-path>/ui_tests/playwright.config.ts
+npx playwright test --config <plugin-path>/ui_tests/playwright.config.ts --project local
 ```
 
+We use `project` flag to define test target, where tests to be run: local servers or Elastic Cloud. Currently we only support local servers.
+
 ### Contributing
 
 We welcome contributions to improve and extend `kbn-scout`. This guide will help you get started, add new features, and align with existing project standards.

src/platform/packages/shared/kbn-scout/index.ts

@@ -12,11 +12,10 @@ export {
   expect,
   test,
   spaceTest,
+  globalSetupHook,
   tags,
   createPlaywrightConfig,
   createLazyPageObject,
-  ingestTestDataHook,
-  ingestSynthtraceDataHook,
 } from './src/playwright';
 export type {
   ScoutPlaywrightOptions,

src/platform/packages/shared/kbn-scout/src/playwright/config/create_config.test.ts

@@ -64,7 +64,8 @@ describe('createPlaywrightConfig', () => {
     expect(config.timeout).toBe(60000);
     expect(config.expect?.timeout).toBe(10000);
     expect(config.outputDir).toBe('./output/test-artifacts');
-    expect(config.projects![0].name).toEqual('chromium');
+    expect(config.projects).toHaveLength(1);
+    expect(config.projects![0].name).toEqual('local');
   });
 
   it('should return a Playwright configuration with Scout reporters', () => {
@@ -96,12 +97,17 @@ describe('createPlaywrightConfig', () => {
     ]);
   });
 
-  it(`should override 'workers' count in Playwright configuration`, () => {
+  it(`should override 'workers' count and add 'setup' project dependency`, () => {
     const testDir = './my_tests';
     const workers = 2;
 
     const config = createPlaywrightConfig({ testDir, workers });
     expect(config.workers).toBe(workers);
+
+    expect(config.projects).toHaveLength(2);
+    expect(config.projects![0].name).toEqual('setup');
+    expect(config.projects![1].name).toEqual('local');
+    expect(config.projects![1]).toHaveProperty('dependencies', ['setup']);
   });
 
   it('should generate and cache runId in process.env.TEST_RUN_ID', () => {

src/platform/packages/shared/kbn-scout/src/playwright/config/create_config.ts

@@ -27,9 +27,34 @@ export function createPlaywrightConfig(options: ScoutPlaywrightOptions): Playwri
     process.env.TEST_RUN_ID = runId;
   }
 
+  const scoutProjects: PlaywrightTestConfig<ScoutTestOptions>['projects'] = [
+    {
+      name: 'local',
+      use: { ...devices['Desktop Chrome'], configName: 'local' },
+    },
+  ];
+
+  /**
+   * For parallel tests, we need to add a setup project that runs before the tests project.
+   */
+  if (options.workers && options.workers > 1) {
+    const parentProject = scoutProjects.find((p) => p.use?.configName);
+
+    scoutProjects.unshift({
+      name: 'setup',
+      use: parentProject?.use ? { ...parentProject.use } : {},
+      testMatch: /global.setup\.ts/,
+    });
+
+    scoutProjects.forEach((project) => {
+      if (project.name !== 'setup') {
+        project.dependencies = ['setup'];
+      }
+    });
+  }
+
   return defineConfig<ScoutTestOptions>({
     testDir: options.testDir,
-    globalSetup: options.globalSetup,
     /* Run tests in files in parallel */
     fullyParallel: false,
     /* Fail the build on CI if you accidentally left test.only in the source code. */
@@ -47,6 +72,7 @@ export function createPlaywrightConfig(options: ScoutPlaywrightOptions): Playwri
     ],
     /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
     use: {
+      // 'configName' is not defined by default to enforce using '--project' flag when running the tests
       testIdAttribute: 'data-test-subj',
       serversConfigDir: SCOUT_SERVERS_ROOT,
       [VALID_CONFIG_MARKER]: true,
@@ -70,24 +96,6 @@ export function createPlaywrightConfig(options: ScoutPlaywrightOptions): Playwri
 
     outputDir: './output/test-artifacts', // For other test artifacts (screenshots, videos, traces)
 
-    /* Configure projects for major browsers */
-    projects: [
-      {
-        name: 'chromium',
-        use: { ...devices['Desktop Chrome'] },
-      },
-
-      // {
-      //   name: 'firefox',
-      //   use: { ...devices['Desktop Firefox'] },
-      // },
-    ],
-
-    /* Run your local dev server before starting the tests */
-    // webServer: {
-    //   command: 'npm run start',
-    //   url: 'http://127.0.0.1:3000',
-    //   reuseExistingServer: !process.env.CI,
-    // },
+    projects: scoutProjects,
   });
 }

src/platform/packages/shared/kbn-scout/src/playwright/fixtures/parallel_run_fixtures.ts

@@ -8,7 +8,13 @@
  */
 
 import { mergeTests } from 'playwright/test';
-import { apiFixtures, coreWorkerFixtures, scoutSpaceParallelFixture } from './worker';
+import {
+  apiFixtures,
+  coreWorkerFixtures,
+  esArchiverFixture,
+  scoutSpaceParallelFixture,
+  synthtraceFixture,
+} from './worker';
 import type {
   ApiParallelWorkerFixtures,
   EsClient,
@@ -52,3 +58,10 @@ export interface ScoutParallelWorkerFixtures extends ApiParallelWorkerFixtures {
   esClient: EsClient;
   scoutSpace: ScoutSpaceParallelFixture;
 }
+
+export const globalSetup = mergeTests(
+  coreWorkerFixtures,
+  esArchiverFixture,
+  synthtraceFixture,
+  apiFixtures
+);

src/platform/packages/shared/kbn-scout/src/playwright/fixtures/worker/core_fixtures.ts

@@ -66,10 +66,15 @@ export const coreWorkerFixtures = base.extend<
    */
   config: [
     ({ log }, use, workerInfo) => {
-      const configName = 'local';
       const projectUse = workerInfo.project.use as ScoutTestOptions;
+      if (!projectUse.configName) {
+        throw new Error(
+          `Failed to read the 'configName' property. Make sure to run tests with '--project' flag and target enviroment (local or cloud),
+          e.g. 'npx playwright test --project local --config <path_to_Playwright.config.ts>'`
+        );
+      }
       const serversConfigDir = projectUse.serversConfigDir;
-      const configInstance = createScoutConfig(serversConfigDir, configName, log);
+      const configInstance = createScoutConfig(serversConfigDir, projectUse.configName, log);
 
       use(configInstance);
     },

src/platform/packages/shared/kbn-scout/src/playwright/fixtures/worker/synthtrace.ts

@@ -8,7 +8,13 @@
  */
 
 import { Readable } from 'stream';
-import type { ApmFields, Fields, InfraDocument, OtelDocument } from '@kbn/apm-synthtrace-client';
+import type {
+  ApmFields,
+  Fields,
+  InfraDocument,
+  OtelDocument,
+  SynthtraceGenerator,
+} from '@kbn/apm-synthtrace-client';
 import Url from 'url';
 import type { SynthtraceEsClient } from '@kbn/apm-synthtrace/src/lib/shared/base_client';
 import {
@@ -17,10 +23,9 @@ import {
   getOtelSynthtraceEsClient,
 } from '../../../common/services/synthtrace';
 import { coreWorkerFixtures } from './core_fixtures';
-import type { SynthtraceEvents } from '../../global_hooks/synthtrace_ingestion';
 
 interface SynthtraceFixtureEsClient<TFields extends Fields> {
-  index: (events: SynthtraceEvents<TFields>) => Promise<void>;
+  index: (events: SynthtraceGenerator<TFields>) => Promise<void>;
   clean: SynthtraceEsClient<TFields>['clean'];
 }
 
@@ -34,15 +39,12 @@ const useSynthtraceClient = async <TFields extends Fields>(
   client: SynthtraceEsClient<TFields>,
   use: (client: SynthtraceFixtureEsClient<TFields>) => Promise<void>
 ) => {
-  const index = async (events: SynthtraceEvents<TFields>) =>
+  const index = async (events: SynthtraceGenerator<TFields>) =>
     await client.index(Readable.from(Array.from(events).flatMap((event) => event.serialize())));
 
   const clean = async () => await client.clean();
 
   await use({ index, clean });
-
-  // cleanup function after all tests have ran
-  await client.clean();
 };
 
 export const synthtraceFixture = coreWorkerFixtures.extend<{}, SynthtraceFixture>({

src/platform/packages/shared/kbn-scout/src/playwright/global_hooks/synthtrace_ingestion.ts

@@ -65,6 +65,9 @@ const getSynthtraceClient = (
   }
 };
 
+/**
+ * @deprecated Use `globalSetupHook` and synthtrace fixtures instead
+ */
 export async function ingestSynthtraceDataHook(config: FullConfig, data: SynthtraceIngestionData) {
   const log = getLogger();
 

src/platform/packages/shared/kbn-scout/src/playwright/index.ts

@@ -7,14 +7,16 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import { scoutFixtures, scoutParallelFixtures } from './fixtures';
+import { scoutFixtures, scoutParallelFixtures, globalSetup } from './fixtures';
 
 // Scout core fixtures: worker & test scope
 export const test = scoutFixtures;
 
 // Scout core 'space aware' fixtures: worker & test scope
 export const spaceTest = scoutParallelFixtures;
 
+export const globalSetupHook = globalSetup;
+
 export { createPlaywrightConfig } from './config';
 export { createLazyPageObject } from './page_objects/utils';
 export { expect } from './expect';

src/platform/packages/shared/kbn-scout/src/playwright/runner/flags.test.ts

@@ -72,6 +72,7 @@ describe('parseTestFlags', () => {
     expect(result).toEqual({
       mode: 'serverless=oblt',
       configPath: '/path/to/config',
+      testTarget: 'local',
       headed: false,
       esFrom: undefined,
       installDir: undefined,
@@ -82,6 +83,7 @@ describe('parseTestFlags', () => {
   it(`should parse with correct config and stateful flags`, async () => {
     const flags = new FlagsReader({
       config: '/path/to/config',
+      testTarget: 'local',
       stateful: true,
       logToFile: false,
       headed: true,
@@ -93,10 +95,41 @@ describe('parseTestFlags', () => {
     expect(result).toEqual({
       mode: 'stateful',
       configPath: '/path/to/config',
+      testTarget: 'local',
       headed: true,
       esFrom: 'snapshot',
       installDir: undefined,
       logsDir: undefined,
     });
   });
+
+  it(`should throw an error with incorrect '--testTarget' flag`, async () => {
+    const flags = new FlagsReader({
+      config: '/path/to/config',
+      testTarget: 'a',
+      stateful: true,
+      logToFile: false,
+      headed: true,
+      esFrom: 'snapshot',
+    });
+
+    await expect(parseTestFlags(flags)).rejects.toThrow(
+      'invalid --testTarget, expected one of "local", "cloud"'
+    );
+  });
+
+  it(`should throw an error with incorrect '--testTarget' flag set to 'cloud'`, async () => {
+    const flags = new FlagsReader({
+      config: '/path/to/config',
+      testTarget: 'cloud',
+      stateful: true,
+      logToFile: false,
+      headed: true,
+      esFrom: 'snapshot',
+    });
+    validatePlaywrightConfigMock.mockResolvedValueOnce();
+    await expect(parseTestFlags(flags)).rejects.toThrow(
+      'Running tests against Cloud / MKI is not supported yet'
+    );
+  });
 });

src/platform/packages/shared/kbn-scout/src/playwright/runner/flags.ts

@@ -19,6 +19,7 @@ export interface RunTestsOptions {
   configPath: string;
   headed: boolean;
   mode: CliSupportedServerModes;
+  testTarget: 'local' | 'cloud';
   esFrom: 'serverless' | 'source' | 'snapshot' | undefined;
   installDir: string | undefined;
   logsDir: string | undefined;
@@ -27,18 +28,24 @@ export interface RunTestsOptions {
 export const TEST_FLAG_OPTIONS: FlagOptions = {
   ...SERVER_FLAG_OPTIONS,
   boolean: [...(SERVER_FLAG_OPTIONS.boolean || []), 'headed'],
-  string: [...(SERVER_FLAG_OPTIONS.string || []), 'config'],
-  default: { headed: false },
+  string: [...(SERVER_FLAG_OPTIONS.string || []), 'config', 'testTarget'],
+  default: { headed: false, testTarget: 'local' },
   help: `${SERVER_FLAG_OPTIONS.help}
     --config             Playwright config file path
     --headed             Run Playwright with browser head
+    --testTarget         Run tests agaist locally started servers or Cloud deployment / MKI project
   `,
 };
 
 export async function parseTestFlags(flags: FlagsReader) {
   const options = parseServerFlags(flags);
   const configPath = flags.string('config');
   const headed = flags.boolean('headed');
+  const testTarget = flags.enum('testTarget', ['local', 'cloud']) || 'local';
+
+  if (testTarget === 'cloud') {
+    throw createFlagError(`Running tests against Cloud / MKI is not supported yet`);
+  }
 
   if (!configPath) {
     throw createFlagError(`Path to playwright config is required: --config <file path>`);
@@ -51,5 +58,6 @@ export async function parseTestFlags(flags: FlagsReader) {
     ...options,
     configPath,
     headed,
+    testTarget,
   };
 }