tests/end-to-end-test-for-report-submission-using-playwright-in-ci (Netflix#2936)

wssheldon · web-flow · commit 0a81df77bc0f · 2023-02-03T11:40:23.000-08:00
* initial work to create a e2e test for report submission on playwright

* add gh action CI job and make login util more robust for CI

* update playwright github actions workflow

* make envs global

* invalid credentials when connecting to database

* still having issues with cross-env dep, debug, and try to force install

* debug node:events:491 error, known previous issue, attempt mentioned solution

* debug node:events:491 error, known previous issue, attempt mentioned solution

* run commands as root

* more node_modules debugging

* remove potentially redundant actions

* revert to hacky workflow, that worked 🐶

* add artifact upload

* update report-submission test and add more inc types to sample data

* don't need to checkout repo twice

* move node setup to beginning

* attempt to resolve flaky chromium with reuseExistingServer arg

* merge root package.json into static dir

* clean up npm install

* try removing the node_modules hack now that things are cleaned up

* forgot to add package.json

* drop global cross-envs install

* add debugging ls commands

* add -D flag to npm ci install

* do what npm wants, install with npm install :(

* debug flaky chrome test and remove npm ls debug line
diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml
@@ -0,0 +1,67 @@
+name: "playwright"
+on: # rebuild any PRs and main branch changes
+  pull_request:
+  push:
+    branches:
+      - main
+
+env:
+  LOG_LEVEL: ERROR
+  STATIC_DIR:
+  DATABASE_HOSTNAME: localhost
+  DATABASE_CREDENTIALS: dispatch:dispatch
+  DISPATCH_ENCRYPTION_KEY: NJHDWDJ3PbHT8h
+  DISPATCH_JWT_SECRET: foo
+
+jobs:
+  end-to-end:
+    runs-on: ubuntu-20.04
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: dispatch
+          POSTGRES_PASSWORD: dispatch
+          POSTGRES_DB: dispatch
+        ports:
+          - 5432:5432
+        # needed because the postgres container does not provide a healthcheck
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v3
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Install python dependencies
+        run: |
+          export DISPATCH_LIGHT_BUILD=1
+          python -m pip install --upgrade pip
+          pip install psycopg[binary]
+          pip install -e ".[dev]"
+      - name: Install npm dependencies
+        run: |
+          npm ci -D --prefix src/dispatch/static/dispatch
+          npm install -D @playwright/test
+      - name: Install playwright browsers
+        run: npx playwright install --with-deps
+      - name: Setup sample database
+        run: dispatch database restore --dump-file data/dispatch-sample-data.dump && dispatch database upgrade
+      - name: Run tests
+        run: npx playwright test
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30
diff --git a/.gitignore b/.gitignore
@@ -207,3 +207,7 @@ docker/.env
 *.dump
 
 .gitconfig
+/test-results/
+/playwright-report/
+/playwright/.cache/
+/tests/e2e/artifacts/*
diff --git a/data/dispatch-sample-data.dump b/data/dispatch-sample-data.dump
@@ -6665,8 +6665,11 @@ COPY dispatch_organization_default.incident_role_tag (incident_role_id, tag_id)
 --
 
 COPY dispatch_organization_default.incident_type (id, name, slug, description, exclude_from_metrics, enabled, "default", visibility, plugin_metadata, incident_template_document_id, executive_template_document_id, review_template_document_id, tracking_template_document_id, commander_service_id, liaison_service_id, search_vector, project_id) FROM stdin;
-2	Employee Investigation	\N	This is an employee investigation.	f	t	f	Restricted	[]	1	2	3	\N	\N	\N	'employe':1,6 'investig':2,7	1
+6	Other	\N	This is a miscellaneous incident.	f	t	f	Open	[]	1	2	3	\N	\N	\N	'oth':1,8 'misc':2,9 'incid':6 'involv':7	1
+5	Denial of Service	\N	This is an incident involving a Denial of Service attack on a compute resource or service.	f	t	f	Open	[]	1	2	3	\N	\N	\N	'ddos':1,8 'den':2,9 'incid':6 'involv':7	1
+4	Malware	\N	This is an incident involving malware on a host.	f	t	f	Open	[]	1	2	3	\N	\N	\N	'mal':1,8 'incid':6 'involv':7	1
 3	Customer Data	\N	This is an incident involving customer data.	f	t	f	Open	[]	1	2	3	\N	\N	\N	'custom':1,8 'data':2,9 'incid':6 'involv':7	1
+2	Employee Investigation	\N	This is an employee investigation.	f	t	f	Restricted	[]	1	2	3	\N	\N	\N	'employe':1,6 'investig':2,7	1
 1	Vulnerability	\N	This is an incident involving a misconfiguration or vulnerability.	f	t	t	Open	[]	1	2	3	\N	\N	\N	'incid':5 'involv':6 'misconfigur':8 'vulner':1,10	1
 \.
 
@@ -12042,4 +12045,3 @@ ALTER TABLE ONLY public.workflow_term
 --
 -- PostgreSQL database dump complete
 --
-
diff --git a/playwright.config.ts b/playwright.config.ts
@@ -0,0 +1,75 @@
+import type { PlaywrightTestConfig } from "@playwright/test"
+import { devices } from "@playwright/test"
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+const config: PlaywrightTestConfig = {
+  testDir: "./tests/e2e",
+  outputDir: "./tests/e2e/artifacts/test-failures",
+  /* Maximum time one test can run for. */
+  timeout: 100 * 1000,
+  expect: {
+    /**
+     * Maximum time expect() should wait for the condition to be met.
+     * For example in `await expect(locator).toHaveText();`
+     */
+    timeout: 10000,
+  },
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: "html",
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
+    actionTimeout: 0,
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    baseURL: "http://localhost:8080/",
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: "on",
+    video: "on",
+    screenshot: "on",
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: "chromium",
+      use: {
+        ...devices["Desktop Chrome"],
+      },
+    },
+
+    {
+      name: "firefox",
+      use: {
+        ...devices["Desktop Firefox"],
+      },
+    },
+
+    {
+      name: "webkit",
+      use: {
+        ...devices["Desktop Safari"],
+      },
+    },
+  ],
+  /* Folder for test artifacts such as screenshots, videos, traces, etc. */
+  // outputDir: 'test-results/',
+
+  /* Run your local dev server before starting the tests */
+  webServer: {
+    command: "dispatch server develop",
+    url: "http://localhost:8080/",
+    reuseExistingServer: !process.env.CI,
+  },
+}
+
+export default config
diff --git a/src/dispatch/static/dispatch/package-lock.json b/src/dispatch/static/dispatch/package-lock.json
diff --git a/src/dispatch/static/dispatch/package.json b/src/dispatch/static/dispatch/package.json
@@ -41,6 +41,7 @@
   },
   "devDependencies": {
     "@mdi/font": "^5.9.55",
+    "@playwright/test": "^1.30.0",
     "@vitejs/plugin-vue2": "^2.0.0",
     "@vue/compiler-sfc": "^3.0.11",
     "babel-runtime": "^6.26.0",
diff --git a/tests/e2e/report-submission.spec.ts b/tests/e2e/report-submission.spec.ts
@@ -0,0 +1,106 @@
+import { test, expect, chromium } from "@playwright/test"
+import login from "./utils/login"
+
+test.describe("Authenticated Dispatch App", () => {
+  test.beforeEach(async ({ page }) => {
+    await login(page)
+  })
+  test("Should allow me to report an incident", async ({ page }) => {
+    /* The ability to report an incident is one of the most critical
+      user stories in the Dispatch application. */
+
+    const project = "default"
+    const type = "Denial of Service"
+    const priority = "Low"
+
+    await page.goto("./default/incidents/report")
+
+    // Title
+    await page.getByLabel("Title").click()
+    await page.getByLabel("Title").fill("Incident Test Created by Playwright")
+
+    // Description
+    await page.getByLabel("Description").click()
+    await page.getByLabel("Description").fill("test")
+
+    // Project
+    await page.getByRole("combobox").filter({ hasText: "Project" }).locator("i").click()
+    await page.getByText(project, { exact: true }).click()
+
+    // Type
+    await page.getByRole("button", { name: "Type" }).click()
+
+    // Soft check that the 'Load More' selector is available upon opening the Project dropdown
+    await expect
+      .soft(
+        page.getByText("Load More"),
+        "The 'Load More' selector should be visible when there are more than 5 options in the Tags combobox."
+      )
+      .toBeVisible()
+
+    // Click load more
+    await page.getByText("Load More").click()
+
+    /* Project Options:
+
+        Other
+        Denial of Service
+        Malware
+        Customer Data
+        Employee Investigation
+        Vulnerability
+    */
+
+    // Click a type to select it
+    await page.getByText(type, { exact: true }).click()
+
+    // Priority
+    await page.getByRole("button", { name: "Priority" }).click()
+    await page.getByText("Low").click()
+
+    // Open the dropdown selection for the Tags field
+    await page.getByRole("combobox").filter({ hasText: "Tags" }).locator("i").click()
+    await page.getByText("default/ExampleTag").click()
+
+    // Click out the dialog
+    await page
+      .locator("span")
+      .filter({
+        hasText: "Report Incident If you suspect an incident and need help, please fill out this f",
+      })
+      .click()
+
+    // Submit the form
+    await page.getByRole("button", { name: "Submit" }).click()
+
+    // Wait for the incident to be created
+    await page.waitForLoadState("networkidle")
+
+    // Soft validate that we get redirected to the incident submission form
+    await expect
+      .soft(page)
+      .toHaveURL(
+        encodeURI(
+          `./default/incidents/report?project=${project}&incident_priority=${priority}&incident_type=${type}`
+        )
+      )
+
+    // Soft validate that we recieve the report form.
+    await expect
+      .soft(
+        page.getByText("Incident Report"),
+        "'Incident Report' text not visible on page after submission of incident report."
+      )
+      .toBeVisible()
+
+    // Soft validate that we recieve the report form.
+    await expect
+      .soft(
+        page.getByText(
+          "This page will be populated with incident resources as they are created (if available). If you have any questions, please feel free to review the Frequently Asked Questions (FAQ) document linked below, and/or reach out to the listed Incident Commander."
+        ),
+        "'Incident Report: Description' not visible on page after submission of incident report."
+      )
+      .toBeVisible()
+  })
+})
diff --git a/tests/e2e/utils/login.ts b/tests/e2e/utils/login.ts
