Test webdriver 4

.github/workflows/pull_request.yml

@@ -150,7 +150,6 @@ jobs:
     timeout-minutes: 35
     env:
       EQ_RUN_FUNCTIONAL_TESTS_HEADLESS: True
-      # :TODO: Revisit & update when 2 instances can be used without adverse effects
       EQ_FUNCTIONAL_TEST_MAX_INSTANCES: 2
     steps:
       - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
@@ -173,8 +172,10 @@ jobs:
         run: npm install
       - name: Docker compose
         run: docker compose --version && RUNNER_ENV_FILE=.functional-tests.env docker compose up --build -d
-      - name: Functional tests
-        run: make test-functional
+      - name: Functional tests (supplementary data)
+        run: make test-functional-sds
+      - name: Functional tests (main suite)
+        run: make test-functional-main
       - name: Docker compose shutdown
         run: RUNNER_ENV_FILE=.functional-tests.env docker compose kill
   docker-push:

Makefile

@@ -45,8 +45,15 @@ test:
 test-unit:
 	poetry run ./scripts/run_tests_unit.sh
 
+test-functional-main: generate-pages
+	npm run test_functional_main
+
+test-functional-sds: generate-pages
+	EQ_FUNCTIONAL_TEST_MAX_INSTANCES=1 npm run test_functional_sds
+
 test-functional: generate-pages
-	npm run test_functional
+	$(MAKE) test-functional-main
+	$(MAKE) test-functional-sds
 
 test-functional-headless: generate-pages
 	EQ_RUN_FUNCTIONAL_TESTS_HEADLESS='True' make test-functional

package.json

@@ -17,6 +17,8 @@
         "lint:tests": "prettier --check \"tests/functional/**/*.js\" && eslint \"tests/functional/**/*.js\"",
         "lint:test-schemas": "prettier --check \"schemas/test/*/*.json\" && eslint \"schemas/test/**/*.json\"",
         "test_functional": "./node_modules/.bin/wdio tests/functional/wdio.conf.js",
+        "test_functional_main": "FUNCTIONAL_MODE=main npm run test_functional",
+        "test_functional_sds": "FUNCTIONAL_MODE=sds npm run test_functional",
         "generate_pages": "rm -rf ./tests/functional/generated_pages && poetry run python -m tests.functional.generate_pages schemas/test/en/ ./tests/functional/generated_pages -r '../../base_pages'",
         "format": "npm run format:tests && npm run format:test-schemas",
         "format:tests": "prettier \"tests/functional/**/*.js\" --write && eslint --fix \"tests/functional/**/*.js\"",
@@ -29,10 +31,10 @@
         "@babel/preset-env": "7.28.5",
         "@babel/register": "7.28.3",
         "@babel/runtime": "7.28.4",
-        "@wdio/cli": "9.21.1",
-        "@wdio/local-runner": "8.46.0",
-        "@wdio/mocha-framework": "9.21.0",
-        "@wdio/spec-reporter": "9.20.0",
+        "@wdio/local-runner": "9.24.0",
+        "@wdio/cli": "9.24.0",
+        "@wdio/mocha-framework": "9.24.0",
+        "@wdio/spec-reporter": "9.24.0",
         "eslint": "8.57.1",
         "eslint-cli": "1.1.1",
         "eslint-config-standard": "17.1.0",
@@ -49,7 +51,7 @@
         "prettier": "3.7.3",
         "typescript": "5.9.3",
         "uuid": "13.0.0",
-        "webdriverio": "9.21.0"
+        "webdriverio": "9.24.0"
     },
     "prettier": {}
 }

tests/functional/helpers.js

@@ -45,7 +45,7 @@ export const click = async (selector) => {
   // but clicks down on the very top of the button which moves down and just below the mouse. When the mouse click is released
   // it's no longer over the button and the click silently fails. This means that when the test comes to do assertions on the following page
   // they fail, as we never navigated to that page.
-  await $(selector).scrollIntoView({ block: "center", inline: "center", behavior: "smooth" });
+  await $(selector).scrollIntoView({ block: "center", inline: "center", behavior: "instant" });
   await $(selector).click();
 
   // Allow time in case the click loads a new page.
@@ -67,3 +67,31 @@ export const verifyUrlContains = async (expectedUrlString) => {
 export const verifyUrlContainsSyncMode = (expectedUrlString) => {
   expect(browser).toHaveUrl(expect.stringContaining(expectedUrlString));
 };
+
+export async function openQuestionnaireWithRetry(schema, options, maxRetries = 1) {
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    await browser.openQuestionnaire(schema, options);
+
+    const bodyText = await $("body").getText();
+    const isServiceError = bodyText.includes("Sorry, there is a problem with this service");
+
+    if (!isServiceError) {
+      return;
+    }
+
+    if (attempt === maxRetries) {
+      throw new Error(`Service error page after ${maxRetries + 1} attempt(s) for schema="${schema}" with options=${JSON.stringify(options)}`);
+    }
+    await browser.reloadSession();
+  }
+}
+
+export async function getRawHTML(selector, options = {}) {
+  const { includeSelectorTag = true } = options;
+  const element = await $(selector);
+
+  return element.getHTML({
+    prettify: false,
+    includeSelectorTag,
+  });
+}

tests/functional/spec/checkbox.spec.js

@@ -2,7 +2,7 @@ import MandatoryCheckboxPage from "../generated_pages/checkbox/mandatory-checkbo
 import NonMandatoryCheckboxPage from "../generated_pages/checkbox/non-mandatory-checkbox.page";
 import singleCheckboxPage from "../generated_pages/checkbox/single-checkbox.page";
 import SubmitPage from "../generated_pages/checkbox/submit.page";
-import { click, verifyUrlContains } from "../helpers";
+import { click, verifyUrlContains, getRawHTML } from "../helpers";
 
 describe('Checkbox with "other" option', () => {
   beforeEach("Load the survey", async () => {
@@ -44,7 +44,7 @@ describe('Checkbox with "other" option', () => {
     // When
     await click(MandatoryCheckboxPage.submit());
     // Then
-    await expect(await $(MandatoryCheckboxPage.error()).getHTML()).toContain(
+    await expect(await getRawHTML(MandatoryCheckboxPage.error())).toContain(
       'Select at least one answer <span class="ons-u-vh">to ‘Which pizza toppings would you like?’</span></a>',
     );
   });

tests/functional/spec/components/address/address.spec.js

@@ -2,7 +2,7 @@ import AddressConfirmation from "../../../generated_pages/address/address-confir
 import AddressMandatory from "../../../generated_pages/address/address-block-mandatory.page";
 import AddressOptional from "../../../generated_pages/address/address-block-optional.page";
 import SubmitPage from "../../../generated_pages/address/submit.page";
-import { click, verifyUrlContains } from "../../../helpers";
+import { click, verifyUrlContains, getRawHTML } from "../../../helpers";
 
 describe("Address Answer Type", () => {
   beforeEach("Launch survey", async () => {
@@ -21,7 +21,7 @@ describe("Address Answer Type", () => {
       await click(AddressConfirmation.submit());
       await verifyUrlContains(SubmitPage.pageName);
       await expect(await $(SubmitPage.addressMandatory()).getText()).toBe("Evelyn Street\nApt 7\nBarry\nCF63 4JG");
-      await expect(await $(SubmitPage.addressMandatory()).getHTML()).toContain("Evelyn Street<br>Apt 7<br>Barry<br>CF63 4JG");
+      await expect(await getRawHTML(SubmitPage.addressMandatory())).toContain("Evelyn Street<br>Apt 7<br>Barry<br>CF63 4JG");
     });
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_currency.spec.js

@@ -5,7 +5,6 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Currency With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
     await browser.url("/questionnaire/mutually-exclusive-currency");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_day_month_year_date.spec.js

@@ -5,7 +5,6 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Day Month Year Date With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
     await browser.url("/questionnaire/mutually-exclusive-date");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_duration.spec.js

@@ -5,7 +5,6 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Duration With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
     await browser.url("/questionnaire/mutually-exclusive-duration");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_month_year_date.spec.js

@@ -5,7 +5,6 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Month Year Date With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
     await browser.url("/questionnaire/mutually-exclusive-month-year-date");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_multiple_day_month_year_date.spec.js

@@ -5,7 +5,6 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Day Month Year Date With Multiple Radio Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive_multiple.json");
-    await browser.pause(100);
     await browser.url("/questionnaire/mutually-exclusive-date");
   });
   describe("Given the user has entered a value for the non-exclusive month year date answer", () => {

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_multiple_textfield.spec.js

@@ -5,7 +5,6 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Textfield With Multiple Radio Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive_multiple.json");
-    await browser.pause(100);
     await browser.url("/questionnaire/mutually-exclusive-textfield");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_number.spec.js

@@ -5,7 +5,6 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Number With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
     await browser.url("/questionnaire/mutually-exclusive-number");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_percentage.spec.js

@@ -5,7 +5,6 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Percentage With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
     await browser.url("/questionnaire/mutually-exclusive-percentage");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_textarea.spec.js

@@ -5,7 +5,6 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive TextArea With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
     await browser.url("/questionnaire/mutually-exclusive-textarea");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_textfield.spec.js

@@ -5,7 +5,6 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Textfield With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
     await browser.url("/questionnaire/mutually-exclusive-textfield");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_unit.spec.js

@@ -5,7 +5,6 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Unit With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
     await browser.url("/questionnaire/mutually-exclusive-unit");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_year_date.spec.js

@@ -5,7 +5,6 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Year Date With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
     await browser.url("/questionnaire/mutually-exclusive-year-date");
   });
 

tests/functional/spec/components/radio/radio.js

@@ -15,7 +15,7 @@ import RadioNonMandatoryDetailAnswerOverriddenPage from "../../../generated_page
 
 import RadioNonMandatoryDetailAnswerPage from "../../../generated_pages/radio_optional_with_detail_answer_mandatory/radio-non-mandatory.page";
 import RadioNonMandatoryDetailAnswerSummary from "../../../generated_pages/radio_optional_with_detail_answer_mandatory/submit.page";
-import { click, verifyUrlContains } from "../../../helpers";
+import { click, verifyUrlContains, getRawHTML } from "../../../helpers";
 describe("Component: Radio", () => {
   describe("Given I start a Mandatory Radio survey", () => {
     before(async () => {
@@ -50,7 +50,7 @@ describe("Component: Radio", () => {
 
     it("When I have submitted the page without any option, Then the question text is hidden in the error message using a span element", async () => {
       await click(RadioMandatoryOverriddenPage.submit());
-      await expect(await $(RadioMandatoryOverriddenPage.errorNumber(1)).getHTML()).toContain(
+      await expect(await getRawHTML(RadioMandatoryOverriddenPage.errorNumber(1))).toContain(
         'Select an answer <span class="ons-u-vh">to ‘What do you prefer for breakfast?’</span></a>',
       );
     });

tests/functional/spec/features/last_viewed_guidance/last_viewed_guidance.spec.js

@@ -22,7 +22,7 @@ describe("Last viewed question guidance", () => {
     it("When the respondent resumes on the first block of a section, then last question guidance is not shown", async () => {
       await $(HouseholdInterstitialPage.saveSignOut()).click();
       await browser.openQuestionnaire("test_last_viewed_question_guidance.json", resumableLaunchParams);
-      await browser.pause(100);
+
       await verifyUrlContains(HouseholdInterstitialPage.url());
       await expect(await $(HouseholdInterstitialPage.lastViewedQuestionGuidance()).isExisting()).toBe(false);
     });
@@ -31,7 +31,7 @@ describe("Last viewed question guidance", () => {
       await click(HouseholdInterstitialPage.submit());
       await $(AddressConfirmationPage.saveSignOut()).click();
       await browser.openQuestionnaire("test_last_viewed_question_guidance.json", resumableLaunchParams);
-      await browser.pause(100);
+
       await verifyUrlContains(AddressConfirmationPage.url());
       await expect(await $(AddressConfirmationPage.lastViewedQuestionGuidanceLink()).getAttribute("href")).toContain(HouseholdInterstitialPage.url());
       await expect(await $(AddressConfirmationPage.lastViewedQuestionGuidance()).isExisting()).toBe(true);

tests/functional/spec/features/last_viewed_guidance/last_viewed_guidance_hub.spec.js

@@ -28,7 +28,7 @@ describe("Last viewed question guidance", () => {
     it("When the respondent saves and resumes from a section which is not started, then last question guidance is not shown", async () => {
       await $(WorkInterstitialPage.saveSignOut()).click();
       await browser.openQuestionnaire("test_last_viewed_question_guidance_hub.json", resumableLaunchParams);
-      await browser.pause(100);
+
       await verifyUrlContains(WorkInterstitialPage.url());
       await expect(await $(WorkInterstitialPage.lastViewedQuestionGuidance()).isExisting()).toBe(false);
     });

tests/functional/spec/features/placeholder/playback_confirmation.spec.js

@@ -1,5 +1,5 @@
 import MandatoryCheckboxPage from "../../../generated_pages/placeholder_playback_list/mandatory-checkbox.page";
-import { click } from "../../../helpers";
+import { click, getRawHTML } from "../../../helpers";
 describe("Feature: Playback Confirmation", () => {
   beforeEach("Open the schema", async () => {
     await browser.openQuestionnaire("test_placeholder_playback_list.json");
@@ -10,7 +10,7 @@ describe("Feature: Playback Confirmation", () => {
     await $(MandatoryCheckboxPage.ham()).click();
     await click(MandatoryCheckboxPage.submit());
 
-    await expect(await $("#confirm-answers-question ul").getHTML()).toContain("Ham");
-    await expect(await $("#confirm-answers-question ul").getHTML()).toContain("Cheese");
+    await expect(await getRawHTML("#confirm-answers-question ul")).toContain("Ham");
+    await expect(await getRawHTML("#confirm-answers-question ul")).toContain("Cheese");
   });
 });

tests/functional/spec/features/view_submitted_response/view_submitted_response.spec.js

@@ -20,7 +20,7 @@ import DependencyQuestionSectionTwo from "../../../generated_pages/view_submitte
 import SkippableBlockSectionTwo from "../../../generated_pages/view_submitted_response_repeating_sections/skippable-block.page";
 import SectionSummarySectionTwo from "../../../generated_pages/new_calculated_summary_cross_section_dependencies_repeating/calculated-summary-section-summary.page";
 import ListCollectorAddPage from "../../../generated_pages/view_submitted_response_repeating_sections//list-collector-add.page";
-import { click, verifyUrlContains } from "../../../helpers";
+import { click, verifyUrlContains, getRawHTML } from "../../../helpers";
 
 describe("View Submitted Response", () => {
   beforeEach("Load the questionnaire", async () => {
@@ -31,7 +31,7 @@ describe("View Submitted Response", () => {
     await click(AddressBlockPage.submit());
     await click(SubmitPage.submit());
     await verifyUrlContains(ThankYouPage.pageName);
-    await expect(await $(ThankYouPage.title()).getHTML()).toContain("Thank you for completing the Test");
+    await expect(await getRawHTML(ThankYouPage.title())).toContain("Thank you for completing the Test");
     await $(ThankYouPage.savePrintAnswersLink()).click();
     await verifyUrlContains(ViewSubmittedResponsePage.pageName);
   });
@@ -55,7 +55,7 @@ describe("View Submitted Response", () => {
       await browser.pause(40000); // Waiting 40 seconds for the timeout to expire (45 minute timeout changed to 35 seconds by overriding VIEW_SUBMITTED_RESPONSE_EXPIRATION_IN_SECONDS for the purpose of the functional test)
       await $(ViewSubmittedResponsePage.downloadButton()).click();
       await expect(await $(ViewSubmittedResponsePage.informationPanel()).isDisplayed()).toBe(true);
-      await expect(await $(ViewSubmittedResponsePage.informationPanel()).getHTML()).toContain(
+      await expect(await getRawHTML(ViewSubmittedResponsePage.informationPanel())).toContain(
         "For security, you can no longer view or get a copy of your answers",
       );
     });
@@ -124,7 +124,7 @@ describe("View Submitted Response Summary Page With Repeating Sections", () => {
 
     await click(HubPage.submit());
     await verifyUrlContains(ThankYouPage.pageName);
-    await expect(await $(ThankYouPage.title()).getHTML()).toContain("Thank you for completing the Test");
+    await expect(await getRawHTML(ThankYouPage.title())).toContain("Thank you for completing the Test");
     await $(ThankYouPage.savePrintAnswersLink()).click();
     await verifyUrlContains(ViewSubmittedResponsePage.pageName);
   });
@@ -141,14 +141,14 @@ describe("View Submitted Response Summary Page With Repeating Sections", () => {
     await expect(await $(ViewSubmittedResponseRepeatingPage.addressDetailsGroupTitle()).getText()).toBe("Address Details");
     await expect(await $(ViewSubmittedResponseRepeatingPage.addressQuestion()).getText()).toBe("What is your address?");
     await expect(await $(ViewSubmittedResponseRepeatingPage.addressAnswer()).getText()).toBe("NP10 8XG");
-    await expect(await $("body").getHTML()).toContain("Marcus Twin");
+    await expect(await getRawHTML("body")).toContain("Marcus Twin");
     await expect(await $(firstGroup).$$(groupTitle)[0].getText()).toBe("Calculated Summary Group");
     await expect(await $(firstGroup).$$(repeatingSectionAnswer)[0].getText()).toBe("40 - calculated summary answer (current section)");
-    await expect(await $("body").getHTML()).toContain("How much did Marcus Twin spend on fruit?");
+    await expect(await getRawHTML("body")).toContain("How much did Marcus Twin spend on fruit?");
     await expect(await $(firstGroup).$$(skippableRepeatingSectionAnswer)[0].getText()).toBe("£100");
-    await expect(await $("body").getHTML()).toContain("John Doe");
+    await expect(await getRawHTML("body")).toContain("John Doe");
     await expect(await $(secondGroup).$$(groupTitle)[0].getText()).toBe("Calculated Summary Group");
     await expect(await $(secondGroup).$$(repeatingSectionAnswer)[0].getText()).toBe("80 - calculated summary answer (current section)");
-    await expect(await $("body").getHTML()).not.toContain("How much did John Doe spend on fruit?");
+    await expect(await getRawHTML("body")).not.toContain("How much did John Doe spend on fruit?");
   });
 });