TESTING

.github/workflows/pull_request.yml

@@ -150,7 +150,6 @@ jobs:
     timeout-minutes: 35
     env:
       EQ_RUN_FUNCTIONAL_TESTS_HEADLESS: True
-      # :TODO: Revisit & update when 2 instances can be used without adverse effects
       EQ_FUNCTIONAL_TEST_MAX_INSTANCES: 2
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
@@ -173,8 +172,10 @@ jobs:
         run: npm install
       - name: Docker compose
         run: docker compose --version && RUNNER_ENV_FILE=.functional-tests.env docker compose up --build -d
-      - name: Functional tests
-        run: make test-functional
+      - name: Functional tests (supplementary data)
+        run: make test-functional-sds
+      - name: Functional tests (main suite)
+        run: make test-functional-main
       - name: Docker compose shutdown
         run: RUNNER_ENV_FILE=.functional-tests.env docker compose kill
   docker-push:

Makefile

@@ -45,8 +45,15 @@ test:
 test-unit:
 	poetry run ./scripts/run_tests_unit.sh
 
+test-functional-main: generate-pages
+	npm run test_functional_main
+
+test-functional-sds: generate-pages
+	EQ_FUNCTIONAL_TEST_MAX_INSTANCES=1 npm run test_functional_sds
+
 test-functional: generate-pages
-	npm run test_functional
+	$(MAKE) test-functional-main
+	$(MAKE) test-functional-sds
 
 test-functional-headless: generate-pages
 	EQ_RUN_FUNCTIONAL_TESTS_HEADLESS='True' make test-functional

package.json

@@ -17,22 +17,24 @@
         "lint:tests": "prettier --check \"tests/functional/**/*.js\" && eslint \"tests/functional/**/*.js\"",
         "lint:test-schemas": "prettier --check \"schemas/test/*/*.json\" && eslint \"schemas/test/**/*.json\"",
         "test_functional": "./node_modules/.bin/wdio tests/functional/wdio.conf.js",
+        "test_functional_main": "FUNCTIONAL_MODE=main npm run test_functional",
+        "test_functional_sds": "FUNCTIONAL_MODE=sds npm run test_functional",
         "generate_pages": "rm -rf ./tests/functional/generated_pages && poetry run python -m tests.functional.generate_pages schemas/test/en/ ./tests/functional/generated_pages -r '../../base_pages'",
         "format": "npm run format:tests && npm run format:test-schemas",
         "format:tests": "prettier \"tests/functional/**/*.js\" --write && eslint --fix \"tests/functional/**/*.js\"",
         "format:test-schemas": "prettier \"schemas/test/*/*.json\" --write && eslint --fix \"schemas/test/*/*.json\"",
         "wdio": "wdio run ./tests/functional/wdio.conf.js"
     },
     "devDependencies": {
-        "@babel/core": "7.28.0",
-        "@babel/plugin-transform-runtime": "7.28.0",
-        "@babel/preset-env": "7.28.0",
-        "@babel/register": "7.27.1",
-        "@babel/runtime": "7.28.2",
-        "@wdio/cli": "9.18.4",
-        "@wdio/local-runner": "8.45.0",
-        "@wdio/mocha-framework": "9.18.0",
-        "@wdio/spec-reporter": "9.18.0",
+        "@babel/core": "7.28.5 ",
+        "@babel/plugin-transform-runtime": "7.28.5",
+        "@babel/preset-env": "7.28.5",
+        "@babel/register": "7.28.3",
+        "@babel/runtime": "7.28.4",
+        "@wdio/cli": "9.20.0",
+        "@wdio/local-runner": "9.20.0",
+        "@wdio/mocha-framework": "9.20.0",
+        "@wdio/spec-reporter": "9.20.0",
         "eslint": "8.57.1",
         "eslint-cli": "1.1.1",
         "eslint-config-standard": "17.1.0",
@@ -49,7 +51,7 @@
         "prettier": "3.6.2",
         "typescript": "5.9.2",
         "uuid": "11.1.0",
-        "webdriverio": "9.18.4"
+        "webdriverio": "9.20.0"
     },
     "prettier": {}
 }

tests/functional/helpers.js

@@ -67,3 +67,21 @@ export const verifyUrlContains = async (expectedUrlString) => {
 export const verifyUrlContainsSyncMode = (expectedUrlString) => {
   expect(browser).toHaveUrl(expect.stringContaining(expectedUrlString));
 };
+
+export async function openQuestionnaireWithRetry(schema, options, maxRetries = 1) {
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    await browser.openQuestionnaire(schema, options);
+
+    const bodyText = await $("body").getText();
+    const isServiceError = bodyText.includes("Sorry, there is a problem with this service");
+
+    if (!isServiceError) {
+      return;
+    }
+
+    if (attempt === maxRetries) {
+      throw new Error(`Service error page after ${maxRetries + 1} attempt(s) for schema="${schema}" with options=${JSON.stringify(options)}`);
+    }
+    await browser.reloadSession();
+  }
+}

tests/functional/spec/checkbox.spec.js

@@ -44,7 +44,7 @@ describe('Checkbox with "other" option', () => {
     // When
     await click(MandatoryCheckboxPage.submit());
     // Then
-    await expect(await $(MandatoryCheckboxPage.error()).getHTML()).toContain(
+    await expect(await $(MandatoryCheckboxPage.error()).getHTML({ prettify: false })).toContain(
       'Select at least one answer <span class="ons-u-vh">to ‘Which pizza toppings would you like?’</span></a>',
     );
   });

tests/functional/spec/components/address/address.spec.js

@@ -21,7 +21,7 @@ describe("Address Answer Type", () => {
       await click(AddressConfirmation.submit());
       await verifyUrlContains(SubmitPage.pageName);
       await expect(await $(SubmitPage.addressMandatory()).getText()).toBe("Evelyn Street\nApt 7\nBarry\nCF63 4JG");
-      await expect(await $(SubmitPage.addressMandatory()).getHTML()).toContain("Evelyn Street<br>Apt 7<br>Barry<br>CF63 4JG");
+      await expect(await $(SubmitPage.addressMandatory()).getHTML({ prettify: false })).toContain("Evelyn Street<br>Apt 7<br>Barry<br>CF63 4JG");
     });
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_currency.spec.js

@@ -5,7 +5,7 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Currency With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
+
     await browser.url("/questionnaire/mutually-exclusive-currency");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_day_month_year_date.spec.js

@@ -5,7 +5,6 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Day Month Year Date With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
     await browser.url("/questionnaire/mutually-exclusive-date");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_duration.spec.js

@@ -5,7 +5,7 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Duration With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
+
     await browser.url("/questionnaire/mutually-exclusive-duration");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_month_year_date.spec.js

@@ -5,7 +5,7 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Month Year Date With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
+
     await browser.url("/questionnaire/mutually-exclusive-month-year-date");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_multiple_day_month_year_date.spec.js

@@ -5,7 +5,7 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Day Month Year Date With Multiple Radio Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive_multiple.json");
-    await browser.pause(100);
+
     await browser.url("/questionnaire/mutually-exclusive-date");
   });
   describe("Given the user has entered a value for the non-exclusive month year date answer", () => {

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_multiple_textfield.spec.js

@@ -5,7 +5,7 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Textfield With Multiple Radio Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive_multiple.json");
-    await browser.pause(100);
+
     await browser.url("/questionnaire/mutually-exclusive-textfield");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_number.spec.js

@@ -5,7 +5,7 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Number With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
+
     await browser.url("/questionnaire/mutually-exclusive-number");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_percentage.spec.js

@@ -5,7 +5,7 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Percentage With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
+
     await browser.url("/questionnaire/mutually-exclusive-percentage");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_textarea.spec.js

@@ -5,7 +5,7 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive TextArea With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
+
     await browser.url("/questionnaire/mutually-exclusive-textarea");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_textfield.spec.js

@@ -5,7 +5,7 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Textfield With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
+
     await browser.url("/questionnaire/mutually-exclusive-textfield");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_unit.spec.js

@@ -5,7 +5,7 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Unit With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
+
     await browser.url("/questionnaire/mutually-exclusive-unit");
   });
 

tests/functional/spec/components/checkbox/mutually_exclusive/mutually_exclusive_year_date.spec.js

@@ -5,7 +5,7 @@ import { click } from "../../../../helpers";
 describe("Component: Mutually Exclusive Year Date With Single Checkbox Override", () => {
   beforeEach(async () => {
     await browser.openQuestionnaire("test_mutually_exclusive.json");
-    await browser.pause(100);
+
     await browser.url("/questionnaire/mutually-exclusive-year-date");
   });
 

tests/functional/spec/components/radio/radio.js

@@ -50,7 +50,7 @@ describe("Component: Radio", () => {
 
     it("When I have submitted the page without any option, Then the question text is hidden in the error message using a span element", async () => {
       await click(RadioMandatoryOverriddenPage.submit());
-      await expect(await $(RadioMandatoryOverriddenPage.errorNumber(1)).getHTML()).toContain(
+      await expect(await $(RadioMandatoryOverriddenPage.errorNumber(1)).getHTML({ prettify: false })).toContain(
         'Select an answer <span class="ons-u-vh">to ‘What do you prefer for breakfast?’</span></a>',
       );
     });

tests/functional/spec/features/last_viewed_guidance/last_viewed_guidance.spec.js

@@ -22,7 +22,7 @@ describe("Last viewed question guidance", () => {
     it("When the respondent resumes on the first block of a section, then last question guidance is not shown", async () => {
       await $(HouseholdInterstitialPage.saveSignOut()).click();
       await browser.openQuestionnaire("test_last_viewed_question_guidance.json", resumableLaunchParams);
-      await browser.pause(100);
+
       await verifyUrlContains(HouseholdInterstitialPage.url());
       await expect(await $(HouseholdInterstitialPage.lastViewedQuestionGuidance()).isExisting()).toBe(false);
     });
@@ -31,7 +31,7 @@ describe("Last viewed question guidance", () => {
       await click(HouseholdInterstitialPage.submit());
       await $(AddressConfirmationPage.saveSignOut()).click();
       await browser.openQuestionnaire("test_last_viewed_question_guidance.json", resumableLaunchParams);
-      await browser.pause(100);
+
       await verifyUrlContains(AddressConfirmationPage.url());
       await expect(await $(AddressConfirmationPage.lastViewedQuestionGuidanceLink()).getAttribute("href")).toContain(HouseholdInterstitialPage.url());
       await expect(await $(AddressConfirmationPage.lastViewedQuestionGuidance()).isExisting()).toBe(true);

tests/functional/spec/features/last_viewed_guidance/last_viewed_guidance_hub.spec.js

@@ -28,7 +28,7 @@ describe("Last viewed question guidance", () => {
     it("When the respondent saves and resumes from a section which is not started, then last question guidance is not shown", async () => {
       await $(WorkInterstitialPage.saveSignOut()).click();
       await browser.openQuestionnaire("test_last_viewed_question_guidance_hub.json", resumableLaunchParams);
-      await browser.pause(100);
+
       await verifyUrlContains(WorkInterstitialPage.url());
       await expect(await $(WorkInterstitialPage.lastViewedQuestionGuidance()).isExisting()).toBe(false);
     });

tests/functional/spec/features/placeholder/playback_confirmation.spec.js

@@ -10,7 +10,7 @@ describe("Feature: Playback Confirmation", () => {
     await $(MandatoryCheckboxPage.ham()).click();
     await click(MandatoryCheckboxPage.submit());
 
-    await expect(await $("#confirm-answers-question ul").getHTML()).toContain("Ham");
-    await expect(await $("#confirm-answers-question ul").getHTML()).toContain("Cheese");
+    await expect(await $("#confirm-answers-question ul").getHTML({ prettify: false })).toContain("Ham");
+    await expect(await $("#confirm-answers-question ul").getHTML({ prettify: false })).toContain("Cheese");
   });
 });

tests/functional/spec/features/view_submitted_response/view_submitted_response.spec.js

@@ -31,7 +31,7 @@ describe("View Submitted Response", () => {
     await click(AddressBlockPage.submit());
     await click(SubmitPage.submit());
     await verifyUrlContains(ThankYouPage.pageName);
-    await expect(await $(ThankYouPage.title()).getHTML()).toContain("Thank you for completing the Test");
+    await expect(await $(ThankYouPage.title()).getHTML({ prettify: false })).toContain("Thank you for completing the Test");
     await $(ThankYouPage.savePrintAnswersLink()).click();
     await verifyUrlContains(ViewSubmittedResponsePage.pageName);
   });
@@ -55,7 +55,7 @@ describe("View Submitted Response", () => {
       await browser.pause(40000); // Waiting 40 seconds for the timeout to expire (45 minute timeout changed to 35 seconds by overriding VIEW_SUBMITTED_RESPONSE_EXPIRATION_IN_SECONDS for the purpose of the functional test)
       await $(ViewSubmittedResponsePage.downloadButton()).click();
       await expect(await $(ViewSubmittedResponsePage.informationPanel()).isDisplayed()).toBe(true);
-      await expect(await $(ViewSubmittedResponsePage.informationPanel()).getHTML()).toContain(
+      await expect(await $(ViewSubmittedResponsePage.informationPanel()).getHTML({ prettify: false })).toContain(
         "For security, you can no longer view or get a copy of your answers",
       );
     });
@@ -124,7 +124,7 @@ describe("View Submitted Response Summary Page With Repeating Sections", () => {
 
     await click(HubPage.submit());
     await verifyUrlContains(ThankYouPage.pageName);
-    await expect(await $(ThankYouPage.title()).getHTML()).toContain("Thank you for completing the Test");
+    await expect(await $(ThankYouPage.title()).getHTML({ prettify: false })).toContain("Thank you for completing the Test");
     await $(ThankYouPage.savePrintAnswersLink()).click();
     await verifyUrlContains(ViewSubmittedResponsePage.pageName);
   });
@@ -141,14 +141,14 @@ describe("View Submitted Response Summary Page With Repeating Sections", () => {
     await expect(await $(ViewSubmittedResponseRepeatingPage.addressDetailsGroupTitle()).getText()).toBe("Address Details");
     await expect(await $(ViewSubmittedResponseRepeatingPage.addressQuestion()).getText()).toBe("What is your address?");
     await expect(await $(ViewSubmittedResponseRepeatingPage.addressAnswer()).getText()).toBe("NP10 8XG");
-    await expect(await $("body").getHTML()).toContain("Marcus Twin");
+    await expect(await $("body").getHTML({ prettify: false })).toContain("Marcus Twin");
     await expect(await $(firstGroup).$$(groupTitle)[0].getText()).toBe("Calculated Summary Group");
     await expect(await $(firstGroup).$$(repeatingSectionAnswer)[0].getText()).toBe("40 - calculated summary answer (current section)");
-    await expect(await $("body").getHTML()).toContain("How much did Marcus Twin spend on fruit?");
+    await expect(await $("body").getHTML({ prettify: false })).toContain("How much did Marcus Twin spend on fruit?");
     await expect(await $(firstGroup).$$(skippableRepeatingSectionAnswer)[0].getText()).toBe("£100");
-    await expect(await $("body").getHTML()).toContain("John Doe");
+    await expect(await $("body").getHTML({ prettify: false })).toContain("John Doe");
     await expect(await $(secondGroup).$$(groupTitle)[0].getText()).toBe("Calculated Summary Group");
     await expect(await $(secondGroup).$$(repeatingSectionAnswer)[0].getText()).toBe("80 - calculated summary answer (current section)");
-    await expect(await $("body").getHTML()).not.toContain("How much did John Doe spend on fruit?");
+    await expect(await $("body").getHTML({ prettify: false })).not.toContain("How much did John Doe spend on fruit?");
   });
 });