[uss_qualifier] constraints sync: verify secondary DSS instances are clean (#1133)

Author: Shastick

monitoring/uss_qualifier/scenarios/astm/utm/dss/fragments/cr/verify_clean_secondary_workspace.md

@@ -0,0 +1,14 @@
+# Verify secondary DSS contains no CRs with a test ID test step fragment
+
+Ensures that a secondary DSS is ready to be used for testing by confirming that no CR bearing an ID used for testing exists on it.
+
+## 🛑 Constraint references can be queried by ID check
+
+If an existing constraint reference cannot directly be queried by its ID, or if for a non-existing one the DSS replies with a status code different than 404,
+the DSS implementation is in violation of **[astm.f3548.v21.DSS0005,3](../../../../../../requirements/astm/f3548/v21.md)**.
+
+## 🛑 Constraint reference with test ID does not exist check
+
+If a CR that was deleted from the primary DSS can still be found on a secondary DSS, either one of them may be improperly pooled
+and in violation of **[astm.f3548.v21.DSS0020](../../../../../../requirements/astm/f3548/v21.md)**.
+

monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/constraint_ref_synchronization.md

@@ -29,9 +29,9 @@ are properly propagated to every other DSS instance participating in the deploym
 
 ## Setup test case
 
-### Ensure clean workspace test step
+### [Ensure clean workspace test step](../clean_workspace_constraints.md)
 
-#### [Clean any existing constraint references with known test IDs](../clean_workspace_constraints.md)
+### [Verify secondary DSS instances are clean test step](../fragments/cr/verify_clean_secondary_workspace.md)
 
 ## CR synchronization test case
 

monitoring/uss_qualifier/scenarios/astm/utm/dss/synchronization/constraint_ref_synchronization.py

@@ -176,11 +176,11 @@ def _setup_case(self):
             time_end=datetime.now() + timedelta(minutes=20),
         )
         self.begin_test_step("Ensure clean workspace")
-        self._ensure_clean_workspace_step()
+        self._ensure_clean_primary_workspace_step()
         self.end_test_step()
         self.end_test_case()
 
-    def _ensure_clean_workspace_step(self):
+    def _ensure_clean_primary_workspace_step(self):
         # Delete any active CRs we might own
         test_step_fragments.cleanup_active_constraint_refs(
             self,
@@ -192,6 +192,13 @@ def _ensure_clean_workspace_step(self):
         # Make sure the CR ID we are going to use is available
         test_step_fragments.cleanup_constraint_ref(self, self._dss, self._cr_id)
 
+    def _verify_clean_secondaries_step(self):
+        self.begin_test_step("Verify secondary DSS instances are clean")
+        for dss in self._secondary_dss_instances:
+            test_step_fragments.verify_constraint_does_not_exist(self, dss, self._cr_id)
+
+        self.end_test_step()
+
     def _create_cr_with_params(self, creation_params: PutConstraintReferenceParameters):
         with self.check(
             "Create constraint reference query succeeds", [self._primary_pid]
@@ -583,5 +590,5 @@ def _confirm_secondary_has_no_oir(self, secondary_dss: DSSInstance):
 
     def cleanup(self):
         self.begin_cleanup()
-        self._ensure_clean_workspace_step()
+        self._ensure_clean_primary_workspace_step()
         self.end_cleanup()

monitoring/uss_qualifier/scenarios/astm/utm/dss/test_step_fragments.py

@@ -273,14 +273,19 @@ def verify_op_intent_does_not_exist(
 
 
 def cleanup_constraint_ref(
-    scenario: TestScenarioType, dss: DSSInstance, cr_id: EntityID
-) -> None:
+    scenario: TestScenarioType,
+    dss: DSSInstance,
+    cr_id: EntityID,
+    delete_if_exists: bool = False,
+) -> bool:
     """
     Remove the specified constraint reference from the DSS, if it exists.
 
     This function implements some of the test step fragment described in clean_workspace.md:
         - Constraint references can be queried by ID
         - Constraint reference removed
+
+    :return: True if the constraint reference was found to exist, False if no constraint reference was found.
     """
 
     with scenario.check(
@@ -289,6 +294,13 @@ def cleanup_constraint_ref(
         try:
             cr, q = dss.get_constraint_ref(cr_id)
             scenario.record_query(q)
+            if cr.ovn is None:
+                check.record_failed(
+                    summary=f"CR {cr_id} is missing OVN",
+                    details="The CR retrieved from the DSS did not include an OVN, despite the CR being owned by uss_qualifier.",
+                    query_timestamps=[q.request.timestamp],
+                )
+                raise ScenarioDidNotStopError(check)
         except fetch.QueryError as e:
             scenario.record_queries(e.queries)
             if e.cause_status_code != 404:
@@ -297,7 +309,25 @@ def cleanup_constraint_ref(
                     details=e.msg,
                     query_timestamps=e.query_timestamps,
                 )
-            else:
-                return
+                raise ScenarioDidNotStopError(check)
+            return False
+
+    if delete_if_exists:
+        remove_constraint_ref(scenario, dss, cr_id, cr.ovn)
 
-    remove_constraint_ref(scenario, dss, cr_id, cr.ovn)
+    return True
+
+
+def verify_constraint_does_not_exist(
+    scenario: TestScenarioType, dss: DSSInstance, cr_id: EntityID
+):
+    cr_found = cleanup_constraint_ref(scenario, dss, cr_id, delete_if_exists=False)
+    with scenario.check(
+        "constraint reference with test ID does not exist",
+        dss.participant_id,
+    ) as check:
+        if cr_found:
+            check.record_failed(
+                summary=f"Constraint intent reference {cr_id} was still found on DSS {dss.participant_id}",
+                details=f"Expected constraint reference {cr_id} to not be found on secondary DSS because it was not present on, or has been removed, from the primary DSS, but it was returned.",
+            )