fix(backend): improve uniqueness constraint query

This should avoid processing useless rows when we have relationship only
constraints.
This especially improves the IPAM use-case, where the uniqueness
constraint is the (address/prefix, namespace) tuple.
Without this change, the query would check for nodes having that same
address and fetch ALL nodes present in the same namespace.
With this change, the query is now FIRST checking nodes having that same
address and THEN checking if these previous nodes are in the same namespace.

Signed-off-by: Fatih Acar <[email protected]>

Author: fatih-acar

backend/infrahub/core/validators/uniqueness/query.py

@@ -30,7 +30,7 @@ def __init__(
     def get_context(self) -> dict[str, str]:
         return {"kind": self.query_request.kind}
 
-    async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa: ARG002
+    async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa: ARG002, PLR0915
         branch_filter, branch_params = self.branch.get_query_filter_path(at=self.at.to_string(), is_isolated=False)
         self.params.update(branch_params)
         from_times = db.render_list_comprehension(items="relationships(potential_path)", item_name="from")
@@ -103,48 +103,75 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa
         )
 
         attr_paths_subquery = """
-        MATCH attr_path = (start_node:%(node_kind)s)-[:HAS_ATTRIBUTE]->(attr:Attribute)-[r:HAS_VALUE]->(attr_value:AttributeValue)
+        MATCH attr_path = (attr_start_node:%(node_kind)s)-[:HAS_ATTRIBUTE]->(attr:Attribute)-[r:HAS_VALUE]->(attr_value:AttributeValue)
         WHERE attr.name in $attribute_names
             AND ([attr.name, type(r)] in $attr_paths
             OR (attr_value.value in $attr_values AND [attr.name, type(r), attr_value.value] in $attr_paths_with_value))
-        RETURN start_node, attr_path as potential_path, NULL as rel_identifier, attr.name as potential_attr, attr_value.value as potential_attr_value
         """ % {"node_kind": self.query_request.kind}
 
         relationship_attr_paths_with_value_subquery = """
-        MATCH rel_path = (start_node:%(node_kind)s)-[:IS_RELATED]-(relationship_node:Relationship)-[:IS_RELATED]-(related_n:Node)-[:HAS_ATTRIBUTE]->(rel_attr:Attribute)-[:HAS_VALUE]->(rel_attr_value:AttributeValue)
+        OPTIONAL MATCH rel_path = (attr_start_node:%(node_kind)s)-[:IS_RELATED]-(relationship_node:Relationship)-[:IS_RELATED]-(related_n:Node)-[:HAS_ATTRIBUTE]->(rel_attr:Attribute)-[:HAS_VALUE]->(rel_attr_value:AttributeValue)
         WHERE relationship_node.name in $relationship_names
             AND ([relationship_node.name, rel_attr.name] in $relationship_attr_paths
             OR (rel_attr_value.value in $relationship_attr_values AND [relationship_node.name, rel_attr.name, rel_attr_value.value] in $relationship_attr_paths_with_value))
-        RETURN start_node, rel_path as potential_path, relationship_node.name as rel_identifier, rel_attr.name as potential_attr, rel_attr_value.value as potential_attr_value
-        """ % {"node_kind": self.query_request.kind}
+         """ % {"node_kind": self.query_request.kind}
 
         relationship_only_attr_paths_subquery = """
-        MATCH rel_path = (start_node:%(node_kind)s)-[:IS_RELATED]-(relationship_node:Relationship)-[:IS_RELATED]-(related_n:Node)
+        OPTIONAL MATCH rel_path = (attr_start_node:%(node_kind)s)-[:IS_RELATED]-(relationship_node:Relationship)-[:IS_RELATED]-(related_n:Node)
         WHERE %(rel_node_filter)s relationship_node.name in $relationship_only_attr_paths
-        RETURN start_node, rel_path as potential_path, relationship_node.name as rel_identifier, "id" as potential_attr, related_n.uuid as potential_attr_value
         """ % {
             "node_kind": self.query_request.kind,
             "rel_node_filter": "related_n.uuid IN $relationship_only_attr_values AND "
             if relationship_only_attr_values
             else "",
         }
 
+        attr_paths_filter_subquery = """
+        WITH attr_start_node, attr_path, attr, attr_value
+        RETURN attr_start_node AS start_node, attr_path AS potential_path, NULL as rel_identifier, attr.name as potential_attr, attr_value.value as potential_attr_value
+        """
+
+        relationship_attr_paths_with_value_filter_subquery = """
+        WITH attr_start_node, rel_path, relationship_node, related_n, rel_attr, rel_attr_value
+        RETURN attr_start_node AS start_node, rel_path AS potential_path, relationship_node.name as rel_identifier, rel_attr.name as potential_attr, rel_attr_value.value as potential_attr_value
+        """
+
+        relationship_only_attr_paths_filter_subquery = """
+        WITH attr_start_node, rel_path, relationship_node, related_n
+        RETURN attr_start_node AS start_node, rel_path as potential_path, relationship_node.name as rel_identifier, "id" as potential_attr, related_n.uuid as potential_attr_value
+        """
+
         select_subqueries = []
+        filter_subqueries = []
+        returned_attributes = ["attr_start_node"]
         if attr_paths or attr_paths_with_value:
             select_subqueries.append(attr_paths_subquery)
+            filter_subqueries.append(attr_paths_filter_subquery)
+            returned_attributes.extend(["attr_path", "attr", "attr_value"])
         if relationship_attr_paths_with_value or relationship_attr_paths:
             select_subqueries.append(relationship_attr_paths_with_value_subquery)
+            filter_subqueries.append(relationship_attr_paths_with_value_filter_subquery)
+            returned_attributes.extend(["rel_path", "relationship_node", "related_n", "rel_attr", "rel_attr_value"])
         if relationship_only_attr_paths:
             select_subqueries.append(relationship_only_attr_paths_subquery)
+            filter_subqueries.append(relationship_only_attr_paths_filter_subquery)
+            returned_attributes.extend(["rel_path", "relationship_node", "related_n"])
 
-        select_subqueries_str = "UNION".join(select_subqueries)
+        select_subqueries_str = "".join(select_subqueries)
+        return_subqueries_str = ", ".join(returned_attributes)
+        filter_subqueries_str = "UNION".join(filter_subqueries)
 
         # ruff: noqa: E501
         query = """
         // get attributes for node and its relationships
         CALL {
             %(select_subqueries_str)s
+            RETURN %(return_subqueries_str)s
+        }
+        CALL {
+            %(filter_subqueries_str)s
         }
+        WITH start_node, potential_path, rel_identifier, potential_attr, potential_attr_value
         CALL {
             WITH potential_path
             WITH potential_path  // workaround for neo4j not allowing WHERE in a WITH of a subquery
@@ -211,6 +238,8 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: Any) -> None:  # noqa
             relationship_identifier
         """ % {
             "select_subqueries_str": select_subqueries_str,
+            "return_subqueries_str": return_subqueries_str,
+            "filter_subqueries_str": filter_subqueries_str,
             "branch_filter": branch_filter,
             "from_times": from_times,
             "branch_name_and_level": branch_name_and_level,