fix pattern lookup matching on F expressions

Author: timgraham

.github/workflows/test-python.yml

@@ -76,7 +76,11 @@ jobs:
           datetimes
           db_functions
           empty
+          expressions.tests.BasicExpressionsTests.test_ticket_11722_iexact_lookup
+          expressions.tests.BasicExpressionsTests.test_ticket_16731_startswith_lookup
           expressions.tests.ExpressionOperatorTests
+          expressions.tests.ExpressionsTests.test_insensitive_patterns_escape
+          expressions.tests.ExpressionsTests.test_patterns_escape
           expressions.tests.FieldTransformTests.test_transform_in_values
           expressions.tests.NegatedExpressionTests
           expressions_case

django_mongodb/base.py

@@ -1,5 +1,3 @@
-import re
-
 from django.core.exceptions import ImproperlyConfigured
 from django.db.backends.base.base import BaseDatabaseWrapper
 from django.db.backends.signals import connection_created
@@ -82,15 +80,15 @@ class DatabaseWrapper(BaseDatabaseWrapper):
         "in": lambda a, b: {"$in": [a, b]},
         "isnull": lambda a, b: {("$eq" if b else "$ne"): [a, None]},
         "range": lambda a, b: {"$and": [{"$gte": [a, b[0]]}, {"$lte": [a, b[1]]}]},
-        "iexact": lambda a, b: regex_match(a, b, "^%s$", re.IGNORECASE),
-        "startswith": lambda a, b: regex_match(a, b, "^%s"),
-        "istartswith": lambda a, b: regex_match(a, b, "^%s", re.IGNORECASE),
-        "endswith": lambda a, b: regex_match(a, b, "%s$"),
-        "iendswith": lambda a, b: regex_match(a, b, "%s$", re.IGNORECASE),
-        "contains": lambda a, b: regex_match(a, b, "%s"),
-        "icontains": lambda a, b: regex_match(a, b, "%s", re.IGNORECASE),
-        "regex": lambda a, b: regex_match(a, "", f"{b}%s"),
-        "iregex": lambda a, b: regex_match(a, "", f"{b}%s", re.IGNORECASE),
+        "iexact": lambda a, b: regex_match(a, ("^", b, {"$literal": "$"}), insensitive=True),
+        "startswith": lambda a, b: regex_match(a, ("^", b)),
+        "istartswith": lambda a, b: regex_match(a, ("^", b), insensitive=True),
+        "endswith": lambda a, b: regex_match(a, (b, {"$literal": "$"})),
+        "iendswith": lambda a, b: regex_match(a, (b, {"$literal": "$"}), insensitive=True),
+        "contains": lambda a, b: regex_match(a, b),
+        "icontains": lambda a, b: regex_match(a, b, insensitive=True),
+        "regex": lambda a, b: regex_match(a, b),
+        "iregex": lambda a, b: regex_match(a, b, insensitive=True),
     }
 
     display_name = "MongoDB"

django_mongodb/features.py

@@ -43,8 +43,6 @@ class DatabaseFeatures(BaseDatabaseFeatures):
         "lookup.tests.LookupTests.test_exact_sliced_queryset_limit_one_offset",
         # Regex lookup doesn't work on non-string fields.
         "lookup.tests.LookupTests.test_regex_non_string",
-        # Substr not implemented.
-        "lookup.tests.LookupTests.test_pattern_lookups_with_substr",
         # Querying ObjectID with string doesn't work.
         "lookup.tests.LookupTests.test_lookup_int_as_str",
         # MongoDB gives the wrong result of log(number, base) when base is a

django_mongodb/lookups.py

@@ -1,6 +1,6 @@
 from django.db import NotSupportedError
 from django.db.models.fields.related_lookups import In, MultiColSource, RelatedIn
-from django.db.models.lookups import BuiltinLookup, IsNull, UUIDTextMixin
+from django.db.models.lookups import BuiltinLookup, IsNull, PatternLookup, UUIDTextMixin
 
 from .query_utils import process_lhs, process_rhs
 
@@ -24,6 +24,24 @@ def is_null(self, compiler, connection):
     return connection.mongo_operators["isnull"](lhs_mql, self.rhs)
 
 
+def pattern_lookup_prep_lookup_value(self, value):
+    if hasattr(self.rhs, "as_mql"):
+        # If value is a column reference, escape regex special characters.
+        # Analogous to PatternLookup.get_rhs_op() / pattern_esc.
+        for find, replacement in (("\\", r"\\"), ("%", r"\%"), ("_", r"\_")):
+            value = {"$replaceAll": {"input": value, "find": find, "replacement": replacement}}
+    else:
+        # If value is a literal, remove percent signs added by
+        # PatternLookup.process_rhs() for LIKE queries.
+        if self.lookup_name in ("startswith", "istartswith"):
+            value = value[:-1]
+        elif self.lookup_name in ("endswith", "iendswith"):
+            value = value[1:]
+        elif self.lookup_name in ("contains", "icontains"):
+            value = value[1:-1]
+    return value
+
+
 def uuid_text_mixin(self, compiler, connection):  # noqa: ARG001
     raise NotSupportedError("Pattern lookups on UUIDField are not supported.")
 
@@ -32,4 +50,5 @@ def register_lookups():
     BuiltinLookup.as_mql = builtin_lookup
     In.as_mql = RelatedIn.as_mql = in_
     IsNull.as_mql = is_null
+    PatternLookup.prep_lookup_value_mongo = pattern_lookup_prep_lookup_value
     UUIDTextMixin.as_mql = uuid_text_mixin

django_mongodb/operations.py

@@ -1,4 +1,5 @@
 import datetime
+import re
 import uuid
 
 from bson.decimal128 import Decimal128
@@ -108,8 +109,8 @@ def combine_expression(self, connector, sub_expressions):
         return {f"${operator}": sub_expressions}
 
     def prep_for_like_query(self, x):
-        # Override value escaping for LIKE queries.
-        return str(x)
+        """Escape "x" for $regexMatch queries."""
+        return re.escape(x)
 
     def quote_name(self, name):
         if name.startswith('"') and name.endswith('"'):

django_mongodb/query_utils.py

@@ -1,5 +1,3 @@
-import re
-
 from django.core.exceptions import FullResultSet
 from django.db.models.expressions import Value
 
@@ -27,25 +25,19 @@ def process_lhs(node, compiler, connection):
 def process_rhs(node, compiler, connection):
     rhs = node.rhs
     if hasattr(rhs, "as_mql"):
-        return rhs.as_mql(compiler, connection)
-    _, value = node.process_rhs(compiler, connection)
-    lookup_name = node.lookup_name
-    # Undo Lookup.get_db_prep_lookup() putting params in a list.
-    if lookup_name not in ("in", "range"):
-        value = value[0]
-    # Remove percent signs added by PatternLookup.process_rhs() for LIKE
-    # queries.
-    if lookup_name in ("startswith", "istartswith"):
-        value = value[:-1]
-    elif lookup_name in ("endswith", "iendswith"):
-        value = value[1:]
-    elif lookup_name in ("contains", "icontains"):
-        value = value[1:-1]
-
+        value = rhs.as_mql(compiler, connection)
+    else:
+        _, value = node.process_rhs(compiler, connection)
+        lookup_name = node.lookup_name
+        # Undo Lookup.get_db_prep_lookup() putting params in a list.
+        if lookup_name not in ("in", "range"):
+            value = value[0]
+    if hasattr(node, "prep_lookup_value_mongo"):
+        value = node.prep_lookup_value_mongo(value)
     return connection.ops.prep_lookup_value(value, node.lhs.output_field, node.lookup_name)
 
 
-def regex_match(field, value, regex, *re_args, **re_kwargs):
-    regex = re.compile(regex % re.escape(value), *re_args, **re_kwargs)
-    options = "i" if regex.flags & re.I else ""
-    return {"$regexMatch": {"input": field, "regex": regex.pattern, "options": options}}
+def regex_match(field, regex_vals, insensitive=False):
+    regex = {"$concat": regex_vals} if isinstance(regex_vals, tuple) else regex_vals
+    options = "i" if insensitive else ""
+    return {"$regexMatch": {"input": field, "regex": regex, "options": options}}