Skip to content

Commit 9019a33

Browse files
WaVEVtimgraham
andcommitted
add support for JSONField
Co-authored-by: Tim Graham <[email protected]>
1 parent 5f2a094 commit 9019a33

File tree

9 files changed

+277
-5
lines changed

9 files changed

+277
-5
lines changed

README.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,17 @@ Migrations for 'admin':
138138
- The `tzinfo` parameter of the `Trunc` database functions doesn't work
139139
properly because MongoDB converts the result back to UTC.
140140

141+
- When querying `JSONField`:
142+
- There is no way to distinguish between a JSON "null" (represented by
143+
`Value(None, JSONField())`) and a SQL null (queried using the `isnull`
144+
lookup). Both of these queries return both of these nulls.
145+
- Some queries with `Q` objects, e.g. `Q(value__foo="bar")`, don't work
146+
properly, particularly with `QuerySet.exclude()`.
147+
- Filtering for a `None` key, e.g. `QuerySet.filter(value__j=None)`
148+
incorrectly returns objects where the key doesn't exist.
149+
- You can study the skipped tests in `DatabaseFeatures.django_test_skips` for
150+
more details on known issues.
151+
141152
## Troubleshooting
142153

143154
TODO

django_mongodb/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,13 @@
77
check_django_compatability()
88

99
from .expressions import register_expressions # noqa: E402
10+
from .fields import register_fields # noqa: E402
1011
from .functions import register_functions # noqa: E402
1112
from .lookups import register_lookups # noqa: E402
1213
from .query import register_nodes # noqa: E402
1314

1415
register_expressions()
16+
register_fields()
1517
register_functions()
1618
register_lookups()
1719
register_nodes()

django_mongodb/base.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ class DatabaseWrapper(BaseDatabaseWrapper):
4242
"IntegerField": "int",
4343
"BigIntegerField": "long",
4444
"GenericIPAddressField": "string",
45+
"JSONField": "object",
4546
"OneToOneField": "int",
4647
"PositiveBigIntegerField": "int",
4748
"PositiveIntegerField": "long",

django_mongodb/features.py

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@
55
class DatabaseFeatures(BaseDatabaseFeatures):
66
greatest_least_ignores_nulls = True
77
has_json_object_function = False
8+
has_native_json_field = True
89
supports_date_lookup_using_string = False
910
supports_foreign_keys = False
1011
supports_ignore_conflicts = False
11-
# Not implemented: https://github.com/mongodb-labs/django-mongodb/issues/8
12-
supports_json_field = False
12+
supports_json_field_contains = False
1313
# BSON Date type doesn't support microsecond precision.
1414
supports_microsecond_precision = False
1515
# MongoDB stores datetimes in UTC.
@@ -41,6 +41,9 @@ class DatabaseFeatures(BaseDatabaseFeatures):
4141
# tuple index out of range in process_rhs()
4242
"lookup.tests.LookupTests.test_exact_sliced_queryset_limit_one",
4343
"lookup.tests.LookupTests.test_exact_sliced_queryset_limit_one_offset",
44+
# Pattern lookups that use regexMatch don't work on JSONField:
45+
# Unsupported conversion from array to string in $convert
46+
"model_fields.test_jsonfield.TestQuerying.test_icontains",
4447
# MongoDB gives the wrong result of log(number, base) when base is a
4548
# fractional Decimal: https://jira.mongodb.org/browse/SERVER-91223
4649
"db_functions.math.test_log.LogTests.test_decimal",
@@ -53,8 +56,14 @@ class DatabaseFeatures(BaseDatabaseFeatures):
5356
# pk__in=queryset doesn't work because subqueries aren't a thing in
5457
# MongoDB.
5558
"annotations.tests.NonAggregateAnnotationTestCase.test_annotation_and_alias_filter_in_subquery",
59+
"model_fields.test_jsonfield.TestQuerying.test_usage_in_subquery",
5660
# Length of null considered zero rather than null.
5761
"db_functions.text.test_length.LengthTests.test_basic",
62+
# Key transforms are incorrectly treated as joins:
63+
# Ordering can't span tables on MongoDB (value_custom__a).
64+
"model_fields.test_jsonfield.TestQuerying.test_order_grouping_custom_decoder",
65+
"model_fields.test_jsonfield.TestQuerying.test_ordering_by_transform",
66+
"model_fields.test_jsonfield.TestQuerying.test_ordering_grouping_by_key_transform",
5867
}
5968
# $bitAnd, #bitOr, and $bitXor are new in MongoDB 6.3.
6069
_django_test_expected_failures_bitwise = {
@@ -221,6 +230,8 @@ def django_test_expected_failures(self):
221230
"db_functions.datetime.test_extract_trunc.DateFunctionTests.test_trunc_subquery_with_parameters",
222231
"expressions_case.tests.CaseExpressionTests.test_in_subquery",
223232
"lookup.tests.LookupQueryingTests.test_filter_subquery_lhs",
233+
"model_fields.test_jsonfield.TestQuerying.test_nested_key_transform_on_subquery",
234+
"model_fields.test_jsonfield.TestQuerying.test_obj_subquery_lookup",
224235
# Invalid $project :: caused by :: Unknown expression $count,
225236
"annotations.tests.NonAggregateAnnotationTestCase.test_combined_expression_annotation_with_aggregation",
226237
"annotations.tests.NonAggregateAnnotationTestCase.test_combined_f_expression_annotation_with_aggregation",
@@ -240,6 +251,7 @@ def django_test_expected_failures(self):
240251
"expressions.tests.NegatedExpressionTests.test_filter",
241252
"expressions_case.tests.CaseExpressionTests.test_annotate_values_not_in_order_by",
242253
"expressions_case.tests.CaseExpressionTests.test_order_by_conditional_implicit",
254+
"model_fields.test_jsonfield.TestQuerying.test_ordering_grouping_by_count",
243255
# annotate().filter().count() gives incorrect results.
244256
"db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_year_exact_lookup",
245257
},
@@ -324,6 +336,7 @@ def django_test_expected_failures(self):
324336
"lookup.tests.LookupTests.test_lookup_collision",
325337
"lookup.tests.LookupTests.test_lookup_rhs",
326338
"lookup.tests.LookupTests.test_isnull_non_boolean_value",
339+
"model_fields.test_jsonfield.TestQuerying.test_join_key_transform_annotation_expression",
327340
"model_fields.test_manytomanyfield.ManyToManyFieldDBTests.test_value_from_object_instance_with_pk",
328341
"model_fields.test_uuid.TestAsPrimaryKey.test_two_level_foreign_keys",
329342
"timezones.tests.LegacyDatabaseTests.test_query_annotation",
@@ -343,6 +356,9 @@ def django_test_expected_failures(self):
343356
},
344357
"Test executes raw SQL.": {
345358
"annotations.tests.NonAggregateAnnotationTestCase.test_raw_sql_with_inherited_field",
359+
"model_fields.test_jsonfield.TestQuerying.test_key_sql_injection_escape",
360+
"model_fields.test_jsonfield.TestQuerying.test_key_transform_raw_expression",
361+
"model_fields.test_jsonfield.TestQuerying.test_nested_key_transform_raw_expression",
346362
"timezones.tests.LegacyDatabaseTests.test_cursor_execute_accepts_naive_datetime",
347363
"timezones.tests.LegacyDatabaseTests.test_cursor_execute_returns_naive_datetime",
348364
"timezones.tests.LegacyDatabaseTests.test_raw_sql",
@@ -401,6 +417,24 @@ def django_test_expected_failures(self):
401417
"db_functions.comparison.test_cast.CastTests.test_cast_from_python_to_datetime",
402418
"db_functions.comparison.test_cast.CastTests.test_cast_to_duration",
403419
},
420+
"Known issue querying JSONField.": {
421+
# An ExpressionWrapper annotation with KeyTransform followed by
422+
# .filter(expr__isnull=False) doesn't use KeyTransformIsNull as it
423+
# needs to.
424+
"model_fields.test_jsonfield.TestQuerying.test_expression_wrapper_key_transform",
425+
# There is no way to distinguish between a JSON "null" (represented
426+
# by Value(None, JSONField())) and a SQL null (queried using the
427+
# isnull lookup). Both of these queries return both nulls.
428+
"model_fields.test_jsonfield.TestSaveLoad.test_json_null_different_from_sql_null",
429+
# Some queries with Q objects, e.g. Q(value__foo="bar"), don't work
430+
# properly, particularly with QuerySet.exclude().
431+
"model_fields.test_jsonfield.TestQuerying.test_lookup_exclude",
432+
"model_fields.test_jsonfield.TestQuerying.test_lookup_exclude_nonexistent_key",
433+
# Queries like like QuerySet.filter(value__j=None) incorrectly
434+
# returns objects where the key doesn't exist.
435+
"model_fields.test_jsonfield.TestQuerying.test_none_key",
436+
"model_fields.test_jsonfield.TestQuerying.test_none_key_exclude",
437+
},
404438
}
405439

406440
@cached_property

django_mongodb/fields/__init__.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
11
from .auto import MongoAutoField
2+
from .json import register_json_field
23

3-
__all__ = ["MongoAutoField"]
4+
__all__ = ["register_fields", "MongoAutoField"]
5+
6+
7+
def register_fields():
8+
register_json_field()

django_mongodb/fields/json.py

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
from django.db import NotSupportedError
2+
from django.db.models.fields.json import (
3+
ContainedBy,
4+
DataContains,
5+
HasAnyKeys,
6+
HasKey,
7+
HasKeyLookup,
8+
HasKeys,
9+
JSONExact,
10+
KeyTransform,
11+
KeyTransformIn,
12+
KeyTransformIsNull,
13+
KeyTransformNumericLookupMixin,
14+
)
15+
16+
from ..lookups import builtin_lookup
17+
from ..query_utils import process_lhs, process_rhs
18+
19+
20+
def contained_by(self, compiler, connection): # noqa: ARG001
21+
raise NotSupportedError("contained_by lookup is not supported on this database backend.")
22+
23+
24+
def data_contains(self, compiler, connection): # noqa: ARG001
25+
raise NotSupportedError("contains lookup is not supported on this database backend.")
26+
27+
28+
def _has_key_predicate(path, root_column, negated=False):
29+
"""Return MQL to check for the existence of `path`."""
30+
result = {
31+
"$and": [
32+
# The path must exist (i.e. not be "missing").
33+
{"$ne": [{"$type": path}, "missing"]},
34+
# If the JSONField value is None, an additional check for not null
35+
# is needed since $type returns null instead of "missing".
36+
{"$ne": [root_column, None]},
37+
]
38+
}
39+
if negated:
40+
result = {"$not": result}
41+
return result
42+
43+
44+
def has_key_lookup(self, compiler, connection):
45+
"""Return MQL to check for the existence of a key."""
46+
rhs = self.rhs
47+
lhs = process_lhs(self, compiler, connection)
48+
if not isinstance(rhs, list | tuple):
49+
rhs = [rhs]
50+
paths = []
51+
# Transform any "raw" keys into KeyTransforms to allow consistent handling
52+
# in the code that follows.
53+
for key in rhs:
54+
rhs_json_path = key if isinstance(key, KeyTransform) else KeyTransform(key, self.lhs)
55+
paths.append(rhs_json_path.as_mql(compiler, connection))
56+
keys = []
57+
for path in paths:
58+
keys.append(_has_key_predicate(path, lhs))
59+
if self.mongo_operator is None:
60+
return keys[0]
61+
return {self.mongo_operator: keys}
62+
63+
64+
_process_rhs = JSONExact.process_rhs
65+
66+
67+
def json_exact_process_rhs(self, compiler, connection):
68+
"""Skip JSONExact.process_rhs()'s conversion of None to "null"."""
69+
return (
70+
super(JSONExact, self).process_rhs(compiler, connection)
71+
if connection.vendor == "mongodb"
72+
else _process_rhs(self, compiler, connection)
73+
)
74+
75+
76+
def key_transform(self, compiler, connection):
77+
"""
78+
Return MQL for this KeyTransform (JSON path).
79+
80+
JSON paths cannot always be represented simply as $var.key1.key2.key3 due
81+
to possible array types. Therefore, indexing arrays requires the use of
82+
`arrayElemAt`. Additionally, $cond is necessary to verify the type before
83+
performing the operation.
84+
"""
85+
key_transforms = [self.key_name]
86+
previous = self.lhs
87+
# Collect all key transforms in order.
88+
while isinstance(previous, KeyTransform):
89+
key_transforms.insert(0, previous.key_name)
90+
previous = previous.lhs
91+
lhs_mql = previous.as_mql(compiler, connection)
92+
result = lhs_mql
93+
# Build the MQL path using the collected key transforms.
94+
for key in key_transforms:
95+
get_field = {"$getField": {"input": result, "field": key}}
96+
# Handle array indexing if the key is a digit. If key is something
97+
# like '001', it's not an array index despite isdigit() returning True.
98+
if key.isdigit() and str(int(key)) == key:
99+
result = {
100+
"$cond": {
101+
"if": {"$isArray": result},
102+
"then": {"$arrayElemAt": [result, int(key)]},
103+
"else": get_field,
104+
}
105+
}
106+
else:
107+
result = get_field
108+
return result
109+
110+
111+
def key_transform_in(self, compiler, connection):
112+
"""
113+
Return MQL to check if a JSON path exists and that its values are in the
114+
set of specified values (rhs).
115+
"""
116+
lhs_mql = process_lhs(self, compiler, connection)
117+
# Traverse to the root column.
118+
previous = self.lhs
119+
while isinstance(previous, KeyTransform):
120+
previous = previous.lhs
121+
root_column = previous.as_mql(compiler, connection)
122+
value = process_rhs(self, compiler, connection)
123+
# Construct the expression to check if lhs_mql values are in rhs values.
124+
expr = connection.mongo_operators[self.lookup_name](lhs_mql, value)
125+
return {"$and": [_has_key_predicate(lhs_mql, root_column), expr]}
126+
127+
128+
def key_transform_is_null(self, compiler, connection):
129+
"""
130+
Return MQL to check the nullability of a key.
131+
132+
If `isnull=True`, the query matches objects where the key is missing or the
133+
root column is null. If `isnull=False`, the query negates the result to
134+
match objects where the key exists.
135+
136+
Reference: https://code.djangoproject.com/ticket/32252
137+
"""
138+
lhs_mql = process_lhs(self, compiler, connection)
139+
rhs_mql = process_rhs(self, compiler, connection)
140+
# Get the root column.
141+
previous = self.lhs
142+
while isinstance(previous, KeyTransform):
143+
previous = previous.lhs
144+
root_column = previous.as_mql(compiler, connection)
145+
return _has_key_predicate(lhs_mql, root_column, negated=rhs_mql)
146+
147+
148+
def key_transform_numeric_lookup_mixin(self, compiler, connection):
149+
"""
150+
Return MQL to check if the field exists (i.e., is not "missing" or "null")
151+
and that the field matches the given numeric lookup expression.
152+
"""
153+
expr = builtin_lookup(self, compiler, connection)
154+
lhs = process_lhs(self, compiler, connection)
155+
# Check if the type of lhs is not "missing" or "null".
156+
not_missing_or_null = {"$not": {"$in": [{"$type": lhs}, ["missing", "null"]]}}
157+
return {"$and": [expr, not_missing_or_null]}
158+
159+
160+
def register_json_field():
161+
ContainedBy.as_mql = contained_by
162+
DataContains.as_mql = data_contains
163+
HasAnyKeys.mongo_operator = "$or"
164+
HasKey.mongo_operator = None
165+
HasKeyLookup.as_mql = has_key_lookup
166+
HasKeys.mongo_operator = "$and"
167+
JSONExact.process_rhs = json_exact_process_rhs
168+
KeyTransform.as_mql = key_transform
169+
KeyTransformIn.as_mql = key_transform_in
170+
KeyTransformIsNull.as_mql = key_transform_is_null
171+
KeyTransformNumericLookupMixin.as_mql = key_transform_numeric_lookup_mixin

django_mongodb/functions.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,11 @@ def cast(self, compiler, connection):
6262
lhs_mql = process_lhs(self, compiler, connection)[0]
6363
if max_length := self.output_field.max_length:
6464
lhs_mql = {"$substrCP": [lhs_mql, 0, max_length]}
65-
lhs_mql = {"$convert": {"input": lhs_mql, "to": output_type}}
65+
# Skip the conversion for "object" as it doesn't need to be transformed for
66+
# interpretation by JSONField, which can handle types including int,
67+
# object, or array.
68+
if output_type != "object":
69+
lhs_mql = {"$convert": {"input": lhs_mql, "to": output_type}}
6670
if decimal_places := getattr(self.output_field, "decimal_places", None):
6771
lhs_mql = {"$trunc": [lhs_mql, decimal_places]}
6872
return lhs_mql

django_mongodb/lookups.py

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
from django.db import NotSupportedError
22
from django.db.models.fields.related_lookups import In, MultiColSource, RelatedIn
3-
from django.db.models.lookups import BuiltinLookup, IsNull, PatternLookup, UUIDTextMixin
3+
from django.db.models.lookups import (
4+
BuiltinLookup,
5+
FieldGetDbPrepValueIterableMixin,
6+
IsNull,
7+
PatternLookup,
8+
UUIDTextMixin,
9+
)
410

511
from .query_utils import process_lhs, process_rhs
612

@@ -11,6 +17,22 @@ def builtin_lookup(self, compiler, connection):
1117
return connection.mongo_operators[self.lookup_name](lhs_mql, value)
1218

1319

20+
_field_resolve_expression_parameter = FieldGetDbPrepValueIterableMixin.resolve_expression_parameter
21+
22+
23+
def field_resolve_expression_parameter(self, compiler, connection, sql, param):
24+
"""For MongoDB, this method must call as_mql() instead of as_sql()."""
25+
sql, sql_params = _field_resolve_expression_parameter(self, compiler, connection, sql, param)
26+
if connection.vendor == "mongodb":
27+
params = [param]
28+
if hasattr(param, "resolve_expression"):
29+
param = param.resolve_expression(compiler.query)
30+
if hasattr(param, "as_mql"):
31+
params = [param.as_mql(compiler, connection)]
32+
return sql, params
33+
return sql, sql_params
34+
35+
1436
def in_(self, compiler, connection):
1537
if isinstance(self.lhs, MultiColSource):
1638
raise NotImplementedError("MultiColSource is not supported.")
@@ -48,6 +70,9 @@ def uuid_text_mixin(self, compiler, connection): # noqa: ARG001
4870

4971
def register_lookups():
5072
BuiltinLookup.as_mql = builtin_lookup
73+
FieldGetDbPrepValueIterableMixin.resolve_expression_parameter = (
74+
field_resolve_expression_parameter
75+
)
5176
In.as_mql = RelatedIn.as_mql = in_
5277
IsNull.as_mql = is_null
5378
PatternLookup.prep_lookup_value_mongo = pattern_lookup_prep_lookup_value

0 commit comments

Comments
 (0)