diff --git a/.github/workflows/mongodb_settings.py b/.github/workflows/mongodb_settings.py index 49d44a5f..28ff964f 100644 --- a/.github/workflows/mongodb_settings.py +++ b/.github/workflows/mongodb_settings.py @@ -15,13 +15,13 @@ else: DATABASES = { "default": { - "ENGINE": "django_mongodb_backend", + "ENGINE": "django_mongodb_backend_gis", "NAME": "djangotests", # Required when connecting to the Atlas image in Docker. "OPTIONS": {"directConnection": True}, }, "other": { - "ENGINE": "django_mongodb_backend", + "ENGINE": "django_mongodb_backend_gis", "NAME": "djangotests-other", "OPTIONS": {"directConnection": True}, }, diff --git a/.github/workflows/runtests.py b/.github/workflows/runtests.py index ebcc4876..4b6a6e59 100755 --- a/.github/workflows/runtests.py +++ b/.github/workflows/runtests.py @@ -4,158 +4,7 @@ import sys test_apps = [ - "admin_changelist", - "admin_checks", - "admin_custom_urls", - "admin_docs", - "admin_filters", - "admin_inlines", - "admin_ordering", - "admin_scripts", - "admin_utils", - "admin_views", - "admin_widgets", - "aggregation", - "aggregation_regress", - "annotations", - "apps", - "async", - "auth_tests", - "backends", - "basic", - "bulk_create", - "cache", - "check_framework", - "constraints", - "contenttypes_tests", - "context_processors", - "custom_columns", - "custom_lookups", - "custom_managers", - "custom_pk", - "datatypes", - "dates", - "datetimes", - "db_functions", - "defer", - "defer_regress", - "delete", - "delete_regress", - "empty", - "empty_models", - "expressions", - "expressions_case", - "field_defaults", - "file_storage", - "file_uploads", - "fixtures", - "fixtures_model_package", - "fixtures_regress", - "flatpages_tests", - "force_insert_update", - "foreign_object", - "forms_tests", - "from_db_value", - "generic_inline_admin", - "generic_relations", - "generic_relations_regress", - "generic_views", - "get_earliest_or_latest", - "get_object_or_404", - "get_or_create", - "i18n", - "indexes", - "inline_formsets", - "introspection", - "invalid_models_tests", - "known_related_objects", - "lookup", - "m2m_and_m2o", - "m2m_intermediary", - "m2m_multiple", - "m2m_recursive", - "m2m_regress", - "m2m_signals", - "m2m_through", - "m2m_through_regress", - "m2o_recursive", - "managers_regress", - "many_to_many", - "many_to_one", - "many_to_one_null", - "max_lengths", - "messages_tests", - "migrate_signals", - "migration_test_data_persistence", - "migrations", - "model_fields", - "model_forms", - "model_formsets", - "model_formsets_regress", - "model_indexes", - "model_inheritance", - "model_inheritance_regress", - "model_options", - "model_package", - "model_regress", - "model_utils", - "modeladmin", - "multiple_database", - "mutually_referential", - "nested_foreign_keys", - "null_fk", - "null_fk_ordering", - "null_queries", - "one_to_one", - "or_lookups", - "order_with_respect_to", - "ordering", - "pagination", - "prefetch_related", - "proxy_model_inheritance", - "proxy_models", - "queries", - "queryset_pickle", - "redirects_tests", - "reserved_names", - "reverse_lookup", - "save_delete_hooks", - "schema", - "select_for_update", - "select_related", - "select_related_onetoone", - "select_related_regress", - "serializers", - "servers", - "sessions_tests", - "shortcuts", - "signals", - "sitemaps_tests", - "sites_framework", - "sites_tests", - "string_lookup", - "swappable_models", - "syndication_tests", - "test_client", - "test_client_regress", - "test_runner", - "test_utils", - "timezones", - "transactions", - "unmanaged_models", - "update", - "update_only_fields", - "user_commands", - "validation", - "view_tests", - "xor_lookups", - # Add directories in django_mongodb_backend/tests - *sorted( - [ - x.name - for x in (pathlib.Path(__file__).parent.parent.parent.resolve() / "tests").iterdir() - ] - ), + "gis_tests", ] runtests = pathlib.Path(__file__).parent.resolve() / "runtests.py" run_tests_cmd = f"python3 {runtests} %s --settings mongodb_settings -v 2" diff --git a/.github/workflows/test-python-atlas.yml b/.github/workflows/test-python-atlas.yml index 175dfe18..c6727e75 100644 --- a/.github/workflows/test-python-atlas.yml +++ b/.github/workflows/test-python-atlas.yml @@ -33,13 +33,13 @@ jobs: uses: actions/checkout@v4 with: repository: 'mongodb-forks/django' - ref: 'mongodb-5.2.x' + ref: 'mongogis' path: 'django_repo' persist-credentials: false - name: Install system packages for Django's Python test dependencies run: | sudo apt-get update - sudo apt-get install libmemcached-dev + sudo apt-get install gdal-bin libmemcached-dev - name: Install Django and its Python test dependencies run: | cd django_repo/tests/ diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index d471f3f2..2dc4663b 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -33,13 +33,13 @@ jobs: uses: actions/checkout@v4 with: repository: 'mongodb-forks/django' - ref: 'mongodb-5.2.x' + ref: 'mongogis' path: 'django_repo' persist-credentials: false - name: Install system packages for Django's Python test dependencies run: | sudo apt-get update - sudo apt-get install libmemcached-dev + sudo apt-get install gdal-bin libmemcached-dev - name: Install Django and its Python test dependencies run: | cd django_repo/tests/ diff --git a/django_mongodb_backend/introspection.py b/django_mongodb_backend/introspection.py index 77e06807..714f2246 100644 --- a/django_mongodb_backend/introspection.py +++ b/django_mongodb_backend/introspection.py @@ -1,12 +1,12 @@ from django.db.backends.base.introspection import BaseDatabaseIntrospection from django.db.models import Index -from pymongo import ASCENDING, DESCENDING +from pymongo import ASCENDING, DESCENDING, GEOSPHERE from django_mongodb_backend.indexes import SearchIndex, VectorSearchIndex class DatabaseIntrospection(BaseDatabaseIntrospection): - ORDER_DIR = {ASCENDING: "ASC", DESCENDING: "DESC"} + ORDER_DIR = {ASCENDING: "ASC", DESCENDING: "DESC", GEOSPHERE: "GEO"} def table_names(self, cursor=None, include_views=False): return sorted([x["name"] for x in self.connection.database.list_collections()]) diff --git a/django_mongodb_backend_gis/__init__.py b/django_mongodb_backend_gis/__init__.py new file mode 100644 index 00000000..8cf3b45c --- /dev/null +++ b/django_mongodb_backend_gis/__init__.py @@ -0,0 +1,5 @@ +from .functions import register_functions +from .lookups import register_lookups + +register_functions() +register_lookups() diff --git a/django_mongodb_backend_gis/adapter.py b/django_mongodb_backend_gis/adapter.py new file mode 100644 index 00000000..e3eba4a4 --- /dev/null +++ b/django_mongodb_backend_gis/adapter.py @@ -0,0 +1,21 @@ +import collections + + +class Adapter(collections.UserDict): + def __init__(self, obj, geography=False): + """ + Initialize on the spatial object. + """ + if obj.__class__.__name__ == "GeometryCollection": + self.data = { + "type": obj.__class__.__name__, + "geometries": [self.get_data(x) for x in obj], + } + else: + self.data = self.get_data(obj) + + def get_data(self, obj): + return { + "type": obj.__class__.__name__, + "coordinates": obj.coords[0] if obj.__class__.__name__ == "Polygon" else obj.coords, + } diff --git a/django_mongodb_backend_gis/base.py b/django_mongodb_backend_gis/base.py new file mode 100644 index 00000000..b9eda04d --- /dev/null +++ b/django_mongodb_backend_gis/base.py @@ -0,0 +1,11 @@ +from django_mongodb_backend.base import DatabaseWrapper as BaseDatabaseWrapper + +from .features import DatabaseFeatures +from .operations import DatabaseOperations +from .schema import DatabaseSchemaEditor + + +class DatabaseWrapper(BaseDatabaseWrapper): + SchemaEditorClass = DatabaseSchemaEditor + features_class = DatabaseFeatures + ops_class = DatabaseOperations diff --git a/django_mongodb_backend_gis/features.py b/django_mongodb_backend_gis/features.py new file mode 100644 index 00000000..f19d08da --- /dev/null +++ b/django_mongodb_backend_gis/features.py @@ -0,0 +1,49 @@ +from django.contrib.gis.db.backends.base.features import BaseSpatialFeatures +from django.utils.functional import cached_property + +from django_mongodb_backend.features import DatabaseFeatures as MongoFeatures + + +class DatabaseFeatures(BaseSpatialFeatures, MongoFeatures): + has_spatialrefsys_table = False + supports_transform = False + + @cached_property + def django_test_expected_failures(self): + expected_failures = super().django_test_expected_failures + expected_failures.update( + { + # SRIDs aren't populated: AssertionError: 4326 != None + # self.assertEqual(4326, nullcity.point.srid) + "gis_tests.geoapp.tests.GeoModelTest.test_proxy", + # MongoDB does not support the within lookup + "gis_tests.relatedapp.tests.RelatedGeoModelTest.test06_f_expressions", + # 'Adapter' object has no attribute 'srid' + "gis_tests.geoapp.test_expressions.GeoExpressionsTests.test_geometry_value_annotation", + # Object of type ObjectId is not JSON serializable + "gis_tests.geoapp.test_serializers.GeoJSONSerializerTests.test_fields_option", + "gis_tests.geoapp.test_serializers.GeoJSONSerializerTests.test_geometry_field_option", + "gis_tests.geoapp.test_serializers.GeoJSONSerializerTests.test_serialization_base", + "gis_tests.geoapp.test_serializers.GeoJSONSerializerTests.test_srid_option", + # KeyError: 'within' connection.ops.gis_operators[self.lookup_name] + "gis_tests.geoapp.tests.GeoModelTest.test_gis_query_as_string", + # No lookups are supported (yet?) + "gis_tests.geoapp.tests.GeoLookupTest.test_gis_lookups_with_complex_expressions", + } + ) + return expected_failures + + @cached_property + def django_test_skips(self): + skips = super().django_test_skips + skips.update( + { + "inspectdb not supported.": { + "gis_tests.inspectapp.tests.InspectDbTests", + }, + "Raw SQL not supported": { + "gis_tests.geoapp.tests.GeoModelTest.test_raw_sql_query", + }, + }, + ) + return skips diff --git a/django_mongodb_backend_gis/functions.py b/django_mongodb_backend_gis/functions.py new file mode 100644 index 00000000..3807f9c6 --- /dev/null +++ b/django_mongodb_backend_gis/functions.py @@ -0,0 +1,6 @@ +# Placeholder if we can support any functions. +# from django.contrib.gis.db.models.functions import Distance, Length, Perimeter + + +def register_functions(): + pass diff --git a/django_mongodb_backend_gis/lookups.py b/django_mongodb_backend_gis/lookups.py new file mode 100644 index 00000000..29c2e1e9 --- /dev/null +++ b/django_mongodb_backend_gis/lookups.py @@ -0,0 +1,10 @@ +from django.contrib.gis.db.models.lookups import GISLookup +from django.db import NotSupportedError + + +def gis_lookup(self, compiler, connection): # noqa: ARG001 + raise NotSupportedError(f"MongoDB does not support the {self.lookup_name} lookup.") + + +def register_lookups(): + GISLookup.as_mql = gis_lookup diff --git a/django_mongodb_backend_gis/operations.py b/django_mongodb_backend_gis/operations.py new file mode 100644 index 00000000..faeda007 --- /dev/null +++ b/django_mongodb_backend_gis/operations.py @@ -0,0 +1,93 @@ +from django.contrib.gis import geos +from django.contrib.gis.db import models +from django.contrib.gis.db.backends.base.operations import BaseSpatialOperations + +from django_mongodb_backend.operations import ( + DatabaseOperations as MongoOperations, +) + +from .adapter import Adapter + + +class DatabaseOperations(BaseSpatialOperations, MongoOperations): + Adapter = Adapter + + disallowed_aggregates = ( + models.Collect, + models.Extent, + models.Extent3D, + models.MakeLine, + models.Union, + ) + + @property + def gis_operators(self): + return {} + + unsupported_functions = { + "Area", + "AsGeoJSON", + "AsGML", + "AsKML", + "AsSVG", + "AsWKB", + "AsWKT", + "Azimuth", + "BoundingCircle", + "Centroid", + "ClosestPoint", + "Difference", + "Distance", + "Envelope", + "ForcePolygonCW", + "FromWKB", + "FromWKT", + "GeoHash", + "GeometryDistance", + "Intersection", + "IsEmpty", + "IsValid", + "Length", + "LineLocatePoint", + "MakeValid", + "MemSize", + "NumGeometries", + "NumPoints", + "Perimeter", + "PointOnSurface", + "Reverse", + "Scale", + "SnapToGrid", + "SymDifference", + "Transform", + "Translate", + "Union", + } + + def geo_db_type(self, f): + return "object" + + def get_geometry_converter(self, expression): + def converter(value, expression, connection): # noqa: ARG001 + if value is None: + return None + geom_class = getattr(geos, value["type"]) + if geom_class.__name__ == "GeometryCollection": + return geom_class( + [getattr(geos, v["type"])(v["coordinates"]) for v in value["geometries"]], + srid=4326, + ) + if issubclass(geom_class, geos.GeometryCollection): + # TODO: confirm this is correct. + return geom_class( + [ + # TODO: For MultiLineString, geom_class._allowed is a + # tuple so this will crash. + geom_class._allowed(value["coordinates"][x][0]) + for x in range(len(value["coordinates"])) + ], + srid=4326, + ) + return geom_class(value["coordinates"], srid=4326) + + return converter diff --git a/django_mongodb_backend_gis/schema.py b/django_mongodb_backend_gis/schema.py new file mode 100644 index 00000000..b9b32fdd --- /dev/null +++ b/django_mongodb_backend_gis/schema.py @@ -0,0 +1,67 @@ +from django.contrib.gis.db.models import GeometryField +from pymongo import GEOSPHERE +from pymongo.operations import IndexModel + +from django_mongodb_backend.schema import DatabaseSchemaEditor as BaseSchemaEditor + + +class DatabaseSchemaEditor(BaseSchemaEditor): + # def _field_should_be_indexed(self, model, field): + # if getattr(field, "spatial_index", False): + # return True + # return super()._field_should_be_indexed(model, field) + + # def _add_field_index(self, model, field, *, column_prefix=""): + # if hasattr(field, "geodetic"): + # self._add_spatial_index(model, field) + # else: + # super()._add_field_index(model, field, column_prefix=column_prefix) + + def _alter_field( + self, + model, + old_field, + new_field, + old_type, + new_type, + old_db_params, + new_db_params, + strict=False, + ): + super()._alter_field( + model, + old_field, + new_field, + old_type, + new_type, + old_db_params, + new_db_params, + strict=strict, + ) + + old_field_spatial_index = isinstance(old_field, GeometryField) and old_field.spatial_index + new_field_spatial_index = isinstance(new_field, GeometryField) and new_field.spatial_index + if not old_field_spatial_index and new_field_spatial_index: + self._add_spatial_index(model, new_field) + elif old_field_spatial_index and not new_field_spatial_index: + self._delete_spatial_index(model, new_field) + + def _add_spatial_index(self, model, field): + index_name = self._create_spatial_index_name(model, field) + self.get_collection(model._meta.db_table).create_indexes( + [ + IndexModel( + [ + (field.column, GEOSPHERE), + ], + name=index_name, + ) + ] + ) + + def _delete_spatial_index(self, model, field): + index_name = self._create_spatial_index_name(model, field) + self.get_collection(model._meta.db_table).drop_index(index_name) + + def _create_spatial_index_name(self, model, field): + return f"{model._meta.db_table}_{field.column}_id"