diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index af043df..7c4724c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,25 +1,25 @@ repos: # Updates all subsequent hooks - - repo: https://gitlab.com/vojko.pribudic.foss/pre-commit-update - rev: v0.5.1 +- repo: https://gitlab.com/vojko.pribudic.foss/pre-commit-update + rev: v0.8.0 hooks: - - id: pre-commit-update + - id: pre-commit-update # Linter hook - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.7 +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.13.0 hooks: - - id: ruff + - id: ruff args: [--fix] # Formatter hook - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.8.0 +- repo: https://github.com/psf/black-pre-commit-mirror + rev: 25.1.0 hooks: - - id: black + - id: black # Static code analysis hook - - repo: https://github.com/PyCQA/bandit - rev: 1.7.10 +- repo: https://github.com/PyCQA/bandit + rev: 1.8.6 hooks: - - id: bandit + - id: bandit diff --git a/requirements.txt b/requirements.txt index e654118..c6b8f8b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,6 +26,7 @@ inflection==0.5.1 itypes==1.2.0 Jinja2==3.1.4 MarkupSafe==2.1.5 +memory-profiler==0.61.0 mypy-extensions==0.4.3 nodeenv==1.9.1 oauthlib==3.1.1 @@ -35,6 +36,7 @@ pathspec==0.9.0 platformdirs==4.2.2 pre-commit==3.8.0 protobuf==3.19.1 +psutil==7.1.0 psycopg2-binary==2.9.3 pyasn1==0.4.8 pyasn1-modules==0.2.8 @@ -51,6 +53,8 @@ sqlparse==0.4.2 tomli==1.2.3 typing_extensions==4.0.1 uritemplate==4.1.1 -urllib3 +urllib3==1.26.20 virtualenv==20.26.3 -google-auth \ No newline at end of file +orjson +drf-orjson +djangorestframework-orjson diff --git a/src/eatery/controllers/populate_eatery.py b/src/eatery/controllers/populate_eatery.py index 13e365a..2191a2c 100644 --- a/src/eatery/controllers/populate_eatery.py +++ b/src/eatery/controllers/populate_eatery.py @@ -3,6 +3,7 @@ from eatery.serializers import EaterySerializer from eatery.models import Eatery from django.core.exceptions import ObjectDoesNotExist +from memory_profiler import profile class PopulateEateryController: @@ -88,6 +89,7 @@ def add_eatery_store(self): else: print(serialized.errors) + @profile def process(self, json_eateries): for json_eatery in json_eateries: self.generate_eatery(json_eatery) diff --git a/src/eatery/serializers.py b/src/eatery/serializers.py index 9f6dae0..0fa4e67 100644 --- a/src/eatery/serializers.py +++ b/src/eatery/serializers.py @@ -1,13 +1,10 @@ from rest_framework import serializers from eatery.models import Eatery -from event.models import Event from event.serializers import ( EventSerializer, EventSerializerSimple, EventSerializerOptimized, ) -from datetime import timedelta, datetime -from zoneinfo import ZoneInfo class EaterySerializer(serializers.ModelSerializer): @@ -107,25 +104,7 @@ class EaterySerializerByDay(serializers.ModelSerializer): allow_null=True, default="https://images-prod.healthline.com/hlcmsresource/images/AN_images/health-benefits-of-apples-1296x728-feature.jpg", ) - events = serializers.SerializerMethodField() - - def get_events(self, obj): - day_offset = self.context.get("day") - now = datetime.now(ZoneInfo("America/New_York")) - day = now.replace(hour=0, minute=0, second=0, microsecond=0) + timedelta( - days=day_offset - ) - day_unix = int(day.timestamp()) - day_end_unix = int((day + timedelta(days=1)).timestamp()) - print(f"Now: {now}") - print(f"Day: {day}") - print(f"Day Unix: {day_unix}") - print(f"Day End Unix: {day_end_unix}") - events = Event.objects.filter( - eatery=obj.id, start__gte=day_unix, start__lt=day_end_unix - ) - serializer = EventSerializerOptimized(instance=events, many=True) - return serializer.data + events = EventSerializerOptimized(many=True, source="filtered_events") class Meta: model = Eatery diff --git a/src/eatery/views.py b/src/eatery/views.py index 7d71021..ee8e9b3 100644 --- a/src/eatery/views.py +++ b/src/eatery/views.py @@ -4,6 +4,7 @@ EaterySerializerByDay, EaterySerializerOptimized, ) +from django.db.models import Prefetch from eatery.util.json import FieldType, error_json, success_json, verify_json_fields from django.http import JsonResponse from django.shortcuts import get_object_or_404 @@ -15,7 +16,11 @@ from .permissions import EateryPermission from eatery.datatype.Eatery import EateryID from eatery.models import Eatery +from event.models import Event from .controllers.update_eatery import UpdateEateryController +from memory_profiler import profile +from datetime import timedelta, datetime +from zoneinfo import ZoneInfo class EateryViewSet(viewsets.ModelViewSet): @@ -27,29 +32,37 @@ class EateryViewSet(viewsets.ModelViewSet): serializer_class = EaterySerializer permission_classes = [EateryPermission] + @profile def get_queryset(self): """ Override to add prefetch_related for optimization """ queryset = super().get_queryset() - - # prefetch all related objects to avoid N+1 query problem - return queryset.prefetch_related( - 'events__menu__items__dietary_preferences', - 'events__menu__items__allergens' - ) + # Prefetching only needed for list to prevent N+1 + if self.action == "list": + return queryset.prefetch_related( + "events__menu__items__dietary_preferences", + "events__menu__items__allergens", + ) + + return queryset + + @profile + @method_decorator(cache_page(60 * 60 * 2)) # cache for 2 hours def retrieve(self, request, *args, **kwargs): instance = self.get_object() serializer = EaterySerializerOptimized(instance) return Response(serializer.data) + @profile @method_decorator(cache_page(60 * 60 * 2)) # cache for 2 hours def list(self, request, *args, **kwargs): queryset = self.filter_queryset(self.get_queryset()) serializer = EaterySerializerOptimized(queryset, many=True) return Response(serializer.data) + @profile def get_object(self): queryset = self.filter_queryset(self.get_queryset()) lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field @@ -109,10 +122,9 @@ class GetEateriesSimple(APIView): View all eateries with less information """ + @profile def get(self, request): - eateries_queryset = Eatery.objects.prefetch_related( - 'events' - ).all() + eateries_queryset = Eatery.objects.prefetch_related("events").all() eateries = EaterySerializerSimple(eateries_queryset, many=True) return Response(eateries.data) @@ -122,16 +134,34 @@ class GetEateriesByDay(APIView): Get all eatery information by day """ + @profile @method_decorator(cache_page(60 * 60 * 2)) # cache for 2 hours def get(self, request, day): + now = datetime.now(ZoneInfo("America/New_York")) + start_date = now.replace(hour=0, minute=0, second=0, microsecond=0) + timedelta( + days=day + ) + end_date = start_date + timedelta(days=1) + + start_unix = int(start_date.timestamp()) + end_unix = int(end_date.timestamp()) + + filtered_events_prefetch = Prefetch( + "events", + queryset=Event.objects.filter( + start__gte=start_unix, start__lt=end_unix + ).prefetch_related( + "menu__items__dietary_preferences", "menu__items__allergens" + ), + to_attr="filtered_events", + ) + eateries_queryset = Eatery.objects.prefetch_related( - 'events__menu__items__dietary_preferences', - 'events__menu__items__allergens' + filtered_events_prefetch ).exclude(events__event_description="Open") - + eateries = EaterySerializerByDay( eateries_queryset, many=True, - context={"day": day}, ) - return Response(eateries.data) \ No newline at end of file + return Response(eateries.data) diff --git a/src/eatery_blue_backend/management/commands/populate_models.py b/src/eatery_blue_backend/management/commands/populate_models.py index 87fc5fe..2f2b823 100644 --- a/src/eatery_blue_backend/management/commands/populate_models.py +++ b/src/eatery_blue_backend/management/commands/populate_models.py @@ -11,6 +11,7 @@ import os import json import shutil +from memory_profiler import profile class Command(BaseCommand): @@ -121,6 +122,7 @@ def logger_wrapper(self, command_obj, log_title, args): print(f"Done ({int(datetime.now().timestamp()) - pre}s) ") return output + @profile def process(self): """ 1. Get JSON from API diff --git a/src/eatery_blue_backend/settings.py b/src/eatery_blue_backend/settings.py index 4d89281..0e48ede 100644 --- a/src/eatery_blue_backend/settings.py +++ b/src/eatery_blue_backend/settings.py @@ -44,6 +44,11 @@ # Application definition +REST_FRAMEWORK = { + "DEFAULT_RENDERER_CLASSES": ("drf_orjson.renderers.ORJSONRenderer",), + "DEFAULT_PARSER_CLASSES": ("drf_orjson.parsers.ORJSONParser",), +} + INSTALLED_APPS = [ "django.contrib.admin", "django.contrib.auth", @@ -115,7 +120,7 @@ "PRE_PING": True, "ECHO": False, "TIMEOUT": 30, - } + }, } }