Fix N+1 Query

requirements.txt

@@ -8,6 +8,7 @@ coreapi==2.3.3
 coreschema==0.0.4
 distlib==0.3.8
 Django==4.0
+django-db-connection-pool==1.2.5
 django-rest-swagger==2.2.0
 djangorestframework==3.13.1
 drf-yasg==1.21.7

src/category/views.py

@@ -6,3 +6,10 @@
 class CategoryViewSet(viewsets.ModelViewSet):
     queryset = Category.objects.all()
     serializer_class = CategorySerializer
+
+    def get_queryset(self):
+        # prefetch items and their related fields
+        return Category.objects.select_related('event__eatery').prefetch_related(
+            'items__dietary_preferences',
+            'items__allergens'
+        ).all()

src/eatery/views.py

@@ -27,6 +27,18 @@ class EateryViewSet(viewsets.ModelViewSet):
     serializer_class = EaterySerializer
     permission_classes = [EateryPermission]
 
+    def get_queryset(self):
+        """
+        Override to add prefetch_related for optimization
+        """
+        queryset = super().get_queryset()
+
+        # prefetch all related objects to avoid N+1 query problem
+        return queryset.prefetch_related(
+            'events__menu__items__dietary_preferences',
+            'events__menu__items__allergens'
+        )
+
     def retrieve(self, request, *args, **kwargs):
         instance = self.get_object()
         serializer = EaterySerializerOptimized(instance)
@@ -98,7 +110,10 @@ class GetEateriesSimple(APIView):
     """
 
     def get(self, request):
-        eateries = EaterySerializerSimple(Eatery.objects.all(), many=True)
+        eateries_queryset = Eatery.objects.prefetch_related(
+            'events'
+        ).all()
+        eateries = EaterySerializerSimple(eateries_queryset, many=True)
         return Response(eateries.data)
 
 
@@ -109,9 +124,14 @@ class GetEateriesByDay(APIView):
 
     @method_decorator(cache_page(60 * 60 * 2))  # cache for 2 hours
     def get(self, request, day):
+        eateries_queryset = Eatery.objects.prefetch_related(
+            'events__menu__items__dietary_preferences',
+            'events__menu__items__allergens'
+        ).exclude(events__event_description="Open")
+
         eateries = EaterySerializerByDay(
-            Eatery.objects.exclude(events__event_description="Open"),
+            eateries_queryset,
             many=True,
             context={"day": day},
         )
-        return Response(eateries.data)
+        return Response(eateries.data)

src/eatery_blue_backend/management/commands/populate_models.py

@@ -10,6 +10,7 @@
 from category.controllers.populate_category import PopulateCategoryController
 import os
 import json
+import shutil
 
 
 class Command(BaseCommand):
@@ -33,18 +34,36 @@ def get_json(self):
             json_eateries = response["data"]["eateries"]
         return json_eateries
 
+    def ensure_external_eateries_exists(self):
+        external_path = "./static_sources/external_eateries.json"
+        static_path = "./static_sources/external_eateries_static.json"
+
+        # if external_eateries.json doesn't exist, copy from static
+        if not os.path.exists(external_path):
+            if os.path.exists(static_path):
+                shutil.copy2(static_path, external_path)
+                print("Created external_eateries.json from static template")
+            else:
+                # create fallback file
+                data = {"eateries": []}
+                with open(external_path, "w") as f:
+                    json.dump(data, f, indent=2)
+                print("Created minimal external_eateries.json")
+
     def update_freedge_external_eatery(self):
+        self.ensure_external_eateries_exists()
+
         GOOGLE_SHEETS_API_KEY = os.environ.get("GOOGLE_SHEETS_API_KEY")
         FREEDGE_SHEET_ID = os.environ.get("FREEDGE_SHEET_ID")
         FREEDGE_APPROVED_EMAILS = os.environ.get("FREEDGE_APPROVED_EMAILS")
         if not GOOGLE_SHEETS_API_KEY:
-            print("GOOGLE_SHEETS_API_KEY not set, cannot update freedge external eatery")
+            print("GOOGLE_SHEETS_API_KEY not set, skipping freedge update")
             return
         if not FREEDGE_SHEET_ID:
-            print("FREEDGE_SHEET_ID not set, cannot update freedge external eatery")
+            print("FREEDGE_SHEET_ID not set, skipping freedge update")
             return
         if not FREEDGE_APPROVED_EMAILS:
-            print("FREEDGE_APPROVED_EMAILS not set, cannot update freedge external eatery")
+            print("FREEDGE_APPROVED_EMAILS not set, skipping freedge update")
             return
 
         approved_emails = FREEDGE_APPROVED_EMAILS.split(",")

src/eatery_blue_backend/settings.py

@@ -102,13 +102,20 @@
 
 DATABASES = {
     "default": {
-        "ENGINE": "django.db.backends.postgresql_psycopg2",
+        "ENGINE": "dj_db_conn_pool.backends.postgresql",
         "NAME": os.getenv("POSTGRES_NAME"),
         "USER": os.getenv("POSTGRES_USER"),
         "PASSWORD": os.getenv("POSTGRES_PASSWORD"),
         "HOST": os.getenv("POSTGRES_HOST"),
         "PORT": os.getenv("POSTGRES_PORT"),
-        'CONN_MAX_AGE': 0,
+        "POOL_OPTIONS": {
+            "POOL_SIZE": 10,
+            "MAX_OVERFLOW": 10,
+            "RECYCLE": 300,
+            "PRE_PING": True,
+            "ECHO": False,
+            "TIMEOUT": 30,
+        }
     }
 }
 

src/event/views.py

@@ -6,3 +6,10 @@
 class EventViewSet(viewsets.ModelViewSet):
     queryset = Event.objects.all()
     serializer_class = EventSerializer
+
+    def get_queryset(self):
+        # prefetch related menu categories and items
+        return Event.objects.select_related('eatery').prefetch_related(
+            'menu__items__dietary_preferences',
+            'menu__items__allergens'
+        ).all()

src/item/controllers/populate_item.py

@@ -1,4 +1,4 @@
-from item.serializers import ItemSerializer
+from item.models import Item, DietaryPreference, Allergen
 import json
 from util.constants import eatery_is_cafe
 
@@ -7,6 +7,30 @@ class PopulateItemController:
     def __init__(self):
         self = self
 
+    def create_item_with_m2m(self, category_id, name, dietary_preferences=None, allergens=None):
+        # handle many to many relationships
+        item, created = Item.objects.get_or_create(
+            category_id=category_id,
+            name=name,
+            defaults={'base_price': 0.0}
+        )
+
+        # handle dietary preferences
+        if dietary_preferences:
+            for pref_name in dietary_preferences:
+                if pref_name:
+                    pref, _ = DietaryPreference.objects.get_or_create(name=pref_name)
+                    item.dietary_preferences.add(pref)
+
+        # handle allergens
+        if allergens:
+            for allergen_name in allergens:
+                if allergen_name:
+                    allergen, _ = Allergen.objects.get_or_create(name=allergen_name)
+                    item.allergens.add(allergen)
+
+        return item
+
     def generate_cafe_items(self, menu, json_eatery):
         for json_item in json_eatery["diningItems"]:
             category_name = json_item["category"].strip()
@@ -18,33 +42,32 @@ def generate_cafe_items(self, menu, json_eatery):
             dietary_preferences = json_item.get("dietaryPreferences", [])
             allergens = json_item.get("allergens", [])
 
-            data = {"category": category_id, "name": json_item["item"], "dietary_preferences": dietary_preferences, "allergens": allergens}
-            try:
-                item = ItemSerializer(data=data)
-                if item.is_valid():
-                    item.save()
-                else:
-                    print(item.errors)
-            except Exception as e:
-                print(f"Error saving item: {e}")
-                print(f"Data: {data}")
-
+            self.create_item_with_m2m(
+                category_id=category_id,
+                name=json_item["item"],
+                dietary_preferences=dietary_preferences,
+                allergens=allergens
+            )
 
     def generate_dining_hall_items(self, menu, json_event, json_eatery):
         json_menus = json_event["menu"]
         for json_menu in json_menus:
             category_name = json_menu["category"].strip()
-            category_id = menu[category_name]
+            try:
+                category_id = menu[category_name]
+            except KeyError:
+                continue
 
             for json_item in json_menu["items"]:
                 dietary_preferences = json_item.get("dietaryPreferences", [])
                 allergens = json_item.get("allergens", [])
-                data = {"category": category_id, "name": json_item["item"], "dietary_preferences": dietary_preferences, "allergens": allergens}
-                item = ItemSerializer(data=data)
-                if item.is_valid():
-                    item.save()
-                else:
-                    print(item.errors)
+
+                self.create_item_with_m2m(
+                    category_id=category_id,
+                    name=json_item["item"],
+                    dietary_preferences=dietary_preferences,
+                    allergens=allergens
+                )
 
     def process(self, categories_dict, json_eateries):
         with open(

src/item/serializers.py

@@ -5,12 +5,16 @@
 class ItemSerializer(serializers.ModelSerializer):
     id = serializers.IntegerField(read_only=True)
     name = serializers.CharField(default="Item")
-    dietary_preferences = serializers.ListField(
-        child=serializers.CharField(), allow_empty=True, default=[]
-    )
-    allergens = serializers.ListField(
-        child=serializers.CharField(), allow_empty=True, default=[]
-    )
+    dietary_preferences = serializers.SerializerMethodField()
+    allergens = serializers.SerializerMethodField()
+
+    def get_dietary_preferences(self, obj):
+        """Get list of dietary preference names"""
+        return list(obj.dietary_preferences.values_list('name', flat=True))
+
+    def get_allergens(self, obj):
+        """Get list of allergen names"""
+        return list(obj.allergens.values_list('name', flat=True))
 
     def create(self, validated_data):
         dietary_prefs = validated_data.pop('dietary_preferences', [])
@@ -34,10 +38,17 @@ def create(self, validated_data):
             item.allergens.add(*allergens_objects)
 
         return item
+
+    def update(self, instance, validated_data):
+        instance.name = validated_data.get('name', instance.name)
+        instance.base_price = validated_data.get('base_price', instance.base_price)
+        instance.save()
+        return instance
 
     class Meta:
         model = Item
-        fields = ["id", "category", "name", "dietary_preferences", "allergens"]
+        fields = ["id", "category", "name", "base_price", "dietary_preferences", "allergens"]
+        read_only_fields = ["id"]
 
 
 class ItemSerializerOptimized(serializers.ModelSerializer):

src/item/views.py

@@ -6,3 +6,11 @@
 class ItemViewSet(viewsets.ModelViewSet):
     queryset = Item.objects.all()
     serializer_class = ItemSerializer
+
+    def get_queryset(self):
+        return Item.objects.select_related(
+            'category__event__eatery'
+        ).prefetch_related(
+            'dietary_preferences',
+            'allergens'
+        ).all()