Feature/4 producttypecode

.gitignore

@@ -43,3 +43,5 @@ local.py
 # Static files
 src/open_producten/static/bundles/
 src/open_producten/fonts/
+
+tmp/*

Dockerfile

@@ -66,19 +66,17 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-reco
 
 WORKDIR /app
 COPY ./bin/docker_start.sh /start.sh
-# Uncomment if you use celery
-# COPY ./bin/celery_worker.sh /celery_worker.sh
-# COPY ./bin/celery_beat.sh /celery_beat.sh
-# COPY ./bin/celery_flower.sh /celery_flower.sh
+COPY ./bin/celery_worker.sh /celery_worker.sh
+COPY ./bin/celery_beat.sh /celery_beat.sh
+COPY ./bin/celery_flower.sh /celery_flower.sh
 RUN mkdir /app/bin /app/log /app/media
 
 VOLUME ["/app/log", "/app/media"]
 
 # copy backend build deps
 COPY --from=backend-build /usr/local/lib/python3.11 /usr/local/lib/python3.11
 COPY --from=backend-build /usr/local/bin/uwsgi /usr/local/bin/uwsgi
-# Uncomment if you use celery
-# COPY --from=backend-build /usr/local/bin/celery /usr/local/bin/celery
+COPY --from=backend-build /usr/local/bin/celery /usr/local/bin/celery
 COPY --from=backend-build /app/src/ /app/src/
 
 # copy frontend build statics

bin/celery_beat.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+LOGLEVEL=${CELERY_LOGLEVEL:-INFO}
+
+mkdir -p celerybeat
+
+echo "Starting celery beat"
+exec celery --workdir src --app open_producten beat \
+    -l $LOGLEVEL \
+    -s ../celerybeat/beat

bin/celery_flower.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+exec celery --workdir src --app open_producten flower

bin/celery_worker.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+LOGLEVEL=${CELERY_LOGLEVEL:-INFO}
+CONCURRENCY=${CELERY_WORKER_CONCURRENCY:-1}
+
+QUEUE=${1:-${CELERY_WORKER_QUEUE:=celery}}
+WORKER_NAME=${2:-${CELERY_WORKER_NAME:="${QUEUE}"@%n}}
+
+echo "Starting celery worker $WORKER_NAME with queue $QUEUE"
+exec celery --workdir src --app open_producten  worker \
+    -Q $QUEUE \
+    -n $WORKER_NAME \
+    -l $LOGLEVEL \
+    -O fair \
+    -c $CONCURRENCY
+

bin/check_celery_worker_liveness.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+#
+# Check the health of a Celery worker.
+#
+# The worker process writes and periodically touches a number of files that indicate it
+# is available and still healthy. If the worker becomes unhealthy for any reason, the
+# timestamp of when the heartbeat file was last touched will not update and the delta
+# becomes too big, allowing (container) orchestration to terminate and restart the
+# worker process.
+#
+# Example usage with Kubernetes, as a liveness probe:
+#
+# .. code-block:: yaml
+#
+#       livenessProbe:
+#         exec:
+#           command:
+#           - python
+#           - /app/bin/check_celery_worker_liveness.py
+#         initialDelaySeconds: 10
+#         periodSeconds: 30  # must be smaller than `MAX_WORKER_LIVENESS_DELTA`
+#
+# Reference: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-a-liveness-command
+#
+# Supported environment variables:
+#
+# * ``MAX_WORKER_LIVENESS_DELTA``: maximum delta between heartbeats before reporting
+#   failure, in seconds. Defaults to 60 (one minute).
+
+
+import os
+import sys
+import time
+from pathlib import Path
+
+HEARTBEAT_FILE = Path(__file__).parent.parent / "tmp" / "celery_worker_heartbeat"
+READINESS_FILE = Path(__file__).parent.parent / "tmp" / "celery_worker_ready"
+MAX_WORKER_LIVENESS_DELTA = int(os.getenv("MAX_WORKER_LIVENESS_DELTA", 60))  # seconds
+
+
+# check if worker is ready
+if not READINESS_FILE.is_file():
+    print("Celery worker not ready.")
+    sys.exit(1)
+
+# check if worker is live
+if not HEARTBEAT_FILE.is_file():
+    print("Celery worker heartbeat not found.")
+    sys.exit(1)
+
+# check if worker heartbeat satisfies constraint
+stats = HEARTBEAT_FILE.stat()
+worker_timestamp = stats.st_mtime
+current_timestamp = time.time()
+time_diff = current_timestamp - worker_timestamp
+
+if time_diff > MAX_WORKER_LIVENESS_DELTA:
+    print("Celery worker heartbeat: interval exceeds constraint (60s).")
+    sys.exit(1)
+
+print("Celery worker heartbeat found: OK.")
+sys.exit(0)

docker-compose.yml

@@ -1,6 +1,3 @@
-# Inspired by https://docs.docker.com/compose/django/
-version: '3'
-
 services:
   db:
     # NOTE: No persistance storage configured.
@@ -12,10 +9,14 @@ services:
     volumes:
       - ./docker-init-db.sql:/docker-entrypoint-initdb.d/init_db.sql
 
+  redis:
+    image: redis
+
   web:
-    build: .
+    build: &web_build
+      context: .
     image: maykinmedia/open-producten:latest
-    environment:
+    environment: &web_env
       - DJANGO_SETTINGS_MODULE=open_producten.conf.docker
       - SECRET_KEY=${SECRET_KEY:-django-insecure-@4wj9(+*bu7*v&%on7+e_8!d1ckl%r=6+sz#d2!pw^@lb0+=}
       - DB_NAME=open_producten
@@ -27,3 +28,37 @@ services:
       - 8000:8000
     depends_on:
       - db
+
+  celery:
+    build: *web_build
+    image: maykinmedia/open-producten:latest
+    environment: *web_env
+    command: /celery_worker.sh
+    healthcheck:
+      test: [ "CMD", "python", "/app/bin/check_celery_worker_liveness.py" ]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+    depends_on:
+      - db
+      - redis
+
+  celery-beat:
+    build: *web_build
+    image: maykinmedia/open-producten:latest
+    environment: *web_env
+    command: /celery_beat.sh
+    depends_on:
+      - celery
+
+  celery-flower:
+    build: *web_build
+    image: maykinmedia/open-producten:latest
+    environment: *web_env
+    command: /celery_flower.sh
+    ports:
+      - "5555:5555"
+    depends_on:
+      - redis
+      - celery

requirements/base.in

@@ -10,5 +10,9 @@ open-api-framework
 django-markdownx
 django-localflavor
 
+django-celery-beat
+flower
+django-timeline-logger
+
 # waiting for > 2.0.1
 commonground-api-common==1.13.4

requirements/base.txt

@@ -32,6 +32,7 @@ cbor2==5.6.5
     # via webauthn
 celery==5.4.0
     # via
+    #   django-celery-beat
     #   flower
     #   notifications-api-common
     #   open-api-framework
@@ -64,6 +65,8 @@ coreapi==2.3.3
     # via commonground-api-common
 coreschema==0.0.4
     # via coreapi
+cron-descriptor==1.4.5
+    # via django-celery-beat
 cryptography==44.0.0
     # via
     #   django-simple-certmanager
@@ -77,6 +80,7 @@ django==4.2.17
     #   django-admin-index
     #   django-appconf
     #   django-axes
+    #   django-celery-beat
     #   django-cors-headers
     #   django-csp
     #   django-filter
@@ -97,6 +101,8 @@ django==4.2.17
     #   django-setup-configuration
     #   django-simple-certmanager
     #   django-solo
+    #   django-timeline-logger
+    #   django-timezone-field
     #   django-two-factor-auth
     #   djangorestframework
     #   djangorestframework-inclusions
@@ -113,9 +119,13 @@ django==4.2.17
 django-admin-index==3.1.1
     # via open-api-framework
 django-appconf==1.0.6
-    # via django-log-outgoing-requests
+    # via
+    #   django-log-outgoing-requests
+    #   django-timeline-logger
 django-axes==7.0.0
     # via open-api-framework
+django-celery-beat==2.7.0
+    # via -r requirements/base.in
 django-cors-headers==4.6.0
     # via open-api-framework
 django-csp==3.8
@@ -168,6 +178,10 @@ django-solo==2.4.0
     #   mozilla-django-oidc-db
     #   notifications-api-common
     #   zgw-consumers
+django-timeline-logger==5.0.0
+    # via -r requirements/base.in
+django-timezone-field==7.0
+    # via django-celery-beat
 django-two-factor-auth==1.17.0
     # via maykin-2fa
 djangorestframework==3.15.2
@@ -207,7 +221,9 @@ elastic-apm==6.23.0
 face==24.0.0
     # via glom
 flower==2.0.1
-    # via open-api-framework
+    # via
+    #   -r requirements/base.in
+    #   open-api-framework
 furl==2.1.3
     # via ape-pie
 gemma-zds-client==2.0.0
@@ -291,10 +307,13 @@ pypng==0.20220715.0
     # via qrcode
 python-creole==1.4.10
     # via django-markup
+python-crontab==3.2.0
+    # via django-celery-beat
 python-dateutil==2.9.0.post0
     # via
     #   celery
     #   django-relativedelta
+    #   python-crontab
 python-decouple==3.8
     # via open-api-framework
 python-dotenv==1.0.1
@@ -358,6 +377,7 @@ typing-extensions==4.12.2
 tzdata==2024.2
     # via
     #   celery
+    #   django-celery-beat
     #   kombu
 uritemplate==4.1.1
     # via

requirements/ci.txt

@@ -62,6 +62,7 @@ celery==5.4.0
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt
+    #   django-celery-beat
     #   flower
     #   notifications-api-common
     #   open-api-framework
@@ -123,6 +124,11 @@ coreschema==0.0.4
     #   coreapi
 coverage==7.6.8
     # via -r requirements/test-tools.in
+cron-descriptor==1.4.5
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   django-celery-beat
 cryptography==44.0.0
     # via
     #   -c requirements/base.txt
@@ -144,6 +150,7 @@ django==4.2.17
     #   django-admin-index
     #   django-appconf
     #   django-axes
+    #   django-celery-beat
     #   django-cors-headers
     #   django-csp
     #   django-filter
@@ -164,6 +171,8 @@ django==4.2.17
     #   django-setup-configuration
     #   django-simple-certmanager
     #   django-solo
+    #   django-timeline-logger
+    #   django-timezone-field
     #   django-two-factor-auth
     #   djangorestframework
     #   djangorestframework-inclusions
@@ -187,11 +196,16 @@ django-appconf==1.0.6
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   django-log-outgoing-requests
+    #   django-timeline-logger
 django-axes==7.0.0
     # via
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   open-api-framework
+django-celery-beat==2.7.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
 django-cors-headers==4.6.0
     # via
     #   -c requirements/base.txt
@@ -302,6 +316,15 @@ django-solo==2.4.0
     #   mozilla-django-oidc-db
     #   notifications-api-common
     #   zgw-consumers
+django-timeline-logger==5.0.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+django-timezone-field==7.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   django-celery-beat
 django-two-factor-auth==1.17.0
     # via
     #   -c requirements/base.txt
@@ -602,6 +625,11 @@ python-creole==1.4.10
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   django-markup
+python-crontab==3.2.0
+    # via
+    #   -c requirements/base.txt
+    #   -r requirements/base.txt
+    #   django-celery-beat
 python-dateutil==2.9.0.post0
     # via
     #   -c requirements/base.txt
@@ -610,6 +638,7 @@ python-dateutil==2.9.0.post0
     #   django-relativedelta
     #   faker
     #   freezegun
+    #   python-crontab
 python-decouple==3.8
     # via
     #   -c requirements/base.txt
@@ -732,6 +761,7 @@ tzdata==2024.2
     #   -c requirements/base.txt
     #   -r requirements/base.txt
     #   celery
+    #   django-celery-beat
     #   kombu
 uritemplate==4.1.1
     # via