Background Workers

.dockerignore

@@ -0,0 +1,24 @@
+*.sqlite3
+*.swp
+*.mo
+*~
+.ve
+.ve2
+*.pyc
+__pycache__
+.pytest_cache/
+media
+.coverage
+htmlcov
+docs/_build
+.cache/*
+.pytest_cache/
+.hypothesis/*
+geckodriver.log
+ghostdriver.log
+cove/lib/org-ids.json
+cove/lib/org-ids.json.lock
+chromedriver/
+src/
+node_modules/
+untracked/

Dockerfile.dev

@@ -0,0 +1,16 @@
+# syntax=docker/dockerfile:1
+# Using python base image: https://hub.docker.com/_/python/
+FROM python:3.12
+WORKDIR /code
+
+# Copy in dependencies & requirements definition
+COPY setup.py setup.py
+COPY tools/ tools/
+COPY lib360dataquality/ lib360dataquality/
+COPY requirements_cove_dev.txt requirements_cove_dev.txt
+
+# Install requirements
+RUN pip install -r requirements_cove_dev.txt
+
+# Copy in rest of the project
+COPY . .

compose.yaml

@@ -0,0 +1,42 @@
+services:
+  web:
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+    command: ["/bin/bash", "-c", "pip install -r requirements_cove_dev.txt; cd /code/cove; python manage.py migrate; python manage.py runserver 0.0.0.0:8000"]
+    ports:
+      - "8000:8000"
+    environment:
+      DB_NAME: "/data/db.sqlite3"
+      MEDIA_ROOT: "/data/media/"
+      RQ_REDIS_HOST: redis
+    volumes:
+      - "dqt_data:/data"
+      - "./:/cove/:ro"
+
+  worker:
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+    command: ["/bin/bash", "-c", "pip install -r requirements_cove_dev.txt; cd /code/cove; python manage.py rqworker default --with-scheduler"]
+    environment:
+      DB_NAME: "/data/db.sqlite3"
+      MEDIA_ROOT: "/data/media/"
+      RQ_REDIS_HOST: redis
+    volumes:
+      - "dqt_data:/data"
+      - "./:/cove/:ro"
+
+  redis:
+    # ValKey is an open source redis drop-in replacement
+    # See: https://hub.docker.com/r/valkey/valkey/
+    #      https://valkey.io/topics/valkey.conf/
+    image: "valkey/valkey:8"
+    ports:
+      - "6379:6379"
+    environment:
+      # disable persistence in dev
+      VALKEY_EXTRA_FLAGS: "--save ''"
+
+volumes:
+  dqt_data: {}

cove/cove_360/views.py

@@ -5,6 +5,7 @@
 import logging
 import re
 import os
+import django_rq
 from decimal import Decimal
 
 from cove.views import explore_data_context, cove_web_input_error
@@ -43,7 +44,9 @@ def results_ready(request, pk):
 def data_loading(request, pk, template='cove_360/data_loading.html'):
     # Backward compatibility if we receive a post request to the loading page
     if request.method == 'POST':
-        return explore_360(request, pk)
+        explore_result = explore_360(request, pk)
+        if explore_result:
+            return explore_result
 
     # Data already loaded so redirect to results page
     try:
@@ -54,9 +57,8 @@ def data_loading(request, pk, template='cove_360/data_loading.html'):
 
     return render(request, template, {"pk": pk})
 
-
 @cove_web_input_error
-def explore_360(request, pk, template='cove_360/explore.html'):
+def explore_360(request, pk, template='cove_360/explore.html') -> HttpResponse | None:
 
     cached_context = cache.get(pk)
 
@@ -91,7 +93,6 @@ def explore_360(request, pk, template='cove_360/explore.html'):
     data_status, dsc = SuppliedDataStatus.objects.get_or_create(
         supplied_data=db_data,
     )
-    context["data_status"] = data_status
     if db_data.source_url:
         context["source_url_domain"] = extract_domain(db_data.source_url)
 
@@ -105,6 +106,19 @@ def explore_360(request, pk, template='cove_360/explore.html'):
         context["submission_tool"] = True
         context["publisher"] = publisher
 
+    explore_360_process.delay(pk, context, db_data, flatten=request.POST.get("flatten"))
+
+    return data_loading(request, pk)
+
+
+@django_rq.job("default", timeout=1800)
+def explore_360_process(pk, context, db_data, flatten):
+
+    data_status, dsc = SuppliedDataStatus.objects.get_or_create(
+        supplied_data=db_data,
+    )
+    context["data_status"] = data_status
+
     lib_cove_config = LibCoveConfig()
     lib_cove_config.config.update(settings.COVE_CONFIG)
 
@@ -140,7 +154,7 @@ def explore_360(request, pk, template='cove_360/explore.html'):
             extension_metadatas = schema_360.resolve_extension(json_data)
 
             context.update(convert_json(upload_dir, upload_url, file_name, schema_url=schema_360.schema_file,
-                                    request=request, flatten=request.POST.get('flatten'),
+                                    flatten=flatten,
                                     lib_cove_config=lib_cove_config))
 
     else:
@@ -242,7 +256,7 @@ def explore_360(request, pk, template='cove_360/explore.html'):
     # Helpful when debugging DQT
     # import pprint
     # pprint.pprint(context, stream=open("/tmp/dqt.py", "w"), indent=2)
-    return render(request, template, context)
+    #return render(request, template, context)
 
 
 def create_passed_tests_context_data(failed_tests, available_tests):

cove/cove_project/settings.py

@@ -8,6 +8,13 @@
 env = environ.Env(  # set default values and casting
     DB_NAME=(str, os.path.join(BASE_DIR, 'db.sqlite3')),
     SENTRY_DSN=(str, ''),
+    MEDIA_ROOT=(str, os.path.join(BASE_DIR, 'media')),
+
+    RQ_REDIS_HOST=(str, 'localhost'),
+    RQ_REDIS_PORT=(str, '6379'),
+    RQ_REDIS_DB=(str, '0'),
+    RQ_REDIS_USERNAME=(str, ''),
+    RQ_REDIS_PASSWORD=(str, ''),
 )
 
 # We use the setting to choose whether to show the section about Sentry in the
@@ -32,7 +39,7 @@
 # We can't take MEDIA_ROOT and MEDIA_URL from cove settings,
 # ... otherwise the files appear under the BASE_DIR that is the Cove library install.
 # That could get messy. We want them to appear in our directory.
-MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
+MEDIA_ROOT = env("MEDIA_ROOT")
 MEDIA_URL = '/media/'
 
 SECRET_KEY = settings.SECRET_KEY
@@ -99,6 +106,7 @@
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
+    'django_rq',
 )
 
 WSGI_APPLICATION = 'cove_project.wsgi.application'
@@ -136,3 +144,16 @@
 # If enabled the grants data can be used in a template to create a browsable
 # table of grants.
 GRANTS_TABLE = False
+
+# Django RQ Queues
+RQ_QUEUES = {
+    'default': {
+        'HOST': env("RQ_REDIS_HOST"),
+        'PORT': env("RQ_REDIS_PORT"),
+        'DB': env("RQ_REDIS_DB"),
+        'USERNAME': env("RQ_REDIS_USERNAME"),
+        'PASSWORD': env("RQ_REDIS_PASSWORD"),
+        'DEFAULT_TIMEOUT': 360,
+        'DEFAULT_RESULT_TTL': 800,
+    },
+}

cove/cove_project/urls.py

@@ -1,6 +1,6 @@
 from django.conf.urls.static import static
 from django.conf import settings
-from django.urls import path
+from django.urls import path, include
 from django.views.generic import TemplateView
 
 from cove.urls import urlpatterns, handler500  # noqa: F401
@@ -18,6 +18,11 @@
     path("additional_checks", cove_360.views.additional_checks, name='additional_checks'),
     path("submit", TemplateView.as_view(template_name="cove_360/publishing.html", extra_context={"submission_tool": True}), name="publishing"),
     path("terms-conditions", TemplateView.as_view(template_name="cove_360/terms.html"), name="terms-conditions"),
+
 ]
 
 urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+
+urlpatterns += [
+    path('django-rq/', include('django_rq.urls'))
+]

requirements_cove.in

@@ -1,5 +1,6 @@
 -e file:./
 Django<5
+django-rq
 flattentool>=0.17.0
 libcove>=0.30.0
 libcoveweb==0.31.1