Dockerfile Rework

.github/workflows/continuous_integration.yml

@@ -1,4 +1,5 @@
-name: Continuous Integration
+name: CI - Build and Test Service
+
 on:
   pull_request:
     branches:
@@ -13,8 +14,21 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Build container
-      run: make build
+    - name: Build Prod
+      uses: Janmtbehrens/OpenSlides/dev/actions/build-service@all-in-one
+      with:
+        service: datastore
+        context: prod
+        module: reader
+        port: 9010
+
+    - name: Build Prod
+      uses: Janmtbehrens/OpenSlides/dev/actions/build-service@all-in-one
+      with:
+        service: datastore
+        context: prod
+        module: writer
+        port: 9011
 
     - name: Create secret
       run: mkdir secrets && echo -n "openslides" > secrets/postgres_password
@@ -30,11 +44,18 @@ jobs:
     - name: Test that writer is up and running
       run: curl -I http://localhost:9011/
 
-  tests:
-    name: "Tests"
+  ci-tests:
+    name: "CI Tests"
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+
+    - name: Build Tests
+      uses: Janmtbehrens/OpenSlides/dev/actions/build-service@all-in-one
+      with:
+        service: datastore
+        context: tests
+
     - name: Execute tests
       run: make run-ci
 
@@ -44,11 +65,33 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Start containers
-      run: make run-dev-standalone
+    - name: Build Dev
+      uses: Janmtbehrens/OpenSlides/dev/actions/build-service@all-in-one
+      with:
+        service: datastore
+        context: dev
+        module: reader
+        port: 9010
+
+    - name: Build Dev
+      uses: Janmtbehrens/OpenSlides/dev/actions/build-service@all-in-one
+      with:
+        service: datastore
+        context: dev
+        module: writer
+        port: 9011
 
-    - name: Execute tests
-      run: make run-full-system-tests-check
+    - name: Run System Tests
+      run: make ci-run-system-tests
+
+  # This setups and runs the dev/run-tests.sh script
+  run-test-script:
+    name: Run-tests.sh Script
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
 
-    - name: Stop tests
-      run: make stop-dev
+      - name: Run Test
+        uses: Janmtbehrens/OpenSlides/dev/actions/build-and-test-service@all-in-one
+        with:
+          service: datastore

Dockerfile

@@ -1,40 +1,109 @@
-FROM python:3.10.17-slim-bookworm
+ARG CONTEXT=prod
 
-RUN apt-get -y update && apt-get -y upgrade && \
-    apt-get install --no-install-recommends -y ncat gcc libpq-dev libc-dev postgresql-client redis-tools cron
+FROM python:3.10.17-slim-bookworm as base
 
+## Setup
+ARG CONTEXT
 WORKDIR /app
+ENV APP_CONTEXT=${CONTEXT}
+
+## Install
+RUN CONTEXT_INSTALLS=$(case "$APP_CONTEXT" in \
+    tests)  echo "curl";; \
+    dev)    echo "";; \
+    *)      echo "cron" ;; esac) && \
+    apt-get -y update && apt-get -y upgrade && apt-get install --no-install-recommends -y \
+    gcc \
+    libc-dev \
+    libpq-dev \
+    ncat \
+    postgresql-client \
+    redis-tools \
+    ${CONTEXT_INSTALLS} && \
+    rm -rf /var/lib/apt/lists/*
+
+## Requirements
+COPY requirements/* scripts/system/* scripts/* ./
+RUN  REQUIREMENTS_FILE=$(case "$APP_CONTEXT" in \
+    tests) echo "testing";; \
+    dev)   echo "testing";; \
+    debug) echo "testing";; \
+    *)     echo "general" ;; esac) && \
+    pip install --no-cache-dir -U -r requirements-${REQUIREMENTS_FILE}.txt
+
 ENV PYTHONPATH /app/
 
-COPY requirements/requirements-general.txt /app/
+## External Information
+LABEL org.opencontainers.image.title="OpenSlides Datastore Service"
+LABEL org.opencontainers.image.description="Service for OpenSlides which wraps the database, which includes reader and writer functionality."
+LABEL org.opencontainers.image.licenses="MIT"
+LABEL org.opencontainers.image.source="https://github.com/OpenSlides/openslides-datastore-service"
+
+## Command
+COPY ./dev/command.sh ./
+RUN chmod +x command.sh
+CMD ["./command.sh"]
+HEALTHCHECK CMD python cli/healthcheck.py
+ENTRYPOINT ["./entrypoint.sh"]
 
-RUN pip install -U -r requirements-general.txt
+# Testing Image
 
-COPY cli cli
-COPY datastore datastore
+FROM base as tests
+
+COPY scripts/* scripts/system/* tests/entrypoint.sh ./
+COPY scripts/ci/* ./ci/
+COPY dev ./dev/
+
+STOPSIGNAL SIGKILL
+
+# Intermediate Image
+
+FROM base as moduled
+
+ARG MODULE
+RUN test -n "$MODULE" || (echo "MODULE not set" && false)
+ENV MODULE=$MODULE
 
 ARG PORT
 RUN test -n "$PORT" || (echo "PORT not set" && false)
 ENV PORT=$PORT
+
 EXPOSE $PORT
 
-ARG MODULE
-RUN test -n "$MODULE" || (echo "MODULE not set" && false)
-ENV MODULE=$MODULE
+COPY $MODULE/entrypoint.sh ./
+
+# Development Image
+
+FROM moduled as dev
+
+COPY scripts/system/* scripts/* ./
+
+ENV FLASK_APP=datastore.$MODULE.app
+ENV FLASK_DEBUG=1
+
+# Debug Image
+
+FROM moduled as debug
+
+ENV FLASK_APP=datastore.$MODULE.app
+ENV FLASK_DEBUG=1
+
+# Production Image
+
+FROM moduled as prod
+
+# Add appuser
+RUN adduser --system --no-create-home appuser && \
+    chown appuser /app/
 
-COPY $MODULE/entrypoint.sh scripts/system/* ./
+COPY cli cli
+COPY datastore datastore
+
+COPY scripts/system/* ./
 
 ENV NUM_WORKERS=1
 ENV WORKER_TIMEOUT=30
 
 RUN echo "20 4 * * * root /app/cron.sh >> /var/log/cron.log 2>&1" > /etc/cron.d/trim-collectionfield-tables
 
-LABEL org.opencontainers.image.title="OpenSlides Datastore Service"
-LABEL org.opencontainers.image.description="Service for OpenSlides which wraps the database, which includes reader and writer functionality."
-LABEL org.opencontainers.image.licenses="MIT"
-LABEL org.opencontainers.image.source="https://github.com/OpenSlides/openslides-datastore-service"
-
-HEALTHCHECK CMD python cli/healthcheck.py
-
-ENTRYPOINT ["./entrypoint.sh"]
-CMD exec gunicorn -w $NUM_WORKERS -b 0.0.0.0:$PORT datastore.$MODULE.app:application -t $WORKER_TIMEOUT
+USER appuser

Makefile

@@ -1,3 +1,18 @@
+SERVICE=datastore
+
+# Build images for different contexts
+
+build-dev:
+	bash ../dev/scripts/makefile/build-service.sh $(SERVICE) dev reader 9010
+	bash ../dev/scripts/makefile/build-service.sh $(SERVICE) dev writer 9011
+
+build-prod:
+	bash ../dev/scripts/makefile/build-service.sh $(SERVICE) prod reader 9010
+	bash ../dev/scripts/makefile/build-service.sh $(SERVICE) prod writer 9011
+
+build-test:
+	bash ../dev/scripts/makefile/build-service.sh $(SERVICE) tests
+
 # TESTS
 
 ifdef MODULE
@@ -9,10 +24,6 @@ build_args=--build-arg MODULE=$(MODULE) --build-arg PORT=$(PORT)
 build:
 	docker build -t openslides-datastore-$(MODULE) $(build_args) .
 
-# DEV
-build-dev:
-	docker build -t openslides-datastore-$(MODULE)-dev -f Dockerfile.dev $(build_args) .
-
 run-dev-standalone: | build-dev
 	docker compose -f dc.dev.yml up -d $(MODULE)
 
@@ -27,23 +38,26 @@ ifndef MODULE
 ## TESTS
 
 build-tests:
-	docker build -t openslides-datastore-test -f Dockerfile.test .
+	make build-test
 
 rebuild-tests:
-	docker build -t openslides-datastore-test -f Dockerfile.test . --no-cache
+	docker build . --tag=openslides-datastore-tests --no-cache --build-arg CONTEXT=tests
 
-setup-docker-compose: | build-tests
+setup-docker-compose: | build-tests-old
 	docker compose -f dc.test.yml up -d
 	docker compose -f dc.test.yml exec -T datastore bash -c "chown -R $$(id -u $${USER}):$$(id -g $${USER}) /app"
 
 run-tests-no-down: | setup-docker-compose
 	docker compose -f dc.test.yml exec datastore ./entrypoint.sh pytest
 
-run-tests: | run-tests-no-down
+run-test:| run-tests-no-down
 	docker compose -f dc.test.yml down
 	@$(MAKE) run-dev
 	@$(MAKE) run-full-system-tests
 
+run-tests: 
+	bash dev/run-tests.sh
+
 run-dev run-bash: | setup-docker-compose
 	docker compose -f dc.test.yml exec -u $$(id -u $${USER}):$$(id -g $${USER}) datastore ./entrypoint.sh bash
 
@@ -54,8 +68,8 @@ run-coverage: | setup-docker-compose
 run-ci-no-down: | setup-docker-compose
 	docker compose -f dc.test.yml exec -T datastore ./entrypoint.sh ./execute-ci.sh
 
-run-ci: | run-ci-no-down
-	docker compose -f dc.test.yml down
+run-ci: 
+	bash dev/run-ci.sh 
 
 run-cleanup: | setup-docker-compose
 	docker compose -f dc.test.yml exec -u $$(id -u $${USER}):$$(id -g $${USER}) datastore ./cleanup.sh
@@ -87,10 +101,10 @@ run-full-system-tests-check: | build-full-system-tests
 
 
 # shared has no dev or prod image
-build build-dev:
-	@$(MAKE) -C reader $@
-	@$(MAKE) -C writer $@
-
+# This runs the target 'build' or 'build-dev' on the Makefile in the reader and writer subdirectory
+#build build-dev:
+#	@$(MAKE) -C reader $@
+#	@$(MAKE) -C writer $@
 run:
 	docker compose up -d
 
@@ -103,6 +117,11 @@ run-dev-standalone: | build-dev
 run-dev-verbose: | build-dev
 	docker compose -f dc.dev.yml up
 
+ci-run-system-tests:
+	docker compose -f dc.dev.yml up -d
+	make run-full-system-tests-check
+	make stop-dev
+
 endif
 
 # stopping is the same everywhere