diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml index 6a3bf0e2..d46b5c63 100644 --- a/.github/workflows/continuous_integration.yml +++ b/.github/workflows/continuous_integration.yml @@ -1,4 +1,5 @@ -name: Continuous Integration +name: CI - Build and Test Service + on: pull_request: branches: @@ -13,8 +14,21 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Build container - run: make build + - name: Build Prod + uses: Janmtbehrens/OpenSlides/dev/actions/build-service@all-in-one + with: + service: datastore + context: prod + module: reader + port: 9010 + + - name: Build Prod + uses: Janmtbehrens/OpenSlides/dev/actions/build-service@all-in-one + with: + service: datastore + context: prod + module: writer + port: 9011 - name: Create secret run: mkdir secrets && echo -n "openslides" > secrets/postgres_password @@ -30,11 +44,18 @@ jobs: - name: Test that writer is up and running run: curl -I http://localhost:9011/ - tests: - name: "Tests" + ci-tests: + name: "CI Tests" runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + + - name: Build Tests + uses: Janmtbehrens/OpenSlides/dev/actions/build-service@all-in-one + with: + service: datastore + context: tests + - name: Execute tests run: make run-ci @@ -44,11 +65,33 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Start containers - run: make run-dev-standalone + - name: Build Dev + uses: Janmtbehrens/OpenSlides/dev/actions/build-service@all-in-one + with: + service: datastore + context: dev + module: reader + port: 9010 + + - name: Build Dev + uses: Janmtbehrens/OpenSlides/dev/actions/build-service@all-in-one + with: + service: datastore + context: dev + module: writer + port: 9011 - - name: Execute tests - run: make run-full-system-tests-check + - name: Run System Tests + run: make ci-run-system-tests + + # This setups and runs the dev/run-tests.sh script + run-test-script: + name: Run-tests.sh Script + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 - - name: Stop tests - run: make stop-dev + - name: Run Test + uses: Janmtbehrens/OpenSlides/dev/actions/build-and-test-service@all-in-one + with: + service: datastore \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 933edac3..d499a485 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,40 +1,109 @@ -FROM python:3.10.17-slim-bookworm +ARG CONTEXT=prod -RUN apt-get -y update && apt-get -y upgrade && \ - apt-get install --no-install-recommends -y ncat gcc libpq-dev libc-dev postgresql-client redis-tools cron +FROM python:3.10.17-slim-bookworm as base +## Setup +ARG CONTEXT WORKDIR /app +ENV APP_CONTEXT=${CONTEXT} + +## Install +RUN CONTEXT_INSTALLS=$(case "$APP_CONTEXT" in \ + tests) echo "curl";; \ + dev) echo "";; \ + *) echo "cron" ;; esac) && \ + apt-get -y update && apt-get -y upgrade && apt-get install --no-install-recommends -y \ + gcc \ + libc-dev \ + libpq-dev \ + ncat \ + postgresql-client \ + redis-tools \ + ${CONTEXT_INSTALLS} && \ + rm -rf /var/lib/apt/lists/* + +## Requirements +COPY requirements/* scripts/system/* scripts/* ./ +RUN REQUIREMENTS_FILE=$(case "$APP_CONTEXT" in \ + tests) echo "testing";; \ + dev) echo "testing";; \ + debug) echo "testing";; \ + *) echo "general" ;; esac) && \ + pip install --no-cache-dir -U -r requirements-${REQUIREMENTS_FILE}.txt + ENV PYTHONPATH /app/ -COPY requirements/requirements-general.txt /app/ +## External Information +LABEL org.opencontainers.image.title="OpenSlides Datastore Service" +LABEL org.opencontainers.image.description="Service for OpenSlides which wraps the database, which includes reader and writer functionality." +LABEL org.opencontainers.image.licenses="MIT" +LABEL org.opencontainers.image.source="https://github.com/OpenSlides/openslides-datastore-service" + +## Command +COPY ./dev/command.sh ./ +RUN chmod +x command.sh +CMD ["./command.sh"] +HEALTHCHECK CMD python cli/healthcheck.py +ENTRYPOINT ["./entrypoint.sh"] -RUN pip install -U -r requirements-general.txt +# Testing Image -COPY cli cli -COPY datastore datastore +FROM base as tests + +COPY scripts/* scripts/system/* tests/entrypoint.sh ./ +COPY scripts/ci/* ./ci/ +COPY dev ./dev/ + +STOPSIGNAL SIGKILL + +# Intermediate Image + +FROM base as moduled + +ARG MODULE +RUN test -n "$MODULE" || (echo "MODULE not set" && false) +ENV MODULE=$MODULE ARG PORT RUN test -n "$PORT" || (echo "PORT not set" && false) ENV PORT=$PORT + EXPOSE $PORT -ARG MODULE -RUN test -n "$MODULE" || (echo "MODULE not set" && false) -ENV MODULE=$MODULE +COPY $MODULE/entrypoint.sh ./ + +# Development Image + +FROM moduled as dev + +COPY scripts/system/* scripts/* ./ + +ENV FLASK_APP=datastore.$MODULE.app +ENV FLASK_DEBUG=1 + +# Debug Image + +FROM moduled as debug + +ENV FLASK_APP=datastore.$MODULE.app +ENV FLASK_DEBUG=1 + +# Production Image + +FROM moduled as prod + +# Add appuser +RUN adduser --system --no-create-home appuser && \ + chown appuser /app/ -COPY $MODULE/entrypoint.sh scripts/system/* ./ +COPY cli cli +COPY datastore datastore + +COPY scripts/system/* ./ ENV NUM_WORKERS=1 ENV WORKER_TIMEOUT=30 RUN echo "20 4 * * * root /app/cron.sh >> /var/log/cron.log 2>&1" > /etc/cron.d/trim-collectionfield-tables -LABEL org.opencontainers.image.title="OpenSlides Datastore Service" -LABEL org.opencontainers.image.description="Service for OpenSlides which wraps the database, which includes reader and writer functionality." -LABEL org.opencontainers.image.licenses="MIT" -LABEL org.opencontainers.image.source="https://github.com/OpenSlides/openslides-datastore-service" - -HEALTHCHECK CMD python cli/healthcheck.py - -ENTRYPOINT ["./entrypoint.sh"] -CMD exec gunicorn -w $NUM_WORKERS -b 0.0.0.0:$PORT datastore.$MODULE.app:application -t $WORKER_TIMEOUT +USER appuser diff --git a/Dockerfile.debug b/Dockerfile.debug deleted file mode 100644 index ca215527..00000000 --- a/Dockerfile.debug +++ /dev/null @@ -1,28 +0,0 @@ -FROM python:3.10.17-slim-bookworm - -RUN apt-get -y update && apt-get -y upgrade && \ - apt-get install --no-install-recommends -y ncat gcc libpq-dev libc-dev postgresql-client redis-tools - -WORKDIR /app -COPY requirements/requirements-* scripts/system/* scripts/* ./ - -RUN pip install -U -r requirements-testing.txt - -ARG MODULE -RUN test -n "$MODULE" || (echo "MODULE not set" && false) - -COPY $MODULE/entrypoint.sh ./ - -ENV PYTHONPATH /app/ - -ENV FLASK_APP=datastore.$MODULE.app -ENV FLASK_DEBUG=1 - -ARG PORT -RUN test -n "$PORT" || (echo "PORT not set" && false) -ENV PORT=$PORT - -EXPOSE $PORT - -ENTRYPOINT ["./entrypoint.sh"] -CMD exec python -m debugpy --listen 0.0.0.0:5678 -m flask run -h 0.0.0.0 -p $PORT --no-reload diff --git a/Dockerfile.dev b/Dockerfile.dev deleted file mode 100644 index 948851ee..00000000 --- a/Dockerfile.dev +++ /dev/null @@ -1,28 +0,0 @@ -FROM python:3.10.17-slim-bookworm - -RUN apt-get -y update && apt-get -y upgrade && \ - apt-get install --no-install-recommends -y ncat gcc libpq-dev libc-dev postgresql-client redis-tools - -WORKDIR /app -COPY requirements/requirements-* scripts/system/* scripts/* ./ - -RUN pip install -U -r requirements-testing.txt - -ARG MODULE -RUN test -n "$MODULE" || (echo "MODULE not set" && false) - -COPY $MODULE/entrypoint.sh ./ - -ENV PYTHONPATH /app/ - -ENV FLASK_APP=datastore.$MODULE.app -ENV FLASK_DEBUG=1 - -ARG PORT -RUN test -n "$PORT" || (echo "PORT not set" && false) -ENV PORT=$PORT - -EXPOSE $PORT - -ENTRYPOINT ["./entrypoint.sh"] -CMD exec python -m flask run -h 0.0.0.0 -p $PORT diff --git a/Dockerfile.test b/Dockerfile.test deleted file mode 100644 index 717f9f2b..00000000 --- a/Dockerfile.test +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.10.17-slim-bookworm - -RUN apt-get -y update && apt-get -y upgrade && \ - apt-get install --no-install-recommends -y ncat vim gcc libpq-dev libc-dev postgresql-client redis-tools curl - -WORKDIR /app -ENV PYTHONPATH /app/ -COPY requirements/ . - -RUN pip install -U -r requirements-testing.txt - -COPY scripts/* scripts/system/* tests/entrypoint.sh ./ -COPY scripts/ci/* ./ci/ - -STOPSIGNAL SIGKILL -ENTRYPOINT ["./entrypoint.sh"] -CMD ["sleep", "inf"] diff --git a/Makefile b/Makefile index 65410686..2bb0700e 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,18 @@ +SERVICE=datastore + +# Build images for different contexts + +build-dev: + bash ../dev/scripts/makefile/build-service.sh $(SERVICE) dev reader 9010 + bash ../dev/scripts/makefile/build-service.sh $(SERVICE) dev writer 9011 + +build-prod: + bash ../dev/scripts/makefile/build-service.sh $(SERVICE) prod reader 9010 + bash ../dev/scripts/makefile/build-service.sh $(SERVICE) prod writer 9011 + +build-test: + bash ../dev/scripts/makefile/build-service.sh $(SERVICE) tests + # TESTS ifdef MODULE @@ -9,10 +24,6 @@ build_args=--build-arg MODULE=$(MODULE) --build-arg PORT=$(PORT) build: docker build -t openslides-datastore-$(MODULE) $(build_args) . -# DEV -build-dev: - docker build -t openslides-datastore-$(MODULE)-dev -f Dockerfile.dev $(build_args) . - run-dev-standalone: | build-dev docker compose -f dc.dev.yml up -d $(MODULE) @@ -27,23 +38,26 @@ ifndef MODULE ## TESTS build-tests: - docker build -t openslides-datastore-test -f Dockerfile.test . + make build-test rebuild-tests: - docker build -t openslides-datastore-test -f Dockerfile.test . --no-cache + docker build . --tag=openslides-datastore-tests --no-cache --build-arg CONTEXT=tests -setup-docker-compose: | build-tests +setup-docker-compose: | build-tests-old docker compose -f dc.test.yml up -d docker compose -f dc.test.yml exec -T datastore bash -c "chown -R $$(id -u $${USER}):$$(id -g $${USER}) /app" run-tests-no-down: | setup-docker-compose docker compose -f dc.test.yml exec datastore ./entrypoint.sh pytest -run-tests: | run-tests-no-down +run-test:| run-tests-no-down docker compose -f dc.test.yml down @$(MAKE) run-dev @$(MAKE) run-full-system-tests +run-tests: + bash dev/run-tests.sh + run-dev run-bash: | setup-docker-compose docker compose -f dc.test.yml exec -u $$(id -u $${USER}):$$(id -g $${USER}) datastore ./entrypoint.sh bash @@ -54,8 +68,8 @@ run-coverage: | setup-docker-compose run-ci-no-down: | setup-docker-compose docker compose -f dc.test.yml exec -T datastore ./entrypoint.sh ./execute-ci.sh -run-ci: | run-ci-no-down - docker compose -f dc.test.yml down +run-ci: + bash dev/run-ci.sh run-cleanup: | setup-docker-compose docker compose -f dc.test.yml exec -u $$(id -u $${USER}):$$(id -g $${USER}) datastore ./cleanup.sh @@ -87,10 +101,10 @@ run-full-system-tests-check: | build-full-system-tests # shared has no dev or prod image -build build-dev: - @$(MAKE) -C reader $@ - @$(MAKE) -C writer $@ - +# This runs the target 'build' or 'build-dev' on the Makefile in the reader and writer subdirectory +#build build-dev: +# @$(MAKE) -C reader $@ +# @$(MAKE) -C writer $@ run: docker compose up -d @@ -103,6 +117,11 @@ run-dev-standalone: | build-dev run-dev-verbose: | build-dev docker compose -f dc.dev.yml up +ci-run-system-tests: + docker compose -f dc.dev.yml up -d + make run-full-system-tests-check + make stop-dev + endif # stopping is the same everywhere diff --git a/dc.dev.yml b/dc.dev.yml index c7010b87..75386894 100644 --- a/dc.dev.yml +++ b/dc.dev.yml @@ -1,6 +1,12 @@ version: "3" services: reader: + build: + target: "dev" + args: + CONTEXT: "dev" + MODULE: "reader" + PORT: "9010" image: openslides-datastore-reader-dev volumes: - ./datastore:/app/datastore @@ -16,6 +22,12 @@ services: stdin_open: true tty: true writer: + build: + target: "dev" + args: + CONTEXT: "dev" + MODULE: "writer" + PORT: "9011" image: openslides-datastore-writer-dev volumes: - ./datastore:/app/datastore diff --git a/dc.test.yml b/dc.test.yml index f3cb3d0e..1d6fb729 100644 --- a/dc.test.yml +++ b/dc.test.yml @@ -1,7 +1,11 @@ version: "3" services: datastore: - image: openslides-datastore-test + build: + target: "tests" + args: + CONTEXT: "tests" + image: openslides-datastore-tests command: [ "sleep", diff --git a/dev/command.sh b/dev/command.sh new file mode 100644 index 00000000..6cac6d6f --- /dev/null +++ b/dev/command.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +if [ "$APP_CONTEXT" = "dev" ]; then exec python -m flask run -h 0.0.0.0 -p "$PORT"; fi +if [ "$APP_CONTEXT" = "tests" ]; then sleep inf; fi +if [ "$APP_CONTEXT" = "debug" ]; then exec python -m debugpy --listen 0.0.0.0:5678 -m flask run -h 0.0.0.0 -p "$PORT" --no-reload; fi +if [ "$APP_CONTEXT" = "prod" ]; then exec gunicorn -w "$NUM_WORKERS" -b 0.0.0.0:"$PORT" datastore."$MODULE".app:application -t "$WORKER_TIMEOUT"; fi \ No newline at end of file diff --git a/dev/run-ci.sh b/dev/run-ci.sh new file mode 100644 index 00000000..4087e720 --- /dev/null +++ b/dev/run-ci.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Executes all tests for the CI/CD Pipeline. Should errors occur, CATCH will be set to 1, causing an erroneous exit code. + +echo "########################################################################" +echo "###################### Run Tests and Linters ###########################" +echo "########################################################################" + +# Parameters +PERSIST_CONTAINERS=$1 + +# Setup +IMAGE_TAG=openslides-datastore-tests +CATCH=0 +CHOWN="$(id -u "${USER}"):$(id -g "${USER}")" + +# Execution +if [ "$(docker images -q $IMAGE_TAG)" = "" ]; then make build-test || CATCH=1; fi +docker compose -f dc.test.yml up -d || CATCH=1 +docker compose -f dc.test.yml exec -T datastore bash -c "chown -R $CHOWN /app" || CATCH=1 +docker compose -f dc.test.yml exec -T datastore ./entrypoint.sh ./execute-ci.sh || CATCH=1 + +if [ -z "$PERSIST_CONTAINERS" ]; then docker compose -f dc.test.yml down || CATCH=1; fi + +exit $CATCH \ No newline at end of file diff --git a/dev/run-tests.sh b/dev/run-tests.sh new file mode 100644 index 00000000..a053fafd --- /dev/null +++ b/dev/run-tests.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Executes all tests. Should errors occur, CATCH will be set to 1, causing an erroneous exit code. + +echo "########################################################################" +echo "###################### Run Tests and Linters ###########################" +echo "########################################################################" + +# Parameters +PERSIST_CONTAINERS=$1 + +# Setup +IMAGE_TAG=openslides-datastore-tests +CATCH=0 +CHOWN="$(id -u "${USER}"):$(id -g "${USER}")" + +# Execution +if [ "$(docker images -q $IMAGE_TAG)" = "" ]; then make build-test || CATCH=1; fi +docker compose -f dc.test.yml up -d || CATCH=1 +docker compose -f dc.test.yml exec -T datastore bash -c "chown -R $CHOWN /app" || CATCH=1 +docker compose -f dc.test.yml exec datastore ./entrypoint.sh pytest || CATCH=1 + +if [ -z "$PERSIST_CONTAINERS" ]; then docker compose -f dc.test.yml down || CATCH=1; fi + +exit $CATCH \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index f92a61a2..483337e0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,10 @@ version: "3.4" services: reader: + build: + target: "prod" + args: + CONTEXT: "prod" image: openslides-datastore-reader ports: - "${OPENSLIDES_DATASTORE_READER_PORT}:${OPENSLIDES_DATASTORE_READER_PORT}" diff --git a/scripts/ci/40-mypy.sh b/scripts/ci/40-mypy.sh index 30ee3c26..58236a48 100755 --- a/scripts/ci/40-mypy.sh +++ b/scripts/ci/40-mypy.sh @@ -2,4 +2,4 @@ set -e -mypy $(ls -d */ | grep -v "ci/") +mypy $(ls -d */ | grep -v "ci/" | grep -v "dev/")