diff --git a/.env-devel b/.env-devel index e2f8baef3ca..bdb406f1e7b 100644 --- a/.env-devel +++ b/.env-devel @@ -83,6 +83,12 @@ DIRECTOR_REGISTRY_CACHING=True DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS=null DIRECTOR_TRACING=null +DOCKER_API_PROXY_HOST=docker-api-proxy +DOCKER_API_PROXY_PASSWORD=null +DOCKER_API_PROXY_PORT=8888 +DOCKER_API_PROXY_SECURE=False +DOCKER_API_PROXY_USER=null + EFS_USER_ID=8006 EFS_USER_NAME=efs EFS_GROUP_ID=8106 diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 54e2dee1fe5..a1778e11092 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -77,6 +77,7 @@ jobs: migration: ${{ steps.filter.outputs.migration }} payments: ${{ steps.filter.outputs.payments }} dynamic-scheduler: ${{ steps.filter.outputs.dynamic-scheduler }} + docker-api-proxy: ${{ steps.filter.outputs.docker-api-proxy }} resource-usage-tracker: ${{ steps.filter.outputs.resource-usage-tracker }} static-webserver: ${{ steps.filter.outputs.static-webserver }} storage: ${{ steps.filter.outputs.storage }} @@ -233,6 +234,9 @@ jobs: - 'services/docker-compose*' - 'scripts/mypy/*' - 'mypy.ini' + docker-api-proxy: + - 'packages/**' + - 'services/docker-api-proxy/**' resource-usage-tracker: - 'packages/**' - 'services/resource-usage-tracker/**' @@ -2190,6 +2194,71 @@ jobs: with: flags: integrationtests #optional + + integration-test-docker-api-proxy: + needs: [changes, build-test-images] + if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.docker-api-proxy == 'true' || github.event_name == 'push'}} + timeout-minutes: 30 # if this timeout gets too small, then split the tests + name: "[int] docker-api-proxy" + runs-on: ${{ matrix.os }} + strategy: + matrix: + python: ["3.11"] + os: [ubuntu-22.04] + fail-fast: false + steps: + - uses: actions/checkout@v4 + - name: setup docker buildx + id: buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker-container + - name: setup python environment + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: expose github runtime for buildx + uses: crazy-max/ghaction-github-runtime@v3 + # FIXME: Workaround for https://github.com/actions/download-artifact/issues/249 + - name: download docker images with retry + uses: Wandalen/wretry.action@master + with: + action: actions/download-artifact@v4 + with: | + name: docker-buildx-images-${{ runner.os }}-${{ github.sha }}-backend + path: /${{ runner.temp }}/build + attempt_limit: 5 + attempt_delay: 1000 + - name: load docker images + run: make load-images local-src=/${{ runner.temp }}/build + - name: install uv + uses: astral-sh/setup-uv@v5 + with: + version: "0.5.x" + enable-cache: false + cache-dependency-glob: "**/docker-api-proxy/requirements/ci.txt" + - name: show system version + run: ./ci/helpers/show_system_versions.bash + - name: install + run: ./ci/github/integration-testing/docker-api-proxy.bash install + - name: test + run: ./ci/github/integration-testing/docker-api-proxy.bash test + - name: upload failed tests logs + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ github.job }}_docker_logs + path: ./services/docker-api-proxy/test_failures + - name: cleanup + if: ${{ !cancelled() }} + run: ./ci/github/integration-testing/docker-api-proxy.bash clean_up + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: integrationtests #optional + integration-test-simcore-sdk: needs: [changes, build-test-images] if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.simcore-sdk == 'true' || github.event_name == 'push' }} @@ -2262,6 +2331,7 @@ jobs: integration-test-director-v2-01, integration-test-director-v2-02, integration-test-dynamic-sidecar, + integration-test-docker-api-proxy, integration-test-simcore-sdk, integration-test-webserver-01, integration-test-webserver-02, diff --git a/Makefile b/Makefile index d8f0ccf252d..71d81fba569 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,7 @@ SERVICES_NAMES_TO_BUILD := \ payments \ resource-usage-tracker \ dynamic-scheduler \ + docker-api-proxy \ service-integration \ static-webserver \ storage \ diff --git a/ci/github/integration-testing/docker-api-proxy.bash b/ci/github/integration-testing/docker-api-proxy.bash new file mode 100755 index 00000000000..c7ad9775c07 --- /dev/null +++ b/ci/github/integration-testing/docker-api-proxy.bash @@ -0,0 +1,40 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes +IFS=$'\n\t' + +install() { + make devenv + # shellcheck source=/dev/null + source .venv/bin/activate + pushd services/docker-api-proxy + make install-ci + popd + uv pip list + make info-images +} + +test() { + # shellcheck source=/dev/null + source .venv/bin/activate + pushd services/docker-api-proxy + make test-ci-integration + popd +} + +clean_up() { + docker images + make down +} + +# Check if the function exists (bash specific) +if declare -f "$1" >/dev/null; then + # call arguments verbatim + "$@" +else + # Show a helpful error + echo "'$1' is not a known function name" >&2 + exit 1 +fi diff --git a/packages/models-library/src/models_library/errors.py b/packages/models-library/src/models_library/errors.py index 26b4aa0d91d..cec882e12b7 100644 --- a/packages/models-library/src/models_library/errors.py +++ b/packages/models-library/src/models_library/errors.py @@ -36,6 +36,7 @@ class ErrorDict(_ErrorDictRequired, total=False): RABBITMQ_CLIENT_UNHEALTHY_MSG = "RabbitMQ client is in a bad state!" REDIS_CLIENT_UNHEALTHY_MSG = "Redis cannot be reached!" +DOCKER_API_PROXY_UNHEALTHY_MSG = "docker-api-proxy service is not reachable!" # NOTE: Here we do not just import as 'from pydantic.error_wrappers import ErrorDict' diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py b/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py new file mode 100644 index 00000000000..1871eefdfed --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py @@ -0,0 +1,52 @@ +import logging + +import pytest +from aiohttp import ClientSession, ClientTimeout +from pydantic import TypeAdapter +from settings_library.docker_api_proxy import DockerApiProxysettings +from tenacity import before_sleep_log, retry, stop_after_delay, wait_fixed + +from .helpers.docker import get_service_published_port +from .helpers.host import get_localhost_ip +from .helpers.typing_env import EnvVarsDict + +_logger = logging.getLogger(__name__) + + +@retry( + wait=wait_fixed(1), + stop=stop_after_delay(10), + before_sleep=before_sleep_log(_logger, logging.INFO), + reraise=True, +) +async def _wait_till_docker_api_proxy_is_responsive( + settings: DockerApiProxysettings, +) -> None: + async with ClientSession(timeout=ClientTimeout(1, 1, 1, 1, 1)) as client: + response = await client.get(f"{settings.base_url}/version") + assert response.status == 200, await response.text() + + +@pytest.fixture +async def docker_api_proxy_settings( + docker_stack: dict, env_vars_for_docker_compose: EnvVarsDict +) -> DockerApiProxysettings: + """Returns the settings of a redis service that is up and responsive""" + + prefix = env_vars_for_docker_compose["SWARM_STACK_NAME"] + assert f"{prefix}_docker-api-proxy" in docker_stack["services"] + + published_port = get_service_published_port( + "docker-api-proxy", int(env_vars_for_docker_compose["DOCKER_API_PROXY_PORT"]) + ) + + settings = TypeAdapter(DockerApiProxysettings).validate_python( + { + "DOCKER_API_PROXY_HOST": get_localhost_ip(), + "DOCKER_API_PROXY_PORT": published_port, + } + ) + + await _wait_till_docker_api_proxy_is_responsive(settings) + + return settings diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py index 11dd165a963..88dba2e4543 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py @@ -52,10 +52,12 @@ "invitations": "/", "payments": "/", "resource-usage-tracker": "/", + "docker-api-proxy": "/version", } AIOHTTP_BASED_SERVICE_PORT: int = 8080 FASTAPI_BASED_SERVICE_PORT: int = 8000 DASK_SCHEDULER_SERVICE_PORT: int = 8787 +DOCKER_API_PROXY_SERVICE_PORT: int = 8888 _SERVICE_NAME_REPLACEMENTS: dict[str, str] = { "dynamic-scheduler": "dynamic-schdlr", @@ -133,6 +135,7 @@ def services_endpoint( AIOHTTP_BASED_SERVICE_PORT, FASTAPI_BASED_SERVICE_PORT, DASK_SCHEDULER_SERVICE_PORT, + DOCKER_API_PROXY_SERVICE_PORT, ] endpoint = URL( f"http://{get_localhost_ip()}:{get_service_published_port(full_service_name, target_ports)}" diff --git a/packages/service-library/src/servicelib/fastapi/docker.py b/packages/service-library/src/servicelib/fastapi/docker.py new file mode 100644 index 00000000000..0fa7706e90a --- /dev/null +++ b/packages/service-library/src/servicelib/fastapi/docker.py @@ -0,0 +1,83 @@ +import asyncio +import logging +from collections.abc import AsyncIterator +from contextlib import AsyncExitStack +from typing import Final + +import aiodocker +import aiohttp +import tenacity +from aiohttp import ClientSession +from fastapi import FastAPI +from fastapi_lifespan_manager import State +from pydantic import NonNegativeInt +from servicelib.fastapi.lifespan_utils import LifespanGenerator +from settings_library.docker_api_proxy import DockerApiProxysettings + +_logger = logging.getLogger(__name__) + +_DEFAULT_DOCKER_API_PROXY_HEALTH_TIMEOUT: Final[NonNegativeInt] = 5 + + +def get_lifespan_remote_docker_client( + settings: DockerApiProxysettings, +) -> LifespanGenerator: + async def _(app: FastAPI) -> AsyncIterator[State]: + + session: ClientSession | None = None + if settings.DOCKER_API_PROXY_USER and settings.DOCKER_API_PROXY_PASSWORD: + session = ClientSession( + auth=aiohttp.BasicAuth( + login=settings.DOCKER_API_PROXY_USER, + password=settings.DOCKER_API_PROXY_PASSWORD.get_secret_value(), + ) + ) + + async with AsyncExitStack() as exit_stack: + if settings.DOCKER_API_PROXY_USER and settings.DOCKER_API_PROXY_PASSWORD: + await exit_stack.enter_async_context( + ClientSession( + auth=aiohttp.BasicAuth( + login=settings.DOCKER_API_PROXY_USER, + password=settings.DOCKER_API_PROXY_PASSWORD.get_secret_value(), + ) + ) + ) + + client = await exit_stack.enter_async_context( + aiodocker.Docker(url=settings.base_url, session=session) + ) + + app.state.remote_docker_client = client + + await wait_till_docker_api_proxy_is_responsive(app) + + # NOTE this has to be inside exit_stack scope + yield {} + + return _ + + +@tenacity.retry( + wait=tenacity.wait_fixed(5), + stop=tenacity.stop_after_delay(60), + before_sleep=tenacity.before_sleep_log(_logger, logging.INFO), + reraise=True, +) +async def wait_till_docker_api_proxy_is_responsive(app: FastAPI) -> None: + await is_docker_api_proxy_ready(app) + + +async def is_docker_api_proxy_ready( + app: FastAPI, *, timeout=_DEFAULT_DOCKER_API_PROXY_HEALTH_TIMEOUT # noqa: ASYNC109 +) -> bool: + try: + await asyncio.wait_for(get_remote_docker_client(app).version(), timeout=timeout) + except (aiodocker.DockerError, TimeoutError): + return False + return True + + +def get_remote_docker_client(app: FastAPI) -> aiodocker.Docker: + assert isinstance(app.state.remote_docker_client, aiodocker.Docker) # nosec + return app.state.remote_docker_client diff --git a/packages/settings-library/src/settings_library/docker_api_proxy.py b/packages/settings-library/src/settings_library/docker_api_proxy.py new file mode 100644 index 00000000000..01dfffbead6 --- /dev/null +++ b/packages/settings-library/src/settings_library/docker_api_proxy.py @@ -0,0 +1,24 @@ +from functools import cached_property + +from pydantic import Field, SecretStr + +from .base import BaseCustomSettings +from .basic_types import PortInt + + +class DockerApiProxysettings(BaseCustomSettings): + DOCKER_API_PROXY_HOST: str = Field( + description="hostname of the docker-api-proxy service" + ) + DOCKER_API_PROXY_PORT: PortInt = Field( + 8888, description="port of the docker-api-proxy service" + ) + DOCKER_API_PROXY_SECURE: bool = False + + DOCKER_API_PROXY_USER: str | None = None + DOCKER_API_PROXY_PASSWORD: SecretStr | None = None + + @cached_property + def base_url(self) -> str: + protocl = "https" if self.DOCKER_API_PROXY_SECURE else "http" + return f"{protocl}://{self.DOCKER_API_PROXY_HOST}:{self.DOCKER_API_PROXY_PORT}" diff --git a/services/docker-api-proxy/Dockerfile b/services/docker-api-proxy/Dockerfile new file mode 100644 index 00000000000..437d549e9fe --- /dev/null +++ b/services/docker-api-proxy/Dockerfile @@ -0,0 +1,44 @@ +FROM alpine:3.21 AS base + +LABEL maintainer=GitHK + +# simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) +ENV SC_USER_ID=8004 \ + SC_USER_NAME=scu \ + SC_BUILD_TARGET=base \ + SC_BOOT_MODE=default + +RUN addgroup -g ${SC_USER_ID} ${SC_USER_NAME} && \ + adduser -u ${SC_USER_ID} -G ${SC_USER_NAME} \ + --disabled-password \ + --gecos "" \ + --shell /bin/sh \ + --home /home/${SC_USER_NAME} \ + ${SC_USER_NAME} + +RUN apk add --no-cache socat curl && \ + curl -L -o /usr/local/bin/gosu https://github.com/tianon/gosu/releases/download/1.16/gosu-amd64 && \ + chmod +x /usr/local/bin/gosu && \ + gosu --version + + +# Health check to ensure the proxy is running +HEALTHCHECK \ + --interval=10s \ + --timeout=5s \ + --start-period=30s \ + --start-interval=1s \ + --retries=5 \ + CMD curl http://localhost:8888/version || exit 1 + +COPY --chown=scu:scu services/docker-api-proxy/docker services/docker-api-proxy/docker +RUN chmod +x services/docker-api-proxy/docker/*.sh + +ENTRYPOINT [ "/bin/sh", "services/docker-api-proxy/docker/entrypoint.sh" ] +CMD ["/bin/sh", "services/docker-api-proxy/docker/boot.sh"] + +FROM base AS development +ENV SC_BUILD_TARGET=development + +FROM base AS production +ENV SC_BUILD_TARGET=production diff --git a/services/docker-api-proxy/Makefile b/services/docker-api-proxy/Makefile new file mode 100644 index 00000000000..82ebf1a73f3 --- /dev/null +++ b/services/docker-api-proxy/Makefile @@ -0,0 +1,2 @@ +include ../../scripts/common.Makefile +include ../../scripts/common-service.Makefile diff --git a/services/docker-api-proxy/docker/boot.sh b/services/docker-api-proxy/docker/boot.sh new file mode 100755 index 00000000000..8fa139339b9 --- /dev/null +++ b/services/docker-api-proxy/docker/boot.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -o errexit +set -o nounset + +IFS=$(printf '\n\t') + +INFO="INFO: [$(basename "$0")] " + +echo "$INFO" "Booting in ${SC_BOOT_MODE} mode ..." +echo "$INFO" "User :$(id "$(whoami)")" + +# +# RUNNING application +# +socat TCP-LISTEN:8888,fork,reuseaddr UNIX-CONNECT:/var/run/docker.sock diff --git a/services/docker-api-proxy/docker/entrypoint.sh b/services/docker-api-proxy/docker/entrypoint.sh new file mode 100755 index 00000000000..d0fbbea5cf9 --- /dev/null +++ b/services/docker-api-proxy/docker/entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# +# - Executes *inside* of the container upon start as --user [default root] +# - Notice that the container *starts* as --user [default root] but +# *runs* as non-root user [scu] +# +set -o errexit +set -o nounset + +IFS=$(printf '\n\t') + +INFO="INFO: [$(basename "$0")] " + +echo "$INFO" "Entrypoint for stage ${SC_BUILD_TARGET} ..." +echo "$INFO" "User :$(id "$(whoami)")" + +# Appends docker group +DOCKER_MOUNT=/var/run/docker.sock +echo "INFO: detected docker socket is mounted, adding user to group..." +GROUPID=$(stat -c %g $DOCKER_MOUNT) # Alpine uses `-c` instead of `--format` +GROUPNAME=scdocker + +# Check if a group with the specified GID exists +if ! addgroup -g "$GROUPID" $GROUPNAME >/dev/null 2>&1; then + echo "WARNING: docker group with GID $GROUPID already exists, getting group name..." + # Get the group name based on GID + GROUPNAME=$(getent group | awk -F: "\$3 == $GROUPID {print \$1}") + echo "WARNING: docker group with GID $GROUPID has name $GROUPNAME" +fi + +# Add the user to the group +adduser "$SC_USER_NAME" $GROUPNAME + +exec gosu "$SC_USER_NAME" "$@" diff --git a/services/docker-api-proxy/requirements/Makefile b/services/docker-api-proxy/requirements/Makefile new file mode 100644 index 00000000000..3f25442b790 --- /dev/null +++ b/services/docker-api-proxy/requirements/Makefile @@ -0,0 +1,6 @@ +# +# Targets to pip-compile requirements +# +include ../../../requirements/base.Makefile + +# Add here any extra explicit dependency: e.g. _migration.txt: _base.txt diff --git a/services/docker-api-proxy/requirements/_base.in b/services/docker-api-proxy/requirements/_base.in new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/docker-api-proxy/requirements/_base.txt b/services/docker-api-proxy/requirements/_base.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/docker-api-proxy/requirements/_test.in b/services/docker-api-proxy/requirements/_test.in new file mode 100644 index 00000000000..321d2e72461 --- /dev/null +++ b/services/docker-api-proxy/requirements/_test.in @@ -0,0 +1,22 @@ +--constraint ../../../requirements/constraints.txt + +--requirement ../../../packages/common-library/requirements/_base.in +--requirement ../../../packages/models-library/requirements/_base.in +--requirement ../../../packages/settings-library/requirements/_base.in +--requirement ../../../packages/service-library/requirements/_base.in + +aiodocker +arrow +asgi_lifespan +docker +faker +fastapi +fastapi-lifespan-manager +flaky +pytest +pytest-asyncio +pytest-cov +pytest-mock +python-dotenv +PyYAML +tenacity diff --git a/services/docker-api-proxy/requirements/_test.txt b/services/docker-api-proxy/requirements/_test.txt new file mode 100644 index 00000000000..4e719d9681e --- /dev/null +++ b/services/docker-api-proxy/requirements/_test.txt @@ -0,0 +1,448 @@ +aio-pika==9.5.4 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiocache==0.12.3 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiodebug==2.3.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiodocker==0.24.0 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/_test.in +aiofiles==24.1.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiohappyeyeballs==2.4.4 + # via aiohttp +aiohttp==3.11.11 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # aiodocker +aiormq==6.8.1 + # via aio-pika +aiosignal==1.3.2 + # via aiohttp +annotated-types==0.7.0 + # via pydantic +anyio==4.8.0 + # via + # fast-depends + # faststream + # starlette +arrow==1.3.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/_test.in +asgi-lifespan==2.1.0 + # via -r requirements/_test.in +attrs==25.1.0 + # via + # aiohttp + # jsonschema + # referencing +certifi==2024.12.14 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # requests +charset-normalizer==3.4.1 + # via requests +click==8.1.8 + # via typer +coverage==7.6.10 + # via pytest-cov +deprecated==1.2.18 + # via + # opentelemetry-api + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-semantic-conventions +dnspython==2.7.0 + # via email-validator +docker==7.1.0 + # via -r requirements/_test.in +email-validator==2.2.0 + # via pydantic +exceptiongroup==1.2.2 + # via aio-pika +faker==35.0.0 + # via -r requirements/_test.in +fast-depends==2.4.12 + # via faststream +fastapi==0.115.7 + # via + # -r requirements/_test.in + # fastapi-lifespan-manager +fastapi-lifespan-manager==0.1.4 + # via -r requirements/_test.in +faststream==0.5.34 + # via -r requirements/../../../packages/service-library/requirements/_base.in +flaky==3.8.1 + # via -r requirements/_test.in +frozenlist==1.5.0 + # via + # aiohttp + # aiosignal +googleapis-common-protos==1.66.0 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +grpcio==1.70.0 + # via opentelemetry-exporter-otlp-proto-grpc +idna==3.10 + # via + # anyio + # email-validator + # requests + # yarl +importlib-metadata==8.5.0 + # via opentelemetry-api +iniconfig==2.0.0 + # via pytest +jsonschema==4.23.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +jsonschema-specifications==2024.10.1 + # via jsonschema +markdown-it-py==3.0.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +multidict==6.1.0 + # via + # aiohttp + # yarl +opentelemetry-api==1.29.0 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-instrumentation + # opentelemetry-instrumentation-logging + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-exporter-otlp==1.29.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-exporter-otlp-proto-common==1.29.0 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-exporter-otlp-proto-grpc==1.29.0 + # via opentelemetry-exporter-otlp +opentelemetry-exporter-otlp-proto-http==1.29.0 + # via opentelemetry-exporter-otlp +opentelemetry-instrumentation==0.50b0 + # via + # opentelemetry-instrumentation-logging + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests +opentelemetry-instrumentation-logging==0.50b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-redis==0.50b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-requests==0.50b0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-proto==1.29.0 + # via + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-sdk==1.29.0 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-semantic-conventions==0.50b0 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests + # opentelemetry-sdk +opentelemetry-util-http==0.50b0 + # via opentelemetry-instrumentation-requests +orjson==3.10.15 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +packaging==24.2 + # via + # opentelemetry-instrumentation + # pytest +pamqp==3.3.0 + # via aiormq +pluggy==1.5.0 + # via pytest +propcache==0.2.1 + # via + # aiohttp + # yarl +protobuf==5.29.3 + # via + # googleapis-common-protos + # opentelemetry-proto +psutil==6.1.1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +pydantic==2.10.6 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in + # fast-depends + # fastapi + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.27.2 + # via pydantic +pydantic-extra-types==2.10.2 + # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.7.1 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in +pygments==2.19.1 + # via rich +pyinstrument==5.0.1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +pytest==8.3.4 + # via + # -r requirements/_test.in + # pytest-asyncio + # pytest-cov + # pytest-mock +pytest-asyncio==0.23.8 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_test.in +pytest-cov==6.0.0 + # via -r requirements/_test.in +pytest-mock==3.14.0 + # via -r requirements/_test.in +python-dateutil==2.9.0.post0 + # via + # arrow + # faker +python-dotenv==1.0.1 + # via + # -r requirements/_test.in + # pydantic-settings +pyyaml==6.0.2 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/_test.in +redis==5.2.1 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/service-library/requirements/_base.in +referencing==0.35.1 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # jsonschema + # jsonschema-specifications +requests==2.32.3 + # via + # docker + # opentelemetry-exporter-otlp-proto-http +rich==13.9.4 + # via + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in + # typer +rpds-py==0.22.3 + # via + # jsonschema + # referencing +shellingham==1.5.4 + # via typer +six==1.17.0 + # via python-dateutil +sniffio==1.3.1 + # via + # anyio + # asgi-lifespan +starlette==0.45.3 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # fastapi +tenacity==9.0.0 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/_test.in +toolz==1.0.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +tqdm==4.67.1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +typer==0.15.1 + # via + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in +types-python-dateutil==2.9.0.20241206 + # via arrow +typing-extensions==4.12.2 + # via + # aiodebug + # anyio + # faker + # fastapi + # faststream + # opentelemetry-sdk + # pydantic + # pydantic-core + # pydantic-extra-types + # typer +urllib3==2.3.0 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # docker + # requests +wrapt==1.17.2 + # via + # deprecated + # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis +yarl==1.18.3 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # aio-pika + # aiohttp + # aiormq +zipp==3.21.0 + # via importlib-metadata diff --git a/services/docker-api-proxy/requirements/_tools.in b/services/docker-api-proxy/requirements/_tools.in new file mode 100644 index 00000000000..140b6ed2e30 --- /dev/null +++ b/services/docker-api-proxy/requirements/_tools.in @@ -0,0 +1,6 @@ +--constraint ../../../requirements/constraints.txt + +--constraint _base.txt +--constraint _test.txt + +--requirement ../../../requirements/devenv.txt diff --git a/services/docker-api-proxy/requirements/_tools.txt b/services/docker-api-proxy/requirements/_tools.txt new file mode 100644 index 00000000000..11e770335f4 --- /dev/null +++ b/services/docker-api-proxy/requirements/_tools.txt @@ -0,0 +1,80 @@ +astroid==3.3.8 + # via pylint +black==25.1.0 + # via -r requirements/../../../requirements/devenv.txt +build==1.2.2.post1 + # via pip-tools +bump2version==1.0.1 + # via -r requirements/../../../requirements/devenv.txt +cfgv==3.4.0 + # via pre-commit +click==8.1.8 + # via + # -c requirements/_test.txt + # black + # pip-tools +dill==0.3.9 + # via pylint +distlib==0.3.9 + # via virtualenv +filelock==3.17.0 + # via virtualenv +identify==2.6.6 + # via pre-commit +isort==6.0.0 + # via + # -r requirements/../../../requirements/devenv.txt + # pylint +mccabe==0.7.0 + # via pylint +mypy==1.14.1 + # via -r requirements/../../../requirements/devenv.txt +mypy-extensions==1.0.0 + # via + # black + # mypy +nodeenv==1.9.1 + # via pre-commit +packaging==24.2 + # via + # -c requirements/_test.txt + # black + # build +pathspec==0.12.1 + # via black +pip==25.0 + # via pip-tools +pip-tools==7.4.1 + # via -r requirements/../../../requirements/devenv.txt +platformdirs==4.3.6 + # via + # black + # pylint + # virtualenv +pre-commit==4.1.0 + # via -r requirements/../../../requirements/devenv.txt +pylint==3.3.4 + # via -r requirements/../../../requirements/devenv.txt +pyproject-hooks==1.2.0 + # via + # build + # pip-tools +pyyaml==6.0.2 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_test.txt + # pre-commit +ruff==0.9.3 + # via -r requirements/../../../requirements/devenv.txt +setuptools==75.8.0 + # via pip-tools +tomlkit==0.13.2 + # via pylint +typing-extensions==4.12.2 + # via + # -c requirements/_test.txt + # mypy +virtualenv==20.29.1 + # via pre-commit +wheel==0.45.1 + # via pip-tools diff --git a/services/docker-api-proxy/requirements/ci.txt b/services/docker-api-proxy/requirements/ci.txt new file mode 100644 index 00000000000..419f091d4d0 --- /dev/null +++ b/services/docker-api-proxy/requirements/ci.txt @@ -0,0 +1,18 @@ +# Shortcut to install all packages for the contigous integration (CI) of 'services/api-server' +# +# - As ci.txt but w/ tests +# +# Usage: +# pip install -r requirements/ci.txt +# + +# installs base + tests requirements +--requirement _base.txt +--requirement _test.txt + +# installs this repo's packages +simcore-common-library @ ../../packages/common-library/ +simcore-models-library @ ../../packages/models-library/ +pytest-simcore @ ../../packages/pytest-simcore/ +simcore-service-library @ ../../packages/service-library +simcore-settings-library @ ../../packages/settings-library/ diff --git a/services/docker-api-proxy/requirements/dev.txt b/services/docker-api-proxy/requirements/dev.txt new file mode 100644 index 00000000000..57a8239ab79 --- /dev/null +++ b/services/docker-api-proxy/requirements/dev.txt @@ -0,0 +1,19 @@ +# Shortcut to install all packages needed to develop 'services/docker-api-proxy' +# +# - As ci.txt but with current and repo packages in develop (edit) mode +# +# Usage: +# pip install -r requirements/dev.txt +# + +# installs this repo's packages +--editable ../../packages/common-library +--editable ../../packages/models-library +--editable ../../packages/pytest-simcore +--editable ../../packages/service-library +--editable ../../packages/settings-library + +# installs base + tests requirements +--requirement _base.txt +--requirement _test.txt +--requirement _tools.txt diff --git a/services/docker-api-proxy/tests/integration/autentication-proxy-docker-compose.yaml b/services/docker-api-proxy/tests/integration/autentication-proxy-docker-compose.yaml new file mode 100644 index 00000000000..44d7e02d21f --- /dev/null +++ b/services/docker-api-proxy/tests/integration/autentication-proxy-docker-compose.yaml @@ -0,0 +1,14 @@ +version: '3.8' +services: + caddy: + image: caddy:2.9.1-alpine + ports: + - 9999:9999 + command: sh -c "echo '${CADDY_FILE}' > /etc/caddy/Caddyfile && cat /etc/caddy/Caddyfile && caddy run --adapter caddyfile --config /etc/caddy/Caddyfile" + networks: + - docker-api-network + +networks: + docker-api-network: + name: pytest-simcore_docker-api-network + external: true diff --git a/services/docker-api-proxy/tests/integration/conftest.py b/services/docker-api-proxy/tests/integration/conftest.py new file mode 100644 index 00000000000..0d02392f917 --- /dev/null +++ b/services/docker-api-proxy/tests/integration/conftest.py @@ -0,0 +1,67 @@ +# pylint:disable=unrecognized-options + +from collections.abc import AsyncIterator, Callable +from contextlib import AbstractAsyncContextManager, asynccontextmanager +from typing import Annotated + +import aiodocker +import pytest +from asgi_lifespan import LifespanManager +from fastapi import FastAPI +from pydantic import Field +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict +from servicelib.fastapi.docker import ( + get_lifespan_remote_docker_client, + get_remote_docker_client, +) +from servicelib.fastapi.lifespan_utils import combine_lifespans +from settings_library.application import BaseApplicationSettings +from settings_library.docker_api_proxy import DockerApiProxysettings + +pytest_plugins = [ + "pytest_simcore.docker_api_proxy", + "pytest_simcore.docker_compose", + "pytest_simcore.docker_swarm", + "pytest_simcore.repository_paths", + "pytest_simcore.simcore_services", +] + + +def pytest_configure(config): + # Set asyncio_mode to "auto" + config.option.asyncio_mode = "auto" + + +def _get_test_app() -> FastAPI: + class ApplicationSetting(BaseApplicationSettings): + DOCKER_API_PROXY: Annotated[ + DockerApiProxysettings, + Field(json_schema_extra={"auto_default_from_env": True}), + ] + + settings = ApplicationSetting.create_from_envs() + + app = FastAPI( + lifespan=combine_lifespans( + get_lifespan_remote_docker_client(settings.DOCKER_API_PROXY) + ) + ) + app.state.settings = settings + + return app + + +@pytest.fixture +async def setup_docker_client( + monkeypatch: pytest.MonkeyPatch, +) -> Callable[[EnvVarsDict], AbstractAsyncContextManager[aiodocker.Docker]]: + @asynccontextmanager + async def _(env_vars: EnvVarsDict) -> AsyncIterator[aiodocker.Docker]: + setenvs_from_dict(monkeypatch, env_vars) + + app = _get_test_app() + + async with LifespanManager(app, startup_timeout=30, shutdown_timeout=30): + yield get_remote_docker_client(app) + + return _ diff --git a/services/docker-api-proxy/tests/integration/test_docker_api_proxy.py b/services/docker-api-proxy/tests/integration/test_docker_api_proxy.py new file mode 100644 index 00000000000..1e3f4641d9e --- /dev/null +++ b/services/docker-api-proxy/tests/integration/test_docker_api_proxy.py @@ -0,0 +1,33 @@ +# pylint: disable=unused-argument + +import json +import sys +from collections.abc import Callable +from contextlib import AbstractAsyncContextManager +from pathlib import Path + +import aiodocker +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict +from settings_library.docker_api_proxy import DockerApiProxysettings + +HERE = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + +pytest_simcore_core_services_selection = [ + "docker-api-proxy", +] + + +async def test_unauthenticated_docker_client( + docker_swarm: None, + docker_api_proxy_settings: DockerApiProxysettings, + setup_docker_client: Callable[ + [EnvVarsDict], AbstractAsyncContextManager[aiodocker.Docker] + ], +): + envs = { + "DOCKER_API_PROXY_HOST": "127.0.0.1", + "DOCKER_API_PROXY_PORT": "8014", + } + async with setup_docker_client(envs) as working_docker: + info = await working_docker.system.info() + print(json.dumps(info, indent=2)) diff --git a/services/docker-api-proxy/tests/integration/test_docker_api_proxy_autenticated.py b/services/docker-api-proxy/tests/integration/test_docker_api_proxy_autenticated.py new file mode 100644 index 00000000000..a40437e3c78 --- /dev/null +++ b/services/docker-api-proxy/tests/integration/test_docker_api_proxy_autenticated.py @@ -0,0 +1,123 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument + +import json +import subprocess +import sys +from collections.abc import Callable, Iterator +from contextlib import AbstractAsyncContextManager +from pathlib import Path + +import aiodocker +import pytest +from pytest_mock import MockerFixture +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict +from settings_library.docker_api_proxy import DockerApiProxysettings +from tenacity import AsyncRetrying, stop_after_delay, wait_fixed + +pytest_simcore_core_services_selection = [ + "docker-api-proxy", +] + +_HERE = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + + +@pytest.fixture +def authentication_proxy_compose_path() -> Path: + compose_spec_path = _HERE / "autentication-proxy-docker-compose.yaml" + assert compose_spec_path.exists() + return compose_spec_path + + +@pytest.fixture +def caddy_file() -> str: + # NOTE: the basicauth encrypeted credentials are `asd:asd`` + return """ +:9999 { + handle { + basicauth { + asd $2a$14$slb.GSUwFUX4jPOMoYTmKePjH.2UPJBkLmTPT5RmOfn38seYld1nu + } + reverse_proxy http://docker-api-proxy:8888 { + health_uri /version + } + } +} + """ + + +@pytest.fixture +def mock_wait_till_docker_api_proxy_is_responsive(mocker: MockerFixture) -> None: + mocker.patch("servicelib.fastapi.docker.wait_till_docker_api_proxy_is_responsive") + + +@pytest.fixture +def authentication_proxy( + mock_wait_till_docker_api_proxy_is_responsive: None, + docker_swarm: None, + docker_api_proxy_settings: DockerApiProxysettings, + caddy_file: str, + authentication_proxy_compose_path: Path, +) -> Iterator[None]: + + stack_name = "with-auth" + subprocess.run( # noqa: S603 + [ # noqa: S607 + "docker", + "stack", + "deploy", + "-c", + authentication_proxy_compose_path, + stack_name, + ], + check=True, + env={"CADDY_FILE": caddy_file}, + ) + + yield + + subprocess.run( # noqa: S603 + ["docker", "stack", "rm", stack_name], check=True # noqa: S607 + ) + + +async def test_with_correct_credentials( + authentication_proxy: None, + setup_docker_client: Callable[ + [EnvVarsDict], AbstractAsyncContextManager[aiodocker.Docker] + ], +): + envs = { + "DOCKER_API_PROXY_HOST": "127.0.0.1", + "DOCKER_API_PROXY_PORT": "9999", + "DOCKER_API_PROXY_USER": "asd", + "DOCKER_API_PROXY_PASSWORD": "asd", + } + async with setup_docker_client(envs) as working_docker: + async for attempt in AsyncRetrying( + wait=wait_fixed(0.1), stop=stop_after_delay(10), reraise=True + ): + with attempt: + info = await working_docker.system.info() + print(json.dumps(info, indent=2)) + + +async def test_wrong_credentials( + authentication_proxy: None, + setup_docker_client: Callable[ + [EnvVarsDict], AbstractAsyncContextManager[aiodocker.Docker] + ], +): + envs = { + "DOCKER_API_PROXY_HOST": "127.0.0.1", + "DOCKER_API_PROXY_PORT": "9999", + "DOCKER_API_PROXY_USER": "wrong", + "DOCKER_API_PROXY_PASSWORD": "wrong", + } + async with setup_docker_client(envs) as failing_docker_client: + async for attempt in AsyncRetrying( + wait=wait_fixed(0.1), stop=stop_after_delay(10), reraise=True + ): + with attempt: # noqa: SIM117 + with pytest.raises(aiodocker.exceptions.DockerError, match="401"): + await failing_docker_client.system.info() diff --git a/services/docker-compose-build.yml b/services/docker-compose-build.yml index becf5ce2a25..7fe0baf7024 100644 --- a/services/docker-compose-build.yml +++ b/services/docker-compose-build.yml @@ -316,6 +316,22 @@ services: org.opencontainers.image.revision: "${VCS_REF}" io.osparc.api-version: "${DYNAMIC_SCHEDULER_API_VERSION}" + docker-api-proxy: + image: local/docker-api-proxy:${BUILD_TARGET:?build_target_required} + build: + context: ../ + dockerfile: services/docker-api-proxy/Dockerfile + cache_from: + - local/docker-api-proxy:${BUILD_TARGET:?build_target_required} + - ${DOCKER_REGISTRY:-itisfoundation}/docker-api-proxy:master-github-latest + - ${DOCKER_REGISTRY:-itisfoundation}/docker-api-proxy:staging-github-latest + - ${DOCKER_REGISTRY:-itisfoundation}/docker-api-proxy:release-github-latest + target: ${BUILD_TARGET:?build_target_required} + labels: + org.opencontainers.image.created: "${BUILD_DATE}" + org.opencontainers.image.source: "${VCS_URL}" + org.opencontainers.image.revision: "${VCS_REF}" + datcore-adapter: image: local/datcore-adapter:${BUILD_TARGET:?build_target_required} build: diff --git a/services/docker-compose-deploy.yml b/services/docker-compose-deploy.yml index 1da5f7933de..f3997a0b11d 100644 --- a/services/docker-compose-deploy.yml +++ b/services/docker-compose-deploy.yml @@ -29,6 +29,8 @@ services: image: ${DOCKER_REGISTRY:-itisfoundation}/payments:${DOCKER_IMAGE_TAG:-latest} dynamic-scheduler: image: ${DOCKER_REGISTRY:-itisfoundation}/dynamic-scheduler:${DOCKER_IMAGE_TAG:-latest} + docker-api-proxy: + image: ${DOCKER_REGISTRY:-itisfoundation}/docker-api-proxy:${DOCKER_IMAGE_TAG:-latest} resource-usage-tracker: image: ${DOCKER_REGISTRY:-itisfoundation}/resource-usage-tracker:${DOCKER_IMAGE_TAG:-latest} service-integration: diff --git a/services/docker-compose.local.yml b/services/docker-compose.local.yml index 945dedc3a28..7222fa212c2 100644 --- a/services/docker-compose.local.yml +++ b/services/docker-compose.local.yml @@ -107,6 +107,10 @@ services: deploy: replicas: 2 + docker-api-proxy: + ports: + - "8014:8888" + resource-usage-tracker: environment: RESOURCE_USAGE_TRACKER_REMOTE_DEBUGGING_PORT : 3000 diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 555f47c4c9d..990f7092bea 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -549,6 +549,7 @@ services: hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" networks: - default + - docker-api-network environment: LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} @@ -573,6 +574,24 @@ services: DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: ${DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT} TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} + DOCKER_API_PROXY_HOST: ${DOCKER_API_PROXY_HOST} + DOCKER_API_PROXY_PASSWORD: ${DOCKER_API_PROXY_PASSWORD} + DOCKER_API_PROXY_PORT: ${DOCKER_API_PROXY_PORT} + DOCKER_API_PROXY_SECURE: ${DOCKER_API_PROXY_SECURE} + DOCKER_API_PROXY_USER: ${DOCKER_API_PROXY_USER} + docker-api-proxy: + image: ${DOCKER_REGISTRY:-itisfoundation}/docker-api-proxy:${DOCKER_IMAGE_TAG:-latest} + init: true + deploy: + placement: + constraints: + - node.role == manager + mode: global + volumes: + - /var/run/docker.sock:/var/run/docker.sock + networks: + - docker-api-network + static-webserver: image: ${DOCKER_REGISTRY:-itisfoundation}/static-webserver:${DOCKER_IMAGE_TAG:-latest} init: true @@ -1352,6 +1371,15 @@ networks: internal: false labels: com.simcore.description: "computational services network" + docker-api-network: + name: ${SWARM_STACK_NAME}_docker-api-network + driver: overlay + attachable: true + internal: false + driver_opts: + encrypted: "true" + labels: + com.simcore.description: "used for internal access to the docker swarm api" secrets: dask_tls_key: diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_health.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_health.py index ff5fe204132..e72f897ca74 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_health.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_health.py @@ -1,17 +1,20 @@ from typing import Annotated import arrow -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, FastAPI from fastapi.responses import PlainTextResponse from models_library.errors import ( + DOCKER_API_PROXY_UNHEALTHY_MSG, RABBITMQ_CLIENT_UNHEALTHY_MSG, REDIS_CLIENT_UNHEALTHY_MSG, ) +from servicelib.fastapi.docker import is_docker_api_proxy_ready from servicelib.rabbitmq import RabbitMQClient, RabbitMQRPCClient from servicelib.redis import RedisClientSDK from settings_library.redis import RedisDatabase from ._dependencies import ( + get_app, get_rabbitmq_client_from_request, get_rabbitmq_rpc_server_from_request, get_redis_clients_from_request, @@ -26,6 +29,7 @@ class HealthCheckError(RuntimeError): @router.get("/health", response_class=PlainTextResponse) async def healthcheck( + app: Annotated[FastAPI, Depends(get_app)], rabbit_client: Annotated[RabbitMQClient, Depends(get_rabbitmq_client_from_request)], rabbit_rpc_server: Annotated[ RabbitMQRPCClient, Depends(get_rabbitmq_rpc_server_from_request) @@ -35,6 +39,9 @@ async def healthcheck( Depends(get_redis_clients_from_request), ], ): + if not await is_docker_api_proxy_ready(app, timeout=1): + raise HealthCheckError(DOCKER_API_PROXY_UNHEALTHY_MSG) + if not rabbit_client.healthy or not rabbit_rpc_server.healthy: raise HealthCheckError(RABBITMQ_CLIENT_UNHEALTHY_MSG) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py index 0b7d56fccda..ed05a7bb265 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py @@ -2,6 +2,7 @@ import os import typer +from settings_library.docker_api_proxy import DockerApiProxysettings from settings_library.rabbit import RabbitSettings from settings_library.utils_cli import ( create_settings_command, @@ -56,6 +57,14 @@ def echo_dotenv(ctx: typer.Context, *, minimal: bool = True): "DYNAMIC_SCHEDULER_UI_STORAGE_SECRET", "replace-with-ui-storage-secret", ), + DYNAMIC_SCHEDULER_DOCKER_API_PROXY=os.environ.get( + "DYNAMIC_SCHEDULER_DOCKER_API_PROXY", + DockerApiProxysettings.create_from_envs( + DOCKER_API_PROXY_HOST=os.environ.get( + "DOCKER_API_PROXY_HOST", "replace-with-proxy-host" + ) + ), + ), ) print_as_envfile( diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py index a7f54a1411c..e2f6f5e8a77 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/application.py @@ -2,6 +2,7 @@ from fastapi import FastAPI from fastapi_lifespan_manager import State +from servicelib.fastapi.docker import get_lifespan_remote_docker_client from servicelib.fastapi.lifespan_utils import LifespanGenerator, combine_lifespans from servicelib.fastapi.openapi import override_fastapi_openapi_method from servicelib.fastapi.profiler import initialize_profiler @@ -26,7 +27,7 @@ from ..services.deferred_manager import lifespan_deferred_manager from ..services.director_v0 import lifespan_director_v0 from ..services.director_v2 import lifespan_director_v2 -from ..services.notifier import get_notifier_lifespans +from ..services.notifier import get_lifespans_notifier from ..services.rabbitmq import lifespan_rabbitmq from ..services.redis import lifespan_redis from ..services.service_tracker import lifespan_service_tracker @@ -50,10 +51,13 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: lifespan_rabbitmq, lifespan_rpc_api_routes, lifespan_redis, - *get_notifier_lifespans(), + *get_lifespans_notifier(), lifespan_service_tracker, lifespan_deferred_manager, lifespan_status_monitor, + get_lifespan_remote_docker_client( + app_settings.DYNAMIC_SCHEDULER_DOCKER_API_PROXY + ), ] if app_settings.DYNAMIC_SCHEDULER_PROMETHEUS_INSTRUMENTATION_ENABLED: diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index 9531641897f..5e38cec62b9 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -7,6 +7,7 @@ from settings_library.basic_types import LogLevel, VersionTag from settings_library.director_v0 import DirectorV0Settings from settings_library.director_v2 import DirectorV2Settings +from settings_library.docker_api_proxy import DockerApiProxysettings from settings_library.http_client_request import ClientRequestSettings from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings @@ -152,6 +153,11 @@ class ApplicationSettings(_BaseApplicationSettings): description="settings for opentelemetry tracing", ) + DYNAMIC_SCHEDULER_DOCKER_API_PROXY: Annotated[ + DockerApiProxysettings, + Field(json_schema_extra={"auto_default_from_env": True}), + ] + @field_validator("DYNAMIC_SCHEDULER_UI_MOUNT_PATH", mode="before") @classmethod def _ensure_ends_with_slash(cls, v: str) -> str: diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/notifier/__init__.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/notifier/__init__.py index e5e1609440b..7daeeb7e2fc 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/notifier/__init__.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/notifier/__init__.py @@ -1,7 +1,7 @@ from ._notifier import notify_service_status_change -from ._setup import get_notifier_lifespans +from ._setup import get_lifespans_notifier __all__: tuple[str, ...] = ( - "get_notifier_lifespans", + "get_lifespans_notifier", "notify_service_status_change", ) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/notifier/_setup.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/notifier/_setup.py index 659eff665c6..5bd140959b2 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/notifier/_setup.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/notifier/_setup.py @@ -3,5 +3,5 @@ from . import _notifier, _socketio -def get_notifier_lifespans() -> list[LifespanGenerator]: +def get_lifespans_notifier() -> list[LifespanGenerator]: return [_socketio.lifespan, _notifier.lifespan] diff --git a/services/dynamic-scheduler/tests/conftest.py b/services/dynamic-scheduler/tests/conftest.py index 3e39bd327c9..f669b22bda7 100644 --- a/services/dynamic-scheduler/tests/conftest.py +++ b/services/dynamic-scheduler/tests/conftest.py @@ -107,7 +107,7 @@ def disable_deferred_manager_lifespan(mocker: MockerFixture) -> None: @pytest.fixture def disable_notifier_lifespan(mocker: MockerFixture) -> None: - mocker.patch(f"{_PATH_APPLICATION}.get_notifier_lifespans") + mocker.patch(f"{_PATH_APPLICATION}.get_lifespans_notifier") @pytest.fixture diff --git a/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__health.py b/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__health.py index cb7939c5824..42bc7396c9c 100644 --- a/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__health.py +++ b/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__health.py @@ -48,8 +48,17 @@ def mock_redis_client( ) +@pytest.fixture +def mock_docker_api_proxy(mocker: MockerFixture, docker_api_proxy_ok: bool) -> None: + base_path = "simcore_service_dynamic_scheduler.api.rest._health" + mocker.patch( + f"{base_path}.is_docker_api_proxy_ready", return_value=docker_api_proxy_ok + ) + + @pytest.fixture def app_environment( + mock_docker_api_proxy: None, mock_rabbitmq_clients: None, mock_redis_client: None, app_environment: EnvVarsDict, @@ -58,12 +67,13 @@ def app_environment( @pytest.mark.parametrize( - "rabbit_client_ok, rabbit_rpc_server_ok, redis_client_ok, is_ok", + "rabbit_client_ok, rabbit_rpc_server_ok, redis_client_ok,, docker_api_proxy_ok, is_ok", [ - pytest.param(True, True, True, True, id="ok"), - pytest.param(False, True, True, False, id="rabbit_client_bad"), - pytest.param(True, False, True, False, id="rabbit_rpc_server_bad"), - pytest.param(True, True, False, False, id="redis_client_bad"), + pytest.param(True, True, True, True, True, id="ok"), + pytest.param(False, True, True, True, False, id="rabbit_client_bad"), + pytest.param(True, False, True, True, False, id="rabbit_rpc_server_bad"), + pytest.param(True, True, False, True, False, id="redis_client_bad"), + pytest.param(True, True, True, False, False, id="docker_api_proxy_bad"), ], ) async def test_health(client: AsyncClient, is_ok: bool):