From ee2c3c24acf408d69634cc92a21194e6fb3526a3 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 30 Oct 2019 18:21:58 +0100 Subject: [PATCH 1/7] Use devpi as PyPI cache Adds a new service for docker-compose that spin up a devpi-server to cache all the Python packages used by the builders. If configured, `--index-url` and `--trusted-host` is added to all the `pip` commands. Fixes https://github.com/readthedocs/readthedocs.org/issues/3553 --- docker-compose-devpi.yml | 16 +++++++++++ docker/entrypoints/devpi.sh | 6 ++++ .../doc_builder/python_environments.py | 28 +++++++++++++++++++ readthedocs/settings/docker.py | 1 + 4 files changed, 51 insertions(+) create mode 100644 docker-compose-devpi.yml create mode 100755 docker/entrypoints/devpi.sh diff --git a/docker-compose-devpi.yml b/docker-compose-devpi.yml new file mode 100644 index 00000000000..a3d4e8faa7a --- /dev/null +++ b/docker-compose-devpi.yml @@ -0,0 +1,16 @@ +# docker-compose-pypi.yml starts "devpi" service for local cache +version: '3' + +volumes: + devpi: + +services: + + devpi: + image: readthedocsorg_server:latest + ports: + - "3141:3141" + volumes: + - ${PWD}/docker/entrypoints/devpi.sh:/usr/src/app/docker/devpi.sh + - devpi:/devpi + command: ["../../docker/devpi.sh"] diff --git a/docker/entrypoints/devpi.sh b/docker/entrypoints/devpi.sh new file mode 100755 index 00000000000..0d31c813b40 --- /dev/null +++ b/docker/entrypoints/devpi.sh @@ -0,0 +1,6 @@ +#! /bin/sh + + +pip3 install devpi-server +devpi-init --serverdir=/devpi +devpi-server --host=0.0.0.0 --absolute-urls --serverdir=/devpi diff --git a/readthedocs/doc_builder/python_environments.py b/readthedocs/doc_builder/python_environments.py index 51749da1adb..b2c4120d909 100644 --- a/readthedocs/doc_builder/python_environments.py +++ b/readthedocs/doc_builder/python_environments.py @@ -105,6 +105,7 @@ def install_package(self, install): '--upgrade-strategy', 'eager', *self._pip_cache_cmd_argument(), + *self._pip_index_cmd_argument(), '{path}{extra_requirements}'.format( path=local_path, extra_requirements=extra_req_param, @@ -122,6 +123,30 @@ def install_package(self, install): bin_path=self.venv_bin(), ) + def _pip_index_cmd_argument(self): + """Return ``--index-url`` to local devpi if using Docker Compose.""" + + def _get_devpi_container_ip(): + client = self.build_env.get_client() + info = client.inspect_container( + getattr(settings, 'RTD_DOCKER_COMPOSE_DEVPI_CONTAINER', None), + ) + networks = info.get('NetworkSettings').get('Networks') + return networks[list(networks.keys())[0]].get('Gateway') + + if all([ + getattr(settings, 'RTD_DOCKER_COMPOSE', None), + getattr(settings, 'RTD_DOCKER_COMPOSE_DEVPI_CONTAINER', None), + ]): + ip = _get_devpi_container_ip() + return [ + '--index-url', + f'http://{ip}:3141/root/pypi/+simple/', + '--trusted-host', + f'{ip}', + ] + return [] + def _pip_cache_cmd_argument(self): """ Return the pip command ``--cache-dir`` or ``--no-cache-dir`` argument. @@ -310,6 +335,7 @@ def install_core_requirements(self): 'install', '--upgrade', *self._pip_cache_cmd_argument(), + *self._pip_index_cmd_argument(), ] # Install latest pip first, @@ -406,6 +432,7 @@ def install_requirements_file(self, install): args += [ '--exists-action=w', *self._pip_cache_cmd_argument(), + *self._pip_index_cmd_argument(), '-r', requirements_file_path, ] @@ -602,6 +629,7 @@ def install_core_requirements(self): 'install', '-U', *self._pip_cache_cmd_argument(), + *self._pip_index_cmd_argument(), ] pip_cmd.extend(pip_requirements) self.build_env.run( diff --git a/readthedocs/settings/docker.py b/readthedocs/settings/docker.py index 77341690a54..714ea18ce78 100644 --- a/readthedocs/settings/docker.py +++ b/readthedocs/settings/docker.py @@ -9,6 +9,7 @@ class DockerBaseSettings(CommunityDevSettings): DOCKER_ENABLE = True RTD_DOCKER_COMPOSE = True RTD_DOCKER_COMPOSE_VOLUME = 'readthedocsorg_build-user-builds' + RTD_DOCKER_COMPOSE_DEVPI_CONTAINER = 'readthedocsorg_devpi_1' RTD_DOCKER_USER = f'{os.geteuid()}:{os.getegid()}' DOCKER_LIMITS = {'memory': '1g', 'time': 900} USE_SUBDOMAIN = True From ce0f69ebbbbd3a67ac9962f8a404687575e60844 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 25 Nov 2019 20:29:27 +0100 Subject: [PATCH 2/7] Create smaller image for devpi --- docker-compose-devpi.yml | 6 ++--- docker/Dockerfile.devpi | 9 ++++++++ .../doc_builder/python_environments.py | 22 +++++++++++++------ 3 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 docker/Dockerfile.devpi diff --git a/docker-compose-devpi.yml b/docker-compose-devpi.yml index a3d4e8faa7a..cb56601de4c 100644 --- a/docker-compose-devpi.yml +++ b/docker-compose-devpi.yml @@ -7,10 +7,10 @@ volumes: services: devpi: - image: readthedocsorg_server:latest + build: + context: . + dockerfile: ${PWD}/docker/Dockerfile.devpi ports: - "3141:3141" volumes: - - ${PWD}/docker/entrypoints/devpi.sh:/usr/src/app/docker/devpi.sh - devpi:/devpi - command: ["../../docker/devpi.sh"] diff --git a/docker/Dockerfile.devpi b/docker/Dockerfile.devpi new file mode 100644 index 00000000000..e6025c37e67 --- /dev/null +++ b/docker/Dockerfile.devpi @@ -0,0 +1,9 @@ +FROM python:3.6-alpine + +# Required to compile cffi +RUN apk add --no-cache openssl-dev libffi-dev build-base + +RUN pip3 install --no-cache-dir --upgrade devpi-server +RUN devpi-init --serverdir=/devpi + +CMD ["devpi-server", "--host=0.0.0.0", "--absolute-urls", "--serverdir=/devpi", "--threads=3"] diff --git a/readthedocs/doc_builder/python_environments.py b/readthedocs/doc_builder/python_environments.py index b2c4120d909..9871e3c55e6 100644 --- a/readthedocs/doc_builder/python_environments.py +++ b/readthedocs/doc_builder/python_environments.py @@ -10,6 +10,8 @@ import shutil import yaml +import docker + from django.conf import settings from readthedocs.config import PIP, SETUPTOOLS, ParseError, parse as parse_yaml @@ -138,13 +140,19 @@ def _get_devpi_container_ip(): getattr(settings, 'RTD_DOCKER_COMPOSE', None), getattr(settings, 'RTD_DOCKER_COMPOSE_DEVPI_CONTAINER', None), ]): - ip = _get_devpi_container_ip() - return [ - '--index-url', - f'http://{ip}:3141/root/pypi/+simple/', - '--trusted-host', - f'{ip}', - ] + try: + ip = _get_devpi_container_ip() + if ip: + return [ + '--index-url', + f'http://{ip}:3141/root/pypi/+simple/', + '--trusted-host', + f'{ip}', + ] + except docker.errors.NotFound: + # devpi container not found + pass + return [] def _pip_cache_cmd_argument(self): From dff5cd811218a19a4386c9abc4851c57b68f1253 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Sun, 1 Dec 2019 15:10:33 +0100 Subject: [PATCH 3/7] Remove unneded file --- docker/entrypoints/devpi.sh | 6 ------ 1 file changed, 6 deletions(-) delete mode 100755 docker/entrypoints/devpi.sh diff --git a/docker/entrypoints/devpi.sh b/docker/entrypoints/devpi.sh deleted file mode 100755 index 0d31c813b40..00000000000 --- a/docker/entrypoints/devpi.sh +++ /dev/null @@ -1,6 +0,0 @@ -#! /bin/sh - - -pip3 install devpi-server -devpi-init --serverdir=/devpi -devpi-server --host=0.0.0.0 --absolute-urls --serverdir=/devpi From 2a6ce3a5d242c9d2604ac104b2fbcd3863a636e5 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Sun, 1 Dec 2019 15:12:25 +0100 Subject: [PATCH 4/7] Log dev container not found --- readthedocs/doc_builder/python_environments.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/readthedocs/doc_builder/python_environments.py b/readthedocs/doc_builder/python_environments.py index 9871e3c55e6..6e9d47c7ef5 100644 --- a/readthedocs/doc_builder/python_environments.py +++ b/readthedocs/doc_builder/python_environments.py @@ -150,8 +150,10 @@ def _get_devpi_container_ip(): f'{ip}', ] except docker.errors.NotFound: - # devpi container not found - pass + log.info( + 'devpi container not found. container=%s', + getattr(settings, 'RTD_DOCKER_COMPOSE_DEVPI_CONTAINER', None), + ) return [] From c0b592e4c0d1650274eca110fa9bb6ac5ce070fb Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 13 Jan 2020 13:29:53 +0100 Subject: [PATCH 5/7] Move file to common --- docker-compose-devpi.yml | 6 ++++-- docker/Dockerfile.devpi | 9 --------- 2 files changed, 4 insertions(+), 11 deletions(-) delete mode 100644 docker/Dockerfile.devpi diff --git a/docker-compose-devpi.yml b/docker-compose-devpi.yml index cb56601de4c..cf3f9a053d8 100644 --- a/docker-compose-devpi.yml +++ b/docker-compose-devpi.yml @@ -1,4 +1,4 @@ -# docker-compose-pypi.yml starts "devpi" service for local cache +# docker-compose-devpi.yml starts "devpi" service for local cache version: '3' volumes: @@ -9,8 +9,10 @@ services: devpi: build: context: . - dockerfile: ${PWD}/docker/Dockerfile.devpi + dockerfile: ${PWD}/common/dockerfiles/Dockerfile.devpi ports: - "3141:3141" volumes: - devpi:/devpi + networks: + community: diff --git a/docker/Dockerfile.devpi b/docker/Dockerfile.devpi deleted file mode 100644 index e6025c37e67..00000000000 --- a/docker/Dockerfile.devpi +++ /dev/null @@ -1,9 +0,0 @@ -FROM python:3.6-alpine - -# Required to compile cffi -RUN apk add --no-cache openssl-dev libffi-dev build-base - -RUN pip3 install --no-cache-dir --upgrade devpi-server -RUN devpi-init --serverdir=/devpi - -CMD ["devpi-server", "--host=0.0.0.0", "--absolute-urls", "--serverdir=/devpi", "--threads=3"] From cb384d86a5d391b5f00c740fe1f0627a7fc7751a Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 13 Jan 2020 13:35:54 +0100 Subject: [PATCH 6/7] Update common to branch with devpi --- common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common b/common index 6c7a67a9c65..608a61bc4ce 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit 6c7a67a9c6567eabb9b95de43809eb6fa3e10fa5 +Subproject commit 608a61bc4ce056f9e7164dc415a981fc71d43075 From c2f18eb041d84abc2104c4ddaa38511db683a58c Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 21 Apr 2020 15:49:32 +0200 Subject: [PATCH 7/7] Update common to the latest required changes --- common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common b/common index ed6bcff6415..74ce9d6df11 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit ed6bcff64159f71548c4baefcde55ec7331abc2c +Subproject commit 74ce9d6df11731da5f003e0b9c67d1e95178b2d1