Use devpi as PyPI cache

Adds a new service for docker-compose that spin up a devpi-server to
cache all the Python packages used by the builders.

If configured, `--index-url` and `--trusted-host` is added to all
the `pip` commands.

Fixes #3553

Author: humitos

docker-compose-devpi.yml

@@ -0,0 +1,16 @@
+# docker-compose-pypi.yml starts "devpi" service for local cache
+version: '3'
+
+volumes:
+  devpi:
+
+services:
+
+  devpi:
+    image: readthedocsorg_server:latest
+    ports:
+      - "3141:3141"
+    volumes:
+      - ${PWD}/docker/entrypoints/devpi.sh:/usr/src/app/docker/devpi.sh
+      - devpi:/devpi
+    command: ["../../docker/devpi.sh"]

docker/entrypoints/devpi.sh

@@ -0,0 +1,6 @@
+#! /bin/sh
+
+
+pip3 install devpi-server
+devpi-init --serverdir=/devpi
+devpi-server --host=0.0.0.0 --absolute-urls --serverdir=/devpi

readthedocs/doc_builder/python_environments.py

@@ -105,6 +105,7 @@ def install_package(self, install):
                 '--upgrade-strategy',
                 'eager',
                 *self._pip_cache_cmd_argument(),
+                *self._pip_index_cmd_argument(),
                 '{path}{extra_requirements}'.format(
                     path=local_path,
                     extra_requirements=extra_req_param,
@@ -122,6 +123,27 @@ def install_package(self, install):
                 bin_path=self.venv_bin(),
             )
 
+    def _pip_index_cmd_argument(self):
+        """Return ``--index-url`` to local devpi if using Docker Compose."""
+
+        def _get_devpi_container_ip():
+            client = self.build_env.get_client()
+            info = client.inspect_container(
+                settings.RTD_DOCKER_COMPOSE_DEVPI_CONTAINER,
+            )
+            networks = info.get('NetworkSettings').get('Networks')
+            return networks[list(networks.keys())[0]].get('Gateway')
+
+        if settings.RTD_DOCKER_COMPOSE:
+            ip = _get_devpi_container_ip()
+            return [
+                '--index-url',
+                f'http://{ip}:3141/root/pypi/+simple/',
+                '--trusted-host',
+                f'{ip}',
+            ]
+        return []
+
     def _pip_cache_cmd_argument(self):
         """
         Return the pip command ``--cache-dir`` or ``--no-cache-dir`` argument.
@@ -310,6 +332,7 @@ def install_core_requirements(self):
             'install',
             '--upgrade',
             *self._pip_cache_cmd_argument(),
+            *self._pip_index_cmd_argument(),
         ]
 
         # Install latest pip first,
@@ -406,6 +429,7 @@ def install_requirements_file(self, install):
             args += [
                 '--exists-action=w',
                 *self._pip_cache_cmd_argument(),
+                *self._pip_index_cmd_argument(),
                 '-r',
                 requirements_file_path,
             ]
@@ -602,6 +626,7 @@ def install_core_requirements(self):
             'install',
             '-U',
             *self._pip_cache_cmd_argument(),
+            *self._pip_index_cmd_argument(),
         ]
         pip_cmd.extend(pip_requirements)
         self.build_env.run(

readthedocs/settings/docker.py

@@ -9,6 +9,7 @@ class DockerBaseSettings(CommunityDevSettings):
     DOCKER_ENABLE = True
     RTD_DOCKER_COMPOSE = True
     RTD_DOCKER_COMPOSE_VOLUME = 'readthedocsorg_build-user-builds'
+    RTD_DOCKER_COMPOSE_DEVPI_CONTAINER = 'readthedocsorg_devpi_1'
     RTD_DOCKER_USER = f'{os.geteuid()}:{os.getegid()}'
     DOCKER_LIMITS = {'memory': '1g', 'time': 900}
     USE_SUBDOMAIN = True