Use devpi as PyPI cache

readthedocs/doc_builder/python_environments.py

@@ -10,6 +10,8 @@
 import shutil
 import yaml
 
+import docker
+
 from django.conf import settings
 
 from readthedocs.builds.constants import EXTERNAL
@@ -106,6 +108,7 @@ def install_package(self, install):
                 '--upgrade-strategy',
                 'eager',
                 *self._pip_cache_cmd_argument(),
+                *self._pip_index_cmd_argument(),
                 '{path}{extra_requirements}'.format(
                     path=local_path,
                     extra_requirements=extra_req_param,
@@ -123,6 +126,38 @@ def install_package(self, install):
                 bin_path=self.venv_bin(),
             )
 
+    def _pip_index_cmd_argument(self):
+        """Return ``--index-url`` to local devpi if using Docker Compose."""
+
+        def _get_devpi_container_ip():
+            client = self.build_env.get_client()
+            info = client.inspect_container(
+                getattr(settings, 'RTD_DOCKER_COMPOSE_DEVPI_CONTAINER', None),
+            )
+            networks = info.get('NetworkSettings').get('Networks')
+            return networks[list(networks.keys())[0]].get('Gateway')
+
+        if all([
+                getattr(settings, 'RTD_DOCKER_COMPOSE', None),
+                getattr(settings, 'RTD_DOCKER_COMPOSE_DEVPI_CONTAINER', None),
+        ]):
+            try:
+                ip = _get_devpi_container_ip()
+                if ip:
+                    return [
+                        '--index-url',
+                        f'http://{ip}:3141/root/pypi/+simple/',
+                        '--trusted-host',
+                        f'{ip}',
+                    ]
+            except docker.errors.NotFound:
+                log.info(
+                    'devpi container not found. container=%s',
+                    getattr(settings, 'RTD_DOCKER_COMPOSE_DEVPI_CONTAINER', None),
+                )
+
+        return []
+
     def _pip_cache_cmd_argument(self):
         """
         Return the pip command ``--cache-dir`` or ``--no-cache-dir`` argument.
@@ -323,6 +358,7 @@ def install_core_requirements(self):
             'install',
             '--upgrade',
             *self._pip_cache_cmd_argument(),
+            *self._pip_index_cmd_argument(),
         ]
 
         # Install latest pip first,
@@ -419,6 +455,7 @@ def install_requirements_file(self, install):
             args += [
                 '--exists-action=w',
                 *self._pip_cache_cmd_argument(),
+                *self._pip_index_cmd_argument(),
                 '-r',
                 requirements_file_path,
             ]
@@ -615,6 +652,7 @@ def install_core_requirements(self):
             'install',
             '-U',
             *self._pip_cache_cmd_argument(),
+            *self._pip_index_cmd_argument(),
         ]
         pip_cmd.extend(pip_requirements)
         self.build_env.run(

readthedocs/settings/docker_compose.py

@@ -11,6 +11,7 @@ class DockerBaseSettings(CommunityDevSettings):
     DOCKER_ENABLE = True
     RTD_DOCKER_COMPOSE = True
     RTD_DOCKER_COMPOSE_VOLUME = 'community_build-user-builds'
+    RTD_DOCKER_COMPOSE_DEVPI_CONTAINER = 'community_devpi_1'
     RTD_DOCKER_USER = f'{os.geteuid()}:{os.getegid()}'
     DOCKER_LIMITS = {'memory': '1g', 'time': 900}
     USE_SUBDOMAIN = True