From 46b58a9d0bfad9b70a5d33792e3330661999c4f4 Mon Sep 17 00:00:00 2001
From: Manuel Kaufmann <humitos@gmail.com>
Date: Wed, 30 Oct 2019 18:21:58 +0100
Subject: [PATCH] Use devpi as PyPI cache

Adds a new service for docker-compose that spin up a devpi-server to
cache all the Python packages used by the builders.

If configured, `--index-url` and `--trusted-host` is added to all
the `pip` commands.

Fixes https://github.com/readthedocs/readthedocs.org/issues/3553
---
 docker-compose-devpi.yml                      | 16 ++++++++++++
 docker/entrypoints/devpi.sh                   |  6 +++++
 .../doc_builder/python_environments.py        | 25 +++++++++++++++++++
 readthedocs/settings/docker.py                |  1 +
 4 files changed, 48 insertions(+)
 create mode 100644 docker-compose-devpi.yml
 create mode 100755 docker/entrypoints/devpi.sh

diff --git a/docker-compose-devpi.yml b/docker-compose-devpi.yml
new file mode 100644
index 00000000000..a3d4e8faa7a
--- /dev/null
+++ b/docker-compose-devpi.yml
@@ -0,0 +1,16 @@
+# docker-compose-pypi.yml starts "devpi" service for local cache
+version: '3'
+
+volumes:
+  devpi:
+
+services:
+
+  devpi:
+    image: readthedocsorg_server:latest
+    ports:
+      - "3141:3141"
+    volumes:
+      - ${PWD}/docker/entrypoints/devpi.sh:/usr/src/app/docker/devpi.sh
+      - devpi:/devpi
+    command: ["../../docker/devpi.sh"]
diff --git a/docker/entrypoints/devpi.sh b/docker/entrypoints/devpi.sh
new file mode 100755
index 00000000000..0d31c813b40
--- /dev/null
+++ b/docker/entrypoints/devpi.sh
@@ -0,0 +1,6 @@
+#! /bin/sh
+
+
+pip3 install devpi-server
+devpi-init --serverdir=/devpi
+devpi-server --host=0.0.0.0 --absolute-urls --serverdir=/devpi
diff --git a/readthedocs/doc_builder/python_environments.py b/readthedocs/doc_builder/python_environments.py
index 51749da1adb..63b2ccd22f1 100644
--- a/readthedocs/doc_builder/python_environments.py
+++ b/readthedocs/doc_builder/python_environments.py
@@ -105,6 +105,7 @@ def install_package(self, install):
                 '--upgrade-strategy',
                 'eager',
                 *self._pip_cache_cmd_argument(),
+                *self._pip_index_cmd_argument(),
                 '{path}{extra_requirements}'.format(
                     path=local_path,
                     extra_requirements=extra_req_param,
@@ -122,6 +123,27 @@ def install_package(self, install):
                 bin_path=self.venv_bin(),
             )
 
+    def _pip_index_cmd_argument(self):
+        """Return ``--index-url`` to local devpi if using Docker Compose."""
+
+        def _get_devpi_container_ip():
+            client = self.build_env.get_client()
+            info = client.inspect_container(
+                settings.RTD_DOCKER_COMPOSE_DEVPI_CONTAINER,
+            )
+            networks = info.get('NetworkSettings').get('Networks')
+            return networks[list(networks.keys())[0]].get('Gateway')
+
+        if settings.RTD_DOCKER_COMPOSE:
+            ip = _get_devpi_container_ip()
+            return [
+                '--index-url',
+                f'http://{ip}:3141/root/pypi/+simple/',
+                '--trusted-host',
+                f'{ip}',
+            ]
+        return []
+
     def _pip_cache_cmd_argument(self):
         """
         Return the pip command ``--cache-dir`` or ``--no-cache-dir`` argument.
@@ -310,6 +332,7 @@ def install_core_requirements(self):
             'install',
             '--upgrade',
             *self._pip_cache_cmd_argument(),
+            *self._pip_index_cmd_argument(),
         ]
 
         # Install latest pip first,
@@ -406,6 +429,7 @@ def install_requirements_file(self, install):
             args += [
                 '--exists-action=w',
                 *self._pip_cache_cmd_argument(),
+                *self._pip_index_cmd_argument(),
                 '-r',
                 requirements_file_path,
             ]
@@ -602,6 +626,7 @@ def install_core_requirements(self):
             'install',
             '-U',
             *self._pip_cache_cmd_argument(),
+            *self._pip_index_cmd_argument(),
         ]
         pip_cmd.extend(pip_requirements)
         self.build_env.run(
diff --git a/readthedocs/settings/docker.py b/readthedocs/settings/docker.py
index 77341690a54..714ea18ce78 100644
--- a/readthedocs/settings/docker.py
+++ b/readthedocs/settings/docker.py
@@ -9,6 +9,7 @@ class DockerBaseSettings(CommunityDevSettings):
     DOCKER_ENABLE = True
     RTD_DOCKER_COMPOSE = True
     RTD_DOCKER_COMPOSE_VOLUME = 'readthedocsorg_build-user-builds'
+    RTD_DOCKER_COMPOSE_DEVPI_CONTAINER = 'readthedocsorg_devpi_1'
     RTD_DOCKER_USER = f'{os.geteuid()}:{os.getegid()}'
     DOCKER_LIMITS = {'memory': '1g', 'time': 900}
     USE_SUBDOMAIN = True