Run mongorestore in a tools pod

docker/mongodb-kubernetes-tests/tests/common/mongodb_tools_pod/mongodb_tools_pod.py

@@ -0,0 +1,142 @@
+import tarfile
+import tempfile
+from typing import Optional
+
+from kubernetes import client
+from kubernetes.client.exceptions import ApiException
+from kubernetes.stream import stream
+from tests import test_logger
+
+logger = test_logger.get_test_logger(__name__)
+
+
+class ToolsPod:
+    namespace: str
+    pod_name: str
+    container_name: str
+    api_client: Optional[client.ApiClient] = None
+
+    def __init__(self, namespace: str, api_client: Optional[client.ApiClient] = None):
+        self.namespace = namespace
+        self.pod_name = "mongodb-tools-pod"
+        self.container_name = "mongodb-tools"
+        self.api_client = api_client
+
+    def run_command(self, cmd: list[str]):
+
+        logger.debug(f"Running command in pod {self.namespace}/{self.pod_name}: {" ".join(cmd)}")
+        api_client = client.CoreV1Api(api_client=self.api_client)
+        resp = stream(
+            api_client.connect_get_namespaced_pod_exec,
+            self.pod_name,
+            self.namespace,
+            container=self.container_name,
+            command=cmd,
+            stdout=True,
+            stderr=True,
+            _preload_content=False,
+        )
+        resp.run_forever(timeout=60)
+        exit_code = resp.returncode
+        output = resp.read_all()
+
+        if exit_code != 0:
+            raise RuntimeError(f"Command failed with exit code {exit_code}: {output}")
+
+        return output
+
+    def copy_file_to_pod(self, src_path: str, dest_path: str):
+        api_client = client.CoreV1Api(api_client=self.api_client)
+        try:
+            exec_command = ["tar", "xvf", "-", "-C", "/"]
+            resp = stream(
+                api_client.connect_get_namespaced_pod_exec,
+                self.pod_name,
+                self.namespace,
+                container=self.container_name,
+                command=exec_command,
+                stderr=True,
+                stdin=True,
+                stdout=True,
+                tty=False,
+                _preload_content=False,
+            )
+
+            stdout_output = []
+            stderr_output = []
+
+            with tempfile.TemporaryFile() as tar_buffer:
+                with tarfile.open(fileobj=tar_buffer, mode="w") as tar:
+                    tar.add(src_path, dest_path)
+
+                tar_buffer.seek(0)
+                commands = [tar_buffer.read()]
+
+                while resp.is_open():
+                    resp.update(timeout=1)
+                    if resp.peek_stdout():
+                        stdout = resp.read_stdout()
+                        stdout_output.append(stdout)
+                    if resp.peek_stderr():
+                        stderr = resp.read_stderr()
+                        stderr_output.append(stderr)
+                    if commands:
+                        c = commands.pop(0)
+                        resp.write_stdin(c.decode())
+                    else:
+                        break
+
+            resp.run_forever(timeout=5)
+            exit_code = resp.returncode
+            if exit_code is not None and exit_code != 0:
+                output = "".join(stdout_output + stderr_output)
+                raise RuntimeError(
+                    f"Failed to copy file to pod: tar command exited with code {exit_code}\nOutput: {output}"
+                )
+
+        except ApiException as e:
+            raise Exception(f"Failed to copy file to the pod: {e}")
+
+    def run_pod_and_wait(self):
+        from kubetester import get_pod_when_ready
+
+        pod_exists = False
+        try:
+            client.CoreV1Api(api_client=self.api_client).read_namespaced_pod(self.pod_name, self.namespace)
+            pod_exists = True
+            logger.debug(f"{self.pod_name} already exists in namespace {self.namespace}")
+        except Exception:
+            pass
+
+        if not pod_exists:
+            pod_body = {
+                "apiVersion": "v1",
+                "kind": "Pod",
+                "metadata": {
+                    "name": self.pod_name,
+                    "labels": {"app": self.pod_name},
+                },
+                "spec": {
+                    "containers": [
+                        {
+                            "name": self.container_name,
+                            "image": "mongodb/mongodb-community-server:8.0-ubi9",
+                            "command": ["/bin/bash", "-c"],
+                            "args": ["trap 'exit 0' SIGTERM; while true; do sleep 1; done"],
+                        }
+                    ],
+                    "restartPolicy": "Never",
+                },
+            }
+            client.CoreV1Api(api_client=self.api_client).create_namespaced_pod(self.namespace, pod_body)
+            logger.info(f"Created {self.pod_name} in namespace {self.namespace}")
+
+        logger.debug(f"Waiting for tools pod ({self.namespace}/{self.pod_name}) to become ready")
+        get_pod_when_ready(self.namespace, f"app={self.pod_name}", default_retry=60)
+
+
+def get_tools_pod(namespace: str) -> ToolsPod:
+    tools_pod = ToolsPod(namespace)
+    tools_pod.run_pod_and_wait()
+
+    return tools_pod

docker/mongodb-kubernetes-tests/tests/common/search/movies_search_helper.py

@@ -3,6 +3,7 @@
 import pymongo.errors
 from kubetester import kubetester
 from tests import test_logger
+from tests.common.mongodb_tools_pod import mongodb_tools_pod
 from tests.common.search.search_tester import SearchTester
 
 logger = test_logger.get_test_logger(__name__)
@@ -13,15 +14,19 @@ class SampleMoviesSearchHelper:
     db_name: str
     col_name: str
     archive_url: str
+    tools_pod: mongodb_tools_pod.ToolsPod
 
-    def __init__(self, search_tester: SearchTester):
+    def __init__(self, search_tester: SearchTester, tools_pod: mongodb_tools_pod.ToolsPod):
         self.search_tester = search_tester
         self.db_name = "sample_mflix"
         self.col_name = "movies"
+        self.tools_pod = tools_pod
 
     def restore_sample_database(self):
         self.search_tester.mongorestore_from_url(
-            "https://atlas-education.s3.amazonaws.com/sample_mflix.archive", f"{self.db_name}.*"
+            "https://atlas-education.s3.amazonaws.com/sample_mflix.archive",
+            f"{self.db_name}.*",
+            tools_pod=self.tools_pod,
         )
 
     def create_search_index(self):

docker/mongodb-kubernetes-tests/tests/common/search/search_tester.py

@@ -1,12 +1,12 @@
 import os
-import tempfile
+from dataclasses import dataclass
 
 import kubetester
-import requests
-from kubetester.helm import process_run_and_check
+from kubetester.kubetester import KubernetesTester
 from kubetester.mongotester import MongoTester
 from pymongo.operations import SearchIndexModel
 from tests import test_logger
+from tests.common.mongodb_tools_pod import mongodb_tools_pod
 
 logger = test_logger.get_test_logger(__name__)
 
@@ -20,19 +20,39 @@ def __init__(
     ):
         super().__init__(connection_string, use_ssl, ca_path)
 
-    def mongorestore_from_url(self, archive_url: str, ns_include: str, mongodb_tools_dir: str = ""):
-        logger.debug(f"running mongorestore from {archive_url}")
-        with tempfile.NamedTemporaryFile(delete=False) as sample_file:
-            resp = requests.get(archive_url)
-            size = sample_file.write(resp.content)
-            logger.debug(f"Downloaded sample file from {archive_url} to {sample_file.name} (size: {size})")
-            mongorestore_path = os.path.join(mongodb_tools_dir, "mongorestore")
-            mongorestore_cmd = f"{mongorestore_path} --archive={sample_file.name} --verbose=1 --drop --nsInclude {ns_include} --uri={self.cnx_string}"
-            if self.default_opts.get("tls", False):
-                mongorestore_cmd += " --ssl"
-            if ca_path := self.default_opts.get("tlsCAFile"):
-                mongorestore_cmd += " --sslCAFile=" + ca_path
-            process_run_and_check(mongorestore_cmd.split(), capture_output=True)
+    def mongorestore_from_url(self, archive_url: str, ns_include: str, tools_pod: mongodb_tools_pod.ToolsPod):
+        logger.debug(f"running mongorestore from {archive_url} in pod {tools_pod.pod_name}")
+
+        archive_file = f"/tmp/mongodb_archive_{os.urandom(4).hex()}.archive"
+
+        logger.debug(f"Downloading archive to {archive_file}")
+        download_cmd = ["curl", "-L", "-o", archive_file, archive_url]
+        tools_pod.run_command(cmd=download_cmd)
+
+        logger.debug("Running mongorestore")
+        mongorestore_cmd = [
+            "mongorestore",
+            f"--archive={archive_file}",
+            "--verbose=1",
+            "--drop",
+            f"--nsInclude={ns_include}",
+            f"--uri={self.cnx_string}",
+        ]
+
+        if self.default_opts.get("tls", False):
+            mongorestore_cmd.append("--ssl")
+        if ca_path := self.default_opts.get("tlsCAFile"):
+            tools_pod.copy_file_to_pod(ca_path, "/tmp/ca.crt")
+            mongorestore_cmd.append(f"--sslCAFile=/tmp/ca.crt")
+
+        result = tools_pod.run_command(cmd=mongorestore_cmd)
+        logger.debug(f"mongorestore completed: {result}")
+
+        logger.debug("Cleaning up archive file")
+        cleanup_cmd = ["rm", "-f", archive_file]
+        tools_pod.run_command(cmd=cleanup_cmd)
+
+        return result
 
     def create_search_index(self, database_name: str, collection_name: str):
         database = self.client[database_name]

docker/mongodb-kubernetes-tests/tests/search/search_community_basic.py

@@ -5,6 +5,7 @@
 from kubetester.phase import Phase
 from pytest import fixture, mark
 from tests import test_logger
+from tests.common.mongodb_tools_pod import mongodb_tools_pod
 from tests.common.search import movies_search_helper
 from tests.common.search.movies_search_helper import SampleMoviesSearchHelper
 from tests.common.search.search_tester import SearchTester
@@ -87,8 +88,11 @@ def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity):
 
 @fixture(scope="function")
 def sample_movies_helper(mdbc: MongoDBCommunity) -> SampleMoviesSearchHelper:
+    from tests.common.mongodb_tools_pod.mongodb_tools_pod import get_tools_pod
+
     return movies_search_helper.SampleMoviesSearchHelper(
-        SearchTester(get_connection_string(mdbc, USER_NAME, USER_PASSWORD))
+        SearchTester(get_connection_string(mdbc, USER_NAME, USER_PASSWORD)),
+        tools_pod=get_tools_pod(mdbc.namespace),
     )
 
 

docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py

@@ -5,6 +5,7 @@
 from kubetester.phase import Phase
 from pytest import fixture, mark
 from tests import test_logger
+from tests.common.mongodb_tools_pod import mongodb_tools_pod
 from tests.common.search import movies_search_helper
 from tests.common.search.movies_search_helper import SampleMoviesSearchHelper
 from tests.common.search.search_tester import SearchTester
@@ -119,8 +120,11 @@ def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity):
 
 @fixture(scope="function")
 def sample_movies_helper(mdbc: MongoDBCommunity) -> SampleMoviesSearchHelper:
+    from tests.common.mongodb_tools_pod.mongodb_tools_pod import get_tools_pod
+
     return movies_search_helper.SampleMoviesSearchHelper(
-        SearchTester(get_connection_string(mdbc, USER_NAME, USER_PASSWORD))
+        SearchTester(get_connection_string(mdbc, USER_NAME, USER_PASSWORD)),
+        tools_pod=get_tools_pod(mdbc.namespace),
     )
 
 

docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py

@@ -6,6 +6,8 @@
 from kubetester.phase import Phase
 from pytest import fixture, mark
 from tests import test_logger
+from tests.common.mongodb_tools_pod import mongodb_tools_pod
+from tests.common.mongodb_tools_pod.mongodb_tools_pod import get_tools_pod
 from tests.common.search import movies_search_helper
 from tests.common.search.movies_search_helper import SampleMoviesSearchHelper
 from tests.common.search.search_tester import SearchTester
@@ -163,7 +165,8 @@ def sample_movies_helper(mdbc: MongoDBCommunity, issuer_ca_filepath: str) -> Sam
             get_connection_string(mdbc, USER_NAME, USER_PASSWORD),
             use_ssl=True,
             ca_path=issuer_ca_filepath,
-        )
+        ),
+        tools_pod=get_tools_pod(mdbc.namespace),
     )
 
 

docker/mongodb-kubernetes-tests/tests/search/search_community_tls.py

@@ -6,6 +6,8 @@
 from kubetester.phase import Phase
 from pytest import fixture, mark
 from tests import test_logger
+from tests.common.mongodb_tools_pod import mongodb_tools_pod
+from tests.common.mongodb_tools_pod.mongodb_tools_pod import get_tools_pod
 from tests.common.search import movies_search_helper
 from tests.common.search.movies_search_helper import SampleMoviesSearchHelper
 from tests.common.search.search_tester import SearchTester
@@ -127,6 +129,7 @@ def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity):
 def sample_movies_helper(mdbc: MongoDBCommunity, issuer_ca_filepath: str) -> SampleMoviesSearchHelper:
     return movies_search_helper.SampleMoviesSearchHelper(
         SearchTester(get_connection_string(mdbc, USER_NAME, USER_PASSWORD), use_ssl=True, ca_path=issuer_ca_filepath),
+        tools_pod=get_tools_pod(mdbc.namespace),
     )
 
 

docker/mongodb-kubernetes-tests/tests/search/search_enterprise_basic.py

@@ -8,6 +8,7 @@
 from kubetester.phase import Phase
 from pytest import fixture, mark
 from tests import test_logger
+from tests.common.mongodb_tools_pod import mongodb_tools_pod
 from tests.common.search import movies_search_helper
 from tests.common.search.movies_search_helper import SampleMoviesSearchHelper
 from tests.common.search.search_tester import SearchTester
@@ -155,27 +156,28 @@ def test_wait_for_database_resource_ready(mdb: MongoDB):
         ), "mongot parameters not found in mongod config"
 
 
-@mark.e2e_search_enterprise_basic
-def test_search_restore_sample_database(mdb: MongoDB):
-    sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper(
-        SearchTester(get_connection_string(mdb, ADMIN_USER_NAME, ADMIN_USER_PASSWORD))
+@fixture(scope="function")
+def sample_movies_helper(mdb: MongoDB, issuer_ca_filepath: str) -> SampleMoviesSearchHelper:
+    from tests.common.mongodb_tools_pod.mongodb_tools_pod import get_tools_pod
+
+    return movies_search_helper.SampleMoviesSearchHelper(
+        SearchTester(get_connection_string(mdb, USER_NAME, USER_PASSWORD), use_ssl=False),
+        tools_pod=get_tools_pod(mdb.namespace),
     )
+
+
+@mark.e2e_search_enterprise_basic
+def test_search_restore_sample_database(sample_movies_helper: SampleMoviesSearchHelper):
     sample_movies_helper.restore_sample_database()
 
 
 @mark.e2e_search_enterprise_basic
-def test_search_create_search_index(mdb: MongoDB):
-    sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper(
-        SearchTester(get_connection_string(mdb, USER_NAME, USER_PASSWORD))
-    )
+def test_search_create_search_index(sample_movies_helper: SampleMoviesSearchHelper):
     sample_movies_helper.create_search_index()
 
 
 @mark.e2e_search_enterprise_basic
-def test_search_assert_search_query(mdb: MongoDB):
-    sample_movies_helper = movies_search_helper.SampleMoviesSearchHelper(
-        SearchTester(get_connection_string(mdb, USER_NAME, USER_PASSWORD))
-    )
+def test_search_assert_search_query(sample_movies_helper: SampleMoviesSearchHelper):
     sample_movies_helper.assert_search_query(retry_timeout=60)