-
Notifications
You must be signed in to change notification settings - Fork 0
feat: Automatically serialize test dependencies #5
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,7 +12,7 @@ | |
| from pytest_in_docker._types import ContainerPrepareError | ||
|
|
||
| if TYPE_CHECKING: | ||
| from collections.abc import Iterable | ||
| from collections.abc import Callable, Iterable | ||
|
|
||
| from testcontainers.core.container import DockerContainer | ||
|
|
||
|
|
@@ -96,8 +96,9 @@ def _check_python_version(container: DockerContainer, python: pathlib.Path) -> N | |
| local_ver = f"{sys.version_info.major}.{sys.version_info.minor}" | ||
| if remote_ver != local_ver: | ||
| msg = ( | ||
| f"Python version mismatch: host has {local_ver} but container has " | ||
| f"{remote_ver}. rpyc teleport requires matching major.minor versions." | ||
| f"Python version mismatch: host has {local_ver} but " | ||
| f"container has {remote_ver}. Matching major.minor " | ||
| f"versions are required for pickle compatibility." | ||
| ) | ||
| raise ContainerPrepareError(msg) | ||
|
|
||
|
|
@@ -113,7 +114,7 @@ def _install_deps(container: DockerContainer, python: pathlib.Path) -> pathlib.P | |
| if venv_ok: | ||
| python = pathlib.Path(f"{_VENV_DIR}/bin/python") | ||
|
|
||
| install_cmd = [str(python), "-m", "pip", "install", "rpyc", "pytest"] | ||
| install_cmd = [str(python), "-m", "pip", "install", "cloudpickle", "rpyc", "pytest"] | ||
| if not venv_ok: | ||
| install_cmd.insert(4, "--break-system-packages") | ||
| _run_or_fail(container, install_cmd, "Failed to install container deps.") | ||
|
|
@@ -170,3 +171,88 @@ def bootstrap_container( | |
| container.get_exposed_port(RPYC_PORT), | ||
| sync_request_timeout=sync_request_timeout, | ||
| ) | ||
|
|
||
|
|
||
| def _make_picklable[T: Callable[..., Any]](func: T) -> T: | ||
| """Return a copy of *func* that cloudpickle will serialise by value. | ||
|
|
||
| Two things prevent a test function from being naively pickled into a | ||
| remote container: | ||
|
|
||
| 1. cloudpickle pickles importable functions *by reference* | ||
| (module + qualname), but the test module does not exist in the | ||
| container. | ||
| 2. pytest's assertion rewriter injects ``@pytest_ar`` into the | ||
| function's ``__globals__``, and that object drags in the test | ||
| module itself. | ||
|
|
||
| We fix both by creating a **new** function object whose | ||
| ``__module__`` is ``"__mp_main__"`` (forces pickle-by-value) and | ||
| whose ``__globals__`` are a *shared* clean dict stripped of the | ||
| assertion-rewriter helper. All sibling callables (same module) are | ||
| cloned into the same ``clean_globals`` dict so transitive calls | ||
| between helpers resolve to the patched versions. | ||
| """ | ||
| import types # noqa: PLC0415 | ||
|
|
||
| original_module = func.__module__ | ||
|
|
||
| # First pass: build clean_globals with non-callable entries, | ||
| # collect names of same-module callables to patch. | ||
| clean_globals: dict[str, Any] = {} | ||
| to_patch: list[str] = [] | ||
| for k, v in func.__globals__.items(): | ||
| if k == "@pytest_ar": | ||
| continue | ||
| if ( | ||
| isinstance(v, types.FunctionType) | ||
| and getattr(v, "__module__", None) == original_module | ||
| ): | ||
| to_patch.append(k) | ||
| else: | ||
| clean_globals[k] = v | ||
|
|
||
| # Second pass: clone callables so they all share clean_globals. | ||
| for k in to_patch: | ||
| orig = func.__globals__[k] | ||
| clone = types.FunctionType( | ||
| orig.__code__, | ||
| clean_globals, | ||
| orig.__name__, | ||
| orig.__defaults__, | ||
| orig.__closure__, | ||
| ) | ||
| clone.__module__ = "__mp_main__" | ||
| clone.__qualname__ = orig.__qualname__ | ||
| clone.__annotations__ = orig.__annotations__ | ||
| clone.__kwdefaults__ = orig.__kwdefaults__ | ||
| clean_globals[k] = clone | ||
|
|
||
| # Clone the test function itself into the same shared dict. | ||
| clone = types.FunctionType( | ||
| func.__code__, | ||
| clean_globals, | ||
| func.__name__, | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Prompt for Agent |
||
| func.__defaults__, | ||
| func.__closure__, | ||
| ) | ||
| clone.__annotations__ = func.__annotations__ | ||
| clone.__kwdefaults__ = func.__kwdefaults__ | ||
| clone.__module__ = "__mp_main__" | ||
| clone.__qualname__ = func.__qualname__ | ||
| return clone # type: ignore[return-value] | ||
|
|
||
|
|
||
| def run_pickled[T]( | ||
| conn: Any, # noqa: ANN401 | ||
| func: Callable[..., T], | ||
| *args: Any, # noqa: ANN401 | ||
| **kwargs: Any, # noqa: ANN401 | ||
| ) -> T: | ||
| """Serialize *func* with cloudpickle, send to container, execute there.""" | ||
| import cloudpickle # noqa: PLC0415 | ||
|
|
||
| payload = cloudpickle.dumps(_make_picklable(func)) | ||
| rpickle = conn.modules["pickle"] | ||
| remote_func = rpickle.loads(payload) | ||
| return remote_func(*args, **kwargs) | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This sample now uses the marker form (
@pytest.mark.in_container) but the snippet no longer importspytest, so copying it verbatim raisesNameError. Either addimport pytestto this block or revert the example back to the decorator version to keep it self‑contained.Prompt for Agent