diff --git a/docker/Dockerfile.nmp-customizer-tasks b/docker/Dockerfile.nmp-customizer-tasks index 359f79df0b..b11aab2972 100644 --- a/docker/Dockerfile.nmp-customizer-tasks +++ b/docker/Dockerfile.nmp-customizer-tasks @@ -71,6 +71,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --overrides /app/docker/customizer/no_override_requirements.txt \ -e /app/sdk/python/nemo-platform \ -e /app/packages/filesets \ + -e /app/packages/models \ -e /app/packages/nemo_platform_plugin \ -e /app/packages/nmp_common \ -e /app/packages/nmp_customization_common \ diff --git a/docker/Dockerfile.nmp-unsloth-training b/docker/Dockerfile.nmp-unsloth-training index 217ad55acd..09767250b7 100644 --- a/docker/Dockerfile.nmp-unsloth-training +++ b/docker/Dockerfile.nmp-unsloth-training @@ -172,6 +172,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --overrides /opt/docker/no_override_requirements.txt \ -e /app/sdk/python/nemo-platform \ -e /app/packages/filesets \ + -e /app/packages/models \ -e /app/packages/nemo_platform_plugin \ -e /app/packages/nmp_common \ -e /app/packages/nmp_customization_common \ diff --git a/docker/automodel/Dockerfile.nmp-automodel-training b/docker/automodel/Dockerfile.nmp-automodel-training index e62e69af65..3f993f5b86 100644 --- a/docker/automodel/Dockerfile.nmp-automodel-training +++ b/docker/automodel/Dockerfile.nmp-automodel-training @@ -36,6 +36,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --overrides /app/docker/automodel/no_override_requirements.txt \ -e /app/sdk/python/nemo-platform \ -e /app/packages/filesets \ + -e /app/packages/models \ -e /app/packages/nemo_platform_plugin \ -e /app/packages/nmp_common \ -e /app/packages/nmp_customization_common \ diff --git a/docker/rl/Dockerfile.nmp-rl-training b/docker/rl/Dockerfile.nmp-rl-training index 5523e74dcc..5109cb1645 100644 --- a/docker/rl/Dockerfile.nmp-rl-training +++ b/docker/rl/Dockerfile.nmp-rl-training @@ -64,6 +64,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --overrides /tmp/base-overrides.txt \ -e /app/sdk/python/nemo-platform \ -e /app/packages/filesets \ + -e /app/packages/models \ -e /app/packages/nemo_platform_plugin \ -e /app/packages/nmp_common \ -e /app/packages/nmp_customization_common \ diff --git a/docker/rl/Dockerfile.platform-workspace b/docker/rl/Dockerfile.platform-workspace index 07b73f8f98..4c8a5b52f6 100644 --- a/docker/rl/Dockerfile.platform-workspace +++ b/docker/rl/Dockerfile.platform-workspace @@ -15,6 +15,7 @@ COPY docs docs COPY openapi openapi COPY packages/nmp_build_tools packages/nmp_build_tools COPY packages/filesets packages/filesets +COPY packages/models packages/models COPY packages/nmp_common packages/nmp_common COPY packages/nmp_customization_common packages/nmp_customization_common COPY packages/nemo_platform_plugin packages/nemo_platform_plugin diff --git a/docker/rl/pyproject.workspace.toml b/docker/rl/pyproject.workspace.toml index 92fc1cceed..d81f0ab81b 100644 --- a/docker/rl/pyproject.workspace.toml +++ b/docker/rl/pyproject.workspace.toml @@ -17,6 +17,7 @@ required-version = ">=0.9.14,<0.10.0" members = [ "packages/nmp_build_tools", "packages/filesets", + "packages/models", "sdk/python/nemo-platform", "packages/nemo_platform_plugin", "packages/nmp_common", @@ -27,6 +28,7 @@ members = [ [tool.uv.sources] nmp-build-tools = { workspace = true } filesets = { workspace = true } +models = { workspace = true } nemo-platform-sdk = { workspace = true } nemo-platform-plugin = { workspace = true } nmp-common = { workspace = true } diff --git a/docker/unsloth/Dockerfile.platform-workspace b/docker/unsloth/Dockerfile.platform-workspace index 80106e743a..864578f6c9 100644 --- a/docker/unsloth/Dockerfile.platform-workspace +++ b/docker/unsloth/Dockerfile.platform-workspace @@ -17,6 +17,7 @@ COPY docs docs COPY openapi openapi COPY packages/nmp_build_tools packages/nmp_build_tools COPY packages/filesets packages/filesets +COPY packages/models packages/models COPY packages/nmp_common packages/nmp_common COPY packages/nmp_customization_common packages/nmp_customization_common COPY packages/nemo_platform_plugin packages/nemo_platform_plugin diff --git a/docker/unsloth/pyproject.workspace.toml b/docker/unsloth/pyproject.workspace.toml index 4b7185abd0..35b86ee4b3 100644 --- a/docker/unsloth/pyproject.workspace.toml +++ b/docker/unsloth/pyproject.workspace.toml @@ -16,6 +16,7 @@ required-version = ">=0.9.14,<0.10.0" members = [ "packages/nmp_build_tools", "packages/filesets", + "packages/models", "sdk/python/nemo-platform", "packages/nemo_platform_plugin", "packages/nmp_common", @@ -26,6 +27,7 @@ members = [ [tool.uv.sources] nmp-build-tools = { workspace = true } filesets = { workspace = true } +models = { workspace = true } nemo-platform-sdk = { workspace = true } nemo-platform-plugin = { workspace = true } nmp-common = { workspace = true } diff --git a/packages/nemo_platform_ext/src/nemo_platform_ext/quickstart/container.py b/packages/nemo_platform_ext/src/nemo_platform_ext/quickstart/container.py index 7d31ace627..f604642545 100644 --- a/packages/nemo_platform_ext/src/nemo_platform_ext/quickstart/container.py +++ b/packages/nemo_platform_ext/src/nemo_platform_ext/quickstart/container.py @@ -325,6 +325,8 @@ def _create_environment(self, platform_config: PlatformConfig) -> dict[str, str] # Configure the models/NIM Docker backend for DonD (Docker-on-Docker) setup. # NIMs need to join the same network as the quickstart container so they can # communicate via container names (e.g., http://md-workspace-name:8000). + env["NEMO_DEPLOYMENTS_DOCKER_NETWORK"] = self.config.network_name + env["NEMO_DEPLOYMENTS_DOCKER_ENDPOINT_MODE"] = "network" env["MODELS_DOCKER_NETWORKING_MODE"] = "dond" env["MODELS_DOCKER_NETWORK"] = self.config.network_name # Pass the container name so the Models service can use it for localhost replacement diff --git a/packages/nemo_platform_ext/tests/quickstart/test_container.py b/packages/nemo_platform_ext/tests/quickstart/test_container.py index 8a01e414be..8f3d3325d2 100644 --- a/packages/nemo_platform_ext/tests/quickstart/test_container.py +++ b/packages/nemo_platform_ext/tests/quickstart/test_container.py @@ -204,6 +204,16 @@ def test_single_gpu_id_format(self): assert env["NMP_DOCKER_RESERVED_GPU_DEVICE_IDS"] == "0" + def test_deployments_docker_backend_uses_quickstart_network_endpoint_mode(self): + manager = self._make_manager() + mock_platform_config = MagicMock() + mock_platform_config.to_env_vars.return_value = {} + + env = manager._create_environment(mock_platform_config) + + assert env["NEMO_DEPLOYMENTS_DOCKER_NETWORK"] == "test-network" + assert env["NEMO_DEPLOYMENTS_DOCKER_ENDPOINT_MODE"] == "network" + class TestCreateEnvironmentRegistryCredentials: """Tests for registry credential env var passthrough.""" diff --git a/packages/nmp_testing/src/nmp/testing/e2e/docker.py b/packages/nmp_testing/src/nmp/testing/e2e/docker.py index c81a54e56e..c3ffef5cd0 100644 --- a/packages/nmp_testing/src/nmp/testing/e2e/docker.py +++ b/packages/nmp_testing/src/nmp/testing/e2e/docker.py @@ -183,6 +183,8 @@ def start(self) -> None: # Models Docker backend: use DonD mode so the API (in a container) can reach NIM # containers by name on the same network. Without this, health_url is localhost:port # and the API container cannot reach the NIM (localhost is the API itself). + self.container.with_env("NEMO_DEPLOYMENTS_DOCKER_NETWORK", self.network.name) + self.container.with_env("NEMO_DEPLOYMENTS_DOCKER_ENDPOINT_MODE", "network") self.container.with_env("MODELS_DOCKER_NETWORKING_MODE", "dond") self.container.with_env("MODELS_DOCKER_NETWORK", self.network.name) self.container.with_env("MODELS_DOCKER_CONTAINER_NAME", NMP_API_NETWORK_ALIAS) diff --git a/packages/nmp_testing/tests/unit/test_e2e_docker_backend.py b/packages/nmp_testing/tests/unit/test_e2e_docker_backend.py index f7f709ea38..3643773eca 100644 --- a/packages/nmp_testing/tests/unit/test_e2e_docker_backend.py +++ b/packages/nmp_testing/tests/unit/test_e2e_docker_backend.py @@ -119,6 +119,8 @@ def test_docker_backend_starts_clickhouse_sidecar_on_api_network( assert api_container.network is backend.network assert api_container.env["NMP_INTAKE_CLICKHOUSE_URL"] == docker_backend._clickhouse_api_url() assert api_container.env["NEMO_JOBS_DEFAULT_DOCKER_NETWORK"] == "nmp-e2e-test-network" + assert api_container.env["NEMO_DEPLOYMENTS_DOCKER_NETWORK"] == "nmp-e2e-test-network" + assert api_container.env["NEMO_DEPLOYMENTS_DOCKER_ENDPOINT_MODE"] == "network" assert api_container.env["NMP_IMAGE_REGISTRY"] == "registry.example/nmp" assert api_container.env["NMP_IMAGE_TAG"] == "test-tag" diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py index a6c929e8ec..4844e5b4fd 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py @@ -287,6 +287,8 @@ async def create_deployment( docker_cfg = parse_docker_backend_config(backend_config) if config.backend_config.docker is not None: docker_cfg = config.backend_config.docker + if docker_cfg.network is None and self._executor_config.network is not None: + docker_cfg = docker_cfg.model_copy(update={"network": self._executor_config.network}) dep_key = deployment_key(workspace, name) gpu_pool = self._gpu_pool @@ -429,7 +431,7 @@ async def create_deployment( status_message=f"Failed to start sidecar {sidecar.name}: {exc}", ) - endpoints = self._build_endpoints(container_spec, host_ports) + endpoints = self._build_endpoints(container_spec, host_ports, target_name=c_name) if config.restart_policy in _ONE_SHOT_RESTART_POLICIES: return await self._observe_one_shot_primary_after_create( workspace=workspace, @@ -730,7 +732,7 @@ async def read_status(self, *, workspace: str, name: str) -> BackendStatusUpdate state = container.status container_id = (container.id or "")[:12] host_ports = self._extract_host_ports(container) - endpoints = self._endpoints_from_container_ports(container, host_ports) + endpoints = self._endpoints_from_container_ports(host_ports, target_name=c_name) if state in ("created", "restarting"): return map_docker_state_to_starting(container_id, state) @@ -740,7 +742,8 @@ async def read_status(self, *, workspace: str, name: str) -> BackendStatusUpdate # it must see only TCP mappings: a UDP-only workload has no TCP listener and # would otherwise be gated STARTING forever. Endpoints still carry every port. tcp_host_ports = self._extract_host_ports(container, protocol="tcp") - host_url = self._primary_host_url(tcp_host_ports) + host_url = self._primary_host_url(tcp_host_ports, target_name=c_name) + probe_ports = self._probe_ports(tcp_host_ports) config = await self._load_config_from_labels(workspace, labels) probe = None if config is not None and config.containers: @@ -749,7 +752,7 @@ async def read_status(self, *, workspace: str, name: str) -> BackendStatusUpdate container=container, probe=probe, host_url=host_url, - host_ports=tcp_host_ports, + host_ports=probe_ports, ) if ready and restart_policy == "Always": sidecar_ok, sidecar_reason = await self._sidecars_healthy(workspace, name, config) @@ -1253,42 +1256,76 @@ def _extract_host_ports(self, container: DockerContainer, *, protocol: str | Non result[container_port] = int(host_port) return result - def _primary_host_url(self, host_ports: dict[int, int]) -> str | None: - if not host_ports: + def _url_for_port( + self, + *, + target_name: str, + container_port: int, + host_port: int | None, + scheme: str = "http", + ) -> str | None: + if self._executor_config.endpoint_mode == "network": + return host_url_for_port(target_name, container_port, scheme=scheme) + if host_port is None: return None - host_port = next(iter(host_ports.values())) host = os.environ.get("NMP_LOOPBACK_ADDRESS", LOOPBACK_ADDRESSES[0]) - return host_url_for_port(host, host_port) + return host_url_for_port(host, host_port, scheme=scheme) - def _build_endpoints(self, container_spec: Container, host_ports: dict[int, int]) -> list[Endpoint]: + def _primary_host_url(self, host_ports: dict[int, int], *, target_name: str) -> str | None: + if not host_ports: + return None + container_port, host_port = next(iter(host_ports.items())) + return self._url_for_port(target_name=target_name, container_port=container_port, host_port=host_port) + + def _probe_ports(self, host_ports: dict[int, int]) -> dict[int, int]: + if self._executor_config.endpoint_mode == "network": + return {container_port: container_port for container_port in host_ports} + return host_ports + + def _build_endpoints( + self, + container_spec: Container, + host_ports: dict[int, int], + *, + target_name: str, + ) -> list[Endpoint]: endpoints: list[Endpoint] = [] - host = os.environ.get("NMP_LOOPBACK_ADDRESS", LOOPBACK_ADDRESSES[0]) for port_spec in container_spec.ports: host_port = host_ports.get(port_spec.container_port) - if host_port is None: + endpoint_url = self._url_for_port( + target_name=target_name, + container_port=port_spec.container_port, + host_port=host_port, + ) + if endpoint_url is None: continue endpoint_name = port_spec.name or f"port-{port_spec.container_port}" protocol = "tcp" if port_spec.protocol == "UDP" else "http" - scheme = "http" endpoints.append( Endpoint( name=endpoint_name, - url=host_url_for_port(host, host_port, scheme=scheme), + url=endpoint_url, protocol=protocol, ) ) return endpoints - def _endpoints_from_container_ports(self, container: DockerContainer, host_ports: dict[int, int]) -> list[Endpoint]: + def _endpoints_from_container_ports(self, host_ports: dict[int, int], *, target_name: str) -> list[Endpoint]: if not host_ports: return [] - host = os.environ.get("NMP_LOOPBACK_ADDRESS", LOOPBACK_ADDRESSES[0]) endpoints: list[Endpoint] = [] for container_port, host_port in host_ports.items(): + endpoint_url = self._url_for_port( + target_name=target_name, + container_port=container_port, + host_port=host_port, + ) + if endpoint_url is None: + continue endpoints.append( Endpoint( name=f"port-{container_port}", - url=host_url_for_port(host, host_port), + url=endpoint_url, protocol="http", ) ) diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/config.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/config.py index 20e25b5d2d..e54ef4efa3 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/config.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/config.py @@ -5,8 +5,41 @@ from __future__ import annotations +import os +from typing import Literal + from nemo_deployments_plugin.backends.labels import DEFAULT_RESOURCE_SCOPE -from pydantic import BaseModel, Field, model_validator +from pydantic import BaseModel, Field, field_validator, model_validator + +DOCKER_ENDPOINT_MODE_ENV_VAR = "NEMO_DEPLOYMENTS_DOCKER_ENDPOINT_MODE" +DOCKER_NETWORK_ENV_VAR = "NEMO_DEPLOYMENTS_DOCKER_NETWORK" +LEGACY_MODELS_DOCKER_NETWORKING_MODE_ENV_VAR = "MODELS_DOCKER_NETWORKING_MODE" +LEGACY_MODELS_DOCKER_NETWORK_ENV_VAR = "MODELS_DOCKER_NETWORK" +DockerEndpointMode = Literal["host", "network"] + + +def _default_network() -> str | None: + for env_var in (DOCKER_NETWORK_ENV_VAR, LEGACY_MODELS_DOCKER_NETWORK_ENV_VAR): + value = os.getenv(env_var) + if value: + return value + return None + + +def _default_endpoint_mode() -> str: + value = os.getenv(DOCKER_ENDPOINT_MODE_ENV_VAR) + if value: + return _normalize_endpoint_mode(value) + return _normalize_endpoint_mode(os.getenv(LEGACY_MODELS_DOCKER_NETWORKING_MODE_ENV_VAR, "host")) + + +def _normalize_endpoint_mode(value: str) -> str: + mode = value.lower() + if mode == "dond": + return "network" + if mode in {"local", "dind"}: + return "host" + return mode class DockerExecutorConfig(BaseModel): @@ -28,6 +61,25 @@ class DockerExecutorConfig(BaseModel): ), ) pull_images: bool = Field(default=True, description="Pull container images before run when missing locally.") + network: str | None = Field( + default_factory=_default_network, + description=( + "Default Docker network for containers created by this executor. " + f"Can also be set with {DOCKER_NETWORK_ENV_VAR}; " + f"{LEGACY_MODELS_DOCKER_NETWORK_ENV_VAR} is accepted for compatibility." + ), + ) + endpoint_mode: DockerEndpointMode = Field( + default_factory=_default_endpoint_mode, + validate_default=True, + description=( + "'host' reports and probes host-published ports. 'network' reports and probes " + "Docker-network container names and container ports, for controller/API " + "processes that run inside a container on the same Docker network. Can also " + f"be set with {DOCKER_ENDPOINT_MODE_ENV_VAR}; " + f"{LEGACY_MODELS_DOCKER_NETWORKING_MODE_ENV_VAR}=dond selects 'network' for compatibility." + ), + ) resource_scope: str = Field( default=DEFAULT_RESOURCE_SCOPE, min_length=1, @@ -49,8 +101,17 @@ class DockerExecutorConfig(BaseModel): description="Last host port (inclusive) to consider when publishing container ports for this executor.", ) + @field_validator("endpoint_mode", mode="before") + @classmethod + def _normalize_endpoint_mode(cls, value: object) -> object: + if isinstance(value, str): + return _normalize_endpoint_mode(value) + return value + @model_validator(mode="after") - def _validate_port_range(self) -> DockerExecutorConfig: + def _validate_config(self) -> DockerExecutorConfig: if self.port_range_start > self.port_range_end: raise ValueError("port_range_start must not exceed port_range_end") + if self.endpoint_mode == "network" and not self.network: + raise ValueError("endpoint_mode='network' requires network") return self diff --git a/plugins/nemo-deployments/tests/unit/backends/docker/test_backend_mocked.py b/plugins/nemo-deployments/tests/unit/backends/docker/test_backend_mocked.py index 5eb09ce568..f71a3a7e66 100644 --- a/plugins/nemo-deployments/tests/unit/backends/docker/test_backend_mocked.py +++ b/plugins/nemo-deployments/tests/unit/backends/docker/test_backend_mocked.py @@ -94,6 +94,118 @@ async def test_create_deployment_starts_container( mock_entities.get.assert_awaited() +@pytest.mark.asyncio +async def test_create_deployment_uses_executor_default_network( + mock_sdk: MagicMock, + mock_entities: AsyncMock, + mock_docker_client: MagicMock, +) -> None: + with ( + patch("nemo_deployments_plugin.backends.docker.backend.client_from_platform"), + patch("nemo_deployments_plugin.backends.docker.backend.NemoEntitiesClient", return_value=mock_entities), + patch("nemo_deployments_plugin.backends.docker.backend.get_shared_gpu_pool", return_value=None), + patch("docker.from_env", return_value=mock_docker_client), + ): + backend = DockerDeploymentBackend( + mock_sdk, + {"docker_timeout": 60, "pull_images": False, "network": "nmp-e2e-test-network"}, + ) + backend._client = mock_docker_client + + mock_entities.get.return_value = sample_config() + mock_docker_client.containers.get.side_effect = NotFound("missing") + mock_docker_client.containers.create.return_value = MagicMock(id="abc123") + + update = await backend.create_deployment( + workspace="default", + name="srv", + config_name="cfg1", + labels={"managed-by": MANAGED_BY_LABEL}, + backend_config={}, + ) + + assert update.status == "STARTING" + _, create_kwargs = mock_docker_client.containers.create.call_args + assert create_kwargs["network"] == "nmp-e2e-test-network" + + +@pytest.mark.asyncio +async def test_create_deployment_backend_config_network_overrides_executor_default( + mock_sdk: MagicMock, + mock_entities: AsyncMock, + mock_docker_client: MagicMock, +) -> None: + with ( + patch("nemo_deployments_plugin.backends.docker.backend.client_from_platform"), + patch("nemo_deployments_plugin.backends.docker.backend.NemoEntitiesClient", return_value=mock_entities), + patch("nemo_deployments_plugin.backends.docker.backend.get_shared_gpu_pool", return_value=None), + patch("docker.from_env", return_value=mock_docker_client), + ): + backend = DockerDeploymentBackend( + mock_sdk, + {"docker_timeout": 60, "pull_images": False, "network": "executor-network"}, + ) + backend._client = mock_docker_client + + mock_entities.get.return_value = sample_config() + mock_docker_client.containers.get.side_effect = NotFound("missing") + mock_docker_client.containers.create.return_value = MagicMock(id="abc123") + + update = await backend.create_deployment( + workspace="default", + name="srv", + config_name="cfg1", + labels={"managed-by": MANAGED_BY_LABEL}, + backend_config={"docker": {"network": "deployment-network"}}, + ) + + assert update.status == "STARTING" + _, create_kwargs = mock_docker_client.containers.create.call_args + assert create_kwargs["network"] == "deployment-network" + + +@pytest.mark.asyncio +async def test_create_deployment_network_endpoint_mode_reports_container_url( + mock_sdk: MagicMock, + mock_entities: AsyncMock, + mock_docker_client: MagicMock, + free_host_ports: None, +) -> None: + del free_host_ports + with ( + patch("nemo_deployments_plugin.backends.docker.backend.client_from_platform"), + patch("nemo_deployments_plugin.backends.docker.backend.NemoEntitiesClient", return_value=mock_entities), + patch("nemo_deployments_plugin.backends.docker.backend.get_shared_gpu_pool", return_value=None), + patch("docker.from_env", return_value=mock_docker_client), + ): + backend = DockerDeploymentBackend( + mock_sdk, + { + "docker_timeout": 60, + "pull_images": False, + "network": "nmp-e2e-test-network", + "endpoint_mode": "network", + }, + ) + backend._client = mock_docker_client + + mock_entities.get.return_value = published_port_config() + mock_docker_client.containers.get.side_effect = NotFound("missing") + mock_docker_client.containers.create.return_value = MagicMock(id="abc123") + + update = await backend.create_deployment( + workspace="default", + name="srv", + config_name="cfg1", + labels={"managed-by": MANAGED_BY_LABEL}, + backend_config={}, + ) + + assert update.status == "STARTING" + assert len(update.endpoints) == 1 + assert update.endpoints[0].url == f"http://{container_name('default', 'srv')}:8000" + + @pytest.mark.asyncio async def test_create_deployment_maps_command_to_entrypoint( docker_backend: DockerDeploymentBackend, @@ -809,6 +921,48 @@ async def test_read_status_starting_when_running_port_not_bound( assert "not ready" in update.status_message +@pytest.mark.asyncio +async def test_read_status_network_endpoint_mode_probes_container_url( + mock_sdk: MagicMock, + mock_entities: AsyncMock, + mock_docker_client: MagicMock, +) -> None: + with ( + patch("nemo_deployments_plugin.backends.docker.backend.client_from_platform"), + patch("nemo_deployments_plugin.backends.docker.backend.NemoEntitiesClient", return_value=mock_entities), + patch("nemo_deployments_plugin.backends.docker.backend.get_shared_gpu_pool", return_value=None), + patch("docker.from_env", return_value=mock_docker_client), + ): + backend = DockerDeploymentBackend( + mock_sdk, + { + "docker_timeout": 60, + "pull_images": False, + "network": "nmp-e2e-test-network", + "endpoint_mode": "network", + }, + ) + backend._client = mock_docker_client + + mock_entities.get.return_value = published_port_config() + mock_docker_client.containers.get.return_value = _running_container_with_published_port(49152) + with patch( + "nemo_deployments_plugin.backends.docker.backend.check_readiness_probe", + new=AsyncMock(return_value=(True, "http probe 200")), + ) as readiness: + update = await backend.read_status(workspace="default", name="srv") + + target_url = f"http://{container_name('default', 'srv')}:8000" + assert update.status == "READY" + assert len(update.endpoints) == 1 + assert update.endpoints[0].url == target_url + readiness.assert_awaited_once() + await_args = readiness.await_args + assert await_args is not None + assert await_args.kwargs["host_url"] == target_url + assert await_args.kwargs["host_ports"] == {8000: 8000} + + @pytest.mark.asyncio async def test_read_status_ready_when_running_udp_only_port( docker_backend: DockerDeploymentBackend, diff --git a/plugins/nemo-deployments/tests/unit/backends/docker/test_executor_config.py b/plugins/nemo-deployments/tests/unit/backends/docker/test_executor_config.py index 2fa9f416bc..efecbe421e 100644 --- a/plugins/nemo-deployments/tests/unit/backends/docker/test_executor_config.py +++ b/plugins/nemo-deployments/tests/unit/backends/docker/test_executor_config.py @@ -7,12 +7,66 @@ from pydantic import ValidationError -def test_docker_executor_config_defaults() -> None: +def test_docker_executor_config_defaults(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("NEMO_DEPLOYMENTS_DOCKER_ENDPOINT_MODE", raising=False) + monkeypatch.delenv("NEMO_DEPLOYMENTS_DOCKER_NETWORK", raising=False) + monkeypatch.delenv("MODELS_DOCKER_NETWORKING_MODE", raising=False) + monkeypatch.delenv("MODELS_DOCKER_NETWORK", raising=False) + cfg = DockerExecutorConfig() assert cfg.port_range_start == 49152 assert cfg.port_range_end == 49251 assert cfg.resource_scope == DEFAULT_RESOURCE_SCOPE assert cfg.oneshot_observe_timeout_seconds == 5 + assert cfg.network is None + assert cfg.endpoint_mode == "host" + + +def test_docker_executor_config_reads_network_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("NEMO_DEPLOYMENTS_DOCKER_NETWORK", "nmp-e2e-test-network") + monkeypatch.setenv("MODELS_DOCKER_NETWORK", "legacy-network") + + cfg = DockerExecutorConfig() + + assert cfg.network == "nmp-e2e-test-network" + + +def test_docker_executor_config_reads_legacy_models_network_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("NEMO_DEPLOYMENTS_DOCKER_NETWORK", raising=False) + monkeypatch.setenv("MODELS_DOCKER_NETWORK", "legacy-network") + + cfg = DockerExecutorConfig() + + assert cfg.network == "legacy-network" + + +def test_docker_executor_config_reads_endpoint_mode_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("NEMO_DEPLOYMENTS_DOCKER_ENDPOINT_MODE", "NETWORK") + monkeypatch.setenv("NEMO_DEPLOYMENTS_DOCKER_NETWORK", "nmp-e2e-test-network") + + cfg = DockerExecutorConfig() + + assert cfg.endpoint_mode == "network" + + +def test_docker_executor_config_reads_legacy_dond_endpoint_mode(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("NEMO_DEPLOYMENTS_DOCKER_ENDPOINT_MODE", raising=False) + monkeypatch.setenv("MODELS_DOCKER_NETWORKING_MODE", "dond") + monkeypatch.setenv("MODELS_DOCKER_NETWORK", "legacy-network") + + cfg = DockerExecutorConfig() + + assert cfg.endpoint_mode == "network" + + +def test_docker_executor_config_rejects_network_endpoint_mode_without_network( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("NEMO_DEPLOYMENTS_DOCKER_NETWORK", raising=False) + monkeypatch.delenv("MODELS_DOCKER_NETWORK", raising=False) + + with pytest.raises(ValidationError, match="endpoint_mode='network' requires network"): + DockerExecutorConfig(endpoint_mode="network") def test_docker_executor_config_rejects_inverted_port_range() -> None: diff --git a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py index ce60701830..f3c3f1a634 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/compiler.py @@ -8,7 +8,6 @@ """ from dataclasses import dataclass -from urllib.parse import urlsplit from nemo_deployments_plugin.entities import ( Container, @@ -25,11 +24,7 @@ VolumeMount, ) from nemo_deployments_plugin.secrets import platform_ngc_secret_ref -from nemo_platform_plugin.config import ( - LOOPBACK_ADDRESSES, - determine_loopback_override, - get_platform_config, -) +from nemo_platform_plugin.config import get_platform_config from nemo_platform_plugin.jobs.image import get_qualified_image from nmp.common.config import Runtime from nmp.core.models.app import ModelWeightsType @@ -45,7 +40,10 @@ tool_call_plugin_init_containers, tool_call_plugin_install_path, ) -from nmp.core.models.controllers.backends.deployments_plugin.resolve import ResolvedPluginDeployment +from nmp.core.models.controllers.backends.deployments_plugin.resolve import ( + ResolvedPluginDeployment, + rewrite_loopback_for_docker_container, +) from nmp.core.models.controllers.backends.engine import ( ENGINE_GENERIC, ENGINE_NIM, @@ -163,17 +161,11 @@ def _container_reachable_platform_base_url(*, runtime: Runtime) -> str: """ platform = get_platform_config() base_url = platform.base_url.rstrip("/") - if runtime != Runtime.DOCKER: - return base_url - - parts = urlsplit(base_url) - hostname = (parts.hostname or "").lower() - if hostname not in LOOPBACK_ADDRESSES: - return base_url - - override = platform.loopback_address or determine_loopback_override() or "host.docker.internal" - netloc = override if parts.port is None else f"{override}:{parts.port}" - return parts._replace(netloc=netloc).geturl() + return rewrite_loopback_for_docker_container( + base_url, + runtime=runtime, + loopback_address=platform.loopback_address, + ) def _lora_sidecar( @@ -278,7 +270,6 @@ def compile_model_deployment( env=_env(puller_env), volumeMounts=[VolumeMount(name=names.volume, mountPath=_WEIGHTS_MOUNT)], ) - _apply_gpu_resources(puller, resolved.view.gpu) puller_config = DeploymentConfig( name=names.puller, workspace=resolved.deployment.workspace, diff --git a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/resolve.py b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/resolve.py index faa301bfd5..a50fe4cd92 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/resolve.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/deployments_plugin/resolve.py @@ -4,12 +4,13 @@ """Resolve models API objects into compiler inputs.""" from dataclasses import dataclass -from urllib.parse import urljoin +from urllib.parse import SplitResult, urljoin, urlsplit from nemo_platform.types.inference.model_deployment import ModelDeployment from nemo_platform.types.inference.model_deployment_config import ModelDeploymentConfig from nemo_platform.types.models.model_entity import ModelEntity from nmp.common.config import Runtime, get_platform_config +from nmp.common.config.base import LOOPBACK_ADDRESSES, determine_loopback_override from nmp.core.models.app import ModelWeightsType, get_model_weights_type, parse_model_name_revision from nmp.core.models.controllers.backends.common import DeploymentConfigView, deployment_config_view from nmp.core.models.controllers.context import ModelContext @@ -32,6 +33,54 @@ class ResolvedPluginDeployment: runtime: Runtime +def _split_url(url: str) -> SplitResult | None: + try: + parsed = urlsplit(url) + parsed.hostname + parsed.port + except ValueError: + return None + return parsed + + +def _format_netloc_with_hostname(parsed: SplitResult, hostname: str) -> str: + host = hostname[1:-1] if hostname.startswith("[") and hostname.endswith("]") else hostname + if ":" in host: + host = f"[{host}]" + + userinfo = "" + if parsed.username is not None: + userinfo = parsed.username + if parsed.password is not None: + userinfo = f"{userinfo}:{parsed.password}" + userinfo = f"{userinfo}@" + + netloc = userinfo + host + if parsed.port is not None: + netloc = f"{netloc}:{parsed.port}" + return netloc + + +def rewrite_loopback_for_docker_container( + url: str, + *, + runtime: Runtime, + loopback_address: str | None = None, +) -> str: + """Rewrite loopback service URLs so Docker deployment containers can reach the host API.""" + if runtime != Runtime.DOCKER: + return url + + parsed = _split_url(url) + hostname = (parsed.hostname or "").lower() if parsed is not None else "" + if parsed is None or hostname not in LOOPBACK_ADDRESSES: + return url + + override = loopback_address or determine_loopback_override() or "host.docker.internal" + netloc = _format_netloc_with_hostname(parsed, override) + return parsed._replace(netloc=netloc).geturl() + + def resolve_model_source( model_entity: ModelEntity | None, view: DeploymentConfigView ) -> tuple[str | None, str | None, str | None]: @@ -55,6 +104,11 @@ def resolve_plugin_deployment(ctx: ModelContext, huggingface_model_puller: str) platform_config = get_platform_config() files_service_url = platform_config.service_discovery.get("files") or platform_config.base_url files_hf_url = urljoin(files_service_url.rstrip("/") + "/", "apis/files/v2/hf") + files_hf_url = rewrite_loopback_for_docker_container( + files_hf_url, + runtime=platform_config.runtime, + loopback_address=platform_config.loopback_address, + ) return ResolvedPluginDeployment( deployment=ctx.model_deployment, config=ctx.model_deployment_config, diff --git a/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py index c2d4b45a72..71e1b16548 100644 --- a/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py +++ b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_compiler.py @@ -89,8 +89,7 @@ def test_vllm_server_command_image_args_and_gpu() -> None: puller_env = {item.name: item.value for item in puller.env} assert puller_env["HF_ENDPOINT"] == resolved.files_hf_url assert puller_env["HF_TOKEN"] == "service:models" - assert puller.resources is not None - assert puller.resources.limits["nvidia.com/gpu"] == "1" + assert "nvidia.com/gpu" not in puller.resources.limits def test_nim_gpu_resources_without_override() -> None: @@ -392,7 +391,7 @@ def test_lora_sidecar_rewrites_loopback_nmp_base_url_for_docker() -> None: return_value=platform, ), patch( - "nmp.core.models.controllers.backends.deployments_plugin.compiler.determine_loopback_override", + "nmp.core.models.controllers.backends.deployments_plugin.resolve.determine_loopback_override", return_value=None, ), ): diff --git a/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_resolve.py b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_resolve.py new file mode 100644 index 0000000000..66da896c44 --- /dev/null +++ b/services/core/models/tests/unit/controllers/backends/deployments_plugin/test_resolve.py @@ -0,0 +1,93 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from unittest.mock import patch +from urllib.parse import urlunsplit + +from nmp.common.config import Runtime +from nmp.core.models.controllers.backends.deployments_plugin.resolve import rewrite_loopback_for_docker_container + + +def test_rewrites_loopback_url_for_docker_with_configured_address() -> None: + result = rewrite_loopback_for_docker_container( + "http://localhost:32827/apis/files/v2/hf", + runtime=Runtime.DOCKER, + loopback_address="host.docker.internal", + ) + + assert result == "http://host.docker.internal:32827/apis/files/v2/hf" + + +def test_rewrites_ipv6_loopback_with_valid_bracket_formatting() -> None: + result = rewrite_loopback_for_docker_container( + "http://[::1]:32827/apis/files/v2/hf", + runtime=Runtime.DOCKER, + loopback_address="fd00::10", + ) + + assert result == "http://[fd00::10]:32827/apis/files/v2/hf" + + +def test_preserves_url_when_loopback_text_is_not_hostname() -> None: + url = "http://api.localhost.example:32827/apis/files/v2/hf?next=http://127.0.0.1:8080" + + result = rewrite_loopback_for_docker_container( + url, + runtime=Runtime.DOCKER, + loopback_address="host.docker.internal", + ) + + assert result == url + + +def test_preserves_loopback_url_for_kubernetes() -> None: + url = "http://localhost:32827/apis/files/v2/hf" + + result = rewrite_loopback_for_docker_container( + url, + runtime=Runtime.KUBERNETES, + loopback_address="host.docker.internal", + ) + + assert result == url + + +def test_uses_detected_loopback_override_when_config_is_unset() -> None: + with patch( + "nmp.core.models.controllers.backends.deployments_plugin.resolve.determine_loopback_override", + return_value="nemo-gateway", + ): + result = rewrite_loopback_for_docker_container( + "http://127.0.0.1:32827/apis/files/v2/hf", + runtime=Runtime.DOCKER, + loopback_address=None, + ) + + assert result == "http://nemo-gateway:32827/apis/files/v2/hf" + + +def test_falls_back_to_host_docker_internal_for_docker_loopback() -> None: + with patch( + "nmp.core.models.controllers.backends.deployments_plugin.resolve.determine_loopback_override", + return_value=None, + ): + result = rewrite_loopback_for_docker_container( + "http://localhost:32827/apis/files/v2/hf", + runtime=Runtime.DOCKER, + loopback_address=None, + ) + + assert result == "http://host.docker.internal:32827/apis/files/v2/hf" + + +def test_rewrites_loopback_hostname_with_userinfo_from_parsed_url() -> None: + url = urlunsplit(("http", "user:pass@localhost:32827", "/apis/files/v2/hf", "", "")) + expected = urlunsplit(("http", "user:pass@host.docker.internal:32827", "/apis/files/v2/hf", "", "")) + + result = rewrite_loopback_for_docker_container( + url, + runtime=Runtime.DOCKER, + loopback_address="host.docker.internal", + ) + + assert result == expected diff --git a/tests/smoke_gpu/test_customizer_automodel.py b/tests/smoke_gpu/test_customizer_automodel.py index a3545d938c..f3316b37d9 100644 --- a/tests/smoke_gpu/test_customizer_automodel.py +++ b/tests/smoke_gpu/test_customizer_automodel.py @@ -16,7 +16,9 @@ the one installed (ABI mismatch) """ +from importlib.machinery import ModuleSpec from pathlib import Path +from typing import Any import pytest from file_removals import assert_file_patterns_absent, read_file_patterns @@ -98,7 +100,7 @@ def test_transformers_audio_backend_probe_is_off(): @pytest.mark.smoke_nmp_automodel_training -def test_peft_lora_dispatch_matches_installed_torchao(): +def test_peft_lora_dispatch_matches_installed_torchao(monkeypatch: pytest.MonkeyPatch): """PEFT's torchao dispatcher must import against the torchao in this image. ``dispatch_torchao`` runs during adapter injection and is gated on ``find_spec("torchao")`` -- @@ -106,6 +108,17 @@ def test_peft_lora_dispatch_matches_installed_torchao(): the NGC base image rather than the venv, so a peft expecting a retired torchao API fails ``merge=true`` jobs at injection, after training has already succeeded. """ + import importlib.util + + original_find_spec = importlib.util.find_spec + + def find_spec_without_transformer_engine(name: str, *args: Any, **kwargs: Any) -> ModuleSpec | None: + if name == "transformer_engine" or name.startswith("transformer_engine."): + return None + return original_find_spec(name, *args, **kwargs) + + monkeypatch.setattr(importlib.util, "find_spec", find_spec_without_transformer_engine) + import torch from peft import LoraConfig from peft.tuners.lora.torchao import dispatch_torchao diff --git a/tests/smoke_gpu/test_customizer_tasks.py b/tests/smoke_gpu/test_customizer_tasks.py index 6884dd878c..2853d5e270 100644 --- a/tests/smoke_gpu/test_customizer_tasks.py +++ b/tests/smoke_gpu/test_customizer_tasks.py @@ -76,6 +76,18 @@ def test_nmp_customizer_tasks_importable(): from nmp.customization_common.tasks.model_entity import __main__ as model_entity_main # noqa: F401 +@pytest.mark.smoke_nmp_customizer_tasks +def test_sdk_alias_resources_importable(): + from nemo_platform import NeMoPlatform + + sdk = NeMoPlatform(base_url="http://127.0.0.1:1") + try: + sdk.files + sdk.models + finally: + sdk.close() + + @pytest.mark.smoke_nmp_customizer_tasks def test_dali_files_removed(): patterns = [ diff --git a/tests/unit/test_docker_workspace_slices.py b/tests/unit/test_docker_workspace_slices.py index a43642d864..c4360a9bb4 100644 --- a/tests/unit/test_docker_workspace_slices.py +++ b/tests/unit/test_docker_workspace_slices.py @@ -17,6 +17,7 @@ Path("docker/automodel/Dockerfile.nmp-automodel-training"), Path("docker/rl/Dockerfile.nmp-rl-training"), ) +SDK_ALIAS_PACKAGES = ("filesets", "models") WANDB_PACKAGE_SPEC_RE = re.compile(r"(?=|<|>)[^\\\s\"']+") DOCKER_IMAGE_WANDB_CONFIG_PATHS = ( Path("docker/Dockerfile.nmp-customizer-tasks"), @@ -99,26 +100,28 @@ def test_docker_workspace_slice_contains_all_workspace_sources(slice_name): "path", [pytest.param(ROOT / path, id=str(path)) for path in SDK_EDITABLE_DOCKERFILES], ) -def test_sdk_editable_image_installs_filesets(path: Path) -> None: - """Editable SDK installs need the filesets source package available separately.""" +def test_sdk_editable_image_installs_sdk_alias_packages(path: Path) -> None: + """Editable SDK installs need source packages for SDK aliases available separately.""" text = path.read_text(encoding="utf-8") assert "-e /app/sdk/python/nemo-platform" in text - assert "-e /app/packages/filesets" in text + for package in SDK_ALIAS_PACKAGES: + assert f"-e /app/packages/{package}" in text @pytest.mark.parametrize("slice_name", WORKSPACE_SLICES) -def test_sdk_editable_workspace_slice_includes_filesets(slice_name: str) -> None: - """The SDK imports ``filesets`` lazily when ``sdk.files`` is accessed.""" +@pytest.mark.parametrize("package", SDK_ALIAS_PACKAGES) +def test_sdk_editable_workspace_slice_includes_sdk_alias_package(slice_name: str, package: str) -> None: + """The SDK imports alias packages lazily when those resources are accessed.""" workspace_path = ROOT / "docker" / slice_name / "pyproject.workspace.toml" workspace = _load_pyproject(workspace_path) member_paths = set(workspace["tool"]["uv"]["workspace"]["members"]) source_names = _workspace_sources(workspace) dockerfile_text = (ROOT / "docker" / slice_name / "Dockerfile.platform-workspace").read_text(encoding="utf-8") - assert "packages/filesets" in member_paths - assert "filesets" in source_names - assert "COPY packages/filesets packages/filesets" in dockerfile_text + assert f"packages/{package}" in member_paths + assert package in source_names + assert f"COPY packages/{package} packages/{package}" in dockerfile_text def test_deployments_plugin_is_optional_for_models_service():