fix(customizer): include models SDK package in task images

docker/Dockerfile.nmp-customizer-tasks

@@ -71,6 +71,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     --overrides /app/docker/customizer/no_override_requirements.txt \
     -e /app/sdk/python/nemo-platform \
     -e /app/packages/filesets \
+    -e /app/packages/models \
     -e /app/packages/nemo_platform_plugin \
     -e /app/packages/nmp_common \
     -e /app/packages/nmp_customization_common \

docker/Dockerfile.nmp-unsloth-training

@@ -172,6 +172,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     --overrides /opt/docker/no_override_requirements.txt \
     -e /app/sdk/python/nemo-platform \
     -e /app/packages/filesets \
+    -e /app/packages/models \
     -e /app/packages/nemo_platform_plugin \
     -e /app/packages/nmp_common \
     -e /app/packages/nmp_customization_common \

docker/automodel/Dockerfile.nmp-automodel-training

@@ -36,6 +36,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     --overrides /app/docker/automodel/no_override_requirements.txt \
     -e /app/sdk/python/nemo-platform \
     -e /app/packages/filesets \
+    -e /app/packages/models \
     -e /app/packages/nemo_platform_plugin \
     -e /app/packages/nmp_common \
     -e /app/packages/nmp_customization_common \

docker/rl/Dockerfile.nmp-rl-training

@@ -64,6 +64,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --overrides /tmp/base-overrides.txt \
         -e /app/sdk/python/nemo-platform \
         -e /app/packages/filesets \
+        -e /app/packages/models \
         -e /app/packages/nemo_platform_plugin \
         -e /app/packages/nmp_common \
         -e /app/packages/nmp_customization_common \

docker/rl/Dockerfile.platform-workspace

@@ -15,6 +15,7 @@ COPY docs docs
 COPY openapi openapi
 COPY packages/nmp_build_tools packages/nmp_build_tools
 COPY packages/filesets packages/filesets
+COPY packages/models packages/models
 COPY packages/nmp_common packages/nmp_common
 COPY packages/nmp_customization_common packages/nmp_customization_common
 COPY packages/nemo_platform_plugin packages/nemo_platform_plugin

docker/rl/pyproject.workspace.toml

@@ -17,6 +17,7 @@ required-version = ">=0.9.14,<0.10.0"
 members = [
   "packages/nmp_build_tools",
   "packages/filesets",
+  "packages/models",
   "sdk/python/nemo-platform",
   "packages/nemo_platform_plugin",
   "packages/nmp_common",
@@ -27,6 +28,7 @@ members = [
 [tool.uv.sources]
 nmp-build-tools = { workspace = true }
 filesets = { workspace = true }
+models = { workspace = true }
 nemo-platform-sdk = { workspace = true }
 nemo-platform-plugin = { workspace = true }
 nmp-common = { workspace = true }

docker/unsloth/Dockerfile.platform-workspace

@@ -17,6 +17,7 @@ COPY docs docs
 COPY openapi openapi
 COPY packages/nmp_build_tools packages/nmp_build_tools
 COPY packages/filesets packages/filesets
+COPY packages/models packages/models
 COPY packages/nmp_common packages/nmp_common
 COPY packages/nmp_customization_common packages/nmp_customization_common
 COPY packages/nemo_platform_plugin packages/nemo_platform_plugin

docker/unsloth/pyproject.workspace.toml

@@ -16,6 +16,7 @@ required-version = ">=0.9.14,<0.10.0"
 members = [
   "packages/nmp_build_tools",
   "packages/filesets",
+  "packages/models",
   "sdk/python/nemo-platform",
   "packages/nemo_platform_plugin",
   "packages/nmp_common",
@@ -26,6 +27,7 @@ members = [
 [tool.uv.sources]
 nmp-build-tools = { workspace = true }
 filesets = { workspace = true }
+models = { workspace = true }
 nemo-platform-sdk = { workspace = true }
 nemo-platform-plugin = { workspace = true }
 nmp-common = { workspace = true }

packages/nemo_platform_ext/src/nemo_platform_ext/quickstart/container.py

@@ -325,6 +325,8 @@ def _create_environment(self, platform_config: PlatformConfig) -> dict[str, str]
         # Configure the models/NIM Docker backend for DonD (Docker-on-Docker) setup.
         # NIMs need to join the same network as the quickstart container so they can
         # communicate via container names (e.g., http://md-workspace-name:8000).
+        env["NEMO_DEPLOYMENTS_DOCKER_NETWORK"] = self.config.network_name
+        env["NEMO_DEPLOYMENTS_DOCKER_ENDPOINT_MODE"] = "network"
         env["MODELS_DOCKER_NETWORKING_MODE"] = "dond"
         env["MODELS_DOCKER_NETWORK"] = self.config.network_name
         # Pass the container name so the Models service can use it for localhost replacement

packages/nemo_platform_ext/tests/quickstart/test_container.py

@@ -204,6 +204,16 @@ def test_single_gpu_id_format(self):
 
         assert env["NMP_DOCKER_RESERVED_GPU_DEVICE_IDS"] == "0"
 
+    def test_deployments_docker_backend_uses_quickstart_network_endpoint_mode(self):
+        manager = self._make_manager()
+        mock_platform_config = MagicMock()
+        mock_platform_config.to_env_vars.return_value = {}
+
+        env = manager._create_environment(mock_platform_config)
+
+        assert env["NEMO_DEPLOYMENTS_DOCKER_NETWORK"] == "test-network"
+        assert env["NEMO_DEPLOYMENTS_DOCKER_ENDPOINT_MODE"] == "network"
+
 
 class TestCreateEnvironmentRegistryCredentials:
     """Tests for registry credential env var passthrough."""

packages/nmp_testing/src/nmp/testing/e2e/docker.py

@@ -183,6 +183,8 @@ def start(self) -> None:
             # Models Docker backend: use DonD mode so the API (in a container) can reach NIM
             # containers by name on the same network. Without this, health_url is localhost:port
             # and the API container cannot reach the NIM (localhost is the API itself).
+            self.container.with_env("NEMO_DEPLOYMENTS_DOCKER_NETWORK", self.network.name)
+            self.container.with_env("NEMO_DEPLOYMENTS_DOCKER_ENDPOINT_MODE", "network")
             self.container.with_env("MODELS_DOCKER_NETWORKING_MODE", "dond")
             self.container.with_env("MODELS_DOCKER_NETWORK", self.network.name)
             self.container.with_env("MODELS_DOCKER_CONTAINER_NAME", NMP_API_NETWORK_ALIAS)

packages/nmp_testing/tests/unit/test_e2e_docker_backend.py

@@ -119,6 +119,8 @@ def test_docker_backend_starts_clickhouse_sidecar_on_api_network(
         assert api_container.network is backend.network
         assert api_container.env["NMP_INTAKE_CLICKHOUSE_URL"] == docker_backend._clickhouse_api_url()
         assert api_container.env["NEMO_JOBS_DEFAULT_DOCKER_NETWORK"] == "nmp-e2e-test-network"
+        assert api_container.env["NEMO_DEPLOYMENTS_DOCKER_NETWORK"] == "nmp-e2e-test-network"
+        assert api_container.env["NEMO_DEPLOYMENTS_DOCKER_ENDPOINT_MODE"] == "network"
         assert api_container.env["NMP_IMAGE_REGISTRY"] == "registry.example/nmp"
         assert api_container.env["NMP_IMAGE_TAG"] == "test-tag"
 

plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py

@@ -287,6 +287,8 @@ async def create_deployment(
         docker_cfg = parse_docker_backend_config(backend_config)
         if config.backend_config.docker is not None:
             docker_cfg = config.backend_config.docker
+        if docker_cfg.network is None and self._executor_config.network is not None:
+            docker_cfg = docker_cfg.model_copy(update={"network": self._executor_config.network})
 
         dep_key = deployment_key(workspace, name)
         gpu_pool = self._gpu_pool
@@ -429,7 +431,7 @@ async def create_deployment(
                     status_message=f"Failed to start sidecar {sidecar.name}: {exc}",
                 )
 
-        endpoints = self._build_endpoints(container_spec, host_ports)
+        endpoints = self._build_endpoints(container_spec, host_ports, target_name=c_name)
         if config.restart_policy in _ONE_SHOT_RESTART_POLICIES:
             return await self._observe_one_shot_primary_after_create(
                 workspace=workspace,
@@ -730,7 +732,7 @@ async def read_status(self, *, workspace: str, name: str) -> BackendStatusUpdate
         state = container.status
         container_id = (container.id or "")[:12]
         host_ports = self._extract_host_ports(container)
-        endpoints = self._endpoints_from_container_ports(container, host_ports)
+        endpoints = self._endpoints_from_container_ports(host_ports, target_name=c_name)
 
         if state in ("created", "restarting"):
             return map_docker_state_to_starting(container_id, state)
@@ -740,7 +742,8 @@ async def read_status(self, *, workspace: str, name: str) -> BackendStatusUpdate
             # it must see only TCP mappings: a UDP-only workload has no TCP listener and
             # would otherwise be gated STARTING forever. Endpoints still carry every port.
             tcp_host_ports = self._extract_host_ports(container, protocol="tcp")
-            host_url = self._primary_host_url(tcp_host_ports)
+            host_url = self._primary_host_url(tcp_host_ports, target_name=c_name)
+            probe_ports = self._probe_ports(tcp_host_ports)
             config = await self._load_config_from_labels(workspace, labels)
             probe = None
             if config is not None and config.containers:
@@ -749,7 +752,7 @@ async def read_status(self, *, workspace: str, name: str) -> BackendStatusUpdate
                 container=container,
                 probe=probe,
                 host_url=host_url,
-                host_ports=tcp_host_ports,
+                host_ports=probe_ports,
             )
             if ready and restart_policy == "Always":
                 sidecar_ok, sidecar_reason = await self._sidecars_healthy(workspace, name, config)
@@ -1253,42 +1256,76 @@ def _extract_host_ports(self, container: DockerContainer, *, protocol: str | Non
                 result[container_port] = int(host_port)
         return result
 
-    def _primary_host_url(self, host_ports: dict[int, int]) -> str | None:
-        if not host_ports:
+    def _url_for_port(
+        self,
+        *,
+        target_name: str,
+        container_port: int,
+        host_port: int | None,
+        scheme: str = "http",
+    ) -> str | None:
+        if self._executor_config.endpoint_mode == "network":
+            return host_url_for_port(target_name, container_port, scheme=scheme)
+        if host_port is None:
             return None
-        host_port = next(iter(host_ports.values()))
         host = os.environ.get("NMP_LOOPBACK_ADDRESS", LOOPBACK_ADDRESSES[0])
-        return host_url_for_port(host, host_port)
+        return host_url_for_port(host, host_port, scheme=scheme)
 
-    def _build_endpoints(self, container_spec: Container, host_ports: dict[int, int]) -> list[Endpoint]:
+    def _primary_host_url(self, host_ports: dict[int, int], *, target_name: str) -> str | None:
+        if not host_ports:
+            return None
+        container_port, host_port = next(iter(host_ports.items()))
+        return self._url_for_port(target_name=target_name, container_port=container_port, host_port=host_port)
+
+    def _probe_ports(self, host_ports: dict[int, int]) -> dict[int, int]:
+        if self._executor_config.endpoint_mode == "network":
+            return {container_port: container_port for container_port in host_ports}
+        return host_ports
+
+    def _build_endpoints(
+        self,
+        container_spec: Container,
+        host_ports: dict[int, int],
+        *,
+        target_name: str,
+    ) -> list[Endpoint]:
         endpoints: list[Endpoint] = []
-        host = os.environ.get("NMP_LOOPBACK_ADDRESS", LOOPBACK_ADDRESSES[0])
         for port_spec in container_spec.ports:
             host_port = host_ports.get(port_spec.container_port)
-            if host_port is None:
+            endpoint_url = self._url_for_port(
+                target_name=target_name,
+                container_port=port_spec.container_port,
+                host_port=host_port,
+            )
+            if endpoint_url is None:
                 continue
             endpoint_name = port_spec.name or f"port-{port_spec.container_port}"
             protocol = "tcp" if port_spec.protocol == "UDP" else "http"
-            scheme = "http"
             endpoints.append(
                 Endpoint(
                     name=endpoint_name,
-                    url=host_url_for_port(host, host_port, scheme=scheme),
+                    url=endpoint_url,
                     protocol=protocol,
                 )
             )
         return endpoints
 
-    def _endpoints_from_container_ports(self, container: DockerContainer, host_ports: dict[int, int]) -> list[Endpoint]:
+    def _endpoints_from_container_ports(self, host_ports: dict[int, int], *, target_name: str) -> list[Endpoint]:
         if not host_ports:
             return []
-        host = os.environ.get("NMP_LOOPBACK_ADDRESS", LOOPBACK_ADDRESSES[0])
         endpoints: list[Endpoint] = []
         for container_port, host_port in host_ports.items():
+            endpoint_url = self._url_for_port(
+                target_name=target_name,
+                container_port=container_port,
+                host_port=host_port,
+            )
+            if endpoint_url is None:
+                continue
             endpoints.append(
                 Endpoint(
                     name=f"port-{container_port}",
-                    url=host_url_for_port(host, host_port),
+                    url=endpoint_url,
                     protocol="http",
                 )
             )

plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/config.py

@@ -5,8 +5,41 @@
 
 from __future__ import annotations
 
+import os
+from typing import Literal
+
 from nemo_deployments_plugin.backends.labels import DEFAULT_RESOURCE_SCOPE
-from pydantic import BaseModel, Field, model_validator
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+DOCKER_ENDPOINT_MODE_ENV_VAR = "NEMO_DEPLOYMENTS_DOCKER_ENDPOINT_MODE"
+DOCKER_NETWORK_ENV_VAR = "NEMO_DEPLOYMENTS_DOCKER_NETWORK"
+LEGACY_MODELS_DOCKER_NETWORKING_MODE_ENV_VAR = "MODELS_DOCKER_NETWORKING_MODE"
+LEGACY_MODELS_DOCKER_NETWORK_ENV_VAR = "MODELS_DOCKER_NETWORK"
+DockerEndpointMode = Literal["host", "network"]
+
+
+def _default_network() -> str | None:
+    for env_var in (DOCKER_NETWORK_ENV_VAR, LEGACY_MODELS_DOCKER_NETWORK_ENV_VAR):
+        value = os.getenv(env_var)
+        if value:
+            return value
+    return None
+
+
+def _default_endpoint_mode() -> str:
+    value = os.getenv(DOCKER_ENDPOINT_MODE_ENV_VAR)
+    if value:
+        return _normalize_endpoint_mode(value)
+    return _normalize_endpoint_mode(os.getenv(LEGACY_MODELS_DOCKER_NETWORKING_MODE_ENV_VAR, "host"))
+
+
+def _normalize_endpoint_mode(value: str) -> str:
+    mode = value.lower()
+    if mode == "dond":
+        return "network"
+    if mode in {"local", "dind"}:
+        return "host"
+    return mode
 
 
 class DockerExecutorConfig(BaseModel):
@@ -28,6 +61,25 @@ class DockerExecutorConfig(BaseModel):
         ),
     )
     pull_images: bool = Field(default=True, description="Pull container images before run when missing locally.")
+    network: str | None = Field(
+        default_factory=_default_network,
+        description=(
+            "Default Docker network for containers created by this executor. "
+            f"Can also be set with {DOCKER_NETWORK_ENV_VAR}; "
+            f"{LEGACY_MODELS_DOCKER_NETWORK_ENV_VAR} is accepted for compatibility."
+        ),
+    )
+    endpoint_mode: DockerEndpointMode = Field(
+        default_factory=_default_endpoint_mode,
+        validate_default=True,
+        description=(
+            "'host' reports and probes host-published ports. 'network' reports and probes "
+            "Docker-network container names and container ports, for controller/API "
+            "processes that run inside a container on the same Docker network. Can also "
+            f"be set with {DOCKER_ENDPOINT_MODE_ENV_VAR}; "
+            f"{LEGACY_MODELS_DOCKER_NETWORKING_MODE_ENV_VAR}=dond selects 'network' for compatibility."
+        ),
+    )
     resource_scope: str = Field(
         default=DEFAULT_RESOURCE_SCOPE,
         min_length=1,
@@ -49,8 +101,17 @@ class DockerExecutorConfig(BaseModel):
         description="Last host port (inclusive) to consider when publishing container ports for this executor.",
     )
 
+    @field_validator("endpoint_mode", mode="before")
+    @classmethod
+    def _normalize_endpoint_mode(cls, value: object) -> object:
+        if isinstance(value, str):
+            return _normalize_endpoint_mode(value)
+        return value
+
     @model_validator(mode="after")
-    def _validate_port_range(self) -> DockerExecutorConfig:
+    def _validate_config(self) -> DockerExecutorConfig:
         if self.port_range_start > self.port_range_end:
             raise ValueError("port_range_start must not exceed port_range_end")
+        if self.endpoint_mode == "network" and not self.network:
+            raise ValueError("endpoint_mode='network' requires network")
         return self