diff --git a/src/srtctl/cli/mixins/frontend_stage.py b/src/srtctl/cli/mixins/frontend_stage.py index b4d15161..ba89be7f 100644 --- a/src/srtctl/cli/mixins/frontend_stage.py +++ b/src/srtctl/cli/mixins/frontend_stage.py @@ -195,6 +195,7 @@ def _generate_nginx_config(self, topology: FrontendTopology) -> str: nginx_raise_ulimit=self.config.frontend.nginx_raise_ulimit, nginx_session_affinity=self.config.frontend.nginx_session_affinity, nginx_session_affinity_header=self.config.frontend.nginx_session_affinity_header, + nginx_keepalive_timeout=self.config.frontend.nginx_keepalive_timeout, ) def start_frontend( diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 00c816dd..b6db317e 100644 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -1456,6 +1456,12 @@ class FrontendConfig: Override per job or set ``nginx_raise_ulimit`` in srtslurm.yaml for the cluster. nginx_session_affinity: Consistently hash ``nginx_session_affinity_header`` to a frontend. Requests without that header use a generated request ID and stay distributed. + nginx_keepalive_timeout: Idle timeout for client and upstream keepalive + connections in the generated nginx.conf (default "3600s"). nginx's own + default is 75s, which closes a session's connection during the long + recorded think-time of an agentic replay; the client's next write on + that pooled socket then fails with "broken pipe" / "server + disconnected" and nothing is logged server-side. nginx_session_affinity_header: Header hashed when affinity is on (default ``X-Dynamo-Session-ID``). Set ``X-Correlation-ID`` for clients (e.g. aiperf) that carry the session id in that header instead. @@ -1470,6 +1476,7 @@ class FrontendConfig: nginx_raise_ulimit: bool = False nginx_session_affinity: bool = False nginx_session_affinity_header: str = "X-Dynamo-Session-ID" + nginx_keepalive_timeout: str = "3600s" args: dict[str, Any] | None = None env: dict[str, str] | None = None # trtllm_serve orchestrator (ser.yaml) options; ignored by other frontends. diff --git a/src/srtctl/templates/nginx.conf.j2 b/src/srtctl/templates/nginx.conf.j2 index 57833705..5a10cd1e 100644 --- a/src/srtctl/templates/nginx.conf.j2 +++ b/src/srtctl/templates/nginx.conf.j2 @@ -25,6 +25,13 @@ http { } {% endif %} + # Agentic replay reproduces recorded think-time, so a session's connection can + # sit idle for minutes between turns. nginx's default keepalive_timeout of 75s + # closes it underneath a pooled client connection, and the client's next write + # on that socket surfaces as "broken pipe" / "server disconnected" with nothing + # logged server-side. + keepalive_timeout {{ nginx_keepalive_timeout }}; + upstream backend_servers { {% if nginx_session_affinity %} hash $frontend_hash_key consistent; @@ -32,6 +39,9 @@ http { {% for frontend_host in frontend_hosts %} server {{ frontend_host }}:{{ backend_port }}; {% endfor %} + # Reuse upstream connections instead of a fresh TCP handshake per request. + keepalive 512; + keepalive_timeout {{ nginx_keepalive_timeout }}; } # The main server block that listens for incoming traffic. @@ -45,6 +55,11 @@ http { proxy_buffering off; proxy_read_timeout 24h; proxy_send_timeout 24h; + + # Required for the upstream keepalive pool above: without these nginx + # talks HTTP/1.0 to the backend and opens a new connection per request. + proxy_http_version 1.1; + proxy_set_header Connection ""; } } } diff --git a/tests/test_nginx_conf.py b/tests/test_nginx_conf.py new file mode 100644 index 00000000..ade32812 --- /dev/null +++ b/tests/test_nginx_conf.py @@ -0,0 +1,59 @@ +"""Tests for the generated nginx.conf, focused on connection keepalive. + +Agentic replay reproduces recorded think-time, so a session's connection sits +idle between turns. nginx's default keepalive_timeout of 75s closes it under a +pooled client connection and the client's next write fails with "broken pipe" / +"server disconnected", with nothing logged server-side. +""" + +from pathlib import Path + +import pytest +from jinja2 import Environment, FileSystemLoader + +from srtctl.core.schema import FrontendConfig + +TEMPLATE_DIR = Path(__file__).parent.parent / "src" / "srtctl" / "templates" + + +def render(**overrides) -> str: + env = Environment(loader=FileSystemLoader(str(TEMPLATE_DIR))) + params = { + "frontend_hosts": ["10.0.0.1", "10.0.0.2"], + "backend_port": 8180, + "listen_port": 8000, + "nginx_raise_ulimit": False, + "nginx_session_affinity": False, + "nginx_session_affinity_header": "X-Dynamo-Session-ID", + "nginx_keepalive_timeout": FrontendConfig().nginx_keepalive_timeout, + } + params.update(overrides) + return env.get_template("nginx.conf.j2").render(**params) + + +def test_default_keepalive_timeout_exceeds_nginx_default(): + """The default must be far above nginx's own 75s or agentic idle gaps break.""" + assert FrontendConfig().nginx_keepalive_timeout == "3600s" + + +def test_client_and_upstream_keepalive_timeouts_rendered(): + conf = render() + assert conf.count("keepalive_timeout 3600s;") == 2 + + +def test_upstream_connection_pool_enabled(): + conf = render() + assert "keepalive 512;" in conf + + +def test_http11_and_cleared_connection_header_for_upstream_keepalive(): + """Without these nginx talks HTTP/1.0 upstream and the pool is a no-op.""" + conf = render() + assert "proxy_http_version 1.1;" in conf + assert 'proxy_set_header Connection "";' in conf + + +@pytest.mark.parametrize("timeout", ["75s", "10m", "3600s"]) +def test_keepalive_timeout_is_configurable(timeout): + conf = render(nginx_keepalive_timeout=timeout) + assert conf.count(f"keepalive_timeout {timeout};") == 2