Keep nginx connections alive across agentic replay idle gaps

src/srtctl/cli/mixins/frontend_stage.py

@@ -195,6 +195,7 @@ def _generate_nginx_config(self, topology: FrontendTopology) -> str:
             nginx_raise_ulimit=self.config.frontend.nginx_raise_ulimit,
             nginx_session_affinity=self.config.frontend.nginx_session_affinity,
             nginx_session_affinity_header=self.config.frontend.nginx_session_affinity_header,
+            nginx_keepalive_timeout=self.config.frontend.nginx_keepalive_timeout,
         )
 
     def start_frontend(

src/srtctl/core/schema.py

@@ -1456,6 +1456,12 @@ class FrontendConfig:
             Override per job or set ``nginx_raise_ulimit`` in srtslurm.yaml for the cluster.
         nginx_session_affinity: Consistently hash ``nginx_session_affinity_header`` to a
             frontend. Requests without that header use a generated request ID and stay distributed.
+        nginx_keepalive_timeout: Idle timeout for client and upstream keepalive
+            connections in the generated nginx.conf (default "3600s"). nginx's own
+            default is 75s, which closes a session's connection during the long
+            recorded think-time of an agentic replay; the client's next write on
+            that pooled socket then fails with "broken pipe" / "server
+            disconnected" and nothing is logged server-side.
         nginx_session_affinity_header: Header hashed when affinity is on (default
             ``X-Dynamo-Session-ID``). Set ``X-Correlation-ID`` for clients (e.g. aiperf) that
             carry the session id in that header instead.
@@ -1470,6 +1476,7 @@ class FrontendConfig:
     nginx_raise_ulimit: bool = False
     nginx_session_affinity: bool = False
     nginx_session_affinity_header: str = "X-Dynamo-Session-ID"
+    nginx_keepalive_timeout: str = "3600s"
     args: dict[str, Any] | None = None
     env: dict[str, str] | None = None
     # trtllm_serve orchestrator (ser.yaml) options; ignored by other frontends.

src/srtctl/templates/nginx.conf.j2

@@ -25,13 +25,23 @@ http {
     }
 {% endif %}
 
+    # Agentic replay reproduces recorded think-time, so a session's connection can
+    # sit idle for minutes between turns. nginx's default keepalive_timeout of 75s
+    # closes it underneath a pooled client connection, and the client's next write
+    # on that socket surfaces as "broken pipe" / "server disconnected" with nothing
+    # logged server-side.
+    keepalive_timeout {{ nginx_keepalive_timeout }};
+
     upstream backend_servers {
 {% if nginx_session_affinity %}
         hash $frontend_hash_key consistent;
 {% endif %}
         {% for frontend_host in frontend_hosts %}
         server {{ frontend_host }}:{{ backend_port }};
         {% endfor %}
+        # Reuse upstream connections instead of a fresh TCP handshake per request.
+        keepalive 512;
+        keepalive_timeout {{ nginx_keepalive_timeout }};
     }
 
     # The main server block that listens for incoming traffic.
@@ -45,6 +55,11 @@ http {
             proxy_buffering off;
             proxy_read_timeout 24h;
             proxy_send_timeout 24h;
+
+            # Required for the upstream keepalive pool above: without these nginx
+            # talks HTTP/1.0 to the backend and opens a new connection per request.
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
         }
     }
 }

tests/test_nginx_conf.py

@@ -0,0 +1,59 @@
+"""Tests for the generated nginx.conf, focused on connection keepalive.
+
+Agentic replay reproduces recorded think-time, so a session's connection sits
+idle between turns. nginx's default keepalive_timeout of 75s closes it under a
+pooled client connection and the client's next write fails with "broken pipe" /
+"server disconnected", with nothing logged server-side.
+"""
+
+from pathlib import Path
+
+import pytest
+from jinja2 import Environment, FileSystemLoader
+
+from srtctl.core.schema import FrontendConfig
+
+TEMPLATE_DIR = Path(__file__).parent.parent / "src" / "srtctl" / "templates"
+
+
+def render(**overrides) -> str:
+    env = Environment(loader=FileSystemLoader(str(TEMPLATE_DIR)))
+    params = {
+        "frontend_hosts": ["10.0.0.1", "10.0.0.2"],
+        "backend_port": 8180,
+        "listen_port": 8000,
+        "nginx_raise_ulimit": False,
+        "nginx_session_affinity": False,
+        "nginx_session_affinity_header": "X-Dynamo-Session-ID",
+        "nginx_keepalive_timeout": FrontendConfig().nginx_keepalive_timeout,
+    }
+    params.update(overrides)
+    return env.get_template("nginx.conf.j2").render(**params)
+
+
+def test_default_keepalive_timeout_exceeds_nginx_default():
+    """The default must be far above nginx's own 75s or agentic idle gaps break."""
+    assert FrontendConfig().nginx_keepalive_timeout == "3600s"
+
+
+def test_client_and_upstream_keepalive_timeouts_rendered():
+    conf = render()
+    assert conf.count("keepalive_timeout 3600s;") == 2
+
+
+def test_upstream_connection_pool_enabled():
+    conf = render()
+    assert "keepalive 512;" in conf
+
+
+def test_http11_and_cleared_connection_header_for_upstream_keepalive():
+    """Without these nginx talks HTTP/1.0 upstream and the pool is a no-op."""
+    conf = render()
+    assert "proxy_http_version 1.1;" in conf
+    assert 'proxy_set_header Connection "";' in conf
+
+
+@pytest.mark.parametrize("timeout", ["75s", "10m", "3600s"])
+def test_keepalive_timeout_is_configurable(timeout):
+    conf = render(nginx_keepalive_timeout=timeout)
+    assert conf.count(f"keepalive_timeout {timeout};") == 2